Adds coverage logic customized from func-e (#596)

This adds coverage the same way as works in func-e, just that there are
slightly different conventions here wrt packages to ignore. Notably, we
should not run coverage using integration tests or examples as they give
false positives.

Signed-off-by: Adrian Cole <[email protected]>

.github/workflows/commit.yaml

@@ -82,6 +82,16 @@ jobs:
 
       - run: make test
 
+      - name: "Generate coverage report"  # only once (not per OS)
+        if: runner.os == 'Linux'
+        run: make coverage
+
+      - name: "Upload coverage report"  # only on main push and only once (not per OS)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && runner.os == 'Linux'
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: bash <(curl -s https://codecov.io/bash)
+
   test_scratch:
     name: ${{ matrix.arch }}, Linux (scratch), Go-${{ matrix.go-version }}
     runs-on: ubuntu-20.04

.gitignore

@@ -26,3 +26,6 @@ go.work
 # AssemblyScript
 node_modules
 package-lock.json
+
+# codecov.io
+/coverage.txt

Makefile

@@ -1,8 +1,26 @@
+
+# Make functions strip spaces and use commas to separate parameters. The below variables escape these characters.
+comma := ,
+space :=
+space +=
+
 goimports := golang.org/x/tools/cmd/[email protected]
 golangci_lint := github.com/golangci/golangci-lint/cmd/[email protected]
 # sync this with netlify.toml!
 hugo          := github.com/gohugoio/[email protected]
 
+# Make 3.81 doesn't support '**' globbing: Set explicitly instead of recursion.
+all_sources   := $(wildcard *.go */*.go */*/*.go */*/*/*.go */*/*/*.go */*/*/*/*.go)
+all_testdata  := $(wildcard testdata/* */testdata/* */*/testdata/* */*/*/testdata/*)
+all_testing   := $(wildcard internal/testing/* internal/testing/*/* internal/testing/*/*/*)
+all_examples  := $(wildcard examples/* examples/*/* examples/*/*/*)
+all_it        := $(wildcard internal/integration_test/* internal/integration_test/*/* internal/integration_test/*/*/*)
+# main_sources exclude any test or example related code
+main_sources  := $(wildcard $(filter-out %_test.go $(all_testdata) $(all_testing) $(all_examples) $(all_it), $(all_sources)))
+# main_packages collect the unique main source directories (sort will dedupe).
+# Paths need to all start with ./, so we do that manually vs foreach which strips it.
+main_packages := $(sort $(foreach f,$(dir $(main_sources)),$(if $(findstring ./,$(f)),./,./$(f))))
+
 ensureCompilerFastest := -ldflags '-X github.com/tetratelabs/wazero/internal/integration_test/vs.ensureCompilerFastest=true'
 .PHONY: bench
 bench:
@@ -71,7 +89,6 @@ build.spectest.v1: # Note: wabt by default uses >1.0 features, so wast2json flag
 			--debug-names $$f; \
 	done
 
-
 .PHONY: build.spectest.v2
 build.spectest.v2: # Note: SIMD cases are placed in the "simd" subdirectory.
 	@mkdir -p $(spectest_v2_testdata_dir)
@@ -88,6 +105,12 @@ test:
 	@go test $$(go list ./... | grep -v spectest) -timeout 120s
 	@cd internal/integration_test/asm && go test ./... -timeout 120s
 
+.PHONY: coverage
+coverpkg = $(subst $(space),$(comma),$(main_packages))
+coverage: ## Generate test coverage
+	@go test -coverprofile=coverage.txt -covermode=atomic --coverpkg=$(coverpkg) $(main_packages)
+	@go tool cover -func coverage.txt
+
 .PHONY: spectest
 spectest:
 	@$(MAKE) spectest.v1
@@ -134,3 +157,8 @@ check:
 site: ## Serve website content
 	@git submodule update --init
 	@cd site && go run $(hugo) server --minify --disableFastRender --baseURL localhost:1313 --cleanDestinationDir -D
+
+.PHONY: clean
+clean: ## Ensure a clean build
+	@rm -rf dist build coverage.txt
+	@go clean -testcache

codecov.yml

@@ -0,0 +1,5 @@
+# Codecov for main is visible here https://app.codecov.io/gh/tetratelabs/wazero
+
+# We use codecov only as a UI, so we disable PR comments.
+# See https://docs.codecov.com/docs/pull-request-comments
+comment: false