feat: improve acceptance tests

* use native Go instrumentation
* report test results per flag (one coverage output file per flag

Author: psturc

.github/workflows/checks-codecov.yaml

@@ -91,6 +91,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-unit.out
+          disable_search: true
           flags: unit
 
       - name: Upload generative test coverage report
@@ -99,6 +100,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-generative.out
+          disable_search: true
           flags: generative
 
       - name: Upload integration test coverage report
@@ -107,6 +109,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-integration.out
+          disable_search: true
           flags: integration
 
   Acceptance:
@@ -143,12 +146,16 @@ jobs:
         run: git log --oneline -n1
 
       - name: Acceptance test
-        run: make acceptance
+        id: acceptance_test
+        run: E2E_INSTRUMENTATION=true make acceptance
 
       - name: Upload coverage report
         uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage-acceptance.out
+          disable_search: true
           flags: acceptance
 
   Tools:

Makefile

@@ -13,6 +13,7 @@ SHELL=$(if $@,$(info ❱ $@))$(_SHELL)
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 COPY:=The Conforma Contributors
 COSIGN_VERSION=$(shell go list -f '{{.Version}}' -m github.com/sigstore/cosign/v2)
+E2E_INSTRUMENTATION_FLAGS := $(if $(filter $(E2E_INSTRUMENTATION),true),-cover -covermode atomic)
 
 ##@ Information
 
@@ -54,7 +55,7 @@ BUILD_BIN_SUFFIX=$(if $(DEBUG_BUILD),_debug,)
 $(ALL_SUPPORTED_OS_ARCH): generate ## Build binaries for specific platform/architecture, e.g. make dist/ec_linux_amd64
 	@GOOS=$(word 2,$(subst _, ,$(notdir $@))); \
 	GOARCH=$(word 3,$(subst _, ,$(notdir $@))); \
-	GOOS=$${GOOS} GOARCH=$${GOARCH} CGO_ENABLED=0 go build $(BUILD_TRIMPATH) $(BUILD_GC_FLAGS) -ldflags="$(BUILD_LD_FLAGS) -X github.com/conforma/cli/internal/version.Version=$(VERSION)" -o dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX); \
+	GOOS=$${GOOS} GOARCH=$${GOARCH} CGO_ENABLED=0 go build $(E2E_INSTRUMENTATION_FLAGS) $(BUILD_TRIMPATH) $(BUILD_GC_FLAGS) -ldflags="$(BUILD_LD_FLAGS) -X github.com/conforma/cli/internal/version.Version=$(VERSION)" -o dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX); \
 	sha256sum -b dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX) > dist/ec_$${GOOS}_$${GOARCH}$(BUILD_BIN_SUFFIX).sha256
 
 .PHONY: dist
@@ -120,15 +121,15 @@ acceptance: ## Run all acceptance tests
 	cleanup() { \
 		cp "$${ACCEPTANCE_WORKDIR}"/features/__snapshots__/* "$(ROOT_DIR)"/features/__snapshots__/; \
 	}; \
+	mkdir -p "$${ACCEPTANCE_WORKDIR}/coverage"; \
 	trap cleanup EXIT; \
 	cp -R . "$$ACCEPTANCE_WORKDIR"; \
 	cd "$$ACCEPTANCE_WORKDIR" && \
-	go run acceptance/coverage/coverage.go && \
 	$(MAKE) build && \
 	export COVERAGE_FILEPATH="$$ACCEPTANCE_WORKDIR"; \
 	export COVERAGE_FILENAME="-acceptance"; \
-	cd acceptance && go test -coverprofile "$$ACCEPTANCE_WORKDIR/coverage-acceptance.out" -timeout $(ACCEPTANCE_TIMEOUT) ./... && \
-	go run -modfile "$$ACCEPTANCE_WORKDIR/tools/go.mod" github.com/wadey/gocovmerge "$$ACCEPTANCE_WORKDIR/coverage-acceptance.out" > "$(ROOT_DIR)/coverage-acceptance.out"
+	export GOCOVERDIR="$${ACCEPTANCE_WORKDIR}/coverage"; \
+	cd acceptance && go test ./... ; go tool covdata textfmt -i=$${GOCOVERDIR} -o="$(ROOT_DIR)/coverage-acceptance.out"
 
 # Add @focus above the feature you're hacking on to use this
 # (Mainly for use with the feature-% target below)

acceptance/cli/cli.go

@@ -89,6 +89,7 @@ func variables(ctx context.Context) (context.Context, []string, map[string]strin
 		"PATH=" + os.Getenv("PATH"),
 		"COVERAGE_FILEPATH=" + os.Getenv("COVERAGE_FILEPATH"), // where to put the coverage file, $COVERAGE_FILEPATH is provided by the Makefile, if empty it'll be $TMPDIR
 		"COVERAGE_FILENAME=" + os.Getenv("COVERAGE_FILENAME"), // suffix for the coverage file
+		"GOCOVERDIR=" + os.Getenv("GOCOVERDIR"),               // directory where the Go coverage raw data is stored
 		"HOME=/tmp",
 	}