From 7e9b6f2e9e10ca090f6ad8fae7186e3edcd1e05b Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Fri, 9 Oct 2026 13:27:48 +0100 Subject: [PATCH] Show PR coverage changes against the exact base commit --- .github/CONTRIBUTING.md | 6 ++ .github/actions/report-coverage/action.yml | 1 + .github/scripts/publish-coverage.cjs | 37 +++++++--- .github/scripts/publish-coverage.test.cjs | 81 +++++++++++++++++++++- .github/workflows/ci.yml | 40 +++++++++++ .github/workflows/coverage-baseline.yml | 79 +++++++++++++++++++++ scripts/lib/coverage_report.rb | 13 ++-- scripts/report_coverage | 3 +- scripts/test/coverage_baseline_test.rb | 57 +++++++++++++++ scripts/test/coverage_report_test.rb | 29 +++++++- 10 files changed, 326 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/coverage-baseline.yml create mode 100644 scripts/test/coverage_baseline_test.rb diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 2fc06595b..d9ed31d51 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -107,6 +107,12 @@ show their status instead of coverage from an older run. GitHub Actions jobs als write their coverage tables to the job summary. Fork builds and builds without a write token keep their artifacts and log summaries without posting a comment. +For affected JavaScript packages, separate baseline jobs measure the PR's exact +base commit. The comment shows changes in percentage points beside each metric, +for example `91.42% (+1.27)`. Deltas require successful measurements for the same +base commit; missing or failed baselines leave the current percentage visible +without a delta. Baseline jobs do not gate the PR or replace its test results. + Platform jobs store numeric results in GitHub checks. The `Coverage Report` workflow is the only comment writer and serializes updates per PR. It refreshes on CI workflow events, reading all current results so simultaneous completions diff --git a/.github/actions/report-coverage/action.yml b/.github/actions/report-coverage/action.yml index 4e4e6f7f0..1f1d5d32b 100644 --- a/.github/actions/report-coverage/action.yml +++ b/.github/actions/report-coverage/action.yml @@ -32,6 +32,7 @@ runs: COVERAGE_TEST_OUTCOME: ${{ inputs.outcome }} COVERAGE_REPOSITORY: ${{ github.repository }} COVERAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + COVERAGE_BASE_SHA: ${{ github.event.pull_request.base.sha }} COVERAGE_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} COVERAGE_TOKEN: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }} COVERAGE_REPORT_URL: ${{ steps.artifact.outputs.artifact-url }} diff --git a/.github/scripts/publish-coverage.cjs b/.github/scripts/publish-coverage.cjs index c106b39b9..07707e9f7 100644 --- a/.github/scripts/publish-coverage.cjs +++ b/.github/scripts/publish-coverage.cjs @@ -53,9 +53,12 @@ function readResult(check, platform, expected, pr, repo, run) { const result = JSON.parse(check.output.text); if (result.version !== 1 || result.platform !== platform.id || result.pr !== pr.number || result.headSha !== pr.head.sha || result.source?.provider !== expected.provider || + (result.revision ?? "head") !== (expected.revision ?? "head") || result.source.runId !== expected.runId || (result.source.runAttempt ?? 1) !== expected.runAttempt || !["success", "failed", "skipped", "unavailable"].includes(result.state) || !Array.isArray(result.rows)) throw new Error("Coverage result does not match this run"); + if (expected.revision === "base" && result.baseSha !== pr.base.sha) + throw new Error("Coverage baseline does not match the PR base"); if (["skipped", "unavailable"].includes(result.state) && result.rows.length) throw new Error("Unexpected measurements for an unmeasured platform"); if (result.rows.length || result.state === "success") { @@ -69,23 +72,26 @@ function readResult(check, platform, expected, pr, repo, run) { return {...result, reportUrl: reportURL(result.reportUrl, repo, run)}; } -function platformResult(platform, snapshot, jobs, pr, repo, core) { +function platformResult(platform, snapshot, jobs, pr, repo, core, revision = "head") { const {run, checks} = snapshot; let state = "pending"; if (!run) return {state}; // A failed-jobs rerun reuses successful jobs from an earlier attempt. Match // each result to the attempt that actually ran that platform's test job. - const job = latest(jobs.filter((item) => item.name === platform.job)) || - latest(jobs.filter((item) => item.name === (platform.skippedJob || platform.title))); - const expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt}; + const baseline = revision === "base"; + const jobName = baseline ? `Coverage baseline (${platform.id}) / Test` : platform.job; + const skippedJob = baseline ? `Coverage baseline (${platform.id})` : platform.skippedJob || platform.title; + const job = latest(jobs.filter((item) => item.name === jobName)) || + latest(jobs.filter((item) => item.name === skippedJob)); + const expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt, revision}; const plan = latest(jobs.filter((item) => item.name === "Detect Changed Areas")); if (plan?.status === "completed" && plan.conclusion !== "success") return {state: "unavailable"}; if (job?.conclusion === "skipped") return {state: "skipped"}; if (job?.status === "completed") state = job.conclusion === "success" ? "unavailable" : "failed"; else if (!job && run.status === "completed") state = "unavailable"; - const externalId = `coverage:${platform.id}:${expected.runId}:${expected.runAttempt}`; + const externalId = `coverage${baseline ? "-base" : ""}:${platform.id}:${expected.runId}:${expected.runAttempt}`; const check = latest(checks.filter((item) => item.app?.slug === expected.provider && - item.external_id === externalId && item.name === `Coverage — ${platform.title}`)); + item.external_id === externalId && item.name === `Coverage${baseline ? " base" : ""} — ${platform.title}`)); if (!check) return {state}; try { return readResult(check, platform, expected, pr, repo, run); @@ -95,15 +101,17 @@ function platformResult(platform, snapshot, jobs, pr, repo, core) { } } -function metric(row) { +function metric(row, baseline) { if (!row) return "—"; const [, covered, total] = row; if (!total) return "N/A"; const percentage = Number((100 * covered / total).toFixed(2)); - return `${percentage}%`; + if (!baseline?.[2]) return `${percentage}%`; + const delta = Number((100 * covered / total - 100 * baseline[1] / baseline[2]).toFixed(2)); + return `${percentage}% **(${delta > 0 ? "+" : ""}${delta})**`; } -function render(results) { +function render(results, baselines = {}) { const lines = [marker, "# Coverage Report", "", "| Status | Platform / target | Lines | Branches | Functions | Report |", "| :---: | --- | ---: | ---: | ---: | --- |"]; @@ -118,8 +126,13 @@ function render(results) { const [emoji, status] = statuses[result.state]; const report = result.reportUrl ? `[Full report](${result.reportUrl})` : status; const find = (name) => rows.find((row) => row[0] === name); - lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${metric(find("Lines"))} | ${metric(find("Branches"))} | ${metric(find("Functions"))} | ${report} |`); + const baseline = baselines[platform.id]; + const comparable = result.state === "success" && baseline?.state === "success" && + result.baseSha && result.baseSha === baseline.baseSha; + const value = (name) => metric(find(name), comparable ? baseline.rows.find((row) => row[0] === name) : undefined); + lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${value("Lines")} | ${value("Branches")} | ${value("Functions")} | ${report} |`); } + lines.push("", "Changes in parentheses are **percentage points versus the PR’s base commit**. No delta appears when matching base coverage is unavailable."); return lines.join("\n") + "\n"; } @@ -136,7 +149,9 @@ async function publish({github, context, core, prNumber}) { }) : []; const results = Object.fromEntries(platforms.map((platform) => [platform.id, platformResult(platform, snapshot, jobs, pr, repo, core)])); - const body = render(results); + const baselines = Object.fromEntries(platforms.map((platform) => + [platform.id, platformResult(platform, snapshot, jobs, pr, repo, core, "base")])); + const body = render(results, baselines); const comments = await github.paginate(github.rest.issues.listComments, {...repo, issue_number: pr.number, per_page: 100}); const existing = comments.find(isComment); // Wait for change detection before opening a comment for an affected platform. diff --git a/.github/scripts/publish-coverage.test.cjs b/.github/scripts/publish-coverage.test.cjs index faa95ee9b..f61c073c5 100644 --- a/.github/scripts/publish-coverage.test.cjs +++ b/.github/scripts/publish-coverage.test.cjs @@ -36,10 +36,11 @@ function fixture() { f.add = (id, overrides = {}) => { const platform = reporter.platforms.find((platform) => platform.id === id); const source = {provider: "github-actions", runId: 100, runAttempt: 1}; - const result = {version: 1, platform: id, pr: 12, headSha: "head", source, state: "success", rows: platform.metrics.map((name) => [name, 7, 10]), ...overrides}; + const result = {version: 1, platform: id, pr: 12, headSha: "head", baseSha: "base", source, state: "success", rows: platform.metrics.map((name) => [name, 7, 10]), ...overrides}; + const baseline = result.revision === "base"; const check = { - id: 400 + f.checks.length, name: `Coverage — ${platform.title}`, head_sha: "head", - app: {slug: source.provider}, external_id: `coverage:${id}:${source.runId}:${source.runAttempt}`, + id: 400 + f.checks.length, name: `Coverage${baseline ? " base" : ""} — ${platform.title}`, head_sha: "head", + app: {slug: source.provider}, external_id: `coverage${baseline ? "-base" : ""}:${id}:${result.source.runId}:${result.source.runAttempt}`, output: {text: JSON.stringify(result)}, }; f.checks.push(check); @@ -48,6 +49,80 @@ function fixture() { return f; } +function addBaseline(f, overrides = {}) { + f.jobs.push({id: 600 + f.jobs.length, name: "Coverage baseline (web) / Test", run_attempt: 1, status: "completed", conclusion: "success"}); + return f.add("web", {revision: "base", ...overrides}); +} + +test("shows signed percentage-point deltas from exact base counts beside each metric", async () => { + const f = fixture(); + f.add("web"); + addBaseline(f, {rows: [["Lines", 6, 10], ["Statements", 7, 10], ["Branches", 8, 10], ["Functions", 14, 20]]}); + await f.publish(); + assert.ok(row(f.writes[0].body, "Web").includes("70% **(+10)** | 70% **(-10)** | 70% **(0)**")); + assert.ok(f.writes[0].body.includes("percentage points versus the PR’s base commit")); +}); + +test("rounds the difference once and does not display negative zero", async () => { + const f = fixture(); + f.add("web", {rows: [["Lines", 667, 1000], ["Statements", 7, 10], ["Branches", 699999, 1000000], ["Functions", 0, 0]]}); + addBaseline(f, {rows: [["Lines", 2, 3], ["Statements", 7, 10], ["Branches", 7, 10], ["Functions", 7, 10]]}); + await f.publish(); + assert.ok(row(f.writes[0].body, "Web").includes("66.7% **(+0.03)** | 70% **(0)** | N/A")); +}); + +test("never presents missing, failed, stale, or empty baselines as zero change", async () => { + for (const overrides of [ + {baseSha: "old-base"}, {state: "failed"}, {state: "unavailable", rows: []}, + {rows: reporter.platforms[0].metrics.map((name) => [name, 0, 0])}, + {source: {provider: "github-actions", runId: 99, runAttempt: 1}}, + ]) { + const f = fixture(); + f.add("web"); + addBaseline(f, overrides); + await f.publish(); + assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |")); + } +}); + +test("legacy, stale-base and failed head reports retain percentages without deltas", async () => { + for (const overrides of [{baseSha: undefined}, {baseSha: "old-base"}, {state: "failed"}]) { + const f = fixture(); + f.add("web", overrides); + addBaseline(f); + await f.publish(); + assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |")); + } +}); + +test("baseline reruns cannot reuse an earlier attempt but preserve the head report", async () => { + const f = fixture(); + f.add("web"); + addBaseline(f); + f.runs[0].run_attempt = 2; + f.jobs.push({id: 999, name: "Coverage baseline (web) / Test", run_attempt: 2, status: "in_progress"}); + await f.publish(); + assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |")); + f.add("web", {revision: "base", source: {provider: "github-actions", runId: 100, runAttempt: 2}}); + await f.publish(); + assert.ok(row(f.writes[1].body, "Web").includes("70% **(0)**")); +}); + +test("malformed and untrusted baselines cannot hide valid head coverage", async () => { + for (const mutate of [ + (check) => { check.app.slug = "untrusted-app"; }, + (check) => { check.output.text = "x".repeat(32769); }, + (check) => { const result = JSON.parse(check.output.text); result.rows[0][1] = 11; check.output.text = JSON.stringify(result); }, + (check) => { const result = JSON.parse(check.output.text); result.revision = "head"; check.output.text = JSON.stringify(result); }, + ]) { + const f = fixture(); + f.add("web"); + mutate(addBaseline(f)); + await f.publish(); + assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |")); + } +}); + test("one comment combines the existing JavaScript reports without native coverage rows", async () => { const f = fixture(); for (const platform of reporter.platforms) f.add(platform.id); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 324c99a93..8bedf43bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,7 @@ jobs: # Common React Native infra: JS-level jobs (jest, lint, packed # files) and shared config that should trigger all RN jobs. reactNativeCommon: + - '.github/workflows/coverage-baseline.yml' - '.github/actions/report-coverage/**' - 'scripts/report_coverage' - 'scripts/lib/coverage_report.rb' @@ -90,6 +91,7 @@ jobs: - '.github/workflows/rn-test-android.yml' - '.github/workflows/rn-build-android.yml' web: + - '.github/workflows/coverage-baseline.yml' - '.github/actions/report-coverage/**' - 'scripts/report_coverage' - 'scripts/lib/coverage_report.rb' @@ -103,6 +105,7 @@ jobs: - '.ci/changed-file-filters.yml' - '.github/workflows/ci.yml' protocol: + - '.github/workflows/coverage-baseline.yml' - '.github/actions/report-coverage/**' - 'scripts/report_coverage' - 'scripts/lib/coverage_report.rb' @@ -113,6 +116,7 @@ jobs: scripts: - '.github/scripts/*coverage*' - '.github/workflows/coverage-report.yml' + - '.github/workflows/coverage-baseline.yml' - '.github/actions/report-coverage/**' - 'scripts/report_coverage' - '.github/scripts/measure-package-size' @@ -220,6 +224,42 @@ jobs: pull-requests: read uses: ./.github/workflows/protocol-test.yml + coverage-base-web: + name: Coverage baseline (web) + needs: changes + if: github.event_name == 'pull_request' && needs.changes.outputs.web == 'true' + permissions: + contents: read + checks: write + pull-requests: read + uses: ./.github/workflows/coverage-baseline.yml + with: + platform: web + + coverage-base-react-native: + name: Coverage baseline (react-native) + needs: changes + if: github.event_name == 'pull_request' && needs.changes.outputs.reactNative == 'true' + permissions: + contents: read + checks: write + pull-requests: read + uses: ./.github/workflows/coverage-baseline.yml + with: + platform: react-native + + coverage-base-protocol: + name: Coverage baseline (protocol) + needs: changes + if: github.event_name == 'pull_request' && needs.changes.outputs.protocol == 'true' + permissions: + contents: read + checks: write + pull-requests: read + uses: ./.github/workflows/coverage-baseline.yml + with: + platform: protocol + scripts-test: name: Scripts needs: changes diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml new file mode 100644 index 000000000..8ac6b88a5 --- /dev/null +++ b/.github/workflows/coverage-baseline.yml @@ -0,0 +1,79 @@ +name: Coverage baseline + +on: + workflow_call: + inputs: + platform: + required: true + type: string + +permissions: + contents: read + +jobs: + test: + name: Test + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + checks: write + pull-requests: read + env: + COVERAGE_PLATFORM: ${{ inputs.platform }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Select coverage package + id: package + shell: bash + run: | + case "$COVERAGE_PLATFORM" in + web) directory=platforms/web ;; + react-native) directory=platforms/react-native ;; + protocol) directory=protocol ;; + *) exit 1 ;; + esac + echo "directory=$directory" >> "$GITHUB_OUTPUT" + + - name: Check out the PR base + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .coverage-base + persist-credentials: false + + - uses: ./.github/actions/setup + with: + node-version-file: .coverage-base/${{ steps.package.outputs.directory }}/package.json + cache-dependency-path: .coverage-base/${{ steps.package.outputs.directory }}/pnpm-lock.yaml + package-json-file: .coverage-base/${{ steps.package.outputs.directory }}/package.json + working-directory: .coverage-base/${{ steps.package.outputs.directory }} + + - name: Measure base coverage + id: tests + working-directory: .coverage-base/${{ steps.package.outputs.directory }} + shell: bash + run: | + case "$COVERAGE_PLATFORM" in + web) pnpm test ;; + react-native) pnpm test --coverage --testPathPatterns="modules/@shopify/checkout-kit-react-native/tests" --coverageReporters=json-summary ;; + protocol) pnpm test --coverage ;; + esac + + - name: Record base coverage + if: ${{ !cancelled() && steps.package.outcome == 'success' }} + env: + COVERAGE_PATH: .coverage-base/${{ steps.package.outputs.directory }}/coverage/coverage-summary.json + COVERAGE_REVISION: base + COVERAGE_BASE_SHA: ${{ github.event.pull_request.base.sha }} + COVERAGE_TEST_OUTCOME: ${{ steps.tests.outcome }} + COVERAGE_REPOSITORY: ${{ github.repository }} + COVERAGE_SHA: ${{ github.event.pull_request.head.sha }} + COVERAGE_PR: ${{ github.event.pull_request.number }} + COVERAGE_TOKEN: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }} + run: ruby scripts/report_coverage "$COVERAGE_PLATFORM" "$COVERAGE_PATH" diff --git a/scripts/lib/coverage_report.rb b/scripts/lib/coverage_report.rb index aee888cb0..50f3d3d87 100644 --- a/scripts/lib/coverage_report.rb +++ b/scripts/lib/coverage_report.rb @@ -86,18 +86,23 @@ def initialize(repository:, pr_number:, sha:, token:, source:, client: nil) end end - def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: false) + def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: false, revision: "head", base_sha: nil) + raise ArgumentError, "Unknown coverage revision" unless %w[head base].include?(revision) + raise ArgumentError, "Base coverage requires a base commit" if revision == "base" && base_sha.to_s.empty? + pr = @client.get("/repos/#{@repository}/pulls/#{@pr_number}") return "Skipped coverage result: pull request is closed or this build is superseded." unless pr["state"] == "open" && pr.dig("head", "sha") == @sha return "Skipped coverage result: fork pull request." unless pr.dig("head", "repo", "full_name") == @repository + return "Skipped coverage result: pull request base changed." if revision == "base" && pr.dig("base", "sha") != base_sha checks = check_runs - external_id = "coverage:#{platform}:#{@source.fetch('runId')}:#{@source.fetch('runAttempt', 1)}" + prefix = revision == "base" ? "coverage-base" : "coverage" + external_id = "#{prefix}:#{platform}:#{@source.fetch('runId')}:#{@source.fetch('runAttempt', 1)}" existing = checks.find { |check| check["external_id"] == external_id && check.dig("app", "slug") == @source.fetch("provider") } unless existing && preserve_existing - result = {version: 1, platform: platform, pr: @pr_number.to_i, headSha: @sha, source: @source, state: state, rows: report&.rows || [], reportUrl: report_url} + result = {version: 1, platform: platform, pr: @pr_number.to_i, headSha: @sha, baseSha: base_sha, revision: revision, source: @source, state: state, rows: report&.rows || [], reportUrl: report_url} payload = { - name: "Coverage — #{CoverageReport::TITLES.fetch(platform)}", + name: "Coverage#{revision == 'base' ? ' base' : ''} — #{CoverageReport::TITLES.fetch(platform)}", external_id: external_id, status: "completed", conclusion: state == "skipped" ? "skipped" : "neutral", diff --git a/scripts/report_coverage b/scripts/report_coverage index 7b980ecdd..9810f5db7 100644 --- a/scripts/report_coverage +++ b/scripts/report_coverage @@ -32,4 +32,5 @@ publisher = CoverageResultPublisher.new( token: token, source: source ) -puts publisher.publish(platform: platform, state: state, report: report, report_url: ENV["COVERAGE_REPORT_URL"]) +puts publisher.publish(platform: platform, state: state, report: report, report_url: ENV["COVERAGE_REPORT_URL"], + revision: ENV.fetch("COVERAGE_REVISION", "head"), base_sha: ENV["COVERAGE_BASE_SHA"]) diff --git a/scripts/test/coverage_baseline_test.rb b/scripts/test/coverage_baseline_test.rb new file mode 100644 index 000000000..9e36001ac --- /dev/null +++ b/scripts/test/coverage_baseline_test.rb @@ -0,0 +1,57 @@ +require "minitest/autorun" +require "yaml" +require "tmpdir" +require "open3" +require "fileutils" + +class CoverageBaselineTest < Minitest::Test + ROOT = File.expand_path("../..", __dir__) + + def setup + @workflow = YAML.safe_load_file(File.join(ROOT, ".github/workflows/coverage-baseline.yml")) + @job = @workflow.fetch("jobs").fetch("test") + @steps = @job.fetch("steps") + @ci = YAML.safe_load_file(File.join(ROOT, ".github/workflows/ci.yml")) + end + + def test_baselines_use_the_exact_pr_base_in_a_separate_checkout_without_persisted_credentials + checkout = @steps.find { |step| step.dig("with", "path") == ".coverage-base" } + assert_equal "${{ github.event.pull_request.base.sha }}", checkout.dig("with", "ref") + @steps.select { |step| step.fetch("uses", "").start_with?("actions/checkout@") }.each do |step| + assert_equal false, step.dig("with", "persist-credentials") + end + assert_equal true, @job.fetch("continue-on-error") + refute @job.fetch("permissions").key?("pull-requests") && @job.dig("permissions", "pull-requests") == "write" + assert_equal "${{ github.event.pull_request.base.sha }}", @steps.last.dig("env", "COVERAGE_BASE_SHA") + assert_equal "base", @steps.last.dig("env", "COVERAGE_REVISION") + end + + def test_only_affected_pr_packages_get_optional_baselines + %w[web react-native protocol].zip(%w[web reactNative protocol]).each do |platform, area| + name = "coverage-base-#{platform}" + job = @ci.fetch("jobs").fetch(name) + assert_equal "github.event_name == 'pull_request' && needs.changes.outputs.#{area} == 'true'", job.fetch("if") + assert_equal platform, job.dig("with", "platform") + refute_includes @ci.dig("jobs", "ci-required", "needs"), name + end + end + + def test_each_baseline_runs_the_same_test_command_as_its_pr_package + Dir.mktmpdir do |directory| + stub = File.join(directory, "pnpm") + File.write(stub, "#!/usr/bin/env ruby\nputs ARGV.inspect\n") + FileUtils.chmod(0o755, stub) + baseline = @steps.find { |step| step["id"] == "tests" }.fetch("run") + {"web" => "web.yml", "react-native" => "rn-test.yml", "protocol" => "protocol-test.yml"}.each do |platform, file| + workflow = YAML.safe_load_file(File.join(ROOT, ".github/workflows", file)) + head = workflow.fetch("jobs").values.flat_map { |job| job.fetch("steps", []) }.find { |step| step["id"] == "coverage_tests" }.fetch("run") + env = {"COVERAGE_PLATFORM" => platform, "PATH" => "#{directory}:#{ENV.fetch('PATH')}"} + expected, head_error, head_status = Open3.capture3(env, "bash", "-c", head) + actual, base_error, base_status = Open3.capture3(env, "bash", "-c", baseline) + assert head_status.success?, head_error + assert base_status.success?, base_error + assert_equal expected, actual, platform + end + end + end +end diff --git a/scripts/test/coverage_report_test.rb b/scripts/test/coverage_report_test.rb index 68d271d85..388a45097 100644 --- a/scripts/test/coverage_report_test.rb +++ b/scripts/test/coverage_report_test.rb @@ -84,7 +84,7 @@ def setup @report = CoverageReport.new("web", JSON.generate("total" => CoverageReport::JAVASCRIPT_METRICS.keys.to_h do |key| [key, {"covered" => 1, "total" => 2}] end)) - @pr = {"state" => "open", "head" => {"sha" => "abc123", "repo" => {"full_name" => "example/sdk"}}} + @pr = {"state" => "open", "head" => {"sha" => "abc123", "repo" => {"full_name" => "example/sdk"}}, "base" => {"sha" => "base123"}} @source = {"provider" => "github-actions", "runId" => 100, "runAttempt" => 2} end @@ -114,6 +114,33 @@ def test_updates_the_matching_check_without_creating_duplicates assert_equal [:patch, "/repos/example/sdk/check-runs/42"], @client.writes.first.take(2) end + def test_base_results_are_stored_separately_on_the_pr_head + publish(revision: "base", base_sha: "base123") + payload = @client.writes.first.last + assert_equal "abc123", payload[:head_sha] + assert_equal "coverage-base:web:100:2", payload[:external_id] + assert_equal "Coverage base — Web", payload[:name] + result = JSON.parse(payload[:output][:text]) + assert_equal "base", result.fetch("revision") + assert_equal "base123", result.fetch("baseSha") + assert_equal @report.rows, result.fetch("rows") + end + + def test_head_results_record_the_tested_base_without_overwriting_base_results + publish([{"id" => 42, "external_id" => "coverage-base:web:100:2", "app" => {"slug" => "github-actions"}}], base_sha: "base123") + assert_equal :post, @client.writes.first.first + result = JSON.parse(@client.writes.first.last[:output][:text]) + assert_equal "head", result.fetch("revision") + assert_equal "base123", result.fetch("baseSha") + end + + def test_base_results_require_the_current_base_commit + assert_raises(ArgumentError) { publish(revision: "base") } + assert_raises(ArgumentError) { publish(revision: "invalid") } + assert_includes publish(revision: "base", base_sha: "old-base"), "Skipped coverage result" + assert_empty @client.writes + end + def test_superseded_closed_and_fork_prs_are_not_published ["stale", "closed", "fork"].each do |scenario| setup