Show PR coverage deltas in the shared report

.github/CONTRIBUTING.md

@@ -107,6 +107,12 @@ show their status instead of coverage from an older run. GitHub Actions jobs als
 write their coverage tables to the job summary. Fork builds and builds without a
 write token keep their artifacts and log summaries without posting a comment.
 
+For affected JavaScript packages, separate baseline jobs measure the PR's exact
+base commit. The comment shows changes in percentage points beside each metric,
+for example `91.42% (+1.27)`. Deltas require successful measurements for the same
+base commit; missing or failed baselines leave the current percentage visible
+without a delta. Baseline jobs do not gate the PR or replace its test results.
+
 Platform jobs store numeric results in GitHub checks. The `Coverage Report`
 workflow is the only comment writer and serializes updates per PR. It refreshes
 on CI workflow events, reading all current results so simultaneous completions

.github/actions/report-coverage/action.yml

@@ -32,6 +32,7 @@ runs:
         COVERAGE_TEST_OUTCOME: ${{ inputs.outcome }}
         COVERAGE_REPOSITORY: ${{ github.repository }}
         COVERAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        COVERAGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
         COVERAGE_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }}
         COVERAGE_TOKEN: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }}
         COVERAGE_REPORT_URL: ${{ steps.artifact.outputs.artifact-url }}

.github/scripts/publish-coverage.cjs

@@ -53,9 +53,12 @@ function readResult(check, platform, expected, pr, repo, run) {
   const result = JSON.parse(check.output.text);
   if (result.version !== 1 || result.platform !== platform.id || result.pr !== pr.number ||
       result.headSha !== pr.head.sha || result.source?.provider !== expected.provider ||
+      (result.revision ?? "head") !== (expected.revision ?? "head") ||
       result.source.runId !== expected.runId || (result.source.runAttempt ?? 1) !== expected.runAttempt ||
       !["success", "failed", "skipped", "unavailable"].includes(result.state) || !Array.isArray(result.rows))
     throw new Error("Coverage result does not match this run");
+  if (expected.revision === "base" && result.baseSha !== pr.base.sha)
+    throw new Error("Coverage baseline does not match the PR base");
   if (["skipped", "unavailable"].includes(result.state) && result.rows.length)
     throw new Error("Unexpected measurements for an unmeasured platform");
   if (result.rows.length || result.state === "success") {
@@ -69,23 +72,26 @@ function readResult(check, platform, expected, pr, repo, run) {
   return {...result, reportUrl: reportURL(result.reportUrl, repo, run)};
 }
 
-function platformResult(platform, snapshot, jobs, pr, repo, core) {
+function platformResult(platform, snapshot, jobs, pr, repo, core, revision = "head") {
   const {run, checks} = snapshot;
   let state = "pending";
   if (!run) return {state};
   // A failed-jobs rerun reuses successful jobs from an earlier attempt. Match
   // each result to the attempt that actually ran that platform's test job.
-  const job = latest(jobs.filter((item) => item.name === platform.job)) ||
-    latest(jobs.filter((item) => item.name === (platform.skippedJob || platform.title)));
-  const expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt};
+  const baseline = revision === "base";
+  const jobName = baseline ? `Coverage baseline (${platform.id}) / Test` : platform.job;
+  const skippedJob = baseline ? `Coverage baseline (${platform.id})` : platform.skippedJob || platform.title;
+  const job = latest(jobs.filter((item) => item.name === jobName)) ||
+    latest(jobs.filter((item) => item.name === skippedJob));
+  const expected = {provider: "github-actions", runId: run.id, runAttempt: job?.run_attempt ?? run.run_attempt, revision};
   const plan = latest(jobs.filter((item) => item.name === "Detect Changed Areas"));
   if (plan?.status === "completed" && plan.conclusion !== "success") return {state: "unavailable"};
   if (job?.conclusion === "skipped") return {state: "skipped"};
   if (job?.status === "completed") state = job.conclusion === "success" ? "unavailable" : "failed";
   else if (!job && run.status === "completed") state = "unavailable";
-  const externalId = `coverage:${platform.id}:${expected.runId}:${expected.runAttempt}`;
+  const externalId = `coverage${baseline ? "-base" : ""}:${platform.id}:${expected.runId}:${expected.runAttempt}`;
   const check = latest(checks.filter((item) => item.app?.slug === expected.provider &&
-    item.external_id === externalId && item.name === `Coverage — ${platform.title}`));
+    item.external_id === externalId && item.name === `Coverage${baseline ? " base" : ""} — ${platform.title}`));
   if (!check) return {state};
   try {
     return readResult(check, platform, expected, pr, repo, run);
@@ -95,15 +101,17 @@ function platformResult(platform, snapshot, jobs, pr, repo, core) {
   }
 }
 
-function metric(row) {
+function metric(row, baseline) {
   if (!row) return "—";
   const [, covered, total] = row;
   if (!total) return "N/A";
   const percentage = Number((100 * covered / total).toFixed(2));
-  return `${percentage}%`;
+  if (!baseline?.[2]) return `${percentage}%`;
+  const delta = Number((100 * covered / total - 100 * baseline[1] / baseline[2]).toFixed(2));
+  return `${percentage}% **(${delta > 0 ? "+" : ""}${delta})**`;
 }
 
-function render(results) {
+function render(results, baselines = {}) {
   const lines = [marker, "# Coverage Report", "",
     "| Status | Platform / target | Lines | Branches | Functions | Report |",
     "| :---: | --- | ---: | ---: | ---: | --- |"];
@@ -118,8 +126,13 @@ function render(results) {
     const [emoji, status] = statuses[result.state];
     const report = result.reportUrl ? `[Full report](${result.reportUrl})` : status;
     const find = (name) => rows.find((row) => row[0] === name);
-    lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${metric(find("Lines"))} | ${metric(find("Branches"))} | ${metric(find("Functions"))} | ${report} |`);
+    const baseline = baselines[platform.id];
+    const comparable = result.state === "success" && baseline?.state === "success" &&
+      result.baseSha && result.baseSha === baseline.baseSha;
+    const value = (name) => metric(find(name), comparable ? baseline.rows.find((row) => row[0] === name) : undefined);
+    lines.push(`| ${emoji} | ${platform.displayTitle || platform.title} | ${value("Lines")} | ${value("Branches")} | ${value("Functions")} | ${report} |`);
   }
+  lines.push("", "Changes in parentheses are **percentage points versus the PR’s base commit**. No delta appears when matching base coverage is unavailable.");
   return lines.join("\n") + "\n";
 }
 
@@ -136,7 +149,9 @@ async function publish({github, context, core, prNumber}) {
   }) : [];
   const results = Object.fromEntries(platforms.map((platform) =>
     [platform.id, platformResult(platform, snapshot, jobs, pr, repo, core)]));
-  const body = render(results);
+  const baselines = Object.fromEntries(platforms.map((platform) =>
+    [platform.id, platformResult(platform, snapshot, jobs, pr, repo, core, "base")]));
+  const body = render(results, baselines);
   const comments = await github.paginate(github.rest.issues.listComments, {...repo, issue_number: pr.number, per_page: 100});
   const existing = comments.find(isComment);
   // Wait for change detection before opening a comment for an affected platform.

.github/scripts/publish-coverage.test.cjs

@@ -36,10 +36,11 @@ function fixture() {
   f.add = (id, overrides = {}) => {
     const platform = reporter.platforms.find((platform) => platform.id === id);
     const source = {provider: "github-actions", runId: 100, runAttempt: 1};
-    const result = {version: 1, platform: id, pr: 12, headSha: "head", source, state: "success", rows: platform.metrics.map((name) => [name, 7, 10]), ...overrides};
+    const result = {version: 1, platform: id, pr: 12, headSha: "head", baseSha: "base", source, state: "success", rows: platform.metrics.map((name) => [name, 7, 10]), ...overrides};
+    const baseline = result.revision === "base";
     const check = {
-      id: 400 + f.checks.length, name: `Coverage — ${platform.title}`, head_sha: "head",
-      app: {slug: source.provider}, external_id: `coverage:${id}:${source.runId}:${source.runAttempt}`,
+      id: 400 + f.checks.length, name: `Coverage${baseline ? " base" : ""} — ${platform.title}`, head_sha: "head",
+      app: {slug: source.provider}, external_id: `coverage${baseline ? "-base" : ""}:${id}:${result.source.runId}:${result.source.runAttempt}`,
       output: {text: JSON.stringify(result)},
     };
     f.checks.push(check);
@@ -48,6 +49,80 @@ function fixture() {
   return f;
 }
 
+function addBaseline(f, overrides = {}) {
+  f.jobs.push({id: 600 + f.jobs.length, name: "Coverage baseline (web) / Test", run_attempt: 1, status: "completed", conclusion: "success"});
+  return f.add("web", {revision: "base", ...overrides});
+}
+
+test("shows signed percentage-point deltas from exact base counts beside each metric", async () => {
+  const f = fixture();
+  f.add("web");
+  addBaseline(f, {rows: [["Lines", 6, 10], ["Statements", 7, 10], ["Branches", 8, 10], ["Functions", 14, 20]]});
+  await f.publish();
+  assert.ok(row(f.writes[0].body, "Web").includes("70% **(+10)** | 70% **(-10)** | 70% **(0)**"));
+  assert.ok(f.writes[0].body.includes("percentage points versus the PR’s base commit"));
+});
+
+test("rounds the difference once and does not display negative zero", async () => {
+  const f = fixture();
+  f.add("web", {rows: [["Lines", 667, 1000], ["Statements", 7, 10], ["Branches", 699999, 1000000], ["Functions", 0, 0]]});
+  addBaseline(f, {rows: [["Lines", 2, 3], ["Statements", 7, 10], ["Branches", 7, 10], ["Functions", 7, 10]]});
+  await f.publish();
+  assert.ok(row(f.writes[0].body, "Web").includes("66.7% **(+0.03)** | 70% **(0)** | N/A"));
+});
+
+test("never presents missing, failed, stale, or empty baselines as zero change", async () => {
+  for (const overrides of [
+    {baseSha: "old-base"}, {state: "failed"}, {state: "unavailable", rows: []},
+    {rows: reporter.platforms[0].metrics.map((name) => [name, 0, 0])},
+    {source: {provider: "github-actions", runId: 99, runAttempt: 1}},
+  ]) {
+    const f = fixture();
+    f.add("web");
+    addBaseline(f, overrides);
+    await f.publish();
+    assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |"));
+  }
+});
+
+test("legacy, stale-base and failed head reports retain percentages without deltas", async () => {
+  for (const overrides of [{baseSha: undefined}, {baseSha: "old-base"}, {state: "failed"}]) {
+    const f = fixture();
+    f.add("web", overrides);
+    addBaseline(f);
+    await f.publish();
+    assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |"));
+  }
+});
+
+test("baseline reruns cannot reuse an earlier attempt but preserve the head report", async () => {
+  const f = fixture();
+  f.add("web");
+  addBaseline(f);
+  f.runs[0].run_attempt = 2;
+  f.jobs.push({id: 999, name: "Coverage baseline (web) / Test", run_attempt: 2, status: "in_progress"});
+  await f.publish();
+  assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |"));
+  f.add("web", {revision: "base", source: {provider: "github-actions", runId: 100, runAttempt: 2}});
+  await f.publish();
+  assert.ok(row(f.writes[1].body, "Web").includes("70% **(0)**"));
+});
+
+test("malformed and untrusted baselines cannot hide valid head coverage", async () => {
+  for (const mutate of [
+    (check) => { check.app.slug = "untrusted-app"; },
+    (check) => { check.output.text = "x".repeat(32769); },
+    (check) => { const result = JSON.parse(check.output.text); result.rows[0][1] = 11; check.output.text = JSON.stringify(result); },
+    (check) => { const result = JSON.parse(check.output.text); result.revision = "head"; check.output.text = JSON.stringify(result); },
+  ]) {
+    const f = fixture();
+    f.add("web");
+    mutate(addBaseline(f));
+    await f.publish();
+    assert.ok(row(f.writes[0].body, "Web").includes("70% | 70% | 70% |"));
+  }
+});
+
 test("one comment combines the existing JavaScript reports without native coverage rows", async () => {
   const f = fixture();
   for (const platform of reporter.platforms) f.add(platform.id);

.github/workflows/ci.yml

@@ -70,6 +70,7 @@ jobs:
             # Common React Native infra: JS-level jobs (jest, lint, packed
             # files) and shared config that should trigger all RN jobs.
             reactNativeCommon:
+              - '.github/workflows/coverage-baseline.yml'
               - '.github/actions/report-coverage/**'
               - 'scripts/report_coverage'
               - 'scripts/lib/coverage_report.rb'
@@ -90,6 +91,7 @@ jobs:
               - '.github/workflows/rn-test-android.yml'
               - '.github/workflows/rn-build-android.yml'
             web:
+              - '.github/workflows/coverage-baseline.yml'
               - '.github/actions/report-coverage/**'
               - 'scripts/report_coverage'
               - 'scripts/lib/coverage_report.rb'
@@ -103,6 +105,7 @@ jobs:
               - '.ci/changed-file-filters.yml'
               - '.github/workflows/ci.yml'
             protocol:
+              - '.github/workflows/coverage-baseline.yml'
               - '.github/actions/report-coverage/**'
               - 'scripts/report_coverage'
               - 'scripts/lib/coverage_report.rb'
@@ -113,6 +116,7 @@ jobs:
             scripts:
               - '.github/scripts/*coverage*'
               - '.github/workflows/coverage-report.yml'
+              - '.github/workflows/coverage-baseline.yml'
               - '.github/actions/report-coverage/**'
               - 'scripts/report_coverage'
               - '.github/scripts/measure-package-size'
@@ -220,6 +224,42 @@ jobs:
       pull-requests: read
     uses: ./.github/workflows/protocol-test.yml
 
+  coverage-base-web:
+    name: Coverage baseline (web)
+    needs: changes
+    if: github.event_name == 'pull_request' && needs.changes.outputs.web == 'true'
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: read
+    uses: ./.github/workflows/coverage-baseline.yml
+    with:
+      platform: web
+
+  coverage-base-react-native:
+    name: Coverage baseline (react-native)
+    needs: changes
+    if: github.event_name == 'pull_request' && needs.changes.outputs.reactNative == 'true'
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: read
+    uses: ./.github/workflows/coverage-baseline.yml
+    with:
+      platform: react-native
+
+  coverage-base-protocol:
+    name: Coverage baseline (protocol)
+    needs: changes
+    if: github.event_name == 'pull_request' && needs.changes.outputs.protocol == 'true'
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: read
+    uses: ./.github/workflows/coverage-baseline.yml
+    with:
+      platform: protocol
+
   scripts-test:
     name: Scripts
     needs: changes

.github/workflows/coverage-baseline.yml

@@ -0,0 +1,79 @@
+name: Coverage baseline
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    continue-on-error: true
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: read
+    env:
+      COVERAGE_PLATFORM: ${{ inputs.platform }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Select coverage package
+        id: package
+        shell: bash
+        run: |
+          case "$COVERAGE_PLATFORM" in
+            web) directory=platforms/web ;;
+            react-native) directory=platforms/react-native ;;
+            protocol) directory=protocol ;;
+            *) exit 1 ;;
+          esac
+          echo "directory=$directory" >> "$GITHUB_OUTPUT"
+
+      - name: Check out the PR base
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: .coverage-base
+          persist-credentials: false
+
+      - uses: ./.github/actions/setup
+        with:
+          node-version-file: .coverage-base/${{ steps.package.outputs.directory }}/package.json
+          cache-dependency-path: .coverage-base/${{ steps.package.outputs.directory }}/pnpm-lock.yaml
+          package-json-file: .coverage-base/${{ steps.package.outputs.directory }}/package.json
+          working-directory: .coverage-base/${{ steps.package.outputs.directory }}
+
+      - name: Measure base coverage
+        id: tests
+        working-directory: .coverage-base/${{ steps.package.outputs.directory }}
+        shell: bash
+        run: |
+          case "$COVERAGE_PLATFORM" in
+            web) pnpm test ;;
+            react-native) pnpm test --coverage --testPathPatterns="modules/@shopify/checkout-kit-react-native/tests" --coverageReporters=json-summary ;;
+            protocol) pnpm test --coverage ;;
+          esac
+
+      - name: Record base coverage
+        if: ${{ !cancelled() && steps.package.outcome == 'success' }}
+        env:
+          COVERAGE_PATH: .coverage-base/${{ steps.package.outputs.directory }}/coverage/coverage-summary.json
+          COVERAGE_REVISION: base
+          COVERAGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          COVERAGE_TEST_OUTCOME: ${{ steps.tests.outcome }}
+          COVERAGE_REPOSITORY: ${{ github.repository }}
+          COVERAGE_SHA: ${{ github.event.pull_request.head.sha }}
+          COVERAGE_PR: ${{ github.event.pull_request.number }}
+          COVERAGE_TOKEN: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }}
+        run: ruby scripts/report_coverage "$COVERAGE_PLATFORM" "$COVERAGE_PATH"

scripts/lib/coverage_report.rb

@@ -86,18 +86,23 @@ def initialize(repository:, pr_number:, sha:, token:, source:, client: nil)
     end
   end
 
-  def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: false)
+  def publish(platform:, state:, report: nil, report_url: nil, preserve_existing: false, revision: "head", base_sha: nil)
+    raise ArgumentError, "Unknown coverage revision" unless %w[head base].include?(revision)
+    raise ArgumentError, "Base coverage requires a base commit" if revision == "base" && base_sha.to_s.empty?
+
     pr = @client.get("/repos/#{@repository}/pulls/#{@pr_number}")
     return "Skipped coverage result: pull request is closed or this build is superseded." unless pr["state"] == "open" && pr.dig("head", "sha") == @sha
     return "Skipped coverage result: fork pull request." unless pr.dig("head", "repo", "full_name") == @repository
+    return "Skipped coverage result: pull request base changed." if revision == "base" && pr.dig("base", "sha") != base_sha
 
     checks = check_runs
-    external_id = "coverage:#{platform}:#{@source.fetch('runId')}:#{@source.fetch('runAttempt', 1)}"
+    prefix = revision == "base" ? "coverage-base" : "coverage"
+    external_id = "#{prefix}:#{platform}:#{@source.fetch('runId')}:#{@source.fetch('runAttempt', 1)}"
     existing = checks.find { |check| check["external_id"] == external_id && check.dig("app", "slug") == @source.fetch("provider") }
     unless existing && preserve_existing
-      result = {version: 1, platform: platform, pr: @pr_number.to_i, headSha: @sha, source: @source, state: state, rows: report&.rows || [], reportUrl: report_url}
+      result = {version: 1, platform: platform, pr: @pr_number.to_i, headSha: @sha, baseSha: base_sha, revision: revision, source: @source, state: state, rows: report&.rows || [], reportUrl: report_url}
       payload = {
-        name: "Coverage — #{CoverageReport::TITLES.fetch(platform)}",
+        name: "Coverage#{revision == 'base' ? ' base' : ''} — #{CoverageReport::TITLES.fetch(platform)}",
         external_id: external_id,
         status: "completed",
         conclusion: state == "skipped" ? "skipped" : "neutral",