From 35a66a349b94aa149fc5989d0661269c5ef4ef6e Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Tue, 6 Oct 2026 16:33:14 +0100 Subject: [PATCH 1/6] Report Swift and Android test coverage --- .github/CONTRIBUTING.md | 20 +++++- .github/workflows/android-test.yml | 12 +++- dev.yml | 11 +++ e2e/bitrise.yml | 8 ++- platforms/android/lib/build.gradle | 14 ++++ platforms/swift/Scripts/test_coverage | 18 +++++ platforms/swift/Scripts/xcode_run | 8 ++- scripts/test/swift_coverage_test.rb | 98 +++++++++++++++++++++++++++ 8 files changed, 182 insertions(+), 7 deletions(-) create mode 100755 platforms/swift/Scripts/test_coverage create mode 100644 scripts/test/swift_coverage_test.rb diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index eac8e669b..50bb67c95 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -96,8 +96,24 @@ pnpm --dir protocol test --coverage A failing threshold fails the test command and its CI job. Add tests for the uncovered behavior instead of lowering the threshold. The thresholds live in `platforms/web/vite.config.ts`, `platforms/react-native/jest.config.js`, and -`protocol/vitest.config.ts`. Swift, Android, and standalone telemetry tests do -not yet enforce coverage thresholds. +`protocol/vitest.config.ts`. + +Swift and Android generate coverage reports without enforcing a minimum yet: + +| Platform | Local command | Reports | CI artifact | +| --- | --- | --- | --- | +| Android | `dev android test coverage` | `platforms/android/lib/build/reports/coverage/test/debug/` (HTML and XML) | `android-coverage` in the GitHub Actions Android test job | +| Swift | `dev swift test coverage` | `.xcresults/coverage/` (JSON and text) | Coverage reports in the Bitrise Swift package test job | + +Android reports cover the library's debug unit tests, including Robolectric +tests and the library's telemetry code. Swift reports contain per-target and +per-file line coverage from the package tests; inspect `ShopifyCheckoutKit` and +`ShopifyAcceleratedCheckouts` for the SDK coverage. Xcode also lists test and +dependency targets, sometimes with no executable lines; compare the named +production targets rather than the overall total. The full Swift result bundle +is available locally at `.xcresults/ShopifyCheckoutKit-Package-coverage.xcresult`. +Neither report includes sample app or end-to-end tests. Standalone TypeScript +telemetry tests do not yet collect coverage. ### Testing PR builds with Tophat diff --git a/.github/workflows/android-test.yml b/.github/workflows/android-test.yml index 574cda06f..207f4d9e3 100644 --- a/.github/workflows/android-test.yml +++ b/.github/workflows/android-test.yml @@ -28,8 +28,16 @@ jobs: java-version: 17 cache: 'gradle' - - name: Run Tests - run: ./gradlew test --console=plain + - name: Run tests and generate library coverage + run: ./gradlew test :lib:createDebugUnitTestCoverageReport --console=plain + + - name: Upload library coverage reports + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: android-coverage + path: platforms/android/lib/build/reports/coverage/test/debug/ + if-no-files-found: error + retention-days: 7 - name: Run Protocol Unit Tests working-directory: protocol/languages/kotlin diff --git a/dev.yml b/dev.yml index 3a9ede7c9..93741df5a 100644 --- a/dev.yml +++ b/dev.yml @@ -377,6 +377,14 @@ commands: platforms/android/gradlew -p platforms/android clean test --console=plain platforms/android/samples/CheckoutKitAndroidDemo/gradlew -p platforms/android/samples/CheckoutKitAndroidDemo :app:testDebugUnitTest --console=plain subcommands: + coverage: + desc: Run Android library unit tests and generate HTML/XML coverage reports + syntax: + optional: --local + run: | + source platforms/android/scripts/parse_local_protocol_flag + parse_local_protocol_flag "$@" + platforms/android/gradlew -p platforms/android :lib:createDebugUnitTestCoverageReport --console=plain samples: desc: Run JVM tests for the Checkout Kit Android sample app syntax: @@ -654,6 +662,9 @@ commands: ./Scripts/test_samples fi subcommands: + coverage: + desc: Run Swift package tests and export Xcode coverage reports + run: cd platforms/swift && ./Scripts/test_coverage sample: desc: Run the sample app test target syntax: "[test_class_name]" diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index c17ac7731..29dfbc674 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -342,14 +342,18 @@ workflows: - git-clone@8: {} - bundle::create-ios-simulator: {} - script@1: - title: Run Swift package tests + title: Run Swift package tests with coverage timeout: 3600 no_output_timeout: 1800 inputs: - content: |- set -euo pipefail cd platforms/swift - ./Scripts/xcode_run test ShopifyCheckoutKit-Package + ./Scripts/test_coverage + - deploy-to-bitrise-io@2: + is_always_run: false + inputs: + - deploy_path: "$BITRISE_SOURCE_DIR/.xcresults/coverage" ci-ios-swift-samples: meta: diff --git a/platforms/android/lib/build.gradle b/platforms/android/lib/build.gradle index 8592eea4d..3c1a14039 100644 --- a/platforms/android/lib/build.gradle +++ b/platforms/android/lib/build.gradle @@ -73,6 +73,9 @@ android { } buildTypes { + debug { + enableUnitTestCoverage true + } release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt') @@ -180,6 +183,17 @@ tasks.withType(Test).configureEach { dependsOn verifyPublishedProtocol } +plugins.withId('jacoco') { + tasks.withType(Test).configureEach { + jacoco { + // Robolectric loads SDK classes without a source location. Include + // those classes, but avoid instrumenting its runtime or JDK internals. + includeNoLocationClasses = true + includes = ['com.shopify.checkoutkit.*'] + } + } +} + // Local protocol sources are for joint development only. Maven Local remains // available for React Native's explicit --local builds. tasks.withType(PublishToMavenRepository).configureEach { diff --git a/platforms/swift/Scripts/test_coverage b/platforms/swift/Scripts/test_coverage new file mode 100755 index 000000000..a20fdf76a --- /dev/null +++ b/platforms/swift/Scripts/test_coverage @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +report_dir="$repo_root/.xcresults/coverage" +result_bundle="$repo_root/.xcresults/ShopifyCheckoutKit-Package-coverage.xcresult" + +# Remove old exports before running so a failed test run cannot publish stale coverage. +mkdir -p "$report_dir" +rm -f "$report_dir/coverage.json" "$report_dir/coverage.txt" + +ENABLE_CODE_COVERAGE=1 "$script_dir/xcode_run" test ShopifyCheckoutKit-Package + +xcrun xccov view --report --json "$result_bundle" > "$report_dir/coverage.json" +xcrun xccov view --report "$result_bundle" > "$report_dir/coverage.txt" +xcrun xccov view --report --only-targets "$result_bundle" +printf '\nCoverage reports: %s\n' "$report_dir" diff --git a/platforms/swift/Scripts/xcode_run b/platforms/swift/Scripts/xcode_run index 90e363cb6..abe3c792b 100755 --- a/platforms/swift/Scripts/xcode_run +++ b/platforms/swift/Scripts/xcode_run @@ -42,15 +42,21 @@ xcodebuild_cmd="xcodebuild $ACTION -scheme $SCHEME -sdk iphonesimulator -destina # Retry failed tests up to three total attempts, including the initial run. if [[ "$ACTION" == *"test"* ]]; then xcodebuild_cmd="$xcodebuild_cmd -retry-tests-on-failure -test-iterations 3" + if [[ "${ENABLE_CODE_COVERAGE:-0}" == "1" ]]; then + xcodebuild_cmd="$xcodebuild_cmd -enableCodeCoverage YES" + fi fi if [[ "$SCHEME" == "ShopifyCheckoutKit-Package" && "$ACTION" == *"build"* && "$ACTION" != *"test"* ]]; then xcodebuild_cmd="$xcodebuild_cmd SWIFT_TREAT_WARNINGS_AS_ERRORS=YES" fi -if [[ "$CI" == "true" && ( "$ACTION" == *"build"* || "$ACTION" == *"test"* ) ]]; then +if [[ ( "$CI" == "true" || "${ENABLE_CODE_COVERAGE:-0}" == "1" ) && ( "$ACTION" == *"build"* || "$ACTION" == *"test"* ) ]]; then result_bundle_dir="$(cd "$SCRIPT_DIR/../../.." && pwd)/.xcresults" result_bundle_path="$result_bundle_dir/$SCHEME.xcresult" + if [[ "${ENABLE_CODE_COVERAGE:-0}" == "1" ]]; then + result_bundle_path="$result_bundle_dir/$SCHEME-coverage.xcresult" + fi rm -rf "$result_bundle_path" mkdir -p "$result_bundle_dir" xcodebuild_cmd="$xcodebuild_cmd -resultBundlePath \"$result_bundle_path\"" diff --git a/scripts/test/swift_coverage_test.rb b/scripts/test/swift_coverage_test.rb new file mode 100644 index 000000000..bead02b62 --- /dev/null +++ b/scripts/test/swift_coverage_test.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require "fileutils" +require "minitest/autorun" +require "open3" +require "tmpdir" + +class SwiftCoverageTest < Minitest::Test + SCRIPTS = File.expand_path("../../platforms/swift/Scripts", __dir__) + + def setup + @directory = Dir.mktmpdir("swift coverage ") + @scripts = File.join(@directory, "platforms/swift/Scripts") + @bin = File.join(@directory, "bin") + @reports = File.join(@directory, ".xcresults/coverage") + @arguments = File.join(@directory, "xcodebuild-arguments") + FileUtils.mkdir_p([@scripts, @bin, @reports]) + File.write(File.join(@directory, "Package.swift"), "// test package\n") + %w[xcode_run test_coverage].each { |name| FileUtils.cp(File.join(SCRIPTS, name), @scripts) } + executable("xcodebuild", <<~BASH) + printf '%s\\n' "$@" > "$COVERAGE_TEST_ARGUMENTS" + exit "${COVERAGE_TEST_BUILD_STATUS:-0}" + BASH + executable("xcbeautify", 'cat') + executable("xcrun", <<~BASH) + [ "$1" = xccov ] || exit 1 + [ "${COVERAGE_TEST_EXPORT_STATUS:-0}" = 0 ] || exit "$COVERAGE_TEST_EXPORT_STATUS" + if [[ " $* " == *" --json "* ]]; then + printf '%s\\n' '{"targets":[]}' + else + printf '%s\\n' 'Swift coverage report' + fi + BASH + end + + def teardown + FileUtils.remove_entry(@directory) + end + + def test_coverage_enables_instrumentation_and_exports_reports_locally + output, error, status = run_script("test_coverage") + + assert status.success?, "#{output}#{error}" + arguments = File.readlines(@arguments, chomp: true) + assert_includes arguments, "-disableAutomaticPackageResolution" + assert_equal "YES", arguments[arguments.index("-enableCodeCoverage") + 1] + assert_equal File.join(@directory, ".xcresults/ShopifyCheckoutKit-Package-coverage.xcresult"), + arguments[arguments.index("-resultBundlePath") + 1] + assert_equal "{\"targets\":[]}\n", File.read(File.join(@reports, "coverage.json")) + assert_equal "Swift coverage report\n", File.read(File.join(@reports, "coverage.txt")) + end + + def test_failed_tests_remove_stale_exports_and_fail_the_command + %w[coverage.json coverage.txt].each { |name| File.write(File.join(@reports, name), "stale") } + + _output, _error, status = run_script("test_coverage", {"COVERAGE_TEST_BUILD_STATUS" => "65"}) + + refute status.success? + assert_empty Dir.children(@reports) + end + + def test_export_failure_fails_the_command + _output, _error, status = run_script("test_coverage", {"COVERAGE_TEST_EXPORT_STATUS" => "1"}) + + refute status.success? + end + + def test_regular_local_tests_do_not_enable_coverage_or_request_a_result_bundle + output, error, status = run_script("xcode_run", {}, "test", "ShopifyCheckoutKit-Package") + + assert status.success?, "#{output}#{error}" + arguments = File.readlines(@arguments, chomp: true) + refute_includes arguments, "-enableCodeCoverage" + refute_includes arguments, "-resultBundlePath" + end + + private + + def executable(name, body) + path = File.join(@bin, name) + File.write(path, "#!/bin/bash\nset -euo pipefail\n#{body}\n") + FileUtils.chmod(0o755, path) + end + + def run_script(name, environment = {}, *arguments) + Open3.capture3( + { + "PATH" => "#{@bin}:#{ENV.fetch("PATH")}", + "CI" => "false", + "ENABLE_CODE_COVERAGE" => "0", + "CURRENT_SIMULATOR_UUID" => "test-simulator", + "COVERAGE_TEST_ARGUMENTS" => @arguments + }.merge(environment), + "bash", File.join(@scripts, name), *arguments, + chdir: @directory + ) + end +end From e119b1fe5c642ddb5cc7a4eafcd9a2ca8a1180c3 Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Tue, 6 Oct 2026 16:42:00 +0100 Subject: [PATCH 2/6] Require pinned package resolution in the coverage test stub --- scripts/test/swift_coverage_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test/swift_coverage_test.rb b/scripts/test/swift_coverage_test.rb index bead02b62..d4ebc67fc 100644 --- a/scripts/test/swift_coverage_test.rb +++ b/scripts/test/swift_coverage_test.rb @@ -18,6 +18,8 @@ def setup File.write(File.join(@directory, "Package.swift"), "// test package\n") %w[xcode_run test_coverage].each { |name| FileUtils.cp(File.join(SCRIPTS, name), @scripts) } executable("xcodebuild", <<~BASH) + # Reject invocations that could silently change the package pins. + [[ " $* " == *" -disableAutomaticPackageResolution "* ]] || exit 2 printf '%s\\n' "$@" > "$COVERAGE_TEST_ARGUMENTS" exit "${COVERAGE_TEST_BUILD_STATUS:-0}" BASH From b3c95b36747c2ca1ec7266253ea10503b7e0ee22 Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Tue, 6 Oct 2026 17:33:39 +0100 Subject: [PATCH 3/6] Publish native coverage summaries on pull requests --- .ci/changed-file-filters.yml | 2 + .github/CONTRIBUTING.md | 6 + .github/workflows/android-test.yml | 16 ++ .github/workflows/ci.yml | 6 + e2e/bitrise.yml | 11 ++ scripts/lib/native_coverage_report.rb | 121 +++++++++++++++ scripts/report_native_coverage | 29 ++++ scripts/test/native_coverage_report_test.rb | 160 ++++++++++++++++++++ 8 files changed, 351 insertions(+) create mode 100644 scripts/lib/native_coverage_report.rb create mode 100644 scripts/report_native_coverage create mode 100644 scripts/test/native_coverage_report_test.rb diff --git a/.ci/changed-file-filters.yml b/.ci/changed-file-filters.yml index 6d26256c9..8690e9229 100644 --- a/.ci/changed-file-filters.yml +++ b/.ci/changed-file-filters.yml @@ -61,6 +61,8 @@ ciFilters: # decide which macOS jobs run, so a change to one has to rerun them all, while a Maestro # flow edit under e2e/ must not. iosCiConfig: + - 'scripts/report_native_coverage' + - 'scripts/lib/native_coverage_report.rb' - 'e2e/bitrise.yml' - 'e2e/config/ios_ci.yml' - 'e2e/lib/ios_ci_run_plan.rb' diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 50bb67c95..797acca09 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -105,6 +105,12 @@ Swift and Android generate coverage reports without enforcing a minimum yet: | Android | `dev android test coverage` | `platforms/android/lib/build/reports/coverage/test/debug/` (HTML and XML) | `android-coverage` in the GitHub Actions Android test job | | Swift | `dev swift test coverage` | `.xcresults/coverage/` (JSON and text) | Coverage reports in the Bitrise Swift package test job | +CI updates a coverage comment per native platform on same-repository pull requests, +with percentages, covered/total counts, the tested PR head, and a link to the full +reports. Android also writes its coverage table to the GitHub Actions job summary. +Fork builds and builds without a write token keep the artifacts and log summaries +without posting a comment. These summaries report coverage without enforcing 85%. + Android reports cover the library's debug unit tests, including Robolectric tests and the library's telemetry code. Swift reports contain per-target and per-file line coverage from the package tests; inspect `ShopifyCheckoutKit` and diff --git a/.github/workflows/android-test.yml b/.github/workflows/android-test.yml index 207f4d9e3..721669496 100644 --- a/.github/workflows/android-test.yml +++ b/.github/workflows/android-test.yml @@ -11,6 +11,9 @@ jobs: test: runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + contents: read + pull-requests: write defaults: run: working-directory: platforms/android @@ -32,6 +35,7 @@ jobs: run: ./gradlew test :lib:createDebugUnitTestCoverageReport --console=plain - name: Upload library coverage reports + id: coverage_artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: android-coverage @@ -39,6 +43,18 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Summarize library coverage + working-directory: ${{ github.workspace }} + env: + COVERAGE_REPOSITORY: ${{ github.repository }} + COVERAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + COVERAGE_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} + # Fork and Dependabot tokens cannot write PR comments. The job summary + # and downloadable reports remain available for those builds. + COVERAGE_TOKEN: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }} + COVERAGE_REPORT_URL: ${{ steps.coverage_artifact.outputs.artifact-url }} + run: ruby scripts/report_native_coverage android platforms/android/lib/build/reports/coverage/test/debug/report.xml + - name: Run Protocol Unit Tests working-directory: protocol/languages/kotlin run: ./gradlew test --console=plain diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e8a97972..0da035a52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,8 @@ jobs: with: filters: | android: + - 'scripts/report_native_coverage' + - 'scripts/lib/native_coverage_report.rb' - '.github/workflows/android-publish.yml' - '.github/workflows/android-protocol-publish.yml' - '.github/workflows/android-test.yml' @@ -102,6 +104,7 @@ jobs: - '.ci/changed-file-filters.yml' - '.github/workflows/ci.yml' scripts: + - 'scripts/report_native_coverage' - '.github/scripts/measure-package-size' - '.github/scripts/*bundle-size*' - '.github/workflows/package-size.yml' @@ -127,6 +130,9 @@ jobs: name: Android needs: changes if: needs.changes.outputs.android == 'true' + permissions: + contents: read + pull-requests: write uses: ./.github/workflows/android-test.yml swift-lint: diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 29dfbc674..ad8157a1b 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -354,6 +354,17 @@ workflows: is_always_run: false inputs: - deploy_path: "$BITRISE_SOURCE_DIR/.xcresults/coverage" + - script@1: + title: Report Swift coverage on the pull request + inputs: + - content: |- + set -euo pipefail + export COVERAGE_REPOSITORY="Shopify/checkout-kit" + export COVERAGE_PR="${BITRISE_PULL_REQUEST:-}" + export COVERAGE_SHA="${BITRISE_GIT_COMMIT:-}" + export COVERAGE_REPORT_URL="${BITRISE_BUILD_URL:-}" + export COVERAGE_TOKEN="${OVERRIDE_GITHUB_TOKEN:-${GIT_HTTP_PASSWORD:-}}" + ruby scripts/report_native_coverage swift .xcresults/coverage/coverage.json ci-ios-swift-samples: meta: diff --git a/scripts/lib/native_coverage_report.rb b/scripts/lib/native_coverage_report.rb new file mode 100644 index 000000000..3e214aebd --- /dev/null +++ b/scripts/lib/native_coverage_report.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require "json" +require "rexml/document" +require_relative "json_http_client" + +class NativeCoverageReport + SWIFT_TARGETS = %w[ShopifyCheckoutKit ShopifyAcceleratedCheckouts].freeze + ANDROID_METRICS = {"LINE" => "Lines", "BRANCH" => "Branches", "METHOD" => "Methods", "INSTRUCTION" => "Instructions"}.freeze + + attr_reader :platform + + def initialize(platform, contents) + @platform = platform + @rows = case platform + when "swift" then swift_rows(contents) + when "android" then android_rows(contents) + else raise ArgumentError, "Unknown coverage platform: #{platform}" + end + end + + def marker + "" + end + + def markdown(sha: nil, report_url: nil) + label = platform == "swift" ? "Swift" : "Android" + column = platform == "swift" ? "SDK target (lines)" : "Library metric" + lines = [marker, "## #{label} coverage", "", "| #{column} | Coverage | Covered / Total |", "| --- | ---: | ---: |"] + @rows.each do |name, covered, total| + percentage = total.zero? ? "N/A" : format("%.2f%%", 100.0 * covered / total) + lines << "| #{name} | #{percentage} | #{covered} / #{total} |" + end + lines.concat(["", "Report only; no native coverage threshold is enforced."]) + lines << if platform == "swift" + "SDK targets only; test and dependency targets are excluded." + else + "Debug library unit tests, including Robolectric and telemetry." + end + lines << "Sample app and end-to-end tests are excluded." + lines.concat(["", "PR head: `#{sha}`."]) unless sha.to_s.empty? + lines.concat(["", "[Full coverage reports](#{report_url})"]) unless report_url.to_s.empty? + lines.join("\n") + "\n" + end + + private + + def swift_rows(contents) + targets = JSON.parse(contents).fetch("targets") + SWIFT_TARGETS.map do |name| + target = targets.find { |entry| entry.fetch("name") == name } + raise "Missing Swift coverage target: #{name}" unless target + + row(name, target.fetch("coveredLines"), target.fetch("executableLines")) + end + end + + def android_rows(contents) + document = REXML::Document.new(contents) + ANDROID_METRICS.map do |type, label| + # Nested package/class counters duplicate the report totals. + counter = document.elements["report/counter[@type='#{type}']"] + raise "Missing Android coverage counter: #{type}" unless counter + + covered = Integer(counter.attributes["covered"]) + missed = Integer(counter.attributes["missed"]) + raise "Invalid Android coverage counter: #{type}" if missed.negative? + + row(label, covered, covered + missed) + end + end + + def row(label, covered, total) + unless covered.is_a?(Integer) && total.is_a?(Integer) && covered >= 0 && total >= covered + raise "Invalid coverage counts for #{label}" + end + + [label, covered, total] + end +end + +class NativeCoverageComment + def initialize(repository:, pr_number:, sha:, token:, client: nil) + @repository = repository + @pr_number = pr_number + @sha = sha + @client = client || JsonHttpClient.new(host: "api.github.com", error_label: "GitHub", default_headers: {"Accept" => "application/vnd.github+json"}) do |request| + request["Authorization"] = "Bearer #{token}" + end + end + + def publish(report, body) + comment = existing_comment(report.marker) + # Check immediately before writing so an older build cannot replace a newer + # head's results. Forks retain artifacts and logs without needing write access. + pr = @client.get("/repos/#{@repository}/pulls/#{@pr_number}") + return "Skipped coverage comment: pull request is closed or this build is superseded." unless pr["state"] == "open" && pr.dig("head", "sha") == @sha + return "Skipped coverage comment: fork pull request." unless pr.dig("head", "repo", "full_name") == @repository + + if comment + @client.patch_json("/repos/#{@repository}/issues/comments/#{comment.fetch("id")}", {body: body}) + else + @client.post_json("/repos/#{@repository}/issues/#{@pr_number}/comments", {body: body}) + end + "Published #{report.platform} coverage comment." + end + + private + + def existing_comment(marker) + page = 1 + loop do + comments = @client.get("/repos/#{@repository}/issues/#{@pr_number}/comments?per_page=100&page=#{page}") + match = comments.find { |comment| comment.dig("user", "type") == "Bot" && comment["body"].to_s.start_with?(marker) } + return match if match + return nil if comments.length < 100 + + page += 1 + end + end +end diff --git a/scripts/report_native_coverage b/scripts/report_native_coverage new file mode 100644 index 000000000..7d365c1e5 --- /dev/null +++ b/scripts/report_native_coverage @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "lib/native_coverage_report" + +abort "Usage: ruby scripts/report_native_coverage " unless ARGV.length == 2 +report = NativeCoverageReport.new(ARGV[0], File.read(ARGV[1])) +sha = ENV["COVERAGE_SHA"] +body = report.markdown(sha: sha, report_url: ENV["COVERAGE_REPORT_URL"]) +puts body +summary_path = ENV["GITHUB_STEP_SUMMARY"] +File.open(summary_path, "a") { |file| file.puts(body) } unless summary_path.to_s.empty? + +pr_number = ENV["COVERAGE_PR"] +token = ENV["COVERAGE_TOKEN"] +if pr_number.to_s.empty? || token.to_s.empty? + puts "Skipped coverage comment: no pull request or GitHub write token." + exit +end +abort "COVERAGE_PR must be a pull request number" unless pr_number.match?(/\A[1-9]\d*\z/) +abort "COVERAGE_SHA is required to publish coverage" if sha.to_s.empty? + +publisher = NativeCoverageComment.new( + repository: ENV.fetch("COVERAGE_REPOSITORY"), + pr_number: pr_number, + sha: sha, + token: token +) +puts publisher.publish(report, body) diff --git a/scripts/test/native_coverage_report_test.rb b/scripts/test/native_coverage_report_test.rb new file mode 100644 index 000000000..59e2eed53 --- /dev/null +++ b/scripts/test/native_coverage_report_test.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "open3" +require "tmpdir" +require_relative "../lib/native_coverage_report" + +class NativeCoverageReportTest < Minitest::Test + def swift_json + JSON.generate("targets" => [ + {"name" => "ShopifyCheckoutKitTests", "coveredLines" => 1000, "executableLines" => 1000}, + {"name" => "ShopifyCheckoutKit", "coveredLines" => 9, "executableLines" => 10}, + {"name" => "ShopifyAcceleratedCheckouts", "coveredLines" => 2, "executableLines" => 3}, + {"name" => "EmbeddedCheckoutProtocol", "coveredLines" => 0, "executableLines" => 0} + ]) + end + + def android_xml + <<~XML + + + + + + + + + + + + XML + end + + def test_swift_reports_sdk_targets_only_even_below_85_percent + markdown = NativeCoverageReport.new("swift", swift_json).markdown + + assert_includes markdown, "| ShopifyCheckoutKit | 90.00% | 9 / 10 |" + assert_includes markdown, "| ShopifyAcceleratedCheckouts | 66.67% | 2 / 3 |" + refute_includes markdown, "ShopifyCheckoutKitTests" + refute_includes markdown, "EmbeddedCheckoutProtocol" + assert_includes markdown, "no native coverage threshold is enforced" + end + + def test_android_uses_report_totals_and_handles_no_executable_branches + markdown = NativeCoverageReport.new("android", android_xml).markdown + + assert_includes markdown, "| Lines | 80.00% | 4 / 5 |" + assert_includes markdown, "| Branches | N/A | 0 / 0 |" + assert_includes markdown, "| Methods | 25.00% | 1 / 4 |" + assert_includes markdown, "| Instructions | 90.00% | 9 / 10 |" + end + + def test_missing_targets_and_counters_fail_instead_of_showing_partial_coverage + assert_raises(RuntimeError) { NativeCoverageReport.new("swift", '{"targets":[]}') } + assert_raises(RuntimeError) { NativeCoverageReport.new("android", '') } + end + + def test_impossible_counts_fail + invalid = swift_json.sub('"coveredLines":9', '"coveredLines":11') + assert_raises(RuntimeError) { NativeCoverageReport.new("swift", invalid) } + assert_raises(RuntimeError) { NativeCoverageReport.new("android", android_xml.sub('missed="1"', 'missed="-1"')) } + end + + def test_cli_keeps_a_job_summary_without_a_comment_token + Dir.mktmpdir("coverage report ") do |directory| + input = File.join(directory, "report.xml") + summary = File.join(directory, "summary.md") + File.write(input, android_xml) + File.write(summary, "Existing summary\n") + output, error, status = Open3.capture3( + {"COVERAGE_TOKEN" => "", "COVERAGE_PR" => "123", "COVERAGE_SHA" => "abc123", + "COVERAGE_REPORT_URL" => "https://example.com/reports", "GITHUB_STEP_SUMMARY" => summary}, + "ruby", File.expand_path("../report_native_coverage", __dir__), "android", input + ) + + assert status.success?, "#{output}#{error}" + assert_includes output, "Skipped coverage comment" + markdown = File.read(summary) + assert markdown.start_with?("Existing summary\n") + assert_includes markdown, "| Lines | 80.00% | 4 / 5 |" + assert_includes markdown, "PR head: `abc123`" + assert_includes markdown, "[Full coverage reports](https://example.com/reports)" + end + end +end + +class NativeCoverageCommentTest < Minitest::Test + class FakeClient + attr_reader :writes, :gets + + def initialize(pr, pages) + @pr = pr + @pages = pages + @writes = [] + @gets = [] + end + + def get(path) + @gets << path + return @pr if path == "/repos/example/sdk/pulls/123" + + page = Integer(path.split("page=").last) + @pages.fetch(page - 1) + end + + def post_json(path, body) + @writes << [:post, path, body] + end + + def patch_json(path, body) + @writes << [:patch, path, body] + end + end + + def setup + @report = NativeCoverageReport.new("swift", JSON.generate("targets" => NativeCoverageReport::SWIFT_TARGETS.map do |name| + {"name" => name, "coveredLines" => 1, "executableLines" => 2} + end)) + @pr = {"state" => "open", "head" => {"sha" => "abc123", "repo" => {"full_name" => "example/sdk"}}} + end + + def publish(pages = [[]]) + @client = FakeClient.new(@pr, pages) + NativeCoverageComment.new(repository: "example/sdk", pr_number: 123, sha: "abc123", token: "test-token", client: @client) + .publish(@report, @report.markdown) + end + + def test_creates_a_comment + assert_includes publish, "Published swift" + assert_equal [[:post, "/repos/example/sdk/issues/123/comments", {body: @report.markdown}]], @client.writes + end + + def test_updates_its_bot_comment_after_paginating_and_preserves_human_comments + human = {"id" => 1, "user" => {"type" => "User"}, "body" => @report.marker} + other_platform = {"id" => 2, "user" => {"type" => "Bot"}, "body" => ""} + existing = {"id" => 3, "user" => {"type" => "Bot"}, "body" => @report.marker + "\nold report"} + publish([[human] * 99 + [other_platform], [existing]]) + + assert_equal [[:patch, "/repos/example/sdk/issues/comments/3", {body: @report.markdown}]], @client.writes + assert_equal "/repos/example/sdk/pulls/123", @client.gets.last + end + + def test_a_superseded_build_cannot_overwrite_the_comment + @pr["head"]["sha"] = "new-head" + assert_includes publish, "superseded" + assert_empty @client.writes + end + + def test_closed_prs_are_not_updated + @pr["state"] = "closed" + assert_includes publish, "closed" + assert_empty @client.writes + end + + def test_forks_do_not_attempt_a_write + @pr["head"]["repo"]["full_name"] = "contributor/sdk" + assert_includes publish, "fork" + assert_empty @client.writes + end +end From 28274903610c3f5b8babee51707cac9eb672c411 Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Tue, 6 Oct 2026 17:41:59 +0100 Subject: [PATCH 4/6] Match coverage comment style and avoid XML dependency on Swift --- scripts/lib/native_coverage_report.rb | 51 +++++++++++++++------ scripts/test/native_coverage_report_test.rb | 32 ++++++++++--- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/scripts/lib/native_coverage_report.rb b/scripts/lib/native_coverage_report.rb index 3e214aebd..844b7d46e 100644 --- a/scripts/lib/native_coverage_report.rb +++ b/scripts/lib/native_coverage_report.rb @@ -1,12 +1,11 @@ # frozen_string_literal: true require "json" -require "rexml/document" require_relative "json_http_client" class NativeCoverageReport SWIFT_TARGETS = %w[ShopifyCheckoutKit ShopifyAcceleratedCheckouts].freeze - ANDROID_METRICS = {"LINE" => "Lines", "BRANCH" => "Branches", "METHOD" => "Methods", "INSTRUCTION" => "Instructions"}.freeze + ANDROID_METRICS = {"LINE" => "Lines", "INSTRUCTION" => "Instructions", "BRANCH" => "Branches", "METHOD" => "Methods"}.freeze attr_reader :platform @@ -25,19 +24,20 @@ def marker def markdown(sha: nil, report_url: nil) label = platform == "swift" ? "Swift" : "Android" - column = platform == "swift" ? "SDK target (lines)" : "Library metric" - lines = [marker, "## #{label} coverage", "", "| #{column} | Coverage | Covered / Total |", "| --- | ---: | ---: |"] - @rows.each do |name, covered, total| - percentage = total.zero? ? "N/A" : format("%.2f%%", 100.0 * covered / total) - lines << "| #{name} | #{percentage} | #{covered} / #{total} |" - end - lines.concat(["", "Report only; no native coverage threshold is enforced."]) - lines << if platform == "swift" - "SDK targets only; test and dependency targets are excluded." + lines = [marker, "# #{label} — Coverage Report", ""] + if platform == "swift" + lines.concat(["| Target | Lines |", "| --- | --- |"]) + @rows.each do |name, covered, total| + lines << "| #{name} | #{metric(covered, total, badge: true, report_url: report_url)} |" + end else - "Debug library unit tests, including Robolectric and telemetry." + lines.concat(["| #{@rows.map(&:first).join(' | ')} |", "| --- | --- | --- | --- |"]) + cells = @rows.map do |name, covered, total| + metric(covered, total, badge: name == "Lines", report_url: report_url) + end + lines << "| #{cells.join(' | ')} |" end - lines << "Sample app and end-to-end tests are excluded." + lines.concat(["", "Report only; no native coverage threshold is enforced."]) lines.concat(["", "PR head: `#{sha}`."]) unless sha.to_s.empty? lines.concat(["", "[Full coverage reports](#{report_url})"]) unless report_url.to_s.empty? lines.join("\n") + "\n" @@ -45,6 +45,28 @@ def markdown(sha: nil, report_url: nil) private + def metric(covered, total, badge:, report_url:) + return "N/A (0/0)" if total.zero? + + percentage = 100.0 * covered / total + formatted = format("%.2f", percentage).sub(/\.?0+\z/, "") + counts = "(#{covered}/#{total})" + return "#{formatted}% #{counts}" unless badge + + # Match the existing jest-coverage-comment badge palette. Colors describe + # coverage, independently of whether a platform enforces a threshold. + color = case percentage + when 0...40 then "red" + when 40...60 then "orange" + when 60...80 then "yellow" + when 80...90 then "green" + else "brightgreen" + end + image = "![Coverage: #{formatted}%](https://img.shields.io/badge/Coverage-#{formatted}%25-#{color}.svg)" + image = "[#{image}](#{report_url})" unless report_url.to_s.empty? + "#{image}
#{counts}" + end + def swift_rows(contents) targets = JSON.parse(contents).fetch("targets") SWIFT_TARGETS.map do |name| @@ -56,6 +78,9 @@ def swift_rows(contents) end def android_rows(contents) + # Bitrise's Swift runner only needs JSON and does not install the XML gem. + require "rexml/document" + document = REXML::Document.new(contents) ANDROID_METRICS.map do |type, label| # Nested package/class counters duplicate the report totals. diff --git a/scripts/test/native_coverage_report_test.rb b/scripts/test/native_coverage_report_test.rb index 59e2eed53..f7b296dbc 100644 --- a/scripts/test/native_coverage_report_test.rb +++ b/scripts/test/native_coverage_report_test.rb @@ -34,8 +34,10 @@ def android_xml def test_swift_reports_sdk_targets_only_even_below_85_percent markdown = NativeCoverageReport.new("swift", swift_json).markdown - assert_includes markdown, "| ShopifyCheckoutKit | 90.00% | 9 / 10 |" - assert_includes markdown, "| ShopifyAcceleratedCheckouts | 66.67% | 2 / 3 |" + assert_includes markdown, "# Swift — Coverage Report" + assert_includes markdown, "| Target | Lines |" + assert_includes markdown, "| ShopifyCheckoutKit | ![Coverage: 90%](https://img.shields.io/badge/Coverage-90%25-brightgreen.svg)
(9/10) |" + assert_includes markdown, "| ShopifyAcceleratedCheckouts | ![Coverage: 66.67%](https://img.shields.io/badge/Coverage-66.67%25-yellow.svg)
(2/3) |" refute_includes markdown, "ShopifyCheckoutKitTests" refute_includes markdown, "EmbeddedCheckoutProtocol" assert_includes markdown, "no native coverage threshold is enforced" @@ -44,10 +46,9 @@ def test_swift_reports_sdk_targets_only_even_below_85_percent def test_android_uses_report_totals_and_handles_no_executable_branches markdown = NativeCoverageReport.new("android", android_xml).markdown - assert_includes markdown, "| Lines | 80.00% | 4 / 5 |" - assert_includes markdown, "| Branches | N/A | 0 / 0 |" - assert_includes markdown, "| Methods | 25.00% | 1 / 4 |" - assert_includes markdown, "| Instructions | 90.00% | 9 / 10 |" + assert_includes markdown, "# Android — Coverage Report" + assert_includes markdown, "| Lines | Instructions | Branches | Methods |" + assert_includes markdown, "| ![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg)
(4/5) | 90% (9/10) | N/A (0/0) | 25% (1/4) |" end def test_missing_targets_and_counters_fail_instead_of_showing_partial_coverage @@ -77,11 +78,28 @@ def test_cli_keeps_a_job_summary_without_a_comment_token assert_includes output, "Skipped coverage comment" markdown = File.read(summary) assert markdown.start_with?("Existing summary\n") - assert_includes markdown, "| Lines | 80.00% | 4 / 5 |" + assert_includes markdown, "[![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg)](https://example.com/reports)
(4/5)" assert_includes markdown, "PR head: `abc123`" assert_includes markdown, "[Full coverage reports](https://example.com/reports)" end end + + def test_swift_cli_runs_without_androids_xml_dependency + Dir.mktmpdir("swift coverage ") do |directory| + Dir.mkdir(File.join(directory, "rexml")) + File.write(File.join(directory, "rexml/document.rb"), 'raise LoadError, "XML is unavailable on the Swift runner"') + input = File.join(directory, "coverage.json") + File.write(input, swift_json) + output, error, status = Open3.capture3( + {"COVERAGE_TOKEN" => "", "GITHUB_STEP_SUMMARY" => ""}, + "ruby", "-I", directory, File.expand_path("../report_native_coverage", __dir__), "swift", input + ) + + assert status.success?, "#{output}#{error}" + assert_includes output, "# Swift — Coverage Report" + assert_includes output, "Coverage: 66.67%" + end + end end class NativeCoverageCommentTest < Minitest::Test From a5aac0ebae577bb1d3d28f4c1095e0725922264a Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Tue, 6 Oct 2026 17:49:35 +0100 Subject: [PATCH 5/6] Simplify native coverage comments to match existing reports --- .github/CONTRIBUTING.md | 5 +++-- scripts/lib/native_coverage_report.rb | 11 +++++------ scripts/report_native_coverage | 2 +- scripts/test/native_coverage_report_test.rb | 10 ++++------ 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 797acca09..0164a98e8 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -106,8 +106,9 @@ Swift and Android generate coverage reports without enforcing a minimum yet: | Swift | `dev swift test coverage` | `.xcresults/coverage/` (JSON and text) | Coverage reports in the Bitrise Swift package test job | CI updates a coverage comment per native platform on same-repository pull requests, -with percentages, covered/total counts, the tested PR head, and a link to the full -reports. Android also writes its coverage table to the GitHub Actions job summary. +using the same headings, line-coverage badges, and metric formatting as the +JavaScript reports, plus a link to the full reports. Android also writes its +coverage table to the GitHub Actions job summary. Fork builds and builds without a write token keep the artifacts and log summaries without posting a comment. These summaries report coverage without enforcing 85%. diff --git a/scripts/lib/native_coverage_report.rb b/scripts/lib/native_coverage_report.rb index 844b7d46e..eac76ed41 100644 --- a/scripts/lib/native_coverage_report.rb +++ b/scripts/lib/native_coverage_report.rb @@ -22,7 +22,7 @@ def marker "" end - def markdown(sha: nil, report_url: nil) + def markdown(report_url: nil) label = platform == "swift" ? "Swift" : "Android" lines = [marker, "# #{label} — Coverage Report", ""] if platform == "swift" @@ -37,8 +37,6 @@ def markdown(sha: nil, report_url: nil) end lines << "| #{cells.join(' | ')} |" end - lines.concat(["", "Report only; no native coverage threshold is enforced."]) - lines.concat(["", "PR head: `#{sha}`."]) unless sha.to_s.empty? lines.concat(["", "[Full coverage reports](#{report_url})"]) unless report_url.to_s.empty? lines.join("\n") + "\n" end @@ -46,12 +44,13 @@ def markdown(sha: nil, report_url: nil) private def metric(covered, total, badge:, report_url:) - return "N/A (0/0)" if total.zero? + return "N/A" if total.zero? && badge + return "N/A (0/0)" if total.zero? percentage = 100.0 * covered / total formatted = format("%.2f", percentage).sub(/\.?0+\z/, "") counts = "(#{covered}/#{total})" - return "#{formatted}% #{counts}" unless badge + return "#{formatted}% #{counts}" unless badge # Match the existing jest-coverage-comment badge palette. Colors describe # coverage, independently of whether a platform enforces a threshold. @@ -64,7 +63,7 @@ def metric(covered, total, badge:, report_url:) end image = "![Coverage: #{formatted}%](https://img.shields.io/badge/Coverage-#{formatted}%25-#{color}.svg)" image = "[#{image}](#{report_url})" unless report_url.to_s.empty? - "#{image}
#{counts}" + image end def swift_rows(contents) diff --git a/scripts/report_native_coverage b/scripts/report_native_coverage index 7d365c1e5..c31183367 100644 --- a/scripts/report_native_coverage +++ b/scripts/report_native_coverage @@ -6,7 +6,7 @@ require_relative "lib/native_coverage_report" abort "Usage: ruby scripts/report_native_coverage " unless ARGV.length == 2 report = NativeCoverageReport.new(ARGV[0], File.read(ARGV[1])) sha = ENV["COVERAGE_SHA"] -body = report.markdown(sha: sha, report_url: ENV["COVERAGE_REPORT_URL"]) +body = report.markdown(report_url: ENV["COVERAGE_REPORT_URL"]) puts body summary_path = ENV["GITHUB_STEP_SUMMARY"] File.open(summary_path, "a") { |file| file.puts(body) } unless summary_path.to_s.empty? diff --git a/scripts/test/native_coverage_report_test.rb b/scripts/test/native_coverage_report_test.rb index f7b296dbc..a738fdc3a 100644 --- a/scripts/test/native_coverage_report_test.rb +++ b/scripts/test/native_coverage_report_test.rb @@ -36,11 +36,10 @@ def test_swift_reports_sdk_targets_only_even_below_85_percent assert_includes markdown, "# Swift — Coverage Report" assert_includes markdown, "| Target | Lines |" - assert_includes markdown, "| ShopifyCheckoutKit | ![Coverage: 90%](https://img.shields.io/badge/Coverage-90%25-brightgreen.svg)
(9/10) |" - assert_includes markdown, "| ShopifyAcceleratedCheckouts | ![Coverage: 66.67%](https://img.shields.io/badge/Coverage-66.67%25-yellow.svg)
(2/3) |" + assert_includes markdown, "| ShopifyCheckoutKit | ![Coverage: 90%](https://img.shields.io/badge/Coverage-90%25-brightgreen.svg) |" + assert_includes markdown, "| ShopifyAcceleratedCheckouts | ![Coverage: 66.67%](https://img.shields.io/badge/Coverage-66.67%25-yellow.svg) |" refute_includes markdown, "ShopifyCheckoutKitTests" refute_includes markdown, "EmbeddedCheckoutProtocol" - assert_includes markdown, "no native coverage threshold is enforced" end def test_android_uses_report_totals_and_handles_no_executable_branches @@ -48,7 +47,7 @@ def test_android_uses_report_totals_and_handles_no_executable_branches assert_includes markdown, "# Android — Coverage Report" assert_includes markdown, "| Lines | Instructions | Branches | Methods |" - assert_includes markdown, "| ![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg)
(4/5) | 90% (9/10) | N/A (0/0) | 25% (1/4) |" + assert_includes markdown, "| ![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg) | 90% (9/10) | N/A (0/0) | 25% (1/4) |" end def test_missing_targets_and_counters_fail_instead_of_showing_partial_coverage @@ -78,8 +77,7 @@ def test_cli_keeps_a_job_summary_without_a_comment_token assert_includes output, "Skipped coverage comment" markdown = File.read(summary) assert markdown.start_with?("Existing summary\n") - assert_includes markdown, "[![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg)](https://example.com/reports)
(4/5)" - assert_includes markdown, "PR head: `abc123`" + assert_includes markdown, "[![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg)](https://example.com/reports)" assert_includes markdown, "[Full coverage reports](https://example.com/reports)" end end From 354e608642e2a8e10cb4f1207ebc53a6ec2a0288 Mon Sep 17 00:00:00 2001 From: Mark Murray Date: Tue, 6 Oct 2026 22:30:42 +0100 Subject: [PATCH 6/6] Report standalone Swift and Kotlin embedded protocol coverage --- .github/workflows/android-test.yml | 23 +++++++++++++-- dev.yml | 10 +++++++ e2e/bitrise.yml | 22 +++++++++++++++ .../embedded-checkout-protocol/build.gradle | 11 ++++++++ protocol/scripts/test_swift_coverage | 11 ++++++++ scripts/lib/native_coverage_report.rb | 26 ++++++++++++++--- scripts/report_native_coverage | 2 +- scripts/test/native_coverage_report_test.rb | 28 +++++++++++++++++++ 8 files changed, 126 insertions(+), 7 deletions(-) create mode 100755 protocol/scripts/test_swift_coverage diff --git a/.github/workflows/android-test.yml b/.github/workflows/android-test.yml index 721669496..3eb497a7b 100644 --- a/.github/workflows/android-test.yml +++ b/.github/workflows/android-test.yml @@ -55,9 +55,28 @@ jobs: COVERAGE_REPORT_URL: ${{ steps.coverage_artifact.outputs.artifact-url }} run: ruby scripts/report_native_coverage android platforms/android/lib/build/reports/coverage/test/debug/report.xml - - name: Run Protocol Unit Tests + - name: Run Embedded Checkout Protocol tests with coverage working-directory: protocol/languages/kotlin - run: ./gradlew test --console=plain + run: ./gradlew test jacocoTestReport --console=plain + + - name: Upload Embedded Checkout Protocol coverage + id: protocol_coverage_artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: protocol-kotlin-coverage + path: protocol/languages/kotlin/embedded-checkout-protocol/build/reports/jacoco/test/ + if-no-files-found: error + retention-days: 7 + + - name: Summarize Embedded Checkout Protocol coverage + working-directory: ${{ github.workspace }} + env: + COVERAGE_REPOSITORY: ${{ github.repository }} + COVERAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + COVERAGE_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} + COVERAGE_TOKEN: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' && github.token || '' }} + COVERAGE_REPORT_URL: ${{ steps.protocol_coverage_artifact.outputs.artifact-url }} + run: ruby scripts/report_native_coverage protocol-kotlin protocol/languages/kotlin/embedded-checkout-protocol/build/reports/jacoco/test/jacocoTestReport.xml - name: Setup sample app environment id: sample_env diff --git a/dev.yml b/dev.yml index 93741df5a..9a4be6dd3 100644 --- a/dev.yml +++ b/dev.yml @@ -237,6 +237,16 @@ commands: *) echo "Usage: dev protocol test [kotlin|swift|typescript] [-- ]"; exit 1 ;; esac + coverage: + desc: "Run native protocol tests with coverage. Usage: dev protocol coverage " + syntax: "" + run: | + case "$1" in + swift) ./protocol/scripts/test_swift_coverage ;; + kotlin) protocol/languages/kotlin/gradlew -p protocol/languages/kotlin :embedded-checkout-protocol:jacocoTestReport --console=plain ;; + *) echo "Usage: dev protocol coverage "; exit 1 ;; + esac + lint: desc: "Lint Kotlin protocol source and TypeScript test files" run: | diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index ad8157a1b..225781427 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -366,6 +366,28 @@ workflows: export COVERAGE_TOKEN="${OVERRIDE_GITHUB_TOKEN:-${GIT_HTTP_PASSWORD:-}}" ruby scripts/report_native_coverage swift .xcresults/coverage/coverage.json + - script@1: + title: Run Embedded Checkout Protocol Swift coverage + inputs: + - content: |- + set -euo pipefail + ./protocol/scripts/test_swift_coverage + - deploy-to-bitrise-io@2: + is_always_run: false + inputs: + - deploy_path: "$BITRISE_SOURCE_DIR/protocol/languages/swift/.build/coverage" + - script@1: + title: Report Embedded Checkout Protocol Swift coverage + inputs: + - content: |- + set -euo pipefail + export COVERAGE_REPOSITORY="Shopify/checkout-kit" + export COVERAGE_PR="${BITRISE_PULL_REQUEST:-}" + export COVERAGE_SHA="${BITRISE_GIT_COMMIT:-}" + export COVERAGE_REPORT_URL="${BITRISE_BUILD_URL:-}" + export COVERAGE_TOKEN="${OVERRIDE_GITHUB_TOKEN:-${GIT_HTTP_PASSWORD:-}}" + ruby scripts/report_native_coverage protocol-swift protocol/languages/swift/.build/coverage/coverage.json + ci-ios-swift-samples: meta: bitrise.io: diff --git a/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle b/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle index 13d523fae..ba37806d1 100644 --- a/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle +++ b/protocol/languages/kotlin/embedded-checkout-protocol/build.gradle @@ -7,6 +7,7 @@ plugins { id 'signing' id 'maven-publish' id 'java-library' + id 'jacoco' id 'org.jetbrains.kotlin.jvm' id 'org.jetbrains.kotlin.plugin.serialization' id 'io.gitlab.arturbosch.detekt' @@ -121,3 +122,13 @@ publishing { } } } + +// The standalone protocol suite measures its own artifact, including generated +// wire models, independently of the Android SDK's published dependency. +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true + } +} diff --git a/protocol/scripts/test_swift_coverage b/protocol/scripts/test_swift_coverage new file mode 100755 index 000000000..8a85eb4e5 --- /dev/null +++ b/protocol/scripts/test_swift_coverage @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +package_path="$repo_root/protocol/languages/swift" +report_dir="$package_path/.build/coverage" +mkdir -p "$report_dir" +rm -f "$report_dir/coverage.json" +swift test --package-path "$package_path" --enable-code-coverage +coverage_path="$(swift test --package-path "$package_path" --show-codecov-path)" +cp "$coverage_path" "$report_dir/coverage.json" diff --git a/scripts/lib/native_coverage_report.rb b/scripts/lib/native_coverage_report.rb index eac76ed41..a561abffd 100644 --- a/scripts/lib/native_coverage_report.rb +++ b/scripts/lib/native_coverage_report.rb @@ -7,13 +7,16 @@ class NativeCoverageReport SWIFT_TARGETS = %w[ShopifyCheckoutKit ShopifyAcceleratedCheckouts].freeze ANDROID_METRICS = {"LINE" => "Lines", "INSTRUCTION" => "Instructions", "BRANCH" => "Branches", "METHOD" => "Methods"}.freeze - attr_reader :platform + TITLES = {"swift" => "Swift", "android" => "Android", "protocol-swift" => "Embedded Checkout Protocol (Swift)", "protocol-kotlin" => "Embedded Checkout Protocol (Kotlin)"}.freeze + + attr_reader :platform, :rows def initialize(platform, contents) @platform = platform @rows = case platform when "swift" then swift_rows(contents) - when "android" then android_rows(contents) + when "android", "protocol-kotlin" then android_rows(contents) + when "protocol-swift" then swift_protocol_rows(contents) else raise ArgumentError, "Unknown coverage platform: #{platform}" end end @@ -23,7 +26,7 @@ def marker end def markdown(report_url: nil) - label = platform == "swift" ? "Swift" : "Android" + label = TITLES.fetch(platform) lines = [marker, "# #{label} — Coverage Report", ""] if platform == "swift" lines.concat(["| Target | Lines |", "| --- | --- |"]) @@ -31,7 +34,7 @@ def markdown(report_url: nil) lines << "| #{name} | #{metric(covered, total, badge: true, report_url: report_url)} |" end else - lines.concat(["| #{@rows.map(&:first).join(' | ')} |", "| --- | --- | --- | --- |"]) + lines.concat(["| #{@rows.map(&:first).join(' | ')} |", "| #{@rows.map { "---" }.join(" | ")} |"]) cells = @rows.map do |name, covered, total| metric(covered, total, badge: name == "Lines", report_url: report_url) end @@ -76,6 +79,21 @@ def swift_rows(contents) end end + def swift_protocol_rows(contents) + files = JSON.parse(contents).fetch("data").flat_map { |data| data.fetch("files") } + .select { |file| file.fetch("filename").include?("/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/") } + raise "Missing Swift protocol coverage files" if files.empty? + raise "Duplicate Swift protocol coverage files" unless files.map { |file| file.fetch("filename") }.uniq.length == files.length + + {"lines" => "Lines", "functions" => "Functions"}.map do |key, label| + counts = files.map do |file| + metric = file.fetch("summary").fetch(key) + row(label, metric.fetch("covered"), metric.fetch("count")) + end + row(label, counts.sum { |entry| entry[1] }, counts.sum { |entry| entry[2] }) + end + end + def android_rows(contents) # Bitrise's Swift runner only needs JSON and does not install the XML gem. require "rexml/document" diff --git a/scripts/report_native_coverage b/scripts/report_native_coverage index c31183367..15ac87d7a 100644 --- a/scripts/report_native_coverage +++ b/scripts/report_native_coverage @@ -3,7 +3,7 @@ require_relative "lib/native_coverage_report" -abort "Usage: ruby scripts/report_native_coverage " unless ARGV.length == 2 +abort "Usage: ruby scripts/report_native_coverage " unless ARGV.length == 2 report = NativeCoverageReport.new(ARGV[0], File.read(ARGV[1])) sha = ENV["COVERAGE_SHA"] body = report.markdown(report_url: ENV["COVERAGE_REPORT_URL"]) diff --git a/scripts/test/native_coverage_report_test.rb b/scripts/test/native_coverage_report_test.rb index a738fdc3a..e9fae5eac 100644 --- a/scripts/test/native_coverage_report_test.rb +++ b/scripts/test/native_coverage_report_test.rb @@ -50,6 +50,34 @@ def test_android_uses_report_totals_and_handles_no_executable_branches assert_includes markdown, "| ![Coverage: 80%](https://img.shields.io/badge/Coverage-80%25-green.svg) | 90% (9/10) | N/A (0/0) | 25% (1/4) |" end + def test_swift_protocol_uses_only_library_files_and_includes_generated_wire_models + files = [ + ["/package/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/Client.swift", 8, 10], + ["/package/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/Generated/Models.swift", 3, 5], + ["/package/Tests/EmbeddedCheckoutProtocolTests/ClientTests.swift", 100, 100], + ["/package/.build/runner.swift", 100, 100] + ].map do |filename, covered, total| + {"filename" => filename, "summary" => {"lines" => {"covered" => covered, "count" => total}, "functions" => {"covered" => 1, "count" => 2}}} + end + report = NativeCoverageReport.new("protocol-swift", JSON.generate("data" => [{"files" => files}])) + assert_equal [["Lines", 11, 15], ["Functions", 2, 4]], report.rows + assert_includes report.markdown, "Embedded Checkout Protocol (Swift)" + assert_includes report.markdown, "| Lines | Functions |" + end + + def test_swift_protocol_rejects_missing_or_duplicate_files + assert_raises(RuntimeError) { NativeCoverageReport.new("protocol-swift", '{"data":[{"files":[]}]}') } + file = {"filename" => "/Sources/UniversalCommerceProtocol/EmbeddedCheckoutProtocol/Client.swift"} + assert_raises(RuntimeError) { NativeCoverageReport.new("protocol-swift", JSON.generate("data" => [{"files" => [file, file]}])) } + end + + def test_kotlin_protocol_has_its_own_title_and_comment_marker + report = NativeCoverageReport.new("protocol-kotlin", android_xml) + assert_equal NativeCoverageReport.new("android", android_xml).rows, report.rows + assert_includes report.markdown, "Embedded Checkout Protocol (Kotlin)" + refute_equal NativeCoverageReport.new("android", android_xml).marker, report.marker + end + def test_missing_targets_and_counters_fail_instead_of_showing_partial_coverage assert_raises(RuntimeError) { NativeCoverageReport.new("swift", '{"targets":[]}') } assert_raises(RuntimeError) { NativeCoverageReport.new("android", '') }