test

#!/bin/bash
set -euo pipefail

# test — the one way to run this repo's tests, for people and for agents.
#
# Three problems, one script:
#
#   1. The incantation. A correct run needs the right scheme, a UDID-resolved
#      destination (see "Selecting a simulator" in AGENTS.md), and — for image
#      snapshots — the scheme carrying the SNAPSHOT_EXPECTED_* / TZ pins. That
#      was copied by hand into the docs, CI, and every agent session, so it
#      drifted.
#
#   2. Scope. Running every bundle for a one-module change was the default
#      nobody chose. With no arguments this runs only the bundles the working
#      tree affects, derived from the two manifests — `swift package
#      dump-package` for the library graph, Project.swift for which bundle links
#      which product — rather than a table here that would go stale.
#
#   3. Silence. A run prints nothing useful for minutes, which is unnerving for
#      a person and useless to an agent tailing the output. This streams
#      progress: an in-place status line on a terminal, and newline-terminated
#      heartbeats when stdout is not one, because a carriage-return redraw is
#      invisible in a captured log.
#
# Raw xcodebuild rather than `tuist test`, for two measured reasons. Both were
# re-verified on 2026-07-28 by corrupting one reference image and running the
# same single test each way on the same simulator.
#
#   1. Tuist's formatter *swallows the failure reason*, and there is no flag
#      that recovers it. Swift Testing's headline for a recorded issue is the
#      contentless "Issue recorded"; the reason arrives in the `↳` block that
#      follows it, carrying "does not match reference" plus the reference and
#      failure image paths. xcbeautify — which `tuist test` pipes through —
#      renders the headline and drops the block, so 100% of the diagnostic
#      content is lost (upstream: cpisciotta/xcbeautify#402, still open; a
#      Tuist maintainer's own advice for this is to run xcodebuild directly).
#      That is how the mismatch that reddened CI run 30367964813 reached the
#      log as "Recorded an issue" — no numbers, no image, no path. Raw output
#      keeps the block, and `--review` adds the magnitude beside it.
#
#   2. `-collect-test-diagnostics never`, which is worth ~10 minutes on every
#      *failing* run. Without it xcodebuild waits out
#      "Failure collecting diagnostics from simulator: Timed out after 600.0
#      seconds" before reporting. Same one-test failure: 28s here, 620s under
#      `tuist test` — 609s of it that fixed timeout, for a test that ran in
#      3.5s. The same stall is in CI run 30367964813, which took 24 minutes.
#      (The flag would pass through `tuist test --`; the point is that the
#      recipe people actually type doesn't.)
#
# The `--timings` / `--review` reports depend on raw output too: they harvest
# SNAPSHOT_TIMING / SNAPSHOT_DIFF lines from the log, and a formatter that
# discards Swift Testing's own detail lines is not one to trust with ours.
#
# Little is given up. Tuist's selective testing does work locally (its hash
# cache skipped 19 unchanged bundles here), but there is no Tuist server — see
# Tuist.swift — so on CI's fresh checkout the cache is empty and it skips
# nothing. The affected-bundle selection below replaces it with better
# semantics anyway: changed against origin/main, rather than since this
# machine's last successful run.
#
# Exits non-zero when tests fail (./profile's posture, not ./flaky's).

DEVICE="iPhone 17"
OS="27.0"
SCOPE=changed          # changed | all | snapshots | everything | bundles | only
BUNDLES=()
ONLY=()
ONLY_FILE=""
RECORD=""
DO_BUILD=true
DO_GENERATE=true
BUILD_ARTIFACTS=""
TEST_ARTIFACTS=""
ENUMERATE_SUITES=""
HEARTBEAT=15
STATUS_FILE=""
TIMINGS=false
REVIEW=false
BASE_REF="origin/main"
SHARED=false
RUN_ARCHITECTURE=true
ARCHITECTURE_ONLY=false
HAS_TEST_ARGUMENTS=false

usage() {
    cat <<'USAGE'
Usage: ./test [options] [BundleName ...]

Runs this repo's tests against the simulator this checkout owns, streaming
progress as it goes. With no arguments it runs only the bundles affected by the
working tree's changes. Every normal invocation first validates and tests the
Bumper Bowling rules, then enforces the architecture. Unit and affected scopes
also run the fast host-side backup-upgrader regression suite.

Scope:
  (no arguments)    Bundles affected by the diff against origin/main, including
                    uncommitted and untracked files
  BundleName ...    Named bundles only (e.g. ./test WhereCoreTests)
  --all             The full unit suite (Stuff-iOS-Tests)
  --snapshots       The image-snapshot suite (StuffSnapshotTests)
  --everything      Both suites — what CI runs
  --only ID         An xcodebuild test identifier, repeatable:
                    Bundle, Bundle/Suite, or 'Bundle/Suite/test()'
  --only-file PATH  Read newline-delimited Bundle/Suite identifiers from PATH
  --base REF        Compare against REF instead of origin/main

Architecture:
  --architecture-only  Run only the Bumper configuration, rule tests, and lint
  --skip-architecture  Skip Bumper checks because another CI job owns them

Running:
  --no-build        Reuse the last build (test-without-building)
  --no-generate     Skip `tuist generate` (assumes the project is current)
  --build-artifacts DIR
                    Build the selected schemes into DIR without running tests
  --test-artifacts DIR
                    Run tests from the manifest and .xctestrun files in DIR
  --enumerate-suites PATH
                    Write the selected artifact's Bundle/Suite identifiers
  --record MODE     Re-record snapshots: all | failed | missing | never
  --device NAME     Simulator device name (default: "iPhone 17")
  --os VERSION      Simulator iOS version (default: "27.0")
  --shared          Use any existing simulator with that name rather than this
                    checkout's own — for a throwaway machine (CI) that is
                    already isolated

Reporting:
  --timings         Collect per-phase snapshot capture timings and print a
                    breakdown (sets SNAPSHOT_TIMING)
  --review          Report how each differing snapshot differs — pixel count,
                    max channel delta, changed region (sets SNAPSHOT_DIFF)
  --heartbeat SECS  Seconds between progress lines when not on a terminal
                    (default: 15)
  --status-file P   Keep only the latest progress line in P, for a watcher that
                    would rather read one small file than scan a log
  -h, --help        Show this help

Examples:
  ./test                                  # just what your change affects
  ./test WhereCoreTests WhereUITests
  ./test --snapshots --review             # snapshots, with the diffs described
  ./test --snapshots --timings            # where the capture time goes
  ./test --everything                     # everything, as CI runs it
  ./test --only 'WhereUITests/WhereStylesheetTests/resolvesTraitAwareTokens()'
USAGE
}

while [ $# -gt 0 ]; do
    case "$1" in
        --all) SCOPE=all; HAS_TEST_ARGUMENTS=true ;;
        --snapshots) SCOPE=snapshots; HAS_TEST_ARGUMENTS=true ;;
        --everything) SCOPE=everything; HAS_TEST_ARGUMENTS=true ;;
        --only)
            HAS_TEST_ARGUMENTS=true
            shift
            ONLY+=("${1:?--only requires a test identifier}")
            [ "$SCOPE" = changed ] && SCOPE=only
            ;;
        --only-file)
            HAS_TEST_ARGUMENTS=true
            shift
            ONLY_FILE="${1:?--only-file requires a path}"
            [ "$SCOPE" = changed ] && SCOPE=only
            ;;
        --base) HAS_TEST_ARGUMENTS=true; shift; BASE_REF="${1:?--base requires a git ref}" ;;
        --no-build) DO_BUILD=false; HAS_TEST_ARGUMENTS=true ;;
        --no-generate) DO_GENERATE=false; HAS_TEST_ARGUMENTS=true ;;
        --build-artifacts)
            HAS_TEST_ARGUMENTS=true
            shift
            BUILD_ARTIFACTS="${1:?--build-artifacts requires a directory}"
            ;;
        --test-artifacts)
            HAS_TEST_ARGUMENTS=true
            shift
            TEST_ARTIFACTS="${1:?--test-artifacts requires a directory}"
            DO_BUILD=false
            DO_GENERATE=false
            ;;
        --enumerate-suites)
            HAS_TEST_ARGUMENTS=true
            shift
            ENUMERATE_SUITES="${1:?--enumerate-suites requires a path}"
            ;;
        --record) HAS_TEST_ARGUMENTS=true; shift; RECORD="${1:?--record requires a mode}" ;;
        --device) HAS_TEST_ARGUMENTS=true; shift; DEVICE="${1:?--device requires a value}" ;;
        --shared) SHARED=true; HAS_TEST_ARGUMENTS=true ;;
        --os) HAS_TEST_ARGUMENTS=true; shift; OS="${1:?--os requires a value}" ;;
        --timings) TIMINGS=true; HAS_TEST_ARGUMENTS=true ;;
        --review) REVIEW=true; HAS_TEST_ARGUMENTS=true ;;
        --heartbeat) HAS_TEST_ARGUMENTS=true; shift; HEARTBEAT="${1:?--heartbeat requires a value}" ;;
        --status-file) HAS_TEST_ARGUMENTS=true; shift; STATUS_FILE="${1:?--status-file requires a path}" ;;
        --architecture-only) ARCHITECTURE_ONLY=true ;;
        --skip-architecture) RUN_ARCHITECTURE=false ;;
        -h|--help) usage; exit 0 ;;
        -*) echo "error: unknown option '$1' (see ./test --help)" >&2; exit 1 ;;
        *) BUNDLES+=("$1"); SCOPE=bundles; HAS_TEST_ARGUMENTS=true ;;
    esac
    shift
done

cd "$(dirname "$0")"

if [ -n "$ONLY_FILE" ]; then
    if [ ! -f "$ONLY_FILE" ]; then
        echo "error: --only-file does not exist: $ONLY_FILE" >&2
        exit 1
    fi
    while IFS= read -r identifier || [ -n "$identifier" ]; do
        [ -n "$identifier" ] && ONLY+=("$identifier")
    done <"$ONLY_FILE"
    if [ "${#ONLY[@]}" -eq 0 ]; then
        echo "error: --only-file contains no test identifiers: $ONLY_FILE" >&2
        exit 1
    fi
fi

case "$RECORD" in
    ""|all|failed|missing|never) ;;
    *) echo "error: --record must be all, failed, missing or never (got '$RECORD')" >&2
       exit 1 ;;
esac

if [ "$ARCHITECTURE_ONLY" = true ] && [ "$RUN_ARCHITECTURE" = false ]; then
    echo "error: --architecture-only cannot be combined with --skip-architecture" >&2
    exit 1
fi
if [ "$ARCHITECTURE_ONLY" = true ] && [ "$HAS_TEST_ARGUMENTS" = true ]; then
    echo "error: --architecture-only cannot be combined with test options or bundles" >&2
    exit 1
fi
if [ -n "$BUILD_ARTIFACTS" ] && [ -n "$TEST_ARTIFACTS" ]; then
    echo "error: --build-artifacts cannot be combined with --test-artifacts" >&2
    exit 1
fi
if [ -n "$BUILD_ARTIFACTS" ] && { [ "$DO_BUILD" = false ] || [ -n "$ONLY_FILE" ] ||
    [ "${#ONLY[@]}" -gt 0 ] || [ -n "$ENUMERATE_SUITES" ]; }
then
    echo "error: --build-artifacts cannot be combined with test filters or test-only modes" >&2
    exit 1
fi
if [ -n "$ENUMERATE_SUITES" ] && [ -z "$TEST_ARTIFACTS" ]; then
    echo "error: --enumerate-suites requires --test-artifacts" >&2
    exit 1
fi
if [ -n "$ENUMERATE_SUITES" ] && [ "$SCOPE" != snapshots ]; then
    echo "error: --enumerate-suites requires --snapshots" >&2
    exit 1
fi

run_architecture_checks() {
    export BUMPER_CACHE_DIR="${BUMPER_CACHE_DIR:-.build/bumper-cache}"
    export BUMPER_RUNNER_BUILD_CONFIGURATION="${BUMPER_RUNNER_BUILD_CONFIGURATION:-debug}"

    echo "==> Validating Bumper Bowling configuration"
    swift run bumper config .
    echo "==> Testing Bumper Bowling rules"
    swift run bumper test .
    echo "==> Enforcing Where architecture"
    swift run bumper lint . --timings
}

if [ "$RUN_ARCHITECTURE" = true ]; then
    run_architecture_checks
fi
if [ "$ARCHITECTURE_ONLY" = true ]; then
    exit 0
fi

WORKSPACE="Stuff.xcworkspace"
UNIT_SCHEME="Stuff-iOS-Tests"
SNAPSHOT_SCHEME="StuffSnapshotTests"
CHECKOUT_HASH="$(printf '%s' "$(pwd -P)" | shasum -a 256 | cut -c1-12)"
# Overridable so CI can collect the logs and result bundles from a known path
# rather than digging through DerivedData. Local runs include the canonical
# checkout path's hash so clones/worktrees cannot overwrite one another's
# logs, progress state, or result bundles while running concurrently.
WORKDIR="${TEST_WORKDIR:-${TMPDIR:-/tmp}/where-test-$CHECKOUT_HASH}"
# Test logs and result metadata can include developer paths and failure details.
# Keep the per-checkout work directory private while preserving its stable path.
mkdir -p "$WORKDIR" && chmod 700 "$WORKDIR"

rule() { printf '%s\n' "============================================================"; }

phase_started() {
    python3 Tools/test_runner.py phase-started
}

emit_phase_timing() {
    python3 Tools/test_runner.py phase-timing "$1" "$2" "${3:-}" "${4:-0}"
}

# The image bundles, read from the manifest rather than listed here: a bundle can
# only run from a scheme that contains it, and StuffSnapshotTests is the only one
# carrying the SNAPSHOT_EXPECTED_* / TZ pins the comparisons rely on.
snapshot_bundles() {
    python3 Tools/test_runner.py snapshot-bundles
}

# The backup upgrader is a host-side unit regression, not an image test. Keep it
# in affected and unit-capable runs, but do not make a snapshot-only invocation
# repeat work owned by the unit CI job.
backup_upgrader_is_needed() {
    local image_bundles bundle identifier
    image_bundles="$(snapshot_bundles)"
    { [ -n "$BUILD_ARTIFACTS" ] || [ -n "$TEST_ARTIFACTS" ]; } && return 1

    case "$SCOPE" in
        changed|all|everything) return 0 ;;
        snapshots|only) ;;
        bundles)
            for bundle in "${BUNDLES[@]}"; do
                if ! printf '%s\n' "$image_bundles" | grep -qx "$bundle"; then
                    return 0
                fi
            done
            return 1
            ;;
    esac

    for identifier in ${ONLY[@]+"${ONLY[@]}"}; do
        bundle="${identifier%%/*}"
        if ! printf '%s\n' "$image_bundles" | grep -qx "$bundle"; then
            return 0
        fi
    done
    return 1
}

if backup_upgrader_is_needed; then
    echo "==> Testing backup upgrader"
    mise exec -- ruby Where/Tools/Tests/upgrade_backup_test.rb
fi

# Which bundles a set of changed paths affects.
affected_bundles() {
    printf '%s\n' "$1" | python3 Tools/test_runner.py affected-bundles
}

# The importable progress filter reads xcodebuild output from stdin. Keeping it
# in Tools makes its terminal, heartbeat, cache, and zero-test behavior directly
# testable without launching Xcode.

# Resolve the run plan:
SCHEMES=()
UNIT_FILTERS=()
SNAPSHOT_FILTERS=()

add_scheme() {
    local candidate="$1" existing
    for existing in ${SCHEMES[@]+"${SCHEMES[@]}"}; do
        [ "$existing" = "$candidate" ] && return 0
    done
    SCHEMES+=("$candidate")
}

case "$SCOPE" in
    all) add_scheme "$UNIT_SCHEME" ;;
    snapshots) add_scheme "$SNAPSHOT_SCHEME" ;;
    everything) add_scheme "$UNIT_SCHEME"; add_scheme "$SNAPSHOT_SCHEME" ;;
    bundles|changed)
        if [ "$SCOPE" = changed ]; then
            if git rev-parse --verify --quiet "$BASE_REF" >/dev/null; then
                base="$(git merge-base HEAD "$BASE_REF")"
            else
                echo "warning: '$BASE_REF' not found; using uncommitted changes only" >&2
                base="HEAD"
            fi
            changed_files="$( { git diff --name-only "$base"
                                git diff --name-only
                                git ls-files --others --exclude-standard; } | sort -u )"
            if [ -z "$changed_files" ]; then
                echo "==> No changes against $BASE_REF — nothing to test."
                exit 0
            fi
            set +e
            selection="$(affected_bundles "$changed_files")"
            selection_status=$?
            set -e
            if [ "$selection_status" -ne 0 ]; then
                echo "error: could not work out which bundles your changes affect." >&2
                echo "       Run an explicit scope instead: ./test --all, ./test --snapshots," >&2
                echo "       or ./test --everything." >&2
                exit 1
            fi
            while IFS= read -r bundle; do
                [ -n "$bundle" ] && BUNDLES+=("$bundle")
            done <<EOF
$selection
EOF
            if [ "${#BUNDLES[@]}" -eq 0 ]; then
                echo "==> No test bundle covers these changes:"
                printf '%s\n' "$changed_files" | sed 's/^/      /'
                exit 0
            fi
            echo "==> Affected bundles: ${BUNDLES[*]}"
        fi
        image_bundles="$(snapshot_bundles)"
        for bundle in "${BUNDLES[@]}"; do
            if printf '%s\n' "$image_bundles" | grep -qx "$bundle"; then
                add_scheme "$SNAPSHOT_SCHEME"
                SNAPSHOT_FILTERS+=("-only-testing:$bundle")
            else
                add_scheme "$UNIT_SCHEME"
                UNIT_FILTERS+=("-only-testing:$bundle")
            fi
        done
        ;;
esac

if [ "${#ONLY[@]}" -gt 0 ]; then
    # An identifier's first component is its bundle, which decides the scheme —
    # so `./test --only 'WhereUISnapshotTests/…'` needs no `--snapshots`. Without
    # this, an identifier copied from the failure report above would run against
    # a scheme that doesn't contain its bundle and match nothing.
    only_images="$(snapshot_bundles)"
    for identifier in "${ONLY[@]}"; do
        if printf '%s\n' "$only_images" | grep -qx "${identifier%%/*}"; then
            add_scheme "$SNAPSHOT_SCHEME"
            [ -z "$ONLY_FILE" ] && SNAPSHOT_FILTERS+=("-only-testing:$identifier")
        else
            add_scheme "$UNIT_SCHEME"
            [ -z "$ONLY_FILE" ] && UNIT_FILTERS+=("-only-testing:$identifier")
        fi
    done
fi

if [ -n "$ONLY_FILE" ]; then
    if [ "${#SCHEMES[@]}" -ne 1 ]; then
        echo "error: --only-file must contain identifiers for one test scheme" >&2
        exit 1
    fi
    if [ "${SCHEMES[0]}" = "$SNAPSHOT_SCHEME" ]; then
        SNAPSHOT_FILTERS=("-only-testing" "@$ONLY_FILE")
    else
        UNIT_FILTERS=("-only-testing" "@$ONLY_FILE")
    fi
fi

if [ "${#SCHEMES[@]}" -eq 0 ]; then
    echo "error: nothing to run (see ./test --help)" >&2
    exit 1
fi

for scheme in "${SCHEMES[@]}"; do
    [ "$scheme" = "$SNAPSHOT_SCHEME" ] || continue

    expected_xcode_build="$(< .xcode-build-version)"
    xcode_build="$(xcodebuild -version | awk '/Build version/{print $3}')"
    if [ "$xcode_build" != "$expected_xcode_build" ]; then
        echo "error: snapshot tests require Xcode build $expected_xcode_build; found $xcode_build" >&2
        echo "       Install the pinned Xcode build. Then select it before capturing snapshots." >&2
        exit 1
    fi
    break
done

if [ "$DO_GENERATE" = true ]; then
    echo "==> Regenerating project (tuist generate --no-open)"
    started="$(phase_started)"
    if mise exec -- tuist generate --no-open >"$WORKDIR/generate.log" 2>&1; then
        emit_phase_timing generate "$started"
    else
        status=$?
        emit_phase_timing generate "$started" "" "$status"
        echo "error: tuist generate failed. Tail of $WORKDIR/generate.log:" >&2
        tail -n 20 "$WORKDIR/generate.log" >&2
        exit "$status"
    fi
fi

if [ -n "$TEST_ARTIFACTS" ]; then
    ARTIFACT_SCHEME_ARGS=()
    for scheme in "${SCHEMES[@]}"; do
        ARTIFACT_SCHEME_ARGS+=(--scheme "$scheme")
    done
    python3 .circleci/test_artifacts.py resolve-all \
        --root "$TEST_ARTIFACTS" \
        "${ARTIFACT_SCHEME_ARGS[@]}" >/dev/null
    echo "Validated test artifacts before simulator boot."
fi

if [ -n "$BUILD_ARTIFACTS" ]; then
    BUILD_ARTIFACTS="$(mkdir -p "$BUILD_ARTIFACTS" && cd "$BUILD_ARTIFACTS" && pwd -P)"
    DESTINATION="generic/platform=iOS Simulator"
    DERIVED_DATA_ARGS=(-derivedDataPath "$BUILD_ARTIFACTS/DerivedData")
else
    SIMULATOR_FLAGS=(--device "$DEVICE" --os "$OS")
    [ "$SHARED" = true ] && SIMULATOR_FLAGS+=(--shared)
    started="$(phase_started)"
    if UDID="$(./simulator "${SIMULATOR_FLAGS[@]}")"; then
        emit_phase_timing simulator "$started"
    else
        status=$?
        emit_phase_timing simulator "$started" "" "$status"
        exit "$status"
    fi
    DESTINATION="platform=iOS Simulator,id=$UDID"
    DERIVED_DATA_ARGS=()
fi

# xcodebuild strips the TEST_RUNNER_ prefix on the way in, which is why the
# pipeline reads SNAPSHOT_* rather than these names.
RUN_ENV=()
[ -n "$RECORD" ] && RUN_ENV+=("TEST_RUNNER_SNAPSHOT_RECORD=$RECORD")
[ "$TIMINGS" = true ] && RUN_ENV+=("TEST_RUNNER_SNAPSHOT_TIMING=1")
[ "$REVIEW" = true ] && RUN_ENV+=("TEST_RUNNER_SNAPSHOT_DIFF=1")
[ -n "${SNAPSHOT_SETTLE_TIMEOUT_MULTIPLIER:-}" ] \
    && RUN_ENV+=("TEST_RUNNER_SNAPSHOT_SETTLE_TIMEOUT_MULTIPLIER=$SNAPSHOT_SETTLE_TIMEOUT_MULTIPLIER")

# SwiftPM's generated `Bundle.module` accessors honor
# PACKAGE_RESOURCE_BUNDLE_PATH as their first lookup candidate (DEBUG-only,
# which test builds are). The schemes set it to $(BUILT_PRODUCTS_DIR) for
# Xcode-IDE runs, but xcodebuild does not expand build-setting macros in
# scheme environment variables, so this script resolves the concrete path
# and delivers it via TEST_RUNNER_ like the pins above. See
# `packageResourceEnvironment` in Project.swift for why the lookup needs an
# override at all (Xcode 27 beta 4's package linking orphans the resource
# bundles copied into each .xctest).
if [ -n "$TEST_ARTIFACTS" ]; then
    BUILT_PRODUCTS_DIR="$(python3 .circleci/test_artifacts.py resolve \
        --root "$TEST_ARTIFACTS" --scheme "${SCHEMES[0]}" --field products)"
elif [ -n "$BUILD_ARTIFACTS" ]; then
    BUILT_PRODUCTS_DIR=""
else
    BUILT_PRODUCTS_DIR="$(xcodebuild -showBuildSettings \
        -workspace "$WORKSPACE" \
        -scheme "${SCHEMES[0]}" \
        -destination "$DESTINATION" 2>/dev/null \
        | awk -F' = ' '/ BUILT_PRODUCTS_DIR = /{print $2; exit}')"
fi
if [ -n "$BUILT_PRODUCTS_DIR" ]; then
    RUN_ENV+=("TEST_RUNNER_PACKAGE_RESOURCE_BUNDLE_PATH=$BUILT_PRODUCTS_DIR")
elif [ -z "$BUILD_ARTIFACTS" ]; then
    echo "warning: could not resolve BUILT_PRODUCTS_DIR; package resource" \
        "bundle lookups will fall back to the linker's placement" >&2
fi

overall_status=0
RESULTS=()
LOGS=()

for scheme in "${SCHEMES[@]}"; do
    filters=()
    if [ "$scheme" = "$SNAPSHOT_SCHEME" ]; then
        filters=(${SNAPSHOT_FILTERS[@]+"${SNAPSHOT_FILTERS[@]}"})
    else
        filters=(${UNIT_FILTERS[@]+"${UNIT_FILTERS[@]}"})
    fi
    log="$WORKDIR/$scheme.log"
    result="$WORKDIR/$scheme.xcresult"
    rm -rf "$result"

    if [ "$DO_BUILD" = true ]; then
        echo "==> Building $scheme for testing"
        started="$(phase_started)"
        if xcodebuild build-for-testing \
            -workspace "$WORKSPACE" \
            -scheme "$scheme" \
            -configuration Debug \
            -destination "$DESTINATION" \
            ${DERIVED_DATA_ARGS[@]+"${DERIVED_DATA_ARGS[@]}"} \
            ${BUILD_ARTIFACTS:+ARCHS=arm64} \
            ${BUILD_ARTIFACTS:+ONLY_ACTIVE_ARCH=YES} \
            >"$WORKDIR/$scheme-build.log" 2>&1
        then
            emit_phase_timing build "$started" "$scheme"
        else
            status=$?
            emit_phase_timing build "$started" "$scheme" "$status"
            echo "error: build failed for $scheme. Tail of $WORKDIR/$scheme-build.log:" >&2
            tail -n 30 "$WORKDIR/$scheme-build.log" >&2
            exit "$status"
        fi
    fi

    [ -n "$BUILD_ARTIFACTS" ] && continue

    if [ -n "$TEST_ARTIFACTS" ]; then
        XCTESTRUN="$(python3 .circleci/test_artifacts.py resolve \
            --root "$TEST_ARTIFACTS" --scheme "$scheme" --field xctestrun)"
        TEST_LOCATION=(-xctestrun "$XCTESTRUN")
    else
        TEST_LOCATION=(-workspace "$WORKSPACE" -scheme "$scheme")
    fi

    if [ -n "$ENUMERATE_SUITES" ]; then
        raw_enumeration="$WORKDIR/$scheme-enumeration.json"
        started="$(phase_started)"
        if xcodebuild test-without-building \
            ${TEST_LOCATION[@]+"${TEST_LOCATION[@]}"} \
            -destination "$DESTINATION" \
            -enumerate-tests \
            -test-enumeration-style hierarchical \
            -test-enumeration-format json \
            -test-enumeration-output-path "$raw_enumeration" \
            >"$WORKDIR/$scheme-enumeration.log" 2>&1
        then
            python3 .circleci/test_artifacts.py suites \
                --input "$raw_enumeration" --output "$ENUMERATE_SUITES"
            emit_phase_timing enumerate "$started" "$scheme"
            exit 0
        else
            status=$?
            emit_phase_timing enumerate "$started" "$scheme" "$status"
            tail -n 30 "$WORKDIR/$scheme-enumeration.log" >&2
            exit "$status"
        fi
    fi

    # One cache entry per scheme + filter set: a run of three bundles and a run
    # of the whole scheme have different totals, and mixing them is what made the
    # progress line read "368/38".
    scope_key="$(printf '%s\n' "$scheme" ${filters[@]+"${filters[@]}"} \
        | shasum | cut -c1-12)"
    counts="$WORKDIR/counts-$scheme-$scope_key.json"
    progress_args=(
        --heartbeat "$HEARTBEAT"
        --counts-file "$counts"
        --scheme "$scheme"
    )
    [ -n "$STATUS_FILE" ] && progress_args+=(--status-file "$STATUS_FILE")
    [ "$TIMINGS" = true ] && progress_args+=(--count-images)

    echo "==> Testing $scheme on $DEVICE / iOS $OS"
    started="$(phase_started)"
    set +e
    env ${RUN_ENV[@]+"${RUN_ENV[@]}"} \
        xcodebuild test-without-building \
        ${TEST_LOCATION[@]+"${TEST_LOCATION[@]}"} \
        -destination "$DESTINATION" \
        -resultBundlePath "$result" \
        -collect-test-diagnostics never \
        ${filters[@]+"${filters[@]}"} \
        2>&1 \
        | tee "$log" \
        | python3 -u Tools/test_runner.py progress "${progress_args[@]}"
    # [0] is xcodebuild; [1] is tee and [2] the progress filter. Prefer the
    # authoritative test status, then surface a logging/reporting failure when
    # Xcode itself succeeded. No pipeline stage may turn a broken run green.
    pipeline_statuses=("${PIPESTATUS[@]}")
    status="${pipeline_statuses[0]}"
    [ "$status" -eq 0 ] && [ "${pipeline_statuses[1]}" -ne 0 ] \
        && status="${pipeline_statuses[1]}"
    [ "$status" -eq 0 ] && [ "${pipeline_statuses[2]}" -ne 0 ] \
        && status="${pipeline_statuses[2]}"
    set -e
    emit_phase_timing test "$started" "$scheme" "$status"
    RESULTS+=("$result")
    LOGS+=("$log")
    [ "$status" -ne 0 ] && overall_status="$status"
    if [ -f "$counts.empty" ]; then
        rm -f "$counts.empty"
        [ "$overall_status" -eq 0 ] && overall_status=1
    fi
done

if [ -n "$BUILD_ARTIFACTS" ]; then
    manifest_args=()
    for scheme in "${SCHEMES[@]}"; do
        manifest_args+=(--scheme "$scheme")
    done
    python3 .circleci/test_artifacts.py create \
        --root "$BUILD_ARTIFACTS" "${manifest_args[@]}"
    artifact_bytes="$(du -sk "$BUILD_ARTIFACTS/DerivedData/Build/Products" | awk '{print $1 * 1024}')"
    printf 'CI_ARTIFACT {"bytes":%s,"schemes":%s}\n' \
        "$artifact_bytes" "${#SCHEMES[@]}"
    exit 0
fi

# Failures, reprinted with the identifier that re-runs each one.
if [ "$overall_status" -ne 0 ]; then
    echo
    rule
    echo "FAILURES"
    rule
    for result in "${RESULTS[@]}"; do
        [ -d "$result" ] || continue
        xcrun xcresulttool get test-results tests --path "$result" \
            >"$WORKDIR/failures.json" 2>/dev/null || true
        python3 Tools/test_runner.py failures \
            --result "$result" --tests-json "$WORKDIR/failures.json"
    done
fi

if [ "$TIMINGS" = true ]; then
    echo
    rule
    echo "SNAPSHOT CAPTURE PHASES"
    rule
    : >"$WORKDIR/timings.jsonl"
    for log in "${LOGS[@]}"; do
        grep -o 'SNAPSHOT_TIMING {.*}' "$log" 2>/dev/null \
            | sed 's/^SNAPSHOT_TIMING //' >>"$WORKDIR/timings.jsonl" || true
    done
    python3 Tools/snapshot_reports.py timings "$WORKDIR/timings.jsonl"
fi

if [ "$REVIEW" = true ]; then
    echo
    rule
    echo "SNAPSHOT DIFFERENCES"
    rule
    : >"$WORKDIR/diffs.jsonl"
    for log in "${LOGS[@]}"; do
        grep -o 'SNAPSHOT_DIFF {.*}' "$log" 2>/dev/null \
            | sed 's/^SNAPSHOT_DIFF //' >>"$WORKDIR/diffs.jsonl" || true
    done
    difference_args=()
    [ -n "$RECORD" ] && [ "$RECORD" != never ] && difference_args+=(--recording)
    python3 Tools/snapshot_reports.py differences \
        "$WORKDIR/diffs.jsonl" ${difference_args[@]+"${difference_args[@]}"}
fi

echo
if [ "$overall_status" -eq 0 ]; then
    echo "==> Passed. Logs and result bundles: $WORKDIR"
else
    echo "==> Failed (exit $overall_status). Logs and result bundles: $WORKDIR"
fi
exit "$overall_status"