|
1 | | -name: Performance Tests |
2 | | - |
3 | | -on: |
4 | | - pull_request: |
5 | | - branches: |
6 | | - - main |
7 | | - - release* |
8 | | - - release/* |
9 | | - - release-* |
10 | | - workflow_dispatch: |
11 | | - |
| 1 | +name: Fixture Windows Matched Control |
| 2 | +'on': |
| 3 | + workflow_dispatch: {} |
12 | 4 | permissions: |
13 | | - actions: read |
14 | 5 | contents: read |
15 | | - pull-requests: write |
16 | | - |
17 | 6 | jobs: |
18 | | - performance: |
19 | | - name: E2E Performance Tests |
20 | | - runs-on: ${{ matrix.os }} |
21 | | - strategy: |
22 | | - fail-fast: false |
23 | | - matrix: |
24 | | - include: |
25 | | - - os: windows-latest |
26 | | - target: x86_64-pc-windows-msvc |
27 | | - platform: Windows |
28 | | - comment_header: perf-windows |
29 | | - - os: ubuntu-latest |
30 | | - target: x86_64-unknown-linux-musl |
31 | | - platform: Linux |
32 | | - comment_header: perf-linux |
33 | | - - os: macos-latest |
34 | | - target: x86_64-apple-darwin |
35 | | - platform: macOS |
36 | | - comment_header: perf-macos |
| 7 | + matched-control: |
| 8 | + runs-on: windows-latest |
| 9 | + timeout-minutes: 35 |
37 | 10 | steps: |
38 | | - - name: Checkout |
39 | | - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
40 | | - |
41 | | - - name: Post In-Progress Comment |
42 | | - if: >- |
43 | | - github.event_name == 'pull_request' && |
44 | | - github.event.pull_request.head.repo.full_name == github.repository |
45 | | - continue-on-error: true |
46 | | - uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 |
47 | | - with: |
48 | | - header: ${{ matrix.comment_header }} |
49 | | - message: | |
50 | | - ## Performance Report (${{ matrix.platform }}) :hourglass_flowing_sand: |
51 | | -
|
52 | | - Running performance tests against baseline `${{ github.event.pull_request.base.sha }}`. |
53 | | -
|
54 | | - - name: Set Python to PATH |
55 | | - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 |
56 | | - with: |
57 | | - python-version: "3.12" |
58 | | - |
59 | | - - name: Validate Snapshot Comparator |
60 | | - run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v |
61 | | - shell: bash |
62 | | - |
63 | | - - name: Add Conda to PATH (Windows) |
64 | | - if: startsWith(matrix.os, 'windows') |
65 | | - run: | |
66 | | - $path = $env:PATH + ";" + $env:CONDA + "\condabin" |
67 | | - echo "PATH=$path" >> $env:GITHUB_ENV |
68 | | -
|
69 | | - - name: Add Conda to PATH (Ubuntu) |
70 | | - if: startsWith(matrix.os, 'ubuntu') |
71 | | - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV |
72 | | - shell: bash |
73 | | - |
74 | | - - name: Install Conda + add to PATH (macOS) |
75 | | - if: startsWith(matrix.os, 'macos') |
76 | | - run: | |
77 | | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh |
78 | | - bash ~/miniconda.sh -b -p ~/miniconda |
79 | | - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV |
80 | | - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV |
81 | | - shell: bash |
82 | | - |
83 | | - - name: Create test Conda environment |
84 | | - run: conda create -n perf-test-env python=3.12 -y |
85 | | - |
86 | | - - name: Create test venv |
87 | | - run: python -m venv .venv |
88 | | - shell: bash |
89 | | - |
90 | | - - name: Rust Tool Chain setup |
91 | | - uses: dtolnay/rust-toolchain@stable |
92 | | - with: |
93 | | - toolchain: stable |
94 | | - targets: ${{ matrix.target }} |
95 | | - |
96 | | - - name: Cargo Fetch |
97 | | - run: cargo fetch |
98 | | - shell: bash |
99 | | - |
100 | | - - name: Build Release |
101 | | - run: cargo build --release --target ${{ matrix.target }} |
102 | | - shell: bash |
103 | | - |
104 | | - - name: Run Performance Tests |
105 | | - id: benchmark |
106 | | - run: | |
107 | | - set -o pipefail |
108 | | - cargo test --release --features ci-perf --target ${{ matrix.target }} --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt |
109 | | - env: |
110 | | - RUST_BACKTRACE: 1 |
111 | | - RUST_LOG: warn |
112 | | - shell: bash |
113 | | - |
114 | | - - name: Extract Performance Metrics |
115 | | - if: steps.benchmark.outcome == 'success' |
116 | | - run: | |
117 | | - if ! grep -q "JSON metrics:" perf-output.txt; then |
118 | | - echo "Performance test produced no JSON metrics" >&2 |
119 | | - exit 1 |
120 | | - fi |
121 | | - sed -n '/JSON metrics:/,/^}/p' perf-output.txt | tail -n +2 > metrics.json |
122 | | - python -m json.tool metrics.json > /dev/null |
123 | | - cat metrics.json |
124 | | - shell: bash |
125 | | - |
126 | | - - name: Wait for Exact PR Base Performance |
127 | | - id: wait_for_base_performance |
128 | | - if: always() && github.event_name == 'pull_request' |
129 | | - run: >- |
130 | | - python scripts/wait_for_baseline.py |
131 | | - --repository "${{ github.repository }}" |
132 | | - --workflow perf-baseline.yml |
133 | | - --commit "${{ github.event.pull_request.base.sha }}" |
134 | | - --artifact "perf-baseline-${{ matrix.os }}" |
135 | | - --timeout-seconds 1200 |
136 | | - --poll-seconds 20 |
137 | | - env: |
138 | | - GITHUB_TOKEN: ${{ github.token }} |
139 | | - shell: bash |
140 | | - |
141 | | - - name: Download Exact PR Base Performance |
142 | | - if: >- |
143 | | - always() && github.event_name == 'pull_request' && |
144 | | - steps.wait_for_base_performance.outcome == 'success' |
145 | | - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 |
146 | | - with: |
147 | | - workflow: perf-baseline.yml |
148 | | - commit: ${{ github.event.pull_request.base.sha }} |
149 | | - workflow_conclusion: success |
150 | | - name: perf-baseline-${{ matrix.os }} |
151 | | - path: baseline-perf |
152 | | - check_artifacts: true |
153 | | - search_artifacts: true |
154 | | - |
155 | | - - name: Download Main Performance for Manual Run |
156 | | - if: always() && github.event_name == 'workflow_dispatch' |
157 | | - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 |
158 | | - with: |
159 | | - workflow: perf-baseline.yml |
160 | | - branch: main |
161 | | - workflow_conclusion: success |
162 | | - name: perf-baseline-${{ matrix.os }} |
163 | | - path: baseline-perf |
164 | | - check_artifacts: true |
165 | | - search_artifacts: true |
166 | | - |
167 | | - - name: Compare Performance Snapshot |
168 | | - if: always() |
169 | | - run: >- |
170 | | - python scripts/quality_snapshot.py performance |
171 | | - --current metrics.json |
172 | | - --baseline baseline-perf/metrics.json |
173 | | - --platform "${{ matrix.platform }}" |
174 | | - --report performance-report.md |
175 | | - --summary "$GITHUB_STEP_SUMMARY" |
176 | | - shell: bash |
177 | | - |
178 | | - - name: Upload PR Performance Results |
179 | | - if: always() |
180 | | - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
181 | | - with: |
182 | | - name: perf-pr-${{ matrix.os }} |
183 | | - path: | |
184 | | - metrics.json |
185 | | - performance-report.md |
186 | | - if-no-files-found: ignore |
187 | | - |
188 | | - - name: Post Performance Comment |
189 | | - if: >- |
190 | | - always() && github.event_name == 'pull_request' && |
191 | | - github.event.pull_request.head.repo.full_name == github.repository |
192 | | - continue-on-error: true |
193 | | - uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 |
194 | | - with: |
195 | | - header: ${{ matrix.comment_header }} |
196 | | - path: performance-report.md |
| 11 | + - name: Checkout exact PR head |
| 12 | + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 |
| 13 | + with: |
| 14 | + ref: 6772e5c3631116386de3953dec083a2d08b4fcfa |
| 15 | + - name: Initialize evidence |
| 16 | + shell: pwsh |
| 17 | + run: | |
| 18 | + New-Item -ItemType Directory -Path "$env:RUNNER_TEMP\fixture-matched-control" -ErrorAction Stop | Out-Null |
| 19 | + - name: Set Python to PATH |
| 20 | + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 |
| 21 | + with: |
| 22 | + python-version: '3.12' |
| 23 | + - name: Create the standard Windows workload |
| 24 | + shell: pwsh |
| 25 | + run: | |
| 26 | + $env:PATH += ";$env:CONDA\condabin" |
| 27 | + "PATH=$env:PATH" >> $env:GITHUB_ENV |
| 28 | + conda create -n perf-test-env python=3.12 -y |
| 29 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 30 | + python -m venv .venv |
| 31 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 32 | + - name: Rust toolchain |
| 33 | + uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 |
| 34 | + with: |
| 35 | + toolchain: 1.98.1 |
| 36 | + targets: x86_64-pc-windows-msvc |
| 37 | + - name: Instrument only the shared test harness after its measured passes |
| 38 | + shell: pwsh |
| 39 | + run: | |
| 40 | + @' |
| 41 | + from pathlib import Path |
| 42 | + path = Path("crates/pet/tests/e2e_performance.rs") |
| 43 | + source = path.read_text() |
| 44 | + for name in ["EnvironmentIdentity", "ManagerIdentity", "Inventory"]: |
| 45 | + marker = f"struct {name} {{" |
| 46 | + assert source.count(marker) == 1 |
| 47 | + source = source.replace(marker, "#[derive(serde::Serialize)]\n" + marker) |
| 48 | + marker = 'expected_inventory.expect("Performance summary must run at least one iteration");' |
| 49 | + assert source.count(marker) == 1 |
| 50 | + source = source.replace(marker, marker + '\n' |
| 51 | + ' if let Some(path) = std::env::var_os("PET_DIAGNOSTIC_INVENTORY") {\n' |
| 52 | + ' std::fs::write(path, serde_json::to_vec(&expected_inventory).unwrap()).unwrap();\n' |
| 53 | + ' }\n') |
| 54 | + path.write_text(source) |
| 55 | + '@ | python - |
| 56 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 57 | + - name: Build exact base and head |
| 58 | + shell: pwsh |
| 59 | + run: | |
| 60 | + git fetch --no-tags --depth=1 origin 3bd7eea22298a0a8e9a08079dd2ed44212368214 |
| 61 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 62 | + git worktree add --detach "$env:RUNNER_TEMP\pet-base" 3bd7eea22298a0a8e9a08079dd2ed44212368214 |
| 63 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 64 | + cargo build --locked --release --target x86_64-pc-windows-msvc --bin pet --manifest-path "$env:RUNNER_TEMP\pet-base\Cargo.toml" --target-dir "$env:RUNNER_TEMP\pet-base-target" |
| 65 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 66 | + Copy-Item "$env:RUNNER_TEMP\pet-base-target\x86_64-pc-windows-msvc\release\pet.exe" "$env:RUNNER_TEMP\pet-base-exe" -ErrorAction Stop |
| 67 | + cargo build --locked --release --target x86_64-pc-windows-msvc --bin pet |
| 68 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 69 | + Copy-Item "target\x86_64-pc-windows-msvc\release\pet.exe" "$env:RUNNER_TEMP\pet-head-exe" -ErrorAction Stop |
| 70 | + cargo test --locked --release --features ci-perf --target x86_64-pc-windows-msvc --test e2e_performance --no-run --message-format=json > "$env:RUNNER_TEMP\pet-test-build.jsonl" |
| 71 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 72 | + if ((git rev-parse HEAD) -ne '6772e5c3631116386de3953dec083a2d08b4fcfa') { throw 'Unexpected head revision' } |
| 73 | + if ((git -C "$env:RUNNER_TEMP\pet-base" rev-parse HEAD) -ne '3bd7eea22298a0a8e9a08079dd2ed44212368214') { throw 'Unexpected base revision' } |
| 74 | + @{ |
| 75 | + base = '3bd7eea22298a0a8e9a08079dd2ed44212368214' |
| 76 | + head = '6772e5c3631116386de3953dec083a2d08b4fcfa' |
| 77 | + baseBinarySha256 = (Get-FileHash "$env:RUNNER_TEMP\pet-base-exe" -Algorithm SHA256).Hash |
| 78 | + headBinarySha256 = (Get-FileHash "$env:RUNNER_TEMP\pet-head-exe" -Algorithm SHA256).Hash |
| 79 | + harnessSha256 = (Get-FileHash 'crates\pet\tests\e2e_performance.rs' -Algorithm SHA256).Hash |
| 80 | + rustc = (rustc -Vv) -join "`n" |
| 81 | + runnerImage = $env:ImageOS |
| 82 | + runnerImageVersion = $env:ImageVersion |
| 83 | + processor = $env:PROCESSOR_IDENTIFIER |
| 84 | + } | ConvertTo-Json | Set-Content "$env:RUNNER_TEMP\fixture-matched-control\provenance.json" -Encoding utf8 |
| 85 | + - name: Counterbalanced same-host measurements |
| 86 | + shell: pwsh |
| 87 | + env: |
| 88 | + RUST_BACKTRACE: '1' |
| 89 | + RUST_LOG: warn |
| 90 | + run: | |
| 91 | + @' |
| 92 | + import hashlib |
| 93 | + import json |
| 94 | + import os |
| 95 | + from pathlib import Path |
| 96 | + import shutil |
| 97 | + import subprocess |
| 98 | +
|
| 99 | + temp = Path(os.environ["RUNNER_TEMP"]) |
| 100 | + workspace = Path(os.environ["GITHUB_WORKSPACE"]) |
| 101 | + artifacts = temp / "fixture-matched-control" |
| 102 | + artifacts.mkdir(exist_ok=True) |
| 103 | + messages = [json.loads(line) for line in (temp / "pet-test-build.jsonl").read_text(encoding="utf-8-sig").splitlines()] |
| 104 | + tests = [m["executable"] for m in messages if m.get("reason") == "compiler-artifact" |
| 105 | + and m["target"]["name"] == "e2e_performance" and m.get("executable")] |
| 106 | + if len(tests) != 1: |
| 107 | + raise RuntimeError(f"Expected one benchmark executable, got {tests!r}") |
| 108 | + executable = workspace / "target" / "x86_64-pc-windows-msvc" / "release" / "pet.exe" |
| 109 | + revisions = {"base": "3bd7eea22298a0a8e9a08079dd2ed44212368214", |
| 110 | + "head": "6772e5c3631116386de3953dec083a2d08b4fcfa"} |
| 111 | + order = ["base", "head", "head", "base", "head", "base", "base", "head"] |
| 112 | + summary = [] |
| 113 | + for index, label in enumerate(order, 1): |
| 114 | + source = temp / f"pet-{label}-exe" |
| 115 | + shutil.copy2(source, executable) |
| 116 | + digest = hashlib.sha256(executable.read_bytes()).hexdigest() |
| 117 | + if digest != hashlib.sha256(source.read_bytes()).hexdigest(): |
| 118 | + raise RuntimeError("Benchmark executable copy did not match source") |
| 119 | + inventory = temp / f"pet-inventory-{index}.json" |
| 120 | + environment = dict(os.environ, PET_DIAGNOSTIC_INVENTORY=str(inventory)) |
| 121 | + log = artifacts / f"{index}-{label}.log" |
| 122 | + try: |
| 123 | + with log.open("w") as output: |
| 124 | + subprocess.run([tests[0], "test_performance_summary", "--exact", "--nocapture"], |
| 125 | + cwd=workspace, stdout=output, stderr=subprocess.STDOUT, |
| 126 | + check=True, timeout=180, env=environment) |
| 127 | + inventory_digest = hashlib.sha256(inventory.read_bytes()).hexdigest() |
| 128 | + finally: |
| 129 | + inventory.unlink(missing_ok=True) |
| 130 | + text = log.read_text() |
| 131 | + metrics, _ = json.JSONDecoder().raw_decode(text.split("JSON metrics:\n", 1)[1]) |
| 132 | + (artifacts / f"{index}-{label}.json").write_text(json.dumps(metrics, indent=2) + "\n") |
| 133 | + result = {"iteration": index, "label": label, "revision": revisions[label], "sha256": digest, |
| 134 | + "inventory_sha256": inventory_digest, |
| 135 | + "environments_count": metrics["environments_count"], |
| 136 | + "managers_count": metrics["managers_count"], |
| 137 | + "stats": metrics["stats"], "locators": metrics["locators"], "phases": metrics["phases"]} |
| 138 | + summary.append(result) |
| 139 | + (artifacts / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") |
| 140 | + print(json.dumps({"iteration": index, "label": label, "revision": revisions[label], |
| 141 | + "sha256": digest, "refresh": metrics["stats"]["refresh_round_trip"], |
| 142 | + "startup": metrics["stats"]["server_startup"]}), flush=True) |
| 143 | + if len({v["inventory_sha256"] for v in summary}) != 1: |
| 144 | + raise RuntimeError("Exact inventory identities differ between base and head workloads") |
| 145 | + '@ | python - |
| 146 | + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } |
| 147 | + - name: Preserve matched control evidence |
| 148 | + if: always() |
| 149 | + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a |
| 150 | + with: |
| 151 | + name: fixture-557-matched-windows |
| 152 | + path: ${{ runner.temp }}\fixture-matched-control |
| 153 | + if-no-files-found: error |
0 commit comments