diff --git a/.gitignore b/.gitignore index f26e74136c0..dc5603d7071 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ __pycache__/ *.pyc +# Swift native-review build output +tools/native-review/swift/.build/ + # lefthook-generated hook scripts (machine-specific) .hooks/ diff --git a/Justfile b/Justfile index ce8647cf77c..7d3fada6c75 100644 --- a/Justfile +++ b/Justfile @@ -999,3 +999,33 @@ benchmark *ARGS: # Stop the benchmark Docker stack (state and channels are kept) benchmark-down: docker compose --project-name buzz-benchmark down + +# Validate macOS native-review tooling and report required OS permissions. +native-review-doctor: + ./tools/native-review/bin/review-native doctor + +# Run one declarative journey against the isolated local desktop fixture. +native-review-desktop JOURNEY="tools/native-review/desktop/tooltip-fresh-dwell.yaml": + ./tools/native-review/bin/review-native run "{{JOURNEY}}" + +# Capture a repeatable native performance cohort (minimum 3 runs). +native-review-benchmark JOURNEY="tools/native-review/desktop/tooltip-fresh-dwell.yaml" RUNS="5": + ./tools/native-review/bin/review-native benchmark "{{JOURNEY}}" --runs "{{RUNS}}" + +# Compare baseline and candidate receipt cohorts with explicit budget policy. +# Pass BASELINE/CANDIDATE as repeated CLI args, e.g. "--baseline a --baseline b". +native-review-compare BASELINE CANDIDATE BUDGET="tools/native-review/performance/tooltip-fresh-dwell.yaml": + ./tools/native-review/bin/review-native compare {{BASELINE}} {{CANDIDATE}} --budget "{{BUDGET}}" + +# Run the native iOS Simulator pairing journey with MP4 and screenshot evidence. +native-review-ios DEVICE="iPhone 17 Pro": + ./tools/native-review/bin/review-ios --device "{{DEVICE}}" + +# Emit a redacted, relay-compatible clip + receipt + focused log excerpt. +native-review-finding RECEIPT OUTPUT MATCH="": + ./tools/native-review/bin/review-native finding-bundle "{{RECEIPT}}" --output "{{OUTPUT}}" {{ if MATCH == "" { "" } else { "--match " + quote(MATCH) } }} + +# Publish the review report with its exact-SHA video and optional timecoded highlights. +# MENTION is the delegator pubkey; pass an empty string only when no callback is required. +native-review-publish RECEIPT SUMMARY CHANNEL REPLY_TO HIGHLIGHTS="" MENTION="": + ./tools/native-review/bin/review-native publish-review "{{RECEIPT}}" --summary "{{SUMMARY}}" --channel "{{CHANNEL}}" --reply-to "{{REPLY_TO}}" {{ if HIGHLIGHTS == "" { "" } else { "--highlights " + quote(HIGHLIGHTS) } }} {{ if MENTION == "" { "" } else { "--mention " + quote(MENTION) } }} diff --git a/desktop/src/main.tsx b/desktop/src/main.tsx index 634a9b171a1..148ac86fe98 100644 --- a/desktop/src/main.tsx +++ b/desktop/src/main.tsx @@ -22,6 +22,7 @@ import { recoverLocalStorageQuotaOnStartup } from "@/shared/lib/localStorageQuot import { startLocalStorageSweep } from "@/shared/lib/localStorageSweep"; import { initializeConversationDensityPreference } from "@/shared/lib/conversationDensityPreference"; import { initializeFontSizePreference } from "@/shared/lib/fontSizePreference"; +import { installNativeReviewSemanticProbe } from "@/testing/nativeReviewSemanticProbe"; type E2eWindow = Window & { __BUZZ_E2E__?: unknown; @@ -31,6 +32,54 @@ const E2E_DEFAULT_PUBKEY = "deadbeef".repeat(8); const E2E_COMMUNITY_ID = "e2e-default-community"; const ONBOARDING_COMPLETION_STORAGE_KEY_PREFIX = "buzz-onboarding-complete.v1:"; const DEV_STATE_RESET_PARAM = "resetDevState"; +const NATIVE_REVIEW_PARAM = "nativeReview"; + +function configureNativeReviewFixtureFromUrl() { + const buildEnabled = import.meta.env.VITE_NATIVE_REVIEW === "1"; + if (!import.meta.env.DEV && !buildEnabled) return; + const url = new URL(window.location.href); + const enabled = + url.searchParams.get(NATIVE_REVIEW_PARAM) === "1" || buildEnabled; + if (!enabled) return; + + const relayUrl = + url.searchParams.get("reviewRelay") ?? + import.meta.env.VITE_NATIVE_REVIEW_RELAY; + const pubkey = + url.searchParams.get("reviewPubkey") ?? + import.meta.env.VITE_NATIVE_REVIEW_PUBKEY; + if ( + !relayUrl || + !pubkey || + !/^(ws|http):\/\/(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?\/?$/.test( + relayUrl, + ) + ) { + throw new Error( + "native review bootstrap requires a loopback relay and pubkey", + ); + } + const communityId = "native-review-local"; + const community = { + addedAt: new Date().toISOString(), + id: communityId, + name: "Native Review", + pubkey, + relayUrl, + }; + window.localStorage.setItem("buzz-communities", JSON.stringify([community])); + window.localStorage.setItem("buzz-active-community-id", communityId); + window.localStorage.setItem( + `buzz-machine-onboarding-complete.v2:${pubkey}`, + "true", + ); + window.localStorage.setItem(`buzz-onboarding-complete.v1:${pubkey}`, "true"); + window.localStorage.setItem( + `buzz-community-onboarding-complete.v1:${encodeURIComponent(relayUrl)}:${pubkey}`, + "true", + ); + installNativeReviewSemanticProbe(); +} function resetDevWebviewStateFromUrl() { if (!import.meta.env.DEV) { @@ -124,6 +173,7 @@ async function installE2eBridgeIfConfigured() { async function bootstrap() { resetDevWebviewStateFromUrl(); + configureNativeReviewFixtureFromUrl(); configureDevE2eBridgeFromUrl(); recoverLocalStorageQuotaOnStartup(); initializeConversationDensityPreference(); diff --git a/desktop/src/testing/nativeReviewSemanticProbe.ts b/desktop/src/testing/nativeReviewSemanticProbe.ts new file mode 100644 index 00000000000..65927d0941c --- /dev/null +++ b/desktop/src/testing/nativeReviewSemanticProbe.ts @@ -0,0 +1,121 @@ +type SemanticNode = { + id?: string; + role?: string; + name?: string; + value?: string; + scrollY: number; + enabled: boolean; + focused: boolean; + frame: { x: number; y: number; width: number; height: number }; + viewport: { width: number; height: number }; +}; + +const IMPLICIT_ROLES: Partial> = { + A: "link", + BUTTON: "button", + INPUT: "text-field", + TEXTAREA: "text-area", +}; + +function accessibleName(element: HTMLElement): string | undefined { + const labelledBy = element.getAttribute("aria-labelledby"); + const labelledText = labelledBy + ?.split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim()) + .filter(Boolean) + .join(" "); + return ( + element.getAttribute("aria-label")?.trim() || + labelledText || + element.getAttribute("title")?.trim() || + (element.getAttribute("role") === "tooltip" + ? element.textContent?.trim() + : undefined) || + undefined + ); +} + +function snapshot(): SemanticNode[] { + const nodes: SemanticNode[] = []; + for (const candidate of document.querySelectorAll( + "[data-testid], [role], button, textarea, input, a[href]", + )) { + const rect = candidate.getBoundingClientRect(); + const style = window.getComputedStyle(candidate); + if ( + rect.width <= 0 || + rect.height <= 0 || + style.display === "none" || + style.visibility === "hidden" + ) { + continue; + } + const id = candidate.dataset.testid; + const role = + candidate.getAttribute("role") ?? IMPLICIT_ROLES[candidate.tagName]; + const name = accessibleName(candidate); + const value = + candidate instanceof HTMLInputElement || + candidate instanceof HTMLTextAreaElement + ? candidate.value + : candidate.isContentEditable + ? candidate.innerText.replace(/\r\n?/g, "\n").replace(/\n$/, "") + : undefined; + if (!id && !role && !name) continue; + nodes.push({ + ...(id ? { id } : {}), + ...(role ? { role } : {}), + ...(name ? { name } : {}), + ...(value !== undefined ? { value } : {}), + scrollY: candidate.scrollTop, + enabled: + !candidate.hasAttribute("disabled") && + candidate.getAttribute("aria-disabled") !== "true", + focused: candidate === document.activeElement, + frame: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }, + viewport: { + width: window.innerWidth, + height: window.innerHeight, + }, + }); + } + return nodes; +} + +export function installNativeReviewSemanticProbe(): void { + let scheduled = false; + const publish = () => { + scheduled = false; + const payload = JSON.stringify(snapshot()); + if ( + !navigator.sendBeacon( + import.meta.env.VITE_NATIVE_REVIEW_PROBE_URL, + payload, + ) + ) { + console.error("native review semantic probe beacon was rejected"); + } + }; + const schedule = () => { + if (scheduled) return; + scheduled = true; + window.requestAnimationFrame(publish); + }; + new MutationObserver(schedule).observe(document.documentElement, { + attributes: true, + childList: true, + subtree: true, + }); + window.addEventListener("input", schedule, true); + window.addEventListener("change", schedule, true); + window.addEventListener("focusin", schedule); + window.addEventListener("focusout", schedule); + window.addEventListener("resize", schedule); + window.addEventListener("scroll", schedule, true); + schedule(); +} diff --git a/mobile/integration_test/native_review_pairing_test.dart b/mobile/integration_test/native_review_pairing_test.dart new file mode 100644 index 00000000000..26eccd1f6e7 --- /dev/null +++ b/mobile/integration_test/native_review_pairing_test.dart @@ -0,0 +1,39 @@ +import 'package:buzz/features/pairing/pairing_page.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:integration_test/integration_test.dart'; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('pairing code reveal, edit, and hide survives native rendering', ( + tester, + ) async { + await tester.pumpWidget( + const ProviderScope(child: MaterialApp(home: PairingPage())), + ); + await tester.pumpAndSettle(); + + expect(find.text('Welcome to Buzz'), findsOneWidget); + expect(find.byKey(const Key('pairing-code-input')), findsNothing); + + await tester.tap(find.byKey(const Key('pairing-code-toggle'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('pairing-code-input')), findsOneWidget); + await tester.pump(const Duration(seconds: 1)); + + await tester.enterText( + find.byKey(const Key('pairing-code-input')), + 'nostrpair://native-review', + ); + await tester.pump(); + expect(find.text('nostrpair://native-review'), findsOneWidget); + expect(find.byKey(const Key('pairing-connect')), findsOneWidget); + await tester.pump(const Duration(seconds: 1)); + + await tester.tap(find.byKey(const Key('pairing-code-toggle'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('pairing-code-input')), findsNothing); + }); +} diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 05267ed7c13..6745d4f3ba0 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -15,9 +15,12 @@ PODS: - FlutterMacOS - image_picker_ios (0.0.1): - Flutter + - integration_test (0.0.1): + - Flutter - local_auth_darwin (0.0.1): - Flutter - FlutterMacOS + - mobile_scanner (7.0.0): - Flutter - FlutterMacOS @@ -48,7 +51,9 @@ DEPENDENCIES: - Flutter (from `Flutter`) - flutter_secure_storage_darwin (from `.symlinks/plugins/flutter_secure_storage_darwin/darwin`) - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`) + - integration_test (from `.symlinks/plugins/integration_test/ios`) - local_auth_darwin (from `.symlinks/plugins/local_auth_darwin/darwin`) + - mobile_scanner (from `.symlinks/plugins/mobile_scanner/darwin`) - open_filex (from `.symlinks/plugins/open_filex/ios`) - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`) @@ -75,8 +80,11 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/flutter_secure_storage_darwin/darwin" image_picker_ios: :path: ".symlinks/plugins/image_picker_ios/ios" + integration_test: + :path: ".symlinks/plugins/integration_test/ios" local_auth_darwin: :path: ".symlinks/plugins/local_auth_darwin/darwin" + mobile_scanner: :path: ".symlinks/plugins/mobile_scanner/darwin" open_filex: @@ -103,7 +111,9 @@ SPEC CHECKSUMS: Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 flutter_secure_storage_darwin: acdb3f316ed05a3e68f856e0353b133eec373a23 image_picker_ios: e0ece4aa2a75771a7de3fa735d26d90817041326 + integration_test: 4a889634ef21a45d28d50d622cf412dc6d9f586e local_auth_darwin: c3ee6cce0a8d56be34c8ccb66ba31f7f180aaebb + mobile_scanner: 9157936403f5a0644ca3779a38ff8404c5434a93 open_filex: 432f3cd11432da3e39f47fcc0df2b1603854eff1 package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499 @@ -115,4 +125,4 @@ SPEC CHECKSUMS: PODFILE CHECKSUM: bd29822c3d5baf6b44b726f00ea3293a19339ef2 -COCOAPODS: 1.16.2 +COCOAPODS: 1.17.0 diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index e4ee6dbd916..ca17285ee69 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -17,7 +17,11 @@ import UserNotifications _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { - UNUserNotificationCenter.current().requestAuthorization(options: [.badge]) { _, _ in } + // Integration tests cannot interact with SpringBoard's notification prompt. + // Production launches still request badge authorization as before. + if ProcessInfo.processInfo.environment["BUZZ_NATIVE_REVIEW"] != "1" { + UNUserNotificationCenter.current().requestAuthorization(options: [.badge]) { _, _ in } + } return super.application(application, didFinishLaunchingWithOptions: launchOptions) } diff --git a/mobile/lib/app.dart b/mobile/lib/app.dart index f2726b9f13a..3dfcd9e1129 100644 --- a/mobile/lib/app.dart +++ b/mobile/lib/app.dart @@ -94,7 +94,9 @@ class App extends HookConsumerWidget { // cold-start link survives until the authenticated UI can dispatch it. ref.watch(pendingDeepLinkProvider); + const nativeReview = bool.fromEnvironment('BUZZ_NATIVE_REVIEW'); void applyBadge(UnreadBadgeState state) { + if (nativeReview) return; if (state.highPriorityCount > 0) { AppBadgePlus.updateBadge(state.highPriorityCount); } else if (state.generalUnreadCount > 0) { diff --git a/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart b/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart index 49fc185133f..1a48ef4a2d2 100644 --- a/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart +++ b/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart @@ -97,6 +97,7 @@ class _PairingWelcomeView extends StatelessWidget { ), const SizedBox(height: Grid.xxs), TextButton( + key: const Key('pairing-code-toggle'), style: _onboardingSecondaryButtonStyle, onPressed: isBusy ? null : onTogglePairingCode, child: Text( @@ -125,6 +126,7 @@ class _PairingWelcomeView extends StatelessWidget { children: [ const SizedBox(height: Grid.twelve), TextField( + key: const Key('pairing-code-input'), controller: codeController, style: context.textTheme.bodyMedium ?.copyWith(color: _onboardingInk), @@ -168,6 +170,7 @@ class _PairingWelcomeView extends StatelessWidget { SizedBox( width: double.infinity, child: FilledButton( + key: const Key('pairing-connect'), style: _onboardingButtonStyle, onPressed: isBusy ? null : onConnect, child: isBusy diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index cb9bc0268c7..b1afc7fa2a3 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -446,6 +446,11 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_driver: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" flutter_hooks: dependency: "direct main" description: @@ -568,6 +573,11 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.0" + fuchsia_remote_debug_protocol: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" glob: dependency: transitive description: @@ -712,6 +722,11 @@ packages: url: "https://pub.dev" source: hosted version: "0.2.2" + integration_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" intl: dependency: "direct main" description: @@ -1080,6 +1095,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.5.2" + process: + dependency: transitive + description: + name: process + sha256: c6248e4526673988586e8c00bb22a49210c258dc91df5227d5da9748ecf79744 + url: "https://pub.dev" + source: hosted + version: "5.0.5" provider: dependency: transitive description: @@ -1325,6 +1348,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.1" + sync_http: + dependency: transitive + description: + name: sync_http + sha256: "7f0cd72eca000d2e026bcd6f990b81d0ca06022ef4e32fb257b30d3d1014a961" + url: "https://pub.dev" + source: hosted + version: "0.3.1" term_glyph: dependency: transitive description: @@ -1557,6 +1588,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + webdriver: + dependency: transitive + description: + name: webdriver + sha256: "2f3a14ca026957870cfd9c635b83507e0e51d8091568e90129fbf805aba7cade" + url: "https://pub.dev" + source: hosted + version: "3.1.0" webkit_inspection_protocol: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 2cc049c5fea..fc34fb59c33 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -47,6 +47,10 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter + integration_test: + sdk: flutter + flutter_driver: + sdk: flutter flutter_lints: ^6.0.0 crypto: ^3.0.7 custom_lint: ^0.8.0 diff --git a/mobile/test_driver/integration_test.dart b/mobile/test_driver/integration_test.dart new file mode 100644 index 00000000000..b38629cca97 --- /dev/null +++ b/mobile/test_driver/integration_test.dart @@ -0,0 +1,3 @@ +import 'package:integration_test/integration_test_driver.dart'; + +Future main() => integrationDriver(); diff --git a/scripts/setup-desktop-test-data.sh b/scripts/setup-desktop-test-data.sh index ff3ebd055c5..da7671f4805 100755 --- a/scripts/setup-desktop-test-data.sh +++ b/scripts/setup-desktop-test-data.sh @@ -15,6 +15,11 @@ BOB_PUBKEY="bb22a5299220cad76ffd46190ccbeede8ab5dc260faa28b6e5a2cb31b9aff260" CHARLIE_PUBKEY="554cef57437abac34522ac2c9f0490d685b72c80478cf9f7ed6f9570ee8624ea" TYLER_PUBKEY="e5ebc6cdb579be112e336cc319b5989b4bb6af11786ea90dbe52b5f08d741b34" AGENT_PUBKEY="db0b028cd36f4d3e36c8300cce87252c1f7fc9495ffecc53f393fcac341ffd36" +REVIEW_PUBKEY="${BUZZ_REVIEW_PUBKEY:-}" +if [[ -n "$REVIEW_PUBKEY" && ! "$REVIEW_PUBKEY" =~ ^[0-9a-fA-F]{64}$ ]]; then + echo "BUZZ_REVIEW_PUBKEY must be exactly 64 hexadecimal characters." >&2 + exit 1 +fi if command -v psql >/dev/null 2>&1; then run_psql() { PGPASSWORD="$DB_PASS" psql -h"$DB_HOST" -p"$DB_PORT" -U"$DB_USER" -d"$DB_NAME" -qtA "$@"; } @@ -129,4 +134,15 @@ ON CONFLICT DO NOTHING ; " +if [[ -n "$REVIEW_PUBKEY" ]]; then + run_sql " +INSERT INTO channel_members + (community_id, channel_id, pubkey, role, invited_by) +VALUES + ('${COMMUNITY_ID}', '${UUID_GENERAL}', decode('${REVIEW_PUBKEY}','hex'), 'member', decode('${SYSTEM_PUBKEY}','hex')) +ON CONFLICT DO NOTHING +; +" +fi + echo "Desktop e2e data ready." diff --git a/tools/native-review/README.md b/tools/native-review/README.md new file mode 100644 index 00000000000..c8d386d37b3 --- /dev/null +++ b/tools/native-review/README.md @@ -0,0 +1,334 @@ +# Buzz native review harness + +The native review harness exercises the app shapes that browser tests cannot: +real Tauri/WKWebView or Flutter launch, OS input delivery, native rendering, +recording, app lifecycle, and cleanup. It writes reviewable evidence tied to the +source revision that produced it. It complements `just ci`, package tests, and +Playwright; it does not replace them. + +Supported lanes: + +- **macOS Desktop:** declarative journeys driven through Accessibility and + CGEvent, with window-only MP4, screenshots, semantic/AX snapshots, app + CPU/RSS sampling, and performance comparison. +- **iOS Simulator:** Flutter integration journeys on an erased simulator, with + full-device MP4, screenshot, Flutter log, simulator provenance, and cleanup + receipt. + +## Safety contract + +Desktop runs accept only loopback `ws://`/`http://` relays. Every run gets an +ephemeral Nostr key, run-specific dev bundle ID, keyring service, HOME, +WebKit/app-data state, and artifact directory. The launcher environment is +allowlisted before the ephemeral key is added; inherited tokens and production +keys never enter the reviewed process. Production bundle IDs, keyring services, +and remote relays fail closed. + +iOS runs create a uniquely named disposable simulator from the selected device +type and runtime, record its run-owned UDID, and delete only that simulator during +cleanup. Pre-existing simulators are never selected, erased, or deleted. The +Flutter child receives +only an allowlist of host tool settings plus review-only flags; inherited tokens, +production keys, and cloud credentials do not enter the reviewed process. The +review-only environment suppresses the launch notification prompt because +Flutter cannot actuate SpringBoard; normal app launches retain production +permission behavior. + +These controls protect reviewer state from accidents. They are **not** +containment for hostile code. Use a dedicated macOS user, disposable simulator, +or VM for untrusted changes. + +## One-time setup + +1. Activate the repository toolchain and install dependencies: + + ```bash + . ./bin/activate-hermit + just setup + ``` + +2. For Desktop, start the isolated `buzz-harness` relay on port 3030 as directed + by `scripts/start-isolated-test-relay.sh`, then grant **Accessibility** and + **Screen Recording** to the terminal or agent process that launches the run. + Verify both permissions independently: + + ```bash + just native-review-doctor + ``` + +3. For iOS, install Xcode with an iOS Simulator runtime, then list available + devices. The exact name is passed to the runner: + + ```bash + xcrun simctl list devices available + ``` + +## Run existing journeys + +```bash +# Desktop +just native-review-desktop tools/native-review/desktop/tooltip-fresh-dwell.yaml +just native-review-desktop tools/native-review/desktop/composer-keyboard.yaml +just native-review-desktop tools/native-review/desktop/search-shortcut-dismissal.yaml + +# iOS (defaults to iPhone 17 Pro) +just native-review-ios +just native-review-ios 'iPhone 17 Pro' + +# Harness tests +python3 -m unittest discover -s tools/native-review/tests -p 'test_*.py' +swift test --package-path tools/native-review/swift +``` + +Artifacts are written under: + +```text +test-results/native-review/<12-char-sha>/// +``` + +A failed locator, postcondition, Flutter test, recording, evidence capture, or +cleanup returns nonzero and leaves a partial failed receipt. A useful report +includes the exact receipt path, full HEAD SHA, dirty status, and whether cleanup +passed. Never describe a dirty receipt as clean-SHA proof. + +## Author a Desktop journey + +Copy the nearest checked-in journey rather than starting from an empty file. +`desktop/composer-keyboard.yaml` demonstrates fallback locators, text entry, +scrolling, focus/value assertions, and cleanup. The schema is +`schemas/journey.schema.json`. + +Every journey declares: + +- `flow`: stable artifact/budget identity using lowercase letters, numbers, + `_`, or `-`; +- `fixture: local_review_channel`: the isolated seeded state; +- recording policy; +- ordered steps; +- explicit termination and state removal. + +Every step must contain an action and an observed postcondition. A step may also +provide ordered fallback locators, a timeout, a sustained assertion, or one +named measurement: + +```yaml +- name: open_search + act: {type: press, key: k, modifiers: [command]} + expect: {exists: {id: search-dialog}} + timeout_ms: 5000 + measure: search_open_latency +``` + +### Locators + +Prefer a stable semantic `id`. Add a role/name fallback when the native +Accessibility tree exposes one: + +```yaml +locate: + - {id: message-input} + - {role: text-area, name: Message} +``` + +Do not use screen coordinates, styling classes, translated prose when a stable +identifier is available, or a broad role such as `button` without a name. If a +production control lacks a stable identity, add a narrowly named semantic/test +identifier to that control. Do not add review-only behavior merely to make a +selector pass. + +### Actions and assertions + +Supported actions are `activate`, `click`, `move_pointer`, `press`, `scroll`, +`type_text`, and `wait`. Keyboard modifiers are `command`, `control`, `option`, +and `shift`. + +Supported assertions cover element existence/nonexistence, focus, enabled state, +text value, and scroll bounds. `expect_for` requires an assertion to remain true +for a duration; use it for dwell or stability behavior rather than a blind +sleep. Keep waits short and use a postcondition that proves the user-visible +state actually changed. + +### Required mutation proof + +A new regression journey is not proven by a green run alone. Deliberately +reintroduce the guarded defect or mutate its critical locator/postcondition and +show that the journey fails for the intended reason. Then restore the source and +show it passes. Existing examples live in `tests/fixtures/broken-*.yaml`. + +Record both outcomes: + +- green receipt with visible artifact and passing cleanup; +- failed receipt with causal diagnostic, preserved screenshot/video/semantic + evidence, and passing cleanup. + +## Author an iOS journey + +iOS journeys are Flutter integration tests under `mobile/integration_test/`. +Use `native_review_pairing_test.dart` as the template and +`test_driver/integration_test.dart` as the shared host driver. + +Guidelines: + +1. Exercise a real app surface with `IntegrationTestWidgetsFlutterBinding`. +2. Select controls by stable `Key`, not coordinates or incidental text. +3. Assert state before and after each actuation. +4. Use `pumpAndSettle()` for animations and bounded `pump(Duration(...))` only + when the state needs to remain visible in the recording. +5. Keep secrets and production relay state out of fixtures. +6. Pass a different test without changing the runner: + + ```bash + ./tools/native-review/bin/review-ios \ + --device 'iPhone 17 Pro' \ + --test mobile/integration_test/your_journey_test.dart + ``` + +The runner erases, boots, and waits for the selected simulator; records before +launch; invokes `flutter drive --keep-app-running`; captures the final app +screen; finalizes the recorder; then shuts down and erases the device even after +failure. Add runner behavior through unit-tested helpers rather than shelling +around this lifecycle. + +Mutation-test an iOS journey by temporarily removing/changing the critical +widget key or expected state. Confirm nonzero exit, exact Flutter diagnostic, +finalized MP4, screenshot/log receipt entries, and passing cleanup. Restore the +source before committing. + +## Performance comparison and budgets + +A Desktop step with `measure: ` persists its complete native +action-to-observed-postcondition duration. During the journey the harness also +samples app CPU and resident memory every 100 ms. Capture a cohort rather than +trusting one noisy laptop run: + +```bash +just native-review-benchmark tools/native-review/desktop/tooltip-fresh-dwell.yaml 5 +``` + +Compare at least three clean baseline receipts with at least three clean +candidate receipts using a checked-in budget policy: + +```bash +./tools/native-review/bin/review-native compare \ + --baseline /path/base-1/receipt.json \ + --baseline /path/base-2/receipt.json \ + --baseline /path/base-3/receipt.json \ + --candidate /path/head-1/receipt.json \ + --candidate /path/head-2/receipt.json \ + --candidate /path/head-3/receipt.json \ + --budget tools/native-review/performance/tooltip-fresh-dwell.yaml \ + --output test-results/native-review/performance-comparison.json +``` + +A budget's `flow` must match the journey. Each metric may set an absolute `max`, +a `max_regression_percent`, or both. Establish ceilings from repeated known-good +runs on representative hardware; do not choose a ceiling merely because the +current candidate slips under it. Name interaction measurements for the user +transition they represent, such as `search_open_latency`. + +Comparison uses cohort medians and preserves raw samples/min/max. It fails closed +for dirty runs, failed cleanup, mixed source revisions, wrong flows, missing +metrics, too few samples, incompatible machine/OS fingerprints, or breached +budgets. Run baseline and candidate on the same host under comparable load. +Recording overhead is intentionally present in both cohorts. + +CPU/RSS sampling measures the app process during the foreground journey. It does +**not** by itself prove background-idle CPU, relay RPS, reconnect storms, +serialization volume, relay saturation, or cross-client scaling. Those require +seeded workload journeys plus the relevant counters/traces before a budget can +guard them. + +## Before opening a PR + +- Run the full package/repository gates required by `AGENTS.md` and `TESTING.md`. +- Run the native-review Python suite. +- Mutation-prove each new regression guard. +- Commit, confirm a clean tree, and run the native journey at that exact HEAD at + least twice for lifecycle-sensitive work. +- Inspect the MP4 and screenshot; a passing log attached to SpringBoard or the + wrong app is not visual evidence. +- Verify receipt SHA, `dirty: false`, selected device/runtime, artifact files, + and cleanup status. +- After merging `origin/main`, rerun any proof whose HEAD changed. + +## Troubleshooting + +- **Doctor denies Accessibility or Screen Recording:** grant the permission to + the actual invoking terminal/agent binary, restart it, and rerun doctor. +- **Desktop relay refused:** use the isolated loopback relay on port 3030. Remote + relays intentionally fail closed. +- **No iOS device with that name:** inspect `xcrun simctl list devices available` + and pass an exact available name. When several runtimes contain that model, + the runner selects the newest runtime and records it. +- **Notification prompt covers iOS:** run through `review-ios`; it provides both + the Dart define and simulator child environment. Direct `flutter drive` does + not provide the complete review contract. +- **Receipt passed but evidence looks wrong:** treat it as a harness defect. The + recording and screenshot are part of the assertion surface, not decoration. +- **Push hooks cannot find Flutter/Rust:** activate Hermit and ensure repository + `bin` remains on `PATH`, for example `PATH="$PWD/bin:$PATH" git push ...`. + +## Share a finding + +Every recorded Desktop and iOS run now retains the original recording and also +finalizes `video-share.mp4`: H.264/yuv420p, fast-start, metadata/chapter/audio +stripped, and bounded to a 2160px longest edge. This is the canonical artifact +for Buzz or GitHub attachment; the original remains the source recording. +Finalization is part of cleanup, so a missing/broken share artifact makes the +receipt fail rather than silently claiming complete evidence. + +To produce a small review bundle from any receipt (passed or failed): + +```bash +just native-review-finding \ + test-results/native-review////receipt.json \ + /tmp/finding \ + 'disposed image|framework exception' +``` + +The output directory is created once and never overwritten. It contains: + +- `finding.mp4` — relay-compatible, metadata-free video; +- `receipt.json` — a minimal copy containing status, source SHA/dirty state, + device/runtime, measurements, and cleanup (no isolation paths or device UDID); +- `log-excerpt.txt` — only matching lines plus bounded context (or the final 200 + lines without `MATCH`), with credential-shaped assignments redacted; +- `manifest.json` — source run id plus SHA-256 and byte size for every file. + +Inspect the clip and excerpt before publishing. Redaction is defense in depth, +not permission to run untrusted code with credentials; the harness isolation +rules still apply. `buzz upload file --file /tmp/finding/finding.mp4` is the +live smoke check for Buzz delivery. PR screenshots still use +`scripts/post-screenshots.sh` as required by `AGENTS.md`; GitHub video should be +attached through GitHub's supported upload UI/API rather than a relay URL. + + +## Publish the review report to Buzz + +A review that exercised Desktop or iOS natively is not complete until its report +and canonical recording are published together in the originating Buzz thread. +Use the receipt-bound publisher rather than a separate `buzz upload`: it rejects +dirty receipts, failed cleanup, missing share video, invalid relay responses, and +highlights outside the recording. + +```bash +just native-review-publish \ + test-results/native-review////receipt.json \ + /tmp/review-summary.md \ + /tmp/highlights.json +``` + +`highlights.json` is a JSON array of `{"seconds": number, "text": string}`. +The command first posts the review summary with `video-share.mp4`, then publishes +each highlight as a direct reply to that video message using Buzz's leading +`[MM:SS.mmm]` video-review timecode syntax. Those chips seek to the highlighted +frame in Desktop. Every material visual finding needs at least one highlight; a +no-finding run may omit the file, but the report must briefly index the journey +that was exercised. The callback mention belongs on the video/report root, not +on every highlight—notifications are not confetti. + +Publication is intentionally fail-closed and non-transactional. If a highlight +send fails after the video root was accepted, the command exits nonzero and +prints the relay error; rerun only after checking the thread to avoid duplicating +the already-published root. Future CLI support for idempotent event publication +can remove that sharp edge. diff --git a/tools/native-review/bin/review-ios b/tools/native-review/bin/review-ios new file mode 100755 index 00000000000..62b01c161a1 --- /dev/null +++ b/tools/native-review/bin/review-ios @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +TOOL_ROOT="$ROOT/tools/native-review" +. "$ROOT/bin/activate-hermit" +exec uv run --with 'pyyaml==6.0.3' python "$TOOL_ROOT/ios_review.py" "$@" diff --git a/tools/native-review/bin/review-native b/tools/native-review/bin/review-native new file mode 100755 index 00000000000..7091e9e8238 --- /dev/null +++ b/tools/native-review/bin/review-native @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# The repository pins Rust/Node tooling through Hermit; native review provisions +# its sole Python dependency explicitly for clean environments. +source "$REPO_ROOT/bin/activate-hermit" +exec uv run --with 'pyyaml==6.0.3' python "$SCRIPT_DIR/review_native.py" "$@" diff --git a/tools/native-review/desktop/composer-keyboard.yaml b/tools/native-review/desktop/composer-keyboard.yaml new file mode 100644 index 00000000000..9f2d6b7be8d --- /dev/null +++ b/tools/native-review/desktop/composer-keyboard.yaml @@ -0,0 +1,49 @@ +schema_version: 1 +flow: composer_keyboard +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: true + accessibility: true +steps: + - name: activate_buzz + act: {type: activate} + expect: {exists: {role: window}} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: {not_exists: {id: app-loading-gate}} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + - {role: button, name: welcome-everyone} + act: {type: click} + expect: {exists: {id: message-composer}} + timeout_ms: 15000 + - name: focus_composer + locate: + - {id: message-input} + - {role: text-area, name: Message} + act: {type: click} + expect: {focused: {id: message-input}} + - name: type_draft + act: {type: type_text, text: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} + expect: {value: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} + - name: keep_draft_while_scrolling + locate: + - {id: message-input-scroll} + act: {type: scroll, delta_y: 240} + expect: {scroll_y_less_than: 1} + - name: draft_survives_scroll + locate: + - {id: message-input} + act: {type: wait, duration_ms: 50} + expect: {value: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} + - name: dismiss_focus + act: {type: press, key: escape} + expect: {value: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/desktop/search-shortcut-dismissal.yaml b/tools/native-review/desktop/search-shortcut-dismissal.yaml new file mode 100644 index 00000000000..0a2876a88bf --- /dev/null +++ b/tools/native-review/desktop/search-shortcut-dismissal.yaml @@ -0,0 +1,39 @@ +schema_version: 1 +flow: search_shortcut_dismissal +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: true + accessibility: true +steps: + - name: activate_buzz + act: {type: activate} + expect: {exists: {role: window}} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: {not_exists: {id: app-loading-gate}} + timeout_ms: 60000 + - name: open_search_with_keyboard + act: {type: press, key: k, modifiers: [command]} + expect: {exists: {id: search-results}} + timeout_ms: 5000 + - name: search_input_receives_focus + locate: + - {id: search-dialog-input} + - {role: text-field, name: "Search everything"} + act: {type: type_text, text: "welcome"} + expect: {value: "welcome"} + - name: dismiss_search + act: {type: press, key: escape} + expect: {not_exists: {id: search-results}} + timeout_ms: 2000 + - name: focus_returns_to_trigger + locate: + - {id: open-search} + act: {type: wait, duration_ms: 50} + expect: {focused: {id: open-search}} +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/desktop/tooltip-fresh-dwell.yaml b/tools/native-review/desktop/tooltip-fresh-dwell.yaml new file mode 100644 index 00000000000..4b7def5b04b --- /dev/null +++ b/tools/native-review/desktop/tooltip-fresh-dwell.yaml @@ -0,0 +1,61 @@ +schema_version: 1 +flow: tooltip_fresh_dwell +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: false + accessibility: false +steps: + - name: activate_buzz + act: {type: activate} + expect: + exists: {role: window} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: + not_exists: {id: app-loading-gate} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + - {role: button, name: welcome-everyone} + act: {type: click} + expect: + exists: {id: message-insert-mention} + timeout_ms: 15000 + - name: settle_pointer_before_fresh_dwell + locate: + - {id: message-composer} + - {role: text-area, name: Message} + act: {type: move_pointer, duration_ms: 80} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + - name: clear_tooltip_skip_delay + act: {type: wait, duration_ms: 800} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + - name: transit_over_mention + locate: + - {id: message-insert-mention} + - {role: button, name: "Mention someone"} + act: {type: move_pointer} + expect: {enabled: true} + - name: complete_dwell + act: {type: wait, duration_ms: 1} + expect: + exists: {role: tooltip, name: "Mention someone"} + timeout_ms: 1000 + measure: tooltip_open_latency + - name: leave_trigger + locate: + - {id: message-composer} + - {role: text-area, name: Message} + act: {type: move_pointer, duration_ms: 80} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + timeout_ms: 1000 +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/evidence_bundle.py b/tools/native-review/evidence_bundle.py new file mode 100644 index 00000000000..3767347a33d --- /dev/null +++ b/tools/native-review/evidence_bundle.py @@ -0,0 +1,180 @@ +"""Build privacy-safe, relay-compatible evidence from a native-review receipt.""" +from __future__ import annotations + +import hashlib +import json +import math +import pathlib +import re +import shutil +import subprocess +from typing import Any + +MAX_VIDEO_EDGE = 2160 +SECRET_KEY = re.compile(r"(?i)(?:auth(?:orization)?|token|secret|password|private[_-]?key|cookie|api[_-]?key)") +SECRET_HEADER = re.compile( + r"(?im)(?P^[^\r\n]*?\b(?:authorization|proxy[-_]authorization)\s*[:=]\s*)" + r"[^\r\n]*(?:\r?\n[ \t]+[^\r\n]*)*" +) +SECRET_VALUE = re.compile( + r"(?i)(?P[\"']?(?:auth|token|secret|password|private[_-]?key|cookie|api[_-]?key)[A-Z0-9_.-]*[\"']?)" + r"(?P\s*[:=]\s*)" + r"(?P\"(?:\\.|[^\"\\\r\n])*\"|'(?:\\.|[^'\\\r\n])*'|[^\r\n]*)(?:\r?\n[ \t]+[^\r\n]*)*" +) + + +class EvidenceError(RuntimeError): + pass + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def relay_safe_video(source: pathlib.Path, destination: pathlib.Path, *, start: float | None = None, + duration: float | None = None) -> pathlib.Path: + """Transcode a recording to Buzz's canonical, metadata-free MP4 profile.""" + if start is not None and (isinstance(start, bool) or not math.isfinite(start) or start < 0): + raise EvidenceError("clip start must be finite and non-negative") + if duration is not None and (isinstance(duration, bool) or not math.isfinite(duration) or duration <= 0): + raise EvidenceError("clip duration must be finite and positive") + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + raise EvidenceError("ffmpeg is required to finalize shareable evidence") + if not source.is_file(): + raise EvidenceError(f"recording does not exist: {source}") + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_suffix(".tmp.mp4") + command = [ffmpeg, "-y"] + if start is not None: + command.extend(["-ss", str(start)]) + command.extend(["-i", str(source)]) + if duration is not None: + command.extend(["-t", str(duration)]) + command.extend([ + "-map_metadata", "-1", "-map_chapters", "-1", "-an", + "-vf", f"scale='min({MAX_VIDEO_EDGE},iw)':'min({MAX_VIDEO_EDGE},ih)':" + "force_original_aspect_ratio=decrease:force_divisible_by=2", + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-preset", "fast", "-crf", "25", + "-movflags", "+faststart", "-fflags", "+bitexact", "-flags:v", "+bitexact", + str(temporary), + ]) + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode: + temporary.unlink(missing_ok=True) + diagnostic = result.stderr.strip().splitlines()[-1] if result.stderr.strip() else "unknown ffmpeg error" + raise EvidenceError(f"video finalization failed: {diagnostic}") + temporary.replace(destination) + return destination + + +def redact_log(text: str) -> str: + def replacement(match: re.Match[str]) -> str: + value = match.group("value") + quote = value[0] if value and value[0] in {"\"", "'"} else "" + return f"{match.group('name')}{match.group('sep')}{quote}[REDACTED]{quote}" + text = SECRET_HEADER.sub(lambda match: f"{match.group('prefix')}[REDACTED]", text) + return SECRET_VALUE.sub(replacement, text) + + +def redact_value(value: Any) -> Any: + """Recursively redact credential-shaped keys and strings copied into bundles.""" + if isinstance(value, dict): + return { + key: "[REDACTED]" if isinstance(key, str) and SECRET_KEY.search(key) else redact_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [redact_value(item) for item in value] + if isinstance(value, str): + return redact_log(value) + return value + + +def focused_log(text: str, match: str | None, context: int) -> str: + lines = text.splitlines() + if match is None: + selected = lines[-200:] + else: + try: + pattern = re.compile(match, re.IGNORECASE) + except re.error as exc: + raise EvidenceError(f"invalid log match expression: {exc}") from exc + indexes = [index for index, line in enumerate(lines) if pattern.search(line)] + if not indexes: + raise EvidenceError(f"log match found no lines: {match}") + included = { + line_index + for index in indexes + for line_index in range(max(0, index - context), min(len(lines), index + context + 1)) + } + selected = [line for index, line in enumerate(lines) if index in included] + return redact_log("\n".join(selected) + ("\n" if selected else "")) + + +def finding_bundle(receipt_path: pathlib.Path, output: pathlib.Path, *, match: str | None = None, + context: int = 8, start: float | None = None, duration: float | None = None) -> dict[str, Any]: + if output.exists(): + raise EvidenceError(f"output already exists: {output}") + if context < 0: + raise EvidenceError("log context cannot be negative") + try: + receipt = json.loads(receipt_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise EvidenceError(f"cannot read receipt {receipt_path}: {exc}") from exc + if not isinstance(receipt, dict) or not isinstance(receipt.get("artifacts"), dict): + raise EvidenceError("receipt has no artifact manifest") + run_dir = receipt_path.parent + video_name = receipt["artifacts"].get("video") + log_name = receipt["artifacts"].get("log") + if not isinstance(video_name, str) or not isinstance(log_name, str): + raise EvidenceError("finding bundles require receipt video and log artifacts") + def artifact(name: str) -> pathlib.Path: + candidate = (run_dir / name).resolve() + if run_dir.resolve() not in candidate.parents: + raise EvidenceError(f"artifact escapes receipt directory: {name}") + return candidate + + output.mkdir(parents=True) + try: + relay_safe_video(artifact(video_name), output / "finding.mp4", start=start, duration=duration) + log_text = artifact(log_name).read_text(errors="replace") + (output / "log-excerpt.txt").write_text(focused_log(log_text, match, context)) + provenance = receipt.get("provenance", {}) + device = receipt.get("isolation", {}).get("simulator", receipt.get("device", {})) + receipt_copy = redact_value({ + "schema_version": receipt.get("schema_version"), + "run_id": receipt.get("run_id"), + "flow": receipt.get("flow"), + "status": receipt.get("status"), + "failure": receipt.get("failure"), + "provenance": {key: provenance.get(key) for key in ("head_sha", "dirty", "artifact_sha256") if key in provenance}, + "device": {key: device.get(key) for key in ("name", "runtime") if key in device}, + "measurements": receipt.get("measurements"), + "performance": receipt.get("performance"), + "cleanup": receipt.get("cleanup"), + }) + (output / "receipt.json").write_text(json.dumps(receipt_copy, indent=2) + "\n") + manifest = { + "schema_version": 1, + "source_receipt": f"{run_dir.name}/receipt.json", + "head_sha": receipt.get("provenance", {}).get("head_sha"), + "dirty": receipt.get("provenance", {}).get("dirty"), + "status": receipt.get("status"), + "cleanup": receipt.get("cleanup", {}).get("status"), + "log_match": match, + "clip": {"start_seconds": start, "duration_seconds": duration}, + "files": { + name: {"sha256": sha256(output / name), "size": (output / name).stat().st_size} + for name in ("finding.mp4", "receipt.json", "log-excerpt.txt") + }, + } + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + except Exception: + shutil.rmtree(output, ignore_errors=True) + raise diff --git a/tools/native-review/ios_review.py b/tools/native-review/ios_review.py new file mode 100755 index 00000000000..227a3f26cba --- /dev/null +++ b/tools/native-review/ios_review.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Run a Flutter integration journey on an isolated iOS Simulator with video evidence.""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import pathlib +import re +import secrets +import selectors +import shutil +import signal +import subprocess +import sys + +_TOOL_ROOT = pathlib.Path(__file__).resolve().parent +if str(_TOOL_ROOT) not in sys.path: + sys.path.insert(0, str(_TOOL_ROOT)) +import time +from typing import Any + +from evidence_bundle import EvidenceError, relay_safe_video + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DEFAULT_TEST = ROOT / "mobile/integration_test/native_review_pairing_test.dart" +SUBPROCESS_ENV_ALLOWLIST = { + "PATH", "HOME", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", + "TERM", "__CF_USER_TEXT_ENCODING", "DEVELOPER_DIR", +} + + +class ReviewError(RuntimeError): + pass + + +def run(command: list[str], *, cwd: pathlib.Path = ROOT, check: bool = True, capture: bool = True, + env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, check=check, text=True, + env=subprocess_environment() if env is None else env, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None) + + +def git(*args: str) -> str: + return run(["git", *args]).stdout.strip() + + +def create_review_device(device_type_name: str, run_id: str) -> dict[str, Any]: + """Create a uniquely named simulator owned by this run; never reuse user state.""" + payload = json.loads(run(["xcrun", "simctl", "list", "devicetypes", "runtimes", "-j"]).stdout) + device_types = [item for item in payload.get("devicetypes", []) if item.get("name") == device_type_name] + if not device_types: + raise ReviewError(f"no iOS Simulator device type named {device_type_name}") + device_type = device_types[0] + runtimes = [ + item for item in payload.get("runtimes", []) + if item.get("isAvailable") and item.get("platform") == "iOS" + and any(supported.get("identifier") == device_type["identifier"] + for supported in item.get("supportedDeviceTypes", [])) + ] + if not runtimes: + raise ReviewError(f"no available iOS Simulator runtime supports {device_type_name}") + runtime = max(runtimes, key=lambda item: tuple(int(part) for part in item["version"].split("."))) + owned_name = f"Buzz Native Review {run_id}" + udid = run(["xcrun", "simctl", "create", owned_name, + device_type["identifier"], runtime["identifier"]]).stdout.strip() + if not re.fullmatch(r"[0-9A-Fa-f-]{36}", udid): + raise ReviewError("simctl create returned an invalid device identifier") + return {"name": owned_name, "udid": udid, "runtimeIdentifier": runtime["identifier"], + "deviceType": device_type_name, "owned": True} + + +def subprocess_environment() -> dict[str, str]: + return {key: value for key, value in os.environ.items() if key in SUBPROCESS_ENV_ALLOWLIST} + + +def flutter_environment() -> dict[str, str]: + env = subprocess_environment() + env.update({"BUZZ_NATIVE_REVIEW": "1", "SIMCTL_CHILD_BUZZ_NATIVE_REVIEW": "1"}) + return env + + +def provenance() -> dict[str, Any]: + status = git("status", "--porcelain=v1", "--untracked-files=all") + return {"head_sha": git("rev-parse", "HEAD"), "dirty": bool(status), "status": status.splitlines()} + + +def wait_for_recording(recorder: subprocess.Popen[str], timeout_seconds: float = 15) -> None: + if recorder.stderr is None: + raise ReviewError("simulator recorder has no diagnostic stream") + selector = selectors.DefaultSelector() + selector.register(recorder.stderr, selectors.EVENT_READ) + deadline = time.monotonic() + timeout_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ReviewError("timed out waiting for Simulator recording") + if not selector.select(remaining): + raise ReviewError("timed out waiting for Simulator recording") + line = recorder.stderr.readline() + if "Recording started" in line: + return + if recorder.poll() is not None: + raise ReviewError(f"simulator recorder exited: {line.strip()}") + finally: + selector.close() + + +def run_review(test: pathlib.Path, device_name: str, output_root: pathlib.Path) -> pathlib.Path: + if sys.platform != "darwin" or not shutil.which("xcrun") or not shutil.which("flutter"): + raise ReviewError("iOS native review requires macOS, Xcode simctl, and Flutter") + if not test.is_file() or ROOT not in test.resolve().parents: + raise ReviewError(f"test must be a repository integration test: {test}") + prov = provenance() + started = dt.datetime.now(dt.timezone.utc).isoformat() + run_id = f"ios-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{secrets.token_hex(3)}" + flow = f"ios_{test.stem}" + run_dir = output_root / git("rev-parse", "--short=12", "HEAD") / flow / run_id + run_dir.mkdir(parents=True) + receipt: dict[str, Any] = { + "schema_version": 1, "run_id": run_id, "flow": flow, "status": "failed", + "started_at": started, "finished_at": started, "failure": None, + "provenance": prov, + "isolation": {"kind": "disposable_ios_simulator", "owned": True}, + "artifacts": {}, "steps": [], "measurements": {}, + "performance": {"machine": { + "system": sys.platform, "release": "ios-simulator", + "machine": os.uname().machine, "cpu": os.uname().machine, + }}, + "cleanup": {"status": "not_started"}, + } + device: dict[str, Any] | None = None + udid: str | None = None + recorder: subprocess.Popen[str] | None = None + try: + device = create_review_device(device_name, run_id) + udid = device["udid"] + receipt["isolation"]["simulator"] = { + "name": device["name"], "device_type": device_name, "udid": udid, + "runtime": device["runtimeIdentifier"], "owned": True, + } + run(["xcrun", "simctl", "boot", udid]) + run(["xcrun", "simctl", "bootstatus", udid, "-b"], capture=False) + video = run_dir / "video.mp4" + recorder = subprocess.Popen(["xcrun", "simctl", "io", udid, "recordVideo", "--codec=h264", "--force", str(video)], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + env=subprocess_environment(), text=True) + wait_for_recording(recorder) + receipt["artifacts"]["video"] = "video.mp4" + result = run(["flutter", "drive", "--driver", "test_driver/integration_test.dart", + "--target", str(test.relative_to(ROOT / "mobile")), "-d", udid, + "--keep-app-running", "--dart-define=BUZZ_NATIVE_REVIEW=true"], + cwd=ROOT / "mobile", check=False, env=flutter_environment()) + (run_dir / "flutter.log").write_text(result.stdout + result.stderr) + receipt["artifacts"]["log"] = "flutter.log" + screenshot = run_dir / "final.png" + run(["xcrun", "simctl", "io", udid, "screenshot", str(screenshot)]) + receipt["artifacts"]["screenshot"] = "final.png" + time.sleep(0.5) + if result.returncode: + raise ReviewError(f"Flutter integration journey failed with exit {result.returncode}") + receipt["status"] = "passed" + except Exception as exc: + receipt["failure"] = str(exc) + finally: + errors = [] + if recorder and recorder.poll() is None: + recorder.send_signal(signal.SIGINT) + try: + recorder.wait(timeout=30) + except subprocess.TimeoutExpired: + recorder.kill(); errors.append("recorder required SIGKILL") + if udid: + try: + run(["xcrun", "simctl", "shutdown", udid], check=False) + except Exception as exc: + errors.append(str(exc)) + try: + run(["xcrun", "simctl", "delete", udid]) + except Exception as exc: + errors.append(str(exc)) + video = run_dir / "video.mp4" + if video.is_file(): + try: + relay_safe_video(video, run_dir / "video-share.mp4") + receipt["artifacts"]["share_video"] = "video-share.mp4" + except EvidenceError as exc: + errors.append(f"shareable video finalization failed: {exc}") + receipt["cleanup"] = {"status": "failed" if errors else "passed", "errors": errors} + if errors: + receipt["status"] = "failed" + receipt["finished_at"] = dt.datetime.now(dt.timezone.utc).isoformat() + (run_dir / "receipt.json").write_text(json.dumps(receipt, indent=2, allow_nan=False) + "\n") + print(run_dir) + if receipt["status"] != "passed": + raise ReviewError(receipt["failure"] or "iOS journey or cleanup failed") + return run_dir + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--test", type=pathlib.Path, default=DEFAULT_TEST) + parser.add_argument("--device", default="iPhone 17 Pro") + parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + args = parser.parse_args() + try: + run_review(args.test.resolve(), args.device, args.output.resolve()) + return 0 + except ReviewError as exc: + print(f"ios-native-review: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/native-review/performance/tooltip-fresh-dwell.yaml b/tools/native-review/performance/tooltip-fresh-dwell.yaml new file mode 100644 index 00000000000..4617fc7c3f7 --- /dev/null +++ b/tools/native-review/performance/tooltip-fresh-dwell.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: tooltip_fresh_dwell +minimum_samples: 3 +metrics: + tooltip_open_latency: + max: 1000 + max_regression_percent: 20 + process.cpu_percent_median: + max: 400 + max_regression_percent: 30 + process.resident_mb_peak: + max: 500 + max_regression_percent: 20 diff --git a/tools/native-review/review_native.py b/tools/native-review/review_native.py new file mode 100755 index 00000000000..75e87d34ee7 --- /dev/null +++ b/tools/native-review/review_native.py @@ -0,0 +1,929 @@ +#!/usr/bin/env python3 +"""Exact-SHA-bound native review orchestrator for Buzz Desktop.""" +from __future__ import annotations + +import argparse +import dataclasses +import datetime as dt +import hashlib +import http.server +import json +import math +import os +import pathlib +import platform +import re +import secrets +import select +import shutil +import statistics +import subprocess +import sys + +_TOOL_ROOT = pathlib.Path(__file__).resolve().parent +if str(_TOOL_ROOT) not in sys.path: + sys.path.insert(0, str(_TOOL_ROOT)) +import tempfile +import threading +import time +import urllib.parse +from typing import Any + +from evidence_bundle import EvidenceError, finding_bundle, relay_safe_video +import review_publish +from review_publish import PublishError, publish_review + +try: + import yaml +except ImportError as exc: # pragma: no cover - environment preflight + raise SystemExit("PyYAML is required (activate the repository Hermit environment)") from exc + +ROOT = pathlib.Path(os.environ.get("BUZZ_NATIVE_REVIEW_ROOT", pathlib.Path(__file__).resolve().parents[2])).resolve() +TOOL_ROOT = pathlib.Path(__file__).resolve().parent +PRODUCTION_BUNDLE_IDS = {"xyz.block.buzz.app", "xyz.block.sprout.app"} +PRODUCTION_KEYRINGS = {"buzz-desktop", "sprout-desktop"} +SECRET_NAME = re.compile(r"(AUTH|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|COOKIE)", re.I) +ALLOWED_TOP = {"schema_version", "flow", "platforms", "fixture", "record", "steps", "cleanup"} +ALLOWED_STEP = {"name", "locate", "act", "expect", "expect_for", "timeout_ms", "measure"} +PERFORMANCE_SAMPLE_MINIMUM = 3 + + +def is_finite_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def is_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +class HarnessError(RuntimeError): + pass + + +def run(command: list[str], *, cwd: pathlib.Path = ROOT, env: dict[str, str] | None = None, + check: bool = True, capture: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, + env=scrubbed_environment(include_home=True) if env is None else env, + check=check, text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None) + + +def git(*args: str) -> str: + return run(["git", *args]).stdout.strip() + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat() + + +def machine_fingerprint() -> dict[str, str]: + """Return comparison-critical host attributes without user-specific data.""" + cpu = run(["sysctl", "-n", "machdep.cpu.brand_string"], check=False).stdout.strip() + return { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "cpu": cpu or platform.processor() or "unknown", + } + + +class ProcessSampler: + """Sample app CPU and resident memory while a native journey is active.""" + + def __init__(self, pid: int, interval_seconds: float = 0.1): + self.pid = pid + self.interval_seconds = interval_seconds + self.samples: list[dict[str, float]] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._sample, daemon=True) + + def start(self) -> None: + self._thread.start() + + def _sample(self) -> None: + while not self._stop.is_set(): + result = run(["ps", "-p", str(self.pid), "-o", "%cpu=", "-o", "rss="], check=False) + fields = result.stdout.split() + if len(fields) == 2: + try: + self.samples.append({ + "elapsed_ms": time.monotonic() * 1000, + "cpu_percent": float(fields[0]), + "resident_mb": int(fields[1]) / 1024, + }) + except ValueError: + pass + self._stop.wait(self.interval_seconds) + + def finish(self) -> dict[str, Any]: + self._stop.set() + self._thread.join(timeout=2) + if not self.samples: + return {"sample_count": 0} + return { + "sample_count": len(self.samples), + "interval_ms": self.interval_seconds * 1000, + "cpu_percent_median": statistics.median(item["cpu_percent"] for item in self.samples), + "cpu_percent_peak": max(item["cpu_percent"] for item in self.samples), + "resident_mb_median": statistics.median(item["resident_mb"] for item in self.samples), + "resident_mb_peak": max(item["resident_mb"] for item in self.samples), + } + + +def validate_locator(locator: Any, where: str) -> None: + if not isinstance(locator, dict) or not locator or set(locator) - {"id", "role", "name"}: + raise HarnessError(f"{where}: locator must contain only id, role, and/or name") + if not all(isinstance(value, str) and value for value in locator.values()): + raise HarnessError(f"{where}: locator values must be non-empty strings") + + +def validate_expectation(expectation: Any, where: str) -> None: + allowed = {"exists", "not_exists", "focused", "enabled", "value", "scroll_y_greater_than", "scroll_y_less_than"} + if ( + not isinstance(expectation, dict) + or len(expectation) != 1 + or set(expectation) - allowed + ): + raise HarnessError(f"{where}: expectation must contain exactly one supported condition") + for key in ("exists", "not_exists"): + if key in expectation: + validate_locator(expectation[key], f"{where}.{key}") + focused = expectation.get("focused") + if focused is not None and not isinstance(focused, bool): + validate_locator(focused, f"{where}.focused") + if "enabled" in expectation and not isinstance(expectation["enabled"], bool): + raise HarnessError(f"{where}.enabled must be boolean") + if "value" in expectation and not isinstance(expectation["value"], str): + raise HarnessError(f"{where}.value must be a string") + for key in ("scroll_y_greater_than", "scroll_y_less_than"): + if key in expectation and not is_finite_number(expectation[key]): + raise HarnessError(f"{where}.{key} must be a finite number") + + +def load_journey(path: pathlib.Path) -> dict[str, Any]: + try: + journey = yaml.safe_load(path.read_text()) + except (OSError, yaml.YAMLError) as exc: + raise HarnessError(f"cannot read journey {path}: {exc}") from exc + if not isinstance(journey, dict) or set(journey) != ALLOWED_TOP: + raise HarnessError(f"journey must contain exactly {sorted(ALLOWED_TOP)}") + if journey["schema_version"] != 1 or journey["platforms"] != ["macos"]: + raise HarnessError("only schema_version 1 and platforms: [macos] are supported") + if journey["fixture"] != "local_review_channel": + raise HarnessError("desktop MVP permits only fixture: local_review_channel") + if not isinstance(journey["flow"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", journey["flow"]): + raise HarnessError("flow must be a lowercase filesystem-safe identifier") + record = journey["record"] + if not isinstance(record, dict) or set(record) != {"video", "screenshots", "accessibility"}: + raise HarnessError("record requires exactly video, screenshots, accessibility") + if record["video"] not in ("window", "off") or not all(isinstance(record[k], bool) for k in ("screenshots", "accessibility")): + raise HarnessError("invalid record policy") + steps = journey["steps"] + if not isinstance(steps, list) or not steps: + raise HarnessError("journey requires at least one step") + measurement_names: set[str] = set() + for index, step in enumerate(steps): + where = f"steps[{index}]" + if not isinstance(step, dict) or set(step) - ALLOWED_STEP or not {"name", "act", "expect"} <= set(step): + raise HarnessError(f"{where}: requires name/act/expect and contains an unsupported field") + if not isinstance(step["name"], str) or not step["name"]: + raise HarnessError(f"{where}.name must be non-empty") + locators = step.get("locate") + if locators is not None: + if not isinstance(locators, list) or not locators: + raise HarnessError(f"{where}.locate must be a non-empty list") + for locator in locators: + validate_locator(locator, f"{where}.locate") + action = step["act"] + action_type = action.get("type") if isinstance(action, dict) else None + if action_type not in {"activate", "click", "move_pointer", "press", "scroll", "type_text", "wait"}: + raise HarnessError(f"{where}.act has unsupported type") + if set(action) - {"type", "duration_ms", "key", "modifiers", "text", "delta_y"}: + raise HarnessError(f"{where}.act contains an unsupported field") + if action_type in {"click", "move_pointer", "scroll"} and locators is None: + raise HarnessError(f"{where}: {action_type} requires locate") + if action_type == "press": + if not isinstance(action.get("key"), str) or not action["key"]: + raise HarnessError(f"{where}: press requires key") + modifiers = action.get("modifiers", []) + allowed_modifiers = {"command", "control", "option", "shift"} + if ( + not isinstance(modifiers, list) + or not all(isinstance(item, str) and item in allowed_modifiers for item in modifiers) + or len(set(modifiers)) != len(modifiers) + ): + raise HarnessError( + f"{where}: press modifiers must be unique command/control/option/shift values" + ) + if action_type == "type_text" and not isinstance(action.get("text"), str): + raise HarnessError(f"{where}: type_text requires text") + if action_type == "scroll" and ( + not is_int(action.get("delta_y")) or not -10000 <= action["delta_y"] <= 10000): + raise HarnessError(f"{where}: scroll requires integer delta_y in -10000..10000") + if action_type == "wait" and ( + not is_int(action.get("duration_ms")) or not 0 < action["duration_ms"] <= 30000): + raise HarnessError(f"{where}: wait requires integer duration_ms in 1..30000") + if action_type != "wait" and "duration_ms" in action and ( + not is_int(action["duration_ms"]) or not 0 <= action["duration_ms"] <= 30000): + raise HarnessError(f"{where}.act.duration_ms must be an integer in 0..30000") + validate_expectation(step["expect"], f"{where}.expect") + if "expect_for" in step: + sustained = step["expect_for"] + if not isinstance(sustained, dict) or set(sustained) != {"duration_ms", "condition"}: + raise HarnessError(f"{where}.expect_for requires duration_ms and condition") + if not is_int(sustained["duration_ms"]) or not 0 < sustained["duration_ms"] <= 30000: + raise HarnessError(f"{where}.expect_for.duration_ms must be an integer in 1..30000") + validate_expectation(sustained["condition"], f"{where}.expect_for.condition") + timeout = step.get("timeout_ms", 5000) + if not is_int(timeout) or not 0 < timeout <= 60000: + raise HarnessError(f"{where}.timeout_ms must be 1..60000") + if "measure" in step: + measurement = step["measure"] + if not isinstance(measurement, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_.-]*", measurement): + raise HarnessError(f"{where}.measure must be a lowercase metric identifier") + if measurement in measurement_names: + raise HarnessError(f"duplicate measurement name: {measurement}") + measurement_names.add(measurement) + cleanup = journey["cleanup"] + if not isinstance(cleanup, dict) or set(cleanup) != {"terminate_app", "remove_state"} or not all(isinstance(v, bool) for v in cleanup.values()): + raise HarnessError("cleanup requires boolean terminate_app and remove_state") + return journey + + +def isolation_manifest(run_id: str, relay_url: str) -> dict[str, str]: + parsed = urllib.parse.urlparse(relay_url) + if parsed.scheme not in {"ws", "http"} or parsed.hostname not in {"localhost", "127.0.0.1", "::1"}: + raise HarnessError(f"refusing non-loopback review relay: {relay_url}") + slug = re.sub(r"[^a-z0-9-]", "-", run_id.lower()) + bundle_id = f"xyz.block.buzz.app.dev.native-review.{slug}" + keyring = f"buzz-desktop-dev.native-review.{slug}" + if bundle_id in PRODUCTION_BUNDLE_IDS or not bundle_id.startswith("xyz.block.buzz.app.dev.native-review."): + raise HarnessError(f"refusing unsafe bundle identifier: {bundle_id}") + if keyring in PRODUCTION_KEYRINGS or not keyring.startswith("buzz-desktop-dev.native-review."): + raise HarnessError(f"refusing unsafe keyring service: {keyring}") + return {"bundle_id": bundle_id, "keyring_service": keyring, "relay_url": relay_url} + + +def provenance() -> dict[str, Any]: + head = git("rev-parse", "HEAD") + status = git("status", "--porcelain=v1", "--untracked-files=all") + diff = run(["git", "diff", "--binary", "HEAD"]).stdout + untracked_hashes = [] + for line in status.splitlines(): + if line.startswith("?? "): + path = ROOT / line[3:] + if path.is_file(): + untracked_hashes.append((line[3:], sha256(path))) + dirty_payload = json.dumps({"diff": diff, "untracked": untracked_hashes}, sort_keys=True).encode() + return { + "head_sha": head, + "dirty": bool(status), + "dirty_state_sha256": hashlib.sha256(dirty_payload).hexdigest() if status else None, + "status": status.splitlines(), + } + + +def scrubbed_environment(*, include_home: bool = False) -> dict[str, str]: + keep = {"PATH", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", "TERM", "__CF_USER_TEXT_ENCODING"} + env = {key: value for key, value in os.environ.items() if key in keep and not SECRET_NAME.search(key)} + env["HOME"] = os.environ.get("HOME", "") if include_home else "" # isolated per run unless tooling needs host caches + return env + + +def fixture_environment(isolation: dict[str, str], review_pubkey: str) -> dict[str, str]: + """Return fixed local fixture coordinates without inheriting host credentials.""" + parsed = urllib.parse.urlparse(isolation["relay_url"]) + port = parsed.port or 80 + if port != 3030: + raise HarnessError("fixture seeding requires the isolated relay at loopback port 3030") + return { + **scrubbed_environment(include_home=True), + "BUZZ_REVIEW_PUBKEY": review_pubkey, + "BUZZ_COMMUNITY_HOST": f"{parsed.hostname}:{port}", + "BUZZ_DB_HOST": "localhost", + "BUZZ_DB_PORT": "5471", + "BUZZ_DB_USER": "buzz", + "BUZZ_DB_PASS": "buzz_dev", + "BUZZ_DB_NAME": "buzz", + "BUZZ_DB_DOCKER_CONTAINER": "buzz-harness-postgres-1", + } + + +def driver_binary() -> pathlib.Path: + override = os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER") + if override: + return pathlib.Path(override).resolve() + return TOOL_ROOT / "swift" / ".build" / "release" / "buzz-native-driver" + + +def build_driver() -> pathlib.Path: + binary = driver_binary() + if os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER"): + if not binary.is_file(): + raise HarnessError(f"configured driver does not exist: {binary}") + return binary + run(["swift", "build", "-c", "release", "--package-path", str(TOOL_ROOT / "swift")], + env=scrubbed_environment(include_home=True), capture=False) + if not binary.is_file(): + raise HarnessError(f"Swift build succeeded without driver binary: {binary}") + return binary + + +class Driver: + def __init__(self, binary: pathlib.Path, pid: int, semantic_snapshot: pathlib.Path): + self.process = subprocess.Popen([str(binary), "serve", "--pid", str(pid), + "--semantic-snapshot", str(semantic_snapshot)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=scrubbed_environment(include_home=True), text=True, bufsize=1) + + def request(self, command: str, **payload: Any) -> dict[str, Any]: + assert self.process.stdin and self.process.stdout + self.process.stdin.write(json.dumps({"command": command, **payload}) + "\n") + self.process.stdin.flush() + timeout_seconds = 45 if command in {"record_start", "record_stop"} else 15 + if command == "act" and isinstance(payload.get("action"), dict): + duration_ms = payload["action"].get("duration_ms", 0) + if is_int(duration_ms): + timeout_seconds = max(timeout_seconds, duration_ms / 1000 + 5) + ready, _, _ = select.select([self.process.stdout], [], [], timeout_seconds) + if not ready: + self.process.kill() + self.process.wait(timeout=5) + stderr = self.process.stderr.read() if self.process.stderr else "" + detail = f"; driver stderr:\n{stderr.strip()}" if stderr.strip() else "" + raise HarnessError(f"native driver timed out during {command}{detail}") + line = self.process.stdout.readline() + if not line: + stderr = self.process.stderr.read() if self.process.stderr else "" + raise HarnessError(f"native driver exited during {command}: {stderr.strip()}") + response = json.loads(line) + if not response.get("ok"): + raise HarnessError(str(response.get("error", f"driver {command} failed"))) + return response + + def close(self) -> None: + if self.process.poll() is None: + try: + self.request("shutdown") + except Exception: + self.process.terminate() + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + if stream: + stream.close() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + + +def doctor(require_permissions: bool = False) -> dict[str, Any]: + checks: list[dict[str, Any]] = [] + checks.append({"name": "platform", "ok": sys.platform == "darwin", "detail": sys.platform}) + for command in ("swift", "xcrun", "git", "ffmpeg"): + path = shutil.which(command) + checks.append({"name": command, "ok": path is not None, "detail": path or "not found"}) + checks.append({"name": "repository", "ok": (ROOT / "desktop/src-tauri/tauri.conf.json").is_file(), "detail": str(ROOT)}) + try: + binary = build_driver() if all(c["ok"] for c in checks[:4]) else driver_binary() + result = run([str(binary), "doctor"], check=False) + native = json.loads(result.stdout) if result.stdout else {"ok": False, "error": result.stderr.strip()} + checks.extend(native.get("checks", [])) + except (HarnessError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + checks.append({"name": "native-driver", "ok": False, "detail": str(exc)}) + hard_names = {"platform", "swift", "xcrun", "git", "ffmpeg", "repository", "native-driver-build"} + failed = [c for c in checks if not c.get("ok") and (require_permissions or c.get("name") in hard_names)] + result = {"ok": not failed, "checks": checks} + print(json.dumps(result, indent=2)) + return result + + +def prepare_fixture(run_dir: pathlib.Path, isolation: dict[str, str]) -> dict[str, Any]: + admin = ROOT / "target" / "debug" / "buzz-admin" + if not admin.is_file(): + run(["cargo", "build", "-p", "buzz-admin"], env=scrubbed_environment(include_home=True), capture=False) + generated = run([str(admin), "generate-key"], env=scrubbed_environment(include_home=True)).stdout + secret_match = re.search(r"Secret key:\s+(\S+)", generated) + public_match = re.search(r"Public key:\s+(\S+)", generated) + if not secret_match or not public_match: + raise HarnessError("buzz-admin generate-key returned an unrecognized response") + secret_path = run_dir / "state" / "identity.key" + secret_path.parent.mkdir(parents=True, exist_ok=True) + secret_path.write_text(secret_match.group(1) + "\n") + secret_path.chmod(0o600) + fixture = { + "kind": "local_review_channel", "identity_pubkey": public_match.group(1), + "secret_path": str(secret_path), "relay_url": isolation["relay_url"], + "seed": "scripts/setup-desktop-test-data.sh", "cleanup_scope": "run-local app state and keyring only", + } + try: + run([str(ROOT / "scripts/setup-desktop-test-data.sh")], + env=fixture_environment(isolation, fixture["identity_pubkey"]), capture=False) + except Exception: + secret_path.unlink(missing_ok=True) + raise + (run_dir / "manifest" / "fixture.json").write_text(json.dumps({k: v for k, v in fixture.items() if k != "secret_path"}, indent=2)) + return fixture + + +def semantic_probe_server(path: pathlib.Path) -> tuple[http.server.ThreadingHTTPServer, str]: + class Handler(http.server.BaseHTTPRequestHandler): + def end_headers(self) -> None: + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.send_header("Access-Control-Allow-Private-Network", "true") + super().end_headers() + + def do_OPTIONS(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self.send_response(204) + self.end_headers() + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", "0")) + payload = self.rfile.read(length) + try: + value = json.loads(payload) + if not isinstance(value, list): + raise ValueError("snapshot must be an array") + temporary = path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(value)) + temporary.replace(path) + self.send_response(204) + except (ValueError, json.JSONDecodeError, OSError): + self.send_response(400) + self.end_headers() + + def log_message(self, _format: str, *_args: Any) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f"http://127.0.0.1:{server.server_port}/snapshot" + + +def build_and_launch(run_dir: pathlib.Path, isolation: dict[str, str], fixture: dict[str, Any], + probe_url: str) -> tuple[subprocess.Popen[str], pathlib.Path, int]: + # Build with the repository toolchain but no inherited credentials. Launch + # the resulting executable separately so only the app receives isolated HOME. + dev_url = ( + "http://localhost:1420?nativeReview=1" + f"&reviewRelay={urllib.parse.quote(isolation['relay_url'], safe='')}" + f"&reviewPubkey={urllib.parse.quote(fixture['identity_pubkey'], safe='')}" + ) + config = json.dumps({ + "build": {"devUrl": dev_url}, + "identifier": isolation["bundle_id"], "productName": "Buzz Native Review", + "bundle": {"externalBin": []}, + }, separators=(",", ":")) + build_env = scrubbed_environment(include_home=True) + build_env["VITE_NATIVE_REVIEW"] = "1" + build_env["VITE_NATIVE_REVIEW_RELAY"] = isolation["relay_url"] + build_env["VITE_NATIVE_REVIEW_PUBKEY"] = fixture["identity_pubkey"] + build_env["VITE_NATIVE_REVIEW_PROBE_URL"] = probe_url + run(["pnpm", "exec", "tauri", "build", "--debug", "--bundles", "app", "--config", config], + cwd=ROOT / "desktop", env=build_env, capture=False) + app_binary = (ROOT / "desktop" / "src-tauri" / "target" / "debug" / "bundle" / "macos" / + "Buzz Native Review.app" / "Contents" / "MacOS" / "buzz-desktop") + if not app_binary.is_file(): + raise HarnessError(f"Tauri build succeeded without app binary: {app_binary}") + + env = scrubbed_environment() + env["HOME"] = str(run_dir / "home") + pathlib.Path(env["HOME"]).mkdir(parents=True, exist_ok=True) + env.update({ + "BUZZ_PRIVATE_KEY": pathlib.Path(fixture["secret_path"]).read_text().strip(), + "BUZZ_RELAY_URL": isolation["relay_url"], "BUZZ_DEV_KEYRING_SERVICE": isolation["keyring_service"], + "BUZZ_NATIVE_REVIEW": "1", "BUZZ_NATIVE_REVIEW_CHANNEL": "general", + }) + log = (run_dir / "logs" / "app.log").open("w") + process = subprocess.Popen([str(app_binary)], cwd=ROOT, env=env, stdout=log, + stderr=subprocess.STDOUT, text=True) + log.close() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if process.poll() is not None: + raise HarnessError(f"Tauri exited during launch; see {run_dir / 'logs/app.log'}") + if run(["ps", "-p", str(process.pid), "-o", "comm="], check=False).stdout.rstrip().endswith("/buzz-desktop"): + return process, app_binary, process.pid + time.sleep(0.25) + process.terminate() + raise HarnessError("timed out waiting for native Buzz process") + + +def wait_for_visible_window(driver: Driver, process: subprocess.Popen[str], timeout_seconds: float = 30) -> dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + last_status: dict[str, Any] | None = None + while time.monotonic() < deadline: + if process.poll() is not None: + raise HarnessError("Tauri exited while waiting for its initial window") + last_status = driver.request("window_status") + if last_status.get("visible"): + return last_status + time.sleep(0.1) + detail = last_status.get("detail") if last_status else "driver returned no status" + raise HarnessError(f"timed out after {timeout_seconds:g}s waiting for visible native window: {detail}") + + +def locate_required(driver: Driver, locators: list[dict[str, str]], timeout_ms: int) -> dict[str, Any]: + """Wait for a semantic target to materialize, then return the locator used.""" + deadline = time.monotonic() + timeout_ms / 1000 + while True: + found = driver.request("locate", locators=locators, required=False).get("element") + if found is not None: + return found + if time.monotonic() >= deadline: + raise HarnessError(f"no accessibility element matched ordered locators within {timeout_ms}ms") + time.sleep(0.025) + + +def expectation_holds(driver: Driver, expectation: dict[str, Any]) -> bool: + if "exists" in expectation: + return driver.request("locate", locators=[expectation["exists"]], required=False).get("element") is not None + if "not_exists" in expectation: + return driver.request("locate", locators=[expectation["not_exists"]], required=False).get("element") is None + if "focused" in expectation and isinstance(expectation["focused"], dict): + found = driver.request("locate", locators=[expectation["focused"]], required=False).get("element") + return bool(found and found.get("focused")) + if "focused" in expectation: + return bool(driver.request("focused").get("focused")) == expectation["focused"] + if "enabled" in expectation: + return bool(driver.request("selected").get("element", {}).get("enabled")) == expectation["enabled"] + if "value" in expectation: + return driver.request("selected").get("element", {}).get("value") == expectation["value"] + if "scroll_y_greater_than" in expectation: + scroll_y = driver.request("selected").get("element", {}).get("scrollY") + return isinstance(scroll_y, (int, float)) and scroll_y > expectation["scroll_y_greater_than"] + if "scroll_y_less_than" in expectation: + scroll_y = driver.request("selected").get("element", {}).get("scrollY") + return isinstance(scroll_y, (int, float)) and scroll_y < expectation["scroll_y_less_than"] + return False + + +def wait_expectation(driver: Driver, expectation: dict[str, Any], timeout_ms: int) -> None: + deadline = time.monotonic() + timeout_ms / 1000 + while True: + if expectation_holds(driver, expectation): + return + if time.monotonic() >= deadline: + raise HarnessError(f"postcondition not met within {timeout_ms}ms: {expectation}") + time.sleep(0.025) + + +def capture_step(driver: Driver, run_dir: pathlib.Path, slug: str, record: dict[str, Any]) -> dict[str, str]: + artifacts: dict[str, str] = {} + if record["screenshots"]: + path = run_dir / "screenshots" / f"{slug}.png" + driver.request("screenshot", path=str(path)) + artifacts["screenshot"] = str(path.relative_to(run_dir)) + if record["accessibility"]: + path = run_dir / "accessibility" / f"{slug}.json" + response = driver.request("snapshot") + path.write_text(json.dumps({key: value for key, value in response.items() if key != "ok"}, indent=2)) + artifacts["accessibility"] = str(path.relative_to(run_dir)) + return artifacts + + +def cleanup_review_state(run_dir: pathlib.Path, isolation: dict[str, str], + fixture: dict[str, Any] | None) -> None: + errors = [] + env = scrubbed_environment() + env["HOME"] = str(run_dir / "home") + try: + run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), + isolation["bundle_id"], isolation["keyring_service"]], env=env) + except Exception as exc: + errors.append(f"desktop state reset failed: {exc}") + if fixture: + try: + pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + except Exception as exc: + errors.append(f"review identity removal failed: {exc}") + if errors: + raise HarnessError("; ".join(errors)) + + +def run_journey(path: pathlib.Path, relay_url: str, output_root: pathlib.Path) -> pathlib.Path: + journey = load_journey(path) + run_id = f"{journey['flow']}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{secrets.token_hex(3)}" + isolation = isolation_manifest(run_id, relay_url) + run_dir = output_root / git("rev-parse", "--short=12", "HEAD") / journey["flow"] / run_id + for child in ("manifest", "logs", "screenshots", "accessibility", "state", "home"): + (run_dir / child).mkdir(parents=True, exist_ok=True) + (run_dir / "journey.yaml").write_text(path.read_text()) + prov = provenance() + (run_dir / "manifest" / "git.json").write_text(json.dumps(prov, indent=2)) + (run_dir / "manifest" / "isolation.json").write_text(json.dumps(isolation, indent=2)) + started = utc_now() + receipt: dict[str, Any] = {"schema_version": 1, "run_id": run_id, "flow": journey["flow"], "status": "failed", + "started_at": started, "finished_at": started, "failure": None, "provenance": prov, "isolation": isolation, + "artifacts": {}, "steps": [], "measurements": {}, "performance": {"machine": machine_fingerprint()}, + "cleanup": {"status": "not_started"}} + process: subprocess.Popen[str] | None = None + driver: Driver | None = None + fixture: dict[str, Any] | None = None + probe_server: http.server.ThreadingHTTPServer | None = None + sampler: ProcessSampler | None = None + try: + if not doctor(require_permissions=True)["ok"]: + raise HarnessError("doctor failed; grant required permissions and rerun") + fixture = prepare_fixture(run_dir, isolation) + probe_server, probe_url = semantic_probe_server(run_dir / "state" / "semantic.json") + process, app_binary, app_pid = build_and_launch(run_dir, isolation, fixture, probe_url) + receipt["provenance"]["artifact_path"] = str(app_binary) + receipt["provenance"]["artifact_sha256"] = sha256(app_binary) + driver = Driver(build_driver(), app_pid, run_dir / "state" / "semantic.json") + sampler = ProcessSampler(app_pid) + sampler.start() + receipt["provenance"]["initial_window"] = wait_for_visible_window(driver, process) + if journey["record"]["video"] == "window": + video = run_dir / "video.mp4" + driver.request("record_start", path=str(video)) + receipt["artifacts"]["video"] = "video.mp4" + for index, step in enumerate(journey["steps"]): + slug = f"{index + 1:02d}-{re.sub(r'[^a-z0-9-]', '-', step['name'].lower())}" + step_start = time.monotonic_ns() + selected = None + step_receipt: dict[str, Any] = {"name": step["name"], "status": "failed", "started_monotonic_ns": step_start} + receipt["steps"].append(step_receipt) + try: + if step.get("locate"): + selected = locate_required(driver, step["locate"], step.get("timeout_ms", 5000)) + step_receipt["locator"] = selected.get("locator") + driver.request("act", action=step["act"], element=selected) + wait_expectation(driver, step["expect"], step.get("timeout_ms", 5000)) + if sustained := step.get("expect_for"): + until = time.monotonic() + sustained["duration_ms"] / 1000 + while time.monotonic() < until: + if not expectation_holds(driver, sustained["condition"]): + raise HarnessError(f"sustained postcondition failed: {sustained['condition']}") + time.sleep(0.025) + step_receipt["status"] = "passed" + finally: + step_receipt["finished_monotonic_ns"] = time.monotonic_ns() + step_receipt["duration_ms"] = (step_receipt["finished_monotonic_ns"] - step_start) / 1_000_000 + if measurement := step.get("measure"): + receipt["measurements"][measurement] = {"value": step_receipt["duration_ms"], "unit": "ms", "step": step["name"]} + step_receipt["artifacts"] = capture_step(driver, run_dir, slug, journey["record"]) + if journey["record"]["video"] == "window": + driver.request("record_stop") + receipt["status"] = "passed" + except Exception as exc: + receipt["failure"] = str(exc) + if driver: + try: + receipt["artifacts"]["failure"] = capture_step(driver, run_dir, "failure", {"screenshots": True, "accessibility": True}) + except Exception as capture_exc: + receipt["artifacts"]["capture_failure"] = str(capture_exc) + try: + driver.request("record_stop") + except Exception: + pass + finally: + cleanup_errors = [] + if sampler: + receipt["performance"]["process"] = sampler.finish() + video = run_dir / "video.mp4" + if video.is_file(): + try: + relay_safe_video(video, run_dir / "video-share.mp4") + receipt["artifacts"]["share_video"] = "video-share.mp4" + except EvidenceError as exc: + cleanup_errors.append(f"shareable video finalization failed: {exc}") + if probe_server: + probe_server.shutdown() + probe_server.server_close() + if driver: + try: + driver.close() + except Exception as exc: + cleanup_errors.append(f"native driver cleanup failed: {exc}") + if process and journey["cleanup"]["terminate_app"]: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill(); cleanup_errors.append("Tauri launcher required SIGKILL") + if journey["cleanup"]["remove_state"]: + try: + cleanup_review_state(run_dir, isolation, fixture) + except Exception as exc: + cleanup_errors.append(str(exc)) + receipt["cleanup"] = {"status": "failed" if cleanup_errors else "passed", "errors": cleanup_errors} + if cleanup_errors: + receipt["status"] = "failed" + receipt["finished_at"] = utc_now() + (run_dir / "receipt.json").write_text(json.dumps(receipt, indent=2)) + report = f"# Native review: {journey['flow']}\n\n**{receipt['status'].upper()}** at `{prov['head_sha']}`.\n\nReceipt: `receipt.json`\n" + if receipt["failure"]: + report += f"\nFailure: `{receipt['failure']}`\n" + (run_dir / "report.md").write_text(report) + print(run_dir) + if receipt["status"] != "passed": + raise HarnessError(receipt["failure"] or "journey or cleanup failed") + return run_dir + + +def load_receipts(paths: list[pathlib.Path], label: str) -> list[dict[str, Any]]: + if len({path.resolve() for path in paths}) != len(paths): + raise HarnessError(f"{label} cohort contains duplicate receipt paths") + receipts = [] + run_ids: set[str] = set() + for path in paths: + try: + receipt = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise HarnessError(f"cannot read {label} receipt {path}: {exc}") from exc + if receipt.get("status") != "passed" or receipt.get("cleanup", {}).get("status") != "passed": + raise HarnessError(f"{label} receipt is not a clean pass: {path}") + if receipt.get("provenance", {}).get("dirty"): + raise HarnessError(f"{label} receipt was captured from a dirty tree: {path}") + run_id = receipt.get("run_id") + if not isinstance(run_id, str) or not run_id or run_id in run_ids: + raise HarnessError(f"{label} cohort contains a missing or duplicate run_id: {run_id}") + run_ids.add(run_id) + receipts.append(receipt) + return receipts + + +def metric_value(receipt: dict[str, Any], metric: str) -> float: + if metric.startswith("process."): + value = receipt.get("performance", {}).get("process", {}).get(metric.removeprefix("process.")) + else: + value = receipt.get("measurements", {}).get(metric, {}).get("value") + if not is_finite_number(value): + raise HarnessError(f"receipt {receipt.get('run_id')} has no finite numeric metric {metric}") + return float(value) + + +def cohort_summary(receipts: list[dict[str, Any]], metrics: list[str]) -> dict[str, Any]: + return { + "sample_count": len(receipts), + "head_sha": receipts[0]["provenance"]["head_sha"], + "artifact_sha256": [receipt["provenance"]["artifact_sha256"] for receipt in receipts], + "metrics": { + metric: { + "median": statistics.median(values := [metric_value(receipt, metric) for receipt in receipts]), + "minimum": min(values), + "maximum": max(values), + "samples": values, + } + for metric in metrics + }, + } + + +def compare_performance(baseline_paths: list[pathlib.Path], candidate_paths: list[pathlib.Path], + budget_path: pathlib.Path, output: pathlib.Path | None = None) -> dict[str, Any]: + try: + budget = yaml.safe_load(budget_path.read_text()) + except (OSError, yaml.YAMLError) as exc: + raise HarnessError(f"cannot read performance budget {budget_path}: {exc}") from exc + if not isinstance(budget, dict) or set(budget) != {"schema_version", "flow", "minimum_samples", "metrics"}: + raise HarnessError("performance budget requires exactly schema_version, flow, minimum_samples, and metrics") + if budget["schema_version"] != 1 or not isinstance(budget["flow"], str): + raise HarnessError("unsupported performance budget schema") + minimum = budget["minimum_samples"] + if not is_int(minimum) or minimum < PERFORMANCE_SAMPLE_MINIMUM: + raise HarnessError(f"minimum_samples must be at least {PERFORMANCE_SAMPLE_MINIMUM}") + metrics = budget["metrics"] + if not isinstance(metrics, dict) or not metrics: + raise HarnessError("performance budget requires at least one metric") + allowed_limits = {"max", "max_regression_percent"} + for name, limits in metrics.items(): + if (not isinstance(name, str) or not name or not isinstance(limits, dict) or not limits + or set(limits) - allowed_limits + or not all(is_finite_number(value) and value >= 0 for value in limits.values())): + raise HarnessError(f"invalid limits for performance metric {name}") + + baseline = load_receipts(baseline_paths, "baseline") + candidate = load_receipts(candidate_paths, "candidate") + if len(baseline) < minimum or len(candidate) < minimum: + raise HarnessError(f"performance comparison requires at least {minimum} clean samples per cohort") + all_paths = baseline_paths + candidate_paths + if len({path.resolve() for path in all_paths}) != len(all_paths): + raise HarnessError("baseline and candidate cohorts must use independent receipt paths") + all_receipts = baseline + candidate + all_run_ids = [receipt["run_id"] for receipt in all_receipts] + if len(set(all_run_ids)) != len(all_run_ids): + raise HarnessError("baseline and candidate cohorts must use independent run_ids") + flows = {receipt.get("flow") for receipt in all_receipts} + machines = {json.dumps(receipt.get("performance", {}).get("machine"), sort_keys=True) for receipt in all_receipts} + if flows != {budget["flow"]}: + raise HarnessError(f"receipt flows {sorted(str(item) for item in flows)} do not match budget flow {budget['flow']}") + if len(machines) != 1: + raise HarnessError("baseline and candidate receipts were captured on incompatible machines") + for label, cohort in (("baseline", baseline), ("candidate", candidate)): + if len({receipt["provenance"].get("head_sha") for receipt in cohort}) != 1: + raise HarnessError(f"{label} cohort mixes source revisions") + + baseline_summary = cohort_summary(baseline, list(metrics)) + candidate_summary = cohort_summary(candidate, list(metrics)) + verdicts = {} + failures = [] + for name, limits in metrics.items(): + baseline_median = baseline_summary["metrics"][name]["median"] + candidate_median = candidate_summary["metrics"][name]["median"] + candidate_maximum = candidate_summary["metrics"][name]["maximum"] + regression = None if baseline_median == 0 and candidate_median > 0 else ( + 0.0 if baseline_median == 0 else (candidate_median - baseline_median) / baseline_median * 100 + ) + reasons = [] + if "max" in limits and candidate_maximum > limits["max"]: + reasons.append(f"maximum {candidate_maximum:.3f} exceeds absolute maximum {limits['max']}") + if "max_regression_percent" in limits: + if regression is None: + reasons.append("relative regression is undefined because the baseline median is zero") + elif regression > limits["max_regression_percent"]: + reasons.append(f"regression {regression:.2f}% exceeds {limits['max_regression_percent']}%") + verdicts[name] = {"status": "failed" if reasons else "passed", "regression_percent": regression, "reasons": reasons} + failures.extend(f"{name}: {reason}" for reason in reasons) + result = { + "schema_version": 1, "status": "failed" if failures else "passed", "flow": budget["flow"], + "machine": all_receipts[0]["performance"]["machine"], "baseline": baseline_summary, + "candidate": candidate_summary, "verdicts": verdicts, "failures": failures, + } + serialized = json.dumps(result, indent=2, allow_nan=False) + if output: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(serialized + "\n") + print(serialized) + if failures: + raise HarnessError("performance budget failed: " + "; ".join(failures)) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + doctor_parser = sub.add_parser("doctor") + doctor_parser.add_argument("--require-permissions", action="store_true") + validate_parser = sub.add_parser("validate") + validate_parser.add_argument("journey", type=pathlib.Path) + run_parser = sub.add_parser("run") + run_parser.add_argument("journey", type=pathlib.Path) + run_parser.add_argument("--relay", default="ws://localhost:3030") + run_parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + benchmark_parser = sub.add_parser("benchmark") + benchmark_parser.add_argument("journey", type=pathlib.Path) + benchmark_parser.add_argument("--runs", type=int, default=5) + benchmark_parser.add_argument("--relay", default="ws://localhost:3030") + benchmark_parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + compare_parser = sub.add_parser("compare") + compare_parser.add_argument("--baseline", type=pathlib.Path, action="append", required=True) + compare_parser.add_argument("--candidate", type=pathlib.Path, action="append", required=True) + compare_parser.add_argument("--budget", type=pathlib.Path, required=True) + compare_parser.add_argument("--output", type=pathlib.Path) + bundle_parser = sub.add_parser("finding-bundle") + bundle_parser.add_argument("receipt", type=pathlib.Path) + bundle_parser.add_argument("--output", type=pathlib.Path, required=True) + bundle_parser.add_argument("--match", help="case-insensitive regular expression selecting log lines") + bundle_parser.add_argument("--context", type=int, default=8) + bundle_parser.add_argument("--start", type=float, help="clip start in seconds") + bundle_parser.add_argument("--duration", type=float, help="clip duration in seconds") + publish_parser = sub.add_parser("publish-review") + publish_parser.add_argument("receipt", type=pathlib.Path) + publish_parser.add_argument("--summary", type=pathlib.Path, required=True) + publish_parser.add_argument("--channel", required=True) + publish_parser.add_argument("--reply-to", required=True) + publish_parser.add_argument("--highlights", type=pathlib.Path) + publish_parser.add_argument("--mention", action="append", default=[]) + args = parser.parse_args() + try: + if args.command == "doctor": + return 0 if doctor(args.require_permissions)["ok"] else 1 + if args.command == "validate": + load_journey(args.journey); print(f"valid: {args.journey}"); return 0 + if args.command == "compare": + compare_performance(args.baseline, args.candidate, args.budget, args.output); return 0 + if args.command == "publish-review": + result = publish_review(args.receipt.resolve(), args.summary.resolve(), args.channel, + args.reply_to, args.highlights.resolve() if args.highlights else None, + args.mention) + print(json.dumps(result, indent=2)); return 0 + if args.command == "finding-bundle": + result = finding_bundle(args.receipt.resolve(), args.output.resolve(), match=args.match, context=args.context, + start=args.start, duration=args.duration) + print(json.dumps({"output": str(args.output.resolve()), "manifest": result}, indent=2)); return 0 + if args.command == "benchmark": + if not is_int(args.runs) or args.runs < PERFORMANCE_SAMPLE_MINIMUM: + raise HarnessError(f"benchmark requires at least {PERFORMANCE_SAMPLE_MINIMUM} runs") + receipts = [run_journey(args.journey, args.relay, args.output) / "receipt.json" for _ in range(args.runs)] + print(json.dumps({"receipts": [str(path) for path in receipts]}, indent=2)); return 0 + run_journey(args.journey, args.relay, args.output); return 0 + except (HarnessError, EvidenceError, PublishError) as exc: + print(f"native-review: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/native-review/review_publish.py b/tools/native-review/review_publish.py new file mode 100644 index 00000000000..4b845408afc --- /dev/null +++ b/tools/native-review/review_publish.py @@ -0,0 +1,153 @@ +"""Publish exact-SHA native-review evidence and timecoded highlights to Buzz.""" +from __future__ import annotations + +import hashlib +import json +import math +import pathlib +import shutil +import subprocess +from typing import Any + + +class PublishError(RuntimeError): + pass + + +def _artifact(run_dir: pathlib.Path, name: str) -> pathlib.Path: + candidate = (run_dir / name).resolve() + if run_dir.resolve() not in candidate.parents or not candidate.is_file(): + raise PublishError(f"artifact is missing or escapes receipt directory: {name}") + return candidate + + +def _run(command: list[str]) -> subprocess.CompletedProcess[str]: + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode: + detail = result.stderr.strip() or result.stdout.strip() or f"exit {result.returncode}" + raise PublishError(f"command failed: {detail}") + return result + + +def _accepted_event(result: subprocess.CompletedProcess[str], operation: str) -> str: + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise PublishError(f"{operation} returned invalid JSON") from exc + event_id = payload.get("event_id") + if payload.get("accepted") is not True or not isinstance(event_id, str) or len(event_id) != 64: + raise PublishError(f"{operation} was not accepted: {payload}") + return event_id + + +def _sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _video_duration(video: pathlib.Path) -> float: + ffprobe = shutil.which("ffprobe") + if not ffprobe: + raise PublishError("ffprobe is required to validate highlight timecodes") + result = _run([ + ffprobe, "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", str(video), + ]) + try: + duration = float(result.stdout.strip()) + except ValueError as exc: + raise PublishError("ffprobe returned an invalid video duration") from exc + if not math.isfinite(duration) or duration <= 0: + raise PublishError("share video has no finite positive duration") + return duration + + +def _load_highlights(path: pathlib.Path | None, duration: float) -> list[dict[str, Any]]: + if path is None: + return [] + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise PublishError(f"cannot read highlights {path}: {exc}") from exc + if not isinstance(payload, list): + raise PublishError("highlights must be a JSON array") + highlights: list[dict[str, Any]] = [] + for index, item in enumerate(payload): + if not isinstance(item, dict) or set(item) != {"seconds", "text"}: + raise PublishError(f"highlight {index} requires exactly seconds and text") + seconds, text = item["seconds"], item["text"] + if (not isinstance(seconds, (int, float)) or isinstance(seconds, bool) + or not math.isfinite(seconds) or seconds < 0 or seconds > duration): + raise PublishError(f"highlight {index} seconds must be within the video (0..{duration:.3f})") + if not isinstance(text, str) or not text.strip(): + raise PublishError(f"highlight {index} text must be non-empty") + highlights.append({"seconds": float(seconds), "text": text.strip()}) + return highlights + + +def format_timecode(seconds: float) -> str: + total_ms = round(seconds * 1000) + hours, remainder = divmod(total_ms, 3_600_000) + minutes, remainder = divmod(remainder, 60_000) + whole_seconds, milliseconds = divmod(remainder, 1000) + if hours: + base = f"{hours}:{minutes:02d}:{whole_seconds:02d}" + else: + base = f"{minutes:02d}:{whole_seconds:02d}" + return f"{base}.{milliseconds:03d}" if milliseconds else base + + +def publish_review(receipt_path: pathlib.Path, summary_path: pathlib.Path, channel: str, + reply_to: str, highlights_path: pathlib.Path | None = None, + mentions: list[str] | None = None) -> dict[str, Any]: + buzz = shutil.which("buzz") + if not buzz: + raise PublishError("buzz CLI is required to publish review evidence") + try: + receipt = json.loads(receipt_path.read_text()) + summary = summary_path.read_text().strip() + except (OSError, json.JSONDecodeError) as exc: + raise PublishError(f"cannot read review input: {exc}") from exc + if not summary: + raise PublishError("review summary must be non-empty") + provenance = receipt.get("provenance", {}) + if provenance.get("dirty") is not False: + raise PublishError("review publication requires a clean source receipt") + head_sha = provenance.get("head_sha") + if not isinstance(head_sha, str) or len(head_sha) != 40: + raise PublishError("receipt has no full head SHA") + if receipt.get("cleanup", {}).get("status") != "passed": + raise PublishError("review publication requires passed cleanup") + share_name = receipt.get("artifacts", {}).get("share_video") + if not isinstance(share_name, str): + raise PublishError("receipt has no relay-safe share video") + video = _artifact(receipt_path.parent, share_name) + duration = _video_duration(video) + highlights = _load_highlights(highlights_path, duration) + evidence = ( + f"\n\n**Native evidence:** `{receipt.get('flow')}` · `{receipt.get('status')}` · " + f"exact clean SHA `{head_sha}` · {duration:.3f}s." + ) + command = [buzz, "messages", "send", "--channel", channel, "--reply-to", reply_to, + "--content", summary + evidence, "--file", str(video)] + for mention in mentions or []: + command.extend(["--mention", mention]) + video_event_id = _accepted_event(_run(command), "review evidence publication") + highlight_event_ids = [] + for highlight in highlights: + content = f"[{format_timecode(highlight['seconds'])}] {highlight['text']}" + event_id = _accepted_event(_run([ + buzz, "messages", "send", "--channel", channel, "--reply-to", video_event_id, + "--content", content, + ]), f"highlight at {highlight['seconds']:.3f}s") + highlight_event_ids.append(event_id) + return { + "video_event_id": video_event_id, + "highlight_event_ids": highlight_event_ids, + "head_sha": head_sha, + "video_sha256": _sha256(video), + "duration_seconds": duration, + } diff --git a/tools/native-review/schemas/journey.schema.json b/tools/native-review/schemas/journey.schema.json new file mode 100644 index 00000000000..0db284ff05e --- /dev/null +++ b/tools/native-review/schemas/journey.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://buzz.local/native-review/journey.schema.json", + "title": "Buzz native review journey", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "flow", "platforms", "fixture", "record", "steps", "cleanup"], + "properties": { + "schema_version": {"const": 1}, + "flow": {"type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$"}, + "platforms": {"type": "array", "minItems": 1, "items": {"enum": ["macos"]}}, + "fixture": {"enum": ["local_review_channel"]}, + "record": { + "type": "object", "additionalProperties": false, + "required": ["video", "screenshots", "accessibility"], + "properties": { + "video": {"enum": ["window", "off"]}, + "screenshots": {"type": "boolean"}, + "accessibility": {"type": "boolean"} + } + }, + "steps": { + "type": "array", "minItems": 1, + "items": { + "type": "object", "additionalProperties": false, + "required": ["name", "act", "expect"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "locate": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/locator"}}, + "act": {"$ref": "#/$defs/action"}, + "expect": {"$ref": "#/$defs/expectation"}, + "expect_for": { + "type": "object", "additionalProperties": false, + "required": ["duration_ms", "condition"], + "properties": { + "duration_ms": {"type": "integer", "minimum": 1, "maximum": 30000}, + "condition": {"$ref": "#/$defs/expectation"} + } + }, + "timeout_ms": {"type": "integer", "minimum": 1, "maximum": 60000}, + "measure": {"type": "string", "minLength": 1} + } + } + }, + "cleanup": { + "type": "object", "additionalProperties": false, + "required": ["terminate_app", "remove_state"], + "properties": {"terminate_app": {"type": "boolean"}, "remove_state": {"type": "boolean"}} + } + }, + "$defs": { + "locator": { + "type": "object", "additionalProperties": false, "minProperties": 1, + "properties": {"id": {"type": "string"}, "role": {"type": "string"}, "name": {"type": "string"}} + }, + "action": { + "type": "object", "additionalProperties": false, "required": ["type"], + "properties": { + "type": {"enum": ["activate", "click", "move_pointer", "press", "scroll", "type_text", "wait"]}, + "duration_ms": {"type": "integer", "minimum": 0, "maximum": 30000}, + "key": {"type": "string"}, + "modifiers": {"type": "array", "uniqueItems": true, "items": {"enum": ["command", "control", "option", "shift"]}}, + "text": {"type": "string"}, + "delta_y": {"type": "integer", "minimum": -10000, "maximum": 10000} + } + }, + "expectation": { + "type": "object", "additionalProperties": false, "minProperties": 1, "maxProperties": 1, + "properties": { + "exists": {"$ref": "#/$defs/locator"}, + "not_exists": {"$ref": "#/$defs/locator"}, + "focused": {"oneOf": [{"type": "boolean"}, {"$ref": "#/$defs/locator"}]}, + "enabled": {"type": "boolean"}, + "value": {"type": "string"}, + "scroll_y_greater_than": {"type": "number"}, + "scroll_y_less_than": {"type": "number"} + } + } + } +} diff --git a/tools/native-review/schemas/receipt.schema.json b/tools/native-review/schemas/receipt.schema.json new file mode 100644 index 00000000000..7839aa36dc2 --- /dev/null +++ b/tools/native-review/schemas/receipt.schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://buzz.local/native-review/receipt.schema.json", + "title": "Buzz native review receipt", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "flow", + "status", + "started_at", + "finished_at", + "provenance", + "isolation", + "artifacts", + "steps", + "cleanup", + "measurements", + "performance" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "run_id": { + "type": "string" + }, + "flow": { + "type": "string" + }, + "status": { + "enum": [ + "passed", + "failed" + ] + }, + "started_at": { + "type": "string" + }, + "finished_at": { + "type": "string" + }, + "failure": { + "type": [ + "string", + "null" + ] + }, + "provenance": { + "type": "object" + }, + "isolation": { + "type": "object" + }, + "artifacts": { + "type": "object" + }, + "steps": { + "type": "array", + "items": { + "type": "object" + } + }, + "cleanup": { + "type": "object" + }, + "measurements": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "value", + "unit", + "step" + ], + "properties": { + "value": { + "type": "number" + }, + "unit": { + "const": "ms" + }, + "step": { + "type": "string" + } + } + } + }, + "performance": { + "type": "object", + "required": [ + "machine" + ], + "properties": { + "machine": { + "type": "object" + }, + "process": { + "type": "object" + } + } + } + } +} diff --git a/tools/native-review/swift/Package.swift b/tools/native-review/swift/Package.swift new file mode 100644 index 00000000000..8e7557542d4 --- /dev/null +++ b/tools/native-review/swift/Package.swift @@ -0,0 +1,13 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "BuzzNativeDriver", + platforms: [.macOS(.v13)], + products: [.executable(name: "buzz-native-driver", targets: ["BuzzNativeDriver"])], + targets: [ + .target(name: "BuzzNativeDriverSupport"), + .executableTarget(name: "BuzzNativeDriver", dependencies: ["BuzzNativeDriverSupport"]), + .testTarget(name: "BuzzNativeDriverSupportTests", dependencies: ["BuzzNativeDriverSupport"]), + ] +) diff --git a/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift b/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift new file mode 100644 index 00000000000..39ebaa0f30f --- /dev/null +++ b/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift @@ -0,0 +1,622 @@ +import AppKit +import ApplicationServices +import AVFoundation +import CoreGraphics +import Foundation +import BuzzNativeDriverSupport + +struct Locator: Codable { + let id: String? + let role: String? + let name: String? +} + +struct Size: Codable { + let width: Double + let height: Double +} + +struct SemanticNode: Codable { + let id: String? + let role: String? + let name: String? + let value: String? + let scrollY: Double + let enabled: Bool + let focused: Bool + let frame: Rect + let viewport: Size +} + +struct ElementDescription: Codable { + let locator: Locator + let role: String? + let name: String? + let identifier: String? + let value: String? + let scrollY: Double? + let enabled: Bool + let focused: Bool + let frame: Rect? +} + +struct Rect: Codable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct AXNode: Codable { + let role: String? + let name: String? + let identifier: String? + let value: String? + let enabled: Bool? + let focused: Bool? + let frame: Rect? + let children: [AXNode] + let truncated: Bool? +} + +enum DriverError: Error, CustomStringConvertible { + case message(String) + var description: String { + switch self { case .message(let value): return value } + } +} + +func enableWebViewAccessibility(_ app: AXUIElement) { + // WebKit does not materialize its remote accessibility tree for ordinary + // automation clients until manual/enhanced accessibility is requested. + // VoiceOver does this implicitly; the review harness must not require it. + let enabled = kCFBooleanTrue as CFTypeRef + for name in ["AXManualAccessibility", "AXEnhancedUserInterface"] { + _ = AXUIElementSetAttributeValue(app, name as CFString, enabled) + } +} + +func attribute(_ element: AXUIElement, _ name: String) -> AnyObject? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else { return nil } + return value +} + +func stringAttribute(_ element: AXUIElement, _ names: [String]) -> String? { + for name in names { + if let value = attribute(element, name) as? String, !value.isEmpty { return value } + } + return nil +} + +func boolAttribute(_ element: AXUIElement, _ name: String, default fallback: Bool = false) -> Bool { + (attribute(element, name) as? Bool) ?? fallback +} + +func frameAttribute(_ element: AXUIElement) -> Rect? { + guard let positionObject = attribute(element, kAXPositionAttribute), + let sizeObject = attribute(element, kAXSizeAttribute), + CFGetTypeID(positionObject) == AXValueGetTypeID(), + CFGetTypeID(sizeObject) == AXValueGetTypeID() else { return nil } + let positionValue = positionObject as! AXValue + let sizeValue = sizeObject as! AXValue + var point = CGPoint.zero + var size = CGSize.zero + guard AXValueGetValue(positionValue, .cgPoint, &point), AXValueGetValue(sizeValue, .cgSize, &size) else { return nil } + return Rect(x: point.x, y: point.y, width: size.width, height: size.height) +} + +func normalizedRole(_ raw: String?) -> String? { + guard var value = raw else { return nil } + if value.hasPrefix("AX") { value.removeFirst(2) } + switch value.lowercased() { + case "textarea", "text area": return "text-area" + default: return value.lowercased() + } +} + +func elementName(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute, kAXValueAttribute]) +} + +func elementIdentifier(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXIdentifierAttribute]) +} + +func elementRole(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXRoleAttribute]) +} + +func matches(_ element: AXUIElement, locator: Locator) -> Bool { + if let id = locator.id, elementIdentifier(element) != id { return false } + if let role = locator.role, normalizedRole(elementRole(element)) != normalizedRole(role) { return false } + if let name = locator.name, elementName(element) != name { return false } + return true +} + +func children(_ element: AXUIElement) -> [AXUIElement] { + let direct = (attribute(element, kAXChildrenAttribute) as? [AXUIElement]) ?? [] + let windows = (attribute(element, kAXWindowsAttribute) as? [AXUIElement]) ?? [] + let singular = [kAXMainWindowAttribute, kAXFocusedWindowAttribute].compactMap { + attribute(element, $0) as! AXUIElement? + } + var seen = Set() + return (direct + windows + singular).filter { child in + guard !CFEqual(child, element) else { return false } + return seen.insert(CFHash(child)).inserted + } +} + +func accessibilityRoots(app: AXUIElement, pid: pid_t) -> [AXUIElement] { + // WKWebView's remote tree is not always attached to AXChildren/AXWindows, + // even after manual accessibility is enabled. Hit-testing the visible app + // window asks the accessibility server for the element actually painted at + // each point and reliably materializes the WebKit subtree without DOM IPC. + var roots = [app] + guard let (_, bounds) = try? windowInfo(pid: pid) else { return roots } + let system = AXUIElementCreateSystemWide() + for xFraction in stride(from: 0.05, through: 0.95, by: 0.1) { + for yFraction in stride(from: 0.05, through: 0.95, by: 0.1) { + var element: AXUIElement? + let x = Float(bounds.minX + bounds.width * xFraction) + let y = Float(bounds.minY + bounds.height * yFraction) + if AXUIElementCopyElementAtPosition(system, x, y, &element) == .success, + let element { + roots.append(element) + } + } + } + var seen = Set() + return roots.filter { seen.insert(CFHash($0)).inserted } +} + +func find(_ roots: [AXUIElement], locator: Locator, maxNodes: Int = 20_000) -> AXUIElement? { + var queue = roots + var seen = Set() + var visited = 0 + while !queue.isEmpty && visited < maxNodes { + let current = queue.removeFirst() + guard seen.insert(CFHash(current)).inserted else { continue } + visited += 1 + if matches(current, locator: locator) { return current } + queue.append(contentsOf: children(current)) + } + return nil +} + +func describe(_ element: AXUIElement, locator: Locator) -> ElementDescription { + ElementDescription(locator: locator, role: normalizedRole(elementRole(element)), name: elementName(element), + identifier: elementIdentifier(element), value: stringAttribute(element, [kAXValueAttribute]), + scrollY: nil, enabled: boolAttribute(element, kAXEnabledAttribute, default: true), + focused: boolAttribute(element, kAXFocusedAttribute), frame: frameAttribute(element)) +} + +func semanticNodes(path: String, pid: pid_t) -> [SemanticNode] { + guard let data = FileManager.default.contents(atPath: path), + let nodes = try? JSONDecoder().decode([SemanticNode].self, from: data) else { return [] } + guard let (_, windowBounds) = try? windowInfo(pid: pid) else { return nodes } + return nodes.map { node in + guard node.viewport.width > 0, node.viewport.height > 0 else { return node } + let xScale = windowBounds.width / node.viewport.width + let yScale = windowBounds.height / node.viewport.height + return SemanticNode( + id: node.id, + role: node.role, + name: node.name, + value: node.value, + scrollY: node.scrollY, + enabled: node.enabled, + focused: node.focused, + frame: Rect( + x: windowBounds.minX + node.frame.x * xScale, + y: windowBounds.minY + node.frame.y * yScale, + width: node.frame.width * xScale, + height: node.frame.height * yScale + ), + viewport: node.viewport + ) + } +} + +func matches(_ element: SemanticNode, locator: Locator) -> Bool { + if let id = locator.id, element.id != id { return false } + if let role = locator.role, normalizedRole(element.role) != normalizedRole(role) { return false } + if let name = locator.name, element.name != name { return false } + return true +} + +func describe(_ element: SemanticNode, locator: Locator) -> ElementDescription { + ElementDescription(locator: locator, role: normalizedRole(element.role), name: element.name, + identifier: element.id, value: element.value, scrollY: element.scrollY, + enabled: element.enabled, focused: element.focused, frame: element.frame) +} + +func snapshot(_ element: AXUIElement, depth: Int = 0, budget: inout Int) -> AXNode { + budget -= 1 + let isTruncated = budget <= 0 || depth >= 80 + let descendants = isTruncated ? [] : children(element).map { snapshot($0, depth: depth + 1, budget: &budget) } + return AXNode(role: normalizedRole(elementRole(element)), name: elementName(element), identifier: elementIdentifier(element), + value: stringAttribute(element, [kAXValueAttribute]), enabled: attribute(element, kAXEnabledAttribute) as? Bool, + focused: attribute(element, kAXFocusedAttribute) as? Bool, frame: frameAttribute(element), + children: descendants, truncated: isTruncated ? true : nil) +} + +func windowInfo(pid: pid_t) throws -> (CGWindowID, CGRect) { + guard let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + throw DriverError.message("cannot enumerate windows; Screen Recording permission may be missing") + } + let candidates = windows.compactMap { window -> (CGWindowID, CGRect)? in + guard (window[kCGWindowOwnerPID as String] as? Int32) == pid, + let number = window[kCGWindowNumber as String] as? CGWindowID, + let boundsValue = window[kCGWindowBounds as String], + CFGetTypeID(boundsValue as CFTypeRef) == CFDictionaryGetTypeID(), + let bounds = CGRect(dictionaryRepresentation: boundsValue as! CFDictionary), + bounds.width > 0, bounds.height > 0 else { return nil } + return (number, bounds) + } + if let largest = candidates.max(by: { $0.1.width * $0.1.height < $1.1.width * $1.1.height }) { + return largest + } + throw DriverError.message("no on-screen window found for pid \(pid)") +} + +func windowStatus(pid: pid_t) -> [String: Any] { + do { + let (windowID, bounds) = try windowInfo(pid: pid) + return [ + "ok": true, + "visible": true, + "window_id": windowID, + "bounds": ["x": bounds.origin.x, "y": bounds.origin.y, "width": bounds.width, "height": bounds.height], + ] + } catch { + return ["ok": true, "visible": false, "detail": String(describing: error)] + } +} + +func captureWindow(pid: pid_t, path: String) throws { + let (windowID, bounds) = try windowInfo(pid: pid) + guard let image = CGWindowListCreateImage(bounds, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution]) else { + throw DriverError.message("window screenshot failed") + } + let destination = URL(fileURLWithPath: path) + try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + guard let bitmap = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]) else { + throw DriverError.message("PNG encoding failed") + } + try bitmap.write(to: destination, options: .atomic) +} + +final class WindowRecorder: @unchecked Sendable { + private var writer: AVAssetWriter? + private var input: AVAssetWriterInput? + private var adaptor: AVAssetWriterInputPixelBufferAdaptor? + private var captureTask: Task? + private var captureError: Error? + + private func trace(_ message: String) { + let timestamp = ISO8601DateFormatter().string(from: Date()) + FileHandle.standardError.write(Data("[\(timestamp)] record: \(message)\n".utf8)) + } + + func start(pid: pid_t, path: String) throws { + let (windowID, bounds) = try windowInfo(pid: pid) + let width = max(Int(bounds.width) & ~1, 2) + let height = max(Int(bounds.height) & ~1, 2) + let destination = URL(fileURLWithPath: path) + try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: destination) + + let assetWriter = try AVAssetWriter(outputURL: destination, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + ] + let writerInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + writerInput.expectsMediaDataInRealTime = true + let attributes: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + kCVPixelBufferCGImageCompatibilityKey as String: true, + kCVPixelBufferCGBitmapContextCompatibilityKey as String: true, + ] + let pixelAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: writerInput, sourcePixelBufferAttributes: attributes) + guard assetWriter.canAdd(writerInput) else { throw DriverError.message("AVAssetWriter rejected window video input") } + assetWriter.add(writerInput) + guard assetWriter.startWriting() else { throw assetWriter.error ?? DriverError.message("AVAssetWriter failed to start") } + assetWriter.startSession(atSourceTime: .zero) + writer = assetWriter; input = writerInput; adaptor = pixelAdaptor + trace("started CGWindow/AVAssetWriter backend window=\(windowID) size=\(width)x\(height)") + + captureTask = Task.detached { [weak self] in + guard let self else { return } + var schedule = CaptureSchedule() + while !Task.isCancelled { + let tick = schedule.advance(isReadyForMoreMediaData: writerInput.isReadyForMoreMediaData) + autoreleasepool { + guard tick.shouldCapture, + let image = CGWindowListCreateImage(bounds, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution]), + let pool = pixelAdaptor.pixelBufferPool else { return } + var optionalBuffer: CVPixelBuffer? + guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &optionalBuffer) == kCVReturnSuccess, + let buffer = optionalBuffer else { return } + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + guard let base = CVPixelBufferGetBaseAddress(buffer), + let context = CGContext(data: base, width: width, height: height, + bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else { return } + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + if !pixelAdaptor.append(buffer, withPresentationTime: tick.presentationTime) { + self.captureError = assetWriter.error ?? DriverError.message("failed to append window video frame") + } + } + try? await Task.sleep(until: tick.deadline) + } + } + } + + func stop() async throws { + captureTask?.cancel() + _ = await captureTask?.result + captureTask = nil + input?.markAsFinished() + if let assetWriter = writer { await assetWriter.finishWriting() } + let error = captureError ?? writer?.error + writer = nil; input = nil; adaptor = nil; captureError = nil + if let error { throw error } + trace("finalized recording") + } +} + +func jsonObject(_ data: Data) throws -> [String: Any] { + guard let value = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw DriverError.message("request must be a JSON object") + } + return value +} + +func locator(from value: Any) throws -> Locator { + let data = try JSONSerialization.data(withJSONObject: value) + return try JSONDecoder().decode(Locator.self, from: data) +} + +func response(_ object: [String: Any]) { + let data = try! JSONSerialization.data(withJSONObject: object) + print(String(data: data, encoding: .utf8)!) + fflush(stdout) +} + +func encoded(_ value: T) throws -> Any { + let data = try JSONEncoder().encode(value) + return try JSONSerialization.jsonObject(with: data) +} + +func keyCode(_ key: String) throws -> CGKeyCode { + switch key.lowercased() { + case "tab": return 48 + case "return", "enter": return 36 + case "escape": return 53 + case "space": return 49 + case "k": return 40 + default: throw DriverError.message("unsupported key: \(key)") + } +} + +func modifierFlags(_ names: [String]) throws -> CGEventFlags { + try names.reduce(into: CGEventFlags()) { flags, name in + switch name.lowercased() { + case "command", "cmd": flags.insert(.maskCommand) + case "control", "ctrl": flags.insert(.maskControl) + case "option", "alt": flags.insert(.maskAlternate) + case "shift": flags.insert(.maskShift) + default: throw DriverError.message("unsupported modifier: \(name)") + } + } +} + +func postKey(_ key: String, modifiers: [String] = []) throws { + let code = try keyCode(key) + let flags = try modifierFlags(modifiers) + for keyDown in [true, false] { + guard let event = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: keyDown) else { + throw DriverError.message("could not create keyboard event") + } + event.flags = flags + event.post(tap: .cghidEventTap) + } +} + +func postUnicodeScalar(_ scalar: Unicode.Scalar) throws { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { + throw DriverError.message("could not create text event") + } + let units = Array(String(scalar).utf16) + units.withUnsafeBufferPointer { buffer in + down.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress) + up.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress) + } + down.flags = [] + up.flags = [] + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) +} + +func postText(_ text: String) throws { + for scalar in text.unicodeScalars { + if scalar == "\n" { + // Buzz submits the composer on plain Enter. A newline in typed text + // therefore has to follow the same native path as a user: Shift+Enter. + try postKey("return", modifiers: ["shift"]) + } else if scalar != "\r" { + try postUnicodeScalar(scalar) + } + } +} + +@main +struct BuzzNativeDriver { + static func main() async { + do { + let arguments = Array(CommandLine.arguments.dropFirst()) + guard let command = arguments.first else { throw DriverError.message("usage: buzz-native-driver doctor | serve --pid PID") } + if command == "doctor" { + let accessibility = AXIsProcessTrusted() + let screen = CGPreflightScreenCaptureAccess() + response(["ok": true, "checks": [ + ["name": "native-driver-build", "ok": true, "detail": "Swift AX/CGEvent/Core Graphics/AVFoundation driver available"], + ["name": "accessibility-permission", "ok": accessibility, "detail": accessibility ? "granted" : "grant Accessibility to the invoking terminal/agent"], + ["name": "screen-recording-permission", "ok": screen, "detail": screen ? "granted" : "grant Screen Recording to the invoking terminal/agent"], + ["name": "recording-api", "ok": ProcessInfo.processInfo.isOperatingSystemAtLeast(.init(majorVersion: 15, minorVersion: 0, patchVersion: 0)), "detail": ProcessInfo.processInfo.operatingSystemVersionString] + ]]) + return + } + guard command == "serve", (arguments.count == 3 || arguments.count == 5), arguments[1] == "--pid", let pid = pid_t(arguments[2]) else { + throw DriverError.message("usage: buzz-native-driver serve --pid PID [--semantic-snapshot PATH]") + } + let semanticSnapshotPath: String? = arguments.count == 5 && arguments[3] == "--semantic-snapshot" ? arguments[4] : nil + guard AXIsProcessTrusted() else { throw DriverError.message("Accessibility permission is not granted") } + let app = AXUIElementCreateApplication(pid) + enableWebViewAccessibility(app) + var selected: (ElementDescription, Locator)? + var recorder: Any? + for try await line in FileHandle.standardInput.bytes.lines { + do { + let request = try jsonObject(Data(line.utf8)) + guard let name = request["command"] as? String else { throw DriverError.message("missing command") } + switch name { + case "locate": + guard let values = request["locators"] as? [Any] else { throw DriverError.message("locate requires locators") } + var found: (ElementDescription, Locator)? + for value in values { + let candidate = try locator(from: value) + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: candidate) }) { + found = (describe(element, locator: candidate), candidate) + break + } + let semanticActive = semanticSnapshotPath.map { FileManager.default.fileExists(atPath: $0) } ?? false + if !semanticActive || candidate.role == "window" { + if let element = find(accessibilityRoots(app: app, pid: pid), locator: candidate) { + found = (describe(element, locator: candidate), candidate) + break + } + } + } + selected = found + if let (element, _) = found { + response(["ok": true, "element": try encoded(element)]) + } else if (request["required"] as? Bool) == true { + throw DriverError.message("no accessibility element matched ordered locators") + } else { response(["ok": true, "element": NSNull()]) } + case "act": + guard let action = request["action"] as? [String: Any], let type = action["type"] as? String else { throw DriverError.message("act requires action.type") } + if type == "activate" { + guard let running = NSRunningApplication(processIdentifier: pid) else { throw DriverError.message("target app is no longer running") } + running.activate(options: [.activateIgnoringOtherApps]) + } else if type == "wait" { + try await Task.sleep(for: .milliseconds(action["duration_ms"] as? Int ?? 0)) + } else if type == "press" { + try postKey(action["key"] as? String ?? "", modifiers: action["modifiers"] as? [String] ?? []) + } else if type == "type_text" { + try postText(action["text"] as? String ?? "") + } else { + guard let (_, used) = selected else { + throw DriverError.message("action requires a freshly selected element with current bounds") + } + var fresh: ElementDescription? + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: used) }) { + fresh = describe(element, locator: used) + } + if fresh == nil, let element = find(accessibilityRoots(app: app, pid: pid), locator: used) { + fresh = describe(element, locator: used) + } + guard let element = fresh, let frame = element.frame else { + throw DriverError.message("action requires a freshly selected element with current bounds") + } + selected = (element, used) + let point = CGPoint(x: frame.x + frame.width / 2, y: frame.y + frame.height / 2) + let duration = max(action["duration_ms"] as? Int ?? 0, 0) + if type == "move_pointer" && duration > 0 { + let current = NSEvent.mouseLocation + let start = CGPoint(x: current.x, y: NSScreen.screens.first.map { $0.frame.height - current.y } ?? current.y) + let steps = max(duration / 8, 2) + for index in 1...steps { + let fraction = Double(index) / Double(steps) + let next = CGPoint(x: start.x + (point.x - start.x) * fraction, y: start.y + (point.y - start.y) * fraction) + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: next, mouseButton: .left)?.post(tap: .cghidEventTap) + try await Task.sleep(for: .milliseconds(duration / steps)) + } + } else if type == "move_pointer" { + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + } else if type == "click" { + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + } else if type == "scroll" { + let deltaY = Int32(action["delta_y"] as? Int ?? 0) + let targetFrame = CGRect(x: frame.x, y: frame.y, width: frame.width, height: frame.height) + guard let event = makeScrollEvent(deltaY: deltaY, targetFrame: targetFrame) else { + throw DriverError.message("could not create scroll event") + } + event.post(tap: .cghidEventTap) + } else { throw DriverError.message("unsupported action: \(type)") } + } + response(["ok": true]) + case "window_status": + response(windowStatus(pid: pid)) + case "snapshot": + var budget = 20_000 + var object: [String: Any] = ["ok": true, "tree": try encoded(snapshot(app, budget: &budget))] + if let path = semanticSnapshotPath { + object["semantic"] = try encoded(semanticNodes(path: path, pid: pid)) + } + response(object) + case "screenshot": + guard let path = request["path"] as? String else { throw DriverError.message("screenshot requires path") } + try captureWindow(pid: pid, path: path); response(["ok": true]) + case "focused": + response(["ok": true, "focused": selected?.0.focused ?? false]) + case "selected": + if let (_, used) = selected { + var fresh: ElementDescription? + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: used) }) { + fresh = describe(element, locator: used) + } + if fresh == nil, let element = find(accessibilityRoots(app: app, pid: pid), locator: used) { + fresh = describe(element, locator: used) + } + if let element = fresh { + selected = (element, used) + response(["ok": true, "element": try encoded(element)]) + } else { + selected = nil + response(["ok": true, "element": NSNull()]) + } + } else { response(["ok": true, "element": NSNull()]) } + case "record_start": + guard let path = request["path"] as? String else { throw DriverError.message("record_start requires path") } + let value = WindowRecorder(); try value.start(pid: pid, path: path); recorder = value; response(["ok": true]) + case "record_stop": + if let value = recorder as? WindowRecorder { try await value.stop() } + recorder = nil; response(["ok": true]) + case "shutdown": response(["ok": true]); return + default: throw DriverError.message("unknown command: \(name)") + } + } catch { + response(["ok": false, "error": String(describing: error)]) + } + } + } catch { + response(["ok": false, "error": String(describing: error)]) + exit(1) + } + } +} diff --git a/tools/native-review/swift/Sources/BuzzNativeDriverSupport/DriverSupport.swift b/tools/native-review/swift/Sources/BuzzNativeDriverSupport/DriverSupport.swift new file mode 100644 index 00000000000..2697e678f11 --- /dev/null +++ b/tools/native-review/swift/Sources/BuzzNativeDriverSupport/DriverSupport.swift @@ -0,0 +1,43 @@ +import CoreGraphics +import CoreMedia +import Foundation + +public struct CaptureTick { + public let shouldCapture: Bool + public let presentationTime: CMTime + public let deadline: ContinuousClock.Instant +} + +public struct CaptureSchedule { + private let start: ContinuousClock.Instant + private let framesPerSecond: Int64 + public private(set) var frame: Int64 = 0 + + public init(start: ContinuousClock.Instant = .now, framesPerSecond: Int64 = 15) { + self.start = start + self.framesPerSecond = framesPerSecond + } + + public mutating func advance(isReadyForMoreMediaData: Bool) -> CaptureTick { + let tick = CaptureTick( + shouldCapture: isReadyForMoreMediaData, + presentationTime: CMTime(value: frame, timescale: Int32(framesPerSecond)), + deadline: start.advanced(by: .milliseconds(Int((frame + 1) * 1000 / framesPerSecond))) + ) + frame += 1 + return tick + } +} + +public func makeScrollEvent(deltaY: Int32, targetFrame: CGRect) -> CGEvent? { + guard let event = CGEvent( + scrollWheelEvent2Source: nil, + units: .pixel, + wheelCount: 1, + wheel1: deltaY, + wheel2: 0, + wheel3: 0 + ) else { return nil } + event.location = CGPoint(x: targetFrame.midX, y: targetFrame.midY) + return event +} diff --git a/tools/native-review/swift/Tests/BuzzNativeDriverSupportTests/DriverSupportTests.swift b/tools/native-review/swift/Tests/BuzzNativeDriverSupportTests/DriverSupportTests.swift new file mode 100644 index 00000000000..4cb54a7d6df --- /dev/null +++ b/tools/native-review/swift/Tests/BuzzNativeDriverSupportTests/DriverSupportTests.swift @@ -0,0 +1,32 @@ +import CoreGraphics +import Testing +@testable import BuzzNativeDriverSupport + +@Test func captureScheduleAdvancesAcrossBackpressure() { + let start = ContinuousClock.now + var schedule = CaptureSchedule(start: start, framesPerSecond: 15) + + let blockedTicks = (0..<4).map { _ in schedule.advance(isReadyForMoreMediaData: false) } + + #expect(blockedTicks.allSatisfy { !$0.shouldCapture }) + #expect(blockedTicks.map(\.presentationTime.value) == [0, 1, 2, 3]) + #expect(blockedTicks.map(\.deadline) == [ + start.advanced(by: .milliseconds(66)), + start.advanced(by: .milliseconds(133)), + start.advanced(by: .milliseconds(200)), + start.advanced(by: .milliseconds(266)), + ]) + #expect(schedule.frame == 4) +} + +@Test func captureScheduleMarksReadyTickForCapture() { + var schedule = CaptureSchedule() + #expect(schedule.advance(isReadyForMoreMediaData: true).shouldCapture) +} + +@Test func scrollEventUsesTargetFrameCenter() throws { + let frame = CGRect(x: 40, y: 80, width: 120, height: 60) + let event = try #require(makeScrollEvent(deltaY: 240, targetFrame: frame)) + + #expect(event.location == CGPoint(x: 100, y: 110)) +} diff --git a/tools/native-review/tests/fixtures/broken-scroll.yaml b/tools/native-review/tests/fixtures/broken-scroll.yaml new file mode 100644 index 00000000000..4eb318cb626 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-scroll.yaml @@ -0,0 +1,34 @@ +schema_version: 1 +flow: broken_scroll +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: {not_exists: {id: app-loading-gate}} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + act: {type: click} + expect: {exists: {id: message-composer}} + timeout_ms: 15000 + - name: focus_composer + locate: + - {id: message-input} + act: {type: click} + expect: {focused: {id: message-input}} + - name: create_scrollable_draft + act: {type: type_text, text: "line 1\nline 2\nline 3\nline 4\nline 5\nline 6\nline 7\nline 8\nline 9\nline 10\nline 11\nline 12\nline 13\nline 14\nline 15"} + expect: {value: "line 1\nline 2\nline 3\nline 4\nline 5\nline 6\nline 7\nline 8\nline 9\nline 10\nline 11\nline 12\nline 13\nline 14\nline 15"} + - name: wrong_scroll_postcondition + locate: + - {id: message-input-scroll} + act: {type: scroll, delta_y: 240} + # The draft starts at the caret-driven bottom (~172px). Scrolling down moves + # it to 0, so this impossible >200 assertion cannot pass on either the + # pre-action snapshot or the settled result. + expect: {scroll_y_greater_than: 200} + timeout_ms: 100 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/fixtures/broken-shortcut.yaml b/tools/native-review/tests/fixtures/broken-shortcut.yaml new file mode 100644 index 00000000000..6e097ac0ae6 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-shortcut.yaml @@ -0,0 +1,11 @@ +schema_version: 1 +flow: broken_shortcut +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: wrong_shortcut + act: {type: press, key: k} + expect: {exists: {id: search-results}} + timeout_ms: 100 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/fixtures/broken-text.yaml b/tools/native-review/tests/fixtures/broken-text.yaml new file mode 100644 index 00000000000..011b928ee42 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-text.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: broken_text +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: impossible_text_postcondition + locate: + - {id: open-search} + act: {type: click} + expect: {value: "this can never match the selected button"} + timeout_ms: 50 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/fixtures/broken-tooltip.yaml b/tools/native-review/tests/fixtures/broken-tooltip.yaml new file mode 100644 index 00000000000..4631454f729 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-tooltip.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: broken_tooltip +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: impossible_locator + locate: [{id: this-element-must-not-exist}] + act: {type: click} + expect: + exists: {id: still-impossible} + timeout_ms: 50 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/test_evidence_bundle.py b/tools/native-review/tests/test_evidence_bundle.py new file mode 100644 index 00000000000..f896609e685 --- /dev/null +++ b/tools/native-review/tests/test_evidence_bundle.py @@ -0,0 +1,177 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).parents[1] / "evidence_bundle.py" +SPEC = importlib.util.spec_from_file_location("evidence_bundle", MODULE_PATH) +evidence = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(evidence) + + +class EvidenceBundleTests(unittest.TestCase): + def test_relay_safe_video_uses_canonical_privacy_profile(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "source.mov" + destination = root / "finding.mp4" + source.write_bytes(b"input") + + def completed(command, **_kwargs): + pathlib.Path(command[-1]).write_bytes(b"canonical") + return mock.Mock(returncode=0, stderr="") + + with mock.patch.object(evidence.shutil, "which", return_value="/usr/bin/ffmpeg"), \ + mock.patch.object(evidence.subprocess, "run", side_effect=completed) as invoked: + evidence.relay_safe_video(source, destination, start=1.5, duration=4.0) + command = invoked.call_args.args[0] + self.assertIn("-map_metadata", command) + self.assertEqual(command[command.index("-ss") + 1], "1.5") + self.assertEqual(command[command.index("-t") + 1], "4.0") + self.assertIn("-map_chapters", command) + self.assertIn("-an", command) + self.assertIn("+faststart", command) + self.assertIn("+bitexact", command) + self.assertTrue(destination.is_file()) + + def test_bundle_emits_provenance_hashes_and_redacted_focused_log(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "video.mp4").write_bytes(b"source") + (root / "flutter.log").write_text( + "before\nBUZZ_PRIVATE_KEY=deadbeef\nBad state: Cannot clone a disposed image\nafter\nunrelated\n" + ) + receipt = { + "status": "failed", + "failure": "Authorization: Bearer failure-secret", + "provenance": {"head_sha": "abc123", "dirty": False}, + "artifacts": {"video": "video.mp4", "log": "flutter.log"}, + "cleanup": {"status": "passed", "errors": [{"token": "nested-secret"}]}, + "isolation": {"secret_path": "/private/path"}, + } + receipt_path = root / "receipt.json" + receipt_path.write_text(json.dumps(receipt)) + output = root / "bundle" + + def finalize(_source, destination, **_kwargs): + destination.write_bytes(b"canonical-video") + return destination + + with mock.patch.object(evidence, "relay_safe_video", side_effect=finalize): + manifest = evidence.finding_bundle( + receipt_path, output, match="disposed image", context=1 + ) + excerpt = (output / "log-excerpt.txt").read_text() + self.assertIn("Cannot clone a disposed image", excerpt) + self.assertIn("BUZZ_PRIVATE_KEY=[REDACTED]", excerpt) + self.assertNotIn("deadbeef", excerpt) + self.assertNotIn("unrelated", excerpt) + self.assertNotIn("isolation", json.loads((output / "receipt.json").read_text())) + bundle_receipt = json.loads((output / "receipt.json").read_text()) + self.assertNotIn("failure-secret", json.dumps(bundle_receipt)) + self.assertNotIn("nested-secret", json.dumps(bundle_receipt)) + self.assertEqual(bundle_receipt["failure"], "Authorization: [REDACTED]") + self.assertEqual(bundle_receipt["cleanup"]["errors"][0]["token"], "[REDACTED]") + self.assertEqual(manifest["head_sha"], "abc123") + self.assertEqual(manifest["status"], "failed") + self.assertEqual(manifest["cleanup"], "passed") + self.assertEqual(set(manifest["files"]), {"finding.mp4", "receipt.json", "log-excerpt.txt"}) + + def test_redacts_header_and_json_secret_forms(self): + sources = { + 'Authorization: Bearer abc123\n{"token": "json-secret", "safe": "visible"}': + 'Authorization: [REDACTED]\n{"token": "[REDACTED]", "safe": "visible"}', + 'Authorization: token ghp_supersecret': + 'Authorization: [REDACTED]', + 'Authorization: Digest username="user", response="secret"': + 'Authorization: [REDACTED]', + 'Proxy-Authorization: Custom opaque credential with spaces': + 'Proxy-Authorization: [REDACTED]', + ' Authorization: token ghp_indented': + ' Authorization: [REDACTED]', + '{"message":"Authorization: token ghp_embedded", "safe":"visible"}': + '{"message":"Authorization: [REDACTED]', + '{"message":"Authorization: Digest username=alice, response=secret"}': + '{"message":"Authorization: [REDACTED]', + '{"message":"Authorization: Negotiate opaque-secret"}': + '{"message":"Authorization: [REDACTED]', + 'prefix Authorization: Custom opaque-secret suffix': + 'prefix Authorization: [REDACTED]', + '{"authorization": "Bearer supersecret"}': + '{"authorization": "[REDACTED]"}', + "{'authorization': 'Bearer supersecret'}": + "{'authorization': '[REDACTED]'}", + "AUTHORIZATION=Bearer supersecret": + "AUTHORIZATION=[REDACTED]", + "AUTHORIZATION=Negotiate opaque-secret": + "AUTHORIZATION=[REDACTED]", + "PROXY_AUTHORIZATION=Custom opaque-secret": + "PROXY_AUTHORIZATION=[REDACTED]", + "Authorization: Negotiate first-part\n second-secret": + "Authorization: [REDACTED]", + "Proxy-Authorization: Custom first-part\r\n\tsecond-secret": + "Proxy-Authorization: [REDACTED]", + "Cookie: session=secret; refresh=supersecret": + "Cookie: [REDACTED]", + "COOKIE=session-secret refresh-secret": + "COOKIE=[REDACTED]", + "PASSWORD=correct horse battery staple": + "PASSWORD=[REDACTED]", + "token=alpha beta gamma": + "token=[REDACTED]", + "X-Api-Key: alpha beta": + "X-Api-Key: [REDACTED]", + "api_key=alpha; beta gamma": + "api_key=[REDACTED]", + "password=first-part\n second-part\nnext=visible": + "password=[REDACTED]\nnext=visible", + 'prefix token=alpha beta; gamma': + 'prefix token=[REDACTED]', + '{"token": "prefix\\\"tail-secret", "safe": "visible"}': + '{"token": "[REDACTED]", "safe": "visible"}', + '{"api_key": "prefix\\\\\\\"tail-secret", "safe": "visible"}': + '{"api_key": "[REDACTED]", "safe": "visible"}', + "{'password': 'prefix\\'tail-secret', 'safe': 'visible'}": + "{'password': '[REDACTED]', 'safe': 'visible'}", + 'token="prefix\\\"tail-secret"\nnext=visible': + 'token="[REDACTED]"\nnext=visible', + } + for source, expected in sources.items(): + with self.subTest(source=source): + self.assertEqual(evidence.redact_log(source), expected) + + def test_non_finite_clip_bounds_fail_before_ffmpeg(self): + with tempfile.TemporaryDirectory() as directory: + source = pathlib.Path(directory) / "source.mp4" + source.write_bytes(b"video") + for start, duration in ((float("nan"), None), (float("inf"), None), (None, float("nan"))): + with self.subTest(start=start, duration=duration), \ + mock.patch.object(evidence.shutil, "which") as which: + with self.assertRaisesRegex(evidence.EvidenceError, "finite"): + evidence.relay_safe_video(source, source.with_name("out.mp4"), start=start, duration=duration) + which.assert_not_called() + + def test_bundle_refuses_missing_match_and_removes_partial_output(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "video.mp4").write_bytes(b"source") + (root / "flutter.log").write_text("ordinary output\n") + receipt_path = root / "receipt.json" + receipt_path.write_text(json.dumps({ + "provenance": {}, "artifacts": {"video": "video.mp4", "log": "flutter.log"} + })) + output = root / "bundle" + with mock.patch.object(evidence, "relay_safe_video", side_effect=lambda _s, d, **_k: d.write_bytes(b"v") or d): + with self.assertRaisesRegex(evidence.EvidenceError, "found no lines"): + evidence.finding_bundle(receipt_path, output, match="missing") + self.assertFalse(output.exists()) + + def test_existing_output_is_not_overwritten(self): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "bundle" + output.mkdir() + with self.assertRaisesRegex(evidence.EvidenceError, "already exists"): + evidence.finding_bundle(pathlib.Path(directory) / "receipt.json", output) diff --git a/tools/native-review/tests/test_ios_review.py b/tools/native-review/tests/test_ios_review.py new file mode 100644 index 00000000000..ca4a4444e56 --- /dev/null +++ b/tools/native-review/tests/test_ios_review.py @@ -0,0 +1,174 @@ +import importlib.util +import json +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).parents[1] / "ios_review.py" +SPEC = importlib.util.spec_from_file_location("ios_review", MODULE_PATH) +ios_review = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(ios_review) + + +class FakeRecorder: + def __init__(self): + self.stderr = self + self.finalized = False + + def readline(self): + return "Recording started\n" + + def poll(self): + return 0 if self.finalized else None + + def send_signal(self, _signal): + self.finalized = True + + def wait(self, timeout): + return 0 + + def kill(self): + self.finalized = True + + +class IosReviewTests(unittest.TestCase): + DEVICE_PAYLOAD = { + "devicetypes": [{"name": "iPhone Test", "identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-Test"}], + "runtimes": [ + {"identifier": "com.apple.CoreSimulator.SimRuntime.iOS-9-3", "version": "9.3", + "platform": "iOS", "isAvailable": True, "supportedDeviceTypes": [ + {"identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-Test"}, + ]}, + {"identifier": "com.apple.CoreSimulator.SimRuntime.iOS-26-0", "version": "26.0", + "platform": "iOS", "isAvailable": True, "supportedDeviceTypes": [ + {"identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-Test"}, + ]}, + ], + } + + def test_device_creation_uses_unique_owned_name_and_latest_runtime(self): + responses = [ + subprocess.CompletedProcess([], 0, json.dumps(self.DEVICE_PAYLOAD), ""), + subprocess.CompletedProcess([], 0, "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE\n", ""), + ] + with mock.patch.object(ios_review, "run", side_effect=responses) as run: + device = ios_review.create_review_device("iPhone Test", "ios-run-123") + create = run.call_args_list[1].args[0] + self.assertEqual(create[:3], ["xcrun", "simctl", "create"]) + self.assertEqual(create[3], "Buzz Native Review ios-run-123") + self.assertEqual(create[-1], "com.apple.CoreSimulator.SimRuntime.iOS-26-0") + self.assertTrue(device["owned"]) + self.assertEqual(device["udid"], "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE") + + def test_missing_device_type_fails_without_creating_or_erasing_any_device(self): + completed = subprocess.CompletedProcess([], 0, json.dumps({"devicetypes": [], "runtimes": []}), "") + with mock.patch.object(ios_review, "run", return_value=completed) as run: + with self.assertRaisesRegex(ios_review.ReviewError, "no iOS Simulator device type"): + ios_review.create_review_device("Absent", "run") + commands = [call.args[0] for call in run.call_args_list] + self.assertFalse(any("erase" in command or "delete" in command for command in commands)) + + def test_flutter_environment_scrubs_host_credentials(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", "GITHUB_TOKEN": "secret"} + with mock.patch.dict(ios_review.os.environ, sentinels, clear=False): + environment = ios_review.flutter_environment() + for name in sentinels: + self.assertNotIn(name, environment) + + def test_run_scrubs_host_credentials_by_default(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", "GITHUB_TOKEN": "secret"} + with mock.patch.dict(ios_review.os.environ, sentinels, clear=False), \ + mock.patch.object(ios_review.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run: + ios_review.run(["xcrun", "simctl", "list"]) + environment = run.call_args.kwargs["env"] + for name in sentinels: + self.assertNotIn(name, environment) + + def test_flutter_failure_finalizes_recording_writes_receipt_and_cleans_device(self): + recorder = FakeRecorder() + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + if command[:2] == ["flutter", "drive"]: + return subprocess.CompletedProcess(command, 1, "journey failed", "diagnostic") + if "screenshot" in command: + pathlib.Path(command[-1]).write_bytes(b"png") + return subprocess.CompletedProcess(command, 0, "", "") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.sys, "platform", "darwin"), \ + mock.patch.object(ios_review.shutil, "which", return_value="/tool"), \ + mock.patch.object(ios_review, "provenance", return_value={"head_sha": "a" * 40, "dirty": False, "status": []}), \ + mock.patch.object(ios_review, "git", return_value="a" * 12), \ + mock.patch.object(ios_review, "create_review_device", return_value={ + "name": "Buzz Native Review owned-run", "udid": "owned-device", + "runtimeIdentifier": "runtime", "deviceType": "iPhone Test", "owned": True, + }), \ + mock.patch.object(ios_review, "run", side_effect=fake_run), \ + mock.patch.object(ios_review, "wait_for_recording"), \ + mock.patch.object(ios_review.subprocess, "Popen", return_value=recorder): + with self.assertRaisesRegex(ios_review.ReviewError, "exit 1"): + ios_review.run_review(ios_review.DEFAULT_TEST, "iPhone Test", pathlib.Path(directory)) + receipts = list(pathlib.Path(directory).rglob("receipt.json")) + self.assertEqual(len(receipts), 1) + receipt = json.loads(receipts[0].read_text()) + + self.assertTrue(recorder.finalized) + self.assertEqual(receipt["status"], "failed") + self.assertEqual(receipt["cleanup"], {"status": "passed", "errors": []}) + self.assertEqual(receipt["artifacts"], { + "video": "video.mp4", "log": "flutter.log", "screenshot": "final.png" + }) + self.assertEqual(receipt["isolation"]["simulator"], { + "name": "Buzz Native Review owned-run", "device_type": "iPhone Test", + "udid": "owned-device", "runtime": "runtime", "owned": True, + }) + self.assertNotIn("device", receipt) + self.assertEqual(receipt["flow"], "ios_native_review_pairing_test") + self.assertEqual(receipt["steps"], []) + self.assertEqual(receipt["measurements"], {}) + self.assertIn("machine", receipt["performance"]) + self.assertIn("started_at", receipt) + self.assertIn("finished_at", receipt) + self.assertIn(["xcrun", "simctl", "shutdown", "owned-device"], commands) + self.assertIn(["xcrun", "simctl", "delete", "owned-device"], commands) + self.assertFalse(any("erase" in command for command in commands)) + self.assertTrue(all("pre-existing-device" not in command for command in commands)) + + def test_recorder_timeout_is_reported_and_device_is_cleaned(self): + recorder = FakeRecorder() + recorder.readline = mock.Mock(return_value="") + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, "", "") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.sys, "platform", "darwin"), \ + mock.patch.object(ios_review.shutil, "which", return_value="/tool"), \ + mock.patch.object(ios_review, "provenance", return_value={"head_sha": "a" * 40, "dirty": False, "status": []}), \ + mock.patch.object(ios_review, "git", return_value="a" * 12), \ + mock.patch.object(ios_review, "create_review_device", return_value={ + "name": "Buzz Native Review timeout-run", "udid": "owned-device", + "runtimeIdentifier": "runtime", "deviceType": "iPhone Test", "owned": True, + }), \ + mock.patch.object(ios_review, "run", side_effect=fake_run), \ + mock.patch.object(ios_review, "wait_for_recording", side_effect=ios_review.ReviewError("timed out waiting for Simulator recording")), \ + mock.patch.object(ios_review.subprocess, "Popen", return_value=recorder): + with self.assertRaisesRegex(ios_review.ReviewError, "timed out"): + ios_review.run_review(ios_review.DEFAULT_TEST, "iPhone Test", pathlib.Path(directory)) + receipt = json.loads(next(pathlib.Path(directory).rglob("receipt.json")).read_text()) + + self.assertTrue(recorder.finalized) + self.assertEqual(receipt["cleanup"]["status"], "passed") + self.assertIn(["xcrun", "simctl", "delete", "owned-device"], commands) + self.assertFalse(any("erase" in command for command in commands)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/native-review/tests/test_review_native.py b/tools/native-review/tests/test_review_native.py new file mode 100644 index 00000000000..acab055937a --- /dev/null +++ b/tools/native-review/tests/test_review_native.py @@ -0,0 +1,532 @@ +import importlib.util +import json +import pathlib +import re +import tempfile +import subprocess +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).parents[1] / "review_native.py" +SPEC = importlib.util.spec_from_file_location("review_native", MODULE_PATH) +review_native = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(review_native) + + +class JourneyTests(unittest.TestCase): + def test_real_journey_validates(self): + journey = review_native.load_journey(MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml") + self.assertEqual(journey["flow"], "tooltip_fresh_dwell") + + def test_broken_mutation_is_schema_valid(self): + journey = review_native.load_journey(MODULE_PATH.parent / "tests/fixtures/broken-tooltip.yaml") + self.assertEqual(journey["steps"][0]["timeout_ms"], 50) + + def test_broader_journeys_and_mutations_validate(self): + expected = { + "desktop/composer-keyboard.yaml": "composer_keyboard", + "desktop/search-shortcut-dismissal.yaml": "search_shortcut_dismissal", + "tests/fixtures/broken-text.yaml": "broken_text", + "tests/fixtures/broken-shortcut.yaml": "broken_shortcut", + "tests/fixtures/broken-scroll.yaml": "broken_scroll", + } + for relative_path, flow in expected.items(): + with self.subTest(relative_path=relative_path): + journey = review_native.load_journey(MODULE_PATH.parent / relative_path) + self.assertEqual(journey["flow"], flow) + + def test_duplicate_measurement_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace(" - name: leave_trigger\n", " - name: duplicate_measure\n act: {type: wait, duration_ms: 1}\n expect: {exists: {role: window}}\n measure: tooltip_open_latency\n - name: leave_trigger\n") + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "duplicate measurement"): + review_native.load_journey(path) + + def test_type_text_requires_text(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + source = re.sub(r'\{type: type_text, text: "[^"]*"\}', "{type: type_text}", source, count=1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "type_text requires text"): + review_native.load_journey(path) + + def test_scroll_requires_integer_delta(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + source = re.sub(r"\{type: scroll, delta_y: -?240\}", "{type: scroll, delta_y: nope}", source, count=1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "scroll requires integer delta_y"): + review_native.load_journey(path) + + def test_action_numeric_bounds_match_schema(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + mutations = ( + ("duration_ms: 100", "duration_ms: 30000", None), + ("duration_ms: 100", "duration_ms: 30001", "wait requires integer duration_ms in 1..30000"), + ("delta_y: 240", "delta_y: -10000", None), + ("delta_y: 240", "delta_y: 10000", None), + ("delta_y: 240", "delta_y: 10001", "scroll requires integer delta_y in -10000..10000"), + ("delta_y: 240", "delta_y: 2147483648", "scroll requires integer delta_y in -10000..10000"), + ) + for old, new, diagnostic in mutations: + with self.subTest(new=new), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "journey.yaml" + path.write_text(source.replace(old, new, 1)) + if diagnostic is None: + review_native.load_journey(path) + else: + with self.assertRaisesRegex(review_native.HarnessError, diagnostic): + review_native.load_journey(path) + + def test_expect_for_duration_bounds_match_schema(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + insertion = " expect_for:\n duration_ms: VALUE\n condition: {exists: {role: window}}\n timeout_ms: 1000\n" + for value, diagnostic in ((1, None), (30000, None), (30001, "expect_for.duration_ms")): + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "journey.yaml" + path.write_text(source.replace(" timeout_ms: 1000\n", insertion.replace("VALUE", str(value)), 1)) + if diagnostic is None: + review_native.load_journey(path) + else: + with self.assertRaisesRegex(review_native.HarnessError, diagnostic): + review_native.load_journey(path) + + def test_boolean_negative_and_vacuous_durations_are_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + mutations = ( + ("duration_ms: 100", "duration_ms: true", "wait requires integer duration_ms"), + ("duration_ms: 800", "duration_ms: -1", "wait requires integer duration_ms"), + ("duration_ms: 800", "duration_ms: 0", "wait requires integer duration_ms"), + ("duration_ms: 800", "", "wait requires integer duration_ms"), + (" timeout_ms: 1000\n", " expect_for:\n duration_ms: 0\n condition: {exists: {role: window}}\n timeout_ms: 1000\n", "expect_for.duration_ms"), + ("timeout_ms: 1000", "timeout_ms: true", "timeout_ms"), + ) + for old, new, diagnostic in mutations: + with self.subTest(new=new), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source.replace(old, new, 1)) + with self.assertRaisesRegex(review_native.HarnessError, diagnostic): + review_native.load_journey(path) + + def test_boolean_and_non_finite_numeric_expectations_are_rejected(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + for value in ("true", ".nan", ".inf"): + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source.replace("scroll_y_less_than: 1", f"scroll_y_less_than: {value}", 1)) + with self.assertRaisesRegex(review_native.HarnessError, "finite number"): + review_native.load_journey(path) + + def test_scroll_requires_locator(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + source = re.sub(r" locate:\n(?: - .*\n)+ act: \{type: scroll", " act: {type: scroll", source, count=1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "scroll requires locate"): + review_native.load_journey(path) + + def test_press_modifiers_match_schema_enum_and_uniqueness(self): + source = (MODULE_PATH.parent / "desktop/search-shortcut-dismissal.yaml").read_text() + mutations = ( + ("modifiers: [command]", "modifiers: [command, shift]", None), + ("modifiers: [command]", "modifiers: [bogus]", "press modifiers must be unique"), + ("modifiers: [command]", "modifiers: [command, command]", "press modifiers must be unique"), + ) + for old, new, diagnostic in mutations: + with self.subTest(new=new), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "journey.yaml" + path.write_text(source.replace(old, new, 1)) + if diagnostic is None: + review_native.load_journey(path) + else: + with self.assertRaisesRegex(review_native.HarnessError, diagnostic): + review_native.load_journey(path) + + def test_driver_timeout_includes_bounded_action_duration(self): + driver = object.__new__(review_native.Driver) + driver.process = mock.Mock() + driver.process.stdin = mock.Mock() + driver.process.stdout = mock.Mock() + driver.process.stdout.readline.return_value = '{"ok": true}\n' + driver.process.stderr = mock.Mock() + with mock.patch.object(review_native.select, "select", return_value=([driver.process.stdout], [], [])) as selected: + driver.request("act", action={"type": "wait", "duration_ms": 30000}) + self.assertEqual(selected.call_args.args[3], 35) + + def test_value_expectation_uses_selected_element(self): + driver = mock.Mock() + driver.request.return_value = {"ok": True, "element": {"value": "draft"}} + self.assertTrue(review_native.expectation_holds(driver, {"value": "draft"})) + self.assertFalse(review_native.expectation_holds(driver, {"value": "wrong"})) + + def test_scroll_expectation_uses_selected_element(self): + driver = mock.Mock() + driver.request.return_value = {"ok": True, "element": {"scrollY": 240}} + self.assertTrue(review_native.expectation_holds(driver, {"scroll_y_greater_than": 0})) + self.assertFalse(review_native.expectation_holds(driver, {"scroll_y_greater_than": 240})) + self.assertTrue(review_native.expectation_holds(driver, {"scroll_y_less_than": 241})) + self.assertFalse(review_native.expectation_holds(driver, {"scroll_y_less_than": 240})) + + def test_action_without_postcondition_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace(" expect:\n exists: {role: window}\n", "", 1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "requires name/act/expect"): + review_native.load_journey(path) + + def test_expectation_with_multiple_conditions_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace( + " expect:\n exists: {role: window}\n", + " expect:\n exists: {role: window}\n enabled: true\n", + 1, + ) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "exactly one"): + review_native.load_journey(path) + + def test_production_and_remote_targets_are_rejected(self): + with self.assertRaisesRegex(review_native.HarnessError, "non-loopback"): + review_native.isolation_manifest("run", "wss://buzz.block.builderlab.xyz") + safe = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + self.assertTrue(safe["bundle_id"].startswith("xyz.block.buzz.app.dev.native-review.")) + self.assertNotIn(safe["bundle_id"], review_native.PRODUCTION_BUNDLE_IDS) + + def test_secret_environment_is_scrubbed(self): + sentinels = { + "BUZZ_PRIVATE_KEY": "must-not-survive", + "SSH_AUTH_SOCK": "/tmp/credential-agent", + "GITHUB_TOKEN": "github-secret", + "AWS_SECRET_ACCESS_KEY": "cloud-secret", + } + with mock.patch.dict(review_native.os.environ, sentinels, clear=False): + environment = review_native.scrubbed_environment() + for name in sentinels: + self.assertNotIn(name, environment) + + def test_fixture_environment_is_fixed_local_and_scrubbed(self): + sentinels = {"BUZZ_DB_HOST": "production-db", "BUZZ_DB_PASS": "production-secret", + "SSH_AUTH_SOCK": "/tmp/credential-agent"} + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.dict(review_native.os.environ, sentinels, clear=False): + environment = review_native.fixture_environment(isolation, "a" * 64) + self.assertEqual(environment["BUZZ_DB_HOST"], "localhost") + self.assertEqual(environment["BUZZ_DB_PORT"], "5471") + self.assertEqual(environment["BUZZ_DB_PASS"], "buzz_dev") + self.assertNotIn("SSH_AUTH_SOCK", environment) + with self.assertRaisesRegex(review_native.HarnessError, "port 3030"): + review_native.fixture_environment( + review_native.isolation_manifest("run", "ws://127.0.0.1:3001"), "a" * 64) + + def test_fixture_seed_failure_removes_generated_identity(self): + generated = mock.Mock(stdout=f"Secret key: {'1' * 64}\nPublic key: {'2' * 64}\n") + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + (run_dir / "manifest").mkdir() + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.object(review_native, "run", side_effect=[generated, RuntimeError("seed failed")]): + with self.assertRaisesRegex(RuntimeError, "seed failed"): + review_native.prepare_fixture(run_dir, isolation) + self.assertFalse((run_dir / "state/identity.key").exists()) + + def test_run_scrubs_repository_controlled_commands_by_default(self): + safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} + with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ + mock.patch.object(review_native.subprocess, "run", return_value=mock.Mock()) as subprocess_run: + review_native.run(["/tmp/repository-tool"]) + self.assertEqual(subprocess_run.call_args.kwargs["env"], safe) + + def test_cleanup_uses_isolated_home_without_host_credentials(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", + "GITHUB_TOKEN": "secret", "AWS_SECRET_ACCESS_KEY": "secret"} + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.dict(review_native.os.environ, sentinels, clear=False), \ + mock.patch.object(review_native, "run") as run: + review_native.cleanup_review_state(run_dir, isolation, None) + environment = run.call_args.kwargs["env"] + self.assertEqual(environment["HOME"], str(run_dir / "home")) + for name in sentinels: + self.assertNotIn(name, environment) + + def test_cleanup_removes_identity_when_state_reset_fails(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + secret_path = run_dir / "state/identity.key" + secret_path.parent.mkdir() + secret_path.write_text("review-secret") + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + fixture = {"secret_path": str(secret_path)} + with mock.patch.object(review_native, "run", side_effect=RuntimeError("reset failed")): + with self.assertRaisesRegex( + review_native.HarnessError, + "desktop state reset failed: reset failed"): + review_native.cleanup_review_state(run_dir, isolation, fixture) + self.assertFalse(secret_path.exists()) + + def test_cleanup_aggregates_reset_and_identity_removal_failures(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + secret_path = run_dir / "state/identity.key" + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + fixture = {"secret_path": str(secret_path)} + with mock.patch.object(review_native, "run", side_effect=RuntimeError("reset failed")), \ + mock.patch.object(pathlib.Path, "unlink", side_effect=OSError("unlink failed")): + with self.assertRaisesRegex( + review_native.HarnessError, + "desktop state reset failed: reset failed; " + "review identity removal failed: unlink failed"): + review_native.cleanup_review_state(run_dir, isolation, fixture) + + def test_repository_controlled_subprocesses_receive_scrubbed_environments(self): + safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} + with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ + mock.patch.object(review_native.subprocess, "Popen") as popen: + review_native.Driver(pathlib.Path("/tmp/driver"), 42, pathlib.Path("/tmp/snapshot")) + self.assertEqual(popen.call_args.kwargs["env"], safe) + + def test_visible_window_waits_for_reveal(self): + driver = mock.Mock() + driver.request.side_effect = [ + {"ok": True, "visible": False, "detail": "not yet"}, + {"ok": True, "visible": True, "window_id": 42}, + ] + process = mock.Mock() + process.poll.return_value = None + with mock.patch.object(review_native.time, "sleep"): + status = review_native.wait_for_visible_window(driver, process, timeout_seconds=1) + self.assertEqual(status["window_id"], 42) + self.assertEqual(driver.request.call_count, 2) + + def test_locate_required_retries_until_target_materializes(self): + driver = mock.Mock() + driver.request.side_effect = [ + {"ok": True, "element": None}, + {"ok": True, "element": {"locator": {"id": "target"}}}, + ] + with mock.patch.object(review_native.time, "sleep"): + found = review_native.locate_required(driver, [{"id": "target"}], 1000) + self.assertEqual(found["locator"]["id"], "target") + self.assertEqual(driver.request.call_count, 2) + + def test_visible_window_fails_if_app_exits(self): + driver = mock.Mock() + process = mock.Mock() + process.poll.return_value = 1 + with self.assertRaisesRegex(review_native.HarnessError, "exited"): + review_native.wait_for_visible_window(driver, process, timeout_seconds=1) + driver.request.assert_not_called() + + +class PerformanceTests(unittest.TestCase): + MACHINE = {"system": "Darwin", "release": "test", "machine": "arm64", "cpu": "test"} + + def receipt(self, path, artifact, timing, cpu=10, memory=100, flow="tooltip_fresh_dwell"): + payload = { + "run_id": path.stem, "flow": flow, "status": "passed", + "cleanup": {"status": "passed"}, + "provenance": {"dirty": False, "head_sha": artifact + "-sha", "artifact_sha256": artifact}, + "measurements": {"tooltip_open_latency": {"value": timing, "unit": "ms", "step": "tooltip"}}, + "performance": {"machine": self.MACHINE, "process": { + "cpu_percent_median": cpu, "resident_mb_peak": memory, + }}, + } + path.write_text(json.dumps(payload)) + return path + + def budget(self, path, regression=20): + path.write_text(f"""schema_version: 1 +flow: tooltip_fresh_dwell +minimum_samples: 3 +metrics: + tooltip_open_latency: + max: 1000 + max_regression_percent: {regression} + process.resident_mb_peak: + max: 500 + max_regression_percent: 20 +""") + return path + + def test_comparison_uses_median_and_passes_with_noise(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", value) for i, value in enumerate((100, 101, 900))] + candidate = [self.receipt(root / f"c{i}.json", "head", value) for i, value in enumerate((110, 111, 5))] + result = review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + self.assertEqual(result["status"], "passed") + self.assertEqual(result["baseline"]["metrics"]["tooltip_open_latency"]["median"], 101) + + def test_comparison_fails_when_one_sample_exceeds_absolute_maximum(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", value) + for i, value in enumerate((100, 100, 10000))] + with self.assertRaisesRegex(review_native.HarnessError, "maximum 10000.000 exceeds absolute maximum 1000"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_fails_on_relative_regression(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 130) for i in range(3)] + with self.assertRaisesRegex(review_native.HarnessError, "regression"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_too_few_samples(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipts = [self.receipt(root / f"r{i}.json", "same", 100) for i in range(2)] + with self.assertRaisesRegex(review_native.HarnessError, "at least 3"): + review_native.compare_performance(receipts, receipts, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_duplicate_receipt_path_and_run_id(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + with self.assertRaisesRegex(review_native.HarnessError, "duplicate receipt paths"): + review_native.compare_performance([baseline[0]] * 3, candidate, self.budget(root / "budget.yaml")) + duplicate = json.loads(baseline[1].read_text()) + duplicate["run_id"] = json.loads(baseline[0].read_text())["run_id"] + baseline[1].write_text(json.dumps(duplicate)) + with self.assertRaisesRegex(review_native.HarnessError, "duplicate run_id"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_receipts_reused_across_cohorts(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + with self.assertRaisesRegex(review_native.HarnessError, "independent receipt paths"): + review_native.compare_performance(baseline, baseline, self.budget(root / "budget.yaml")) + + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + for source, destination in zip(baseline, candidate, strict=True): + payload = json.loads(destination.read_text()) + payload["run_id"] = json.loads(source.read_text())["run_id"] + destination.write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "independent run_ids"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_boolean_and_non_finite_numbers(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + for value in (True, float("nan"), float("inf")): + with self.subTest(value=value): + payload = json.loads(candidate[0].read_text()) + payload["measurements"]["tooltip_open_latency"]["value"] = value + candidate[0].write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "finite numeric metric"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_incompatible_machine(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + payload = json.loads(candidate[0].read_text()) + payload["performance"]["machine"]["machine"] = "x86_64" + candidate[0].write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "incompatible machines"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_mixed_revisions(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + payload = json.loads(candidate[0].read_text()) + payload["provenance"]["head_sha"] = "other" + candidate[0].write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "mixes source revisions"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + +class PublishReviewTests(unittest.TestCase): + def inputs(self, root): + video = root / "video-share.mp4" + video.write_bytes(b"video") + receipt = root / "receipt.json" + receipt.write_text(json.dumps({ + "flow": "journey", "status": "failed", + "provenance": {"dirty": False, "head_sha": "a" * 40}, + "cleanup": {"status": "passed"}, + "artifacts": {"share_video": video.name}, + })) + summary = root / "summary.md" + summary.write_text("Found a regression.") + return receipt, summary + + @mock.patch.object(review_native.review_publish.shutil, "which", side_effect=lambda name: f"/usr/bin/{name}") + @mock.patch.object(review_native.review_publish, "_video_duration", return_value=12.5) + @mock.patch.object(review_native.review_publish, "_run") + def test_publishes_video_root_then_timecoded_highlights(self, run, _duration, _which): + run.side_effect = [ + subprocess.CompletedProcess([], 0, json.dumps({"accepted": True, "event_id": "b" * 64}), ""), + subprocess.CompletedProcess([], 0, json.dumps({"accepted": True, "event_id": "c" * 64}), ""), + ] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipt, summary = self.inputs(root) + highlights = root / "highlights.json" + highlights.write_text(json.dumps([{"seconds": 3.25, "text": "The broken state appears."}])) + result = review_native.publish_review(receipt, summary, "channel", "thread", highlights, ["d" * 64]) + first, second = [call.args[0] for call in run.call_args_list] + self.assertIn("--file", first) + self.assertIn("--mention", first) + self.assertEqual(second[-1], "[00:03.250] The broken state appears.") + self.assertEqual(second[second.index("--reply-to") + 1], "b" * 64) + self.assertEqual(result["highlight_event_ids"], ["c" * 64]) + + @mock.patch.object(review_native.review_publish.shutil, "which", return_value="/usr/bin/tool") + @mock.patch.object(review_native.review_publish, "_video_duration", return_value=10.0) + def test_rejects_highlight_outside_video_before_publish(self, _duration, _which): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipt, summary = self.inputs(root) + highlights = root / "highlights.json" + highlights.write_text(json.dumps([{"seconds": 11, "text": "impossible"}])) + with self.assertRaisesRegex(review_native.PublishError, "within the video"): + review_native.publish_review(receipt, summary, "channel", "thread", highlights) + + @mock.patch.object(review_native.review_publish.shutil, "which", return_value="/usr/bin/tool") + @mock.patch.object(review_native.review_publish, "_video_duration", return_value=10.0) + def test_rejects_boolean_and_non_finite_highlights(self, _duration, _which): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipt, summary = self.inputs(root) + for value in (True, float("nan"), float("inf")): + with self.subTest(value=value): + highlights = root / "highlights.json" + highlights.write_text(json.dumps([{"seconds": value, "text": "invalid"}])) + with self.assertRaisesRegex(review_native.PublishError, "within the video"): + review_native.publish_review(receipt, summary, "channel", "thread", highlights) + + @mock.patch.object(review_native.review_publish.shutil, "which", return_value="/usr/bin/tool") + def test_rejects_dirty_receipt_before_upload(self, _which): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipt, summary = self.inputs(root) + payload = json.loads(receipt.read_text()) + payload["provenance"]["dirty"] = True + receipt.write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.PublishError, "clean source"): + review_native.publish_review(receipt, summary, "channel", "thread") + + +if __name__ == "__main__": + unittest.main()