diff --git a/.gitignore b/.gitignore index f26e74136c0..dc5603d7071 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ __pycache__/ *.pyc +# Swift native-review build output +tools/native-review/swift/.build/ + # lefthook-generated hook scripts (machine-specific) .hooks/ diff --git a/Justfile b/Justfile index 6c7740bc7ac..8eb51717f71 100644 --- a/Justfile +++ b/Justfile @@ -1043,3 +1043,24 @@ benchmark-check: # Stop the benchmark Docker stack (state and channels are kept) benchmark-down: docker compose --project-name buzz-benchmark down + +# Validate macOS native-review tooling and report required OS permissions. +native-review-doctor: + ./tools/native-review/bin/review-native doctor + +# Run one declarative journey against the isolated local desktop fixture. +native-review-desktop JOURNEY="tools/native-review/desktop/tooltip-fresh-dwell.yaml": + ./tools/native-review/bin/review-native run "{{JOURNEY}}" + +# Capture a repeatable native performance cohort (minimum 3 runs). +native-review-benchmark JOURNEY="tools/native-review/desktop/tooltip-fresh-dwell.yaml" RUNS="5": + ./tools/native-review/bin/review-native benchmark "{{JOURNEY}}" --runs "{{RUNS}}" + +# Compare baseline and candidate receipt cohorts with explicit budget policy. +# Pass BASELINE/CANDIDATE as repeated CLI args, e.g. "--baseline a --baseline b". +native-review-compare BASELINE CANDIDATE BUDGET="tools/native-review/performance/tooltip-fresh-dwell.yaml": + ./tools/native-review/bin/review-native compare {{BASELINE}} {{CANDIDATE}} --budget "{{BUDGET}}" + +# Run the native iOS Simulator pairing journey with MP4 and screenshot evidence. +native-review-ios DEVICE="iPhone 17 Pro": + ./tools/native-review/bin/review-ios --device "{{DEVICE}}" diff --git a/desktop/src/main.tsx b/desktop/src/main.tsx index 634a9b171a1..9284fdfdab2 100644 --- a/desktop/src/main.tsx +++ b/desktop/src/main.tsx @@ -20,6 +20,7 @@ import { Toaster } from "@/shared/ui/sonner"; import { TooltipProvider } from "@/shared/ui/tooltip"; import { recoverLocalStorageQuotaOnStartup } from "@/shared/lib/localStorageQuota"; import { startLocalStorageSweep } from "@/shared/lib/localStorageSweep"; +import { installNativeReviewSemanticProbe } from "@/testing/nativeReviewSemanticProbe"; import { initializeConversationDensityPreference } from "@/shared/lib/conversationDensityPreference"; import { initializeFontSizePreference } from "@/shared/lib/fontSizePreference"; @@ -31,6 +32,54 @@ const E2E_DEFAULT_PUBKEY = "deadbeef".repeat(8); const E2E_COMMUNITY_ID = "e2e-default-community"; const ONBOARDING_COMPLETION_STORAGE_KEY_PREFIX = "buzz-onboarding-complete.v1:"; const DEV_STATE_RESET_PARAM = "resetDevState"; +const NATIVE_REVIEW_PARAM = "nativeReview"; + +function configureNativeReviewFixtureFromUrl() { + const buildEnabled = import.meta.env.VITE_NATIVE_REVIEW === "1"; + if (!import.meta.env.DEV && !buildEnabled) return; + const url = new URL(window.location.href); + const enabled = + url.searchParams.get(NATIVE_REVIEW_PARAM) === "1" || buildEnabled; + if (!enabled) return; + + const relayUrl = + url.searchParams.get("reviewRelay") ?? + import.meta.env.VITE_NATIVE_REVIEW_RELAY; + const pubkey = + url.searchParams.get("reviewPubkey") ?? + import.meta.env.VITE_NATIVE_REVIEW_PUBKEY; + if ( + !relayUrl || + !pubkey || + !/^(ws|http):\/\/(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?\/?$/.test( + relayUrl, + ) + ) { + throw new Error( + "native review bootstrap requires a loopback relay and pubkey", + ); + } + const communityId = "native-review-local"; + const community = { + addedAt: new Date().toISOString(), + id: communityId, + name: "Native Review", + pubkey, + relayUrl, + }; + window.localStorage.setItem("buzz-communities", JSON.stringify([community])); + window.localStorage.setItem("buzz-active-community-id", communityId); + window.localStorage.setItem( + `buzz-machine-onboarding-complete.v2:${pubkey}`, + "true", + ); + window.localStorage.setItem(`buzz-onboarding-complete.v1:${pubkey}`, "true"); + window.localStorage.setItem( + `buzz-community-onboarding-complete.v1:${encodeURIComponent(relayUrl)}:${pubkey}`, + "true", + ); + installNativeReviewSemanticProbe(); +} function resetDevWebviewStateFromUrl() { if (!import.meta.env.DEV) { @@ -89,7 +138,7 @@ function renderApp() { enabled={huddleWindowChannelId() === null} > - + @@ -124,6 +173,7 @@ async function installE2eBridgeIfConfigured() { async function bootstrap() { resetDevWebviewStateFromUrl(); + configureNativeReviewFixtureFromUrl(); configureDevE2eBridgeFromUrl(); recoverLocalStorageQuotaOnStartup(); initializeConversationDensityPreference(); diff --git a/desktop/src/testing/nativeReviewSemanticProbe.ts b/desktop/src/testing/nativeReviewSemanticProbe.ts new file mode 100644 index 00000000000..65927d0941c --- /dev/null +++ b/desktop/src/testing/nativeReviewSemanticProbe.ts @@ -0,0 +1,121 @@ +type SemanticNode = { + id?: string; + role?: string; + name?: string; + value?: string; + scrollY: number; + enabled: boolean; + focused: boolean; + frame: { x: number; y: number; width: number; height: number }; + viewport: { width: number; height: number }; +}; + +const IMPLICIT_ROLES: Partial> = { + A: "link", + BUTTON: "button", + INPUT: "text-field", + TEXTAREA: "text-area", +}; + +function accessibleName(element: HTMLElement): string | undefined { + const labelledBy = element.getAttribute("aria-labelledby"); + const labelledText = labelledBy + ?.split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim()) + .filter(Boolean) + .join(" "); + return ( + element.getAttribute("aria-label")?.trim() || + labelledText || + element.getAttribute("title")?.trim() || + (element.getAttribute("role") === "tooltip" + ? element.textContent?.trim() + : undefined) || + undefined + ); +} + +function snapshot(): SemanticNode[] { + const nodes: SemanticNode[] = []; + for (const candidate of document.querySelectorAll( + "[data-testid], [role], button, textarea, input, a[href]", + )) { + const rect = candidate.getBoundingClientRect(); + const style = window.getComputedStyle(candidate); + if ( + rect.width <= 0 || + rect.height <= 0 || + style.display === "none" || + style.visibility === "hidden" + ) { + continue; + } + const id = candidate.dataset.testid; + const role = + candidate.getAttribute("role") ?? IMPLICIT_ROLES[candidate.tagName]; + const name = accessibleName(candidate); + const value = + candidate instanceof HTMLInputElement || + candidate instanceof HTMLTextAreaElement + ? candidate.value + : candidate.isContentEditable + ? candidate.innerText.replace(/\r\n?/g, "\n").replace(/\n$/, "") + : undefined; + if (!id && !role && !name) continue; + nodes.push({ + ...(id ? { id } : {}), + ...(role ? { role } : {}), + ...(name ? { name } : {}), + ...(value !== undefined ? { value } : {}), + scrollY: candidate.scrollTop, + enabled: + !candidate.hasAttribute("disabled") && + candidate.getAttribute("aria-disabled") !== "true", + focused: candidate === document.activeElement, + frame: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }, + viewport: { + width: window.innerWidth, + height: window.innerHeight, + }, + }); + } + return nodes; +} + +export function installNativeReviewSemanticProbe(): void { + let scheduled = false; + const publish = () => { + scheduled = false; + const payload = JSON.stringify(snapshot()); + if ( + !navigator.sendBeacon( + import.meta.env.VITE_NATIVE_REVIEW_PROBE_URL, + payload, + ) + ) { + console.error("native review semantic probe beacon was rejected"); + } + }; + const schedule = () => { + if (scheduled) return; + scheduled = true; + window.requestAnimationFrame(publish); + }; + new MutationObserver(schedule).observe(document.documentElement, { + attributes: true, + childList: true, + subtree: true, + }); + window.addEventListener("input", schedule, true); + window.addEventListener("change", schedule, true); + window.addEventListener("focusin", schedule); + window.addEventListener("focusout", schedule); + window.addEventListener("resize", schedule); + window.addEventListener("scroll", schedule, true); + schedule(); +} diff --git a/mobile/integration_test/native_review_pairing_test.dart b/mobile/integration_test/native_review_pairing_test.dart new file mode 100644 index 00000000000..022893de40b --- /dev/null +++ b/mobile/integration_test/native_review_pairing_test.dart @@ -0,0 +1,59 @@ +import 'dart:io'; + +import 'package:buzz/features/pairing/pairing_page.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:integration_test/integration_test.dart'; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('pairing code reveal, edit, and hide survives native rendering', ( + tester, + ) async { + await tester.pumpWidget( + const ProviderScope(child: MaterialApp(home: PairingPage())), + ); + await tester.pumpAndSettle(); + expect(find.text('Welcome to Buzz'), findsOneWidget); + expect(find.byKey(const Key('pairing-code-input')), findsNothing); + debugPrint('BUZZ_NATIVE_REVIEW_RECORDING_READY'); + debugPrint('BUZZ_NATIVE_REVIEW_STATE:initial-hidden'); + final proceedUrl = Platform.environment['BUZZ_NATIVE_REVIEW_PROCEED_URL']; + expect(proceedUrl, isNotNull); + const proceedTimeout = Duration(minutes: 3); + final client = HttpClient(); + try { + final proceed = await client + .getUrl(Uri.parse(proceedUrl!)) + .then((request) => request.close()) + .timeout(proceedTimeout); + expect(proceed.statusCode, HttpStatus.noContent); + await proceed.drain().timeout(proceedTimeout); + } finally { + client.close(force: true); + } + + await tester.tap(find.byKey(const Key('pairing-code-toggle'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('pairing-code-input')), findsOneWidget); + debugPrint('BUZZ_NATIVE_REVIEW_STATE:revealed'); + await tester.pump(const Duration(seconds: 1)); + + await tester.enterText( + find.byKey(const Key('pairing-code-input')), + 'nostrpair://native-review', + ); + await tester.pump(); + expect(find.text('nostrpair://native-review'), findsOneWidget); + expect(find.byKey(const Key('pairing-connect')), findsOneWidget); + debugPrint('BUZZ_NATIVE_REVIEW_STATE:edited'); + await tester.pump(const Duration(seconds: 1)); + + await tester.tap(find.byKey(const Key('pairing-code-toggle'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('pairing-code-input')), findsNothing); + debugPrint('BUZZ_NATIVE_REVIEW_STATE:final-hidden'); + }); +} diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 491da241e8f..d02a6884d91 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -45,6 +45,8 @@ PODS: - GTMSessionFetcher/Core (3.5.0) - image_picker_ios (0.0.1): - Flutter + - integration_test (0.0.1): + - Flutter - local_auth_darwin (0.0.1): - Flutter - FlutterMacOS @@ -108,6 +110,7 @@ DEPENDENCIES: - google_mlkit_commons (from `.symlinks/plugins/google_mlkit_commons/ios`) - google_mlkit_selfie_segmentation (from `.symlinks/plugins/google_mlkit_selfie_segmentation/ios`) - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`) + - integration_test (from `.symlinks/plugins/integration_test/ios`) - local_auth_darwin (from `.symlinks/plugins/local_auth_darwin/darwin`) - mobile_scanner (from `.symlinks/plugins/mobile_scanner/darwin`) - open_filex (from `.symlinks/plugins/open_filex/ios`) @@ -155,6 +158,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/google_mlkit_selfie_segmentation/ios" image_picker_ios: :path: ".symlinks/plugins/image_picker_ios/ios" + integration_test: + :path: ".symlinks/plugins/integration_test/ios" local_auth_darwin: :path: ".symlinks/plugins/local_auth_darwin/darwin" mobile_scanner: @@ -190,6 +195,7 @@ SPEC CHECKSUMS: GoogleUtilities: 26a3abef001b6533cf678d3eb38fd3f614b7872d GTMSessionFetcher: 5aea5ba6bd522a239e236100971f10cb71b96ab6 image_picker_ios: e0ece4aa2a75771a7de3fa735d26d90817041326 + integration_test: 4a889634ef21a45d28d50d622cf412dc6d9f586e local_auth_darwin: c3ee6cce0a8d56be34c8ccb66ba31f7f180aaebb MLImage: 0de5c6c2bf9e93b80ef752e2797f0836f03b58c0 MLKitCommon: 47d47b50a031d00db62f1b0efe5a1d8b09a3b2e6 diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index 1ef121fff4a..27cf7bbbcf1 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -19,7 +19,11 @@ import UserNotifications _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { - UNUserNotificationCenter.current().requestAuthorization(options: [.badge]) { _, _ in } + // Integration tests cannot interact with SpringBoard's notification prompt. + // Production launches still request badge authorization as before. + if ProcessInfo.processInfo.environment["BUZZ_NATIVE_REVIEW"] != "1" { + UNUserNotificationCenter.current().requestAuthorization(options: [.badge]) { _, _ in } + } return super.application(application, didFinishLaunchingWithOptions: launchOptions) } diff --git a/mobile/lib/app.dart b/mobile/lib/app.dart index 655095045ef..4c565088e0e 100644 --- a/mobile/lib/app.dart +++ b/mobile/lib/app.dart @@ -332,7 +332,9 @@ class App extends HookConsumerWidget { // cold-start link survives until the authenticated UI can dispatch it. ref.watch(pendingDeepLinkProvider); + const nativeReview = bool.fromEnvironment('BUZZ_NATIVE_REVIEW'); void applyBadge(UnreadBadgeState state) { + if (nativeReview) return; if (state.highPriorityCount > 0) { AppBadgePlus.updateBadge(state.highPriorityCount); } else if (state.generalUnreadCount > 0) { diff --git a/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart b/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart index 49fc185133f..1a48ef4a2d2 100644 --- a/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart +++ b/mobile/lib/features/pairing/pairing_page/pairing_welcome_view.dart @@ -97,6 +97,7 @@ class _PairingWelcomeView extends StatelessWidget { ), const SizedBox(height: Grid.xxs), TextButton( + key: const Key('pairing-code-toggle'), style: _onboardingSecondaryButtonStyle, onPressed: isBusy ? null : onTogglePairingCode, child: Text( @@ -125,6 +126,7 @@ class _PairingWelcomeView extends StatelessWidget { children: [ const SizedBox(height: Grid.twelve), TextField( + key: const Key('pairing-code-input'), controller: codeController, style: context.textTheme.bodyMedium ?.copyWith(color: _onboardingInk), @@ -168,6 +170,7 @@ class _PairingWelcomeView extends StatelessWidget { SizedBox( width: double.infinity, child: FilledButton( + key: const Key('pairing-connect'), style: _onboardingButtonStyle, onPressed: isBusy ? null : onConnect, child: isBusy diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 46ea8cf564a..44f0c2b9540 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -462,6 +462,11 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_driver: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" flutter_hooks: dependency: "direct main" description: @@ -584,6 +589,11 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.0" + fuchsia_remote_debug_protocol: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" glob: dependency: transitive description: @@ -752,6 +762,11 @@ packages: url: "https://pub.dev" source: hosted version: "0.2.2" + integration_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" intl: dependency: "direct main" description: @@ -1128,6 +1143,14 @@ packages: url: "https://pub.dev" source: hosted version: "6.5.2" + process: + dependency: transitive + description: + name: process + sha256: c6248e4526673988586e8c00bb22a49210c258dc91df5227d5da9748ecf79744 + url: "https://pub.dev" + source: hosted + version: "5.0.5" provider: dependency: transitive description: @@ -1373,6 +1396,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.1" + sync_http: + dependency: transitive + description: + name: sync_http + sha256: "7f0cd72eca000d2e026bcd6f990b81d0ca06022ef4e32fb257b30d3d1014a961" + url: "https://pub.dev" + source: hosted + version: "0.3.1" term_glyph: dependency: transitive description: @@ -1605,6 +1636,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + webdriver: + dependency: transitive + description: + name: webdriver + sha256: "2f3a14ca026957870cfd9c635b83507e0e51d8091568e90129fbf805aba7cade" + url: "https://pub.dev" + source: hosted + version: "3.1.0" webkit_inspection_protocol: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 4543888551b..5543e7af5be 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -50,6 +50,10 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter + integration_test: + sdk: flutter + flutter_driver: + sdk: flutter flutter_lints: ^6.0.0 camera_platform_interface: ^2.13.1 crypto: ^3.0.7 diff --git a/mobile/test_driver/integration_test.dart b/mobile/test_driver/integration_test.dart new file mode 100644 index 00000000000..b38629cca97 --- /dev/null +++ b/mobile/test_driver/integration_test.dart @@ -0,0 +1,3 @@ +import 'package:integration_test/integration_test_driver.dart'; + +Future main() => integrationDriver(); diff --git a/scripts/setup-desktop-test-data.sh b/scripts/setup-desktop-test-data.sh index ff3ebd055c5..da7671f4805 100755 --- a/scripts/setup-desktop-test-data.sh +++ b/scripts/setup-desktop-test-data.sh @@ -15,6 +15,11 @@ BOB_PUBKEY="bb22a5299220cad76ffd46190ccbeede8ab5dc260faa28b6e5a2cb31b9aff260" CHARLIE_PUBKEY="554cef57437abac34522ac2c9f0490d685b72c80478cf9f7ed6f9570ee8624ea" TYLER_PUBKEY="e5ebc6cdb579be112e336cc319b5989b4bb6af11786ea90dbe52b5f08d741b34" AGENT_PUBKEY="db0b028cd36f4d3e36c8300cce87252c1f7fc9495ffecc53f393fcac341ffd36" +REVIEW_PUBKEY="${BUZZ_REVIEW_PUBKEY:-}" +if [[ -n "$REVIEW_PUBKEY" && ! "$REVIEW_PUBKEY" =~ ^[0-9a-fA-F]{64}$ ]]; then + echo "BUZZ_REVIEW_PUBKEY must be exactly 64 hexadecimal characters." >&2 + exit 1 +fi if command -v psql >/dev/null 2>&1; then run_psql() { PGPASSWORD="$DB_PASS" psql -h"$DB_HOST" -p"$DB_PORT" -U"$DB_USER" -d"$DB_NAME" -qtA "$@"; } @@ -129,4 +134,15 @@ ON CONFLICT DO NOTHING ; " +if [[ -n "$REVIEW_PUBKEY" ]]; then + run_sql " +INSERT INTO channel_members + (community_id, channel_id, pubkey, role, invited_by) +VALUES + ('${COMMUNITY_ID}', '${UUID_GENERAL}', decode('${REVIEW_PUBKEY}','hex'), 'member', decode('${SYSTEM_PUBKEY}','hex')) +ON CONFLICT DO NOTHING +; +" +fi + echo "Desktop e2e data ready." diff --git a/tools/native-review/README.md b/tools/native-review/README.md new file mode 100644 index 00000000000..f70a15c7abf --- /dev/null +++ b/tools/native-review/README.md @@ -0,0 +1,95 @@ +# Buzz native review harness + +This macOS-only MVP drives the real Tauri/WKWebView app with Accessibility and +CGEvent, captures its window with Core Graphics and AVFoundation, and writes an +exact-SHA run receipt. It is a targeted review lane, not a replacement for +`just ci` or Playwright. + +## Safety contract + +- Only loopback `ws://`/`http://` relays are accepted. +- Every run gets an ephemeral Nostr key, run-specific dev bundle ID, keyring + service, HOME, WebKit/app-data state, and artifact directory. +- The launcher environment is allowlisted before the ephemeral key is added; + inherited tokens and production keys never enter the reviewed process. +- Production bundle IDs, keyring services, and remote relays fail closed. +- This protects reviewer state from accidents. It is **not** containment for + hostile code; use a dedicated macOS user or disposable VM for untrusted PRs. + +## Commands + +```bash +just native-review-doctor +just native-review-desktop tools/native-review/desktop/tooltip-fresh-dwell.yaml +python3 -m unittest discover -s tools/native-review/tests -p 'test_*.py' +``` + +The desktop command expects the isolated `buzz-harness` relay on port 3030 +(`scripts/start-isolated-test-relay.sh`). Doctor reports Accessibility and +Screen Recording separately and the run refuses to proceed unless both are +already granted to the invoking terminal/agent. + +Runs are written under +`test-results/native-review////`. A failed locator, +postcondition, recording, evidence capture, or cleanup produces a failed partial +receipt. `tests/fixtures/broken-tooltip.yaml` is the deliberate fail-loud +mutation. + +## Performance comparison and budgets + +A step with `measure: ` persists its complete native action-to-observed- +postcondition duration in the receipt. While the journey runs, the harness also +samples the app process every 100 ms and records median/peak CPU percentage and +resident memory. Capture a cohort rather than trusting one noisy laptop run: + +```bash +just native-review-benchmark tools/native-review/desktop/tooltip-fresh-dwell.yaml 5 +``` + +Compare at least three clean baseline receipts with at least three clean +candidate receipts using `compare` and a checked-in policy such as +`performance/tooltip-fresh-dwell.yaml`: + +```bash +./tools/native-review/bin/review-native compare \ + --baseline /path/base-1/receipt.json --baseline /path/base-2/receipt.json --baseline /path/base-3/receipt.json \ + --candidate /path/head-1/receipt.json --candidate /path/head-2/receipt.json --candidate /path/head-3/receipt.json \ + --budget tools/native-review/performance/tooltip-fresh-dwell.yaml \ + --output test-results/native-review/performance-comparison.json +``` + +Comparison uses cohort medians, reports every raw sample and min/max, and exits +nonzero when an absolute ceiling or relative regression limit is breached. It +fails closed for dirty-tree runs, failed cleanup, mixed source revisions within a +cohort, wrong flows, missing metrics, too few samples, or different machine/OS +fingerprints. Baseline and candidate therefore need to run on the same host; +thermal/load noise is reduced by repeated samples, not disguised as universal +lab-grade benchmarking. Recording overhead is intentionally present in both +cohorts because this tool measures the reviewer-visible workflow. + +## iOS Simulator + +The iOS lane runs the pairing integration journey on a simulator created uniquely +for that run. Build and installation happen before canonical capture: the journey +must report its rendered initial state, and the harness must foreground Buzz, +before recording begins. It records video, Flutter logs, and a final screenshot, +writes the same schema-versioned receipt shape as Desktop, then shuts down and +deletes only the simulator UDID it created. It never erases, boots, or deletes an +existing simulator. Host credentials are removed from child environments. + +```bash +just native-review-ios 'iPhone 17 Pro' +``` + +Treat the owned-device boundary as mandatory: do not replace the generated UDID +with a personal or shared simulator. The runner is reviewer-state protection, +not hostile-code containment; use a disposable macOS account or VM for untrusted +changes. + +## Current limits + +macOS desktop, the local review-channel fixture, and the iOS pairing Simulator +journey are implemented. The desktop schema covers role/name/identifier locators; +native click, hover, text entry, keyboard shortcuts, scrolling, and waits; +value/focus/existence assertions; window screenshots/video; and per-step semantic ++ AX snapshots. Additional iOS journeys remain future work. diff --git a/tools/native-review/bin/review-ios b/tools/native-review/bin/review-ios new file mode 100755 index 00000000000..ed2fd323fa3 --- /dev/null +++ b/tools/native-review/bin/review-ios @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +TOOL_ROOT="$ROOT/tools/native-review" +. "$ROOT/bin/activate-hermit" +exec uv run --frozen --project "$TOOL_ROOT" python "$TOOL_ROOT/ios_review.py" "$@" diff --git a/tools/native-review/bin/review-native b/tools/native-review/bin/review-native new file mode 100755 index 00000000000..e09e4704948 --- /dev/null +++ b/tools/native-review/bin/review-native @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# Run through the locked native-review Python environment so a clean checkout +# never depends on host-installed packages. +source "$REPO_ROOT/bin/activate-hermit" +exec uv run --frozen --project "$SCRIPT_DIR" python "$SCRIPT_DIR/review_native.py" "$@" diff --git a/tools/native-review/desktop/composer-keyboard.yaml b/tools/native-review/desktop/composer-keyboard.yaml new file mode 100644 index 00000000000..9f2d6b7be8d --- /dev/null +++ b/tools/native-review/desktop/composer-keyboard.yaml @@ -0,0 +1,49 @@ +schema_version: 1 +flow: composer_keyboard +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: true + accessibility: true +steps: + - name: activate_buzz + act: {type: activate} + expect: {exists: {role: window}} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: {not_exists: {id: app-loading-gate}} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + - {role: button, name: welcome-everyone} + act: {type: click} + expect: {exists: {id: message-composer}} + timeout_ms: 15000 + - name: focus_composer + locate: + - {id: message-input} + - {role: text-area, name: Message} + act: {type: click} + expect: {focused: {id: message-input}} + - name: type_draft + act: {type: type_text, text: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} + expect: {value: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} + - name: keep_draft_while_scrolling + locate: + - {id: message-input-scroll} + act: {type: scroll, delta_y: 240} + expect: {scroll_y_less_than: 1} + - name: draft_survives_scroll + locate: + - {id: message-input} + act: {type: wait, duration_ms: 50} + expect: {value: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} + - name: dismiss_focus + act: {type: press, key: escape} + expect: {value: "native review line 1\nnative review line 2\nnative review line 3\nnative review line 4\nnative review line 5\nnative review line 6\nnative review line 7\nnative review line 8\nnative review line 9\nnative review line 10\nnative review line 11\nnative review line 12\nnative review line 13\nnative review line 14\nnative review line 15"} +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/desktop/search-shortcut-dismissal.yaml b/tools/native-review/desktop/search-shortcut-dismissal.yaml new file mode 100644 index 00000000000..0a2876a88bf --- /dev/null +++ b/tools/native-review/desktop/search-shortcut-dismissal.yaml @@ -0,0 +1,39 @@ +schema_version: 1 +flow: search_shortcut_dismissal +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: true + accessibility: true +steps: + - name: activate_buzz + act: {type: activate} + expect: {exists: {role: window}} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: {not_exists: {id: app-loading-gate}} + timeout_ms: 60000 + - name: open_search_with_keyboard + act: {type: press, key: k, modifiers: [command]} + expect: {exists: {id: search-results}} + timeout_ms: 5000 + - name: search_input_receives_focus + locate: + - {id: search-dialog-input} + - {role: text-field, name: "Search everything"} + act: {type: type_text, text: "welcome"} + expect: {value: "welcome"} + - name: dismiss_search + act: {type: press, key: escape} + expect: {not_exists: {id: search-results}} + timeout_ms: 2000 + - name: focus_returns_to_trigger + locate: + - {id: open-search} + act: {type: wait, duration_ms: 50} + expect: {focused: {id: open-search}} +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/desktop/tooltip-fresh-dwell.yaml b/tools/native-review/desktop/tooltip-fresh-dwell.yaml new file mode 100644 index 00000000000..5665beb159c --- /dev/null +++ b/tools/native-review/desktop/tooltip-fresh-dwell.yaml @@ -0,0 +1,61 @@ +schema_version: 1 +flow: tooltip_fresh_dwell +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: false + accessibility: false +steps: + - name: activate_buzz + act: {type: activate} + expect: + exists: {role: window} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: + not_exists: {id: app-loading-gate} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + - {role: button, name: welcome-everyone} + act: {type: click} + expect: + exists: {id: message-insert-mention} + timeout_ms: 15000 + - name: settle_pointer_before_fresh_dwell + locate: + - {id: message-composer} + - {role: text-area, name: Message} + act: {type: move_pointer, duration_ms: 80} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + - name: clear_tooltip_skip_delay + act: {type: wait, duration_ms: 800} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + - name: transit_over_mention + locate: + - {id: message-insert-mention} + - {role: button, name: "Mention someone"} + act: {type: move_pointer} + expect: {enabled: true} + - name: complete_dwell + act: {type: wait, duration_ms: 0} + expect: + exists: {role: tooltip, name: "Mention someone"} + timeout_ms: 1000 + measure: tooltip_open_latency + - name: leave_trigger + locate: + - {id: message-composer} + - {role: text-area, name: Message} + act: {type: move_pointer, duration_ms: 80} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + timeout_ms: 1000 +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/ios_review.py b/tools/native-review/ios_review.py new file mode 100755 index 00000000000..7a727f9634e --- /dev/null +++ b/tools/native-review/ios_review.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +"""Run a Flutter integration journey on an isolated iOS Simulator with video evidence.""" +from __future__ import annotations + +import argparse +import datetime as dt +import http.server +import json +import os +import pathlib +import platform +import re +import secrets +import selectors +import shutil +import signal +import subprocess +import sys +import threading +import time +from typing import Any + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DEFAULT_TEST = ROOT / "mobile/integration_test/native_review_pairing_test.dart" +IOS_BUNDLE_ID = "com.buzz.buzzMobile" +RECORDING_READY_MARKER = "BUZZ_NATIVE_REVIEW_RECORDING_READY" +STATE_MARKER_PREFIX = "BUZZ_NATIVE_REVIEW_STATE:" +BOOT_TIMEOUT_SECONDS = 120 +FLUTTER_TIMEOUT_SECONDS = 180 +SIMCTL_TIMEOUT_SECONDS = 30 +SUBPROCESS_ENV_ALLOWLIST = { + "PATH", "HOME", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", + "TERM", "__CF_USER_TEXT_ENCODING", "DEVELOPER_DIR", +} + + +class ReviewError(RuntimeError): + pass + + +def run(command: list[str], *, cwd: pathlib.Path = ROOT, check: bool = True, capture: bool = True, + env: dict[str, str] | None = None, timeout: float | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, check=check, text=True, + env=subprocess_environment() if env is None else env, timeout=timeout, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None) + + +def git(*args: str) -> str: + return run(["git", *args]).stdout.strip() + + +def create_review_device(device_type_name: str, run_id: str) -> dict[str, Any]: + """Create a uniquely named simulator owned by this run; never reuse user state.""" + payload = json.loads(run( + ["xcrun", "simctl", "list", "devicetypes", "runtimes", "-j"], + timeout=SIMCTL_TIMEOUT_SECONDS, + ).stdout) + device_types = [item for item in payload.get("devicetypes", []) if item.get("name") == device_type_name] + if not device_types: + raise ReviewError(f"no iOS Simulator device type named {device_type_name}") + device_type = device_types[0] + runtimes = [ + item for item in payload.get("runtimes", []) + if item.get("isAvailable") and item.get("platform") == "iOS" + and any(supported.get("identifier") == device_type["identifier"] + for supported in item.get("supportedDeviceTypes", [])) + ] + if not runtimes: + raise ReviewError(f"no available iOS Simulator runtime supports {device_type_name}") + runtime = max(runtimes, key=lambda item: tuple(int(part) for part in item["version"].split("."))) + owned_name = f"Buzz Native Review {run_id}" + udid = run( + ["xcrun", "simctl", "create", owned_name, + device_type["identifier"], runtime["identifier"]], + timeout=SIMCTL_TIMEOUT_SECONDS, + ).stdout.strip() + if not re.fullmatch(r"[0-9A-Fa-f-]{36}", udid): + raise ReviewError("simctl create returned an invalid device identifier") + return {"name": owned_name, "udid": udid, "runtimeIdentifier": runtime["identifier"], + "deviceType": device_type_name, "owned": True} + + +def subprocess_environment() -> dict[str, str]: + return {key: value for key, value in os.environ.items() if key in SUBPROCESS_ENV_ALLOWLIST} + + +def flutter_environment(proceed_url: str | None = None) -> dict[str, str]: + env = subprocess_environment() + env.update({"BUZZ_NATIVE_REVIEW": "1", "SIMCTL_CHILD_BUZZ_NATIVE_REVIEW": "1"}) + if proceed_url: + env["BUZZ_NATIVE_REVIEW_PROCEED_URL"] = proceed_url + env["SIMCTL_CHILD_BUZZ_NATIVE_REVIEW_PROCEED_URL"] = proceed_url + return env + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat() + + +def machine_fingerprint() -> dict[str, str]: + cpu = run(["sysctl", "-n", "machdep.cpu.brand_string"], check=False).stdout.strip() + return { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "cpu": cpu or "unknown", + } + + +def provenance() -> dict[str, Any]: + status = git("status", "--porcelain=v1", "--untracked-files=all") + return {"head_sha": git("rev-parse", "HEAD"), "dirty": bool(status), "status": status.splitlines()} + + +def wait_for_recording(recorder: subprocess.Popen[str], timeout_seconds: float = 15) -> None: + if recorder.stderr is None: + raise ReviewError("simulator recorder has no diagnostic stream") + selector = selectors.DefaultSelector() + selector.register(recorder.stderr, selectors.EVENT_READ) + deadline = time.monotonic() + timeout_seconds + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ReviewError("timed out waiting for Simulator recording") + if not selector.select(remaining): + raise ReviewError("timed out waiting for Simulator recording") + line = recorder.stderr.readline() + if "Recording started" in line: + return + if recorder.poll() is not None: + raise ReviewError(f"simulator recorder exited: {line.strip()}") + finally: + selector.close() + + +def wait_for_recording_ready(process: subprocess.Popen[str], log: pathlib.Path, + timeout_seconds: float = 180) -> None: + """Wait for the running journey to render its initial evidence state.""" + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if log.is_file() and RECORDING_READY_MARKER in log.read_text(errors="replace"): + return + if process.poll() is not None: + raise ReviewError("Flutter integration journey exited before recording readiness") + time.sleep(0.1) + raise ReviewError("timed out waiting for Flutter recording readiness") + + +class ProceedServer(http.server.ThreadingHTTPServer): + review_ready: threading.Event + + +def proceed_server() -> tuple[ProceedServer, str]: + token = secrets.token_hex(32) + ready = threading.Event() + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + if self.path != f"/proceed/{token}": + self.send_error(404) + return + if not ready.wait(timeout=FLUTTER_TIMEOUT_SECONDS): + self.send_error(504) + return + self.send_response(204) + self.end_headers() + + def log_message(self, _format: str, *_args: Any) -> None: + pass + + server = ProceedServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + server.review_ready = ready + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f"http://127.0.0.1:{server.server_port}/proceed/{token}" + + +def release_journey(server: ProceedServer) -> None: + server.review_ready.set() + + +def evidence_steps(log: pathlib.Path) -> list[dict[str, Any]]: + states = [line.split(STATE_MARKER_PREFIX, 1)[1].strip() + for line in log.read_text(errors="replace").splitlines() + if STATE_MARKER_PREFIX in line] + expected = ["initial-hidden", "revealed", "edited", "final-hidden"] + if states != expected: + raise ReviewError(f"journey state evidence incomplete: expected {expected}, got {states}") + return [{"name": state, "status": "passed"} for state in states] + + +def start_flutter_review(test: pathlib.Path, udid: str, log: pathlib.Path, + proceed_url: str) -> tuple[subprocess.Popen[str], Any]: + log_handle = log.open("w") + command = ["flutter", "drive", "--driver", "test_driver/integration_test.dart", + "--target", str(test.relative_to(ROOT / "mobile")), "-d", udid, + "--keep-app-running", "--dart-define=BUZZ_NATIVE_REVIEW=true"] + try: + process = subprocess.Popen(command, cwd=ROOT / "mobile", stdout=log_handle, + stderr=subprocess.STDOUT, env=flutter_environment(proceed_url), text=True) + except Exception: + log_handle.close() + raise + return process, log_handle + + +def finalize_recording(recorder: subprocess.Popen[str], video: pathlib.Path) -> None: + if recorder.poll() is None: + recorder.send_signal(signal.SIGINT) + try: + returncode = recorder.wait(timeout=30) + except subprocess.TimeoutExpired as exc: + recorder.kill() + try: + recorder.wait(timeout=5) + except subprocess.TimeoutExpired as reap_exc: + raise ReviewError("simulator recorder did not exit after SIGKILL") from reap_exc + raise ReviewError("simulator recorder required SIGKILL") from exc + if returncode: + diagnostic = recorder.stderr.read().strip() if recorder.stderr else "" + detail = f": {diagnostic}" if diagnostic else "" + raise ReviewError(f"simulator recorder failed with exit {returncode}{detail}") + if not video.is_file() or video.stat().st_size == 0: + raise ReviewError("simulator recorder produced no video") + probe = video.with_name("video-validation.mov") + try: + result = run(["/usr/bin/avconvert", "--source", str(video), "--preset", "PresetHighestQuality", + "--output", str(probe), "--duration", "0.1", "--replace"], check=False, timeout=30) + except subprocess.TimeoutExpired as exc: + raise ReviewError("timed out validating simulator video") from exc + finally: + probe.unlink(missing_ok=True) + if result.returncode: + diagnostic = (result.stderr or result.stdout).strip() + detail = f": {diagnostic}" if diagnostic else "" + raise ReviewError(f"simulator recorder produced an invalid video{detail}") + + +def terminate_child(process: subprocess.Popen[str], name: str) -> list[str]: + errors: list[str] = [] + if process.poll() is not None: + try: + process.wait(timeout=0) + except Exception as exc: + errors.append(f"{name} reap failed: {exc}") + return errors + try: + process.terminate() + except Exception as exc: + errors.append(f"{name} terminate failed: {exc}") + try: + process.wait(timeout=10) + return errors + except subprocess.TimeoutExpired: + errors.append(f"{name} required SIGKILL") + except Exception as exc: + errors.append(f"{name} first reap failed: {exc}") + try: + process.kill() + except Exception as exc: + errors.append(f"{name} kill failed: {exc}") + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + errors.append(f"{name} did not exit after SIGKILL") + except Exception as exc: + errors.append(f"{name} final reap failed: {exc}") + return errors + + +def run_review(test: pathlib.Path, device_name: str, output_root: pathlib.Path) -> pathlib.Path: + if sys.platform != "darwin" or not shutil.which("xcrun") or not shutil.which("flutter"): + raise ReviewError("iOS native review requires macOS, Xcode simctl, and Flutter") + if not test.is_file() or ROOT not in test.resolve().parents: + raise ReviewError(f"test must be a repository integration test: {test}") + prov = provenance() + run_id = f"ios-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{secrets.token_hex(3)}" + run_dir = output_root / git("rev-parse", "--short=12", "HEAD") / "ios_pairing" / run_id + run_dir.mkdir(parents=True) + started = utc_now() + receipt: dict[str, Any] = { + "schema_version": 1, + "run_id": run_id, + "flow": "ios_pairing", + "status": "failed", + "started_at": started, + "finished_at": started, + "failure": None, + "provenance": prov, + "isolation": {"kind": "run_owned_simulator", "device_type": device_name}, + "artifacts": {}, + "steps": [], + "measurements": {}, + "performance": {"machine": machine_fingerprint()}, + "cleanup": {"status": "not_started"}, + } + device: dict[str, Any] | None = None + udid: str | None = None + recorder: subprocess.Popen[str] | None = None + flutter_process: subprocess.Popen[str] | None = None + flutter_log: Any | None = None + video: pathlib.Path | None = None + gate_server: ProceedServer | None = None + try: + device = create_review_device(device_name, run_id) + udid = device["udid"] + receipt["isolation"]["device"] = { + "name": device["name"], "device_type": device_name, "udid": udid, + "runtime": device["runtimeIdentifier"], "owned": True, + } + run(["xcrun", "simctl", "boot", udid], timeout=SIMCTL_TIMEOUT_SECONDS) + run(["xcrun", "simctl", "bootstatus", udid, "-b"], capture=False, timeout=BOOT_TIMEOUT_SECONDS) + log = run_dir / "flutter.log" + gate_server, proceed_url = proceed_server() + flutter_process, flutter_log = start_flutter_review(test, udid, log, proceed_url) + receipt["artifacts"]["log"] = "flutter.log" + wait_for_recording_ready(flutter_process, log) + # Readiness proves the reviewed UI rendered. Foreground it explicitly before + # capture so build/install/SpringBoard can never become canonical evidence. + run(["xcrun", "simctl", "launch", udid, IOS_BUNDLE_ID], timeout=SIMCTL_TIMEOUT_SECONDS) + video = run_dir / "video.mp4" + recorder = subprocess.Popen(["xcrun", "simctl", "io", udid, "recordVideo", "--codec=h264", "--force", str(video)], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + env=subprocess_environment(), text=True) + wait_for_recording(recorder) + receipt["artifacts"]["video"] = "video.mp4" + release_journey(gate_server) + try: + returncode = flutter_process.wait(timeout=FLUTTER_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as exc: + raise ReviewError("timed out waiting for Flutter integration journey") from exc + flutter_log.flush() + if returncode: + raise ReviewError(f"Flutter integration journey failed with exit {returncode}") + receipt["steps"] = evidence_steps(log) + flutter_log.close() + flutter_log = None + screenshot = run_dir / "final.png" + run(["xcrun", "simctl", "io", udid, "screenshot", str(screenshot)], timeout=SIMCTL_TIMEOUT_SECONDS) + receipt["artifacts"]["screenshot"] = "final.png" + time.sleep(0.5) + except Exception as exc: + receipt["failure"] = str(exc) + finally: + errors = [] + if flutter_process: + errors.extend(terminate_child(flutter_process, "Flutter integration journey")) + if flutter_log: + try: + flutter_log.close() + except Exception as exc: + errors.append(f"Flutter log cleanup failed: {exc}") + if gate_server: + try: + release_journey(gate_server) + gate_server.shutdown() + gate_server.server_close() + except Exception as exc: + errors.append(f"journey gate cleanup failed: {exc}") + if recorder: + try: + if video is None: + raise ReviewError("simulator recorder video path was not initialized") + finalize_recording(recorder, video) + except Exception as exc: + errors.append(str(exc)) + if udid: + try: + run(["xcrun", "simctl", "shutdown", udid], check=False, timeout=SIMCTL_TIMEOUT_SECONDS) + except Exception as exc: + errors.append(str(exc)) + try: + run(["xcrun", "simctl", "delete", udid], timeout=SIMCTL_TIMEOUT_SECONDS) + except Exception as exc: + errors.append(str(exc)) + receipt["cleanup"] = {"status": "failed" if errors else "passed", "errors": errors} + if errors: + receipt["status"] = "failed" + if receipt["failure"] is None: + receipt["failure"] = errors[0] + elif receipt["failure"] is None: + receipt["status"] = "passed" + receipt["finished_at"] = utc_now() + (run_dir / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(run_dir) + if receipt["status"] != "passed": + raise ReviewError(receipt["failure"] or "iOS journey or cleanup failed") + return run_dir + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--test", type=pathlib.Path, default=DEFAULT_TEST) + parser.add_argument("--device", default="iPhone 17 Pro") + parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + args = parser.parse_args() + try: + run_review(args.test.resolve(), args.device, args.output.resolve()) + return 0 + except ReviewError as exc: + print(f"ios-native-review: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/native-review/performance/tooltip-fresh-dwell.yaml b/tools/native-review/performance/tooltip-fresh-dwell.yaml new file mode 100644 index 00000000000..4617fc7c3f7 --- /dev/null +++ b/tools/native-review/performance/tooltip-fresh-dwell.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: tooltip_fresh_dwell +minimum_samples: 3 +metrics: + tooltip_open_latency: + max: 1000 + max_regression_percent: 20 + process.cpu_percent_median: + max: 400 + max_regression_percent: 30 + process.resident_mb_peak: + max: 500 + max_regression_percent: 20 diff --git a/tools/native-review/pyproject.toml b/tools/native-review/pyproject.toml new file mode 100644 index 00000000000..7b022c50926 --- /dev/null +++ b/tools/native-review/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "buzz-native-review" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "PyYAML==6.0.3", +] diff --git a/tools/native-review/review_native.py b/tools/native-review/review_native.py new file mode 100755 index 00000000000..0ab6c29423e --- /dev/null +++ b/tools/native-review/review_native.py @@ -0,0 +1,895 @@ +#!/usr/bin/env python3 +"""Exact-SHA-bound native review orchestrator for Buzz Desktop.""" +from __future__ import annotations + +import argparse +import dataclasses +import datetime as dt +import hashlib +import http.server +import json +import math +import os +import pathlib +import platform +import re +import secrets +import select +import shutil +import statistics +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse +from typing import Any + +try: + import yaml +except ImportError as exc: # pragma: no cover - environment preflight + raise SystemExit("PyYAML is required (activate the repository Hermit environment)") from exc + +ROOT = pathlib.Path(os.environ.get("BUZZ_NATIVE_REVIEW_ROOT", pathlib.Path(__file__).resolve().parents[2])).resolve() +TOOL_ROOT = pathlib.Path(__file__).resolve().parent +PRODUCTION_BUNDLE_IDS = {"xyz.block.buzz.app", "xyz.block.sprout.app"} +PRODUCTION_KEYRINGS = {"buzz-desktop", "sprout-desktop"} +SECRET_NAME = re.compile(r"(AUTH|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|COOKIE)", re.I) +ALLOWED_TOP = {"schema_version", "flow", "platforms", "fixture", "record", "steps", "cleanup"} +ALLOWED_STEP = {"name", "locate", "act", "expect", "expect_for", "timeout_ms", "measure"} +PERFORMANCE_SAMPLE_MINIMUM = 3 + + +class HarnessError(RuntimeError): + pass + + +def run(command: list[str], *, cwd: pathlib.Path = ROOT, env: dict[str, str] | None = None, + check: bool = True, capture: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, + env=scrubbed_environment(include_home=True) if env is None else env, + check=check, text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None) + + +def git(*args: str) -> str: + return run(["git", *args]).stdout.strip() + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat() + + +def machine_fingerprint() -> dict[str, str]: + """Return comparison-critical host attributes without user-specific data.""" + cpu = run(["sysctl", "-n", "machdep.cpu.brand_string"], check=False).stdout.strip() + return { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "cpu": cpu or platform.processor() or "unknown", + } + + +class ProcessSampler: + """Sample app CPU and resident memory while a native journey is active.""" + + def __init__(self, pid: int, interval_seconds: float = 0.1): + self.pid = pid + self.interval_seconds = interval_seconds + self.samples: list[dict[str, float]] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._sample, daemon=True) + + def start(self) -> None: + self._thread.start() + + def _sample(self) -> None: + while not self._stop.is_set(): + result = run(["ps", "-p", str(self.pid), "-o", "%cpu=", "-o", "rss="], check=False) + fields = result.stdout.split() + if len(fields) == 2: + try: + self.samples.append({ + "elapsed_ms": time.monotonic() * 1000, + "cpu_percent": float(fields[0]), + "resident_mb": int(fields[1]) / 1024, + }) + except ValueError: + pass + self._stop.wait(self.interval_seconds) + + def finish(self) -> dict[str, Any]: + self._stop.set() + self._thread.join(timeout=2) + if not self.samples: + return {"sample_count": 0} + return { + "sample_count": len(self.samples), + "interval_ms": self.interval_seconds * 1000, + "cpu_percent_median": statistics.median(item["cpu_percent"] for item in self.samples), + "cpu_percent_peak": max(item["cpu_percent"] for item in self.samples), + "resident_mb_median": statistics.median(item["resident_mb"] for item in self.samples), + "resident_mb_peak": max(item["resident_mb"] for item in self.samples), + } + + +def validate_locator(locator: Any, where: str) -> None: + if not isinstance(locator, dict) or not locator or set(locator) - {"id", "role", "name"}: + raise HarnessError(f"{where}: locator must contain only id, role, and/or name") + if not all(isinstance(value, str) and value for value in locator.values()): + raise HarnessError(f"{where}: locator values must be non-empty strings") + + +def validate_expectation(expectation: Any, where: str) -> None: + allowed = {"exists", "not_exists", "focused", "enabled", "value", "scroll_y_greater_than", "scroll_y_less_than"} + if ( + not isinstance(expectation, dict) + or len(expectation) != 1 + or set(expectation) - allowed + ): + raise HarnessError(f"{where}: expectation must contain exactly one supported condition") + for key in ("exists", "not_exists"): + if key in expectation: + validate_locator(expectation[key], f"{where}.{key}") + focused = expectation.get("focused") + if focused is not None and not isinstance(focused, bool): + validate_locator(focused, f"{where}.focused") + if "enabled" in expectation and not isinstance(expectation["enabled"], bool): + raise HarnessError(f"{where}.enabled must be boolean") + if "value" in expectation and not isinstance(expectation["value"], str): + raise HarnessError(f"{where}.value must be a string") + for key in ("scroll_y_greater_than", "scroll_y_less_than"): + value = expectation.get(key) + if key in expectation and ( + not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value) + ): + raise HarnessError(f"{where}.{key} must be a finite number") + + +def load_journey(path: pathlib.Path) -> dict[str, Any]: + try: + journey = yaml.safe_load(path.read_text()) + except (OSError, yaml.YAMLError) as exc: + raise HarnessError(f"cannot read journey {path}: {exc}") from exc + if not isinstance(journey, dict) or set(journey) != ALLOWED_TOP: + raise HarnessError(f"journey must contain exactly {sorted(ALLOWED_TOP)}") + if journey["schema_version"] != 1 or journey["platforms"] != ["macos"]: + raise HarnessError("only schema_version 1 and platforms: [macos] are supported") + if journey["fixture"] != "local_review_channel": + raise HarnessError("desktop MVP permits only fixture: local_review_channel") + if not isinstance(journey["flow"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", journey["flow"]): + raise HarnessError("flow must be a lowercase filesystem-safe identifier") + record = journey["record"] + if not isinstance(record, dict) or set(record) != {"video", "screenshots", "accessibility"}: + raise HarnessError("record requires exactly video, screenshots, accessibility") + if record["video"] not in ("window", "off") or not all(isinstance(record[k], bool) for k in ("screenshots", "accessibility")): + raise HarnessError("invalid record policy") + steps = journey["steps"] + if not isinstance(steps, list) or not steps: + raise HarnessError("journey requires at least one step") + measurement_names: set[str] = set() + for index, step in enumerate(steps): + where = f"steps[{index}]" + if not isinstance(step, dict) or set(step) - ALLOWED_STEP or not {"name", "act", "expect"} <= set(step): + raise HarnessError(f"{where}: requires name/act/expect and contains an unsupported field") + if not isinstance(step["name"], str) or not step["name"]: + raise HarnessError(f"{where}.name must be non-empty") + locators = step.get("locate") + if locators is not None: + if not isinstance(locators, list) or not locators: + raise HarnessError(f"{where}.locate must be a non-empty list") + for locator in locators: + validate_locator(locator, f"{where}.locate") + action = step["act"] + action_type = action.get("type") if isinstance(action, dict) else None + if action_type not in {"activate", "click", "move_pointer", "press", "scroll", "type_text", "wait"}: + raise HarnessError(f"{where}.act has unsupported type") + if set(action) - {"type", "duration_ms", "key", "modifiers", "text", "delta_y"}: + raise HarnessError(f"{where}.act contains an unsupported field") + if action_type in {"click", "move_pointer", "scroll"} and locators is None: + raise HarnessError(f"{where}: {action_type} requires locate") + if action_type == "press": + if not isinstance(action.get("key"), str) or not action["key"]: + raise HarnessError(f"{where}: press requires key") + modifiers = action.get("modifiers", []) + if not isinstance(modifiers, list) or not all(isinstance(item, str) and item for item in modifiers): + raise HarnessError(f"{where}: press modifiers must be strings") + if action_type == "type_text" and not isinstance(action.get("text"), str): + raise HarnessError(f"{where}: type_text requires text") + duration = action.get("duration_ms") + if duration is not None and ( + not isinstance(duration, int) or isinstance(duration, bool) or not 0 <= duration <= 30000 + ): + raise HarnessError(f"{where}.act.duration_ms must be 0..30000") + if action_type == "scroll": + delta_y = action.get("delta_y") + if ( + not isinstance(delta_y, int) + or isinstance(delta_y, bool) + or not -10000 <= delta_y <= 10000 + ): + raise HarnessError(f"{where}: scroll delta_y must be an integer from -10000..10000") + validate_expectation(step["expect"], f"{where}.expect") + if "expect_for" in step: + sustained = step["expect_for"] + if not isinstance(sustained, dict) or set(sustained) != {"duration_ms", "condition"}: + raise HarnessError(f"{where}.expect_for requires duration_ms and condition") + sustained_duration = sustained["duration_ms"] + if ( + not isinstance(sustained_duration, int) + or isinstance(sustained_duration, bool) + or not 1 <= sustained_duration <= 30000 + ): + raise HarnessError(f"{where}.expect_for.duration_ms must be 1..30000") + validate_expectation(sustained["condition"], f"{where}.expect_for.condition") + timeout = step.get("timeout_ms", 5000) + if not isinstance(timeout, int) or isinstance(timeout, bool) or not 0 < timeout <= 60000: + raise HarnessError(f"{where}.timeout_ms must be 1..60000") + if "measure" in step: + measurement = step["measure"] + if not isinstance(measurement, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_.-]*", measurement): + raise HarnessError(f"{where}.measure must be a lowercase metric identifier") + if measurement in measurement_names: + raise HarnessError(f"duplicate measurement name: {measurement}") + measurement_names.add(measurement) + cleanup = journey["cleanup"] + if not isinstance(cleanup, dict) or set(cleanup) != {"terminate_app", "remove_state"} or not all(isinstance(v, bool) for v in cleanup.values()): + raise HarnessError("cleanup requires boolean terminate_app and remove_state") + return journey + + +def isolation_manifest(run_id: str, relay_url: str) -> dict[str, str]: + parsed = urllib.parse.urlparse(relay_url) + if parsed.scheme not in {"ws", "http"} or parsed.hostname not in {"localhost", "127.0.0.1", "::1"}: + raise HarnessError(f"refusing non-loopback review relay: {relay_url}") + slug = re.sub(r"[^a-z0-9-]", "-", run_id.lower()) + bundle_id = f"xyz.block.buzz.app.dev.native-review.{slug}" + keyring = f"buzz-desktop-dev.native-review.{slug}" + if bundle_id in PRODUCTION_BUNDLE_IDS or not bundle_id.startswith("xyz.block.buzz.app.dev.native-review."): + raise HarnessError(f"refusing unsafe bundle identifier: {bundle_id}") + if keyring in PRODUCTION_KEYRINGS or not keyring.startswith("buzz-desktop-dev.native-review."): + raise HarnessError(f"refusing unsafe keyring service: {keyring}") + return {"bundle_id": bundle_id, "keyring_service": keyring, "relay_url": relay_url} + + +def provenance() -> dict[str, Any]: + head = git("rev-parse", "HEAD") + status = git("status", "--porcelain=v1", "--untracked-files=all") + diff = run(["git", "diff", "--binary", "HEAD"]).stdout + untracked_hashes = [] + for line in status.splitlines(): + if line.startswith("?? "): + path = ROOT / line[3:] + if path.is_file(): + untracked_hashes.append((line[3:], sha256(path))) + dirty_payload = json.dumps({"diff": diff, "untracked": untracked_hashes}, sort_keys=True).encode() + return { + "head_sha": head, + "dirty": bool(status), + "dirty_state_sha256": hashlib.sha256(dirty_payload).hexdigest() if status else None, + "status": status.splitlines(), + } + + +def scrubbed_environment(*, include_home: bool = False) -> dict[str, str]: + keep = {"PATH", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", "TERM", "__CF_USER_TEXT_ENCODING"} + env = {key: value for key, value in os.environ.items() if key in keep and not SECRET_NAME.search(key)} + env["HOME"] = os.environ.get("HOME", "") if include_home else "" # isolated per run unless tooling needs host caches + return env + + +def fixture_environment(isolation: dict[str, str], review_pubkey: str) -> dict[str, str]: + """Return fixed local fixture coordinates without inheriting host credentials.""" + parsed = urllib.parse.urlparse(isolation["relay_url"]) + port = parsed.port or 80 + if port != 3030: + raise HarnessError("fixture seeding requires the isolated relay at loopback port 3030") + return { + **scrubbed_environment(include_home=True), + "BUZZ_REVIEW_PUBKEY": review_pubkey, + "BUZZ_COMMUNITY_HOST": f"{parsed.hostname}:{port}", + "BUZZ_DB_HOST": "localhost", + "BUZZ_DB_PORT": "5471", + "BUZZ_DB_USER": "buzz", + "BUZZ_DB_PASS": "buzz_dev", + "BUZZ_DB_NAME": "buzz", + "BUZZ_DB_DOCKER_CONTAINER": "buzz-harness-postgres-1", + } + + +def driver_binary() -> pathlib.Path: + override = os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER") + if override: + return pathlib.Path(override).resolve() + return TOOL_ROOT / "swift" / ".build" / "release" / "buzz-native-driver" + + +def build_driver() -> pathlib.Path: + binary = driver_binary() + if os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER"): + if not binary.is_file(): + raise HarnessError(f"configured driver does not exist: {binary}") + return binary + run(["swift", "build", "-c", "release", "--package-path", str(TOOL_ROOT / "swift")], + env=scrubbed_environment(include_home=True), capture=False) + if not binary.is_file(): + raise HarnessError(f"Swift build succeeded without driver binary: {binary}") + return binary + + +class Driver: + def __init__(self, binary: pathlib.Path, pid: int, semantic_snapshot: pathlib.Path): + self.process = subprocess.Popen([str(binary), "serve", "--pid", str(pid), + "--semantic-snapshot", str(semantic_snapshot)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=scrubbed_environment(include_home=True), text=True, bufsize=1) + + def request(self, command: str, **payload: Any) -> dict[str, Any]: + assert self.process.stdin and self.process.stdout + self.process.stdin.write(json.dumps({"command": command, **payload}) + "\n") + self.process.stdin.flush() + timeout_seconds = 45 if command in {"record_start", "record_stop"} else 15 + ready, _, _ = select.select([self.process.stdout], [], [], timeout_seconds) + if not ready: + self.process.kill() + self.process.wait(timeout=5) + stderr = self.process.stderr.read() if self.process.stderr else "" + detail = f"; driver stderr:\n{stderr.strip()}" if stderr.strip() else "" + raise HarnessError(f"native driver timed out during {command}{detail}") + line = self.process.stdout.readline() + if not line: + stderr = self.process.stderr.read() if self.process.stderr else "" + raise HarnessError(f"native driver exited during {command}: {stderr.strip()}") + response = json.loads(line) + if not response.get("ok"): + raise HarnessError(str(response.get("error", f"driver {command} failed"))) + return response + + def close(self) -> None: + if self.process.poll() is None: + try: + self.request("shutdown") + except Exception: + self.process.terminate() + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + if stream: + stream.close() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + + +def doctor(require_permissions: bool = False) -> dict[str, Any]: + checks: list[dict[str, Any]] = [] + checks.append({"name": "platform", "ok": sys.platform == "darwin", "detail": sys.platform}) + for command in ("swift", "xcrun", "git"): + path = shutil.which(command) + checks.append({"name": command, "ok": path is not None, "detail": path or "not found"}) + checks.append({"name": "repository", "ok": (ROOT / "desktop/src-tauri/tauri.conf.json").is_file(), "detail": str(ROOT)}) + try: + binary = build_driver() if all(c["ok"] for c in checks[:4]) else driver_binary() + result = run([str(binary), "doctor"], check=False) + native = json.loads(result.stdout) if result.stdout else {"ok": False, "error": result.stderr.strip()} + checks.extend(native.get("checks", [])) + except (HarnessError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + checks.append({"name": "native-driver", "ok": False, "detail": str(exc)}) + hard_names = {"platform", "swift", "xcrun", "git", "repository", "native-driver-build"} + failed = [c for c in checks if not c.get("ok") and (require_permissions or c.get("name") in hard_names)] + result = {"ok": not failed, "checks": checks} + print(json.dumps(result, indent=2)) + return result + + +def prepare_fixture(run_dir: pathlib.Path, isolation: dict[str, str]) -> dict[str, Any]: + admin = ROOT / "target" / "debug" / "buzz-admin" + if not admin.is_file(): + run(["cargo", "build", "-p", "buzz-admin"], env=scrubbed_environment(include_home=True), capture=False) + generated = run([str(admin), "generate-key"], env=scrubbed_environment(include_home=True)).stdout + secret_match = re.search(r"Secret key:\s+(\S+)", generated) + public_match = re.search(r"Public key:\s+(\S+)", generated) + if not secret_match or not public_match: + raise HarnessError("buzz-admin generate-key returned an unrecognized response") + secret_path = run_dir / "state" / "identity.key" + secret_path.parent.mkdir(parents=True, exist_ok=True) + secret_path.write_text(secret_match.group(1) + "\n") + secret_path.chmod(0o600) + fixture = { + "kind": "local_review_channel", "identity_pubkey": public_match.group(1), + "secret_path": str(secret_path), "relay_url": isolation["relay_url"], + "seed": "scripts/setup-desktop-test-data.sh", "cleanup_scope": "run-local app state and keyring only", + } + try: + run([str(ROOT / "scripts/setup-desktop-test-data.sh")], + env=fixture_environment(isolation, fixture["identity_pubkey"]), capture=False) + except Exception: + secret_path.unlink(missing_ok=True) + raise + (run_dir / "manifest" / "fixture.json").write_text(json.dumps({k: v for k, v in fixture.items() if k != "secret_path"}, indent=2)) + return fixture + + +def semantic_probe_server(path: pathlib.Path) -> tuple[http.server.ThreadingHTTPServer, str]: + class Handler(http.server.BaseHTTPRequestHandler): + def end_headers(self) -> None: + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.send_header("Access-Control-Allow-Private-Network", "true") + super().end_headers() + + def do_OPTIONS(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self.send_response(204) + self.end_headers() + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", "0")) + payload = self.rfile.read(length) + try: + value = json.loads(payload) + if not isinstance(value, list): + raise ValueError("snapshot must be an array") + temporary = path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(value)) + temporary.replace(path) + self.send_response(204) + except (ValueError, json.JSONDecodeError, OSError): + self.send_response(400) + self.end_headers() + + def log_message(self, _format: str, *_args: Any) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f"http://127.0.0.1:{server.server_port}/snapshot" + + +def build_and_launch(run_dir: pathlib.Path, isolation: dict[str, str], fixture: dict[str, Any], + probe_url: str) -> tuple[subprocess.Popen[str], pathlib.Path, int]: + # Build with the repository toolchain but no inherited credentials. Launch + # the resulting executable separately so only the app receives isolated HOME. + dev_url = ( + "http://localhost:1420?nativeReview=1" + f"&reviewRelay={urllib.parse.quote(isolation['relay_url'], safe='')}" + f"&reviewPubkey={urllib.parse.quote(fixture['identity_pubkey'], safe='')}" + ) + config = json.dumps({ + "build": {"devUrl": dev_url}, + "identifier": isolation["bundle_id"], "productName": "Buzz Native Review", + "bundle": {"externalBin": []}, + }, separators=(",", ":")) + build_env = scrubbed_environment(include_home=True) + build_env["VITE_NATIVE_REVIEW"] = "1" + build_env["VITE_NATIVE_REVIEW_RELAY"] = isolation["relay_url"] + build_env["VITE_NATIVE_REVIEW_PUBKEY"] = fixture["identity_pubkey"] + build_env["VITE_NATIVE_REVIEW_PROBE_URL"] = probe_url + run(["pnpm", "exec", "tauri", "build", "--debug", "--bundles", "app", "--config", config], + cwd=ROOT / "desktop", env=build_env, capture=False) + app_binary = (ROOT / "desktop" / "src-tauri" / "target" / "debug" / "bundle" / "macos" / + "Buzz Native Review.app" / "Contents" / "MacOS" / "buzz-desktop") + if not app_binary.is_file(): + raise HarnessError(f"Tauri build succeeded without app binary: {app_binary}") + + env = scrubbed_environment() + env["HOME"] = str(run_dir / "home") + pathlib.Path(env["HOME"]).mkdir(parents=True, exist_ok=True) + env.update({ + "BUZZ_PRIVATE_KEY": pathlib.Path(fixture["secret_path"]).read_text().strip(), + "BUZZ_RELAY_URL": isolation["relay_url"], "BUZZ_DEV_KEYRING_SERVICE": isolation["keyring_service"], + "BUZZ_NATIVE_REVIEW": "1", "BUZZ_NATIVE_REVIEW_CHANNEL": "general", + }) + log = (run_dir / "logs" / "app.log").open("w") + process = subprocess.Popen([str(app_binary)], cwd=ROOT, env=env, stdout=log, + stderr=subprocess.STDOUT, text=True) + log.close() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if process.poll() is not None: + raise HarnessError(f"Tauri exited during launch; see {run_dir / 'logs/app.log'}") + if run(["ps", "-p", str(process.pid), "-o", "comm="], check=False).stdout.rstrip().endswith("/buzz-desktop"): + return process, app_binary, process.pid + time.sleep(0.25) + process.terminate() + raise HarnessError("timed out waiting for native Buzz process") + + +def wait_for_visible_window(driver: Driver, process: subprocess.Popen[str], timeout_seconds: float = 30) -> dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + last_status: dict[str, Any] | None = None + while time.monotonic() < deadline: + if process.poll() is not None: + raise HarnessError("Tauri exited while waiting for its initial window") + last_status = driver.request("window_status") + if last_status.get("visible"): + return last_status + time.sleep(0.1) + detail = last_status.get("detail") if last_status else "driver returned no status" + raise HarnessError(f"timed out after {timeout_seconds:g}s waiting for visible native window: {detail}") + + +def locate_required(driver: Driver, locators: list[dict[str, str]], timeout_ms: int) -> dict[str, Any]: + """Wait for a semantic target to materialize, then return the locator used.""" + deadline = time.monotonic() + timeout_ms / 1000 + while True: + found = driver.request("locate", locators=locators, required=False).get("element") + if found is not None: + return found + if time.monotonic() >= deadline: + raise HarnessError(f"no accessibility element matched ordered locators within {timeout_ms}ms") + time.sleep(0.025) + + +def expectation_holds(driver: Driver, expectation: dict[str, Any]) -> bool: + if "exists" in expectation: + return driver.request("locate", locators=[expectation["exists"]], required=False).get("element") is not None + if "not_exists" in expectation: + return driver.request("locate", locators=[expectation["not_exists"]], required=False).get("element") is None + if "focused" in expectation and isinstance(expectation["focused"], dict): + found = driver.request("locate", locators=[expectation["focused"]], required=False).get("element") + return bool(found and found.get("focused")) + if "focused" in expectation: + return bool(driver.request("focused").get("focused")) == expectation["focused"] + if "enabled" in expectation: + return bool(driver.request("selected").get("element", {}).get("enabled")) == expectation["enabled"] + if "value" in expectation: + return driver.request("selected").get("element", {}).get("value") == expectation["value"] + if "scroll_y_greater_than" in expectation: + scroll_y = driver.request("selected").get("element", {}).get("scrollY") + return isinstance(scroll_y, (int, float)) and scroll_y > expectation["scroll_y_greater_than"] + if "scroll_y_less_than" in expectation: + scroll_y = driver.request("selected").get("element", {}).get("scrollY") + return isinstance(scroll_y, (int, float)) and scroll_y < expectation["scroll_y_less_than"] + return False + + +def wait_expectation(driver: Driver, expectation: dict[str, Any], timeout_ms: int) -> None: + deadline = time.monotonic() + timeout_ms / 1000 + while True: + if expectation_holds(driver, expectation): + return + if time.monotonic() >= deadline: + raise HarnessError(f"postcondition not met within {timeout_ms}ms: {expectation}") + time.sleep(0.025) + + +def capture_step(driver: Driver, run_dir: pathlib.Path, slug: str, record: dict[str, Any]) -> dict[str, str]: + artifacts: dict[str, str] = {} + if record["screenshots"]: + path = run_dir / "screenshots" / f"{slug}.png" + driver.request("screenshot", path=str(path)) + artifacts["screenshot"] = str(path.relative_to(run_dir)) + if record["accessibility"]: + path = run_dir / "accessibility" / f"{slug}.json" + response = driver.request("snapshot") + path.write_text(json.dumps({key: value for key, value in response.items() if key != "ok"}, indent=2)) + artifacts["accessibility"] = str(path.relative_to(run_dir)) + return artifacts + + +def cleanup_review_state(run_dir: pathlib.Path, isolation: dict[str, str], + fixture: dict[str, Any] | None) -> None: + errors = [] + env = scrubbed_environment() + env["HOME"] = str(run_dir / "home") + try: + run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), + isolation["bundle_id"], isolation["keyring_service"]], env=env) + except Exception as exc: + errors.append(f"desktop state reset failed: {exc}") + if fixture: + try: + pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + except Exception as exc: + errors.append(f"review identity removal failed: {exc}") + if errors: + raise HarnessError("; ".join(errors)) + + +def run_journey(path: pathlib.Path, relay_url: str, output_root: pathlib.Path) -> pathlib.Path: + journey = load_journey(path) + run_id = f"{journey['flow']}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{secrets.token_hex(3)}" + isolation = isolation_manifest(run_id, relay_url) + run_dir = output_root / git("rev-parse", "--short=12", "HEAD") / journey["flow"] / run_id + for child in ("manifest", "logs", "screenshots", "accessibility", "state", "home"): + (run_dir / child).mkdir(parents=True, exist_ok=True) + (run_dir / "journey.yaml").write_text(path.read_text()) + prov = provenance() + (run_dir / "manifest" / "git.json").write_text(json.dumps(prov, indent=2)) + (run_dir / "manifest" / "isolation.json").write_text(json.dumps(isolation, indent=2)) + started = utc_now() + receipt: dict[str, Any] = {"schema_version": 1, "run_id": run_id, "flow": journey["flow"], "status": "failed", + "started_at": started, "finished_at": started, "failure": None, "provenance": prov, "isolation": isolation, + "artifacts": {}, "steps": [], "measurements": {}, "performance": {"machine": machine_fingerprint()}, + "cleanup": {"status": "not_started"}} + process: subprocess.Popen[str] | None = None + driver: Driver | None = None + fixture: dict[str, Any] | None = None + probe_server: http.server.ThreadingHTTPServer | None = None + sampler: ProcessSampler | None = None + try: + if not doctor(require_permissions=True)["ok"]: + raise HarnessError("doctor failed; grant required permissions and rerun") + fixture = prepare_fixture(run_dir, isolation) + probe_server, probe_url = semantic_probe_server(run_dir / "state" / "semantic.json") + process, app_binary, app_pid = build_and_launch(run_dir, isolation, fixture, probe_url) + receipt["provenance"]["artifact_path"] = str(app_binary) + receipt["provenance"]["artifact_sha256"] = sha256(app_binary) + driver = Driver(build_driver(), app_pid, run_dir / "state" / "semantic.json") + sampler = ProcessSampler(app_pid) + sampler.start() + receipt["provenance"]["initial_window"] = wait_for_visible_window(driver, process) + if journey["record"]["video"] == "window": + video = run_dir / "video.mp4" + driver.request("record_start", path=str(video)) + receipt["artifacts"]["video"] = "video.mp4" + for index, step in enumerate(journey["steps"]): + slug = f"{index + 1:02d}-{re.sub(r'[^a-z0-9-]', '-', step['name'].lower())}" + step_start = time.monotonic_ns() + selected = None + step_receipt: dict[str, Any] = {"name": step["name"], "status": "failed", "started_monotonic_ns": step_start} + receipt["steps"].append(step_receipt) + try: + if step.get("locate"): + selected = locate_required(driver, step["locate"], step.get("timeout_ms", 5000)) + step_receipt["locator"] = selected.get("locator") + driver.request("act", action=step["act"], element=selected) + wait_expectation(driver, step["expect"], step.get("timeout_ms", 5000)) + if sustained := step.get("expect_for"): + until = time.monotonic() + sustained["duration_ms"] / 1000 + while time.monotonic() < until: + if not expectation_holds(driver, sustained["condition"]): + raise HarnessError(f"sustained postcondition failed: {sustained['condition']}") + time.sleep(0.025) + step_receipt["status"] = "passed" + finally: + step_receipt["finished_monotonic_ns"] = time.monotonic_ns() + step_receipt["duration_ms"] = (step_receipt["finished_monotonic_ns"] - step_start) / 1_000_000 + if measurement := step.get("measure"): + receipt["measurements"][measurement] = {"value": step_receipt["duration_ms"], "unit": "ms", "step": step["name"]} + step_receipt["artifacts"] = capture_step(driver, run_dir, slug, journey["record"]) + if journey["record"]["video"] == "window": + driver.request("record_stop") + receipt["status"] = "passed" + except Exception as exc: + receipt["failure"] = str(exc) + if driver: + try: + receipt["artifacts"]["failure"] = capture_step(driver, run_dir, "failure", {"screenshots": True, "accessibility": True}) + except Exception as capture_exc: + receipt["artifacts"]["capture_failure"] = str(capture_exc) + try: + driver.request("record_stop") + except Exception: + pass + finally: + cleanup_errors = [] + if sampler: + receipt["performance"]["process"] = sampler.finish() + if probe_server: + probe_server.shutdown() + probe_server.server_close() + if driver: + try: + driver.close() + except Exception as exc: + cleanup_errors.append(f"native driver cleanup failed: {exc}") + if process and journey["cleanup"]["terminate_app"]: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill(); cleanup_errors.append("Tauri launcher required SIGKILL") + if journey["cleanup"]["remove_state"]: + try: + cleanup_review_state(run_dir, isolation, fixture) + except Exception as exc: + cleanup_errors.append(str(exc)) + receipt["cleanup"] = {"status": "failed" if cleanup_errors else "passed", "errors": cleanup_errors} + if cleanup_errors: + receipt["status"] = "failed" + receipt["finished_at"] = utc_now() + (run_dir / "receipt.json").write_text(json.dumps(receipt, indent=2)) + report = f"# Native review: {journey['flow']}\n\n**{receipt['status'].upper()}** at `{prov['head_sha']}`.\n\nReceipt: `receipt.json`\n" + if receipt["failure"]: + report += f"\nFailure: `{receipt['failure']}`\n" + (run_dir / "report.md").write_text(report) + print(run_dir) + if receipt["status"] != "passed": + raise HarnessError(receipt["failure"] or "journey or cleanup failed") + return run_dir + + +def load_receipts(paths: list[pathlib.Path], label: str) -> list[dict[str, Any]]: + receipts = [] + for path in paths: + try: + receipt = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise HarnessError(f"cannot read {label} receipt {path}: {exc}") from exc + if receipt.get("status") != "passed" or receipt.get("cleanup", {}).get("status") != "passed": + raise HarnessError(f"{label} receipt is not a clean pass: {path}") + if receipt.get("provenance", {}).get("dirty"): + raise HarnessError(f"{label} receipt was captured from a dirty tree: {path}") + receipts.append(receipt) + run_ids = [receipt.get("run_id") for receipt in receipts] + if len(set(run_ids)) != len(run_ids): + raise HarnessError(f"{label} cohort contains duplicate run_id values") + return receipts + + +def metric_value(receipt: dict[str, Any], metric: str) -> float: + if metric.startswith("process."): + value = receipt.get("performance", {}).get("process", {}).get(metric.removeprefix("process.")) + else: + value = receipt.get("measurements", {}).get(metric, {}).get("value") + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + ): + raise HarnessError(f"receipt {receipt.get('run_id')} has no finite numeric metric {metric}") + return float(value) + + +def cohort_summary(receipts: list[dict[str, Any]], metrics: list[str]) -> dict[str, Any]: + return { + "sample_count": len(receipts), + "head_sha": receipts[0]["provenance"]["head_sha"], + "artifact_sha256": [receipt["provenance"]["artifact_sha256"] for receipt in receipts], + "metrics": { + metric: { + "median": statistics.median(values := [metric_value(receipt, metric) for receipt in receipts]), + "minimum": min(values), + "maximum": max(values), + "samples": values, + } + for metric in metrics + }, + } + + +def compare_performance(baseline_paths: list[pathlib.Path], candidate_paths: list[pathlib.Path], + budget_path: pathlib.Path, output: pathlib.Path | None = None) -> dict[str, Any]: + try: + budget = yaml.safe_load(budget_path.read_text()) + except (OSError, yaml.YAMLError) as exc: + raise HarnessError(f"cannot read performance budget {budget_path}: {exc}") from exc + if not isinstance(budget, dict) or set(budget) != {"schema_version", "flow", "minimum_samples", "metrics"}: + raise HarnessError("performance budget requires exactly schema_version, flow, minimum_samples, and metrics") + if budget["schema_version"] != 1 or not isinstance(budget["flow"], str): + raise HarnessError("unsupported performance budget schema") + minimum = budget["minimum_samples"] + if ( + not isinstance(minimum, int) + or isinstance(minimum, bool) + or minimum < PERFORMANCE_SAMPLE_MINIMUM + ): + raise HarnessError(f"minimum_samples must be at least {PERFORMANCE_SAMPLE_MINIMUM}") + metrics = budget["metrics"] + if not isinstance(metrics, dict) or not metrics: + raise HarnessError("performance budget requires at least one metric") + allowed_limits = {"max", "max_regression_percent"} + for name, limits in metrics.items(): + if (not isinstance(name, str) or not name or not isinstance(limits, dict) or not limits + or set(limits) - allowed_limits + or not all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + for value in limits.values() + )): + raise HarnessError(f"invalid limits for performance metric {name}") + + baseline = load_receipts(baseline_paths, "baseline") + candidate = load_receipts(candidate_paths, "candidate") + if len(baseline) < minimum or len(candidate) < minimum: + raise HarnessError(f"performance comparison requires at least {minimum} clean samples per cohort") + all_receipts = baseline + candidate + run_ids = [receipt.get("run_id") for receipt in all_receipts] + if len(set(run_ids)) != len(run_ids): + raise HarnessError("baseline and candidate cohorts contain duplicate run_id values") + flows = {receipt.get("flow") for receipt in all_receipts} + machines = {json.dumps(receipt.get("performance", {}).get("machine"), sort_keys=True) for receipt in all_receipts} + if flows != {budget["flow"]}: + raise HarnessError(f"receipt flows {sorted(str(item) for item in flows)} do not match budget flow {budget['flow']}") + if len(machines) != 1: + raise HarnessError("baseline and candidate receipts were captured on incompatible machines") + for label, cohort in (("baseline", baseline), ("candidate", candidate)): + if len({receipt["provenance"].get("head_sha") for receipt in cohort}) != 1: + raise HarnessError(f"{label} cohort mixes source revisions") + baseline_sha = baseline[0]["provenance"].get("head_sha") + candidate_sha = candidate[0]["provenance"].get("head_sha") + if baseline_sha == candidate_sha: + raise HarnessError("baseline and candidate cohorts must use different source revisions") + + baseline_summary = cohort_summary(baseline, list(metrics)) + candidate_summary = cohort_summary(candidate, list(metrics)) + verdicts = {} + failures = [] + for name, limits in metrics.items(): + baseline_median = baseline_summary["metrics"][name]["median"] + candidate_median = candidate_summary["metrics"][name]["median"] + candidate_maximum = candidate_summary["metrics"][name]["maximum"] + regression = None if baseline_median == 0 and candidate_median > 0 else ( + 0.0 if baseline_median == 0 else (candidate_median - baseline_median) / baseline_median * 100 + ) + reasons = [] + if "max" in limits and candidate_maximum > limits["max"]: + reasons.append(f"maximum {candidate_maximum:.3f} exceeds absolute maximum {limits['max']}") + if "max_regression_percent" in limits: + if regression is None: + reasons.append("relative regression is undefined because the baseline median is zero") + elif regression > limits["max_regression_percent"]: + reasons.append(f"regression {regression:.2f}% exceeds {limits['max_regression_percent']}%") + verdicts[name] = {"status": "failed" if reasons else "passed", "regression_percent": regression, "reasons": reasons} + failures.extend(f"{name}: {reason}" for reason in reasons) + result = { + "schema_version": 1, "status": "failed" if failures else "passed", "flow": budget["flow"], + "machine": all_receipts[0]["performance"]["machine"], "baseline": baseline_summary, + "candidate": candidate_summary, "verdicts": verdicts, "failures": failures, + } + serialized = json.dumps(result, indent=2, allow_nan=False) + if output: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(serialized + "\n") + print(serialized) + if failures: + raise HarnessError("performance budget failed: " + "; ".join(failures)) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + doctor_parser = sub.add_parser("doctor") + doctor_parser.add_argument("--require-permissions", action="store_true") + validate_parser = sub.add_parser("validate") + validate_parser.add_argument("journey", type=pathlib.Path) + run_parser = sub.add_parser("run") + run_parser.add_argument("journey", type=pathlib.Path) + run_parser.add_argument("--relay", default="ws://localhost:3030") + run_parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + benchmark_parser = sub.add_parser("benchmark") + benchmark_parser.add_argument("journey", type=pathlib.Path) + benchmark_parser.add_argument("--runs", type=int, default=5) + benchmark_parser.add_argument("--relay", default="ws://localhost:3030") + benchmark_parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + compare_parser = sub.add_parser("compare") + compare_parser.add_argument("--baseline", type=pathlib.Path, action="append", required=True) + compare_parser.add_argument("--candidate", type=pathlib.Path, action="append", required=True) + compare_parser.add_argument("--budget", type=pathlib.Path, required=True) + compare_parser.add_argument("--output", type=pathlib.Path) + args = parser.parse_args() + try: + if args.command == "doctor": + return 0 if doctor(args.require_permissions)["ok"] else 1 + if args.command == "validate": + load_journey(args.journey); print(f"valid: {args.journey}"); return 0 + if args.command == "compare": + compare_performance(args.baseline, args.candidate, args.budget, args.output); return 0 + if args.command == "benchmark": + if args.runs < PERFORMANCE_SAMPLE_MINIMUM: + raise HarnessError(f"benchmark requires at least {PERFORMANCE_SAMPLE_MINIMUM} runs") + receipts = [run_journey(args.journey, args.relay, args.output) / "receipt.json" for _ in range(args.runs)] + print(json.dumps({"receipts": [str(path) for path in receipts]}, indent=2)); return 0 + run_journey(args.journey, args.relay, args.output); return 0 + except HarnessError as exc: + print(f"native-review: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/native-review/schemas/journey.schema.json b/tools/native-review/schemas/journey.schema.json new file mode 100644 index 00000000000..0db284ff05e --- /dev/null +++ b/tools/native-review/schemas/journey.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://buzz.local/native-review/journey.schema.json", + "title": "Buzz native review journey", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "flow", "platforms", "fixture", "record", "steps", "cleanup"], + "properties": { + "schema_version": {"const": 1}, + "flow": {"type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$"}, + "platforms": {"type": "array", "minItems": 1, "items": {"enum": ["macos"]}}, + "fixture": {"enum": ["local_review_channel"]}, + "record": { + "type": "object", "additionalProperties": false, + "required": ["video", "screenshots", "accessibility"], + "properties": { + "video": {"enum": ["window", "off"]}, + "screenshots": {"type": "boolean"}, + "accessibility": {"type": "boolean"} + } + }, + "steps": { + "type": "array", "minItems": 1, + "items": { + "type": "object", "additionalProperties": false, + "required": ["name", "act", "expect"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "locate": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/locator"}}, + "act": {"$ref": "#/$defs/action"}, + "expect": {"$ref": "#/$defs/expectation"}, + "expect_for": { + "type": "object", "additionalProperties": false, + "required": ["duration_ms", "condition"], + "properties": { + "duration_ms": {"type": "integer", "minimum": 1, "maximum": 30000}, + "condition": {"$ref": "#/$defs/expectation"} + } + }, + "timeout_ms": {"type": "integer", "minimum": 1, "maximum": 60000}, + "measure": {"type": "string", "minLength": 1} + } + } + }, + "cleanup": { + "type": "object", "additionalProperties": false, + "required": ["terminate_app", "remove_state"], + "properties": {"terminate_app": {"type": "boolean"}, "remove_state": {"type": "boolean"}} + } + }, + "$defs": { + "locator": { + "type": "object", "additionalProperties": false, "minProperties": 1, + "properties": {"id": {"type": "string"}, "role": {"type": "string"}, "name": {"type": "string"}} + }, + "action": { + "type": "object", "additionalProperties": false, "required": ["type"], + "properties": { + "type": {"enum": ["activate", "click", "move_pointer", "press", "scroll", "type_text", "wait"]}, + "duration_ms": {"type": "integer", "minimum": 0, "maximum": 30000}, + "key": {"type": "string"}, + "modifiers": {"type": "array", "uniqueItems": true, "items": {"enum": ["command", "control", "option", "shift"]}}, + "text": {"type": "string"}, + "delta_y": {"type": "integer", "minimum": -10000, "maximum": 10000} + } + }, + "expectation": { + "type": "object", "additionalProperties": false, "minProperties": 1, "maxProperties": 1, + "properties": { + "exists": {"$ref": "#/$defs/locator"}, + "not_exists": {"$ref": "#/$defs/locator"}, + "focused": {"oneOf": [{"type": "boolean"}, {"$ref": "#/$defs/locator"}]}, + "enabled": {"type": "boolean"}, + "value": {"type": "string"}, + "scroll_y_greater_than": {"type": "number"}, + "scroll_y_less_than": {"type": "number"} + } + } + } +} diff --git a/tools/native-review/schemas/receipt.schema.json b/tools/native-review/schemas/receipt.schema.json new file mode 100644 index 00000000000..7839aa36dc2 --- /dev/null +++ b/tools/native-review/schemas/receipt.schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://buzz.local/native-review/receipt.schema.json", + "title": "Buzz native review receipt", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "flow", + "status", + "started_at", + "finished_at", + "provenance", + "isolation", + "artifacts", + "steps", + "cleanup", + "measurements", + "performance" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "run_id": { + "type": "string" + }, + "flow": { + "type": "string" + }, + "status": { + "enum": [ + "passed", + "failed" + ] + }, + "started_at": { + "type": "string" + }, + "finished_at": { + "type": "string" + }, + "failure": { + "type": [ + "string", + "null" + ] + }, + "provenance": { + "type": "object" + }, + "isolation": { + "type": "object" + }, + "artifacts": { + "type": "object" + }, + "steps": { + "type": "array", + "items": { + "type": "object" + } + }, + "cleanup": { + "type": "object" + }, + "measurements": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "value", + "unit", + "step" + ], + "properties": { + "value": { + "type": "number" + }, + "unit": { + "const": "ms" + }, + "step": { + "type": "string" + } + } + } + }, + "performance": { + "type": "object", + "required": [ + "machine" + ], + "properties": { + "machine": { + "type": "object" + }, + "process": { + "type": "object" + } + } + } + } +} diff --git a/tools/native-review/swift/Package.swift b/tools/native-review/swift/Package.swift new file mode 100644 index 00000000000..3fc591eaa1a --- /dev/null +++ b/tools/native-review/swift/Package.swift @@ -0,0 +1,9 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "BuzzNativeDriver", + platforms: [.macOS(.v13)], + products: [.executable(name: "buzz-native-driver", targets: ["BuzzNativeDriver"])], + targets: [.executableTarget(name: "BuzzNativeDriver")] +) diff --git a/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift b/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift new file mode 100644 index 00000000000..7685e7e14a3 --- /dev/null +++ b/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift @@ -0,0 +1,648 @@ +import AppKit +import ApplicationServices +import AVFoundation +import CoreGraphics +import Foundation + +struct Locator: Codable { + let id: String? + let role: String? + let name: String? +} + +struct Size: Codable { + let width: Double + let height: Double +} + +struct SemanticNode: Codable { + let id: String? + let role: String? + let name: String? + let value: String? + let scrollY: Double + let enabled: Bool + let focused: Bool + let frame: Rect + let viewport: Size +} + +struct ElementDescription: Codable { + let locator: Locator + let role: String? + let name: String? + let identifier: String? + let value: String? + let scrollY: Double? + let enabled: Bool + let focused: Bool + let frame: Rect? +} + +struct Rect: Codable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct AXNode: Codable { + let role: String? + let name: String? + let identifier: String? + let value: String? + let enabled: Bool? + let focused: Bool? + let frame: Rect? + let children: [AXNode] + let truncated: Bool? +} + +enum DriverError: Error, CustomStringConvertible { + case message(String) + var description: String { + switch self { case .message(let value): return value } + } +} + +func enableWebViewAccessibility(_ app: AXUIElement) { + // WebKit does not materialize its remote accessibility tree for ordinary + // automation clients until manual/enhanced accessibility is requested. + // VoiceOver does this implicitly; the review harness must not require it. + let enabled = kCFBooleanTrue as CFTypeRef + for name in ["AXManualAccessibility", "AXEnhancedUserInterface"] { + _ = AXUIElementSetAttributeValue(app, name as CFString, enabled) + } +} + +func attribute(_ element: AXUIElement, _ name: String) -> AnyObject? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else { return nil } + return value +} + +func stringAttribute(_ element: AXUIElement, _ names: [String]) -> String? { + for name in names { + if let value = attribute(element, name) as? String, !value.isEmpty { return value } + } + return nil +} + +func boolAttribute(_ element: AXUIElement, _ name: String, default fallback: Bool = false) -> Bool { + (attribute(element, name) as? Bool) ?? fallback +} + +func frameAttribute(_ element: AXUIElement) -> Rect? { + guard let positionObject = attribute(element, kAXPositionAttribute), + let sizeObject = attribute(element, kAXSizeAttribute), + CFGetTypeID(positionObject) == AXValueGetTypeID(), + CFGetTypeID(sizeObject) == AXValueGetTypeID() else { return nil } + let positionValue = positionObject as! AXValue + let sizeValue = sizeObject as! AXValue + var point = CGPoint.zero + var size = CGSize.zero + guard AXValueGetValue(positionValue, .cgPoint, &point), AXValueGetValue(sizeValue, .cgSize, &size) else { return nil } + return Rect(x: point.x, y: point.y, width: size.width, height: size.height) +} + +func normalizedRole(_ raw: String?) -> String? { + guard var value = raw else { return nil } + if value.hasPrefix("AX") { value.removeFirst(2) } + switch value.lowercased() { + case "textarea", "text area": return "text-area" + default: return value.lowercased() + } +} + +func elementName(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute, kAXValueAttribute]) +} + +func elementIdentifier(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXIdentifierAttribute]) +} + +func elementRole(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXRoleAttribute]) +} + +func matches(_ element: AXUIElement, locator: Locator) -> Bool { + if let id = locator.id, elementIdentifier(element) != id { return false } + if let role = locator.role, normalizedRole(elementRole(element)) != normalizedRole(role) { return false } + if let name = locator.name, elementName(element) != name { return false } + return true +} + +func children(_ element: AXUIElement) -> [AXUIElement] { + let direct = (attribute(element, kAXChildrenAttribute) as? [AXUIElement]) ?? [] + let windows = (attribute(element, kAXWindowsAttribute) as? [AXUIElement]) ?? [] + let singular = [kAXMainWindowAttribute, kAXFocusedWindowAttribute].compactMap { + attribute(element, $0) as! AXUIElement? + } + var seen = Set() + return (direct + windows + singular).filter { child in + guard !CFEqual(child, element) else { return false } + return seen.insert(CFHash(child)).inserted + } +} + +func accessibilityRoots(app: AXUIElement, pid: pid_t) -> [AXUIElement] { + // WKWebView's remote tree is not always attached to AXChildren/AXWindows, + // even after manual accessibility is enabled. Hit-testing the visible app + // window asks the accessibility server for the element actually painted at + // each point and reliably materializes the WebKit subtree without DOM IPC. + var roots = [app] + guard let (_, bounds) = try? windowInfo(pid: pid) else { return roots } + let system = AXUIElementCreateSystemWide() + for xFraction in stride(from: 0.05, through: 0.95, by: 0.1) { + for yFraction in stride(from: 0.05, through: 0.95, by: 0.1) { + var element: AXUIElement? + let x = Float(bounds.minX + bounds.width * xFraction) + let y = Float(bounds.minY + bounds.height * yFraction) + if AXUIElementCopyElementAtPosition(system, x, y, &element) == .success, + let element { + roots.append(element) + } + } + } + var seen = Set() + return roots.filter { seen.insert(CFHash($0)).inserted } +} + +func find(_ roots: [AXUIElement], locator: Locator, maxNodes: Int = 20_000) -> AXUIElement? { + var queue = roots + var seen = Set() + var visited = 0 + while !queue.isEmpty && visited < maxNodes { + let current = queue.removeFirst() + guard seen.insert(CFHash(current)).inserted else { continue } + visited += 1 + if matches(current, locator: locator) { return current } + queue.append(contentsOf: children(current)) + } + return nil +} + +func describe(_ element: AXUIElement, locator: Locator) -> ElementDescription { + ElementDescription(locator: locator, role: normalizedRole(elementRole(element)), name: elementName(element), + identifier: elementIdentifier(element), value: stringAttribute(element, [kAXValueAttribute]), + scrollY: nil, enabled: boolAttribute(element, kAXEnabledAttribute, default: true), + focused: boolAttribute(element, kAXFocusedAttribute), frame: frameAttribute(element)) +} + +func semanticNodes(path: String, pid: pid_t) -> [SemanticNode] { + guard let data = FileManager.default.contents(atPath: path), + let nodes = try? JSONDecoder().decode([SemanticNode].self, from: data) else { return [] } + guard let (_, windowBounds) = try? windowInfo(pid: pid) else { return nodes } + return nodes.map { node in + guard node.viewport.width > 0, node.viewport.height > 0 else { return node } + let xScale = windowBounds.width / node.viewport.width + let yScale = windowBounds.height / node.viewport.height + return SemanticNode( + id: node.id, + role: node.role, + name: node.name, + value: node.value, + scrollY: node.scrollY, + enabled: node.enabled, + focused: node.focused, + frame: Rect( + x: windowBounds.minX + node.frame.x * xScale, + y: windowBounds.minY + node.frame.y * yScale, + width: node.frame.width * xScale, + height: node.frame.height * yScale + ), + viewport: node.viewport + ) + } +} + +func matches(_ element: SemanticNode, locator: Locator) -> Bool { + if let id = locator.id, element.id != id { return false } + if let role = locator.role, normalizedRole(element.role) != normalizedRole(role) { return false } + if let name = locator.name, element.name != name { return false } + return true +} + +func describe(_ element: SemanticNode, locator: Locator) -> ElementDescription { + ElementDescription(locator: locator, role: normalizedRole(element.role), name: element.name, + identifier: element.id, value: element.value, scrollY: element.scrollY, + enabled: element.enabled, focused: element.focused, frame: element.frame) +} + +func snapshot(_ element: AXUIElement, depth: Int = 0, budget: inout Int) -> AXNode { + budget -= 1 + let isTruncated = budget <= 0 || depth >= 80 + let descendants = isTruncated ? [] : children(element).map { snapshot($0, depth: depth + 1, budget: &budget) } + return AXNode(role: normalizedRole(elementRole(element)), name: elementName(element), identifier: elementIdentifier(element), + value: stringAttribute(element, [kAXValueAttribute]), enabled: attribute(element, kAXEnabledAttribute) as? Bool, + focused: attribute(element, kAXFocusedAttribute) as? Bool, frame: frameAttribute(element), + children: descendants, truncated: isTruncated ? true : nil) +} + +func windowInfo(pid: pid_t) throws -> (CGWindowID, CGRect) { + guard let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + throw DriverError.message("cannot enumerate windows; Screen Recording permission may be missing") + } + let candidates = windows.compactMap { window -> (CGWindowID, CGRect)? in + guard (window[kCGWindowOwnerPID as String] as? Int32) == pid, + let number = window[kCGWindowNumber as String] as? CGWindowID, + let boundsValue = window[kCGWindowBounds as String], + CFGetTypeID(boundsValue as CFTypeRef) == CFDictionaryGetTypeID(), + let bounds = CGRect(dictionaryRepresentation: boundsValue as! CFDictionary), + bounds.width > 0, bounds.height > 0 else { return nil } + return (number, bounds) + } + if let largest = candidates.max(by: { $0.1.width * $0.1.height < $1.1.width * $1.1.height }) { + return largest + } + throw DriverError.message("no on-screen window found for pid \(pid)") +} + +func windowStatus(pid: pid_t) -> [String: Any] { + do { + let (windowID, bounds) = try windowInfo(pid: pid) + return [ + "ok": true, + "visible": true, + "window_id": windowID, + "bounds": ["x": bounds.origin.x, "y": bounds.origin.y, "width": bounds.width, "height": bounds.height], + ] + } catch { + return ["ok": true, "visible": false, "detail": String(describing: error)] + } +} + +func captureWindow(pid: pid_t, path: String) throws { + let (windowID, bounds) = try windowInfo(pid: pid) + guard let image = CGWindowListCreateImage(bounds, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution]) else { + throw DriverError.message("window screenshot failed") + } + let destination = URL(fileURLWithPath: path) + try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + guard let bitmap = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]) else { + throw DriverError.message("PNG encoding failed") + } + try bitmap.write(to: destination, options: .atomic) +} + +final class WindowRecorder: @unchecked Sendable { + private var writer: AVAssetWriter? + private var input: AVAssetWriterInput? + private var adaptor: AVAssetWriterInputPixelBufferAdaptor? + private var captureTask: Task? + private var captureError: Error? + + private func trace(_ message: String) { + let timestamp = ISO8601DateFormatter().string(from: Date()) + FileHandle.standardError.write(Data("[\(timestamp)] record: \(message)\n".utf8)) + } + + func start(pid: pid_t, path: String) throws { + let (windowID, bounds) = try windowInfo(pid: pid) + let width = max(Int(bounds.width) & ~1, 2) + let height = max(Int(bounds.height) & ~1, 2) + let destination = URL(fileURLWithPath: path) + try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: destination) + + let assetWriter = try AVAssetWriter(outputURL: destination, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + ] + let writerInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + writerInput.expectsMediaDataInRealTime = true + let attributes: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + kCVPixelBufferCGImageCompatibilityKey as String: true, + kCVPixelBufferCGBitmapContextCompatibilityKey as String: true, + ] + let pixelAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: writerInput, sourcePixelBufferAttributes: attributes) + guard assetWriter.canAdd(writerInput) else { throw DriverError.message("AVAssetWriter rejected window video input") } + assetWriter.add(writerInput) + guard assetWriter.startWriting() else { throw assetWriter.error ?? DriverError.message("AVAssetWriter failed to start") } + assetWriter.startSession(atSourceTime: .zero) + writer = assetWriter; input = writerInput; adaptor = pixelAdaptor + trace("started CGWindow/AVAssetWriter backend window=\(windowID) size=\(width)x\(height)") + + captureTask = Task.detached { [weak self] in + guard let self else { return } + let frameInterval = Duration.milliseconds(1000 / 15) + var frame: Int64 = 0 + while !Task.isCancelled { + if !writerInput.isReadyForMoreMediaData { + try? await Task.sleep(for: frameInterval) + continue + } + autoreleasepool { + guard let image = CGWindowListCreateImage(bounds, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution]), + let pool = pixelAdaptor.pixelBufferPool else { return } + var optionalBuffer: CVPixelBuffer? + guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &optionalBuffer) == kCVReturnSuccess, + let buffer = optionalBuffer else { return } + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + guard let base = CVPixelBufferGetBaseAddress(buffer), + let context = CGContext(data: base, width: width, height: height, + bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else { return } + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + let time = CMTime(value: frame, timescale: 15) + if !pixelAdaptor.append(buffer, withPresentationTime: time) { + self.captureError = assetWriter.error ?? DriverError.message("failed to append window video frame") + } + frame += 1 + } + try? await Task.sleep(for: frameInterval) + } + } + } + + func stop() async throws { + captureTask?.cancel() + _ = await captureTask?.result + captureTask = nil + input?.markAsFinished() + if let assetWriter = writer { await assetWriter.finishWriting() } + let error = captureError ?? writer?.error + writer = nil; input = nil; adaptor = nil; captureError = nil + if let error { throw error } + trace("finalized recording") + } +} + +func jsonObject(_ data: Data) throws -> [String: Any] { + guard let value = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw DriverError.message("request must be a JSON object") + } + return value +} + +func locator(from value: Any) throws -> Locator { + let data = try JSONSerialization.data(withJSONObject: value) + return try JSONDecoder().decode(Locator.self, from: data) +} + +func response(_ object: [String: Any]) { + let data = try! JSONSerialization.data(withJSONObject: object) + print(String(data: data, encoding: .utf8)!) + fflush(stdout) +} + +func encoded(_ value: T) throws -> Any { + let data = try JSONEncoder().encode(value) + return try JSONSerialization.jsonObject(with: data) +} + +func keyCode(_ key: String) throws -> CGKeyCode { + switch key.lowercased() { + case "tab": return 48 + case "return", "enter": return 36 + case "escape": return 53 + case "space": return 49 + case "k": return 40 + default: throw DriverError.message("unsupported key: \(key)") + } +} + +func modifierFlags(_ names: [String]) throws -> CGEventFlags { + try names.reduce(into: CGEventFlags()) { flags, name in + switch name.lowercased() { + case "command", "cmd": flags.insert(.maskCommand) + case "control", "ctrl": flags.insert(.maskControl) + case "option", "alt": flags.insert(.maskAlternate) + case "shift": flags.insert(.maskShift) + default: throw DriverError.message("unsupported modifier: \(name)") + } + } +} + +func postKey(_ key: String, modifiers: [String] = []) throws { + let code = try keyCode(key) + let flags = try modifierFlags(modifiers) + for keyDown in [true, false] { + guard let event = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: keyDown) else { + throw DriverError.message("could not create keyboard event") + } + event.flags = flags + event.post(tap: .cghidEventTap) + } +} + +func postUnicodeScalar(_ scalar: Unicode.Scalar) throws { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { + throw DriverError.message("could not create text event") + } + let units = Array(String(scalar).utf16) + units.withUnsafeBufferPointer { buffer in + down.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress) + up.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress) + } + down.flags = [] + up.flags = [] + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) +} + +func postText(_ text: String) throws { + for scalar in text.unicodeScalars { + if scalar == "\n" { + // Buzz submits the composer on plain Enter. A newline in typed text + // therefore has to follow the same native path as a user: Shift+Enter. + try postKey("return", modifiers: ["shift"]) + } else if scalar != "\r" { + try postUnicodeScalar(scalar) + } + } +} + +@main +struct BuzzNativeDriver { + static func main() async { + do { + let arguments = Array(CommandLine.arguments.dropFirst()) + guard let command = arguments.first else { throw DriverError.message("usage: buzz-native-driver doctor | serve --pid PID") } + if command == "doctor" { + let accessibility = AXIsProcessTrusted() + let screen = CGPreflightScreenCaptureAccess() + response(["ok": true, "checks": [ + ["name": "native-driver-build", "ok": true, "detail": "Swift AX/CGEvent/Core Graphics/AVFoundation driver available"], + ["name": "accessibility-permission", "ok": accessibility, "detail": accessibility ? "granted" : "grant Accessibility to the invoking terminal/agent"], + ["name": "screen-recording-permission", "ok": screen, "detail": screen ? "granted" : "grant Screen Recording to the invoking terminal/agent"], + ["name": "recording-api", "ok": ProcessInfo.processInfo.isOperatingSystemAtLeast(.init(majorVersion: 15, minorVersion: 0, patchVersion: 0)), "detail": ProcessInfo.processInfo.operatingSystemVersionString] + ]]) + return + } + guard command == "serve", (arguments.count == 3 || arguments.count == 5), arguments[1] == "--pid", let pid = pid_t(arguments[2]) else { + throw DriverError.message("usage: buzz-native-driver serve --pid PID [--semantic-snapshot PATH]") + } + let semanticSnapshotPath: String? = arguments.count == 5 && arguments[3] == "--semantic-snapshot" ? arguments[4] : nil + guard AXIsProcessTrusted() else { throw DriverError.message("Accessibility permission is not granted") } + let app = AXUIElementCreateApplication(pid) + enableWebViewAccessibility(app) + var selected: (ElementDescription, Locator)? + var recorder: Any? + for try await line in FileHandle.standardInput.bytes.lines { + do { + let request = try jsonObject(Data(line.utf8)) + guard let name = request["command"] as? String else { throw DriverError.message("missing command") } + switch name { + case "locate": + guard let values = request["locators"] as? [Any] else { throw DriverError.message("locate requires locators") } + var found: (ElementDescription, Locator)? + for value in values { + let candidate = try locator(from: value) + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: candidate) }) { + found = (describe(element, locator: candidate), candidate) + break + } + let semanticActive = semanticSnapshotPath.map { FileManager.default.fileExists(atPath: $0) } ?? false + if !semanticActive || candidate.role == "window" { + if let element = find(accessibilityRoots(app: app, pid: pid), locator: candidate) { + found = (describe(element, locator: candidate), candidate) + break + } + } + } + selected = found + if let (element, _) = found { + response(["ok": true, "element": try encoded(element)]) + } else if (request["required"] as? Bool) == true { + throw DriverError.message("no accessibility element matched ordered locators") + } else { response(["ok": true, "element": NSNull()]) } + case "act": + guard let action = request["action"] as? [String: Any], let type = action["type"] as? String else { throw DriverError.message("act requires action.type") } + if type == "activate" { + guard let running = NSRunningApplication(processIdentifier: pid) else { throw DriverError.message("target app is no longer running") } + running.activate(options: [.activateIgnoringOtherApps]) + } else if type == "wait" { + try await Task.sleep(for: .milliseconds(action["duration_ms"] as? Int ?? 0)) + } else if type == "press" { + try postKey(action["key"] as? String ?? "", modifiers: action["modifiers"] as? [String] ?? []) + } else if type == "type_text" { + try postText(action["text"] as? String ?? "") + } else if type == "scroll" { + guard let (_, used) = selected else { + throw DriverError.message("scroll requires a freshly selected element with current bounds") + } + var fresh: ElementDescription? + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: used) }) { + fresh = describe(element, locator: used) + } + if fresh == nil, let element = find(accessibilityRoots(app: app, pid: pid), locator: used) { + fresh = describe(element, locator: used) + } + guard let element = fresh, let frame = element.frame else { + throw DriverError.message("scroll requires a freshly selected element with current bounds") + } + selected = (element, used) + let point = CGPoint(x: frame.x + frame.width / 2, y: frame.y + frame.height / 2) + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, + mouseButton: .left)?.post(tap: .cghidEventTap) + guard let rawDeltaY = action["delta_y"] as? Int, + let deltaY = Int32(exactly: rawDeltaY) else { + throw DriverError.message("scroll delta_y must fit in Int32") + } + guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 1, + wheel1: deltaY, wheel2: 0, wheel3: 0) else { + throw DriverError.message("could not create scroll event") + } + event.location = point + event.post(tap: .cghidEventTap) + } else { + guard let (_, used) = selected else { + throw DriverError.message("action requires a freshly selected element with current bounds") + } + var fresh: ElementDescription? + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: used) }) { + fresh = describe(element, locator: used) + } + if fresh == nil, let element = find(accessibilityRoots(app: app, pid: pid), locator: used) { + fresh = describe(element, locator: used) + } + guard let element = fresh, let frame = element.frame else { + throw DriverError.message("action requires a freshly selected element with current bounds") + } + selected = (element, used) + let point = CGPoint(x: frame.x + frame.width / 2, y: frame.y + frame.height / 2) + let duration = max(action["duration_ms"] as? Int ?? 0, 0) + if type == "move_pointer" && duration > 0 { + let current = NSEvent.mouseLocation + let start = CGPoint(x: current.x, y: NSScreen.screens.first.map { $0.frame.height - current.y } ?? current.y) + let steps = max(duration / 8, 2) + for index in 1...steps { + let fraction = Double(index) / Double(steps) + let next = CGPoint(x: start.x + (point.x - start.x) * fraction, y: start.y + (point.y - start.y) * fraction) + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: next, mouseButton: .left)?.post(tap: .cghidEventTap) + try await Task.sleep(for: .milliseconds(duration / steps)) + } + } else if type == "move_pointer" { + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + } else if type == "click" { + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + } else { throw DriverError.message("unsupported action: \(type)") } + } + response(["ok": true]) + case "window_status": + response(windowStatus(pid: pid)) + case "snapshot": + var budget = 20_000 + var object: [String: Any] = ["ok": true, "tree": try encoded(snapshot(app, budget: &budget))] + if let path = semanticSnapshotPath { + object["semantic"] = try encoded(semanticNodes(path: path, pid: pid)) + } + response(object) + case "screenshot": + guard let path = request["path"] as? String else { throw DriverError.message("screenshot requires path") } + try captureWindow(pid: pid, path: path); response(["ok": true]) + case "focused": + response(["ok": true, "focused": selected?.0.focused ?? false]) + case "selected": + if let (_, used) = selected { + var fresh: ElementDescription? + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: used) }) { + fresh = describe(element, locator: used) + } + if fresh == nil, let element = find(accessibilityRoots(app: app, pid: pid), locator: used) { + fresh = describe(element, locator: used) + } + if let element = fresh { + selected = (element, used) + response(["ok": true, "element": try encoded(element)]) + } else { + selected = nil + response(["ok": true, "element": NSNull()]) + } + } else { response(["ok": true, "element": NSNull()]) } + case "record_start": + guard let path = request["path"] as? String else { throw DriverError.message("record_start requires path") } + let value = WindowRecorder(); try value.start(pid: pid, path: path); recorder = value; response(["ok": true]) + case "record_stop": + if let value = recorder as? WindowRecorder { try await value.stop() } + recorder = nil; response(["ok": true]) + case "shutdown": response(["ok": true]); return + default: throw DriverError.message("unknown command: \(name)") + } + } catch { + response(["ok": false, "error": String(describing: error)]) + } + } + } catch { + response(["ok": false, "error": String(describing: error)]) + exit(1) + } + } +} diff --git a/tools/native-review/tests/fixtures/broken-scroll.yaml b/tools/native-review/tests/fixtures/broken-scroll.yaml new file mode 100644 index 00000000000..4eb318cb626 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-scroll.yaml @@ -0,0 +1,34 @@ +schema_version: 1 +flow: broken_scroll +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: {not_exists: {id: app-loading-gate}} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + act: {type: click} + expect: {exists: {id: message-composer}} + timeout_ms: 15000 + - name: focus_composer + locate: + - {id: message-input} + act: {type: click} + expect: {focused: {id: message-input}} + - name: create_scrollable_draft + act: {type: type_text, text: "line 1\nline 2\nline 3\nline 4\nline 5\nline 6\nline 7\nline 8\nline 9\nline 10\nline 11\nline 12\nline 13\nline 14\nline 15"} + expect: {value: "line 1\nline 2\nline 3\nline 4\nline 5\nline 6\nline 7\nline 8\nline 9\nline 10\nline 11\nline 12\nline 13\nline 14\nline 15"} + - name: wrong_scroll_postcondition + locate: + - {id: message-input-scroll} + act: {type: scroll, delta_y: 240} + # The draft starts at the caret-driven bottom (~172px). Scrolling down moves + # it to 0, so this impossible >200 assertion cannot pass on either the + # pre-action snapshot or the settled result. + expect: {scroll_y_greater_than: 200} + timeout_ms: 100 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/fixtures/broken-shortcut.yaml b/tools/native-review/tests/fixtures/broken-shortcut.yaml new file mode 100644 index 00000000000..6e097ac0ae6 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-shortcut.yaml @@ -0,0 +1,11 @@ +schema_version: 1 +flow: broken_shortcut +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: wrong_shortcut + act: {type: press, key: k} + expect: {exists: {id: search-results}} + timeout_ms: 100 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/fixtures/broken-text.yaml b/tools/native-review/tests/fixtures/broken-text.yaml new file mode 100644 index 00000000000..011b928ee42 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-text.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: broken_text +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: impossible_text_postcondition + locate: + - {id: open-search} + act: {type: click} + expect: {value: "this can never match the selected button"} + timeout_ms: 50 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/fixtures/broken-tooltip.yaml b/tools/native-review/tests/fixtures/broken-tooltip.yaml new file mode 100644 index 00000000000..4631454f729 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-tooltip.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: broken_tooltip +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: impossible_locator + locate: [{id: this-element-must-not-exist}] + act: {type: click} + expect: + exists: {id: still-impossible} + timeout_ms: 50 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/test_ios_review.py b/tools/native-review/tests/test_ios_review.py new file mode 100644 index 00000000000..c08f1b438e8 --- /dev/null +++ b/tools/native-review/tests/test_ios_review.py @@ -0,0 +1,386 @@ +import importlib.util +import json +import pathlib +import subprocess +import tempfile +import threading +import unittest +import urllib.request +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).parents[1] / "ios_review.py" +SPEC = importlib.util.spec_from_file_location("ios_review", MODULE_PATH) +ios_review = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(ios_review) + + +class FakeRecorder: + def __init__(self): + self.stderr = self + self.finalized = False + + def readline(self): + return "Recording started\n" + + def poll(self): + return 0 if self.finalized else None + + def send_signal(self, _signal): + self.finalized = True + + def wait(self, timeout): + return 0 + + def kill(self): + self.finalized = True + + +class FakeFlutterProcess: + def __init__(self, returncode=0): + self.returncode = returncode + self.running = True + self.terminated = False + + def poll(self): + return None if self.running else self.returncode + + def wait(self, timeout=None): + self.running = False + return self.returncode + + def terminate(self): + self.terminated = True + self.running = False + + def kill(self): + self.terminated = True + self.running = False + + +class IosReviewTests(unittest.TestCase): + DEVICE_PAYLOAD = { + "devicetypes": [{"name": "iPhone Test", "identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-Test"}], + "runtimes": [ + {"identifier": "com.apple.CoreSimulator.SimRuntime.iOS-9-3", "version": "9.3", + "platform": "iOS", "isAvailable": True, "supportedDeviceTypes": [ + {"identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-Test"}, + ]}, + {"identifier": "com.apple.CoreSimulator.SimRuntime.iOS-26-0", "version": "26.0", + "platform": "iOS", "isAvailable": True, "supportedDeviceTypes": [ + {"identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-Test"}, + ]}, + ], + } + + def test_device_creation_uses_unique_owned_name_and_latest_runtime(self): + responses = [ + subprocess.CompletedProcess([], 0, json.dumps(self.DEVICE_PAYLOAD), ""), + subprocess.CompletedProcess([], 0, "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE\n", ""), + ] + with mock.patch.object(ios_review, "run", side_effect=responses) as run: + device = ios_review.create_review_device("iPhone Test", "ios-run-123") + create = run.call_args_list[1].args[0] + self.assertEqual(create[:3], ["xcrun", "simctl", "create"]) + self.assertEqual(create[3], "Buzz Native Review ios-run-123") + self.assertEqual(create[-1], "com.apple.CoreSimulator.SimRuntime.iOS-26-0") + self.assertTrue(device["owned"]) + self.assertEqual(device["udid"], "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE") + + def test_missing_device_type_fails_without_creating_or_erasing_any_device(self): + completed = subprocess.CompletedProcess([], 0, json.dumps({"devicetypes": [], "runtimes": []}), "") + with mock.patch.object(ios_review, "run", return_value=completed) as run: + with self.assertRaisesRegex(ios_review.ReviewError, "no iOS Simulator device type"): + ios_review.create_review_device("Absent", "run") + commands = [call.args[0] for call in run.call_args_list] + self.assertFalse(any("erase" in command or "delete" in command for command in commands)) + + def test_flutter_environment_scrubs_host_credentials(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", "GITHUB_TOKEN": "secret"} + with mock.patch.dict(ios_review.os.environ, sentinels, clear=False): + environment = ios_review.flutter_environment() + for name in sentinels: + self.assertNotIn(name, environment) + + def test_flutter_environment_carries_private_proceed_url(self): + environment = ios_review.flutter_environment("http://127.0.0.1:1/proceed/token") + self.assertEqual(environment["BUZZ_NATIVE_REVIEW_PROCEED_URL"], "http://127.0.0.1:1/proceed/token") + self.assertEqual(environment["SIMCTL_CHILD_BUZZ_NATIVE_REVIEW_PROCEED_URL"], environment["BUZZ_NATIVE_REVIEW_PROCEED_URL"]) + + def test_run_scrubs_host_credentials_by_default(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", "GITHUB_TOKEN": "secret"} + with mock.patch.dict(ios_review.os.environ, sentinels, clear=False), \ + mock.patch.object(ios_review.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run: + ios_review.run(["xcrun", "simctl", "list"]) + environment = run.call_args.kwargs["env"] + for name in sentinels: + self.assertNotIn(name, environment) + + def test_proceed_gate_waits_for_explicit_recorder_release(self): + server, url = ios_review.proceed_server() + result = [] + + def request_proceed(): + with urllib.request.urlopen(url, timeout=2) as response: + result.append(response.status) + + request = threading.Thread(target=request_proceed) + request.start() + try: + self.assertFalse(server.review_ready.is_set()) + request.join(timeout=0.05) + self.assertTrue(request.is_alive()) + ios_review.release_journey(server) + request.join(timeout=1) + self.assertFalse(request.is_alive()) + self.assertEqual(result, [204]) + finally: + ios_review.release_journey(server) + server.shutdown() + server.server_close() + request.join(timeout=1) + + def test_state_evidence_requires_every_reviewed_transition_in_order(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "flutter.log" + log.write_text("\n".join( + f"{ios_review.STATE_MARKER_PREFIX}{state}" for state in + ["initial-hidden", "revealed", "edited", "final-hidden"] + )) + self.assertEqual( + [step["name"] for step in ios_review.evidence_steps(log)], + ["initial-hidden", "revealed", "edited", "final-hidden"], + ) + log.write_text(f"{ios_review.STATE_MARKER_PREFIX}final-hidden\n") + with self.assertRaisesRegex(ios_review.ReviewError, "evidence incomplete"): + ios_review.evidence_steps(log) + + def test_second_reap_timeout_is_collected_without_aborting_teardown(self): + process = mock.Mock() + process.poll.return_value = None + process.wait.side_effect = [ + subprocess.TimeoutExpired("flutter", 10), + subprocess.TimeoutExpired("flutter", 5), + ] + errors = ios_review.terminate_child(process, "Flutter") + self.assertEqual(errors, ["Flutter required SIGKILL", "Flutter did not exit after SIGKILL"]) + process.kill.assert_called_once() + + def test_all_simctl_commands_and_flutter_waits_are_bounded(self): + source = MODULE_PATH.read_text() + self.assertIn( + '["xcrun", "simctl", "list", "devicetypes", "runtimes", "-j"],\n' + ' timeout=SIMCTL_TIMEOUT_SECONDS', + source, + ) + self.assertIn( + 'device_type["identifier"], runtime["identifier"]],\n' + ' timeout=SIMCTL_TIMEOUT_SECONDS', + source, + ) + self.assertIn( + '"bootstatus", udid, "-b"], capture=False, ' + 'timeout=BOOT_TIMEOUT_SECONDS', + source, + ) + self.assertIn( + "flutter_process.wait(timeout=FLUTTER_TIMEOUT_SECONDS)", source + ) + self.assertIn( + '"screenshot", str(screenshot)], timeout=SIMCTL_TIMEOUT_SECONDS', + source, + ) + + def test_recording_readiness_requires_journey_marker(self): + process = FakeFlutterProcess() + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.time, "monotonic", side_effect=[0, 2]), \ + mock.patch.object(ios_review.time, "sleep"): + with self.assertRaisesRegex(ios_review.ReviewError, "timed out waiting"): + ios_review.wait_for_recording_ready( + process, pathlib.Path(directory) / "flutter.log", timeout_seconds=1 + ) + + def test_recording_readiness_accepts_rendered_journey_marker(self): + process = FakeFlutterProcess() + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "flutter.log" + log.write_text(f"build output\n{ios_review.RECORDING_READY_MARKER}\n") + ios_review.wait_for_recording_ready(process, log, timeout_seconds=0.1) + + def test_flutter_failure_finalizes_recording_writes_receipt_and_cleans_device(self): + recorder = FakeRecorder() + flutter = FakeFlutterProcess(1) + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + if command[:2] == ["flutter", "drive"]: + return subprocess.CompletedProcess(command, 1, "journey failed", "diagnostic") + if "screenshot" in command: + pathlib.Path(command[-1]).write_bytes(b"png") + return subprocess.CompletedProcess(command, 0, "", "") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.sys, "platform", "darwin"), \ + mock.patch.object(ios_review.shutil, "which", return_value="/tool"), \ + mock.patch.object(ios_review, "provenance", return_value={"head_sha": "a" * 40, "dirty": False, "status": []}), \ + mock.patch.object(ios_review, "git", return_value="a" * 12), \ + mock.patch.object(ios_review, "create_review_device", return_value={ + "name": "Buzz Native Review owned-run", "udid": "owned-device", + "runtimeIdentifier": "runtime", "deviceType": "iPhone Test", "owned": True, + }), \ + mock.patch.object(ios_review, "run", side_effect=fake_run), \ + mock.patch.object(ios_review, "wait_for_recording_ready"), \ + mock.patch.object(ios_review, "wait_for_recording"), \ + mock.patch.object(ios_review, "evidence_steps", return_value=[{"name": "state", "status": "passed"}]), \ + mock.patch.object(ios_review, "finalize_recording") as finalize_recording, \ + mock.patch.object(ios_review.subprocess, "Popen", side_effect=[flutter, recorder]): + with self.assertRaisesRegex(ios_review.ReviewError, "exit 1"): + ios_review.run_review(ios_review.DEFAULT_TEST, "iPhone Test", pathlib.Path(directory)) + receipts = list(pathlib.Path(directory).rglob("receipt.json")) + self.assertEqual(len(receipts), 1) + receipt = json.loads(receipts[0].read_text()) + + finalize_recording.assert_called_once() + self.assertEqual(receipt["status"], "failed") + self.assertEqual(receipt["cleanup"], {"status": "passed", "errors": []}) + self.assertEqual(receipt["artifacts"], { + "video": "video.mp4", "log": "flutter.log" + }) + self.assertEqual(receipt["isolation"]["device"], { + "name": "Buzz Native Review owned-run", "device_type": "iPhone Test", + "udid": "owned-device", "runtime": "runtime", "owned": True, + }) + self.assertEqual(receipt["flow"], "ios_pairing") + self.assertIn("started_at", receipt) + self.assertIn("finished_at", receipt) + self.assertEqual(receipt["steps"], []) + self.assertEqual(receipt["measurements"], {}) + self.assertIn("machine", receipt["performance"]) + schema = json.loads((MODULE_PATH.parent / "schemas/receipt.schema.json").read_text()) + self.assertTrue(set(schema["required"]) <= set(receipt)) + self.assertFalse(set(receipt) - set(schema["properties"])) + self.assertIn(["xcrun", "simctl", "launch", "owned-device", ios_review.IOS_BUNDLE_ID], commands) + self.assertIn(["xcrun", "simctl", "shutdown", "owned-device"], commands) + self.assertIn(["xcrun", "simctl", "delete", "owned-device"], commands) + self.assertFalse(any("erase" in command for command in commands)) + self.assertTrue(all("pre-existing-device" not in command for command in commands)) + + def test_recorder_failure_after_start_fails_successful_journey_and_cleans_device(self): + recorder = FakeRecorder() + flutter = FakeFlutterProcess() + recorder.finalized = True + recorder.poll = mock.Mock(return_value=9) + recorder.wait = mock.Mock(return_value=9) + recorder.read = mock.Mock(return_value="encoder failed") + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + if command[:2] == ["flutter", "drive"]: + return subprocess.CompletedProcess(command, 0, "journey passed", "") + if "screenshot" in command: + pathlib.Path(command[-1]).write_bytes(b"png") + return subprocess.CompletedProcess(command, 0, "", "") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.sys, "platform", "darwin"), \ + mock.patch.object(ios_review.shutil, "which", return_value="/tool"), \ + mock.patch.object(ios_review, "provenance", return_value={"head_sha": "a" * 40, "dirty": False, "status": []}), \ + mock.patch.object(ios_review, "git", return_value="a" * 12), \ + mock.patch.object(ios_review, "create_review_device", return_value={ + "name": "Buzz Native Review failed-recorder", "udid": "owned-device", + "runtimeIdentifier": "runtime", "deviceType": "iPhone Test", "owned": True, + }), \ + mock.patch.object(ios_review, "run", side_effect=fake_run), \ + mock.patch.object(ios_review, "wait_for_recording_ready"), \ + mock.patch.object(ios_review, "wait_for_recording"), \ + mock.patch.object(ios_review, "evidence_steps", return_value=[{"name": "state", "status": "passed"}]), \ + mock.patch.object(ios_review.subprocess, "Popen", side_effect=[flutter, recorder]): + with self.assertRaisesRegex(ios_review.ReviewError, "recorder failed with exit 9"): + ios_review.run_review(ios_review.DEFAULT_TEST, "iPhone Test", pathlib.Path(directory)) + receipt = json.loads(next(pathlib.Path(directory).rglob("receipt.json")).read_text()) + + self.assertEqual(receipt["status"], "failed") + self.assertIn("recorder failed with exit 9", receipt["failure"]) + self.assertEqual(receipt["cleanup"]["status"], "failed") + self.assertIn(["xcrun", "simctl", "delete", "owned-device"], commands) + + def test_video_validation_timeout_fails_journey_and_still_cleans_device(self): + recorder = FakeRecorder() + flutter = FakeFlutterProcess() + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + if command[:2] == ["flutter", "drive"]: + return subprocess.CompletedProcess(command, 0, "journey passed", "") + if "screenshot" in command: + pathlib.Path(command[-1]).write_bytes(b"png") + if command[0] == "/usr/bin/avconvert": + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + return subprocess.CompletedProcess(command, 0, "", "") + + def fake_wait_for_recording(_recorder): + run_dir = next((pathlib.Path(directory) / ("a" * 12) / "ios_pairing").iterdir()) + (run_dir / "video.mp4").write_bytes(b"video") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.sys, "platform", "darwin"), \ + mock.patch.object(ios_review.shutil, "which", return_value="/tool"), \ + mock.patch.object(ios_review, "provenance", return_value={"head_sha": "a" * 40, "dirty": False, "status": []}), \ + mock.patch.object(ios_review, "git", return_value="a" * 12), \ + mock.patch.object(ios_review, "create_review_device", return_value={ + "name": "Buzz Native Review timeout", "udid": "owned-device", + "runtimeIdentifier": "runtime", "deviceType": "iPhone Test", "owned": True, + }), \ + mock.patch.object(ios_review, "run", side_effect=fake_run), \ + mock.patch.object(ios_review, "wait_for_recording_ready"), \ + mock.patch.object(ios_review, "wait_for_recording", side_effect=fake_wait_for_recording), \ + mock.patch.object(ios_review, "evidence_steps", return_value=[{"name": "state", "status": "passed"}]), \ + mock.patch.object(ios_review.subprocess, "Popen", side_effect=[flutter, recorder]): + with self.assertRaisesRegex(ios_review.ReviewError, "timed out validating simulator video"): + ios_review.run_review(ios_review.DEFAULT_TEST, "iPhone Test", pathlib.Path(directory)) + receipt = json.loads(next(pathlib.Path(directory).rglob("receipt.json")).read_text()) + + self.assertEqual(receipt["status"], "failed") + self.assertEqual(receipt["failure"], "timed out validating simulator video") + self.assertIn(["xcrun", "simctl", "delete", "owned-device"], commands) + + def test_recorder_timeout_is_reported_and_device_is_cleaned(self): + recorder = FakeRecorder() + flutter = FakeFlutterProcess() + recorder.readline = mock.Mock(return_value="") + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, "", "") + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(ios_review.sys, "platform", "darwin"), \ + mock.patch.object(ios_review.shutil, "which", return_value="/tool"), \ + mock.patch.object(ios_review, "provenance", return_value={"head_sha": "a" * 40, "dirty": False, "status": []}), \ + mock.patch.object(ios_review, "git", return_value="a" * 12), \ + mock.patch.object(ios_review, "create_review_device", return_value={ + "name": "Buzz Native Review timeout-run", "udid": "owned-device", + "runtimeIdentifier": "runtime", "deviceType": "iPhone Test", "owned": True, + }), \ + mock.patch.object(ios_review, "run", side_effect=fake_run), \ + mock.patch.object(ios_review, "wait_for_recording_ready"), \ + mock.patch.object(ios_review, "wait_for_recording", side_effect=ios_review.ReviewError("timed out waiting for Simulator recording")), \ + mock.patch.object(ios_review, "finalize_recording", side_effect=lambda process, _video: process.send_signal(0)), \ + mock.patch.object(ios_review.subprocess, "Popen", side_effect=[flutter, recorder]): + with self.assertRaisesRegex(ios_review.ReviewError, "timed out"): + ios_review.run_review(ios_review.DEFAULT_TEST, "iPhone Test", pathlib.Path(directory)) + receipt = json.loads(next(pathlib.Path(directory).rglob("receipt.json")).read_text()) + + self.assertTrue(recorder.finalized) + self.assertEqual(receipt["cleanup"]["status"], "passed") + self.assertIn(["xcrun", "simctl", "delete", "owned-device"], commands) + self.assertFalse(any("erase" in command for command in commands)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/native-review/tests/test_review_native.py b/tools/native-review/tests/test_review_native.py new file mode 100644 index 00000000000..11ab57f0e7e --- /dev/null +++ b/tools/native-review/tests/test_review_native.py @@ -0,0 +1,413 @@ +import importlib.util +import json +import pathlib +import re +import tempfile +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).parents[1] / "review_native.py" +SPEC = importlib.util.spec_from_file_location("review_native", MODULE_PATH) +review_native = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(review_native) + + +class JourneyTests(unittest.TestCase): + def test_real_journey_validates(self): + journey = review_native.load_journey(MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml") + self.assertEqual(journey["flow"], "tooltip_fresh_dwell") + + def test_broken_mutation_is_schema_valid(self): + journey = review_native.load_journey(MODULE_PATH.parent / "tests/fixtures/broken-tooltip.yaml") + self.assertEqual(journey["steps"][0]["timeout_ms"], 50) + + def test_broader_journeys_and_mutations_validate(self): + expected = { + "desktop/composer-keyboard.yaml": "composer_keyboard", + "desktop/search-shortcut-dismissal.yaml": "search_shortcut_dismissal", + "tests/fixtures/broken-text.yaml": "broken_text", + "tests/fixtures/broken-shortcut.yaml": "broken_shortcut", + "tests/fixtures/broken-scroll.yaml": "broken_scroll", + } + for relative_path, flow in expected.items(): + with self.subTest(relative_path=relative_path): + journey = review_native.load_journey(MODULE_PATH.parent / relative_path) + self.assertEqual(journey["flow"], flow) + + def test_duplicate_measurement_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace(" - name: leave_trigger\n", " - name: duplicate_measure\n act: {type: wait, duration_ms: 1}\n expect: {exists: {role: window}}\n measure: tooltip_open_latency\n - name: leave_trigger\n") + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "duplicate measurement"): + review_native.load_journey(path) + + def test_type_text_requires_text(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + source = re.sub(r'\{type: type_text, text: "[^"]*"\}', "{type: type_text}", source, count=1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "type_text requires text"): + review_native.load_journey(path) + + def test_scroll_requires_integer_delta(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + source = re.sub(r"\{type: scroll, delta_y: -?240\}", "{type: scroll, delta_y: nope}", source, count=1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "scroll delta_y must be an integer"): + review_native.load_journey(path) + + def test_scroll_delta_respects_schema_bounds(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + for value in (-10000, 10000): + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "valid.yaml" + path.write_text(source.replace("delta_y: 240", f"delta_y: {value}", 1)) + self.assertEqual(review_native.load_journey(path)["flow"], "composer_keyboard") + for value in (-10001, 10001, 9223372036854775807): + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source.replace("delta_y: 240", f"delta_y: {value}", 1)) + with self.assertRaisesRegex(review_native.HarnessError, "-10000..10000"): + review_native.load_journey(path) + + def test_scroll_requires_locator(self): + source = (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text() + source = source.replace(" locate:\n - {id: message-input-scroll}\n", "", 1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "scroll requires locate"): + review_native.load_journey(path) + + def test_durations_and_boolean_scroll_delta_fail_closed(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + invalid_sources = { + "action": source.replace("duration_ms: 100", "duration_ms: -1", 1), + "sustained": source.replace( + " expect:\n exists: {role: window}\n", + " expect:\n exists: {role: window}\n expect_for:\n duration_ms: 0\n condition: {exists: {role: window}}\n", + 1, + ), + "timeout": source.replace("timeout_ms: 1000", "timeout_ms: true", 1), + "scroll": (MODULE_PATH.parent / "desktop/composer-keyboard.yaml").read_text().replace( + "delta_y: 240", "delta_y: true", 1 + ), + } + for name, invalid in invalid_sources.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(invalid) + with self.assertRaises(review_native.HarnessError): + review_native.load_journey(path) + + def test_non_finite_and_boolean_scroll_expectations_fail_closed(self): + for value in (True, float("inf"), float("nan")): + with self.subTest(value=value), self.assertRaisesRegex( + review_native.HarnessError, "finite number" + ): + review_native.validate_expectation({"scroll_y_greater_than": value}, "expect") + + def test_value_expectation_uses_selected_element(self): + driver = mock.Mock() + driver.request.return_value = {"ok": True, "element": {"value": "draft"}} + self.assertTrue(review_native.expectation_holds(driver, {"value": "draft"})) + self.assertFalse(review_native.expectation_holds(driver, {"value": "wrong"})) + + def test_scroll_expectation_uses_selected_element(self): + driver = mock.Mock() + driver.request.return_value = {"ok": True, "element": {"scrollY": 240}} + self.assertTrue(review_native.expectation_holds(driver, {"scroll_y_greater_than": 0})) + self.assertFalse(review_native.expectation_holds(driver, {"scroll_y_greater_than": 240})) + self.assertTrue(review_native.expectation_holds(driver, {"scroll_y_less_than": 241})) + self.assertFalse(review_native.expectation_holds(driver, {"scroll_y_less_than": 240})) + + def test_action_without_postcondition_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace(" expect:\n exists: {role: window}\n", "", 1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "requires name/act/expect"): + review_native.load_journey(path) + + def test_expectation_with_multiple_conditions_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace( + " expect:\n exists: {role: window}\n", + " expect:\n exists: {role: window}\n enabled: true\n", + 1, + ) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "exactly one"): + review_native.load_journey(path) + + def test_production_and_remote_targets_are_rejected(self): + with self.assertRaisesRegex(review_native.HarnessError, "non-loopback"): + review_native.isolation_manifest("run", "wss://buzz.block.builderlab.xyz") + safe = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + self.assertTrue(safe["bundle_id"].startswith("xyz.block.buzz.app.dev.native-review.")) + self.assertNotIn(safe["bundle_id"], review_native.PRODUCTION_BUNDLE_IDS) + + def test_secret_environment_is_scrubbed(self): + sentinels = { + "BUZZ_PRIVATE_KEY": "must-not-survive", + "SSH_AUTH_SOCK": "/tmp/credential-agent", + "GITHUB_TOKEN": "github-secret", + "AWS_SECRET_ACCESS_KEY": "cloud-secret", + } + with mock.patch.dict(review_native.os.environ, sentinels, clear=False): + environment = review_native.scrubbed_environment() + for name in sentinels: + self.assertNotIn(name, environment) + + def test_fixture_environment_is_fixed_local_and_scrubbed(self): + sentinels = {"BUZZ_DB_HOST": "production-db", "BUZZ_DB_PASS": "production-secret", + "SSH_AUTH_SOCK": "/tmp/credential-agent"} + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.dict(review_native.os.environ, sentinels, clear=False): + environment = review_native.fixture_environment(isolation, "a" * 64) + self.assertEqual(environment["BUZZ_DB_HOST"], "localhost") + self.assertEqual(environment["BUZZ_DB_PORT"], "5471") + self.assertEqual(environment["BUZZ_DB_PASS"], "buzz_dev") + self.assertNotIn("SSH_AUTH_SOCK", environment) + with self.assertRaisesRegex(review_native.HarnessError, "port 3030"): + review_native.fixture_environment( + review_native.isolation_manifest("run", "ws://127.0.0.1:3001"), "a" * 64) + + def test_fixture_seed_failure_removes_generated_identity(self): + generated = mock.Mock(stdout=f"Secret key: {'1' * 64}\nPublic key: {'2' * 64}\n") + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + (run_dir / "manifest").mkdir() + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.object(review_native, "run", side_effect=[generated, RuntimeError("seed failed")]): + with self.assertRaisesRegex(RuntimeError, "seed failed"): + review_native.prepare_fixture(run_dir, isolation) + self.assertFalse((run_dir / "state/identity.key").exists()) + + def test_run_scrubs_repository_controlled_commands_by_default(self): + safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} + with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ + mock.patch.object(review_native.subprocess, "run", return_value=mock.Mock()) as subprocess_run: + review_native.run(["/tmp/repository-tool"]) + self.assertEqual(subprocess_run.call_args.kwargs["env"], safe) + + def test_cleanup_uses_isolated_home_without_host_credentials(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", + "GITHUB_TOKEN": "secret", "AWS_SECRET_ACCESS_KEY": "secret"} + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.dict(review_native.os.environ, sentinels, clear=False), \ + mock.patch.object(review_native, "run") as run: + review_native.cleanup_review_state(run_dir, isolation, None) + environment = run.call_args.kwargs["env"] + self.assertEqual(environment["HOME"], str(run_dir / "home")) + for name in sentinels: + self.assertNotIn(name, environment) + + def test_cleanup_removes_identity_when_state_reset_fails(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + secret_path = run_dir / "state/identity.key" + secret_path.parent.mkdir() + secret_path.write_text("review-secret") + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + fixture = {"secret_path": str(secret_path)} + with mock.patch.object(review_native, "run", side_effect=RuntimeError("reset failed")): + with self.assertRaisesRegex( + review_native.HarnessError, + "desktop state reset failed: reset failed"): + review_native.cleanup_review_state(run_dir, isolation, fixture) + self.assertFalse(secret_path.exists()) + + def test_cleanup_aggregates_reset_and_identity_removal_failures(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + secret_path = run_dir / "state/identity.key" + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + fixture = {"secret_path": str(secret_path)} + with mock.patch.object(review_native, "run", side_effect=RuntimeError("reset failed")), \ + mock.patch.object(pathlib.Path, "unlink", side_effect=OSError("unlink failed")): + with self.assertRaisesRegex( + review_native.HarnessError, + "desktop state reset failed: reset failed; " + "review identity removal failed: unlink failed"): + review_native.cleanup_review_state(run_dir, isolation, fixture) + + def test_repository_controlled_subprocesses_receive_scrubbed_environments(self): + safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} + with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ + mock.patch.object(review_native.subprocess, "Popen") as popen: + review_native.Driver(pathlib.Path("/tmp/driver"), 42, pathlib.Path("/tmp/snapshot")) + self.assertEqual(popen.call_args.kwargs["env"], safe) + + def test_visible_window_waits_for_reveal(self): + driver = mock.Mock() + driver.request.side_effect = [ + {"ok": True, "visible": False, "detail": "not yet"}, + {"ok": True, "visible": True, "window_id": 42}, + ] + process = mock.Mock() + process.poll.return_value = None + with mock.patch.object(review_native.time, "sleep"): + status = review_native.wait_for_visible_window(driver, process, timeout_seconds=1) + self.assertEqual(status["window_id"], 42) + self.assertEqual(driver.request.call_count, 2) + + def test_locate_required_retries_until_target_materializes(self): + driver = mock.Mock() + driver.request.side_effect = [ + {"ok": True, "element": None}, + {"ok": True, "element": {"locator": {"id": "target"}}}, + ] + with mock.patch.object(review_native.time, "sleep"): + found = review_native.locate_required(driver, [{"id": "target"}], 1000) + self.assertEqual(found["locator"]["id"], "target") + self.assertEqual(driver.request.call_count, 2) + + def test_visible_window_fails_if_app_exits(self): + driver = mock.Mock() + process = mock.Mock() + process.poll.return_value = 1 + with self.assertRaisesRegex(review_native.HarnessError, "exited"): + review_native.wait_for_visible_window(driver, process, timeout_seconds=1) + driver.request.assert_not_called() + + +class PerformanceTests(unittest.TestCase): + MACHINE = {"system": "Darwin", "release": "test", "machine": "arm64", "cpu": "test"} + + def receipt(self, path, artifact, timing, cpu=10, memory=100, flow="tooltip_fresh_dwell"): + payload = { + "run_id": path.stem, "flow": flow, "status": "passed", + "cleanup": {"status": "passed"}, + "provenance": {"dirty": False, "head_sha": artifact + "-sha", "artifact_sha256": artifact}, + "measurements": {"tooltip_open_latency": {"value": timing, "unit": "ms", "step": "tooltip"}}, + "performance": {"machine": self.MACHINE, "process": { + "cpu_percent_median": cpu, "resident_mb_peak": memory, + }}, + } + path.write_text(json.dumps(payload)) + return path + + def budget(self, path, regression=20): + path.write_text(f"""schema_version: 1 +flow: tooltip_fresh_dwell +minimum_samples: 3 +metrics: + tooltip_open_latency: + max: 1000 + max_regression_percent: {regression} + process.resident_mb_peak: + max: 500 + max_regression_percent: 20 +""") + return path + + def test_comparison_uses_median_and_passes_with_noise(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", value) for i, value in enumerate((100, 101, 900))] + candidate = [self.receipt(root / f"c{i}.json", "head", value) for i, value in enumerate((110, 111, 5))] + result = review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + self.assertEqual(result["status"], "passed") + self.assertEqual(result["baseline"]["metrics"]["tooltip_open_latency"]["median"], 101) + + def test_comparison_fails_when_one_sample_exceeds_absolute_maximum(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", value) + for i, value in enumerate((100, 100, 10000))] + with self.assertRaisesRegex(review_native.HarnessError, "maximum 10000.000 exceeds absolute maximum 1000"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_fails_on_relative_regression(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 130) for i in range(3)] + with self.assertRaisesRegex(review_native.HarnessError, "regression"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_too_few_samples(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipts = [self.receipt(root / f"r{i}.json", "same", 100) for i in range(2)] + with self.assertRaisesRegex(review_native.HarnessError, "at least 3"): + review_native.compare_performance(receipts, receipts, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_incompatible_machine(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + payload = json.loads(candidate[0].read_text()) + payload["performance"]["machine"]["machine"] = "x86_64" + candidate[0].write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "incompatible machines"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_duplicate_receipts(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + baseline[1] = baseline[0] + with self.assertRaisesRegex(review_native.HarnessError, "duplicate run_id"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_cross_cohort_run_reuse(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + receipts = [self.receipt(root / f"r{i}.json", "same", 100) for i in range(3)] + with self.assertRaisesRegex(review_native.HarnessError, "cohorts contain duplicate run_id"): + review_native.compare_performance(receipts, receipts, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_equal_cohort_revisions(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "same", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "same", 100) for i in range(3)] + with self.assertRaisesRegex(review_native.HarnessError, "different source revisions"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + def test_comparison_rejects_boolean_minimum_and_non_finite_metrics(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + budget = self.budget(root / "budget.yaml") + budget.write_text(budget.read_text().replace("minimum_samples: 3", "minimum_samples: true")) + with self.assertRaisesRegex(review_native.HarnessError, "minimum_samples"): + review_native.compare_performance(baseline, candidate, budget) + budget = self.budget(root / "budget.yaml") + payload = json.loads(candidate[0].read_text()) + payload["measurements"]["tooltip_open_latency"]["value"] = float("inf") + candidate[0].write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "finite numeric metric"): + review_native.compare_performance(baseline, candidate, budget) + + def test_comparison_rejects_mixed_revisions(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + baseline = [self.receipt(root / f"b{i}.json", "base", 100) for i in range(3)] + candidate = [self.receipt(root / f"c{i}.json", "head", 100) for i in range(3)] + payload = json.loads(candidate[0].read_text()) + payload["provenance"]["head_sha"] = "other" + candidate[0].write_text(json.dumps(payload)) + with self.assertRaisesRegex(review_native.HarnessError, "mixes source revisions"): + review_native.compare_performance(baseline, candidate, self.budget(root / "budget.yaml")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/native-review/uv.lock b/tools/native-review/uv.lock new file mode 100644 index 00000000000..482446a9125 --- /dev/null +++ b/tools/native-review/uv.lock @@ -0,0 +1,60 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "buzz-native-review" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = "==6.0.3" }] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/simple" } +sdist = { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f" } +wheels = [ + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b" }, +]