From 8b3450af0ef952f5a0b5b8e593f29deb523e5d87 Mon Sep 17 00:00:00 2001 From: Carl <5f365698229751c0461f57bb03a4e93134e6e936bd7039ebe7b737282a43c754@buzz.block.builderlab.xyz> Date: Fri, 14 Aug 2026 20:54:07 -0700 Subject: [PATCH 1/4] feat: add native desktop review harness Signed-off-by: Carl <5f365698229751c0461f57bb03a4e93134e6e936bd7039ebe7b737282a43c754@buzz.block.builderlab.xyz> --- .gitignore | 3 + Justfile | 8 + desktop/src/main.tsx | 50 ++ .../src/testing/nativeReviewSemanticProbe.ts | 108 ++++ scripts/setup-desktop-test-data.sh | 16 + tools/native-review/README.md | 43 ++ tools/native-review/bin/review-native | 8 + .../desktop/tooltip-fresh-dwell.yaml | 61 ++ tools/native-review/review_native.py | 584 ++++++++++++++++++ .../native-review/schemas/journey.schema.json | 74 +++ .../native-review/schemas/receipt.schema.json | 15 + tools/native-review/swift/Package.swift | 9 + .../swift/Sources/BuzzNativeDriver/main.swift | 542 ++++++++++++++++ .../tests/fixtures/broken-tooltip.yaml | 13 + .../native-review/tests/test_review_native.py | 97 +++ 15 files changed, 1631 insertions(+) create mode 100644 desktop/src/testing/nativeReviewSemanticProbe.ts create mode 100644 tools/native-review/README.md create mode 100755 tools/native-review/bin/review-native create mode 100644 tools/native-review/desktop/tooltip-fresh-dwell.yaml create mode 100755 tools/native-review/review_native.py create mode 100644 tools/native-review/schemas/journey.schema.json create mode 100644 tools/native-review/schemas/receipt.schema.json create mode 100644 tools/native-review/swift/Package.swift create mode 100644 tools/native-review/swift/Sources/BuzzNativeDriver/main.swift create mode 100644 tools/native-review/tests/fixtures/broken-tooltip.yaml create mode 100644 tools/native-review/tests/test_review_native.py diff --git a/.gitignore b/.gitignore index f26e74136c..dc5603d707 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ __pycache__/ *.pyc +# Swift native-review build output +tools/native-review/swift/.build/ + # lefthook-generated hook scripts (machine-specific) .hooks/ diff --git a/Justfile b/Justfile index 9e47178427..b68f374ef9 100644 --- a/Justfile +++ b/Justfile @@ -973,3 +973,11 @@ benchmark *ARGS: # Stop the benchmark Docker stack (state and channels are kept) benchmark-down: docker compose --project-name buzz-benchmark down + +# Validate macOS native-review tooling and report required OS permissions. +native-review-doctor: + ./tools/native-review/bin/review-native doctor + +# Run one declarative journey against the isolated local desktop fixture. +native-review-desktop JOURNEY="tools/native-review/desktop/tooltip-fresh-dwell.yaml": + ./tools/native-review/bin/review-native run "{{JOURNEY}}" diff --git a/desktop/src/main.tsx b/desktop/src/main.tsx index bbb4c5fa42..a80cf88a32 100644 --- a/desktop/src/main.tsx +++ b/desktop/src/main.tsx @@ -19,6 +19,7 @@ import { Toaster } from "@/shared/ui/sonner"; import { TooltipProvider } from "@/shared/ui/tooltip"; import { recoverLocalStorageQuotaOnStartup } from "@/shared/lib/localStorageQuota"; import { startLocalStorageSweep } from "@/shared/lib/localStorageSweep"; +import { installNativeReviewSemanticProbe } from "@/testing/nativeReviewSemanticProbe"; type E2eWindow = Window & { __BUZZ_E2E__?: unknown; @@ -28,6 +29,54 @@ const E2E_DEFAULT_PUBKEY = "deadbeef".repeat(8); const E2E_COMMUNITY_ID = "e2e-default-community"; const ONBOARDING_COMPLETION_STORAGE_KEY_PREFIX = "buzz-onboarding-complete.v1:"; const DEV_STATE_RESET_PARAM = "resetDevState"; +const NATIVE_REVIEW_PARAM = "nativeReview"; + +function configureNativeReviewFixtureFromUrl() { + const buildEnabled = import.meta.env.VITE_NATIVE_REVIEW === "1"; + if (!import.meta.env.DEV && !buildEnabled) return; + const url = new URL(window.location.href); + const enabled = + url.searchParams.get(NATIVE_REVIEW_PARAM) === "1" || buildEnabled; + if (!enabled) return; + + const relayUrl = + url.searchParams.get("reviewRelay") ?? + import.meta.env.VITE_NATIVE_REVIEW_RELAY; + const pubkey = + url.searchParams.get("reviewPubkey") ?? + import.meta.env.VITE_NATIVE_REVIEW_PUBKEY; + if ( + !relayUrl || + !pubkey || + !/^(ws|http):\/\/(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?\/?$/.test( + relayUrl, + ) + ) { + throw new Error( + "native review bootstrap requires a loopback relay and pubkey", + ); + } + const communityId = "native-review-local"; + const community = { + addedAt: new Date().toISOString(), + id: communityId, + name: "Native Review", + pubkey, + relayUrl, + }; + window.localStorage.setItem("buzz-communities", JSON.stringify([community])); + window.localStorage.setItem("buzz-active-community-id", communityId); + window.localStorage.setItem( + `buzz-machine-onboarding-complete.v2:${pubkey}`, + "true", + ); + window.localStorage.setItem(`buzz-onboarding-complete.v1:${pubkey}`, "true"); + window.localStorage.setItem( + `buzz-community-onboarding-complete.v1:${encodeURIComponent(relayUrl)}:${pubkey}`, + "true", + ); + installNativeReviewSemanticProbe(); +} function resetDevWebviewStateFromUrl() { if (!import.meta.env.DEV) { @@ -121,6 +170,7 @@ async function installE2eBridgeIfConfigured() { async function bootstrap() { resetDevWebviewStateFromUrl(); + configureNativeReviewFixtureFromUrl(); configureDevE2eBridgeFromUrl(); recoverLocalStorageQuotaOnStartup(); startLocalStorageSweep(); diff --git a/desktop/src/testing/nativeReviewSemanticProbe.ts b/desktop/src/testing/nativeReviewSemanticProbe.ts new file mode 100644 index 0000000000..1acecaf4bc --- /dev/null +++ b/desktop/src/testing/nativeReviewSemanticProbe.ts @@ -0,0 +1,108 @@ +type SemanticNode = { + id?: string; + role?: string; + name?: string; + enabled: boolean; + focused: boolean; + frame: { x: number; y: number; width: number; height: number }; + viewport: { width: number; height: number }; +}; + +const IMPLICIT_ROLES: Partial> = { + A: "link", + BUTTON: "button", + INPUT: "text-field", + TEXTAREA: "text-area", +}; + +function accessibleName(element: HTMLElement): string | undefined { + const labelledBy = element.getAttribute("aria-labelledby"); + const labelledText = labelledBy + ?.split(/\s+/) + .map((id) => document.getElementById(id)?.textContent?.trim()) + .filter(Boolean) + .join(" "); + return ( + element.getAttribute("aria-label")?.trim() || + labelledText || + element.getAttribute("title")?.trim() || + (element.getAttribute("role") === "tooltip" + ? element.textContent?.trim() + : undefined) || + undefined + ); +} + +function snapshot(): SemanticNode[] { + const nodes: SemanticNode[] = []; + for (const candidate of document.querySelectorAll( + "[data-testid], [role], button, textarea, input, a[href]", + )) { + const rect = candidate.getBoundingClientRect(); + const style = window.getComputedStyle(candidate); + if ( + rect.width <= 0 || + rect.height <= 0 || + style.display === "none" || + style.visibility === "hidden" + ) { + continue; + } + const id = candidate.dataset.testid; + const role = + candidate.getAttribute("role") ?? IMPLICIT_ROLES[candidate.tagName]; + const name = accessibleName(candidate); + if (!id && !role && !name) continue; + nodes.push({ + ...(id ? { id } : {}), + ...(role ? { role } : {}), + ...(name ? { name } : {}), + enabled: + !candidate.hasAttribute("disabled") && + candidate.getAttribute("aria-disabled") !== "true", + focused: candidate === document.activeElement, + frame: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }, + viewport: { + width: window.innerWidth, + height: window.innerHeight, + }, + }); + } + return nodes; +} + +export function installNativeReviewSemanticProbe(): void { + let scheduled = false; + const publish = () => { + scheduled = false; + const payload = JSON.stringify(snapshot()); + if ( + !navigator.sendBeacon( + import.meta.env.VITE_NATIVE_REVIEW_PROBE_URL, + payload, + ) + ) { + console.error("native review semantic probe beacon was rejected"); + } + }; + const schedule = () => { + if (scheduled) return; + scheduled = true; + window.requestAnimationFrame(publish); + }; + new MutationObserver(schedule).observe(document.documentElement, { + attributes: true, + childList: true, + subtree: true, + }); + window.addEventListener("focusin", schedule); + window.addEventListener("focusout", schedule); + window.addEventListener("resize", schedule); + window.addEventListener("scroll", schedule, true); + schedule(); +} diff --git a/scripts/setup-desktop-test-data.sh b/scripts/setup-desktop-test-data.sh index ff3ebd055c..da7671f480 100755 --- a/scripts/setup-desktop-test-data.sh +++ b/scripts/setup-desktop-test-data.sh @@ -15,6 +15,11 @@ BOB_PUBKEY="bb22a5299220cad76ffd46190ccbeede8ab5dc260faa28b6e5a2cb31b9aff260" CHARLIE_PUBKEY="554cef57437abac34522ac2c9f0490d685b72c80478cf9f7ed6f9570ee8624ea" TYLER_PUBKEY="e5ebc6cdb579be112e336cc319b5989b4bb6af11786ea90dbe52b5f08d741b34" AGENT_PUBKEY="db0b028cd36f4d3e36c8300cce87252c1f7fc9495ffecc53f393fcac341ffd36" +REVIEW_PUBKEY="${BUZZ_REVIEW_PUBKEY:-}" +if [[ -n "$REVIEW_PUBKEY" && ! "$REVIEW_PUBKEY" =~ ^[0-9a-fA-F]{64}$ ]]; then + echo "BUZZ_REVIEW_PUBKEY must be exactly 64 hexadecimal characters." >&2 + exit 1 +fi if command -v psql >/dev/null 2>&1; then run_psql() { PGPASSWORD="$DB_PASS" psql -h"$DB_HOST" -p"$DB_PORT" -U"$DB_USER" -d"$DB_NAME" -qtA "$@"; } @@ -129,4 +134,15 @@ ON CONFLICT DO NOTHING ; " +if [[ -n "$REVIEW_PUBKEY" ]]; then + run_sql " +INSERT INTO channel_members + (community_id, channel_id, pubkey, role, invited_by) +VALUES + ('${COMMUNITY_ID}', '${UUID_GENERAL}', decode('${REVIEW_PUBKEY}','hex'), 'member', decode('${SYSTEM_PUBKEY}','hex')) +ON CONFLICT DO NOTHING +; +" +fi + echo "Desktop e2e data ready." diff --git a/tools/native-review/README.md b/tools/native-review/README.md new file mode 100644 index 0000000000..94b38fe169 --- /dev/null +++ b/tools/native-review/README.md @@ -0,0 +1,43 @@ +# Buzz native review harness + +This macOS-only MVP drives the real Tauri/WKWebView app with Accessibility and +CGEvent, captures its window with Core Graphics and AVFoundation, and writes an +exact-SHA run receipt. It is a targeted review lane, not a replacement for +`just ci` or Playwright. + +## Safety contract + +- Only loopback `ws://`/`http://` relays are accepted. +- Every run gets an ephemeral Nostr key, run-specific dev bundle ID, keyring + service, HOME, WebKit/app-data state, and artifact directory. +- The launcher environment is allowlisted before the ephemeral key is added; + inherited tokens and production keys never enter the reviewed process. +- Production bundle IDs, keyring services, and remote relays fail closed. +- This protects reviewer state from accidents. It is **not** containment for + hostile code; use a dedicated macOS user or disposable VM for untrusted PRs. + +## Commands + +```bash +just native-review-doctor +just native-review-desktop tools/native-review/desktop/tooltip-fresh-dwell.yaml +python3 -m unittest discover -s tools/native-review/tests -p 'test_*.py' +``` + +The desktop command expects the isolated `buzz-harness` relay on port 3030 +(`scripts/start-isolated-test-relay.sh`). Doctor reports Accessibility and +Screen Recording separately and the run refuses to proceed unless both are +already granted to the invoking terminal/agent. + +Runs are written under +`test-results/native-review////`. A failed locator, +postcondition, recording, evidence capture, or cleanup produces a failed partial +receipt. `tests/fixtures/broken-tooltip.yaml` is the deliberate fail-loud +mutation. + +## MVP limits + +Only macOS desktop, the local review-channel fixture, role/name/identifier AX +locators, click/hover/key/wait actions, window screenshots/video, and step AX +snapshots are implemented. iOS Simulator and repeated base/head performance +comparison remain later phases. diff --git a/tools/native-review/bin/review-native b/tools/native-review/bin/review-native new file mode 100755 index 0000000000..f4a1741cfe --- /dev/null +++ b/tools/native-review/bin/review-native @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# The repository pins Rust/Node tooling through Hermit; native review must build +# with the same toolchain as the artifact it records. +source "$REPO_ROOT/bin/activate-hermit" +exec python3 "$SCRIPT_DIR/review_native.py" "$@" diff --git a/tools/native-review/desktop/tooltip-fresh-dwell.yaml b/tools/native-review/desktop/tooltip-fresh-dwell.yaml new file mode 100644 index 0000000000..6dda5c2900 --- /dev/null +++ b/tools/native-review/desktop/tooltip-fresh-dwell.yaml @@ -0,0 +1,61 @@ +schema_version: 1 +flow: tooltip_fresh_dwell +platforms: [macos] +fixture: local_review_channel +record: + video: window + screenshots: false + accessibility: false +steps: + - name: activate_buzz + act: {type: activate} + expect: + exists: {role: window} + timeout_ms: 15000 + - name: await_interactive_app + act: {type: wait, duration_ms: 100} + expect: + not_exists: {id: app-loading-gate} + timeout_ms: 60000 + - name: reach_seeded_channel + locate: + - {id: channel-welcome-everyone} + - {role: button, name: welcome-everyone} + act: {type: click} + expect: + exists: {id: message-insert-mention} + timeout_ms: 15000 + - name: settle_pointer_before_fresh_dwell + locate: + - {id: message-composer} + - {role: text-area, name: Message} + act: {type: move_pointer, duration_ms: 80} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + - name: clear_tooltip_skip_delay + act: {type: wait, duration_ms: 800} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + - name: transit_over_mention + locate: + - {id: message-insert-mention} + - {role: button, name: "Mention someone"} + act: {type: move_pointer} + expect: {enabled: true} + - name: complete_dwell + act: {type: wait, duration_ms: 550} + expect: + exists: {role: tooltip, name: "Mention someone"} + timeout_ms: 1000 + measure: tooltip_open_latency + - name: leave_trigger + locate: + - {id: message-composer} + - {role: text-area, name: Message} + act: {type: move_pointer, duration_ms: 80} + expect: + not_exists: {role: tooltip, name: "Mention someone"} + timeout_ms: 1000 +cleanup: + terminate_app: true + remove_state: true diff --git a/tools/native-review/review_native.py b/tools/native-review/review_native.py new file mode 100755 index 0000000000..559446770a --- /dev/null +++ b/tools/native-review/review_native.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +"""Exact-SHA-bound native review orchestrator for Buzz Desktop.""" +from __future__ import annotations + +import argparse +import dataclasses +import datetime as dt +import hashlib +import http.server +import json +import os +import pathlib +import re +import secrets +import select +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse +from typing import Any + +try: + import yaml +except ImportError as exc: # pragma: no cover - environment preflight + raise SystemExit("PyYAML is required (activate the repository Hermit environment)") from exc + +ROOT = pathlib.Path(__file__).resolve().parents[2] +TOOL_ROOT = pathlib.Path(__file__).resolve().parent +PRODUCTION_BUNDLE_IDS = {"xyz.block.buzz.app", "xyz.block.sprout.app"} +PRODUCTION_KEYRINGS = {"buzz-desktop", "sprout-desktop"} +SECRET_NAME = re.compile(r"(AUTH|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|COOKIE)", re.I) +ALLOWED_TOP = {"schema_version", "flow", "platforms", "fixture", "record", "steps", "cleanup"} +ALLOWED_STEP = {"name", "locate", "act", "expect", "expect_for", "timeout_ms", "measure"} + + +class HarnessError(RuntimeError): + pass + + +def run(command: list[str], *, cwd: pathlib.Path = ROOT, env: dict[str, str] | None = None, + check: bool = True, capture: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, env=env, check=check, text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None) + + +def git(*args: str) -> str: + return run(["git", *args]).stdout.strip() + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat() + + +def validate_locator(locator: Any, where: str) -> None: + if not isinstance(locator, dict) or not locator or set(locator) - {"id", "role", "name"}: + raise HarnessError(f"{where}: locator must contain only id, role, and/or name") + if not all(isinstance(value, str) and value for value in locator.values()): + raise HarnessError(f"{where}: locator values must be non-empty strings") + + +def validate_expectation(expectation: Any, where: str) -> None: + allowed = {"exists", "not_exists", "focused", "enabled"} + if ( + not isinstance(expectation, dict) + or len(expectation) != 1 + or set(expectation) - allowed + ): + raise HarnessError(f"{where}: expectation must contain exactly one supported condition") + for key in ("exists", "not_exists"): + if key in expectation: + validate_locator(expectation[key], f"{where}.{key}") + focused = expectation.get("focused") + if focused is not None and not isinstance(focused, bool): + validate_locator(focused, f"{where}.focused") + if "enabled" in expectation and not isinstance(expectation["enabled"], bool): + raise HarnessError(f"{where}.enabled must be boolean") + + +def load_journey(path: pathlib.Path) -> dict[str, Any]: + try: + journey = yaml.safe_load(path.read_text()) + except (OSError, yaml.YAMLError) as exc: + raise HarnessError(f"cannot read journey {path}: {exc}") from exc + if not isinstance(journey, dict) or set(journey) != ALLOWED_TOP: + raise HarnessError(f"journey must contain exactly {sorted(ALLOWED_TOP)}") + if journey["schema_version"] != 1 or journey["platforms"] != ["macos"]: + raise HarnessError("only schema_version 1 and platforms: [macos] are supported") + if journey["fixture"] != "local_review_channel": + raise HarnessError("desktop MVP permits only fixture: local_review_channel") + if not isinstance(journey["flow"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", journey["flow"]): + raise HarnessError("flow must be a lowercase filesystem-safe identifier") + record = journey["record"] + if not isinstance(record, dict) or set(record) != {"video", "screenshots", "accessibility"}: + raise HarnessError("record requires exactly video, screenshots, accessibility") + if record["video"] not in ("window", "off") or not all(isinstance(record[k], bool) for k in ("screenshots", "accessibility")): + raise HarnessError("invalid record policy") + steps = journey["steps"] + if not isinstance(steps, list) or not steps: + raise HarnessError("journey requires at least one step") + for index, step in enumerate(steps): + where = f"steps[{index}]" + if not isinstance(step, dict) or set(step) - ALLOWED_STEP or not {"name", "act", "expect"} <= set(step): + raise HarnessError(f"{where}: requires name/act/expect and contains an unsupported field") + if not isinstance(step["name"], str) or not step["name"]: + raise HarnessError(f"{where}.name must be non-empty") + locators = step.get("locate") + if locators is not None: + if not isinstance(locators, list) or not locators: + raise HarnessError(f"{where}.locate must be a non-empty list") + for locator in locators: + validate_locator(locator, f"{where}.locate") + action = step["act"] + if not isinstance(action, dict) or action.get("type") not in {"activate", "click", "move_pointer", "press", "wait"}: + raise HarnessError(f"{where}.act has unsupported type") + if action["type"] in {"click", "move_pointer"} and locators is None: + raise HarnessError(f"{where}: {action['type']} requires locate") + if action["type"] == "press" and not isinstance(action.get("key"), str): + raise HarnessError(f"{where}: press requires key") + validate_expectation(step["expect"], f"{where}.expect") + if "expect_for" in step: + sustained = step["expect_for"] + if not isinstance(sustained, dict) or set(sustained) != {"duration_ms", "condition"}: + raise HarnessError(f"{where}.expect_for requires duration_ms and condition") + validate_expectation(sustained["condition"], f"{where}.expect_for.condition") + timeout = step.get("timeout_ms", 5000) + if not isinstance(timeout, int) or not 0 < timeout <= 60000: + raise HarnessError(f"{where}.timeout_ms must be 1..60000") + cleanup = journey["cleanup"] + if not isinstance(cleanup, dict) or set(cleanup) != {"terminate_app", "remove_state"} or not all(isinstance(v, bool) for v in cleanup.values()): + raise HarnessError("cleanup requires boolean terminate_app and remove_state") + return journey + + +def isolation_manifest(run_id: str, relay_url: str) -> dict[str, str]: + parsed = urllib.parse.urlparse(relay_url) + if parsed.scheme not in {"ws", "http"} or parsed.hostname not in {"localhost", "127.0.0.1", "::1"}: + raise HarnessError(f"refusing non-loopback review relay: {relay_url}") + slug = re.sub(r"[^a-z0-9-]", "-", run_id.lower()) + bundle_id = f"xyz.block.buzz.app.dev.native-review.{slug}" + keyring = f"buzz-desktop-dev.native-review.{slug}" + if bundle_id in PRODUCTION_BUNDLE_IDS or not bundle_id.startswith("xyz.block.buzz.app.dev.native-review."): + raise HarnessError(f"refusing unsafe bundle identifier: {bundle_id}") + if keyring in PRODUCTION_KEYRINGS or not keyring.startswith("buzz-desktop-dev.native-review."): + raise HarnessError(f"refusing unsafe keyring service: {keyring}") + return {"bundle_id": bundle_id, "keyring_service": keyring, "relay_url": relay_url} + + +def provenance() -> dict[str, Any]: + head = git("rev-parse", "HEAD") + status = git("status", "--porcelain=v1", "--untracked-files=all") + diff = run(["git", "diff", "--binary", "HEAD"]).stdout + untracked_hashes = [] + for line in status.splitlines(): + if line.startswith("?? "): + path = ROOT / line[3:] + if path.is_file(): + untracked_hashes.append((line[3:], sha256(path))) + dirty_payload = json.dumps({"diff": diff, "untracked": untracked_hashes}, sort_keys=True).encode() + return { + "head_sha": head, + "dirty": bool(status), + "dirty_state_sha256": hashlib.sha256(dirty_payload).hexdigest() if status else None, + "status": status.splitlines(), + } + + +def scrubbed_environment() -> dict[str, str]: + keep = {"PATH", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", "TERM", "SSH_AUTH_SOCK", "__CF_USER_TEXT_ENCODING"} + env = {key: value for key, value in os.environ.items() if key in keep and not SECRET_NAME.search(key)} + env["HOME"] = "" # filled per run + return env + + +def driver_binary() -> pathlib.Path: + override = os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER") + if override: + return pathlib.Path(override).resolve() + return TOOL_ROOT / "swift" / ".build" / "release" / "buzz-native-driver" + + +def build_driver() -> pathlib.Path: + binary = driver_binary() + if os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER"): + if not binary.is_file(): + raise HarnessError(f"configured driver does not exist: {binary}") + return binary + run(["swift", "build", "-c", "release", "--package-path", str(TOOL_ROOT / "swift")], capture=False) + if not binary.is_file(): + raise HarnessError(f"Swift build succeeded without driver binary: {binary}") + return binary + + +class Driver: + def __init__(self, binary: pathlib.Path, pid: int, semantic_snapshot: pathlib.Path): + self.process = subprocess.Popen([str(binary), "serve", "--pid", str(pid), + "--semantic-snapshot", str(semantic_snapshot)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) + + def request(self, command: str, **payload: Any) -> dict[str, Any]: + assert self.process.stdin and self.process.stdout + self.process.stdin.write(json.dumps({"command": command, **payload}) + "\n") + self.process.stdin.flush() + timeout_seconds = 45 if command in {"record_start", "record_stop"} else 15 + ready, _, _ = select.select([self.process.stdout], [], [], timeout_seconds) + if not ready: + self.process.kill() + self.process.wait(timeout=5) + stderr = self.process.stderr.read() if self.process.stderr else "" + detail = f"; driver stderr:\n{stderr.strip()}" if stderr.strip() else "" + raise HarnessError(f"native driver timed out during {command}{detail}") + line = self.process.stdout.readline() + if not line: + stderr = self.process.stderr.read() if self.process.stderr else "" + raise HarnessError(f"native driver exited during {command}: {stderr.strip()}") + response = json.loads(line) + if not response.get("ok"): + raise HarnessError(str(response.get("error", f"driver {command} failed"))) + return response + + def close(self) -> None: + if self.process.poll() is None: + try: + self.request("shutdown") + except Exception: + self.process.terminate() + for stream in (self.process.stdin, self.process.stdout, self.process.stderr): + if stream: + stream.close() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + + +def doctor(require_permissions: bool = False) -> dict[str, Any]: + checks: list[dict[str, Any]] = [] + checks.append({"name": "platform", "ok": sys.platform == "darwin", "detail": sys.platform}) + for command in ("swift", "xcrun", "git"): + path = shutil.which(command) + checks.append({"name": command, "ok": path is not None, "detail": path or "not found"}) + checks.append({"name": "repository", "ok": (ROOT / "desktop/src-tauri/tauri.conf.json").is_file(), "detail": str(ROOT)}) + try: + binary = build_driver() if all(c["ok"] for c in checks[:4]) else driver_binary() + result = run([str(binary), "doctor"], check=False) + native = json.loads(result.stdout) if result.stdout else {"ok": False, "error": result.stderr.strip()} + checks.extend(native.get("checks", [])) + except (HarnessError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + checks.append({"name": "native-driver", "ok": False, "detail": str(exc)}) + hard_names = {"platform", "swift", "xcrun", "git", "repository", "native-driver-build"} + failed = [c for c in checks if not c.get("ok") and (require_permissions or c.get("name") in hard_names)] + result = {"ok": not failed, "checks": checks} + print(json.dumps(result, indent=2)) + return result + + +def prepare_fixture(run_dir: pathlib.Path, isolation: dict[str, str]) -> dict[str, Any]: + admin = ROOT / "target" / "debug" / "buzz-admin" + if not admin.is_file(): + run(["cargo", "build", "-p", "buzz-admin"], capture=False) + generated = run([str(admin), "generate-key"]).stdout + secret_match = re.search(r"Secret key:\s+(\S+)", generated) + public_match = re.search(r"Public key:\s+(\S+)", generated) + if not secret_match or not public_match: + raise HarnessError("buzz-admin generate-key returned an unrecognized response") + secret_path = run_dir / "state" / "identity.key" + secret_path.parent.mkdir(parents=True, exist_ok=True) + secret_path.write_text(secret_match.group(1) + "\n") + secret_path.chmod(0o600) + fixture = { + "kind": "local_review_channel", "identity_pubkey": public_match.group(1), + "secret_path": str(secret_path), "relay_url": isolation["relay_url"], + "seed": "scripts/setup-desktop-test-data.sh", "cleanup_scope": "run-local app state and keyring only", + } + parsed = urllib.parse.urlparse(isolation["relay_url"]) + port = parsed.port or 80 + seed_env = {**os.environ, "BUZZ_REVIEW_PUBKEY": fixture["identity_pubkey"], + "BUZZ_COMMUNITY_HOST": f"{parsed.hostname}:{port}"} + if port == 3030: + seed_env.update({"BUZZ_DB_HOST": "localhost", "BUZZ_DB_PORT": "5471", "BUZZ_DB_USER": "buzz", + "BUZZ_DB_PASS": "buzz_dev", "BUZZ_DB_NAME": "buzz", + "BUZZ_DB_DOCKER_CONTAINER": "buzz-harness-postgres-1"}) + run([str(ROOT / "scripts/setup-desktop-test-data.sh")], env=seed_env, capture=False) + (run_dir / "manifest" / "fixture.json").write_text(json.dumps({k: v for k, v in fixture.items() if k != "secret_path"}, indent=2)) + return fixture + + +def semantic_probe_server(path: pathlib.Path) -> tuple[http.server.ThreadingHTTPServer, str]: + class Handler(http.server.BaseHTTPRequestHandler): + def end_headers(self) -> None: + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.send_header("Access-Control-Allow-Private-Network", "true") + super().end_headers() + + def do_OPTIONS(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self.send_response(204) + self.end_headers() + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", "0")) + payload = self.rfile.read(length) + try: + value = json.loads(payload) + if not isinstance(value, list): + raise ValueError("snapshot must be an array") + temporary = path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(value)) + temporary.replace(path) + self.send_response(204) + except (ValueError, json.JSONDecodeError, OSError): + self.send_response(400) + self.end_headers() + + def log_message(self, _format: str, *_args: Any) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f"http://127.0.0.1:{server.server_port}/snapshot" + + +def build_and_launch(run_dir: pathlib.Path, isolation: dict[str, str], fixture: dict[str, Any], + probe_url: str) -> tuple[subprocess.Popen[str], pathlib.Path, int]: + # Build with the repository toolchain but no inherited credentials. Launch + # the resulting executable separately so only the app receives isolated HOME. + dev_url = ( + "http://localhost:1420?nativeReview=1" + f"&reviewRelay={urllib.parse.quote(isolation['relay_url'], safe='')}" + f"&reviewPubkey={urllib.parse.quote(fixture['identity_pubkey'], safe='')}" + ) + config = json.dumps({ + "build": {"devUrl": dev_url}, + "identifier": isolation["bundle_id"], "productName": "Buzz Native Review", + "bundle": {"externalBin": []}, + }, separators=(",", ":")) + build_env = scrubbed_environment() + build_env["HOME"] = os.environ["HOME"] + build_env["VITE_NATIVE_REVIEW"] = "1" + build_env["VITE_NATIVE_REVIEW_RELAY"] = isolation["relay_url"] + build_env["VITE_NATIVE_REVIEW_PUBKEY"] = fixture["identity_pubkey"] + build_env["VITE_NATIVE_REVIEW_PROBE_URL"] = probe_url + run(["pnpm", "exec", "tauri", "build", "--debug", "--bundles", "app", "--config", config], + cwd=ROOT / "desktop", env=build_env, capture=False) + app_binary = (ROOT / "desktop" / "src-tauri" / "target" / "debug" / "bundle" / "macos" / + "Buzz Native Review.app" / "Contents" / "MacOS" / "buzz-desktop") + if not app_binary.is_file(): + raise HarnessError(f"Tauri build succeeded without app binary: {app_binary}") + + env = scrubbed_environment() + env["HOME"] = str(run_dir / "home") + pathlib.Path(env["HOME"]).mkdir(parents=True, exist_ok=True) + env.update({ + "BUZZ_PRIVATE_KEY": pathlib.Path(fixture["secret_path"]).read_text().strip(), + "BUZZ_RELAY_URL": isolation["relay_url"], "BUZZ_DEV_KEYRING_SERVICE": isolation["keyring_service"], + "BUZZ_NATIVE_REVIEW": "1", "BUZZ_NATIVE_REVIEW_CHANNEL": "general", + }) + log = (run_dir / "logs" / "app.log").open("w") + process = subprocess.Popen([str(app_binary)], cwd=ROOT, env=env, stdout=log, + stderr=subprocess.STDOUT, text=True) + log.close() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if process.poll() is not None: + raise HarnessError(f"Tauri exited during launch; see {run_dir / 'logs/app.log'}") + if run(["ps", "-p", str(process.pid), "-o", "comm="], check=False).stdout.rstrip().endswith("/buzz-desktop"): + return process, app_binary, process.pid + time.sleep(0.25) + process.terminate() + raise HarnessError("timed out waiting for native Buzz process") + + +def wait_for_visible_window(driver: Driver, process: subprocess.Popen[str], timeout_seconds: float = 30) -> dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + last_status: dict[str, Any] | None = None + while time.monotonic() < deadline: + if process.poll() is not None: + raise HarnessError("Tauri exited while waiting for its initial window") + last_status = driver.request("window_status") + if last_status.get("visible"): + return last_status + time.sleep(0.1) + detail = last_status.get("detail") if last_status else "driver returned no status" + raise HarnessError(f"timed out after {timeout_seconds:g}s waiting for visible native window: {detail}") + + +def locate_required(driver: Driver, locators: list[dict[str, str]], timeout_ms: int) -> dict[str, Any]: + """Wait for a semantic target to materialize, then return the locator used.""" + deadline = time.monotonic() + timeout_ms / 1000 + while True: + found = driver.request("locate", locators=locators, required=False).get("element") + if found is not None: + return found + if time.monotonic() >= deadline: + raise HarnessError(f"no accessibility element matched ordered locators within {timeout_ms}ms") + time.sleep(0.025) + + +def expectation_holds(driver: Driver, expectation: dict[str, Any]) -> bool: + if "exists" in expectation: + return driver.request("locate", locators=[expectation["exists"]], required=False).get("element") is not None + if "not_exists" in expectation: + return driver.request("locate", locators=[expectation["not_exists"]], required=False).get("element") is None + if "focused" in expectation and isinstance(expectation["focused"], dict): + found = driver.request("locate", locators=[expectation["focused"]], required=False).get("element") + return bool(found and found.get("focused")) + if "focused" in expectation: + return bool(driver.request("focused").get("focused")) == expectation["focused"] + if "enabled" in expectation: + return bool(driver.request("selected").get("element", {}).get("enabled")) == expectation["enabled"] + return False + + +def wait_expectation(driver: Driver, expectation: dict[str, Any], timeout_ms: int) -> None: + deadline = time.monotonic() + timeout_ms / 1000 + while True: + if expectation_holds(driver, expectation): + return + if time.monotonic() >= deadline: + raise HarnessError(f"postcondition not met within {timeout_ms}ms: {expectation}") + time.sleep(0.025) + + +def capture_step(driver: Driver, run_dir: pathlib.Path, slug: str, record: dict[str, Any]) -> dict[str, str]: + artifacts: dict[str, str] = {} + if record["screenshots"]: + path = run_dir / "screenshots" / f"{slug}.png" + driver.request("screenshot", path=str(path)) + artifacts["screenshot"] = str(path.relative_to(run_dir)) + if record["accessibility"]: + path = run_dir / "accessibility" / f"{slug}.json" + response = driver.request("snapshot") + path.write_text(json.dumps({key: value for key, value in response.items() if key != "ok"}, indent=2)) + artifacts["accessibility"] = str(path.relative_to(run_dir)) + return artifacts + + +def run_journey(path: pathlib.Path, relay_url: str, output_root: pathlib.Path) -> pathlib.Path: + journey = load_journey(path) + run_id = f"{journey['flow']}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{secrets.token_hex(3)}" + isolation = isolation_manifest(run_id, relay_url) + run_dir = output_root / git("rev-parse", "--short=12", "HEAD") / journey["flow"] / run_id + for child in ("manifest", "logs", "screenshots", "accessibility", "state", "home"): + (run_dir / child).mkdir(parents=True, exist_ok=True) + (run_dir / "journey.yaml").write_text(path.read_text()) + prov = provenance() + (run_dir / "manifest" / "git.json").write_text(json.dumps(prov, indent=2)) + (run_dir / "manifest" / "isolation.json").write_text(json.dumps(isolation, indent=2)) + started = utc_now() + receipt: dict[str, Any] = {"schema_version": 1, "run_id": run_id, "flow": journey["flow"], "status": "failed", + "started_at": started, "finished_at": started, "failure": None, "provenance": prov, "isolation": isolation, + "artifacts": {}, "steps": [], "cleanup": {"status": "not_started"}} + process: subprocess.Popen[str] | None = None + driver: Driver | None = None + fixture: dict[str, Any] | None = None + probe_server: http.server.ThreadingHTTPServer | None = None + try: + if not doctor(require_permissions=True)["ok"]: + raise HarnessError("doctor failed; grant required permissions and rerun") + fixture = prepare_fixture(run_dir, isolation) + probe_server, probe_url = semantic_probe_server(run_dir / "state" / "semantic.json") + process, app_binary, app_pid = build_and_launch(run_dir, isolation, fixture, probe_url) + receipt["provenance"]["artifact_path"] = str(app_binary) + receipt["provenance"]["artifact_sha256"] = sha256(app_binary) + driver = Driver(build_driver(), app_pid, run_dir / "state" / "semantic.json") + receipt["provenance"]["initial_window"] = wait_for_visible_window(driver, process) + if journey["record"]["video"] == "window": + video = run_dir / "video.mp4" + driver.request("record_start", path=str(video)) + receipt["artifacts"]["video"] = "video.mp4" + for index, step in enumerate(journey["steps"]): + slug = f"{index + 1:02d}-{re.sub(r'[^a-z0-9-]', '-', step['name'].lower())}" + step_start = time.monotonic_ns() + selected = None + step_receipt: dict[str, Any] = {"name": step["name"], "status": "failed", "started_monotonic_ns": step_start} + receipt["steps"].append(step_receipt) + try: + if step.get("locate"): + selected = locate_required(driver, step["locate"], step.get("timeout_ms", 5000)) + step_receipt["locator"] = selected.get("locator") + driver.request("act", action=step["act"], element=selected) + wait_expectation(driver, step["expect"], step.get("timeout_ms", 5000)) + if sustained := step.get("expect_for"): + until = time.monotonic() + sustained["duration_ms"] / 1000 + while time.monotonic() < until: + if not expectation_holds(driver, sustained["condition"]): + raise HarnessError(f"sustained postcondition failed: {sustained['condition']}") + time.sleep(0.025) + step_receipt["status"] = "passed" + finally: + step_receipt["finished_monotonic_ns"] = time.monotonic_ns() + step_receipt["duration_ms"] = (step_receipt["finished_monotonic_ns"] - step_start) / 1_000_000 + step_receipt["artifacts"] = capture_step(driver, run_dir, slug, journey["record"]) + if journey["record"]["video"] == "window": + driver.request("record_stop") + receipt["status"] = "passed" + except Exception as exc: + receipt["failure"] = str(exc) + if driver: + try: + receipt["artifacts"]["failure"] = capture_step(driver, run_dir, "failure", {"screenshots": True, "accessibility": True}) + except Exception as capture_exc: + receipt["artifacts"]["capture_failure"] = str(capture_exc) + try: + driver.request("record_stop") + except Exception: + pass + finally: + cleanup_errors = [] + if probe_server: + probe_server.shutdown() + probe_server.server_close() + if driver: + try: + driver.close() + except Exception as exc: + cleanup_errors.append(f"native driver cleanup failed: {exc}") + if process and journey["cleanup"]["terminate_app"]: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill(); cleanup_errors.append("Tauri launcher required SIGKILL") + if journey["cleanup"]["remove_state"]: + try: + run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), isolation["bundle_id"], isolation["keyring_service"]], + env={**os.environ, "HOME": str(run_dir / "home")}) + if fixture: + pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + except Exception as exc: + cleanup_errors.append(str(exc)) + receipt["cleanup"] = {"status": "failed" if cleanup_errors else "passed", "errors": cleanup_errors} + if cleanup_errors: + receipt["status"] = "failed" + receipt["finished_at"] = utc_now() + (run_dir / "receipt.json").write_text(json.dumps(receipt, indent=2)) + report = f"# Native review: {journey['flow']}\n\n**{receipt['status'].upper()}** at `{prov['head_sha']}`.\n\nReceipt: `receipt.json`\n" + if receipt["failure"]: + report += f"\nFailure: `{receipt['failure']}`\n" + (run_dir / "report.md").write_text(report) + print(run_dir) + if receipt["status"] != "passed": + raise HarnessError(receipt["failure"] or "journey or cleanup failed") + return run_dir + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + doctor_parser = sub.add_parser("doctor") + doctor_parser.add_argument("--require-permissions", action="store_true") + validate_parser = sub.add_parser("validate") + validate_parser.add_argument("journey", type=pathlib.Path) + run_parser = sub.add_parser("run") + run_parser.add_argument("journey", type=pathlib.Path) + run_parser.add_argument("--relay", default="ws://localhost:3030") + run_parser.add_argument("--output", type=pathlib.Path, default=ROOT / "test-results/native-review") + args = parser.parse_args() + try: + if args.command == "doctor": + return 0 if doctor(args.require_permissions)["ok"] else 1 + if args.command == "validate": + load_journey(args.journey); print(f"valid: {args.journey}"); return 0 + run_journey(args.journey, args.relay, args.output); return 0 + except HarnessError as exc: + print(f"native-review: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/native-review/schemas/journey.schema.json b/tools/native-review/schemas/journey.schema.json new file mode 100644 index 0000000000..c61bab6317 --- /dev/null +++ b/tools/native-review/schemas/journey.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://buzz.local/native-review/journey.schema.json", + "title": "Buzz native review journey", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "flow", "platforms", "fixture", "record", "steps", "cleanup"], + "properties": { + "schema_version": {"const": 1}, + "flow": {"type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$"}, + "platforms": {"type": "array", "minItems": 1, "items": {"enum": ["macos"]}}, + "fixture": {"enum": ["local_review_channel"]}, + "record": { + "type": "object", "additionalProperties": false, + "required": ["video", "screenshots", "accessibility"], + "properties": { + "video": {"enum": ["window", "off"]}, + "screenshots": {"type": "boolean"}, + "accessibility": {"type": "boolean"} + } + }, + "steps": { + "type": "array", "minItems": 1, + "items": { + "type": "object", "additionalProperties": false, + "required": ["name", "act", "expect"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "locate": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/locator"}}, + "act": {"$ref": "#/$defs/action"}, + "expect": {"$ref": "#/$defs/expectation"}, + "expect_for": { + "type": "object", "additionalProperties": false, + "required": ["duration_ms", "condition"], + "properties": { + "duration_ms": {"type": "integer", "minimum": 1, "maximum": 30000}, + "condition": {"$ref": "#/$defs/expectation"} + } + }, + "timeout_ms": {"type": "integer", "minimum": 1, "maximum": 60000}, + "measure": {"type": "string", "minLength": 1} + } + } + }, + "cleanup": { + "type": "object", "additionalProperties": false, + "required": ["terminate_app", "remove_state"], + "properties": {"terminate_app": {"type": "boolean"}, "remove_state": {"type": "boolean"}} + } + }, + "$defs": { + "locator": { + "type": "object", "additionalProperties": false, "minProperties": 1, + "properties": {"id": {"type": "string"}, "role": {"type": "string"}, "name": {"type": "string"}} + }, + "action": { + "type": "object", "additionalProperties": false, "required": ["type"], + "properties": { + "type": {"enum": ["activate", "click", "move_pointer", "press", "wait"]}, + "duration_ms": {"type": "integer", "minimum": 0, "maximum": 30000}, + "key": {"type": "string"} + } + }, + "expectation": { + "type": "object", "additionalProperties": false, "minProperties": 1, "maxProperties": 1, + "properties": { + "exists": {"$ref": "#/$defs/locator"}, + "not_exists": {"$ref": "#/$defs/locator"}, + "focused": {"oneOf": [{"type": "boolean"}, {"$ref": "#/$defs/locator"}]}, + "enabled": {"type": "boolean"} + } + } + } +} diff --git a/tools/native-review/schemas/receipt.schema.json b/tools/native-review/schemas/receipt.schema.json new file mode 100644 index 0000000000..404d183a9c --- /dev/null +++ b/tools/native-review/schemas/receipt.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://buzz.local/native-review/receipt.schema.json", + "title": "Buzz native review receipt", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "run_id", "flow", "status", "started_at", "finished_at", "provenance", "isolation", "artifacts", "steps", "cleanup"], + "properties": { + "schema_version": {"const": 1}, "run_id": {"type": "string"}, "flow": {"type": "string"}, + "status": {"enum": ["passed", "failed"]}, "started_at": {"type": "string"}, "finished_at": {"type": "string"}, + "failure": {"type": ["string", "null"]}, + "provenance": {"type": "object"}, "isolation": {"type": "object"}, "artifacts": {"type": "object"}, + "steps": {"type": "array", "items": {"type": "object"}}, "cleanup": {"type": "object"} + } +} diff --git a/tools/native-review/swift/Package.swift b/tools/native-review/swift/Package.swift new file mode 100644 index 0000000000..3fc591eaa1 --- /dev/null +++ b/tools/native-review/swift/Package.swift @@ -0,0 +1,9 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "BuzzNativeDriver", + platforms: [.macOS(.v13)], + products: [.executable(name: "buzz-native-driver", targets: ["BuzzNativeDriver"])], + targets: [.executableTarget(name: "BuzzNativeDriver")] +) diff --git a/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift b/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift new file mode 100644 index 0000000000..668b04907d --- /dev/null +++ b/tools/native-review/swift/Sources/BuzzNativeDriver/main.swift @@ -0,0 +1,542 @@ +import AppKit +import ApplicationServices +import AVFoundation +import CoreGraphics +import Foundation + +struct Locator: Codable { + let id: String? + let role: String? + let name: String? +} + +struct Size: Codable { + let width: Double + let height: Double +} + +struct SemanticNode: Codable { + let id: String? + let role: String? + let name: String? + let enabled: Bool + let focused: Bool + let frame: Rect + let viewport: Size +} + +struct ElementDescription: Codable { + let locator: Locator + let role: String? + let name: String? + let identifier: String? + let enabled: Bool + let focused: Bool + let frame: Rect? +} + +struct Rect: Codable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct AXNode: Codable { + let role: String? + let name: String? + let identifier: String? + let value: String? + let enabled: Bool? + let focused: Bool? + let frame: Rect? + let children: [AXNode] + let truncated: Bool? +} + +enum DriverError: Error, CustomStringConvertible { + case message(String) + var description: String { + switch self { case .message(let value): return value } + } +} + +func enableWebViewAccessibility(_ app: AXUIElement) { + // WebKit does not materialize its remote accessibility tree for ordinary + // automation clients until manual/enhanced accessibility is requested. + // VoiceOver does this implicitly; the review harness must not require it. + let enabled = kCFBooleanTrue as CFTypeRef + for name in ["AXManualAccessibility", "AXEnhancedUserInterface"] { + _ = AXUIElementSetAttributeValue(app, name as CFString, enabled) + } +} + +func attribute(_ element: AXUIElement, _ name: String) -> AnyObject? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else { return nil } + return value +} + +func stringAttribute(_ element: AXUIElement, _ names: [String]) -> String? { + for name in names { + if let value = attribute(element, name) as? String, !value.isEmpty { return value } + } + return nil +} + +func boolAttribute(_ element: AXUIElement, _ name: String, default fallback: Bool = false) -> Bool { + (attribute(element, name) as? Bool) ?? fallback +} + +func frameAttribute(_ element: AXUIElement) -> Rect? { + guard let positionObject = attribute(element, kAXPositionAttribute), + let sizeObject = attribute(element, kAXSizeAttribute), + CFGetTypeID(positionObject) == AXValueGetTypeID(), + CFGetTypeID(sizeObject) == AXValueGetTypeID() else { return nil } + let positionValue = positionObject as! AXValue + let sizeValue = sizeObject as! AXValue + var point = CGPoint.zero + var size = CGSize.zero + guard AXValueGetValue(positionValue, .cgPoint, &point), AXValueGetValue(sizeValue, .cgSize, &size) else { return nil } + return Rect(x: point.x, y: point.y, width: size.width, height: size.height) +} + +func normalizedRole(_ raw: String?) -> String? { + guard var value = raw else { return nil } + if value.hasPrefix("AX") { value.removeFirst(2) } + switch value.lowercased() { + case "textarea", "text area": return "text-area" + default: return value.lowercased() + } +} + +func elementName(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute, kAXValueAttribute]) +} + +func elementIdentifier(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXIdentifierAttribute]) +} + +func elementRole(_ element: AXUIElement) -> String? { + stringAttribute(element, [kAXRoleAttribute]) +} + +func matches(_ element: AXUIElement, locator: Locator) -> Bool { + if let id = locator.id, elementIdentifier(element) != id { return false } + if let role = locator.role, normalizedRole(elementRole(element)) != normalizedRole(role) { return false } + if let name = locator.name, elementName(element) != name { return false } + return true +} + +func children(_ element: AXUIElement) -> [AXUIElement] { + let direct = (attribute(element, kAXChildrenAttribute) as? [AXUIElement]) ?? [] + let windows = (attribute(element, kAXWindowsAttribute) as? [AXUIElement]) ?? [] + let singular = [kAXMainWindowAttribute, kAXFocusedWindowAttribute].compactMap { + attribute(element, $0) as! AXUIElement? + } + var seen = Set() + return (direct + windows + singular).filter { child in + guard !CFEqual(child, element) else { return false } + return seen.insert(CFHash(child)).inserted + } +} + +func accessibilityRoots(app: AXUIElement, pid: pid_t) -> [AXUIElement] { + // WKWebView's remote tree is not always attached to AXChildren/AXWindows, + // even after manual accessibility is enabled. Hit-testing the visible app + // window asks the accessibility server for the element actually painted at + // each point and reliably materializes the WebKit subtree without DOM IPC. + var roots = [app] + guard let (_, bounds) = try? windowInfo(pid: pid) else { return roots } + let system = AXUIElementCreateSystemWide() + for xFraction in stride(from: 0.05, through: 0.95, by: 0.1) { + for yFraction in stride(from: 0.05, through: 0.95, by: 0.1) { + var element: AXUIElement? + let x = Float(bounds.minX + bounds.width * xFraction) + let y = Float(bounds.minY + bounds.height * yFraction) + if AXUIElementCopyElementAtPosition(system, x, y, &element) == .success, + let element { + roots.append(element) + } + } + } + var seen = Set() + return roots.filter { seen.insert(CFHash($0)).inserted } +} + +func find(_ roots: [AXUIElement], locator: Locator, maxNodes: Int = 20_000) -> AXUIElement? { + var queue = roots + var seen = Set() + var visited = 0 + while !queue.isEmpty && visited < maxNodes { + let current = queue.removeFirst() + guard seen.insert(CFHash(current)).inserted else { continue } + visited += 1 + if matches(current, locator: locator) { return current } + queue.append(contentsOf: children(current)) + } + return nil +} + +func describe(_ element: AXUIElement, locator: Locator) -> ElementDescription { + ElementDescription(locator: locator, role: normalizedRole(elementRole(element)), name: elementName(element), + identifier: elementIdentifier(element), enabled: boolAttribute(element, kAXEnabledAttribute, default: true), + focused: boolAttribute(element, kAXFocusedAttribute), frame: frameAttribute(element)) +} + +func semanticNodes(path: String, pid: pid_t) -> [SemanticNode] { + guard let data = FileManager.default.contents(atPath: path), + let nodes = try? JSONDecoder().decode([SemanticNode].self, from: data) else { return [] } + guard let (_, windowBounds) = try? windowInfo(pid: pid) else { return nodes } + return nodes.map { node in + guard node.viewport.width > 0, node.viewport.height > 0 else { return node } + let xScale = windowBounds.width / node.viewport.width + let yScale = windowBounds.height / node.viewport.height + return SemanticNode( + id: node.id, + role: node.role, + name: node.name, + enabled: node.enabled, + focused: node.focused, + frame: Rect( + x: windowBounds.minX + node.frame.x * xScale, + y: windowBounds.minY + node.frame.y * yScale, + width: node.frame.width * xScale, + height: node.frame.height * yScale + ), + viewport: node.viewport + ) + } +} + +func matches(_ element: SemanticNode, locator: Locator) -> Bool { + if let id = locator.id, element.id != id { return false } + if let role = locator.role, normalizedRole(element.role) != normalizedRole(role) { return false } + if let name = locator.name, element.name != name { return false } + return true +} + +func describe(_ element: SemanticNode, locator: Locator) -> ElementDescription { + ElementDescription(locator: locator, role: normalizedRole(element.role), name: element.name, + identifier: element.id, enabled: element.enabled, focused: element.focused, + frame: element.frame) +} + +func snapshot(_ element: AXUIElement, depth: Int = 0, budget: inout Int) -> AXNode { + budget -= 1 + let isTruncated = budget <= 0 || depth >= 80 + let descendants = isTruncated ? [] : children(element).map { snapshot($0, depth: depth + 1, budget: &budget) } + return AXNode(role: normalizedRole(elementRole(element)), name: elementName(element), identifier: elementIdentifier(element), + value: stringAttribute(element, [kAXValueAttribute]), enabled: attribute(element, kAXEnabledAttribute) as? Bool, + focused: attribute(element, kAXFocusedAttribute) as? Bool, frame: frameAttribute(element), + children: descendants, truncated: isTruncated ? true : nil) +} + +func windowInfo(pid: pid_t) throws -> (CGWindowID, CGRect) { + guard let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + throw DriverError.message("cannot enumerate windows; Screen Recording permission may be missing") + } + let candidates = windows.compactMap { window -> (CGWindowID, CGRect)? in + guard (window[kCGWindowOwnerPID as String] as? Int32) == pid, + let number = window[kCGWindowNumber as String] as? CGWindowID, + let boundsValue = window[kCGWindowBounds as String], + CFGetTypeID(boundsValue as CFTypeRef) == CFDictionaryGetTypeID(), + let bounds = CGRect(dictionaryRepresentation: boundsValue as! CFDictionary), + bounds.width > 0, bounds.height > 0 else { return nil } + return (number, bounds) + } + if let largest = candidates.max(by: { $0.1.width * $0.1.height < $1.1.width * $1.1.height }) { + return largest + } + throw DriverError.message("no on-screen window found for pid \(pid)") +} + +func windowStatus(pid: pid_t) -> [String: Any] { + do { + let (windowID, bounds) = try windowInfo(pid: pid) + return [ + "ok": true, + "visible": true, + "window_id": windowID, + "bounds": ["x": bounds.origin.x, "y": bounds.origin.y, "width": bounds.width, "height": bounds.height], + ] + } catch { + return ["ok": true, "visible": false, "detail": String(describing: error)] + } +} + +func captureWindow(pid: pid_t, path: String) throws { + let (windowID, bounds) = try windowInfo(pid: pid) + guard let image = CGWindowListCreateImage(bounds, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution]) else { + throw DriverError.message("window screenshot failed") + } + let destination = URL(fileURLWithPath: path) + try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + guard let bitmap = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]) else { + throw DriverError.message("PNG encoding failed") + } + try bitmap.write(to: destination, options: .atomic) +} + +final class WindowRecorder: @unchecked Sendable { + private var writer: AVAssetWriter? + private var input: AVAssetWriterInput? + private var adaptor: AVAssetWriterInputPixelBufferAdaptor? + private var captureTask: Task? + private var captureError: Error? + + private func trace(_ message: String) { + let timestamp = ISO8601DateFormatter().string(from: Date()) + FileHandle.standardError.write(Data("[\(timestamp)] record: \(message)\n".utf8)) + } + + func start(pid: pid_t, path: String) throws { + let (windowID, bounds) = try windowInfo(pid: pid) + let width = max(Int(bounds.width) & ~1, 2) + let height = max(Int(bounds.height) & ~1, 2) + let destination = URL(fileURLWithPath: path) + try FileManager.default.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: destination) + + let assetWriter = try AVAssetWriter(outputURL: destination, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + ] + let writerInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + writerInput.expectsMediaDataInRealTime = true + let attributes: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + kCVPixelBufferCGImageCompatibilityKey as String: true, + kCVPixelBufferCGBitmapContextCompatibilityKey as String: true, + ] + let pixelAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: writerInput, sourcePixelBufferAttributes: attributes) + guard assetWriter.canAdd(writerInput) else { throw DriverError.message("AVAssetWriter rejected window video input") } + assetWriter.add(writerInput) + guard assetWriter.startWriting() else { throw assetWriter.error ?? DriverError.message("AVAssetWriter failed to start") } + assetWriter.startSession(atSourceTime: .zero) + writer = assetWriter; input = writerInput; adaptor = pixelAdaptor + trace("started CGWindow/AVAssetWriter backend window=\(windowID) size=\(width)x\(height)") + + captureTask = Task.detached { [weak self] in + guard let self else { return } + let start = ContinuousClock.now + var frame: Int64 = 0 + while !Task.isCancelled { + autoreleasepool { + guard writerInput.isReadyForMoreMediaData, + let image = CGWindowListCreateImage(bounds, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution]), + let pool = pixelAdaptor.pixelBufferPool else { return } + var optionalBuffer: CVPixelBuffer? + guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &optionalBuffer) == kCVReturnSuccess, + let buffer = optionalBuffer else { return } + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + guard let base = CVPixelBufferGetBaseAddress(buffer), + let context = CGContext(data: base, width: width, height: height, + bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) else { return } + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + let time = CMTime(value: frame, timescale: 15) + if !pixelAdaptor.append(buffer, withPresentationTime: time) { + self.captureError = assetWriter.error ?? DriverError.message("failed to append window video frame") + } + frame += 1 + } + let target = start.advanced(by: .milliseconds(Int(frame * 1000 / 15))) + try? await Task.sleep(until: target) + } + } + } + + func stop() async throws { + captureTask?.cancel() + _ = await captureTask?.result + captureTask = nil + input?.markAsFinished() + if let assetWriter = writer { await assetWriter.finishWriting() } + let error = captureError ?? writer?.error + writer = nil; input = nil; adaptor = nil; captureError = nil + if let error { throw error } + trace("finalized recording") + } +} + +func jsonObject(_ data: Data) throws -> [String: Any] { + guard let value = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw DriverError.message("request must be a JSON object") + } + return value +} + +func locator(from value: Any) throws -> Locator { + let data = try JSONSerialization.data(withJSONObject: value) + return try JSONDecoder().decode(Locator.self, from: data) +} + +func response(_ object: [String: Any]) { + let data = try! JSONSerialization.data(withJSONObject: object) + print(String(data: data, encoding: .utf8)!) + fflush(stdout) +} + +func encoded(_ value: T) throws -> Any { + let data = try JSONEncoder().encode(value) + return try JSONSerialization.jsonObject(with: data) +} + +func keyCode(_ key: String) throws -> CGKeyCode { + switch key.lowercased() { + case "tab": return 48 + case "return", "enter": return 36 + case "escape": return 53 + case "space": return 49 + default: throw DriverError.message("unsupported key: \(key)") + } +} + +@main +struct BuzzNativeDriver { + static func main() async { + do { + let arguments = Array(CommandLine.arguments.dropFirst()) + guard let command = arguments.first else { throw DriverError.message("usage: buzz-native-driver doctor | serve --pid PID") } + if command == "doctor" { + let accessibility = AXIsProcessTrusted() + let screen = CGPreflightScreenCaptureAccess() + response(["ok": true, "checks": [ + ["name": "native-driver-build", "ok": true, "detail": "Swift AX/CGEvent/Core Graphics/AVFoundation driver available"], + ["name": "accessibility-permission", "ok": accessibility, "detail": accessibility ? "granted" : "grant Accessibility to the invoking terminal/agent"], + ["name": "screen-recording-permission", "ok": screen, "detail": screen ? "granted" : "grant Screen Recording to the invoking terminal/agent"], + ["name": "recording-api", "ok": ProcessInfo.processInfo.isOperatingSystemAtLeast(.init(majorVersion: 15, minorVersion: 0, patchVersion: 0)), "detail": ProcessInfo.processInfo.operatingSystemVersionString] + ]]) + return + } + guard command == "serve", (arguments.count == 3 || arguments.count == 5), arguments[1] == "--pid", let pid = pid_t(arguments[2]) else { + throw DriverError.message("usage: buzz-native-driver serve --pid PID [--semantic-snapshot PATH]") + } + let semanticSnapshotPath: String? = arguments.count == 5 && arguments[3] == "--semantic-snapshot" ? arguments[4] : nil + guard AXIsProcessTrusted() else { throw DriverError.message("Accessibility permission is not granted") } + let app = AXUIElementCreateApplication(pid) + enableWebViewAccessibility(app) + var selected: (ElementDescription, Locator)? + var recorder: Any? + for try await line in FileHandle.standardInput.bytes.lines { + do { + let request = try jsonObject(Data(line.utf8)) + guard let name = request["command"] as? String else { throw DriverError.message("missing command") } + switch name { + case "locate": + guard let values = request["locators"] as? [Any] else { throw DriverError.message("locate requires locators") } + var found: (ElementDescription, Locator)? + for value in values { + let candidate = try locator(from: value) + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: candidate) }) { + found = (describe(element, locator: candidate), candidate) + break + } + let semanticActive = semanticSnapshotPath.map { FileManager.default.fileExists(atPath: $0) } ?? false + if !semanticActive || candidate.role == "window" { + if let element = find(accessibilityRoots(app: app, pid: pid), locator: candidate) { + found = (describe(element, locator: candidate), candidate) + break + } + } + } + selected = found + if let (element, _) = found { + response(["ok": true, "element": try encoded(element)]) + } else if (request["required"] as? Bool) == true { + throw DriverError.message("no accessibility element matched ordered locators") + } else { response(["ok": true, "element": NSNull()]) } + case "act": + guard let action = request["action"] as? [String: Any], let type = action["type"] as? String else { throw DriverError.message("act requires action.type") } + if type == "activate" { + guard let running = NSRunningApplication(processIdentifier: pid) else { throw DriverError.message("target app is no longer running") } + running.activate(options: [.activateIgnoringOtherApps]) + } else if type == "wait" { + try await Task.sleep(for: .milliseconds(action["duration_ms"] as? Int ?? 0)) + } else if type == "press" { + let code = try keyCode(action["key"] as? String ?? "") + CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true)?.post(tap: .cghidEventTap) + CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)?.post(tap: .cghidEventTap) + } else { + guard let (_, used) = selected else { + throw DriverError.message("action requires a freshly selected element with current bounds") + } + var fresh: ElementDescription? + if let path = semanticSnapshotPath, + let element = semanticNodes(path: path, pid: pid).first(where: { matches($0, locator: used) }) { + fresh = describe(element, locator: used) + } + if fresh == nil, let element = find(accessibilityRoots(app: app, pid: pid), locator: used) { + fresh = describe(element, locator: used) + } + guard let element = fresh, let frame = element.frame else { + throw DriverError.message("action requires a freshly selected element with current bounds") + } + selected = (element, used) + let point = CGPoint(x: frame.x + frame.width / 2, y: frame.y + frame.height / 2) + let duration = max(action["duration_ms"] as? Int ?? 0, 0) + if type == "move_pointer" && duration > 0 { + let current = NSEvent.mouseLocation + let start = CGPoint(x: current.x, y: NSScreen.screens.first.map { $0.frame.height - current.y } ?? current.y) + let steps = max(duration / 8, 2) + for index in 1...steps { + let fraction = Double(index) / Double(steps) + let next = CGPoint(x: start.x + (point.x - start.x) * fraction, y: start.y + (point.y - start.y) * fraction) + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: next, mouseButton: .left)?.post(tap: .cghidEventTap) + try await Task.sleep(for: .milliseconds(duration / steps)) + } + } else if type == "move_pointer" { + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + } else if type == "click" { + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left)?.post(tap: .cghidEventTap) + } else { throw DriverError.message("unsupported action: \(type)") } + } + response(["ok": true]) + case "window_status": + response(windowStatus(pid: pid)) + case "snapshot": + var budget = 20_000 + var object: [String: Any] = ["ok": true, "tree": try encoded(snapshot(app, budget: &budget))] + if let path = semanticSnapshotPath { + object["semantic"] = try encoded(semanticNodes(path: path, pid: pid)) + } + response(object) + case "screenshot": + guard let path = request["path"] as? String else { throw DriverError.message("screenshot requires path") } + try captureWindow(pid: pid, path: path); response(["ok": true]) + case "focused": + response(["ok": true, "focused": selected?.0.focused ?? false]) + case "selected": + if let (element, _) = selected { response(["ok": true, "element": try encoded(element)]) } + else { response(["ok": true, "element": NSNull()]) } + case "record_start": + guard let path = request["path"] as? String else { throw DriverError.message("record_start requires path") } + let value = WindowRecorder(); try value.start(pid: pid, path: path); recorder = value; response(["ok": true]) + case "record_stop": + if let value = recorder as? WindowRecorder { try await value.stop() } + recorder = nil; response(["ok": true]) + case "shutdown": response(["ok": true]); return + default: throw DriverError.message("unknown command: \(name)") + } + } catch { + response(["ok": false, "error": String(describing: error)]) + } + } + } catch { + response(["ok": false, "error": String(describing: error)]) + exit(1) + } + } +} diff --git a/tools/native-review/tests/fixtures/broken-tooltip.yaml b/tools/native-review/tests/fixtures/broken-tooltip.yaml new file mode 100644 index 0000000000..4631454f72 --- /dev/null +++ b/tools/native-review/tests/fixtures/broken-tooltip.yaml @@ -0,0 +1,13 @@ +schema_version: 1 +flow: broken_tooltip +platforms: [macos] +fixture: local_review_channel +record: {video: "off", screenshots: true, accessibility: true} +steps: + - name: impossible_locator + locate: [{id: this-element-must-not-exist}] + act: {type: click} + expect: + exists: {id: still-impossible} + timeout_ms: 50 +cleanup: {terminate_app: true, remove_state: true} diff --git a/tools/native-review/tests/test_review_native.py b/tools/native-review/tests/test_review_native.py new file mode 100644 index 0000000000..397cb22122 --- /dev/null +++ b/tools/native-review/tests/test_review_native.py @@ -0,0 +1,97 @@ +import importlib.util +import pathlib +import tempfile +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).parents[1] / "review_native.py" +SPEC = importlib.util.spec_from_file_location("review_native", MODULE_PATH) +review_native = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(review_native) + + +class JourneyTests(unittest.TestCase): + def test_real_journey_validates(self): + journey = review_native.load_journey(MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml") + self.assertEqual(journey["flow"], "tooltip_fresh_dwell") + + def test_broken_mutation_is_schema_valid(self): + journey = review_native.load_journey(MODULE_PATH.parent / "tests/fixtures/broken-tooltip.yaml") + self.assertEqual(journey["steps"][0]["timeout_ms"], 50) + + def test_action_without_postcondition_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace(" expect:\n exists: {role: window}\n", "", 1) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "requires name/act/expect"): + review_native.load_journey(path) + + def test_expectation_with_multiple_conditions_is_rejected(self): + source = (MODULE_PATH.parent / "desktop/tooltip-fresh-dwell.yaml").read_text() + source = source.replace( + " expect:\n exists: {role: window}\n", + " expect:\n exists: {role: window}\n enabled: true\n", + 1, + ) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "invalid.yaml" + path.write_text(source) + with self.assertRaisesRegex(review_native.HarnessError, "exactly one"): + review_native.load_journey(path) + + def test_production_and_remote_targets_are_rejected(self): + with self.assertRaisesRegex(review_native.HarnessError, "non-loopback"): + review_native.isolation_manifest("run", "wss://buzz.block.builderlab.xyz") + safe = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + self.assertTrue(safe["bundle_id"].startswith("xyz.block.buzz.app.dev.native-review.")) + self.assertNotIn(safe["bundle_id"], review_native.PRODUCTION_BUNDLE_IDS) + + def test_secret_environment_is_scrubbed(self): + old = review_native.os.environ.get("BUZZ_PRIVATE_KEY") + review_native.os.environ["BUZZ_PRIVATE_KEY"] = "must-not-survive" + try: + self.assertNotIn("BUZZ_PRIVATE_KEY", review_native.scrubbed_environment()) + finally: + if old is None: + review_native.os.environ.pop("BUZZ_PRIVATE_KEY", None) + else: + review_native.os.environ["BUZZ_PRIVATE_KEY"] = old + + def test_visible_window_waits_for_reveal(self): + driver = mock.Mock() + driver.request.side_effect = [ + {"ok": True, "visible": False, "detail": "not yet"}, + {"ok": True, "visible": True, "window_id": 42}, + ] + process = mock.Mock() + process.poll.return_value = None + with mock.patch.object(review_native.time, "sleep"): + status = review_native.wait_for_visible_window(driver, process, timeout_seconds=1) + self.assertEqual(status["window_id"], 42) + self.assertEqual(driver.request.call_count, 2) + + def test_locate_required_retries_until_target_materializes(self): + driver = mock.Mock() + driver.request.side_effect = [ + {"ok": True, "element": None}, + {"ok": True, "element": {"locator": {"id": "target"}}}, + ] + with mock.patch.object(review_native.time, "sleep"): + found = review_native.locate_required(driver, [{"id": "target"}], 1000) + self.assertEqual(found["locator"]["id"], "target") + self.assertEqual(driver.request.call_count, 2) + + def test_visible_window_fails_if_app_exits(self): + driver = mock.Mock() + process = mock.Mock() + process.poll.return_value = 1 + with self.assertRaisesRegex(review_native.HarnessError, "exited"): + review_native.wait_for_visible_window(driver, process, timeout_seconds=1) + driver.request.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From cf6729f95b5a5492ebe6ab8398415ec9c1b2457e Mon Sep 17 00:00:00 2001 From: Wes Date: Sat, 15 Aug 2026 15:21:04 -0700 Subject: [PATCH 2/4] fix: isolate native review host credentials Fail closed on non-standard fixture ports, use fixed loopback database coordinates, scrub repository-controlled subprocess environments, and remove generated review keys when fixture seeding fails. Co-authored-by: Carl <5f365698229751c0461f57bb03a4e93134e6e936bd7039ebe7b737282a43c754@buzz.block.builderlab.xyz> Signed-off-by: Wes --- tools/native-review/review_native.py | 53 ++++++++++++------- .../native-review/tests/test_review_native.py | 51 ++++++++++++++---- 2 files changed, 77 insertions(+), 27 deletions(-) diff --git a/tools/native-review/review_native.py b/tools/native-review/review_native.py index 559446770a..dc2700e206 100755 --- a/tools/native-review/review_native.py +++ b/tools/native-review/review_native.py @@ -176,13 +176,32 @@ def provenance() -> dict[str, Any]: } -def scrubbed_environment() -> dict[str, str]: - keep = {"PATH", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", "TERM", "SSH_AUTH_SOCK", "__CF_USER_TEXT_ENCODING"} +def scrubbed_environment(*, include_home: bool = False) -> dict[str, str]: + keep = {"PATH", "TMPDIR", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", "TERM", "__CF_USER_TEXT_ENCODING"} env = {key: value for key, value in os.environ.items() if key in keep and not SECRET_NAME.search(key)} - env["HOME"] = "" # filled per run + env["HOME"] = os.environ.get("HOME", "") if include_home else "" # isolated per run unless tooling needs host caches return env +def fixture_environment(isolation: dict[str, str], review_pubkey: str) -> dict[str, str]: + """Return fixed local fixture coordinates without inheriting host credentials.""" + parsed = urllib.parse.urlparse(isolation["relay_url"]) + port = parsed.port or 80 + if port != 3030: + raise HarnessError("fixture seeding requires the isolated relay at loopback port 3030") + return { + **scrubbed_environment(include_home=True), + "BUZZ_REVIEW_PUBKEY": review_pubkey, + "BUZZ_COMMUNITY_HOST": f"{parsed.hostname}:{port}", + "BUZZ_DB_HOST": "localhost", + "BUZZ_DB_PORT": "5471", + "BUZZ_DB_USER": "buzz", + "BUZZ_DB_PASS": "buzz_dev", + "BUZZ_DB_NAME": "buzz", + "BUZZ_DB_DOCKER_CONTAINER": "buzz-harness-postgres-1", + } + + def driver_binary() -> pathlib.Path: override = os.environ.get("BUZZ_NATIVE_REVIEW_DRIVER") if override: @@ -196,7 +215,8 @@ def build_driver() -> pathlib.Path: if not binary.is_file(): raise HarnessError(f"configured driver does not exist: {binary}") return binary - run(["swift", "build", "-c", "release", "--package-path", str(TOOL_ROOT / "swift")], capture=False) + run(["swift", "build", "-c", "release", "--package-path", str(TOOL_ROOT / "swift")], + env=scrubbed_environment(include_home=True), capture=False) if not binary.is_file(): raise HarnessError(f"Swift build succeeded without driver binary: {binary}") return binary @@ -206,7 +226,8 @@ class Driver: def __init__(self, binary: pathlib.Path, pid: int, semantic_snapshot: pathlib.Path): self.process = subprocess.Popen([str(binary), "serve", "--pid", str(pid), "--semantic-snapshot", str(semantic_snapshot)], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=scrubbed_environment(include_home=True), text=True, bufsize=1) def request(self, command: str, **payload: Any) -> dict[str, Any]: assert self.process.stdin and self.process.stdout @@ -268,8 +289,8 @@ def doctor(require_permissions: bool = False) -> dict[str, Any]: def prepare_fixture(run_dir: pathlib.Path, isolation: dict[str, str]) -> dict[str, Any]: admin = ROOT / "target" / "debug" / "buzz-admin" if not admin.is_file(): - run(["cargo", "build", "-p", "buzz-admin"], capture=False) - generated = run([str(admin), "generate-key"]).stdout + run(["cargo", "build", "-p", "buzz-admin"], env=scrubbed_environment(include_home=True), capture=False) + generated = run([str(admin), "generate-key"], env=scrubbed_environment(include_home=True)).stdout secret_match = re.search(r"Secret key:\s+(\S+)", generated) public_match = re.search(r"Public key:\s+(\S+)", generated) if not secret_match or not public_match: @@ -283,15 +304,12 @@ def prepare_fixture(run_dir: pathlib.Path, isolation: dict[str, str]) -> dict[st "secret_path": str(secret_path), "relay_url": isolation["relay_url"], "seed": "scripts/setup-desktop-test-data.sh", "cleanup_scope": "run-local app state and keyring only", } - parsed = urllib.parse.urlparse(isolation["relay_url"]) - port = parsed.port or 80 - seed_env = {**os.environ, "BUZZ_REVIEW_PUBKEY": fixture["identity_pubkey"], - "BUZZ_COMMUNITY_HOST": f"{parsed.hostname}:{port}"} - if port == 3030: - seed_env.update({"BUZZ_DB_HOST": "localhost", "BUZZ_DB_PORT": "5471", "BUZZ_DB_USER": "buzz", - "BUZZ_DB_PASS": "buzz_dev", "BUZZ_DB_NAME": "buzz", - "BUZZ_DB_DOCKER_CONTAINER": "buzz-harness-postgres-1"}) - run([str(ROOT / "scripts/setup-desktop-test-data.sh")], env=seed_env, capture=False) + try: + run([str(ROOT / "scripts/setup-desktop-test-data.sh")], + env=fixture_environment(isolation, fixture["identity_pubkey"]), capture=False) + except Exception: + secret_path.unlink(missing_ok=True) + raise (run_dir / "manifest" / "fixture.json").write_text(json.dumps({k: v for k, v in fixture.items() if k != "secret_path"}, indent=2)) return fixture @@ -347,8 +365,7 @@ def build_and_launch(run_dir: pathlib.Path, isolation: dict[str, str], fixture: "identifier": isolation["bundle_id"], "productName": "Buzz Native Review", "bundle": {"externalBin": []}, }, separators=(",", ":")) - build_env = scrubbed_environment() - build_env["HOME"] = os.environ["HOME"] + build_env = scrubbed_environment(include_home=True) build_env["VITE_NATIVE_REVIEW"] = "1" build_env["VITE_NATIVE_REVIEW_RELAY"] = isolation["relay_url"] build_env["VITE_NATIVE_REVIEW_PUBKEY"] = fixture["identity_pubkey"] diff --git a/tools/native-review/tests/test_review_native.py b/tools/native-review/tests/test_review_native.py index 397cb22122..93a70a4440 100644 --- a/tools/native-review/tests/test_review_native.py +++ b/tools/native-review/tests/test_review_native.py @@ -50,15 +50,48 @@ def test_production_and_remote_targets_are_rejected(self): self.assertNotIn(safe["bundle_id"], review_native.PRODUCTION_BUNDLE_IDS) def test_secret_environment_is_scrubbed(self): - old = review_native.os.environ.get("BUZZ_PRIVATE_KEY") - review_native.os.environ["BUZZ_PRIVATE_KEY"] = "must-not-survive" - try: - self.assertNotIn("BUZZ_PRIVATE_KEY", review_native.scrubbed_environment()) - finally: - if old is None: - review_native.os.environ.pop("BUZZ_PRIVATE_KEY", None) - else: - review_native.os.environ["BUZZ_PRIVATE_KEY"] = old + sentinels = { + "BUZZ_PRIVATE_KEY": "must-not-survive", + "SSH_AUTH_SOCK": "/tmp/credential-agent", + "GITHUB_TOKEN": "github-secret", + "AWS_SECRET_ACCESS_KEY": "cloud-secret", + } + with mock.patch.dict(review_native.os.environ, sentinels, clear=False): + environment = review_native.scrubbed_environment() + for name in sentinels: + self.assertNotIn(name, environment) + + def test_fixture_environment_is_fixed_local_and_scrubbed(self): + sentinels = {"BUZZ_DB_HOST": "production-db", "BUZZ_DB_PASS": "production-secret", + "SSH_AUTH_SOCK": "/tmp/credential-agent"} + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.dict(review_native.os.environ, sentinels, clear=False): + environment = review_native.fixture_environment(isolation, "a" * 64) + self.assertEqual(environment["BUZZ_DB_HOST"], "localhost") + self.assertEqual(environment["BUZZ_DB_PORT"], "5471") + self.assertEqual(environment["BUZZ_DB_PASS"], "buzz_dev") + self.assertNotIn("SSH_AUTH_SOCK", environment) + with self.assertRaisesRegex(review_native.HarnessError, "port 3030"): + review_native.fixture_environment( + review_native.isolation_manifest("run", "ws://127.0.0.1:3001"), "a" * 64) + + def test_fixture_seed_failure_removes_generated_identity(self): + generated = mock.Mock(stdout=f"Secret key: {'1' * 64}\nPublic key: {'2' * 64}\n") + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + (run_dir / "manifest").mkdir() + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.object(review_native, "run", side_effect=[generated, RuntimeError("seed failed")]): + with self.assertRaisesRegex(RuntimeError, "seed failed"): + review_native.prepare_fixture(run_dir, isolation) + self.assertFalse((run_dir / "state/identity.key").exists()) + + def test_repository_controlled_subprocesses_receive_scrubbed_environments(self): + safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} + with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ + mock.patch.object(review_native.subprocess, "Popen") as popen: + review_native.Driver(pathlib.Path("/tmp/driver"), 42, pathlib.Path("/tmp/snapshot")) + self.assertEqual(popen.call_args.kwargs["env"], safe) def test_visible_window_waits_for_reveal(self): driver = mock.Mock() From ba6b17e158c57a1b71720ea925c7536ceb523a72 Mon Sep 17 00:00:00 2001 From: Wes Date: Sat, 15 Aug 2026 15:35:04 -0700 Subject: [PATCH 3/4] fix: scrub remaining review subprocesses Default harness commands to a credential-free environment and isolate cleanup HOME without reconstructing the host environment. Co-authored-by: Carl <5f365698229751c0461f57bb03a4e93134e6e936bd7039ebe7b737282a43c754@buzz.block.builderlab.xyz> Signed-off-by: Wes --- tools/native-review/review_native.py | 19 ++++++++++++----- .../native-review/tests/test_review_native.py | 21 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tools/native-review/review_native.py b/tools/native-review/review_native.py index dc2700e206..16c44ffc45 100755 --- a/tools/native-review/review_native.py +++ b/tools/native-review/review_native.py @@ -42,7 +42,9 @@ class HarnessError(RuntimeError): def run(command: list[str], *, cwd: pathlib.Path = ROOT, env: dict[str, str] | None = None, check: bool = True, capture: bool = True) -> subprocess.CompletedProcess[str]: - return subprocess.run(command, cwd=cwd, env=env, check=check, text=True, + return subprocess.run(command, cwd=cwd, + env=scrubbed_environment(include_home=True) if env is None else env, + check=check, text=True, stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE if capture else None) @@ -465,6 +467,16 @@ def capture_step(driver: Driver, run_dir: pathlib.Path, slug: str, record: dict[ return artifacts +def cleanup_review_state(run_dir: pathlib.Path, isolation: dict[str, str], + fixture: dict[str, Any] | None) -> None: + env = scrubbed_environment() + env["HOME"] = str(run_dir / "home") + run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), + isolation["bundle_id"], isolation["keyring_service"]], env=env) + if fixture: + pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + + def run_journey(path: pathlib.Path, relay_url: str, output_root: pathlib.Path) -> pathlib.Path: journey = load_journey(path) run_id = f"{journey['flow']}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{secrets.token_hex(3)}" @@ -553,10 +565,7 @@ def run_journey(path: pathlib.Path, relay_url: str, output_root: pathlib.Path) - process.kill(); cleanup_errors.append("Tauri launcher required SIGKILL") if journey["cleanup"]["remove_state"]: try: - run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), isolation["bundle_id"], isolation["keyring_service"]], - env={**os.environ, "HOME": str(run_dir / "home")}) - if fixture: - pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + cleanup_review_state(run_dir, isolation, fixture) except Exception as exc: cleanup_errors.append(str(exc)) receipt["cleanup"] = {"status": "failed" if cleanup_errors else "passed", "errors": cleanup_errors} diff --git a/tools/native-review/tests/test_review_native.py b/tools/native-review/tests/test_review_native.py index 93a70a4440..c99ee63a22 100644 --- a/tools/native-review/tests/test_review_native.py +++ b/tools/native-review/tests/test_review_native.py @@ -86,6 +86,27 @@ def test_fixture_seed_failure_removes_generated_identity(self): review_native.prepare_fixture(run_dir, isolation) self.assertFalse((run_dir / "state/identity.key").exists()) + def test_run_scrubs_repository_controlled_commands_by_default(self): + safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} + with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ + mock.patch.object(review_native.subprocess, "run", return_value=mock.Mock()) as subprocess_run: + review_native.run(["/tmp/repository-tool"]) + self.assertEqual(subprocess_run.call_args.kwargs["env"], safe) + + def test_cleanup_uses_isolated_home_without_host_credentials(self): + sentinels = {"BUZZ_PRIVATE_KEY": "secret", "SSH_AUTH_SOCK": "/tmp/agent", + "GITHUB_TOKEN": "secret", "AWS_SECRET_ACCESS_KEY": "secret"} + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + with mock.patch.dict(review_native.os.environ, sentinels, clear=False), \ + mock.patch.object(review_native, "run") as run: + review_native.cleanup_review_state(run_dir, isolation, None) + environment = run.call_args.kwargs["env"] + self.assertEqual(environment["HOME"], str(run_dir / "home")) + for name in sentinels: + self.assertNotIn(name, environment) + def test_repository_controlled_subprocesses_receive_scrubbed_environments(self): safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \ From 36356873c94b47d381a07900bf47484500b243a5 Mon Sep 17 00:00:00 2001 From: Carl <5f365698229751c0461f57bb03a4e93134e6e936bd7039ebe7b737282a43c754@buzz.block.builderlab.xyz> Date: Sat, 15 Aug 2026 16:51:14 -0700 Subject: [PATCH 4/4] fix: preserve identity cleanup after reset failure Signed-off-by: Carl <5f365698229751c0461f57bb03a4e93134e6e936bd7039ebe7b737282a43c754@buzz.block.builderlab.xyz> --- tools/native-review/review_native.py | 15 ++++++++-- .../native-review/tests/test_review_native.py | 29 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tools/native-review/review_native.py b/tools/native-review/review_native.py index 16c44ffc45..691d326daa 100755 --- a/tools/native-review/review_native.py +++ b/tools/native-review/review_native.py @@ -469,12 +469,21 @@ def capture_step(driver: Driver, run_dir: pathlib.Path, slug: str, record: dict[ def cleanup_review_state(run_dir: pathlib.Path, isolation: dict[str, str], fixture: dict[str, Any] | None) -> None: + errors = [] env = scrubbed_environment() env["HOME"] = str(run_dir / "home") - run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), - isolation["bundle_id"], isolation["keyring_service"]], env=env) + try: + run([str(ROOT / "scripts/reset-desktop-standalone-state.sh"), + isolation["bundle_id"], isolation["keyring_service"]], env=env) + except Exception as exc: + errors.append(f"desktop state reset failed: {exc}") if fixture: - pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + try: + pathlib.Path(fixture["secret_path"]).unlink(missing_ok=True) + except Exception as exc: + errors.append(f"review identity removal failed: {exc}") + if errors: + raise HarnessError("; ".join(errors)) def run_journey(path: pathlib.Path, relay_url: str, output_root: pathlib.Path) -> pathlib.Path: diff --git a/tools/native-review/tests/test_review_native.py b/tools/native-review/tests/test_review_native.py index c99ee63a22..59f440d631 100644 --- a/tools/native-review/tests/test_review_native.py +++ b/tools/native-review/tests/test_review_native.py @@ -107,6 +107,35 @@ def test_cleanup_uses_isolated_home_without_host_credentials(self): for name in sentinels: self.assertNotIn(name, environment) + def test_cleanup_removes_identity_when_state_reset_fails(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + secret_path = run_dir / "state/identity.key" + secret_path.parent.mkdir() + secret_path.write_text("review-secret") + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + fixture = {"secret_path": str(secret_path)} + with mock.patch.object(review_native, "run", side_effect=RuntimeError("reset failed")): + with self.assertRaisesRegex( + review_native.HarnessError, + "desktop state reset failed: reset failed"): + review_native.cleanup_review_state(run_dir, isolation, fixture) + self.assertFalse(secret_path.exists()) + + def test_cleanup_aggregates_reset_and_identity_removal_failures(self): + with tempfile.TemporaryDirectory() as directory: + run_dir = pathlib.Path(directory) + secret_path = run_dir / "state/identity.key" + isolation = review_native.isolation_manifest("run", "ws://127.0.0.1:3030") + fixture = {"secret_path": str(secret_path)} + with mock.patch.object(review_native, "run", side_effect=RuntimeError("reset failed")), \ + mock.patch.object(pathlib.Path, "unlink", side_effect=OSError("unlink failed")): + with self.assertRaisesRegex( + review_native.HarnessError, + "desktop state reset failed: reset failed; " + "review identity removal failed: unlink failed"): + review_native.cleanup_review_state(run_dir, isolation, fixture) + def test_repository_controlled_subprocesses_receive_scrubbed_environments(self): safe = {"PATH": "/usr/bin", "HOME": "/tmp/home"} with mock.patch.object(review_native, "scrubbed_environment", return_value=safe), \