From 9c33eef654ef82366e9fab4da646bc9d433fe9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 8 Jul 2026 09:30:56 +0200 Subject: [PATCH 01/15] agent: add script healing capability Adds a `--heal` CLI flag and interactive REPL prompt to automatically diagnose and repair failing scripts using the LLM. The agent attempts to fix the script, then validates the revision in a fresh session before overwriting the original file. --- build.zig.zon | 4 +- src/Config.zig | 1 + src/agent/Agent.zig | 284 ++++++++++++++++++++++++++++++++++------ src/help.zon | 7 + src/script/Recorder.zig | 52 ++++++++ 5 files changed, 309 insertions(+), 39 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 92f960358a..028b5d257e 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -35,8 +35,8 @@ .hash = "sqlite3-3.51.0-DMxLWssOAABZ8cAvU_LfBIbp0kZjm824PU8sSLXpEDdr", }, .zenai = .{ - .url = "git+https://github.com/lightpanda-io/zenai.git#e58635146733157eb4936388b21a071fcabfa687", - .hash = "zenai-0.0.0-iOY_VIf0BADH7PvSOlnVf564krXnAlns_1IIhh9ZDdwz", + .url = "git+https://github.com/lightpanda-io/zenai.git#9ac20be6c5d46580b8571b32d19ca5d136eae29c", + .hash = "zenai-0.0.0-iOY_VKL5BADSmxXtp3vVa6bAYrwEogLO936_5SKslkiD", }, .isocline = .{ .url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47", diff --git a/src/Config.zig b/src/Config.zig index c5c574f8b9..8f6789c125 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -254,6 +254,7 @@ const Commands = cli.Builder(.{ .{ .name = "system_prompt", .type = ?[:0]const u8 }, .{ .name = "task", .type = ?[]const u8 }, .{ .name = "save", .type = ?[]const u8 }, + .{ .name = "heal", .type = bool }, .{ .name = "attach", .short = 'a', .type = []const u8, .multiple = true }, .{ .name = "verbosity", .type = ?AgentVerbosity }, .{ .name = "effort", .type = ?Effort }, diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 207d846994..c9d0d08be5 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -249,6 +249,7 @@ model: []u8, /// Per-turn reasoning budget for LLM turns. Mutable at runtime via `/effort`. effort: Config.Effort, script_file: ?[]const u8, +heal: bool, one_shot_task: ?[]const u8, one_shot_save: ?[]const u8, one_shot_attachments: ?[]const []const u8, @@ -305,6 +306,20 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent return error.InvalidFilename; } } + if (opts.heal) { + if (opts.script_file == null) { + log.fatal(.app, "conflicting flags", .{ + .hint = "--heal repairs a failing script run; pass a script positional", + }); + return error.ConflictingFlags; + } + if (opts.no_llm) { + log.fatal(.app, "conflicting flags", .{ + .hint = "--heal needs an LLM to repair the script; drop --no-llm", + }); + return error.ConflictingFlags; + } + } if (opts.no_llm and opts.provider != null) { log.warn(.app, "ignoring --provider", .{ .reason = "--no-llm takes precedence" }); } @@ -314,10 +329,12 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent const is_one_shot = opts.task != null; const will_repl = !is_one_shot and opts.script_file == null; + // One-shot --task and --heal both need a model before any interaction. + const needs_llm_upfront = is_one_shot or opts.heal; // Load remembered selection up front so a saved null provider can flip the // REPL into basic mode before resolution. Pure script runs need nothing. - const remembered: ?settings.Remembered = if (will_repl or is_one_shot) settings.loadRemembered(allocator) else null; + const remembered: ?settings.Remembered = if (will_repl or needs_llm_upfront) settings.loadRemembered(allocator) else null; defer if (remembered) |r| std.zon.parse.free(allocator, r); // A remembered null provider means the user disabled the LLM via @@ -330,7 +347,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent // null provider. Without it the REPL accepts natural language, so an absent // API key would only surface at the first non-slash-command line — too late. // Pure JavaScript script runs stay allowed: no REPL, no LLM. - const requires_llm = is_one_shot or (will_repl and !opts.no_llm and !remembered_no_llm); + const requires_llm = needs_llm_upfront or (will_repl and !opts.no_llm and !remembered_no_llm); // Skip resolve when no client is wanted — else resolveCredentials prints // "No API key detected" for a run that does not need one. @@ -415,6 +432,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent .model = model, .effort = effort, .script_file = opts.script_file, + .heal = opts.heal, .one_shot_task = opts.task, .one_shot_save = opts.save, .one_shot_attachments = if (opts.attach.items.len == 0) null else opts.attach.items, @@ -480,6 +498,8 @@ fn startSession(self: *Agent) !void { self.session = try self.browser.newSession(self.notification); self.session.cancel_hook = .{ .context = @ptrCast(self), .check = checkCancel }; try self.session.enableConsoleCapture(); + // Node IDs are session-scoped; drop them with the session they point into. + self.node_registry.reset(); } // Compile-time constant; projected once per process to avoid rebuilding per call. @@ -587,6 +607,7 @@ pub fn run(self: *Agent) bool { return ok; } if (self.script_file) |path| { + if (self.heal) return self.runScriptWithHeal(path); return self.runScript(path); } self.runRepl(); @@ -834,7 +855,30 @@ fn handleLoad(self: *Agent, rest: []const u8) void { self.terminal.printError("usage: /load ", .{}); return; } - _ = self.runScript(path); + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok, .fatal => {}, + .script_error => |script_error| { + if (self.ai_client == null) return; + if (!promptHeal(path)) return; + _ = self.healLoop(arena.allocator(), path, script_error); + }, + } +} + +/// Defaults to no: healing spends tokens and its validation step resets the +/// browser session. +fn promptHeal(path: []const u8) bool { + var header_buf: [256]u8 = undefined; + const header = std.fmt.bufPrint(&header_buf, "{s} failed. Heal it with the model?", .{path}) catch + "Script failed. Heal it with the model?"; + const labels: []const [:0]const u8 = &.{ + "heal — diagnose and fix, then validate in a fresh session (drops page/cookies like /reset)", + "no — leave the script and session as they are", + }; + const idx = Terminal.promptNumberedChoice(header, labels, 1) catch return false; + return idx == 0; } const api_keys_hint = settings.api_keys_hint; @@ -1117,15 +1161,17 @@ fn promptSaveMode(self: *Agent, path: []const u8) ?save.Mode { return modes[idx]; } -fn failSave(self: *Agent, reason: []const u8) void { - self.terminal.printError("save failed: {s}", .{reason}); +fn failSynthesis(self: *Agent, label: []const u8, reason: []const u8) ?[]const u8 { + self.terminal.printError("{s} failed: {s}", .{ label, reason }); + return null; } -/// Roll the in-flight save turn back out of the conversation, then report the -/// failure — so a doomed `/save` synthesis never leaks its messages into history. -fn abortSave(self: *Agent, baseline: usize, reason: []const u8) void { +/// Roll the in-flight synthesis turn back out of the conversation, then report +/// the failure — so a doomed `/save` or heal synthesis never leaks its messages +/// into history. +fn abortSynthesis(self: *Agent, label: []const u8, baseline: usize, reason: []const u8) ?[]const u8 { self.conversation.rollback(baseline); - self.failSave(reason); + return self.failSynthesis(label, reason); } /// Save synthesis warrants more reasoning than a normal turn. `.none` stays off @@ -1171,8 +1217,6 @@ fn saveOneShot(self: *Agent) void { /// LLM synthesis + write for an already-resolved destination. Shared by the /// interactive `/save` and one-shot `--save`. fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mode: save.Mode, prompt: ?[]const u8) void { - const provider_client = self.ai_client.?; - // Only update feeds the saved script back to the model; append stays // blind — the script is synthesized from this session alone and written // after the existing content. @@ -1184,7 +1228,27 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo else null; - self.conversation.ensureSystemPrompt() catch return self.failSave("out of memory"); + const script = self.synthesizeScriptText(arena, "save", path, previous_script, prompt) orelse return; + + save.writeContentFile(path, script, mode) catch |err| { + self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) }); + return; + }; + + self.rememberSavePath(path); + self.save_buffer.reset(); + self.terminal.printInfo("Saved synthesized script to {s}", .{path}); +} + +/// The synthesis turn shared by `/save` and `--heal`: hand the model the +/// conversation, the deterministic record of what ran, and the previous script +/// when revising, then return the scrubbed script text (arena-owned). Reports +/// the failure under `label` and returns null on any error or cancellation; +/// the synthesis turn never stays in history. +fn synthesizeScriptText(self: *Agent, arena: std.mem.Allocator, label: []const u8, path: []const u8, previous_script: ?[]const u8, prompt: ?[]const u8) ?[]const u8 { + const provider_client = self.ai_client.?; + + self.conversation.ensureSystemPrompt() catch return self.failSynthesis(label, "out of memory"); // Swap the dedicated save_system_prompt in as the system prompt for this one turn; // regular turns keep the driver prompt. (`messages[0]` is the system @@ -1196,8 +1260,8 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo const ma = self.conversation.arena.allocator(); const baseline = self.conversation.messages.items.len; - const user_msg = self.buildSaveSynthesisMessage(ma, path, previous_script, prompt) catch return self.failSave("out of memory"); - self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSave("out of memory"); + const user_msg = self.buildSaveSynthesisMessage(ma, path, previous_script, prompt) catch return self.failSynthesis(label, "out of memory"); + self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); self.http_interrupt.reset(); self.terminal.spinner.start(); @@ -1219,10 +1283,10 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo self.terminal.spinner.cancel(); if (self.cancel_requested.load(.acquire)) { self.resetAfterCancel(baseline); - return; + return null; } - log.err(.app, "AI save synthesis error", .{ .err = err }); - return self.abortSave(baseline, @errorName(err)); + log.err(.app, "AI synthesis error", .{ .label = label, .err = err }); + return self.abortSynthesis(label, baseline, @errorName(err)); }; self.terminal.spinner.stop(); defer result.deinit(); @@ -1230,28 +1294,20 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo if (result.cancelled) { self.resetAfterCancel(baseline); - return; + return null; } - const raw = result.text orelse return self.abortSave(baseline, "the model returned no script"); + const raw = result.text orelse return self.abortSynthesis(label, baseline, "the model returned no script"); // `result.text` lives in the conversation arena, freed by the rollback - // below; copy into the command arena first (scrubbing may return its input + // below; copy into the caller's arena first (scrubbing may return its input // as-is). - const owned = arena.dupe(u8, save.stripCodeFence(raw)) catch return self.abortSave(baseline, "out of memory"); - const script = browser_tools.reverseSubstituteEnvVars(arena, owned) catch return self.abortSave(baseline, "out of memory"); + const owned = arena.dupe(u8, save.stripCodeFence(raw)) catch return self.abortSynthesis(label, baseline, "out of memory"); + const script = browser_tools.reverseSubstituteEnvVars(arena, owned) catch return self.abortSynthesis(label, baseline, "out of memory"); - // The save turn is a meta-action; keep it out of the ongoing conversation. + // The synthesis turn is a meta-action; keep it out of the ongoing conversation. self.conversation.rollback(baseline); - - save.writeContentFile(path, script, mode) catch |err| { - self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) }); - return; - }; - - self.rememberSavePath(path); - self.save_buffer.reset(); - self.terminal.printInfo("Saved synthesized script to {s}", .{path}); + return script; } /// Persist `path` as the destination reused by a subsequent bare `/save`. @@ -1452,18 +1508,41 @@ const ScriptOutput = struct { } }; +/// `fatal` covers setup failures (unreadable file, runtime init, OOM) that a +/// retry can't help. +const ScriptRunOutcome = union(enum) { + ok, + fatal, + script_error: ScriptError, +}; + +/// Both slices are duped into the caller's arena. `source` is the exact text +/// that ran, so a heal diagnoses what actually failed instead of re-reading a +/// possibly-changed file. +const ScriptError = struct { + /// Formatted error (line, stack). + detail: []const u8, + source: []const u8, +}; + fn runScript(self: *Agent, path: []const u8) bool { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + return self.runScriptOutcome(arena.allocator(), path) == .ok; +} + +fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) ScriptRunOutcome { var script_arena: std.heap.ArenaAllocator = .init(self.allocator); defer script_arena.deinit(); const content = std.fs.cwd().readFileAlloc(script_arena.allocator(), path, 10 * 1024 * 1024) catch |err| { self.terminal.printError("Failed to read script '{s}': {s}", .{ path, @errorName(err) }); - return false; + return .fatal; }; const runtime = ScriptRuntime.init(self.allocator, self.browser.app, self.session, &self.node_registry) catch |err| { self.terminal.printError("Failed to initialize script runtime: {s}", .{@errorName(err)}); - return false; + return .fatal; }; defer runtime.deinit(); self.script_runtime_mutex.lock(); @@ -1486,16 +1565,147 @@ fn runScript(self: *Agent, path: []const u8) bool { if (result catch |err| { self.terminal.printError("Script failed: {s}", .{@errorName(err)}); - return false; + return .fatal; }) |message| { self.terminal.printError("{s}", .{message}); - return false; + // A Ctrl-C termination is not a script defect — never heal it. + if (self.cancel_requested.load(.acquire)) return .fatal; + // The message lives in the runtime's call arena and the source in + // script_arena; both are freed by the defers above — dupe first. + return .{ .script_error = .{ + .detail = arena.dupe(u8, message) catch return .fatal, + .source = arena.dupe(u8, content) catch return .fatal, + } }; } // A script that printed nothing leaves no trace, so freeze the spinner into // a green bullet (like /goto); one that printed already showed its result. if (!output.emitted) self.terminal.printScriptDone("script", path); - return true; + return .ok; +} + +/// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. +fn runScriptWithHeal(self: *Agent, path: []const u8) bool { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + + switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok => return true, + .fatal => return false, + .script_error => |script_error| { + // Healing spends tokens; report the cost the way --task does. + defer self.printUsageSummary(); + return self.healLoop(arena.allocator(), path, script_error); + }, + } +} + +const max_heal_attempts = 2; + +fn buildHealDiagnoseMessage(arena: std.mem.Allocator, path: []const u8, source: []const u8, error_detail: []const u8) ![]const u8 { + return std.fmt.allocPrint(arena, + \\Replaying the saved script {s} failed. The browser session is still + \\at the failure state. + \\ + \\The script (its comments and structure carry the intent): + \\```js + \\{s} + \\``` + \\ + \\The error: + \\{s} + \\ + \\Diagnose the failure: inspect the live page (tree, findElement, + \\markdown) to see how the site differs from what the script expects, + \\then perform the corrected step(s) with tools to prove they work — + \\verify selectors against the live page, never guess. If the failing + \\step gated the rest of the script (a login, a navigation), carry on + \\far enough to show the script's goal is reachable again. + , .{ path, source, error_detail }); +} + +/// Heal synthesis instruction; rides on the regular save revision system prompt. +const heal_revision_prompt = + \\Fix the script so it replays successfully against the current site: the + \\error and this session's working tool calls identify the repair. Keep + \\every step, selector, and output shape that still works unchanged. +; + +/// Only a revision that passed validation in a fresh session replaces `path`; +/// the original survives a failed heal. +fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: ScriptError) bool { + var source = first.source; + var error_detail = first.detail; + + const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch { + self.terminal.printError("heal failed: out of memory", .{}); + return false; + }; + + // The recorder is /save's stream: heal must neither synthesize from the + // REPL's prior recordings nor leak its diagnose actions into a later /save. + const recorded = self.save_buffer.snapshot(arena) catch { + self.terminal.printError("heal failed: out of memory", .{}); + return false; + }; + self.save_buffer.reset(); + defer self.save_buffer.restore(recorded) catch { + self.terminal.printWarning("commands recorded before the heal were lost (out of memory)", .{}); + }; + + var attempt: usize = 1; + while (attempt <= max_heal_attempts) : (attempt += 1) { + self.terminal.printInfo("Healing {s} (attempt {d}/{d})", .{ path, attempt, max_heal_attempts }); + + const diagnose = buildHealDiagnoseMessage(arena, path, source, error_detail) catch { + self.terminal.printError("heal failed: out of memory", .{}); + return false; + }; + if (!self.runTurn(.{ .prompt = diagnose, .capture_for_save = true, .label = "Heal" })) return false; + + const revised = self.synthesizeScriptText(arena, "heal", path, source, heal_revision_prompt) orelse return false; + + save.writeContentFile(tmp_path, revised, .replace) catch |err| { + self.terminal.printError("heal failed: could not write {s}: {s}", .{ tmp_path, @errorName(err) }); + return false; + }; + + // Validate in a fresh session so failure-state cookies and pages can't + // mask a still-broken script. + self.startSession() catch |err| { + self.removeTempScript(tmp_path); + self.terminal.printError("heal failed: could not start a fresh session: {s}", .{@errorName(err)}); + return false; + }; + + switch (self.runScriptOutcome(arena, tmp_path)) { + .ok => { + std.fs.cwd().rename(tmp_path, path) catch |err| { + self.terminal.printError("healed script validated but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }); + return false; + }; + self.terminal.printInfo("Healed {s}: the revised script validated in a fresh session.", .{path}); + return true; + }, + .fatal => { + self.removeTempScript(tmp_path); + return false; + }, + .script_error => |script_error| { + source = revised; + error_detail = script_error.detail; + }, + } + } + self.removeTempScript(tmp_path); + self.terminal.printError("heal gave up after {d} attempts; {s} is unchanged", .{ max_heal_attempts, path }); + return false; +} + +fn removeTempScript(self: *Agent, tmp_path: []const u8) void { + std.fs.cwd().deleteFile(tmp_path) catch |err| { + self.terminal.printWarning("could not remove temp script {s}: {s}", .{ tmp_path, @errorName(err) }); + }; } /// Mirror a user-typed slash command into `self.conversation.messages` as if the diff --git a/src/help.zon b/src/help.zon index 2be7e812f3..e44cdb3fab 100644 --- a/src/help.zon +++ b/src/help.zon @@ -144,6 +144,7 @@ \\ {0s} agent --provider ollama --model qwen3.5:latest \\ {0s} agent --no-llm (basic slash-command-only REPL) \\ {0s} agent script.js (run a saved script, then exit) + \\ {0s} agent script.js --heal (run it; LLM repairs it on failure) \\ {0s} agent --task "..." --save out.js (synthesize a replayable script) \\ \\Arguments: @@ -176,6 +177,12 @@ \\ default (Mistral defaults to none, as its default model rejects \\ effort). In the REPL, use /effort to change it. \\ Allowed values: none, minimal, low, medium, high, xhigh. + \\ --heal + \\ When the positional script fails, diagnose the failure with the + \\ LLM against the live page, revise the script, and validate the + \\ revision by re-running it in a fresh session. The script file is + \\ overwritten only when the revised version passes. Requires the + \\ script positional and an LLM provider. \\ --list-models \\ Print the model IDs usable with `agent` for --provider, one per \\ line, sorted, and exit. Auto-detects the provider from env when diff --git a/src/script/Recorder.zig b/src/script/Recorder.zig index 2ab8b381bc..5532373055 100644 --- a/src/script/Recorder.zig +++ b/src/script/Recorder.zig @@ -68,6 +68,29 @@ pub fn reset(self: *Recorder) void { _ = self.arena.reset(.retain_capacity); } +pub const Snapshot = struct { + content: []const u8, + lines: u32, + page_declared: bool, +}; + +/// Copy the current recording so a scoped consumer (the heal loop) can reset +/// the recorder for its own actions and hand the original back afterwards. +pub fn snapshot(self: *Recorder, allocator: std.mem.Allocator) !Snapshot { + return .{ + .content = try allocator.dupe(u8, self.bytes()), + .lines = self.lines, + .page_declared = self.page_declared, + }; +} + +pub fn restore(self: *Recorder, snap: Snapshot) !void { + self.reset(); + try self.content.writer.writeAll(snap.content); + self.lines = snap.lines; + self.page_declared = snap.page_declared; +} + pub fn record(self: *Recorder, cmd: Command) !void { if (!cmd.isRecorded()) return; self.buf.clearRetainingCapacity(); @@ -157,6 +180,35 @@ test "record filters state-mutating commands and comments" { try std.testing.expectEqual(@as(u32, 2), recorder.lines); } +test "snapshot/restore scopes recordings" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + var recorder: Recorder = .init(std.testing.allocator); + defer recorder.deinit(); + + try recorder.record(parseLine(aa, "/goto https://example.com")); + const snap = try recorder.snapshot(aa); + + recorder.reset(); + try recorder.record(parseLine(aa, "/scroll y=200")); + + try recorder.restore(snap); + try std.testing.expectEqualStrings( + "const page = new Page();\nawait page.goto(\"https://example.com\");\n", + recorder.bytes(), + ); + try std.testing.expectEqual(@as(u32, 2), recorder.lines); + + // page_declared came back with the snapshot: no second declaration. + try recorder.record(parseLine(aa, "/scroll y=200")); + try std.testing.expectEqualStrings( + "const page = new Page();\nawait page.goto(\"https://example.com\");\npage.scroll({ y: 200 });\n", + recorder.bytes(), + ); +} + test "recordRaw writes the JS line verbatim" { var recorder: Recorder = .init(std.testing.allocator); defer recorder.deinit(); From 19a722a28a32bb4d3c9231251a2166fd50278d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 8 Jul 2026 11:01:23 +0200 Subject: [PATCH 02/15] agent: heal scripts that return empty data Detects when a script runs successfully but returns no data (e.g., due to a stale selector). Treats this as a healable error and prompts the user to heal it with the model. --- src/agent/Agent.zig | 60 +++++++++++----- src/script/Runtime.zig | 160 ++++++++++++++++++++++++++++++++--------- 2 files changed, 169 insertions(+), 51 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index c9d0d08be5..228f4f3c28 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -861,7 +861,7 @@ fn handleLoad(self: *Agent, rest: []const u8) void { .ok, .fatal => {}, .script_error => |script_error| { if (self.ai_client == null) return; - if (!promptHeal(path)) return; + if (!promptHeal(path, script_error.kind)) return; _ = self.healLoop(arena.allocator(), path, script_error); }, } @@ -869,9 +869,13 @@ fn handleLoad(self: *Agent, rest: []const u8) void { /// Defaults to no: healing spends tokens and its validation step resets the /// browser session. -fn promptHeal(path: []const u8) bool { +fn promptHeal(path: []const u8, kind: ScriptError.Kind) bool { + const symptom = switch (kind) { + .threw => "failed", + .empty => "ran but returned no data", + }; var header_buf: [256]u8 = undefined; - const header = std.fmt.bufPrint(&header_buf, "{s} failed. Heal it with the model?", .{path}) catch + const header = std.fmt.bufPrint(&header_buf, "{s} {s}. Heal it with the model?", .{ path, symptom }) catch "Script failed. Heal it with the model?"; const labels: []const [:0]const u8 = &.{ "heal — diagnose and fix, then validate in a fresh session (drops page/cookies like /reset)", @@ -1520,15 +1524,26 @@ const ScriptRunOutcome = union(enum) { /// that ran, so a heal diagnoses what actually failed instead of re-reading a /// possibly-changed file. const ScriptError = struct { - /// Formatted error (line, stack). + kind: Kind, + /// Formatted error (line, stack) — or, for `empty`, what came back. detail: []const u8, source: []const u8, + + /// `empty` is a run that completed but returned a value with no data in + /// it — the usual symptom of a stale selector, which matches nothing + /// instead of throwing. Only heal treats it as a failure; a plain replay + /// still exits 0, since an empty answer can be the right answer. + const Kind = enum { threw, empty }; }; fn runScript(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); - return self.runScriptOutcome(arena.allocator(), path) == .ok; + return switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok => true, + .fatal => false, + .script_error => |script_error| script_error.kind == .empty, + }; } fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) ScriptRunOutcome { @@ -1563,19 +1578,32 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc const result = runtime.runSource(content, path); self.terminal.endTool(); - if (result catch |err| { + switch (result catch |err| { self.terminal.printError("Script failed: {s}", .{@errorName(err)}); return .fatal; - }) |message| { - self.terminal.printError("{s}", .{message}); - // A Ctrl-C termination is not a script defect — never heal it. - if (self.cancel_requested.load(.acquire)) return .fatal; - // The message lives in the runtime's call arena and the source in - // script_arena; both are freed by the defers above — dupe first. - return .{ .script_error = .{ - .detail = arena.dupe(u8, message) catch return .fatal, - .source = arena.dupe(u8, content) catch return .fatal, - } }; + }) { + .err => |message| { + self.terminal.printError("{s}", .{message}); + // A Ctrl-C termination is not a script defect — never heal it. + if (self.cancel_requested.load(.acquire)) return .fatal; + // The message lives in the runtime's call arena and the source in + // script_arena; both are freed by the defers above — dupe first. + return .{ .script_error = .{ + .kind = .threw, + .detail = arena.dupe(u8, message) catch return .fatal, + .source = arena.dupe(u8, content) catch return .fatal, + } }; + }, + .ok => |maybe_completion| { + if (maybe_completion) |completion| { + if (completion.empty) return .{ .script_error = .{ + .kind = .empty, + .detail = std.fmt.allocPrint(arena, "The script completed without throwing, but its return value carries no data: {s}\n" ++ + "A selector probably no longer matches anything on the page.", .{completion.text}) catch return .fatal, + .source = arena.dupe(u8, content) catch return .fatal, + } }; + } + }, } // A script that printed nothing leaves no trace, so freeze the spinner into diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index b29d3a308e..c6dc82bfa2 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -262,19 +262,35 @@ fn setObjectProperty( if (!out.has_value or !out.value) return error.RuntimeInitFailed; } -/// Run script source in the agent context. Returns null on success; on a JS -/// compile/runtime exception returns a formatted error allocated in this -/// runtime's call arena and valid until deinit or the next run. -pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!?[]const u8 { +/// The value a script ran to completion with. +pub const Completion = struct { + /// Display form of the returned value: objects and arrays as JSON. + text: []const u8, + /// True when the value carries no data — null, "", or an array/object all + /// of whose members are empty. + empty: bool, +}; + +/// A script run's outcome. `err` is a formatted JS compile/runtime exception; +/// `ok` carries the value the script returned, null when it returned +/// `undefined`. All slices are allocated in this runtime's call arena and +/// valid until deinit or the next run. +pub const RunResult = union(enum) { + ok: ?Completion, + err: []const u8, +}; + +/// Run script source in the agent context. +pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!RunResult { _ = self.call_arena.reset(.retain_capacity); - self.run_timer = std.time.Timer.start() catch return try self.dupeError("internal: timer unavailable"); + self.run_timer = std.time.Timer.start() catch return .{ .err = try self.dupeError("internal: timer unavailable") }; var hs: lp.js.HandleScope = undefined; hs.init(self.env.isolate); defer hs.deinit(); const context: *const v8.Context = @ptrCast(v8.v8__Global__Get(&self.context, self.env.isolate.handle) orelse - return try self.dupeError("agent script context is not available")); + return .{ .err = try self.dupeError("agent script context is not available") }); v8.v8__Context__Enter(context); defer v8.v8__Context__Exit(context); @@ -289,7 +305,7 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError! // trailing expression no longer auto-prints — `await` and a script // completion value are mutually exclusive in JS.) const wrapped = std.fmt.allocPrint(self.call_arena.allocator(), "(async () => {{\n{s}\n}})()", .{source}) catch - return try self.dupeError("out of memory"); + return .{ .err = try self.dupeError("out of memory") }; const script_source = self.env.isolate.initStringHandle(wrapped); var origin: v8.ScriptOrigin = undefined; @@ -304,27 +320,26 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError! &compiler_source, v8.kNoCompileOptions, v8.kNoCacheNoReason, - ) orelse return try self.formatCaught(context, &try_catch, "compile failed"); + ) orelse return .{ .err = try self.formatCaught(context, &try_catch, "compile failed") }; - const completion = v8.v8__Script__Run(script, context) orelse - return try self.formatCaught(context, &try_catch, "script failed"); + const completion_promise = v8.v8__Script__Run(script, context) orelse + return .{ .err = try self.formatCaught(context, &try_catch, "script failed") }; // `goto` only *starts* a navigation, so the root Promise is usually still // pending; drive the in-flight ones to completion. - const root: *const v8.Promise = @ptrCast(completion); + const root: *const v8.Promise = @ptrCast(completion_promise); self.driveAsync(context, &try_catch, root); if (v8.v8__TryCatch__HasCaught(&try_catch)) { - return try self.formatCaught(context, &try_catch, "script failed"); + return .{ .err = try self.formatCaught(context, &try_catch, "script failed") }; } // A still-pending root means the script awaited something we can't settle // (no async navigation is in flight) — stay silent. const state = promiseState(root); - if (state != v8.kFulfilled and state != v8.kRejected) return null; - const completion_value = v8.v8__Promise__Result(root) orelse return null; - if (state == v8.kRejected) return try self.formatRejection(context, completion_value); - self.printCompletion(context, completion_value); - return null; + if (state != v8.kFulfilled and state != v8.kRejected) return .{ .ok = null }; + const completion_value = v8.v8__Promise__Result(root) orelse return .{ .ok = null }; + if (state == v8.kRejected) return .{ .err = try self.formatRejection(context, completion_value) }; + return .{ .ok = try self.completion(context, completion_value) }; } /// `v8__Promise__State` returns the `c_uint` `PromiseState`, but the `k*` @@ -353,15 +368,55 @@ fn errorMessage(self: *Runtime, context: *const v8.Context, reason: *const v8.Va } /// Echo a script's output — the value it `return`s from the async wrapper, so a -/// script ending in `return page.extract(...)` prints without `console.log`. -/// `undefined` — no `return`, or a bare trailing expression — stays silent. -fn printCompletion(self: *Runtime, context: *const v8.Context, value: *const v8.Value) void { - if (v8.v8__Value__IsUndefined(value)) return; +/// script ending in `return page.extract(...)` prints without `console.log` — +/// and hand it back to the caller. `undefined` — no `return`, or a bare +/// trailing expression — stays silent and yields null, as does a value whose +/// display form can't be computed. +fn completion(self: *Runtime, context: *const v8.Context, value: *const v8.Value) error{OutOfMemory}!?Completion { + if (v8.v8__Value__IsUndefined(value)) return null; - var arena_state: std.heap.ArenaAllocator = .init(self.allocator); - defer arena_state.deinit(); - const text = self.displayString(arena_state.allocator(), context, value) catch return; + const arena = self.call_arena.allocator(); + const text = self.displayString(arena, context, value) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.JsException => return null, + }; self.writeConsoleLine(.log, text); + return .{ .text = text, .empty = self.isEmptyValue(value, text) }; +} + +/// Whether a completion value carries no data. `text` is the value's display +/// form: JSON for objects and arrays (a fallback coercion — function, circular +/// reference — won't parse and counts as data), plain coercion otherwise. +fn isEmptyValue(self: *Runtime, value: *const v8.Value, text: []const u8) bool { + if (v8.v8__Value__IsNull(value)) return true; + if (v8.v8__Value__IsObject(value)) { + // Scratch arena: the parse tree is as large as the extract output and + // must not sit in the call arena until the next run. + var arena_state: std.heap.ArenaAllocator = .init(self.allocator); + defer arena_state.deinit(); + const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena_state.allocator(), text, .{}) catch return false; + return jsonIsEmpty(parsed); + } + return text.len == 0; +} + +/// Null, "", and arrays/objects all of whose members are empty carry no data; +/// numbers and booleans always do (0 and false are answers). +fn jsonIsEmpty(value: std.json.Value) bool { + switch (value) { + .null => return true, + .string => |s| return s.len == 0, + .array => |a| { + for (a.items) |item| if (!jsonIsEmpty(item)) return false; + return true; + }, + .object => |o| { + var it = o.iterator(); + while (it.next()) |entry| if (!jsonIsEmpty(entry.value_ptr.*)) return false; + return true; + }, + else => return false, + } } /// Unwrap a callback's info handle and its `External` payload (the `*T` passed @@ -979,9 +1034,12 @@ fn dupeError(self: *Runtime, message: []const u8) RunError![]const u8 { const testing = @import("../testing.zig"); fn runTestScript(runtime: *Runtime, source: []const u8) !void { - if (try runtime.runSource(source, "agent-runtime-test.js")) |message| { - std.debug.print("agent script failed:\n{s}\n", .{message}); - return error.AgentScriptFailed; + switch (try runtime.runSource(source, "agent-runtime-test.js")) { + .ok => {}, + .err => |message| { + std.debug.print("agent script failed:\n{s}\n", .{message}); + return error.AgentScriptFailed; + }, } } @@ -1021,7 +1079,7 @@ test "agent script runtime: Page must be called with new" { const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); defer runtime.deinit(); - const message = (try runtime.runSource("Page();", "agent-runtime-page-no-new.js")).?; + const message = (try runtime.runSource("Page();", "agent-runtime-page-no-new.js")).err; try testing.expect(std.mem.indexOf(u8, message, "must be called with new") != null); } @@ -1037,7 +1095,7 @@ test "agent script runtime: a method on an un-navigated page errors" { const message = (try runtime.runSource( \\const page = new Page(); \\page.extract({ btn: "#btn" }); - , "agent-runtime-not-navigated.js")).?; + , "agent-runtime-not-navigated.js")).err; try testing.expect(std.mem.indexOf(u8, message, "not navigated") != null); } @@ -1058,7 +1116,7 @@ test "agent script runtime: page.close stales the handle" { \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); \\page.close(); \\page.extract({ btn: "#btn" }); - , "agent-runtime-close.js")).?; + , "agent-runtime-close.js")).err; try testing.expect(std.mem.indexOf(u8, message, "closed") != null); } @@ -1353,8 +1411,8 @@ test "agent script runtime: terminate interrupts local JavaScript" { defer runtime.cancelTerminate(); defer thread.join(); - const message = try runtime.runSource("while (true) {}", "agent-runtime-terminate-test.js"); - try testing.expect(message != null); + const result = try runtime.runSource("while (true) {}", "agent-runtime-terminate-test.js"); + try testing.expect(result == .err); } test "agent script runtime: agent variables persist and page globals are isolated" { @@ -1434,7 +1492,7 @@ test "agent script runtime: tool errors throw and stop execution" { \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); \\page.click({ selector: "#does-not-exist" }); \\globalThis.marker = "after"; - , "agent-runtime-failure.js")).?; + , "agent-runtime-failure.js")).err; try testing.expect(std.mem.indexOf(u8, message, "click") != null or std.mem.indexOf(u8, message, "NodeNotFound") != null or @@ -1489,7 +1547,7 @@ test "agent script runtime: builtin argument marshalling (positional + options)" { const message = (try runtime.runSource( \\await new Page().goto("http://localhost:9582/src/browser/tests/mcp_actions.html", { url: "http://other" }); - , "agent-runtime-conflict.js")).?; + , "agent-runtime-conflict.js")).err; try testing.expect(std.mem.indexOf(u8, message, "invalid arguments") != null); } @@ -1499,7 +1557,7 @@ test "agent script runtime: builtin argument marshalling (positional + options)" \\const page = new Page(); \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); \\page.click("#btn", "#extra"); - , "agent-runtime-arity.js")).?; + , "agent-runtime-arity.js")).err; try testing.expect(std.mem.indexOf(u8, message, "invalid arguments") != null); } } @@ -1521,3 +1579,35 @@ test "agent script runtime: top-level await runs in an async wrapper" { \\return x + 2; ); } + +test "agent script runtime: completion value emptiness" { + defer testing.reset(); + + var registry = CDPNode.Registry.init(testing.allocator); + defer registry.deinit(); + + const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); + defer runtime.deinit(); + + const cases = [_]struct { source: []const u8, empty: bool }{ + // The shapes a stale extract selector produces: an empty list, or rows + // whose every field missed. + .{ .source = "return [];", .empty = true }, + .{ .source = "return {};", .empty = true }, + .{ .source = "return null;", .empty = true }, + .{ .source = "return { stories: [] };", .empty = true }, + .{ .source = "return [{ id: null, title: \"\" }];", .empty = true }, + .{ .source = "return [{ id: null, title: \"HN\" }];", .empty = false }, + .{ .source = "return \"text\";", .empty = false }, + .{ .source = "return 0;", .empty = false }, + .{ .source = "return false;", .empty = false }, + }; + for (cases) |case| { + const result = try runtime.runSource(case.source, "agent-runtime-emptiness.js"); + try testing.expectEqual(case.empty, result.ok.?.empty); + } + + // No `return` — nothing to classify, nothing to heal. + const silent = try runtime.runSource("const x = 1;", "agent-runtime-emptiness.js"); + try testing.expectEqual(null, silent.ok); +} From 70109b4d514c4ec2797af1f8bcba89b93ff7ed14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 8 Jul 2026 14:01:49 +0200 Subject: [PATCH 03/15] agent: detect and heal empty extract fields Tracks extract calls during script execution. If a list field in an extract schema returns empty across all calls, it is treated as a script error (`dry_extracts`) to trigger the heal prompt. --- src/agent/Agent.zig | 52 +++++++++-- src/script/Runtime.zig | 192 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 215 insertions(+), 29 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 228f4f3c28..7501f5e362 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -873,6 +873,7 @@ fn promptHeal(path: []const u8, kind: ScriptError.Kind) bool { const symptom = switch (kind) { .threw => "failed", .empty => "ran but returned no data", + .dry_extracts => "ran but some extracts came back empty", }; var header_buf: [256]u8 = undefined; const header = std.fmt.bufPrint(&header_buf, "{s} {s}. Heal it with the model?", .{ path, symptom }) catch @@ -881,6 +882,8 @@ fn promptHeal(path: []const u8, kind: ScriptError.Kind) bool { "heal — diagnose and fix, then validate in a fresh session (drops page/cookies like /reset)", "no — leave the script and session as they are", }; + // Blank line so the question doesn't run into the script's output dump. + std.debug.print("\n", .{}); const idx = Terminal.promptNumberedChoice(header, labels, 1) catch return false; return idx == 0; } @@ -1530,10 +1533,12 @@ const ScriptError = struct { source: []const u8, /// `empty` is a run that completed but returned a value with no data in - /// it — the usual symptom of a stale selector, which matches nothing - /// instead of throwing. Only heal treats it as a failure; a plain replay - /// still exits 0, since an empty answer can be the right answer. - const Kind = enum { threw, empty }; + /// it; `dry_extracts` one whose return value had data, but where some + /// extract list field came back empty on every call. Both are the usual + /// symptom of a stale selector, which matches nothing instead of throwing. + /// Only heal treats them as failures; a plain replay still exits 0, since + /// an empty answer can be the right answer. + const Kind = enum { threw, empty, dry_extracts }; }; fn runScript(self: *Agent, path: []const u8) bool { @@ -1542,7 +1547,7 @@ fn runScript(self: *Agent, path: []const u8) bool { return switch (self.runScriptOutcome(arena.allocator(), path)) { .ok => true, .fatal => false, - .script_error => |script_error| script_error.kind == .empty, + .script_error => |script_error| script_error.kind != .threw, }; } @@ -1594,8 +1599,8 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc .source = arena.dupe(u8, content) catch return .fatal, } }; }, - .ok => |maybe_completion| { - if (maybe_completion) |completion| { + .ok => |ok| { + if (ok.completion) |completion| { if (completion.empty) return .{ .script_error = .{ .kind = .empty, .detail = std.fmt.allocPrint(arena, "The script completed without throwing, but its return value carries no data: {s}\n" ++ @@ -1603,6 +1608,15 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc .source = arena.dupe(u8, content) catch return .fatal, } }; } + // Checked even without a completion: a no-`return` script can + // still have dry extracts. + if (dryExtractsDetail(arena, ok.extract_stats) catch return .fatal) |detail| { + return .{ .script_error = .{ + .kind = .dry_extracts, + .detail = detail, + .source = arena.dupe(u8, content) catch return .fatal, + } }; + } }, } @@ -1612,6 +1626,30 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc return .ok; } +/// Detail for `.dry_extracts`: one line per extract list field that came back +/// empty on every call, or null when none did. +fn dryExtractsDetail(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) !?[]const u8 { + var aw: std.Io.Writer.Allocating = .init(arena); + var any = false; + for (stats) |stat| { + if (stat.empty != stat.calls) continue; + if (!any) { + try aw.writer.writeAll("The script completed without throwing, but some extracts came back empty on every call:\n"); + any = true; + } + if (stat.field) |field| { + try aw.writer.print("- the \"{s}\" list in extract({s}) came back empty", .{ field, stat.schema }); + } else { + try aw.writer.print("- extract({s}) returned no data", .{stat.schema}); + } + if (stat.calls != 1) try aw.writer.print(" in all {d} calls", .{stat.calls}); + try aw.writer.writeAll("\n"); + } + if (!any) return null; + try aw.writer.writeAll("A selector probably no longer matches the page — but an empty list can also be legitimate; verify against the live page before changing anything."); + return aw.written(); +} + /// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. fn runScriptWithHeal(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index c6dc82bfa2..6efcc20bb9 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -46,6 +46,9 @@ console_observer: ?ConsoleObserver = null, pending_gotos: std.ArrayList(PendingGoto), /// Restarted per `runSource`; backs `PendingGoto.deadline_ms`. run_timer: std.time.Timer, +/// Per-run tally of extract list-field emptiness. call_arena-backed, so it is +/// re-zeroed alongside the arena reset at the top of `runSource`. +extract_stats: std.ArrayList(ExtractStat) = .empty, /// The runtime installs exactly the recorded browser tools as script /// primitives — the same set the recorder writes — so every recorded call @@ -271,18 +274,36 @@ pub const Completion = struct { empty: bool, }; -/// A script run's outcome. `err` is a formatted JS compile/runtime exception; -/// `ok` carries the value the script returned, null when it returned -/// `undefined`. All slices are allocated in this runtime's call arena and -/// valid until deinit or the next run. +/// One extract schema's top-level list field, tallied across the run. +pub const ExtractStat = struct { + /// Schema JSON as the script wrote it (array schemas are shown without the + /// internal `__root` wrapper). + schema: []const u8, + /// Top-level list field of the result; null when the schema itself is a list. + field: ?[]const u8, + calls: u32, + empty: u32, +}; + +/// A script run's outcome. `err` is a formatted JS compile/runtime exception. +/// All slices are allocated in this runtime's call arena and valid until +/// deinit or the next run. pub const RunResult = union(enum) { - ok: ?Completion, + ok: Ok, err: []const u8, }; +pub const Ok = struct { + /// The value the script returned; null when it returned `undefined`. + completion: ?Completion, + /// Per-(schema, list-field) extract tallies for the run. + extract_stats: []const ExtractStat, +}; + /// Run script source in the agent context. pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!RunResult { _ = self.call_arena.reset(.retain_capacity); + self.extract_stats = .empty; self.run_timer = std.time.Timer.start() catch return .{ .err = try self.dupeError("internal: timer unavailable") }; var hs: lp.js.HandleScope = undefined; @@ -336,10 +357,17 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError! // A still-pending root means the script awaited something we can't settle // (no async navigation is in flight) — stay silent. const state = promiseState(root); - if (state != v8.kFulfilled and state != v8.kRejected) return .{ .ok = null }; - const completion_value = v8.v8__Promise__Result(root) orelse return .{ .ok = null }; + if (state != v8.kFulfilled and state != v8.kRejected) return self.runOk(null); + const completion_value = v8.v8__Promise__Result(root) orelse return self.runOk(null); if (state == v8.kRejected) return .{ .err = try self.formatRejection(context, completion_value) }; - return .{ .ok = try self.completion(context, completion_value) }; + return self.runOk(try self.completion(context, completion_value)); +} + +fn runOk(self: *Runtime, completion_value: ?Completion) RunResult { + return .{ .ok = .{ + .completion = completion_value, + .extract_stats = self.extract_stats.items, + } }; } /// `v8__Promise__State` returns the `c_uint` `PromiseState`, but the `k*` @@ -518,10 +546,18 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf switch (result) { .ok => |text| switch (tool) { .extract => { - const normalized = self.normalizeExtractReturnJson(arena, text) catch |err| switch (err) { + const normalized = normalizeExtractReturnJson(arena, text) catch |err| switch (err) { error.OutOfMemory => return self.throwError("out of memory"), }; - self.setReturnJson(context, info, normalized); + // Recording happens after `callTool` returns, so a re-entrant + // extract (via a `toJSON` during argument marshalling) tallies + // inner-then-outer; the map is append-only. Failed extracts + // threw above and are intentionally not counted. + if (normalized.parsed) |parsed| { + self.recordExtractStats(args, parsed) catch + return self.throwError("out of memory"); + } + self.setReturnJson(context, info, normalized.text); }, else => self.setReturnString(info, text), }, @@ -529,6 +565,48 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf } } +/// Tally each top-level list field of an extract result: a field that stays +/// empty across every call of its schema is the heal trigger's signal, while +/// scalar fields (legitimately sparse) are never tracked. A whole-array result +/// (`__root` schema) tallies as the single null field. +fn recordExtractStats(self: *Runtime, args: ?std.json.Value, result: std.json.Value) error{OutOfMemory}!void { + const raw_schema = switch ((args orelse return).object.get("schema") orelse return) { + .string => |s| s, + else => return, + }; + const schema = stripExtractSchemaRoot(raw_schema); + switch (result) { + .array => try self.bumpExtractStat(schema, null, jsonIsEmpty(result)), + .object => |obj| { + var it = obj.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.* != .array) continue; + try self.bumpExtractStat(schema, entry.key_ptr.*, jsonIsEmpty(entry.value_ptr.*)); + } + }, + else => {}, + } +} + +fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_empty: bool) error{OutOfMemory}!void { + for (self.extract_stats.items) |*stat| { + if (!std.mem.eql(u8, stat.schema, schema)) continue; + const same_field = if (stat.field) |f| field != null and std.mem.eql(u8, f, field.?) else field == null; + if (!same_field) continue; + stat.calls += 1; + if (is_empty) stat.empty += 1; + return; + } + // `schema`/`field` live in invoke's per-call arena; the stat outlives it. + const arena = self.call_arena.allocator(); + try self.extract_stats.append(arena, .{ + .schema = try arena.dupe(u8, schema), + .field = if (field) |f| try arena.dupe(u8, f) else null, + .calls = 1, + .empty = @intFromBool(is_empty), + }); +} + /// Start the receiver Page's navigation and return a *pending* Promise of the /// page object; `driveAsync` settles it once the frame loads. This is what lets /// `Promise.all([a.goto(x), b.goto(y)])` fetch both in parallel. @@ -866,10 +944,24 @@ fn extractSchemaString(arena: std.mem.Allocator, value: std.json.Value) error{Ou }; } +/// Key under which `normalizeExtractSchemaString` nests an array schema so the +/// walker always receives an object; wrap, unwrap, and display-strip must agree. +const extract_root_key = "__root"; + fn normalizeExtractSchemaString(arena: std.mem.Allocator, schema: []const u8) error{OutOfMemory}![]const u8 { const trimmed = std.mem.trim(u8, schema, &std.ascii.whitespace); if (trimmed.len == 0 or trimmed[0] != '[') return schema; - return try std.fmt.allocPrint(arena, "{{\"__root\":{s}}}", .{schema}); + return try std.fmt.allocPrint(arena, "{{\"" ++ extract_root_key ++ "\":{s}}}", .{schema}); +} + +/// Inverse of the wrapping above, for showing an array schema the way the +/// script wrote it. +fn stripExtractSchemaRoot(schema: []const u8) []const u8 { + const prefix = "{\"" ++ extract_root_key ++ "\":"; + if (std.mem.startsWith(u8, schema, prefix) and std.mem.endsWith(u8, schema, "}")) { + return schema[prefix.len .. schema.len - 1]; + } + return schema; } fn argJson( @@ -903,19 +995,27 @@ fn objectWith(arena: std.mem.Allocator, key: []const u8, value: std.json.Value) /// Unwraps only the `__root` sentinel that `normalizeExtractSchemaString` injects /// for array schemas; a real single-field object schema keeps its shape. -fn normalizeExtractReturnJson(_: *Runtime, arena: std.mem.Allocator, value: []const u8) error{OutOfMemory}![]const u8 { - if (value.len == 0) return value; +/// `parsed` is the post-unwrap value, null when the raw text is empty or not +/// JSON — the stats hook classifies it without a second parse. +fn normalizeExtractReturnJson(arena: std.mem.Allocator, value: []const u8) error{OutOfMemory}!struct { + text: []const u8, + parsed: ?std.json.Value, +} { + if (value.len == 0) return .{ .text = value, .parsed = null }; const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, value, .{}) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, - else => return value, + else => return .{ .text = value, .parsed = null }, }; - if (parsed != .object or parsed.object.count() != 1) return value; - - var it = parsed.object.iterator(); - const entry = it.next() orelse return value; - if (!std.mem.eql(u8, entry.key_ptr.*, "__root")) return value; - return try std.json.Stringify.valueAlloc(arena, entry.value_ptr.*, .{}); + if (parsed == .object and parsed.object.count() == 1) { + var it = parsed.object.iterator(); + const entry = it.next().?; + if (std.mem.eql(u8, entry.key_ptr.*, extract_root_key)) return .{ + .text = try std.json.Stringify.valueAlloc(arena, entry.value_ptr.*, .{}), + .parsed = entry.value_ptr.*, + }; + } + return .{ .text = value, .parsed = parsed }; } fn setReturnString(self: *Runtime, info: *const v8.FunctionCallbackInfo, value: []const u8) void { @@ -1604,10 +1704,58 @@ test "agent script runtime: completion value emptiness" { }; for (cases) |case| { const result = try runtime.runSource(case.source, "agent-runtime-emptiness.js"); - try testing.expectEqual(case.empty, result.ok.?.empty); + try testing.expectEqual(case.empty, result.ok.completion.?.empty); } // No `return` — nothing to classify, nothing to heal. const silent = try runtime.runSource("const x = 1;", "agent-runtime-emptiness.js"); - try testing.expectEqual(null, silent.ok); + try testing.expectEqual(null, silent.ok.completion); +} + +test "agent script runtime: extract stats tally list-field emptiness" { + defer testing.reset(); + defer testing.test_session.closeAllPages(); + + var registry = CDPNode.Registry.init(testing.allocator); + defer registry.deinit(); + + const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); + defer runtime.deinit(); + + const result = try runtime.runSource( + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\page.extract({ items: [".no-such"] }); + \\page.extract({ items: [".no-such"] }); + \\page.extract({ btn: "#btn", buttons: ["button"] }); + \\page.extract([".no-such-root"]); + \\const other = new Page(); + \\await other.goto("http://localhost:9582/src/browser/tests/runner/runner1.html"); + \\page.extract({ sel: ["#sel0"] }); + \\other.extract({ sel: ["#sel0"] }); + , "agent-runtime-extract-stats.js"); + + const stats = result.ok.extract_stats; + try testing.expectEqual(4, stats.len); + + // Scalar `btn` never records a stat. + try testing.expectString("items", stats[0].field.?); + try testing.expectEqual(2, stats[0].calls); + try testing.expectEqual(2, stats[0].empty); + + try testing.expectString("buttons", stats[1].field.?); + try testing.expectEqual(1, stats[1].calls); + try testing.expectEqual(0, stats[1].empty); + + try testing.expectEqual(null, stats[2].field); + try testing.expectString("[\".no-such-root\"]", stats[2].schema); + try testing.expectEqual(1, stats[2].calls); + try testing.expectEqual(1, stats[2].empty); + + try testing.expectString("sel", stats[3].field.?); + try testing.expectEqual(2, stats[3].calls); + try testing.expectEqual(1, stats[3].empty); + + const rerun = try runtime.runSource("return 1;", "agent-runtime-extract-stats.js"); + try testing.expectEqual(0, rerun.ok.extract_stats.len); } From 8d1ef30b7b8f736110cfe4889015e3c411695d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 8 Jul 2026 14:33:28 +0200 Subject: [PATCH 04/15] agent: add extract baselines for script healing Tracks extract results and persists them as comments in saved scripts. This allows healing to compare replays against record-time reality. --- src/agent/Agent.zig | 314 ++++++++++++++++++++++++++++++++++------- src/agent/Baseline.zig | 223 +++++++++++++++++++++++++++++ src/script/Runtime.zig | 49 ++++--- 3 files changed, 519 insertions(+), 67 deletions(-) create mode 100644 src/agent/Baseline.zig diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 7501f5e362..b5ac122a4c 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -29,6 +29,7 @@ const Command = lp.Command; const Schema = lp.Schema; const Recorder = lp.Recorder; const ScriptRuntime = lp.Runtime; +const Baseline = @import("Baseline.zig"); const Credentials = zenai.provider.Credentials; const App = @import("../App.zig"); @@ -241,6 +242,7 @@ session: *lp.Session, node_registry: CDPNode.Registry, terminal: Terminal, save_buffer: Recorder, +baseline: Baseline, save_path: ?[]u8, script_runtime_mutex: std.Thread.Mutex = .{}, active_script_runtime: ?*ScriptRuntime = null, @@ -427,6 +429,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent .node_registry = .init(allocator), .terminal = .init(allocator, history_paths, verbosity, will_repl), .save_buffer = .init(allocator), + .baseline = .init(allocator), .save_path = null, .conversation = .init(allocator, opts.system_prompt orelse default_system_prompt), .model = model, @@ -471,6 +474,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent pub fn deinit(self: *Agent) void { self.terminal.uninstallLogSink(); self.save_buffer.deinit(); + self.baseline.deinit(); if (self.save_path) |p| self.allocator.free(p); self.terminal.deinit(); self.conversation.deinit(); @@ -825,6 +829,7 @@ fn handleUsage(self: *Agent) void { fn clearConversation(self: *Agent) void { self.conversation.rollback(0); self.save_buffer.reset(); + self.baseline.reset(); if (self.save_path) |p| self.allocator.free(p); self.save_path = null; self.total_usage = .{}; @@ -857,19 +862,30 @@ fn handleLoad(self: *Agent, rest: []const u8) void { } var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); - switch (self.runScriptOutcome(arena.allocator(), path)) { - .ok, .fatal => {}, - .script_error => |script_error| { - if (self.ai_client == null) return; - if (!promptHeal(path, script_error.kind)) return; - _ = self.healLoop(arena.allocator(), path, script_error); - }, + while (true) { + switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok, .fatal => return, + .script_error => |script_error| { + if (self.ai_client == null) return; + switch (promptHeal(path, script_error.kind)) { + .no => return, + .retry => {}, + .heal => { + _ = self.healLoop(arena.allocator(), path, script_error); + return; + }, + } + }, + } } } +const HealChoice = enum { heal, retry, no }; + /// Defaults to no: healing spends tokens and its validation step resets the -/// browser session. -fn promptHeal(path: []const u8, kind: ScriptError.Kind) bool { +/// browser session. Retry is the antidote to a transient failure (a flaky +/// page load) that would otherwise send a correct script to the model. +fn promptHeal(path: []const u8, kind: ScriptError.Kind) HealChoice { const symptom = switch (kind) { .threw => "failed", .empty => "ran but returned no data", @@ -880,12 +896,17 @@ fn promptHeal(path: []const u8, kind: ScriptError.Kind) bool { "Script failed. Heal it with the model?"; const labels: []const [:0]const u8 = &.{ "heal — diagnose and fix, then validate in a fresh session (drops page/cookies like /reset)", + "retry — run the script again, in case the failure was transient (no tokens)", "no — leave the script and session as they are", }; // Blank line so the question doesn't run into the script's output dump. std.debug.print("\n", .{}); - const idx = Terminal.promptNumberedChoice(header, labels, 1) catch return false; - return idx == 0; + const idx = Terminal.promptNumberedChoice(header, labels, 2) catch return .no; + return switch (idx) { + 0 => .heal, + 1 => .retry, + else => .no, + }; } const api_keys_hint = settings.api_keys_hint; @@ -1127,7 +1148,7 @@ fn handleSave(self: *Agent, arena: std.mem.Allocator, rest: []const u8) void { null; defer if (new_save_path) |p| self.allocator.free(p); - save.writeContentFile(path, self.save_buffer.bytes(), mode) catch |err| { + save.writeContentFile(path, self.appendSessionBaseline(arena, self.save_buffer.bytes()), mode) catch |err| { self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) }); return; }; @@ -1141,6 +1162,14 @@ fn handleSave(self: *Agent, arena: std.mem.Allocator, rest: []const u8) void { self.terminal.printInfo("Saved {d} line(s) to {s}", .{ saved_lines, self.save_path.? }); } +/// `script` with the session's extract baseline as its trailing comment (stale +/// baseline lines stripped); the script itself on any failure — the baseline +/// is telemetry, never worth failing a save over. +fn appendSessionBaseline(self: *Agent, arena: std.mem.Allocator, script: []const u8) []const u8 { + const line = self.baseline.serialize(arena) catch return script; + return Baseline.withBaseline(arena, script, line) catch script; +} + fn promptSaveMode(self: *Agent, path: []const u8) ?save.Mode { var header_buf: [256]u8 = undefined; const header = std.fmt.bufPrint(&header_buf, "{s} already exists. Pick save mode:", .{path}) catch @@ -1237,7 +1266,8 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo const script = self.synthesizeScriptText(arena, "save", path, previous_script, prompt) orelse return; - save.writeContentFile(path, script, mode) catch |err| { + const content = self.appendSessionBaseline(arena, script); + save.writeContentFile(path, content, mode) catch |err| { self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) }); return; }; @@ -1464,7 +1494,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool .tool_call => |t| t, else => return .{ .text = "internal: command has no tool mapping", .is_error = true }, }; - return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| .{ + const result = browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| return .{ .text = switch (err) { error.OutOfMemory => "out of memory", error.FrameNotLoaded => "no page loaded — run /goto first", @@ -1472,6 +1502,8 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool }, .is_error = true, }; + if (tc.tool == .extract and !result.is_error) self.baseline.noteExtractResult(result.text) catch {}; + return result; } /// Data output (/extract, /evaluate, /markdown, /tree, …) → plain stdout on @@ -1518,11 +1550,19 @@ const ScriptOutput = struct { /// `fatal` covers setup failures (unreadable file, runtime init, OOM) that a /// retry can't help. const ScriptRunOutcome = union(enum) { - ok, + ok: RunFacts, fatal, script_error: ScriptError, }; +/// Facts about a passing run, for heal validation's cure check. Duped into the +/// caller's arena — the runtime dies with `runScriptOutcome`. +const RunFacts = struct { + /// The script returned a value that carries data. + returned_data: bool, + extract_stats: []const ScriptRuntime.ExtractStat, +}; + /// Both slices are duped into the caller's arena. `source` is the exact text /// that ran, so a heal diagnoses what actually failed instead of re-reading a /// possibly-changed file. @@ -1531,6 +1571,10 @@ const ScriptError = struct { /// Formatted error (line, stack) — or, for `empty`, what came back. detail: []const u8, source: []const u8, + /// For `dry_extracts`: the field names that were empty on every call + /// (null = a whole-array schema). The cure check requires each one to + /// come back with data before a heal may replace the file. + dry_fields: []const ?[]const u8 = &.{}, /// `empty` is a run that completed but returned a value with no data in /// it; `dry_extracts` one whose return value had data, but where some @@ -1559,6 +1603,7 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc self.terminal.printError("Failed to read script '{s}': {s}", .{ path, @errorName(err) }); return .fatal; }; + const baseline = Baseline.parse(script_arena.allocator(), content); const runtime = ScriptRuntime.init(self.allocator, self.browser.app, self.session, &self.node_registry) catch |err| { self.terminal.printError("Failed to initialize script runtime: {s}", .{@errorName(err)}); @@ -1583,7 +1628,7 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc const result = runtime.runSource(content, path); self.terminal.endTool(); - switch (result catch |err| { + const ok = switch (result catch |err| { self.terminal.printError("Script failed: {s}", .{@errorName(err)}); return .fatal; }) { @@ -1599,55 +1644,106 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc .source = arena.dupe(u8, content) catch return .fatal, } }; }, - .ok => |ok| { + .ok => |ok| blk: { if (ok.completion) |completion| { - if (completion.empty) return .{ .script_error = .{ - .kind = .empty, - .detail = std.fmt.allocPrint(arena, "The script completed without throwing, but its return value carries no data: {s}\n" ++ - "A selector probably no longer matches anything on the page.", .{completion.text}) catch return .fatal, - .source = arena.dupe(u8, content) catch return .fatal, - } }; + if (completion.empty) { + const value = capDetail(arena, completion.text) catch return .fatal; + return .{ .script_error = .{ + .kind = .empty, + .detail = std.fmt.allocPrint(arena, "The script completed without throwing, but its return value carries no data: {s}\n" ++ + "A selector probably no longer matches anything on the page.", .{value}) catch return .fatal, + .source = arena.dupe(u8, content) catch return .fatal, + } }; + } } // Checked even without a completion: a no-`return` script can // still have dry extracts. - if (dryExtractsDetail(arena, ok.extract_stats) catch return .fatal) |detail| { + if (dryExtractsFinding(arena, ok.extract_stats, baseline) catch return .fatal) |finding| { return .{ .script_error = .{ .kind = .dry_extracts, - .detail = detail, + .detail = finding.detail, .source = arena.dupe(u8, content) catch return .fatal, + .dry_fields = finding.fields, } }; } + break :blk ok; }, - } + }; // A script that printed nothing leaves no trace, so freeze the spinner into // a green bullet (like /goto); one that printed already showed its result. if (!output.emitted) self.terminal.printScriptDone("script", path); - return .ok; + return .{ .ok = .{ + .returned_data = ok.completion != null, + .extract_stats = dupeExtractStats(arena, ok.extract_stats) catch return .fatal, + } }; +} + +fn dupeExtractStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) error{OutOfMemory}![]const ScriptRuntime.ExtractStat { + const out = try arena.alloc(ScriptRuntime.ExtractStat, stats.len); + for (stats, out) |stat, *o| { + o.* = .{ + .schema = try arena.dupe(u8, stat.schema), + .field = if (stat.field) |f| try arena.dupe(u8, f) else null, + .is_list = stat.is_list, + .calls = stat.calls, + .empty = stat.empty, + }; + } + return out; } -/// Detail for `.dry_extracts`: one line per extract list field that came back -/// empty on every call, or null when none did. -fn dryExtractsDetail(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) !?[]const u8 { +/// Bound a value or schema echoed into a heal message; a degenerate empty-ish +/// result (hundreds of all-null rows) would otherwise bloat the LLM turn. +const detail_max_bytes: usize = 2048; + +fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]const u8 { + if (text.len <= detail_max_bytes) return text; + return std.mem.concat(arena, u8, &.{ string.truncateUtf8(text, detail_max_bytes), "…[truncated]" }); +} + +const DryFinding = struct { + detail: []const u8, + fields: []const ?[]const u8, +}; + +/// The `.dry_extracts` finding: one detail line per extract field that came +/// back empty on every call, plus the field names for the cure check. +/// Null when no field was dry. +fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat, baseline: ?Baseline.Fields) !?DryFinding { var aw: std.Io.Writer.Allocating = .init(arena); - var any = false; + var fields: std.ArrayList(?[]const u8) = .empty; for (stats) |stat| { if (stat.empty != stat.calls) continue; - if (!any) { + if (!dryCanTrigger(stat, baseline)) continue; + if (fields.items.len == 0) { try aw.writer.writeAll("The script completed without throwing, but some extracts came back empty on every call:\n"); - any = true; } + try fields.append(arena, if (stat.field) |f| try arena.dupe(u8, f) else null); + const schema = try capDetail(arena, stat.schema); if (stat.field) |field| { - try aw.writer.print("- the \"{s}\" list in extract({s}) came back empty", .{ field, stat.schema }); + const shape: []const u8 = if (stat.is_list) "list" else "field"; + try aw.writer.print("- the \"{s}\" {s} in extract({s}) came back empty", .{ field, shape, schema }); } else { - try aw.writer.print("- extract({s}) returned no data", .{stat.schema}); + try aw.writer.print("- extract({s}) returned no data", .{schema}); } if (stat.calls != 1) try aw.writer.print(" in all {d} calls", .{stat.calls}); try aw.writer.writeAll("\n"); } - if (!any) return null; - try aw.writer.writeAll("A selector probably no longer matches the page — but an empty list can also be legitimate; verify against the live page before changing anything."); - return aw.written(); + if (fields.items.len == 0) return null; + try aw.writer.writeAll("A selector probably no longer matches the page — but an empty result can also be legitimate; verify against the live page before changing anything."); + return .{ .detail = aw.written(), .fields = fields.items }; +} + +/// Whether an all-empty stat is heal-worthy. With a baseline, record-time +/// reality decides: a field that carried data then is a finding now (scalars +/// included), one that was already empty then is legitimately sparse. Without +/// one, only list fields qualify — lists signal collection intent. +fn dryCanTrigger(stat: ScriptRuntime.ExtractStat, baseline: ?Baseline.Fields) bool { + if (baseline) |b| { + if (b.get(stat.field orelse "")) |base| return base.nonempty > 0; + } + return stat.is_list; } /// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. @@ -1655,6 +1751,15 @@ fn runScriptWithHeal(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); + switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok => return true, + .fatal => return false, + .script_error => {}, + } + + // A re-run is free; healing is not. One retry filters transient failures + // (a flaky page load) before the model is asked to fix a correct script. + self.terminal.printInfo("Script failed; retrying once before healing.", .{}); switch (self.runScriptOutcome(arena.allocator(), path)) { .ok => return true, .fatal => return false, @@ -1745,13 +1850,29 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr }; switch (self.runScriptOutcome(arena, tmp_path)) { - .ok => { - std.fs.cwd().rename(tmp_path, path) catch |err| { - self.terminal.printError("healed script validated but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }); + .ok => |facts| { + if (cureFailure(arena, first, facts) catch { + self.removeTempScript(tmp_path); + self.terminal.printError("heal failed: out of memory", .{}); return false; - }; - self.terminal.printInfo("Healed {s}: the revised script validated in a fresh session.", .{path}); - return true; + }) |failure| { + self.terminal.printWarning("{s}", .{failure}); + source = revised; + error_detail = failure; + } else { + // Refresh the baseline from the validation run — synthesis + // may have copied the stale one from the broken script. + // Best-effort: the validated revision is already on disk. + if (refreshedBaselineScript(arena, revised, facts.extract_stats)) |updated| { + save.writeContentFile(tmp_path, updated, .replace) catch {}; + } + std.fs.cwd().rename(tmp_path, path) catch |err| { + self.terminal.printError("healed script validated but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }); + return false; + }; + self.terminal.printInfo("Healed {s}: the revised script validated in a fresh session.", .{path}); + return true; + } }, .fatal => { self.removeTempScript(tmp_path); @@ -1768,6 +1889,38 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr return false; } +/// Null when the validation run cured the original finding; otherwise the +/// message fed to the next heal attempt. Running clean is not a cure on its +/// own — a revision that deletes the failing extract (or the `return`) also +/// runs clean. +fn cureFailure(arena: std.mem.Allocator, first: ScriptError, facts: RunFacts) error{OutOfMemory}!?[]const u8 { + switch (first.kind) { + .threw => return null, + .empty => return if (facts.returned_data) + null + else + "The revised script ran, but still returns no data (or no longer returns anything) — the original returned a value.", + .dry_extracts => { + for (first.dry_fields) |dry| { + const cured = for (facts.extract_stats) |stat| { + const same = if (dry) |d| + stat.field != null and std.mem.eql(u8, stat.field.?, d) + else + stat.field == null; + if (same and stat.empty < stat.calls) break true; + } else false; + if (!cured) return try std.fmt.allocPrint(arena, "The revised script ran, but the \"{s}\" extract still came back empty on every call (or was removed) — keep it and fix its selector.", .{dry orelse ""}); + } + return null; + }, + } +} + +fn refreshedBaselineScript(arena: std.mem.Allocator, revised: []const u8, stats: []const ScriptRuntime.ExtractStat) ?[]const u8 { + const line = Baseline.serializeStats(arena, stats) catch return null; + return Baseline.withBaseline(arena, revised, line) catch null; +} + fn removeTempScript(self: *Agent, tmp_path: []const u8) void { std.fs.cwd().deleteFile(tmp_path) catch |err| { self.terminal.printWarning("could not remove temp script {s}: {s}", .{ tmp_path, @errorName(err) }); @@ -2085,10 +2238,14 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co self.terminal.spinner.setTool(tool_name, args_str); defer self.terminal.spinner.setThinking(); - const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments)) |result| - .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error } - else |err| - .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; + const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments)) |result| blk: { + // Pre-cap text: a truncated result would parse as malformed and + // record nothing. + if (std.mem.eql(u8, tool_name, "extract") and !result.is_error) { + self.baseline.noteExtractResult(result.text) catch {}; + } + break :blk .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error }; + } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error); if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error); @@ -2186,6 +2343,67 @@ fn completionModels(context: *anyopaque, _: std.mem.Allocator) []const []const u test { _ = save; _ = settings; + _ = Baseline; +} + +test "cureFailure: running clean is not a cure" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const dry: ScriptError = .{ + .kind = .dry_extracts, + .detail = "", + .source = "", + .dry_fields = &.{ @as(?[]const u8, "comments"), null }, + }; + const cured_stats: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, + .{ .schema = "[]", .field = null, .is_list = true, .calls = 1, .empty = 0 }, + }; + try std.testing.expectEqual(null, try cureFailure(aa, dry, .{ .returned_data = true, .extract_stats = cured_stats })); + + // Fix-by-deletion: the dry field is simply gone from the revised run. + const deleted = (try cureFailure(aa, dry, .{ .returned_data = true, .extract_stats = cured_stats[1..] })).?; + try std.testing.expect(std.mem.indexOf(u8, deleted, "\"comments\"") != null); + + // Still dry counts as uncured. + const still_dry_stats: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 5 }, + cured_stats[1], + }; + try std.testing.expect((try cureFailure(aa, dry, .{ .returned_data = true, .extract_stats = still_dry_stats })) != null); + + // .empty is cured only by a data-carrying return. + const empty: ScriptError = .{ .kind = .empty, .detail = "", .source = "" }; + try std.testing.expectEqual(null, try cureFailure(aa, empty, .{ .returned_data = true, .extract_stats = &.{} })); + try std.testing.expect((try cureFailure(aa, empty, .{ .returned_data = false, .extract_stats = &.{} })) != null); + + // .threw needs nothing beyond running clean. + const threw: ScriptError = .{ .kind = .threw, .detail = "", .source = "" }; + try std.testing.expectEqual(null, try cureFailure(aa, threw, .{ .returned_data = false, .extract_stats = &.{} })); +} + +test "dryCanTrigger: baseline overrides the list-only rule" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const scalar: ScriptRuntime.ExtractStat = .{ .schema = "{}", .field = "title", .is_list = false, .calls = 3, .empty = 3 }; + const list: ScriptRuntime.ExtractStat = .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 3, .empty = 3 }; + + // No baseline: list-only. + try std.testing.expectEqual(false, dryCanTrigger(scalar, null)); + try std.testing.expectEqual(true, dryCanTrigger(list, null)); + + var fields: Baseline.Fields = .empty; + try fields.put(aa, "title", .{ .calls = 2, .nonempty = 2 }); + try fields.put(aa, "comments", .{ .calls = 2, .nonempty = 0 }); + + // Baseline: a scalar that carried data at record time triggers; a list + // that was already empty then is legitimately sparse. + try std.testing.expectEqual(true, dryCanTrigger(scalar, fields)); + try std.testing.expectEqual(false, dryCanTrigger(list, fields)); } test "capToolOutput: passes through when under cap" { diff --git a/src/agent/Baseline.zig b/src/agent/Baseline.zig new file mode 100644 index 0000000000..606e49e7f7 --- /dev/null +++ b/src/agent/Baseline.zig @@ -0,0 +1,223 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Session-scoped record of what the agent's extract calls actually returned. +//! `/save` persists it into the script as a `// lp:baseline {...}` comment, so +//! a replay compares its own extract results against record-time reality — +//! which fields carried data when the script was made — instead of guessing. + +const std = @import("std"); +const lp = @import("lightpanda"); +const ScriptRuntime = lp.Runtime; + +const Baseline = @This(); + +pub const marker = "// lp:baseline "; + +pub const FieldStat = struct { + calls: u32 = 0, + nonempty: u32 = 0, +}; + +/// Keyed by top-level result field name; "" for a whole-array result. +pub const Fields = std.StringArrayHashMapUnmanaged(FieldStat); + +arena: std.heap.ArenaAllocator, +fields: Fields = .empty, + +pub fn init(allocator: std.mem.Allocator) Baseline { + return .{ .arena = .init(allocator) }; +} + +pub fn deinit(self: *Baseline) void { + self.arena.deinit(); +} + +pub fn reset(self: *Baseline) void { + self.fields = .empty; + _ = self.arena.reset(.retain_capacity); +} + +/// Tally one successful extract result (the tool's JSON output). Malformed +/// output records nothing — this is best-effort telemetry. +pub fn noteExtractResult(self: *Baseline, result_text: []const u8) error{OutOfMemory}!void { + var scratch: std.heap.ArenaAllocator = .init(self.arena.child_allocator); + defer scratch.deinit(); + const parsed = std.json.parseFromSliceLeaky(std.json.Value, scratch.allocator(), result_text, .{}) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return, + }; + switch (parsed) { + .array => try self.bump("", !ScriptRuntime.jsonIsEmpty(parsed)), + .object => |obj| { + var it = obj.iterator(); + while (it.next()) |entry| { + try self.bump(entry.key_ptr.*, !ScriptRuntime.jsonIsEmpty(entry.value_ptr.*)); + } + }, + else => {}, + } +} + +fn bump(self: *Baseline, field: []const u8, nonempty: bool) error{OutOfMemory}!void { + const arena = self.arena.allocator(); + const gop = try self.fields.getOrPut(arena, field); + if (!gop.found_existing) { + // The probe key may live in caller scratch; own it. + gop.key_ptr.* = try arena.dupe(u8, field); + gop.value_ptr.* = .{}; + } + gop.value_ptr.calls += 1; + if (nonempty) gop.value_ptr.nonempty += 1; +} + +/// The session's baseline as a full `// lp:baseline {...}` line, or null when +/// no extract ran. +pub fn serialize(self: *const Baseline, arena: std.mem.Allocator) error{OutOfMemory}!?[]const u8 { + return fieldsToLine(arena, &self.fields); +} + +/// Baseline line from a script run's extract stats (per-field totals folded +/// across schemas), for refreshing a healed script from its validation run. +pub fn serializeStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) error{OutOfMemory}!?[]const u8 { + var fields: Fields = .empty; + for (stats) |stat| { + const gop = try fields.getOrPut(arena, stat.field orelse ""); + if (!gop.found_existing) gop.value_ptr.* = .{}; + gop.value_ptr.calls += stat.calls; + gop.value_ptr.nonempty += stat.calls - stat.empty; + } + return fieldsToLine(arena, &fields); +} + +fn fieldsToLine(arena: std.mem.Allocator, fields: *const Fields) error{OutOfMemory}!?[]const u8 { + if (fields.count() == 0) return null; + var fields_obj: std.json.ObjectMap = .init(arena); + var it = fields.iterator(); + while (it.next()) |entry| { + var stat_obj: std.json.ObjectMap = .init(arena); + try stat_obj.put("calls", .{ .integer = entry.value_ptr.calls }); + try stat_obj.put("nonempty", .{ .integer = entry.value_ptr.nonempty }); + try fields_obj.put(entry.key_ptr.*, .{ .object = stat_obj }); + } + var root: std.json.ObjectMap = .init(arena); + try root.put("fields", .{ .object = fields_obj }); + const json = try std.json.Stringify.valueAlloc(arena, std.json.Value{ .object = root }, .{}); + return try std.mem.concat(arena, u8, &.{ marker, json }); +} + +/// Parse the baseline comment out of script source; null when absent or +/// malformed (a hand-written script simply has none). Field-name strings may +/// reference `source` — keep it alive as long as the result. +pub fn parse(arena: std.mem.Allocator, source: []const u8) ?Fields { + var lines = std.mem.splitScalar(u8, source, '\n'); + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, " \t\r"); + if (!std.mem.startsWith(u8, trimmed, marker)) continue; + return parseJson(arena, trimmed[marker.len..]); + } + return null; +} + +fn parseJson(arena: std.mem.Allocator, json_text: []const u8) ?Fields { + const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, json_text, .{}) catch return null; + if (parsed != .object) return null; + const fields_value = parsed.object.get("fields") orelse return null; + if (fields_value != .object) return null; + + var out: Fields = .empty; + var it = fields_value.object.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.* != .object) return null; + const calls = intField(entry.value_ptr.object, "calls") orelse return null; + const nonempty = intField(entry.value_ptr.object, "nonempty") orelse return null; + out.put(arena, entry.key_ptr.*, .{ .calls = calls, .nonempty = nonempty }) catch return null; + } + return out; +} + +fn intField(obj: std.json.ObjectMap, key: []const u8) ?u32 { + const value = obj.get(key) orelse return null; + if (value != .integer) return null; + if (value.integer < 0 or value.integer > std.math.maxInt(u32)) return null; + return @intCast(value.integer); +} + +/// `script` with any existing baseline lines dropped and `line` (a full +/// baseline line, or null) appended — synthesis may have copied a stale +/// baseline from the previous script verbatim. +pub fn withBaseline(arena: std.mem.Allocator, script: []const u8, line: ?[]const u8) error{OutOfMemory}![]const u8 { + var aw: std.Io.Writer.Allocating = .init(arena); + var lines = std.mem.splitScalar(u8, script, '\n'); + var pending_newline = false; + while (lines.next()) |script_line| { + if (std.mem.startsWith(u8, std.mem.trim(u8, script_line, " \t\r"), marker)) continue; + if (pending_newline) aw.writer.writeByte('\n') catch return error.OutOfMemory; + aw.writer.writeAll(script_line) catch return error.OutOfMemory; + pending_newline = true; + } + if (line) |l| { + // A split of "a\nb\n" ends with an empty segment, so the writer is + // already newline-terminated in the common case. + if (aw.written().len != 0 and aw.written()[aw.written().len - 1] != '\n') { + aw.writer.writeByte('\n') catch return error.OutOfMemory; + } + aw.writer.writeAll(l) catch return error.OutOfMemory; + aw.writer.writeByte('\n') catch return error.OutOfMemory; + } + return aw.written(); +} + +test "baseline: note, serialize, parse round-trip" { + var baseline: Baseline = .init(std.testing.allocator); + defer baseline.deinit(); + + try baseline.noteExtractResult("{\"stories\":[{\"title\":\"a\"}],\"empty_list\":[],\"scalar\":\"x\"}"); + try baseline.noteExtractResult("{\"stories\":[],\"empty_list\":[]}"); + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const line = (try baseline.serialize(arena.allocator())).?; + try std.testing.expect(std.mem.startsWith(u8, line, marker)); + + const parsed = Baseline.parse(arena.allocator(), line).?; + try std.testing.expectEqual(FieldStat{ .calls = 2, .nonempty = 1 }, parsed.get("stories").?); + try std.testing.expectEqual(FieldStat{ .calls = 2, .nonempty = 0 }, parsed.get("empty_list").?); + try std.testing.expectEqual(FieldStat{ .calls = 1, .nonempty = 1 }, parsed.get("scalar").?); + + // Malformed results record nothing; malformed baselines parse to null. + try baseline.noteExtractResult("not json"); + try std.testing.expectEqual(null, Baseline.parse(arena.allocator(), "// lp:baseline {broken")); + try std.testing.expectEqual(null, Baseline.parse(arena.allocator(), "const x = 1;")); +} + +test "baseline: withBaseline strips stale lines and appends" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const script = "const x = 1;\n// lp:baseline {\"fields\":{}}\nreturn x;\n"; + const out = try withBaseline(aa, script, marker ++ "{\"fields\":{\"a\":{\"calls\":1,\"nonempty\":1}}}"); + try std.testing.expectEqualStrings( + "const x = 1;\nreturn x;\n" ++ marker ++ "{\"fields\":{\"a\":{\"calls\":1,\"nonempty\":1}}}\n", + out, + ); + + const stripped = try withBaseline(aa, script, null); + try std.testing.expectEqualStrings("const x = 1;\nreturn x;\n", stripped); +} diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index 6efcc20bb9..8fd9903916 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -274,13 +274,17 @@ pub const Completion = struct { empty: bool, }; -/// One extract schema's top-level list field, tallied across the run. +/// One extract schema's top-level result field, tallied across the run. pub const ExtractStat = struct { /// Schema JSON as the script wrote it (array schemas are shown without the /// internal `__root` wrapper). schema: []const u8, - /// Top-level list field of the result; null when the schema itself is a list. + /// Top-level field of the result; null when the schema itself is a list. field: ?[]const u8, + /// The field's value was an array on some call. Consumers gate the + /// no-baseline heal policy on this: lists signal collection intent, + /// scalars are legitimately sparse. + is_list: bool, calls: u32, empty: u32, }; @@ -430,7 +434,7 @@ fn isEmptyValue(self: *Runtime, value: *const v8.Value, text: []const u8) bool { /// Null, "", and arrays/objects all of whose members are empty carry no data; /// numbers and booleans always do (0 and false are answers). -fn jsonIsEmpty(value: std.json.Value) bool { +pub fn jsonIsEmpty(value: std.json.Value) bool { switch (value) { .null => return true, .string => |s| return s.len == 0, @@ -565,10 +569,9 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf } } -/// Tally each top-level list field of an extract result: a field that stays -/// empty across every call of its schema is the heal trigger's signal, while -/// scalar fields (legitimately sparse) are never tracked. A whole-array result -/// (`__root` schema) tallies as the single null field. +/// Tally each top-level field of an extract result; the Agent's heal policy +/// decides which stats can trigger. A whole-array result (`__root` schema) +/// tallies as the single null field. fn recordExtractStats(self: *Runtime, args: ?std.json.Value, result: std.json.Value) error{OutOfMemory}!void { const raw_schema = switch ((args orelse return).object.get("schema") orelse return) { .string => |s| s, @@ -576,24 +579,24 @@ fn recordExtractStats(self: *Runtime, args: ?std.json.Value, result: std.json.Va }; const schema = stripExtractSchemaRoot(raw_schema); switch (result) { - .array => try self.bumpExtractStat(schema, null, jsonIsEmpty(result)), + .array => try self.bumpExtractStat(schema, null, true, jsonIsEmpty(result)), .object => |obj| { var it = obj.iterator(); while (it.next()) |entry| { - if (entry.value_ptr.* != .array) continue; - try self.bumpExtractStat(schema, entry.key_ptr.*, jsonIsEmpty(entry.value_ptr.*)); + try self.bumpExtractStat(schema, entry.key_ptr.*, entry.value_ptr.* == .array, jsonIsEmpty(entry.value_ptr.*)); } }, else => {}, } } -fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_empty: bool) error{OutOfMemory}!void { +fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_list: bool, is_empty: bool) error{OutOfMemory}!void { for (self.extract_stats.items) |*stat| { if (!std.mem.eql(u8, stat.schema, schema)) continue; const same_field = if (stat.field) |f| field != null and std.mem.eql(u8, f, field.?) else field == null; if (!same_field) continue; stat.calls += 1; + if (is_list) stat.is_list = true; if (is_empty) stat.empty += 1; return; } @@ -602,6 +605,7 @@ fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_em try self.extract_stats.append(arena, .{ .schema = try arena.dupe(u8, schema), .field = if (field) |f| try arena.dupe(u8, f) else null, + .is_list = is_list, .calls = 1, .empty = @intFromBool(is_empty), }); @@ -1736,26 +1740,33 @@ test "agent script runtime: extract stats tally list-field emptiness" { , "agent-runtime-extract-stats.js"); const stats = result.ok.extract_stats; - try testing.expectEqual(4, stats.len); + try testing.expectEqual(5, stats.len); - // Scalar `btn` never records a stat. try testing.expectString("items", stats[0].field.?); + try testing.expectEqual(true, stats[0].is_list); try testing.expectEqual(2, stats[0].calls); try testing.expectEqual(2, stats[0].empty); - try testing.expectString("buttons", stats[1].field.?); + try testing.expectString("btn", stats[1].field.?); + try testing.expectEqual(false, stats[1].is_list); try testing.expectEqual(1, stats[1].calls); try testing.expectEqual(0, stats[1].empty); - try testing.expectEqual(null, stats[2].field); - try testing.expectString("[\".no-such-root\"]", stats[2].schema); + try testing.expectString("buttons", stats[2].field.?); + try testing.expectEqual(true, stats[2].is_list); try testing.expectEqual(1, stats[2].calls); - try testing.expectEqual(1, stats[2].empty); + try testing.expectEqual(0, stats[2].empty); - try testing.expectString("sel", stats[3].field.?); - try testing.expectEqual(2, stats[3].calls); + try testing.expectEqual(null, stats[3].field); + try testing.expectEqual(true, stats[3].is_list); + try testing.expectString("[\".no-such-root\"]", stats[3].schema); + try testing.expectEqual(1, stats[3].calls); try testing.expectEqual(1, stats[3].empty); + try testing.expectString("sel", stats[4].field.?); + try testing.expectEqual(2, stats[4].calls); + try testing.expectEqual(1, stats[4].empty); + const rerun = try runtime.runSource("return 1;", "agent-runtime-extract-stats.js"); try testing.expectEqual(0, rerun.ok.extract_stats.len); } From 3cd1e76a305fdb67e2bd342699f6ea2cdfe9461e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 8 Jul 2026 14:43:35 +0200 Subject: [PATCH 05/15] refactor: unify extract field classification - Add `classifyExtractFields` to share parsing between Baseline and Runtime. - Reuse a scratch arena allocator in Baseline to avoid overhead. - Simplify the retry loop in `Agent.runScriptWithHeal`. --- src/agent/Agent.zig | 43 ++++++++++++++---------------- src/agent/Baseline.zig | 35 ++++++++++++------------- src/script/Runtime.zig | 59 +++++++++++++++++++++++++++++++----------- 3 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index b5ac122a4c..12c884b0b4 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -1751,24 +1751,25 @@ fn runScriptWithHeal(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); - switch (self.runScriptOutcome(arena.allocator(), path)) { - .ok => return true, - .fatal => return false, - .script_error => {}, - } - - // A re-run is free; healing is not. One retry filters transient failures - // (a flaky page load) before the model is asked to fix a correct script. - self.terminal.printInfo("Script failed; retrying once before healing.", .{}); - switch (self.runScriptOutcome(arena.allocator(), path)) { - .ok => return true, - .fatal => return false, - .script_error => |script_error| { - // Healing spends tokens; report the cost the way --task does. - defer self.printUsageSummary(); - return self.healLoop(arena.allocator(), path, script_error); - }, + for (0..2) |attempt| { + switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok => return true, + .fatal => return false, + .script_error => |script_error| { + if (attempt == 0) { + // A re-run is free; healing is not — one retry filters + // transient failures (a flaky page load) before the model + // is asked to fix a correct script. + self.terminal.printInfo("Script failed; retrying once before healing.", .{}); + continue; + } + // Healing spends tokens; report the cost the way --task does. + defer self.printUsageSummary(); + return self.healLoop(arena.allocator(), path, script_error); + }, + } } + unreachable; } const max_heal_attempts = 2; @@ -1903,11 +1904,7 @@ fn cureFailure(arena: std.mem.Allocator, first: ScriptError, facts: RunFacts) er .dry_extracts => { for (first.dry_fields) |dry| { const cured = for (facts.extract_stats) |stat| { - const same = if (dry) |d| - stat.field != null and std.mem.eql(u8, stat.field.?, d) - else - stat.field == null; - if (same and stat.empty < stat.calls) break true; + if (ScriptRuntime.fieldEql(stat.field, dry) and stat.empty < stat.calls) break true; } else false; if (!cured) return try std.fmt.allocPrint(arena, "The revised script ran, but the \"{s}\" extract still came back empty on every call (or was removed) — keep it and fix its selector.", .{dry orelse ""}); } @@ -2241,7 +2238,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments)) |result| blk: { // Pre-cap text: a truncated result would parse as malformed and // record nothing. - if (std.mem.eql(u8, tool_name, "extract") and !result.is_error) { + if (std.mem.eql(u8, tool_name, @tagName(BrowserTool.extract)) and !result.is_error) { self.baseline.noteExtractResult(result.text) catch {}; } break :blk .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error }; diff --git a/src/agent/Baseline.zig b/src/agent/Baseline.zig index 606e49e7f7..0329ce7f95 100644 --- a/src/agent/Baseline.zig +++ b/src/agent/Baseline.zig @@ -38,14 +38,18 @@ pub const FieldStat = struct { pub const Fields = std.StringArrayHashMapUnmanaged(FieldStat); arena: std.heap.ArenaAllocator, +/// Reused per `noteExtractResult` for the transient parse tree; keeps pages +/// warm on the every-tool-call path instead of an init/deinit pair each time. +scratch: std.heap.ArenaAllocator, fields: Fields = .empty, pub fn init(allocator: std.mem.Allocator) Baseline { - return .{ .arena = .init(allocator) }; + return .{ .arena = .init(allocator), .scratch = .init(allocator) }; } pub fn deinit(self: *Baseline) void { self.arena.deinit(); + self.scratch.deinit(); } pub fn reset(self: *Baseline) void { @@ -56,21 +60,14 @@ pub fn reset(self: *Baseline) void { /// Tally one successful extract result (the tool's JSON output). Malformed /// output records nothing — this is best-effort telemetry. pub fn noteExtractResult(self: *Baseline, result_text: []const u8) error{OutOfMemory}!void { - var scratch: std.heap.ArenaAllocator = .init(self.arena.child_allocator); - defer scratch.deinit(); - const parsed = std.json.parseFromSliceLeaky(std.json.Value, scratch.allocator(), result_text, .{}) catch |err| switch (err) { + _ = self.scratch.reset(.retain_capacity); + const scratch = self.scratch.allocator(); + const parsed = std.json.parseFromSliceLeaky(std.json.Value, scratch, result_text, .{}) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return, }; - switch (parsed) { - .array => try self.bump("", !ScriptRuntime.jsonIsEmpty(parsed)), - .object => |obj| { - var it = obj.iterator(); - while (it.next()) |entry| { - try self.bump(entry.key_ptr.*, !ScriptRuntime.jsonIsEmpty(entry.value_ptr.*)); - } - }, - else => {}, + for (try ScriptRuntime.classifyExtractFields(scratch, parsed)) |fc| { + try self.bump(fc.field orelse "", !fc.empty); } } @@ -161,24 +158,24 @@ fn intField(obj: std.json.ObjectMap, key: []const u8) ?u32 { /// `script` with any existing baseline lines dropped and `line` (a full /// baseline line, or null) appended — synthesis may have copied a stale /// baseline from the previous script verbatim. -pub fn withBaseline(arena: std.mem.Allocator, script: []const u8, line: ?[]const u8) error{OutOfMemory}![]const u8 { +pub fn withBaseline(arena: std.mem.Allocator, script: []const u8, line: ?[]const u8) ![]const u8 { var aw: std.Io.Writer.Allocating = .init(arena); var lines = std.mem.splitScalar(u8, script, '\n'); var pending_newline = false; while (lines.next()) |script_line| { if (std.mem.startsWith(u8, std.mem.trim(u8, script_line, " \t\r"), marker)) continue; - if (pending_newline) aw.writer.writeByte('\n') catch return error.OutOfMemory; - aw.writer.writeAll(script_line) catch return error.OutOfMemory; + if (pending_newline) try aw.writer.writeByte('\n'); + try aw.writer.writeAll(script_line); pending_newline = true; } if (line) |l| { // A split of "a\nb\n" ends with an empty segment, so the writer is // already newline-terminated in the common case. if (aw.written().len != 0 and aw.written()[aw.written().len - 1] != '\n') { - aw.writer.writeByte('\n') catch return error.OutOfMemory; + try aw.writer.writeByte('\n'); } - aw.writer.writeAll(l) catch return error.OutOfMemory; - aw.writer.writeByte('\n') catch return error.OutOfMemory; + try aw.writer.writeAll(l); + try aw.writer.writeByte('\n'); } return aw.written(); } diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index 8fd9903916..a160be0f63 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -274,6 +274,44 @@ pub const Completion = struct { empty: bool, }; +/// One top-level field of a parsed extract result: an object yields an entry +/// per key, an array (a `__root` schema) the single null field. Slices +/// reference the parsed value. +pub const ExtractField = struct { + field: ?[]const u8, + is_list: bool, + empty: bool, +}; + +pub fn classifyExtractFields(arena: std.mem.Allocator, result: std.json.Value) error{OutOfMemory}![]const ExtractField { + switch (result) { + .array => { + const out = try arena.alloc(ExtractField, 1); + out[0] = .{ .field = null, .is_list = true, .empty = jsonIsEmpty(result) }; + return out; + }, + .object => |obj| { + const out = try arena.alloc(ExtractField, obj.count()); + var it = obj.iterator(); + var i: usize = 0; + while (it.next()) |entry| : (i += 1) { + out[i] = .{ + .field = entry.key_ptr.*, + .is_list = entry.value_ptr.* == .array, + .empty = jsonIsEmpty(entry.value_ptr.*), + }; + } + return out; + }, + else => return &.{}, + } +} + +pub fn fieldEql(a: ?[]const u8, b: ?[]const u8) bool { + if (a) |x| return b != null and std.mem.eql(u8, x, b.?); + return b == null; +} + /// One extract schema's top-level result field, tallied across the run. pub const ExtractStat = struct { /// Schema JSON as the script wrote it (array schemas are shown without the @@ -558,7 +596,7 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf // inner-then-outer; the map is append-only. Failed extracts // threw above and are intentionally not counted. if (normalized.parsed) |parsed| { - self.recordExtractStats(args, parsed) catch + self.recordExtractStats(arena, args, parsed) catch return self.throwError("out of memory"); } self.setReturnJson(context, info, normalized.text); @@ -570,31 +608,22 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf } /// Tally each top-level field of an extract result; the Agent's heal policy -/// decides which stats can trigger. A whole-array result (`__root` schema) -/// tallies as the single null field. -fn recordExtractStats(self: *Runtime, args: ?std.json.Value, result: std.json.Value) error{OutOfMemory}!void { +/// decides which stats can trigger. +fn recordExtractStats(self: *Runtime, arena: std.mem.Allocator, args: ?std.json.Value, result: std.json.Value) error{OutOfMemory}!void { const raw_schema = switch ((args orelse return).object.get("schema") orelse return) { .string => |s| s, else => return, }; const schema = stripExtractSchemaRoot(raw_schema); - switch (result) { - .array => try self.bumpExtractStat(schema, null, true, jsonIsEmpty(result)), - .object => |obj| { - var it = obj.iterator(); - while (it.next()) |entry| { - try self.bumpExtractStat(schema, entry.key_ptr.*, entry.value_ptr.* == .array, jsonIsEmpty(entry.value_ptr.*)); - } - }, - else => {}, + for (try classifyExtractFields(arena, result)) |fc| { + try self.bumpExtractStat(schema, fc.field, fc.is_list, fc.empty); } } fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_list: bool, is_empty: bool) error{OutOfMemory}!void { for (self.extract_stats.items) |*stat| { if (!std.mem.eql(u8, stat.schema, schema)) continue; - const same_field = if (stat.field) |f| field != null and std.mem.eql(u8, f, field.?) else field == null; - if (!same_field) continue; + if (!fieldEql(stat.field, field)) continue; stat.calls += 1; if (is_list) stat.is_list = true; if (is_empty) stat.empty += 1; From dc768aaa70d0db0f3a1e296c1511673b548024c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 8 Jul 2026 15:08:01 +0200 Subject: [PATCH 06/15] feat(agent): use LLM verdict to judge empty script outputs Replaces hardcoded heuristics and programmatic baseline parsing with an LLM turn that judges if empty outputs or dry extracts indicate a broken script or legitimate sparseness. Extracts a `metaTurn` helper to share out-of-conversation LLM execution logic. --- src/agent/Agent.zig | 409 +++++++++++++++++++++++++---------------- src/agent/Baseline.zig | 63 ++----- 2 files changed, 259 insertions(+), 213 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 12c884b0b4..68e1f7ec8b 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -863,18 +863,18 @@ fn handleLoad(self: *Agent, rest: []const u8) void { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); while (true) { - switch (self.runScriptOutcome(arena.allocator(), path)) { - .ok, .fatal => return, - .script_error => |script_error| { - if (self.ai_client == null) return; - switch (promptHeal(path, script_error.kind)) { - .no => return, - .retry => {}, - .heal => { - _ = self.healLoop(arena.allocator(), path, script_error); - return; - }, - } + const finding: ScriptError = switch (self.runScriptOutcome(arena.allocator(), path)) { + .fatal => return, + .ok => |facts| self.judgedFinding(arena.allocator(), path, facts) orelse return, + .script_error => |script_error| script_error, + }; + if (self.ai_client == null) return; + switch (promptHeal(path, finding.kind)) { + .no => return, + .retry => {}, + .heal => { + _ = self.healLoop(arena.allocator(), path, finding); + return; }, } } @@ -1202,12 +1202,62 @@ fn failSynthesis(self: *Agent, label: []const u8, reason: []const u8) ?[]const u return null; } -/// Roll the in-flight synthesis turn back out of the conversation, then report -/// the failure — so a doomed `/save` or heal synthesis never leaks its messages -/// into history. -fn abortSynthesis(self: *Agent, label: []const u8, baseline: usize, reason: []const u8) ?[]const u8 { - self.conversation.rollback(baseline); - return self.failSynthesis(label, reason); +/// One out-of-conversation LLM turn — the transport shared by `/save`/heal +/// synthesis and the replay verdict: swap `system` in for this turn, send +/// `user_msg`, and return the response text duped into `arena`. The turn is +/// always rolled back out of history; null (with a `label`-tagged report) on +/// error, no text, or cancellation. +fn metaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: []const u8, user_msg: []const u8, max_tokens: i32, effort: Config.Effort) ?[]const u8 { + const provider_client = self.ai_client.?; + self.conversation.ensureSystemPrompt() catch return self.failSynthesis(label, "out of memory"); + + // Regular turns keep the driver prompt. (`messages[0]` is the system + // message — rollback and prune never touch it.) + const plain_system = self.conversation.messages.items[0].content; + self.conversation.messages.items[0].content = system; + defer self.conversation.messages.items[0].content = plain_system; + + const baseline = self.conversation.messages.items.len; + self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); + defer self.conversation.rollback(baseline); + + self.http_interrupt.reset(); + self.terminal.spinner.start(); + var result = provider_client.runTools( + self.model, + &self.conversation.messages, + self.allocator, + self.conversation.arena.allocator(), + .{ .context = @ptrCast(self), .callFn = handleToolCall }, + .{ + .tools = &.{}, + .max_turns = 1, + .max_tokens = max_tokens, + .tool_choice = .none, + .effort = effort, + .cancel = .{ .context = @ptrCast(self), .checkFn = checkCancel }, + }, + ) catch |err| { + self.terminal.spinner.cancel(); + if (self.cancel_requested.load(.acquire)) { + self.resetAfterCancel(baseline); + return null; + } + log.err(.app, "AI meta-turn error", .{ .label = label, .err = err }); + return self.failSynthesis(label, @errorName(err)); + }; + self.terminal.spinner.stop(); + defer result.deinit(); + self.total_usage.add(result.usage); + if (result.cancelled) { + self.resetAfterCancel(baseline); + return null; + } + + // `result.text` lives in the conversation arena, freed by the deferred + // rollback; the dupe happens before defers run. + const raw = result.text orelse return self.failSynthesis(label, "the model returned no text"); + return arena.dupe(u8, raw) catch self.failSynthesis(label, "out of memory"); } /// Save synthesis warrants more reasoning than a normal turn. `.none` stays off @@ -1283,68 +1333,12 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo /// the failure under `label` and returns null on any error or cancellation; /// the synthesis turn never stays in history. fn synthesizeScriptText(self: *Agent, arena: std.mem.Allocator, label: []const u8, path: []const u8, previous_script: ?[]const u8, prompt: ?[]const u8) ?[]const u8 { - const provider_client = self.ai_client.?; - - self.conversation.ensureSystemPrompt() catch return self.failSynthesis(label, "out of memory"); - - // Swap the dedicated save_system_prompt in as the system prompt for this one turn; - // regular turns keep the driver prompt. (`messages[0]` is the system - // message — rollback and prune never touch it.) - const plain_system = self.conversation.messages.items[0].content; - self.conversation.messages.items[0].content = if (previous_script != null) save_revision_system_prompt else save_system_prompt; - defer self.conversation.messages.items[0].content = plain_system; - - const ma = self.conversation.arena.allocator(); - const baseline = self.conversation.messages.items.len; - - const user_msg = self.buildSaveSynthesisMessage(ma, path, previous_script, prompt) catch return self.failSynthesis(label, "out of memory"); - self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); - - self.http_interrupt.reset(); - self.terminal.spinner.start(); - var result = provider_client.runTools( - self.model, - &self.conversation.messages, - self.allocator, - ma, - .{ .context = @ptrCast(self), .callFn = handleToolCall }, - .{ - .tools = &.{}, - .max_turns = 1, - .max_tokens = 8192, - .tool_choice = .none, - .effort = bumpedEffort(self.effort), - .cancel = .{ .context = @ptrCast(self), .checkFn = checkCancel }, - }, - ) catch |err| { - self.terminal.spinner.cancel(); - if (self.cancel_requested.load(.acquire)) { - self.resetAfterCancel(baseline); - return null; - } - log.err(.app, "AI synthesis error", .{ .label = label, .err = err }); - return self.abortSynthesis(label, baseline, @errorName(err)); - }; - self.terminal.spinner.stop(); - defer result.deinit(); - self.total_usage.add(result.usage); - - if (result.cancelled) { - self.resetAfterCancel(baseline); - return null; - } - - const raw = result.text orelse return self.abortSynthesis(label, baseline, "the model returned no script"); - - // `result.text` lives in the conversation arena, freed by the rollback - // below; copy into the caller's arena first (scrubbing may return its input - // as-is). - const owned = arena.dupe(u8, save.stripCodeFence(raw)) catch return self.abortSynthesis(label, baseline, "out of memory"); - const script = browser_tools.reverseSubstituteEnvVars(arena, owned) catch return self.abortSynthesis(label, baseline, "out of memory"); - - // The synthesis turn is a meta-action; keep it out of the ongoing conversation. - self.conversation.rollback(baseline); - return script; + const user_msg = self.buildSaveSynthesisMessage(self.conversation.arena.allocator(), path, previous_script, prompt) catch + return self.failSynthesis(label, "out of memory"); + const system = if (previous_script != null) save_revision_system_prompt else save_system_prompt; + const raw = self.metaTurn(arena, label, system, user_msg, 8192, bumpedEffort(self.effort)) orelse return null; + return browser_tools.reverseSubstituteEnvVars(arena, save.stripCodeFence(raw)) catch + return self.failSynthesis(label, "out of memory"); } /// Persist `path` as the destination reused by a subsequent bare `/save`. @@ -1555,12 +1549,17 @@ const ScriptRunOutcome = union(enum) { script_error: ScriptError, }; -/// Facts about a passing run, for heal validation's cure check. Duped into the -/// caller's arena — the runtime dies with `runScriptOutcome`. +/// Facts about a run that completed without throwing — suspicion is judged by +/// the model, never here. Duped into the caller's arena — the runtime dies +/// with `runScriptOutcome`. const RunFacts = struct { /// The script returned a value that carries data. returned_data: bool, + /// The returned value when it was deep-empty (capped display text). + empty_completion: ?[]const u8, extract_stats: []const ScriptRuntime.ExtractStat, + /// The exact text that ran. + source: []const u8, }; /// Both slices are duped into the caller's arena. `source` is the exact text @@ -1590,8 +1589,7 @@ fn runScript(self: *Agent, path: []const u8) bool { defer arena.deinit(); return switch (self.runScriptOutcome(arena.allocator(), path)) { .ok => true, - .fatal => false, - .script_error => |script_error| script_error.kind != .threw, + .fatal, .script_error => false, }; } @@ -1603,7 +1601,6 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc self.terminal.printError("Failed to read script '{s}': {s}", .{ path, @errorName(err) }); return .fatal; }; - const baseline = Baseline.parse(script_arena.allocator(), content); const runtime = ScriptRuntime.init(self.allocator, self.browser.app, self.session, &self.node_registry) catch |err| { self.terminal.printError("Failed to initialize script runtime: {s}", .{@errorName(err)}); @@ -1644,38 +1641,21 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc .source = arena.dupe(u8, content) catch return .fatal, } }; }, - .ok => |ok| blk: { - if (ok.completion) |completion| { - if (completion.empty) { - const value = capDetail(arena, completion.text) catch return .fatal; - return .{ .script_error = .{ - .kind = .empty, - .detail = std.fmt.allocPrint(arena, "The script completed without throwing, but its return value carries no data: {s}\n" ++ - "A selector probably no longer matches anything on the page.", .{value}) catch return .fatal, - .source = arena.dupe(u8, content) catch return .fatal, - } }; - } - } - // Checked even without a completion: a no-`return` script can - // still have dry extracts. - if (dryExtractsFinding(arena, ok.extract_stats, baseline) catch return .fatal) |finding| { - return .{ .script_error = .{ - .kind = .dry_extracts, - .detail = finding.detail, - .source = arena.dupe(u8, content) catch return .fatal, - .dry_fields = finding.fields, - } }; - } - break :blk ok; - }, + .ok => |ok| ok, }; // A script that printed nothing leaves no trace, so freeze the spinner into // a green bullet (like /goto); one that printed already showed its result. if (!output.emitted) self.terminal.printScriptDone("script", path); + const empty_completion: ?[]const u8 = if (ok.completion) |c| + (if (c.empty) capDetail(arena, c.text) catch return .fatal else null) + else + null; return .{ .ok = .{ - .returned_data = ok.completion != null, + .returned_data = if (ok.completion) |c| !c.empty else false, + .empty_completion = empty_completion, .extract_stats = dupeExtractStats(arena, ok.extract_stats) catch return .fatal, + .source = arena.dupe(u8, content) catch return .fatal, } }; } @@ -1702,22 +1682,40 @@ fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]co return std.mem.concat(arena, u8, &.{ string.truncateUtf8(text, detail_max_bytes), "…[truncated]" }); } +/// Facts worth a verdict, not yet a finding: the return value was deep-empty, +/// or some extract field came back empty on every call — any field, scalar or +/// list, baseline or not. Whether that is breakage or legitimate sparseness +/// is the model's judgment, not encoded here. +const Suspicion = struct { + kind: ScriptError.Kind, + detail: []const u8, + dry_fields: []const ?[]const u8, +}; + +fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?Suspicion { + if (facts.empty_completion) |text| return .{ + .kind = .empty, + .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, + .dry_fields = &.{}, + }; + const finding = (dryExtractsFinding(arena, facts.extract_stats) catch return null) orelse return null; + return .{ .kind = .dry_extracts, .detail = finding.detail, .dry_fields = finding.fields }; +} + const DryFinding = struct { detail: []const u8, fields: []const ?[]const u8, }; -/// The `.dry_extracts` finding: one detail line per extract field that came -/// back empty on every call, plus the field names for the cure check. -/// Null when no field was dry. -fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat, baseline: ?Baseline.Fields) !?DryFinding { +/// One detail line per extract field that came back empty on every call, plus +/// the field names for the cure check. Null when no field was dry. +fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) !?DryFinding { var aw: std.Io.Writer.Allocating = .init(arena); var fields: std.ArrayList(?[]const u8) = .empty; for (stats) |stat| { if (stat.empty != stat.calls) continue; - if (!dryCanTrigger(stat, baseline)) continue; if (fields.items.len == 0) { - try aw.writer.writeAll("The script completed without throwing, but some extracts came back empty on every call:\n"); + try aw.writer.writeAll("some extracts came back empty on every call:\n"); } try fields.append(arena, if (stat.field) |f| try arena.dupe(u8, f) else null); const schema = try capDetail(arena, stat.schema); @@ -1731,43 +1729,30 @@ fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.Ext try aw.writer.writeAll("\n"); } if (fields.items.len == 0) return null; - try aw.writer.writeAll("A selector probably no longer matches the page — but an empty result can also be legitimate; verify against the live page before changing anything."); return .{ .detail = aw.written(), .fields = fields.items }; } -/// Whether an all-empty stat is heal-worthy. With a baseline, record-time -/// reality decides: a field that carried data then is a finding now (scalars -/// included), one that was already empty then is legitimately sparse. Without -/// one, only list fields qualify — lists signal collection intent. -fn dryCanTrigger(stat: ScriptRuntime.ExtractStat, baseline: ?Baseline.Fields) bool { - if (baseline) |b| { - if (b.get(stat.field orelse "")) |base| return base.nonempty > 0; - } - return stat.is_list; -} - /// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. fn runScriptWithHeal(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); for (0..2) |attempt| { - switch (self.runScriptOutcome(arena.allocator(), path)) { - .ok => return true, + const finding: ScriptError = switch (self.runScriptOutcome(arena.allocator(), path)) { .fatal => return false, - .script_error => |script_error| { - if (attempt == 0) { - // A re-run is free; healing is not — one retry filters - // transient failures (a flaky page load) before the model - // is asked to fix a correct script. - self.terminal.printInfo("Script failed; retrying once before healing.", .{}); - continue; - } - // Healing spends tokens; report the cost the way --task does. - defer self.printUsageSummary(); - return self.healLoop(arena.allocator(), path, script_error); - }, + .ok => |facts| self.judgedFinding(arena.allocator(), path, facts) orelse return true, + .script_error => |script_error| script_error, + }; + if (attempt == 0) { + // A re-run is free; healing is not — one retry filters transient + // failures (a flaky page load) before the model is asked to fix + // a correct script. + self.terminal.printInfo("Script run failed or looks broken; retrying once before healing.", .{}); + continue; } + // Healing spends tokens; report the cost the way --task does. + defer self.printUsageSummary(); + return self.healLoop(arena.allocator(), path, finding); } unreachable; } @@ -1890,6 +1875,87 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr return false; } +const verdict_system_prompt = + \\You judge replays of saved browser-automation scripts. Given the script + \\and facts about a run that completed without errors, decide whether the + \\empty output means the script is broken (stale selectors after a site + \\change) or legitimate (the page genuinely has no such data right now). + \\A `// lp:baseline` comment, when present, records how often each output + \\field carried data when the script was saved — weigh it as evidence. + \\Respond with JSON only: + \\{"broken": true|false, "fields": ["", ...], "reason": ""} + \\`fields` names the output fields you judge broken; use "" for the whole + \\return value. +; + +const Verdict = struct { + broken: bool, + /// Output fields the model judged broken; null = the whole return value. + fields: []const ?[]const u8, + reason: []const u8, +}; + +/// One tool-free LLM turn deciding whether suspicious facts are breakage. +/// Null on transport/parse failure or cancellation — the caller falls back to +/// surfacing the facts instead of guessing. +fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, source: []const u8, suspicion: Suspicion) ?Verdict { + const user_msg = std.fmt.allocPrint(self.conversation.arena.allocator(), + \\Replay of {s} completed without errors, but {s} + \\ + \\The script (its comments and structure carry the intent): + \\```js + \\{s} + \\``` + , .{ path, suspicion.detail, source }) catch return null; + const raw = self.metaTurn(arena, "verdict", verdict_system_prompt, user_msg, 1024, self.effort) orelse return null; + return parseVerdict(arena, raw); +} + +fn parseVerdict(arena: std.mem.Allocator, raw: []const u8) ?Verdict { + const start = std.mem.indexOfScalar(u8, raw, '{') orelse return null; + const end = std.mem.lastIndexOfScalar(u8, raw, '}') orelse return null; + if (end < start) return null; + const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, raw[start .. end + 1], .{}) catch return null; + if (parsed != .object) return null; + const broken = parsed.object.get("broken") orelse return null; + if (broken != .bool) return null; + + var fields: std.ArrayList(?[]const u8) = .empty; + if (parsed.object.get("fields")) |fv| { + if (fv == .array) for (fv.array.items) |item| { + if (item != .string) continue; + fields.append(arena, if (item.string.len == 0) null else item.string) catch return null; + }; + } + const reason: []const u8 = if (parsed.object.get("reason")) |r| + (if (r == .string) r.string else "") + else + ""; + return .{ .broken = broken.bool, .fields = fields.items, .reason = reason }; +} + +/// The finding to heal, per the model's verdict on suspicious facts; null when +/// the run is fine (no suspicion, no model, or a not-broken verdict). A failed +/// verdict call falls back to the raw facts — the heal prompt, defaulting to +/// no, becomes the judgment of last resort. +fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts: RunFacts) ?ScriptError { + const suspicion = suspicionOf(arena, facts) orelse return null; + if (self.ai_client == null) return null; + if (self.judgeSuspicion(arena, path, facts.source, suspicion)) |verdict| { + if (!verdict.broken) { + self.terminal.printInfo("Output has empty fields, but the model judged the run consistent: {s}", .{verdict.reason}); + return null; + } + return .{ + .kind = suspicion.kind, + .detail = std.fmt.allocPrint(arena, "{s}\nVerdict: {s}", .{ suspicion.detail, verdict.reason }) catch return null, + .source = facts.source, + .dry_fields = if (verdict.fields.len > 0) verdict.fields else suspicion.dry_fields, + }; + } + return .{ .kind = suspicion.kind, .detail = suspicion.detail, .source = facts.source, .dry_fields = suspicion.dry_fields }; +} + /// Null when the validation run cured the original finding; otherwise the /// message fed to the next heal attempt. Running clean is not a cure on its /// own — a revision that deletes the failing extract (or the `return`) also @@ -2358,10 +2424,10 @@ test "cureFailure: running clean is not a cure" { .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, .{ .schema = "[]", .field = null, .is_list = true, .calls = 1, .empty = 0 }, }; - try std.testing.expectEqual(null, try cureFailure(aa, dry, .{ .returned_data = true, .extract_stats = cured_stats })); + try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(true, cured_stats))); // Fix-by-deletion: the dry field is simply gone from the revised run. - const deleted = (try cureFailure(aa, dry, .{ .returned_data = true, .extract_stats = cured_stats[1..] })).?; + const deleted = (try cureFailure(aa, dry, testFacts(true, cured_stats[1..]))).?; try std.testing.expect(std.mem.indexOf(u8, deleted, "\"comments\"") != null); // Still dry counts as uncured. @@ -2369,38 +2435,59 @@ test "cureFailure: running clean is not a cure" { .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 5 }, cured_stats[1], }; - try std.testing.expect((try cureFailure(aa, dry, .{ .returned_data = true, .extract_stats = still_dry_stats })) != null); + try std.testing.expect((try cureFailure(aa, dry, testFacts(true, still_dry_stats))) != null); // .empty is cured only by a data-carrying return. const empty: ScriptError = .{ .kind = .empty, .detail = "", .source = "" }; - try std.testing.expectEqual(null, try cureFailure(aa, empty, .{ .returned_data = true, .extract_stats = &.{} })); - try std.testing.expect((try cureFailure(aa, empty, .{ .returned_data = false, .extract_stats = &.{} })) != null); + try std.testing.expectEqual(null, try cureFailure(aa, empty, testFacts(true, &.{}))); + try std.testing.expect((try cureFailure(aa, empty, testFacts(false, &.{}))) != null); // .threw needs nothing beyond running clean. const threw: ScriptError = .{ .kind = .threw, .detail = "", .source = "" }; - try std.testing.expectEqual(null, try cureFailure(aa, threw, .{ .returned_data = false, .extract_stats = &.{} })); + try std.testing.expectEqual(null, try cureFailure(aa, threw, testFacts(false, &.{}))); +} + +fn testFacts(returned_data: bool, stats: []const ScriptRuntime.ExtractStat) RunFacts { + return .{ .returned_data = returned_data, .empty_completion = null, .extract_stats = stats, .source = "" }; } -test "dryCanTrigger: baseline overrides the list-only rule" { +test "suspicionOf: any all-empty field is suspect, none is not" { var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); const aa = arena.allocator(); - const scalar: ScriptRuntime.ExtractStat = .{ .schema = "{}", .field = "title", .is_list = false, .calls = 3, .empty = 3 }; - const list: ScriptRuntime.ExtractStat = .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 3, .empty = 3 }; + const sparse: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, + }; + try std.testing.expectEqual(null, suspicionOf(aa, testFacts(true, sparse))); - // No baseline: list-only. - try std.testing.expectEqual(false, dryCanTrigger(scalar, null)); - try std.testing.expectEqual(true, dryCanTrigger(list, null)); + // Scalar all-empty is suspect too — judgment belongs to the model now. + const dry_scalar: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "title", .is_list = false, .calls = 3, .empty = 3 }, + }; + const s = suspicionOf(aa, testFacts(true, dry_scalar)).?; + try std.testing.expectEqual(ScriptError.Kind.dry_extracts, s.kind); + try std.testing.expectEqual(1, s.dry_fields.len); + + var empty_facts = testFacts(false, &.{}); + empty_facts.empty_completion = "[]"; + try std.testing.expectEqual(ScriptError.Kind.empty, suspicionOf(aa, empty_facts).?.kind); +} + +test "parseVerdict: tolerant of prose around the JSON" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); - var fields: Baseline.Fields = .empty; - try fields.put(aa, "title", .{ .calls = 2, .nonempty = 2 }); - try fields.put(aa, "comments", .{ .calls = 2, .nonempty = 0 }); + const v = parseVerdict(aa, "Sure!\n{\"broken\": true, \"fields\": [\"comments\", \"\"], \"reason\": \"selector drift\"}").?; + try std.testing.expect(v.broken); + try std.testing.expectEqual(2, v.fields.len); + try std.testing.expectEqualStrings("comments", v.fields[0].?); + try std.testing.expectEqual(null, v.fields[1]); + try std.testing.expectEqualStrings("selector drift", v.reason); - // Baseline: a scalar that carried data at record time triggers; a list - // that was already empty then is legitimately sparse. - try std.testing.expectEqual(true, dryCanTrigger(scalar, fields)); - try std.testing.expectEqual(false, dryCanTrigger(list, fields)); + try std.testing.expectEqual(null, parseVerdict(aa, "no json here")); + try std.testing.expectEqual(null, parseVerdict(aa, "{\"fields\": []}")); } test "capToolOutput: passes through when under cap" { diff --git a/src/agent/Baseline.zig b/src/agent/Baseline.zig index 0329ce7f95..fe4c68a85f 100644 --- a/src/agent/Baseline.zig +++ b/src/agent/Baseline.zig @@ -17,9 +17,10 @@ // along with this program. If not, see . //! Session-scoped record of what the agent's extract calls actually returned. -//! `/save` persists it into the script as a `// lp:baseline {...}` comment, so -//! a replay compares its own extract results against record-time reality — -//! which fields carried data when the script was made — instead of guessing. +//! `/save` persists it into the script as a `// lp:baseline {...}` comment — +//! evidence for the replay verdict turn, which reads it in the script source +//! to judge whether empty output is breakage or record-time-normal sparseness. +//! Nothing parses it programmatically; the model is the only consumer. const std = @import("std"); const lp = @import("lightpanda"); @@ -118,43 +119,6 @@ fn fieldsToLine(arena: std.mem.Allocator, fields: *const Fields) error{OutOfMemo return try std.mem.concat(arena, u8, &.{ marker, json }); } -/// Parse the baseline comment out of script source; null when absent or -/// malformed (a hand-written script simply has none). Field-name strings may -/// reference `source` — keep it alive as long as the result. -pub fn parse(arena: std.mem.Allocator, source: []const u8) ?Fields { - var lines = std.mem.splitScalar(u8, source, '\n'); - while (lines.next()) |line| { - const trimmed = std.mem.trim(u8, line, " \t\r"); - if (!std.mem.startsWith(u8, trimmed, marker)) continue; - return parseJson(arena, trimmed[marker.len..]); - } - return null; -} - -fn parseJson(arena: std.mem.Allocator, json_text: []const u8) ?Fields { - const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, json_text, .{}) catch return null; - if (parsed != .object) return null; - const fields_value = parsed.object.get("fields") orelse return null; - if (fields_value != .object) return null; - - var out: Fields = .empty; - var it = fields_value.object.iterator(); - while (it.next()) |entry| { - if (entry.value_ptr.* != .object) return null; - const calls = intField(entry.value_ptr.object, "calls") orelse return null; - const nonempty = intField(entry.value_ptr.object, "nonempty") orelse return null; - out.put(arena, entry.key_ptr.*, .{ .calls = calls, .nonempty = nonempty }) catch return null; - } - return out; -} - -fn intField(obj: std.json.ObjectMap, key: []const u8) ?u32 { - const value = obj.get(key) orelse return null; - if (value != .integer) return null; - if (value.integer < 0 or value.integer > std.math.maxInt(u32)) return null; - return @intCast(value.integer); -} - /// `script` with any existing baseline lines dropped and `line` (a full /// baseline line, or null) appended — synthesis may have copied a stale /// baseline from the previous script verbatim. @@ -180,27 +144,22 @@ pub fn withBaseline(arena: std.mem.Allocator, script: []const u8, line: ?[]const return aw.written(); } -test "baseline: note, serialize, parse round-trip" { +test "baseline: note and serialize tally per-field data presence" { var baseline: Baseline = .init(std.testing.allocator); defer baseline.deinit(); try baseline.noteExtractResult("{\"stories\":[{\"title\":\"a\"}],\"empty_list\":[],\"scalar\":\"x\"}"); try baseline.noteExtractResult("{\"stories\":[],\"empty_list\":[]}"); + // Malformed results record nothing. + try baseline.noteExtractResult("not json"); var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); const line = (try baseline.serialize(arena.allocator())).?; - try std.testing.expect(std.mem.startsWith(u8, line, marker)); - - const parsed = Baseline.parse(arena.allocator(), line).?; - try std.testing.expectEqual(FieldStat{ .calls = 2, .nonempty = 1 }, parsed.get("stories").?); - try std.testing.expectEqual(FieldStat{ .calls = 2, .nonempty = 0 }, parsed.get("empty_list").?); - try std.testing.expectEqual(FieldStat{ .calls = 1, .nonempty = 1 }, parsed.get("scalar").?); - - // Malformed results record nothing; malformed baselines parse to null. - try baseline.noteExtractResult("not json"); - try std.testing.expectEqual(null, Baseline.parse(arena.allocator(), "// lp:baseline {broken")); - try std.testing.expectEqual(null, Baseline.parse(arena.allocator(), "const x = 1;")); + try std.testing.expectEqualStrings( + marker ++ "{\"fields\":{\"stories\":{\"calls\":2,\"nonempty\":1},\"empty_list\":{\"calls\":2,\"nonempty\":0},\"scalar\":{\"calls\":1,\"nonempty\":1}}}", + line, + ); } test "baseline: withBaseline strips stale lines and appends" { From bd67911059829db018f6e011d2709fb2f60f1096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Thu, 9 Jul 2026 10:06:52 +0200 Subject: [PATCH 07/15] agent: simplify tool calls and suspicion handling Consolidates tool execution into a shared `callTool` helper. Replaces the temporary `Suspicion` and `DryFinding` structs with `ScriptError` to streamline validation and judgment paths. --- src/agent/Agent.zig | 74 +++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 68e1f7ec8b..979b821b9c 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -1488,7 +1488,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool .tool_call => |t| t, else => return .{ .text = "internal: command has no tool mapping", .is_error = true }, }; - const result = browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| return .{ + return self.callTool(arena, tc.name(), tc.args) catch |err| .{ .text = switch (err) { error.OutOfMemory => "out of memory", error.FrameNotLoaded => "no page loaded — run /goto first", @@ -1496,7 +1496,16 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool }, .is_error = true, }; - if (tc.tool == .extract and !result.is_error) self.baseline.noteExtractResult(result.text) catch {}; +} + +/// `browser_tools.call` plus the session-baseline note owed on a successful +/// extract, shared by both dispatch paths. The note runs on the uncapped +/// result — a truncated one parses as malformed and records nothing. +fn callTool(self: *Agent, arena: std.mem.Allocator, tool_name: []const u8, args: ?std.json.Value) browser_tools.ToolError!browser_tools.ToolResult { + const result = try browser_tools.call(arena, self.session, &self.node_registry, tool_name, args); + if (!result.is_error and std.mem.eql(u8, tool_name, @tagName(BrowserTool.extract))) { + self.baseline.noteExtractResult(result.text) catch {}; + } return result; } @@ -1558,7 +1567,6 @@ const RunFacts = struct { /// The returned value when it was deep-empty (capped display text). empty_completion: ?[]const u8, extract_stats: []const ScriptRuntime.ExtractStat, - /// The exact text that ran. source: []const u8, }; @@ -1682,34 +1690,23 @@ fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]co return std.mem.concat(arena, u8, &.{ string.truncateUtf8(text, detail_max_bytes), "…[truncated]" }); } -/// Facts worth a verdict, not yet a finding: the return value was deep-empty, -/// or some extract field came back empty on every call — any field, scalar or -/// list, baseline or not. Whether that is breakage or legitimate sparseness -/// is the model's judgment, not encoded here. -const Suspicion = struct { - kind: ScriptError.Kind, - detail: []const u8, - dry_fields: []const ?[]const u8, -}; - -fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?Suspicion { +/// A finding worth a verdict, not yet confirmed: the return value was +/// deep-empty, or some extract field came back empty on every call — any field, +/// scalar or list, baseline or not. Whether that is breakage or legitimate +/// sparseness is the model's judgment, not encoded here. +fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?ScriptError { if (facts.empty_completion) |text| return .{ .kind = .empty, .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, - .dry_fields = &.{}, + .source = facts.source, }; - const finding = (dryExtractsFinding(arena, facts.extract_stats) catch return null) orelse return null; - return .{ .kind = .dry_extracts, .detail = finding.detail, .dry_fields = finding.fields }; + return dryExtractsFinding(arena, facts.source, facts.extract_stats) catch return null; } -const DryFinding = struct { - detail: []const u8, - fields: []const ?[]const u8, -}; - -/// One detail line per extract field that came back empty on every call, plus -/// the field names for the cure check. Null when no field was dry. -fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) !?DryFinding { +/// A `dry_extracts` finding with one detail line per extract field that came +/// back empty on every call, plus the field names for the cure check. Null when +/// no field was dry. +fn dryExtractsFinding(arena: std.mem.Allocator, source: []const u8, stats: []const ScriptRuntime.ExtractStat) !?ScriptError { var aw: std.Io.Writer.Allocating = .init(arena); var fields: std.ArrayList(?[]const u8) = .empty; for (stats) |stat| { @@ -1717,7 +1714,8 @@ fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.Ext if (fields.items.len == 0) { try aw.writer.writeAll("some extracts came back empty on every call:\n"); } - try fields.append(arena, if (stat.field) |f| try arena.dupe(u8, f) else null); + // `stat.field` already lives in `arena` (facts were duped into it). + try fields.append(arena, stat.field); const schema = try capDetail(arena, stat.schema); if (stat.field) |field| { const shape: []const u8 = if (stat.is_list) "list" else "field"; @@ -1729,7 +1727,7 @@ fn dryExtractsFinding(arena: std.mem.Allocator, stats: []const ScriptRuntime.Ext try aw.writer.writeAll("\n"); } if (fields.items.len == 0) return null; - return .{ .detail = aw.written(), .fields = fields.items }; + return .{ .kind = .dry_extracts, .detail = aw.written(), .source = source, .dry_fields = fields.items }; } /// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. @@ -1898,7 +1896,7 @@ const Verdict = struct { /// One tool-free LLM turn deciding whether suspicious facts are breakage. /// Null on transport/parse failure or cancellation — the caller falls back to /// surfacing the facts instead of guessing. -fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, source: []const u8, suspicion: Suspicion) ?Verdict { +fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, suspicion: ScriptError) ?Verdict { const user_msg = std.fmt.allocPrint(self.conversation.arena.allocator(), \\Replay of {s} completed without errors, but {s} \\ @@ -1906,7 +1904,7 @@ fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, sour \\```js \\{s} \\``` - , .{ path, suspicion.detail, source }) catch return null; + , .{ path, suspicion.detail, suspicion.source }) catch return null; const raw = self.metaTurn(arena, "verdict", verdict_system_prompt, user_msg, 1024, self.effort) orelse return null; return parseVerdict(arena, raw); } @@ -1941,7 +1939,7 @@ fn parseVerdict(arena: std.mem.Allocator, raw: []const u8) ?Verdict { fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts: RunFacts) ?ScriptError { const suspicion = suspicionOf(arena, facts) orelse return null; if (self.ai_client == null) return null; - if (self.judgeSuspicion(arena, path, facts.source, suspicion)) |verdict| { + if (self.judgeSuspicion(arena, path, suspicion)) |verdict| { if (!verdict.broken) { self.terminal.printInfo("Output has empty fields, but the model judged the run consistent: {s}", .{verdict.reason}); return null; @@ -1949,11 +1947,11 @@ fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts return .{ .kind = suspicion.kind, .detail = std.fmt.allocPrint(arena, "{s}\nVerdict: {s}", .{ suspicion.detail, verdict.reason }) catch return null, - .source = facts.source, + .source = suspicion.source, .dry_fields = if (verdict.fields.len > 0) verdict.fields else suspicion.dry_fields, }; } - return .{ .kind = suspicion.kind, .detail = suspicion.detail, .source = facts.source, .dry_fields = suspicion.dry_fields }; + return suspicion; } /// Null when the validation run cured the original finding; otherwise the @@ -2301,14 +2299,10 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co self.terminal.spinner.setTool(tool_name, args_str); defer self.terminal.spinner.setThinking(); - const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments)) |result| blk: { - // Pre-cap text: a truncated result would parse as malformed and - // record nothing. - if (std.mem.eql(u8, tool_name, @tagName(BrowserTool.extract)) and !result.is_error) { - self.baseline.noteExtractResult(result.text) catch {}; - } - break :blk .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error }; - } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; + const outcome: zenai.provider.Client.ToolHandler.Result = if (self.callTool(allocator, tool_name, arguments)) |result| + .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error } + else |err| + .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error); if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error); From 7a4bbd107ba0b2d44a06c0f5300cd70c14896b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Thu, 9 Jul 2026 11:24:53 +0200 Subject: [PATCH 08/15] fix(agent): filter hallucinated fields and refactor script judging --- src/agent/Agent.zig | 163 ++++++++++++++++++++++++++++++-------------- 1 file changed, 113 insertions(+), 50 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 979b821b9c..d5ca022b5c 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -863,10 +863,11 @@ fn handleLoad(self: *Agent, rest: []const u8) void { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); while (true) { - const finding: ScriptError = switch (self.runScriptOutcome(arena.allocator(), path)) { - .fatal => return, - .ok => |facts| self.judgedFinding(arena.allocator(), path, facts) orelse return, - .script_error => |script_error| script_error, + // Nothing from a prior retry is read again. + _ = arena.reset(.retain_capacity); + const finding: ScriptError = switch (self.runAndJudge(arena.allocator(), path)) { + .fatal, .clean => return, + .broken => |f| f, }; if (self.ai_client == null) return; switch (promptHeal(path, finding.kind)) { @@ -1558,14 +1559,21 @@ const ScriptRunOutcome = union(enum) { script_error: ScriptError, }; +/// What a completed run returned, as far as heal cares. +const Returned = union(enum) { + /// No `return`, or a value whose display form couldn't be computed. + none, + /// A value carrying data. + data, + /// A deep-empty value, carrying its capped display text. + empty: []const u8, +}; + /// Facts about a run that completed without throwing — suspicion is judged by /// the model, never here. Duped into the caller's arena — the runtime dies /// with `runScriptOutcome`. const RunFacts = struct { - /// The script returned a value that carries data. - returned_data: bool, - /// The returned value when it was deep-empty (capped display text). - empty_completion: ?[]const u8, + returned: Returned, extract_stats: []const ScriptRuntime.ExtractStat, source: []const u8, }; @@ -1601,6 +1609,23 @@ fn runScript(self: *Agent, path: []const u8) bool { }; } +/// Terminal `fatal`/`clean` states, or the `broken` finding to heal — the model +/// gets to judge a clean-but-suspicious run. Shared by the `/load` heal offer +/// and one-shot `--heal`. +const RunJudgement = union(enum) { + fatal, + clean, + broken: ScriptError, +}; + +fn runAndJudge(self: *Agent, arena: std.mem.Allocator, path: []const u8) RunJudgement { + return switch (self.runScriptOutcome(arena, path)) { + .fatal => .fatal, + .ok => |facts| if (self.judgedFinding(arena, path, facts)) |finding| .{ .broken = finding } else .clean, + .script_error => |script_error| .{ .broken = script_error }, + }; +} + fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) ScriptRunOutcome { var script_arena: std.heap.ArenaAllocator = .init(self.allocator); defer script_arena.deinit(); @@ -1655,13 +1680,12 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc // A script that printed nothing leaves no trace, so freeze the spinner into // a green bullet (like /goto); one that printed already showed its result. if (!output.emitted) self.terminal.printScriptDone("script", path); - const empty_completion: ?[]const u8 = if (ok.completion) |c| - (if (c.empty) capDetail(arena, c.text) catch return .fatal else null) + const returned: Returned = if (ok.completion) |c| + (if (c.empty) .{ .empty = capDetail(arena, c.text) catch return .fatal } else .data) else - null; + .none; return .{ .ok = .{ - .returned_data = if (ok.completion) |c| !c.empty else false, - .empty_completion = empty_completion, + .returned = returned, .extract_stats = dupeExtractStats(arena, ok.extract_stats) catch return .fatal, .source = arena.dupe(u8, content) catch return .fatal, } }; @@ -1695,11 +1719,14 @@ fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]co /// scalar or list, baseline or not. Whether that is breakage or legitimate /// sparseness is the model's judgment, not encoded here. fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?ScriptError { - if (facts.empty_completion) |text| return .{ - .kind = .empty, - .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, - .source = facts.source, - }; + switch (facts.returned) { + .empty => |text| return .{ + .kind = .empty, + .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, + .source = facts.source, + }, + .none, .data => {}, + } return dryExtractsFinding(arena, facts.source, facts.extract_stats) catch return null; } @@ -1735,24 +1762,22 @@ fn runScriptWithHeal(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); - for (0..2) |attempt| { - const finding: ScriptError = switch (self.runScriptOutcome(arena.allocator(), path)) { - .fatal => return false, - .ok => |facts| self.judgedFinding(arena.allocator(), path, facts) orelse return true, - .script_error => |script_error| script_error, - }; - if (attempt == 0) { - // A re-run is free; healing is not — one retry filters transient - // failures (a flaky page load) before the model is asked to fix - // a correct script. - self.terminal.printInfo("Script run failed or looks broken; retrying once before healing.", .{}); - continue; - } - // Healing spends tokens; report the cost the way --task does. - defer self.printUsageSummary(); - return self.healLoop(arena.allocator(), path, finding); - } - unreachable; + switch (self.runAndJudge(arena.allocator(), path)) { + .fatal => return false, + .clean => return true, + .broken => {}, + } + // A re-run is free; healing is not — one retry filters a transient failure + // (a flaky page load) before the model is asked to fix a correct script. + self.terminal.printInfo("Script run failed or looks broken; retrying once before healing.", .{}); + const finding: ScriptError = switch (self.runAndJudge(arena.allocator(), path)) { + .fatal => return false, + .clean => return true, + .broken => |f| f, + }; + // Healing spends tokens; report the cost the way --task does. + defer self.printUsageSummary(); + return self.healLoop(arena.allocator(), path, finding); } const max_heal_attempts = 2; @@ -1948,12 +1973,29 @@ fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts .kind = suspicion.kind, .detail = std.fmt.allocPrint(arena, "{s}\nVerdict: {s}", .{ suspicion.detail, verdict.reason }) catch return null, .source = suspicion.source, - .dry_fields = if (verdict.fields.len > 0) verdict.fields else suspicion.dry_fields, + .dry_fields = confirmedDryFields(arena, verdict.fields, suspicion.dry_fields) catch return null, }; } return suspicion; } +/// Verdict field names minus any that weren't actually dry — a hallucinated +/// name never matches an extract stat, so the cure check could never clear it. +/// Falls back to every dry field when none match. +fn confirmedDryFields(arena: std.mem.Allocator, judged: []const ?[]const u8, actual: []const ?[]const u8) error{OutOfMemory}![]const ?[]const u8 { + if (judged.len == 0) return actual; + var kept: std.ArrayList(?[]const u8) = .empty; + for (judged) |f| { + for (actual) |a| { + if (ScriptRuntime.fieldEql(f, a)) { + try kept.append(arena, f); + break; + } + } + } + return if (kept.items.len == 0) actual else kept.items; +} + /// Null when the validation run cured the original finding; otherwise the /// message fed to the next heal attempt. Running clean is not a cure on its /// own — a revision that deletes the failing extract (or the `return`) also @@ -1961,7 +2003,7 @@ fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts fn cureFailure(arena: std.mem.Allocator, first: ScriptError, facts: RunFacts) error{OutOfMemory}!?[]const u8 { switch (first.kind) { .threw => return null, - .empty => return if (facts.returned_data) + .empty => return if (facts.returned == .data) null else "The revised script ran, but still returns no data (or no longer returns anything) — the original returned a value.", @@ -2418,10 +2460,10 @@ test "cureFailure: running clean is not a cure" { .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, .{ .schema = "[]", .field = null, .is_list = true, .calls = 1, .empty = 0 }, }; - try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(true, cured_stats))); + try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(.data, cured_stats))); // Fix-by-deletion: the dry field is simply gone from the revised run. - const deleted = (try cureFailure(aa, dry, testFacts(true, cured_stats[1..]))).?; + const deleted = (try cureFailure(aa, dry, testFacts(.data, cured_stats[1..]))).?; try std.testing.expect(std.mem.indexOf(u8, deleted, "\"comments\"") != null); // Still dry counts as uncured. @@ -2429,20 +2471,42 @@ test "cureFailure: running clean is not a cure" { .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 5 }, cured_stats[1], }; - try std.testing.expect((try cureFailure(aa, dry, testFacts(true, still_dry_stats))) != null); + try std.testing.expect((try cureFailure(aa, dry, testFacts(.data, still_dry_stats))) != null); // .empty is cured only by a data-carrying return. const empty: ScriptError = .{ .kind = .empty, .detail = "", .source = "" }; - try std.testing.expectEqual(null, try cureFailure(aa, empty, testFacts(true, &.{}))); - try std.testing.expect((try cureFailure(aa, empty, testFacts(false, &.{}))) != null); + try std.testing.expectEqual(null, try cureFailure(aa, empty, testFacts(.data, &.{}))); + try std.testing.expect((try cureFailure(aa, empty, testFacts(.none, &.{}))) != null); // .threw needs nothing beyond running clean. const threw: ScriptError = .{ .kind = .threw, .detail = "", .source = "" }; - try std.testing.expectEqual(null, try cureFailure(aa, threw, testFacts(false, &.{}))); + try std.testing.expectEqual(null, try cureFailure(aa, threw, testFacts(.none, &.{}))); +} + +fn testFacts(returned: Returned, stats: []const ScriptRuntime.ExtractStat) RunFacts { + return .{ .returned = returned, .extract_stats = stats, .source = "" }; } -fn testFacts(returned_data: bool, stats: []const ScriptRuntime.ExtractStat) RunFacts { - return .{ .returned_data = returned_data, .empty_completion = null, .extract_stats = stats, .source = "" }; +test "confirmedDryFields drops hallucinated names, keeps the narrowing" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const actual: []const ?[]const u8 = &.{ @as(?[]const u8, "comments"), @as(?[]const u8, "score"), null }; + + const narrowed = try confirmedDryFields(aa, &.{@as(?[]const u8, "comments")}, actual); + try std.testing.expectEqual(1, narrowed.len); + try std.testing.expectEqualStrings("comments", narrowed[0].?); + + const mixed = try confirmedDryFields(aa, &.{ @as(?[]const u8, "nonexistent"), @as(?[]const u8, "score") }, actual); + try std.testing.expectEqual(1, mixed.len); + try std.testing.expectEqualStrings("score", mixed[0].?); + + // All hallucinated → the full dry set, not an empty one. + const fallback = try confirmedDryFields(aa, &.{@as(?[]const u8, "bogus")}, actual); + try std.testing.expectEqual(3, fallback.len); + + try std.testing.expectEqual(actual.ptr, (try confirmedDryFields(aa, &.{}, actual)).ptr); } test "suspicionOf: any all-empty field is suspect, none is not" { @@ -2453,18 +2517,17 @@ test "suspicionOf: any all-empty field is suspect, none is not" { const sparse: []const ScriptRuntime.ExtractStat = &.{ .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, }; - try std.testing.expectEqual(null, suspicionOf(aa, testFacts(true, sparse))); + try std.testing.expectEqual(null, suspicionOf(aa, testFacts(.data, sparse))); // Scalar all-empty is suspect too — judgment belongs to the model now. const dry_scalar: []const ScriptRuntime.ExtractStat = &.{ .{ .schema = "{}", .field = "title", .is_list = false, .calls = 3, .empty = 3 }, }; - const s = suspicionOf(aa, testFacts(true, dry_scalar)).?; + const s = suspicionOf(aa, testFacts(.data, dry_scalar)).?; try std.testing.expectEqual(ScriptError.Kind.dry_extracts, s.kind); try std.testing.expectEqual(1, s.dry_fields.len); - var empty_facts = testFacts(false, &.{}); - empty_facts.empty_completion = "[]"; + const empty_facts = testFacts(.{ .empty = "[]" }, &.{}); try std.testing.expectEqual(ScriptError.Kind.empty, suspicionOf(aa, empty_facts).?.kind); } From 718d7e7137d50708a287204334df38f84af15635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Fri, 10 Jul 2026 16:48:05 +0200 Subject: [PATCH 09/15] agent: update heal prompt to preserve comments Instructs the agent to preserve, update, and add intent comments when healing scripts. --- src/agent/Agent.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 34b7104c71..ae775e05e9 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -1822,6 +1822,9 @@ const heal_revision_prompt = \\Fix the script so it replays successfully against the current site: the \\error and this session's working tool calls identify the repair. Keep \\every step, selector, and output shape that still works unchanged. + \\Preserve the script's `//` intent comments; where you change a block, + \\update its comment so it still describes what the revised code does, and + \\add one for any block that lacks it. ; /// Only a revision that passed validation in a fresh session replaces `path`; From f18dce65620c4a395758df381f1c986f4dd59767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Thu, 16 Jul 2026 14:33:19 +0200 Subject: [PATCH 10/15] agent: simplify script healing and verdict logic - Add `soloTurn` to run LLM verdicts without conversation history. - Swap `Recorder` instead of snapshotting during healing. - Remove `is_list` tracking from extract stats. - Simplify JSON parsing in `parseVerdict`. --- src/agent/Agent.zig | 113 ++++++++++++++++++++-------------------- src/script/Recorder.zig | 52 ------------------ src/script/Runtime.zig | 26 +++------ 3 files changed, 64 insertions(+), 127 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index eec860be8c..3d403a31dd 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -1192,7 +1192,6 @@ fn failSynthesis(self: *Agent, label: []const u8, reason: []const u8) ?[]const u /// always rolled back out of history; null (with a `label`-tagged report) on /// error, no text, or cancellation. fn metaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: []const u8, user_msg: []const u8, max_tokens: i32, effort: Config.Effort) ?[]const u8 { - const provider_client = self.ai_client.?; self.conversation.ensureSystemPrompt() catch return self.failSynthesis(label, "out of memory"); // Regular turns keep the driver prompt. (`messages[0]` is the system @@ -1205,13 +1204,29 @@ fn metaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: [ self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); defer self.conversation.rollback(baseline); + return self.sendMetaTurn(arena, label, &self.conversation.messages, self.conversation.arena.allocator(), max_tokens, effort); +} + +/// `metaTurn` minus the conversation: the turn sends `system` + `user_msg` +/// alone, for judgments that must come from the message itself (the replay +/// verdict) — session history would only add tokens and sway the verdict. +fn soloTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: []const u8, user_msg: []const u8, max_tokens: i32, effort: Config.Effort) ?[]const u8 { + var messages: std.ArrayList(zenai.provider.Message) = .empty; + messages.append(arena, .{ .role = .system, .content = system }) catch return self.failSynthesis(label, "out of memory"); + messages.append(arena, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); + return self.sendMetaTurn(arena, label, &messages, arena, max_tokens, effort); +} + +fn sendMetaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, messages: *std.ArrayList(zenai.provider.Message), message_arena: std.mem.Allocator, max_tokens: i32, effort: Config.Effort) ?[]const u8 { + const provider_client = self.ai_client.?; + const baseline = self.conversation.messages.items.len; self.http_interrupt.reset(); self.terminal.spinner.start(); var result = provider_client.runTools( self.model, - &self.conversation.messages, + messages, self.allocator, - self.conversation.arena.allocator(), + message_arena, .{ .context = @ptrCast(self), .callFn = handleToolCall }, .{ .tools = &.{}, @@ -1238,8 +1253,8 @@ fn metaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: [ return null; } - // `result.text` lives in the conversation arena, freed by the deferred - // rollback; the dupe happens before defers run. + // `result.text` lives in `message_arena`, which the caller may roll back + // on return; the dupe happens before that. const raw = result.text orelse return self.failSynthesis(label, "the model returned no text"); return arena.dupe(u8, raw) catch self.failSynthesis(label, "out of memory"); } @@ -1685,7 +1700,6 @@ fn dupeExtractStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.Extra o.* = .{ .schema = try arena.dupe(u8, stat.schema), .field = if (stat.field) |f| try arena.dupe(u8, f) else null, - .is_list = stat.is_list, .calls = stat.calls, .empty = stat.empty, }; @@ -1733,8 +1747,7 @@ fn dryExtractsFinding(arena: std.mem.Allocator, source: []const u8, stats: []con try fields.append(arena, stat.field); const schema = try capDetail(arena, stat.schema); if (stat.field) |field| { - const shape: []const u8 = if (stat.is_list) "list" else "field"; - try aw.writer.print("- the \"{s}\" {s} in extract({s}) came back empty", .{ field, shape, schema }); + try aw.writer.print("- the \"{s}\" field in extract({s}) came back empty", .{ field, schema }); } else { try aw.writer.print("- extract({s}) returned no data", .{schema}); } @@ -1750,13 +1763,15 @@ fn runScriptWithHeal(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); - switch (self.runAndJudge(arena.allocator(), path)) { + // A re-run is free; a verdict turn and healing are not. The first pass + // gates on the local suspicion check alone — one retry filters a transient + // failure (a flaky page load), and the model judges the re-run's facts, + // the ones healing would act on. + switch (self.runScriptOutcome(arena.allocator(), path)) { .fatal => return false, - .clean => return true, - .broken => {}, + .ok => |facts| if (suspicionOf(arena.allocator(), facts) == null) return true, + .script_error => {}, } - // A re-run is free; healing is not — one retry filters a transient failure - // (a flaky page load) before the model is asked to fix a correct script. self.terminal.printInfo("Script run failed or looks broken; retrying once before healing.", .{}); const finding: ScriptError = switch (self.runAndJudge(arena.allocator(), path)) { .fatal => return false, @@ -1808,30 +1823,22 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr var source = first.source; var error_detail = first.detail; - const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch { - self.terminal.printError("heal failed: out of memory", .{}); - return false; - }; + const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch return self.healOom(); // The recorder is /save's stream: heal must neither synthesize from the // REPL's prior recordings nor leak its diagnose actions into a later /save. - const recorded = self.save_buffer.snapshot(arena) catch { - self.terminal.printError("heal failed: out of memory", .{}); - return false; - }; - self.save_buffer.reset(); - defer self.save_buffer.restore(recorded) catch { - self.terminal.printWarning("commands recorded before the heal were lost (out of memory)", .{}); - }; + var prior: Recorder = .init(self.allocator); + std.mem.swap(Recorder, &self.save_buffer, &prior); + defer { + std.mem.swap(Recorder, &self.save_buffer, &prior); + prior.deinit(); + } var attempt: usize = 1; while (attempt <= max_heal_attempts) : (attempt += 1) { self.terminal.printInfo("Healing {s} (attempt {d}/{d})", .{ path, attempt, max_heal_attempts }); - const diagnose = buildHealDiagnoseMessage(arena, path, source, error_detail) catch { - self.terminal.printError("heal failed: out of memory", .{}); - return false; - }; + const diagnose = buildHealDiagnoseMessage(arena, path, source, error_detail) catch return self.healOom(); if (!self.runTurn(.{ .prompt = diagnose, .capture_for_save = true, .label = "Heal" })) return false; const revised = self.synthesizeScriptText(arena, "heal", path, source, heal_revision_prompt) orelse return false; @@ -1853,8 +1860,7 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr .ok => |facts| { if (cureFailure(arena, first, facts) catch { self.removeTempScript(tmp_path); - self.terminal.printError("heal failed: out of memory", .{}); - return false; + return self.healOom(); }) |failure| { self.terminal.printWarning("{s}", .{failure}); source = revised; @@ -1889,12 +1895,19 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr return false; } +fn healOom(self: *Agent) bool { + self.terminal.printError("heal failed: out of memory", .{}); + return false; +} + const verdict_system_prompt = \\You judge replays of saved browser-automation scripts. Given the script \\and facts about a run that completed without errors, decide whether the \\empty output means the script is broken (stale selectors after a site \\change) or legitimate (the page genuinely has no such data right now). - \\A `// lp:baseline` comment, when present, records how often each output + \\A ` +++ std.mem.trimRight(u8, Baseline.marker, " ") ++ + \\` comment, when present, records how often each output \\field carried data when the script was saved — weigh it as evidence. \\Respond with JSON only: \\{"broken": true|false, "fields": ["", ...], "reason": ""} @@ -1913,7 +1926,7 @@ const Verdict = struct { /// Null on transport/parse failure or cancellation — the caller falls back to /// surfacing the facts instead of guessing. fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, suspicion: ScriptError) ?Verdict { - const user_msg = std.fmt.allocPrint(self.conversation.arena.allocator(), + const user_msg = std.fmt.allocPrint(arena, \\Replay of {s} completed without errors, but {s} \\ \\The script (its comments and structure carry the intent): @@ -1921,7 +1934,7 @@ fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, susp \\{s} \\``` , .{ path, suspicion.detail, suspicion.source }) catch return null; - const raw = self.metaTurn(arena, "verdict", verdict_system_prompt, user_msg, 1024, self.effort) orelse return null; + const raw = self.soloTurn(arena, "verdict", verdict_system_prompt, user_msg, 1024, self.effort) orelse return null; return parseVerdict(arena, raw); } @@ -1929,23 +1942,11 @@ fn parseVerdict(arena: std.mem.Allocator, raw: []const u8) ?Verdict { const start = std.mem.indexOfScalar(u8, raw, '{') orelse return null; const end = std.mem.lastIndexOfScalar(u8, raw, '}') orelse return null; if (end < start) return null; - const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, raw[start .. end + 1], .{}) catch return null; - if (parsed != .object) return null; - const broken = parsed.object.get("broken") orelse return null; - if (broken != .bool) return null; - - var fields: std.ArrayList(?[]const u8) = .empty; - if (parsed.object.get("fields")) |fv| { - if (fv == .array) for (fv.array.items) |item| { - if (item != .string) continue; - fields.append(arena, if (item.string.len == 0) null else item.string) catch return null; - }; - } - const reason: []const u8 = if (parsed.object.get("reason")) |r| - (if (r == .string) r.string else "") - else - ""; - return .{ .broken = broken.bool, .fields = fields.items, .reason = reason }; + const Wire = struct { broken: bool, fields: []const []const u8 = &.{}, reason: []const u8 = "" }; + const wire = std.json.parseFromSliceLeaky(Wire, arena, raw[start .. end + 1], .{ .ignore_unknown_fields = true }) catch return null; + const fields = arena.alloc(?[]const u8, wire.fields.len) catch return null; + for (wire.fields, fields) |f, *out| out.* = if (f.len == 0) null else f; + return .{ .broken = wire.broken, .fields = fields, .reason = wire.reason }; } /// The finding to heal, per the model's verdict on suspicious facts; null when @@ -2464,8 +2465,8 @@ test "cureFailure: running clean is not a cure" { .dry_fields = &.{ @as(?[]const u8, "comments"), null }, }; const cured_stats: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, - .{ .schema = "[]", .field = null, .is_list = true, .calls = 1, .empty = 0 }, + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, + .{ .schema = "[]", .field = null, .calls = 1, .empty = 0 }, }; try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(.data, cured_stats))); @@ -2475,7 +2476,7 @@ test "cureFailure: running clean is not a cure" { // Still dry counts as uncured. const still_dry_stats: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 5 }, + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 5 }, cured_stats[1], }; try std.testing.expect((try cureFailure(aa, dry, testFacts(.data, still_dry_stats))) != null); @@ -2522,13 +2523,13 @@ test "suspicionOf: any all-empty field is suspect, none is not" { const aa = arena.allocator(); const sparse: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "comments", .is_list = true, .calls = 5, .empty = 2 }, + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, }; try std.testing.expectEqual(null, suspicionOf(aa, testFacts(.data, sparse))); // Scalar all-empty is suspect too — judgment belongs to the model now. const dry_scalar: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "title", .is_list = false, .calls = 3, .empty = 3 }, + .{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }, }; const s = suspicionOf(aa, testFacts(.data, dry_scalar)).?; try std.testing.expectEqual(ScriptError.Kind.dry_extracts, s.kind); diff --git a/src/script/Recorder.zig b/src/script/Recorder.zig index 5532373055..2ab8b381bc 100644 --- a/src/script/Recorder.zig +++ b/src/script/Recorder.zig @@ -68,29 +68,6 @@ pub fn reset(self: *Recorder) void { _ = self.arena.reset(.retain_capacity); } -pub const Snapshot = struct { - content: []const u8, - lines: u32, - page_declared: bool, -}; - -/// Copy the current recording so a scoped consumer (the heal loop) can reset -/// the recorder for its own actions and hand the original back afterwards. -pub fn snapshot(self: *Recorder, allocator: std.mem.Allocator) !Snapshot { - return .{ - .content = try allocator.dupe(u8, self.bytes()), - .lines = self.lines, - .page_declared = self.page_declared, - }; -} - -pub fn restore(self: *Recorder, snap: Snapshot) !void { - self.reset(); - try self.content.writer.writeAll(snap.content); - self.lines = snap.lines; - self.page_declared = snap.page_declared; -} - pub fn record(self: *Recorder, cmd: Command) !void { if (!cmd.isRecorded()) return; self.buf.clearRetainingCapacity(); @@ -180,35 +157,6 @@ test "record filters state-mutating commands and comments" { try std.testing.expectEqual(@as(u32, 2), recorder.lines); } -test "snapshot/restore scopes recordings" { - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const aa = arena.allocator(); - - var recorder: Recorder = .init(std.testing.allocator); - defer recorder.deinit(); - - try recorder.record(parseLine(aa, "/goto https://example.com")); - const snap = try recorder.snapshot(aa); - - recorder.reset(); - try recorder.record(parseLine(aa, "/scroll y=200")); - - try recorder.restore(snap); - try std.testing.expectEqualStrings( - "const page = new Page();\nawait page.goto(\"https://example.com\");\n", - recorder.bytes(), - ); - try std.testing.expectEqual(@as(u32, 2), recorder.lines); - - // page_declared came back with the snapshot: no second declaration. - try recorder.record(parseLine(aa, "/scroll y=200")); - try std.testing.expectEqualStrings( - "const page = new Page();\nawait page.goto(\"https://example.com\");\npage.scroll({ y: 200 });\n", - recorder.bytes(), - ); -} - test "recordRaw writes the JS line verbatim" { var recorder: Recorder = .init(std.testing.allocator); defer recorder.deinit(); diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index 3fe1022ebb..e9140fc910 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -46,7 +46,7 @@ console_observer: ?ConsoleObserver = null, pending_gotos: std.ArrayList(PendingGoto), /// Restarted per `runSource`; backs `PendingGoto.deadline_ms`. run_timer: std.time.Timer, -/// Per-run tally of extract list-field emptiness. call_arena-backed, so it is +/// Per-run tally of extract field emptiness. call_arena-backed, so it is /// re-zeroed alongside the arena reset at the top of `runSource`. extract_stats: std.ArrayList(ExtractStat) = .empty, @@ -279,7 +279,6 @@ pub const Completion = struct { /// reference the parsed value. pub const ExtractField = struct { field: ?[]const u8, - is_list: bool, empty: bool, }; @@ -287,7 +286,7 @@ pub fn classifyExtractFields(arena: std.mem.Allocator, result: std.json.Value) e switch (result) { .array => { const out = try arena.alloc(ExtractField, 1); - out[0] = .{ .field = null, .is_list = true, .empty = jsonIsEmpty(result) }; + out[0] = .{ .field = null, .empty = jsonIsEmpty(result) }; return out; }, .object => |obj| { @@ -297,7 +296,6 @@ pub fn classifyExtractFields(arena: std.mem.Allocator, result: std.json.Value) e while (it.next()) |entry| : (i += 1) { out[i] = .{ .field = entry.key_ptr.*, - .is_list = entry.value_ptr.* == .array, .empty = jsonIsEmpty(entry.value_ptr.*), }; } @@ -319,10 +317,6 @@ pub const ExtractStat = struct { schema: []const u8, /// Top-level field of the result; null when the schema itself is a list. field: ?[]const u8, - /// The field's value was an array on some call. Consumers gate the - /// no-baseline heal policy on this: lists signal collection intent, - /// scalars are legitimately sparse. - is_list: bool, calls: u32, empty: u32, }; @@ -338,7 +332,7 @@ pub const RunResult = union(enum) { pub const Ok = struct { /// The value the script returned; null when it returned `undefined`. completion: ?Completion, - /// Per-(schema, list-field) extract tallies for the run. + /// Per-(schema, field) extract tallies for the run. extract_stats: []const ExtractStat, }; @@ -607,8 +601,8 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf } } -/// Tally each top-level field of an extract result; the Agent's heal policy -/// decides which stats can trigger. +/// Tally each top-level field of an extract result; callers judge what the +/// tallies mean. fn recordExtractStats(self: *Runtime, arena: std.mem.Allocator, args: ?std.json.Value, result: std.json.Value) error{OutOfMemory}!void { const raw_schema = switch ((args orelse return).object.get("schema") orelse return) { .string => |s| s, @@ -616,16 +610,15 @@ fn recordExtractStats(self: *Runtime, arena: std.mem.Allocator, args: ?std.json. }; const schema = stripExtractSchemaRoot(raw_schema); for (try classifyExtractFields(arena, result)) |fc| { - try self.bumpExtractStat(schema, fc.field, fc.is_list, fc.empty); + try self.bumpExtractStat(schema, fc.field, fc.empty); } } -fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_list: bool, is_empty: bool) error{OutOfMemory}!void { +fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_empty: bool) error{OutOfMemory}!void { for (self.extract_stats.items) |*stat| { if (!std.mem.eql(u8, stat.schema, schema)) continue; if (!fieldEql(stat.field, field)) continue; stat.calls += 1; - if (is_list) stat.is_list = true; if (is_empty) stat.empty += 1; return; } @@ -634,7 +627,6 @@ fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_li try self.extract_stats.append(arena, .{ .schema = try arena.dupe(u8, schema), .field = if (field) |f| try arena.dupe(u8, f) else null, - .is_list = is_list, .calls = 1, .empty = @intFromBool(is_empty), }); @@ -1775,22 +1767,18 @@ test "agent script runtime: extract stats tally list-field emptiness" { try testing.expectEqual(5, stats.len); try testing.expectString("items", stats[0].field.?); - try testing.expectEqual(true, stats[0].is_list); try testing.expectEqual(2, stats[0].calls); try testing.expectEqual(2, stats[0].empty); try testing.expectString("btn", stats[1].field.?); - try testing.expectEqual(false, stats[1].is_list); try testing.expectEqual(1, stats[1].calls); try testing.expectEqual(0, stats[1].empty); try testing.expectString("buttons", stats[2].field.?); - try testing.expectEqual(true, stats[2].is_list); try testing.expectEqual(1, stats[2].calls); try testing.expectEqual(0, stats[2].empty); try testing.expectEqual(null, stats[3].field); - try testing.expectEqual(true, stats[3].is_list); try testing.expectString("[\".no-such-root\"]", stats[3].schema); try testing.expectEqual(1, stats[3].calls); try testing.expectEqual(1, stats[3].empty); From f3f551d673eed97dd29827c3171ee3a0a16d622a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Mon, 20 Jul 2026 16:18:08 +0200 Subject: [PATCH 11/15] agent: isolate baseline during heal and fix meta-turn cancel` - Swap `self.baseline` during `healLoop` to prevent state leakage. - Use `defer` with a `keep_revision` flag for robust temp file cleanup. - Avoid rolling back conversation state on meta-turn cancellation. - Generalize `capToolOutput` to `capOutput` and use it for details. - Remove persistent scratch arena from `Baseline`. --- src/agent/Agent.zig | 73 +++++++++++++++++++++++++----------------- src/agent/Baseline.zig | 12 +++---- src/agent/Terminal.zig | 2 +- 3 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 3d403a31dd..de12bd9291 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -496,8 +496,14 @@ fn drainCancellation(self: *Agent, baseline: usize) error{UserCancelled} { /// The side effects of `drainCancellation` without surfacing the error, for /// void callers (e.g. `/save` synthesis) that just need to clean up. fn resetAfterCancel(self: *Agent, baseline: usize) void { - self.endStreamedText(); + self.clearCancelState(); self.conversation.rollback(baseline); +} + +/// Cancel cleanup that touches no conversation state, for turns whose message +/// cleanup is owned elsewhere (meta-turns roll back via their own defer). +fn clearCancelState(self: *Agent) void { + self.endStreamedText(); self.browser.env.cancelTerminate(); self.cancel_requested.store(false, .release); self.http_interrupt.reset(); @@ -1219,7 +1225,6 @@ fn soloTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: [ fn sendMetaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, messages: *std.ArrayList(zenai.provider.Message), message_arena: std.mem.Allocator, max_tokens: i32, effort: Config.Effort) ?[]const u8 { const provider_client = self.ai_client.?; - const baseline = self.conversation.messages.items.len; self.http_interrupt.reset(); self.terminal.spinner.start(); var result = provider_client.runTools( @@ -1239,7 +1244,7 @@ fn sendMetaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, messa ) catch |err| { self.terminal.spinner.cancel(); if (self.cancel_requested.load(.acquire)) { - self.resetAfterCancel(baseline); + self.clearCancelState(); return null; } log.err(.app, "AI meta-turn error", .{ .label = label, .err = err }); @@ -1249,7 +1254,7 @@ fn sendMetaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, messa defer result.deinit(); self.total_usage.add(result.usage); if (result.cancelled) { - self.resetAfterCancel(baseline); + self.clearCancelState(); return null; } @@ -1711,9 +1716,10 @@ fn dupeExtractStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.Extra /// result (hundreds of all-null rows) would otherwise bloat the LLM turn. const detail_max_bytes: usize = 2048; +/// `capOutput` at `detail_max_bytes`, always duped — `RunFacts` details must +/// outlive the runtime whose arena the text came from. fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]const u8 { - if (text.len <= detail_max_bytes) return text; - return std.mem.concat(arena, u8, &.{ string.truncateUtf8(text, detail_max_bytes), "…[truncated]" }); + return arena.dupe(u8, capOutput(arena, text, detail_max_bytes)); } /// A finding worth a verdict, not yet confirmed: the return value was @@ -1825,13 +1831,23 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch return self.healOom(); - // The recorder is /save's stream: heal must neither synthesize from the - // REPL's prior recordings nor leak its diagnose actions into a later /save. + // Every exit leaves no stray revision behind; a failed rename is the one + // deliberate keep (its error message points at tmp_path). + var keep_revision = false; + defer if (!keep_revision) self.removeTempScript(tmp_path); + + // The recorder and baseline are /save's stream: heal must neither + // synthesize from the REPL's prior recordings nor leak its diagnose + // actions into a later /save. var prior: Recorder = .init(self.allocator); + var prior_baseline: Baseline = .init(self.allocator); std.mem.swap(Recorder, &self.save_buffer, &prior); + std.mem.swap(Baseline, &self.baseline, &prior_baseline); defer { std.mem.swap(Recorder, &self.save_buffer, &prior); + std.mem.swap(Baseline, &self.baseline, &prior_baseline); prior.deinit(); + prior_baseline.deinit(); } var attempt: usize = 1; @@ -1851,17 +1867,13 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr // Validate in a fresh session so failure-state cookies and pages can't // mask a still-broken script. self.startSession() catch |err| { - self.removeTempScript(tmp_path); self.terminal.printError("heal failed: could not start a fresh session: {s}", .{@errorName(err)}); return false; }; switch (self.runScriptOutcome(arena, tmp_path)) { .ok => |facts| { - if (cureFailure(arena, first, facts) catch { - self.removeTempScript(tmp_path); - return self.healOom(); - }) |failure| { + if (cureFailure(arena, first, facts) catch return self.healOom()) |failure| { self.terminal.printWarning("{s}", .{failure}); source = revised; error_detail = failure; @@ -1872,6 +1884,7 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr if (refreshedBaselineScript(arena, revised, facts.extract_stats)) |updated| { save.writeContentFile(tmp_path, updated, .replace) catch {}; } + keep_revision = true; std.fs.cwd().rename(tmp_path, path) catch |err| { self.terminal.printError("healed script validated but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }); return false; @@ -1880,17 +1893,13 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr return true; } }, - .fatal => { - self.removeTempScript(tmp_path); - return false; - }, + .fatal => return false, .script_error => |script_error| { source = revised; error_detail = script_error.detail; }, } } - self.removeTempScript(tmp_path); self.terminal.printError("heal gave up after {d} attempts; {s} is unchanged", .{ max_heal_attempts, path }); return false; } @@ -2017,8 +2026,10 @@ fn refreshedBaselineScript(arena: std.mem.Allocator, revised: []const u8, stats: } fn removeTempScript(self: *Agent, tmp_path: []const u8) void { - std.fs.cwd().deleteFile(tmp_path) catch |err| { - self.terminal.printWarning("could not remove temp script {s}: {s}", .{ tmp_path, @errorName(err) }); + std.fs.cwd().deleteFile(tmp_path) catch |err| switch (err) { + // Covers exits before the first write. + error.FileNotFound => {}, + else => self.terminal.printWarning("could not remove temp script {s}: {s}", .{ tmp_path, @errorName(err) }), }; } @@ -2047,9 +2058,9 @@ fn recordSlashToolCall( .arguments = if (args) |v| try zenai.json.dupeValue(ma, v) else null, }; - // capToolOutput returns its input unchanged under the cap; dupe so content + // capOutput returns its input unchanged under the cap; dupe so content // doesn't alias the caller's per-iteration arena. - const capped = capToolOutput(ma, result.text); + const capped = capOutput(ma, result.text, tool_output_max_bytes); const content = if (capped.ptr == result.text.ptr) try ma.dupe(u8, capped) else capped; const tool_results = try ma.alloc(zenai.provider.ToolResult, 1); @@ -2324,9 +2335,11 @@ fn buildUserMessageParts( // the next request body) without bound. const tool_output_max_bytes: usize = 1 * 1024 * 1024; -fn capToolOutput(allocator: std.mem.Allocator, output: []const u8) []const u8 { - if (output.len <= tool_output_max_bytes) return output; - const prefix = string.truncateUtf8(output, tool_output_max_bytes); +/// Returns `output` unchanged when under `max_bytes`; a truncated-and-marked +/// copy in `allocator` otherwise (the bare prefix on OOM — never an error). +fn capOutput(allocator: std.mem.Allocator, output: []const u8, max_bytes: usize) []const u8 { + if (output.len <= max_bytes) return output; + const prefix = string.truncateUtf8(output, max_bytes); var suffix_buf: [64]u8 = undefined; const suffix = std.fmt.bufPrint(&suffix_buf, "\n...[truncated, original {d} bytes]", .{output.len}) catch return prefix; return std.mem.concat(allocator, u8, &.{ prefix, suffix }) catch prefix; @@ -2349,7 +2362,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co defer self.terminal.spinner.setThinking(); const outcome: zenai.provider.Client.ToolHandler.Result = if (self.callTool(allocator, tool_name, arguments)) |result| - .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error } + .{ .content = capOutput(allocator, result.text, tool_output_max_bytes), .is_error = result.is_error } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; @@ -2565,16 +2578,16 @@ test "savePrompt: save instructions followed by the rendered script skill" { try std.testing.expect(std.mem.endsWith(u8, revision, lp.skill.text())); } -test "capToolOutput: passes through when under cap" { +test "capOutput: passes through when under cap" { const ta = std.testing.allocator; - const out = capToolOutput(ta, "short"); + const out = capOutput(ta, "short", tool_output_max_bytes); try std.testing.expectEqualStrings("short", out); } // Boundary correctness lives in string.zig's `truncateUtf8` tests; here we only // assert the agent-specific policy: an over-cap body keeps valid UTF-8 and gains // the truncation marker. -test "capToolOutput: appends a marker when truncating" { +test "capOutput: appends a marker when truncating" { const ta = std.testing.allocator; // 3-byte Hangul codepoint (U+D55C '한' = 0xED 0x95 0x9C) straddling the cap. @@ -2587,7 +2600,7 @@ test "capToolOutput: appends a marker when truncating" { buf[cap + 1] = 0x9C; @memset(buf[cap + 2 ..], 'b'); - const out = capToolOutput(ta, buf); + const out = capOutput(ta, buf, tool_output_max_bytes); defer if (out.ptr != buf.ptr) ta.free(out); try std.testing.expect(std.unicode.utf8ValidateSlice(out)); diff --git a/src/agent/Baseline.zig b/src/agent/Baseline.zig index fe4c68a85f..3a0731a1c9 100644 --- a/src/agent/Baseline.zig +++ b/src/agent/Baseline.zig @@ -39,18 +39,14 @@ pub const FieldStat = struct { pub const Fields = std.StringArrayHashMapUnmanaged(FieldStat); arena: std.heap.ArenaAllocator, -/// Reused per `noteExtractResult` for the transient parse tree; keeps pages -/// warm on the every-tool-call path instead of an init/deinit pair each time. -scratch: std.heap.ArenaAllocator, fields: Fields = .empty, pub fn init(allocator: std.mem.Allocator) Baseline { - return .{ .arena = .init(allocator), .scratch = .init(allocator) }; + return .{ .arena = .init(allocator) }; } pub fn deinit(self: *Baseline) void { self.arena.deinit(); - self.scratch.deinit(); } pub fn reset(self: *Baseline) void { @@ -61,8 +57,9 @@ pub fn reset(self: *Baseline) void { /// Tally one successful extract result (the tool's JSON output). Malformed /// output records nothing — this is best-effort telemetry. pub fn noteExtractResult(self: *Baseline, result_text: []const u8) error{OutOfMemory}!void { - _ = self.scratch.reset(.retain_capacity); - const scratch = self.scratch.allocator(); + var scratch_state: std.heap.ArenaAllocator = .init(self.arena.child_allocator); + defer scratch_state.deinit(); + const scratch = scratch_state.allocator(); const parsed = std.json.parseFromSliceLeaky(std.json.Value, scratch, result_text, .{}) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return, @@ -123,6 +120,7 @@ fn fieldsToLine(arena: std.mem.Allocator, fields: *const Fields) error{OutOfMemo /// baseline line, or null) appended — synthesis may have copied a stale /// baseline from the previous script verbatim. pub fn withBaseline(arena: std.mem.Allocator, script: []const u8, line: ?[]const u8) ![]const u8 { + if (line == null and std.mem.indexOf(u8, script, marker) == null) return script; var aw: std.Io.Writer.Allocating = .init(arena); var lines = std.mem.splitScalar(u8, script, '\n'); var pending_newline = false; diff --git a/src/agent/Terminal.zig b/src/agent/Terminal.zig index 654e644c74..ed7e80c5a0 100644 --- a/src/agent/Terminal.zig +++ b/src/agent/Terminal.zig @@ -269,7 +269,7 @@ pub fn endAssistantStream(self: *Terminal) void { // Must exceed the downstream LLM-judge's snapshot window for full grounding // evidence. Does not cap the agent's own LLM, which gets up to -// tool_output_max_bytes (1 MiB) via Agent.zig:capToolOutput. Bypassed in REPL +// tool_output_max_bytes (1 MiB) via Agent.zig:capOutput. Bypassed in REPL // where the human can scroll. const max_result_display_len = 2000; From 83cbf916f923f755b616bfffa61aba7fb84a84be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <1671644+arrufat@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:45:22 +0200 Subject: [PATCH 12/15] agent: use trimEnd to help zig 0.16.0 Co-authored-by: Karl Seguin --- src/agent/Agent.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index de12bd9291..c3e6af9dac 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -1915,7 +1915,7 @@ const verdict_system_prompt = \\empty output means the script is broken (stale selectors after a site \\change) or legitimate (the page genuinely has no such data right now). \\A ` -++ std.mem.trimRight(u8, Baseline.marker, " ") ++ +++ std.mem.trimEnd(u8, Baseline.marker, " ") ++ \\` comment, when present, records how often each output \\field carried data when the script was saved — weigh it as evidence. \\Respond with JSON only: From 9866c8f73585865c4af454aa2df91859b1d0e3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 22 Jul 2026 14:04:49 +0200 Subject: [PATCH 13/15] agent: anchor verdict parsing with marker --- src/agent/Agent.zig | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index c3e6af9dac..a3d4559381 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -1816,7 +1816,8 @@ fn buildHealDiagnoseMessage(arena: std.mem.Allocator, path: []const u8, source: /// Heal synthesis instruction; rides on the regular save revision system prompt. const heal_revision_prompt = \\Fix the script so it replays successfully against the current site: the - \\error and this session's working tool calls identify the repair. Keep + \\error names what broke, and the diagnosis tool calls above that + \\succeeded against the live page show the repair. Keep \\every step, selector, and output shape that still works unchanged. \\Preserve the script's `//` intent comments; where you change a block, \\update its comment so it still describes what the revised code does, and @@ -1918,12 +1919,16 @@ const verdict_system_prompt = ++ std.mem.trimEnd(u8, Baseline.marker, " ") ++ \\` comment, when present, records how often each output \\field carried data when the script was saved — weigh it as evidence. - \\Respond with JSON only: - \\{"broken": true|false, "fields": ["", ...], "reason": ""} + \\Respond with your verdict on its own final line: + \\ +++ verdict_marker ++ + \\ {"broken": true|false, "fields": ["", ...], "reason": ""} \\`fields` names the output fields you judge broken; use "" for the whole \\return value. ; +const verdict_marker = "VERDICT:"; + const Verdict = struct { broken: bool, /// Output fields the model judged broken; null = the whole return value. @@ -1948,11 +1953,13 @@ fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, susp } fn parseVerdict(arena: std.mem.Allocator, raw: []const u8) ?Verdict { - const start = std.mem.indexOfScalar(u8, raw, '{') orelse return null; - const end = std.mem.lastIndexOfScalar(u8, raw, '}') orelse return null; + const at = std.mem.lastIndexOf(u8, raw, verdict_marker) orelse return null; + const rest = raw[at + verdict_marker.len ..]; + const start = std.mem.indexOfScalar(u8, rest, '{') orelse return null; + const end = std.mem.lastIndexOfScalar(u8, rest, '}') orelse return null; if (end < start) return null; const Wire = struct { broken: bool, fields: []const []const u8 = &.{}, reason: []const u8 = "" }; - const wire = std.json.parseFromSliceLeaky(Wire, arena, raw[start .. end + 1], .{ .ignore_unknown_fields = true }) catch return null; + const wire = std.json.parseFromSliceLeaky(Wire, arena, rest[start .. end + 1], .{ .ignore_unknown_fields = true }) catch return null; const fields = arena.alloc(?[]const u8, wire.fields.len) catch return null; for (wire.fields, fields) |f, *out| out.* = if (f.len == 0) null else f; return .{ .broken = wire.broken, .fields = fields, .reason = wire.reason }; @@ -2552,20 +2559,24 @@ test "suspicionOf: any all-empty field is suspect, none is not" { try std.testing.expectEqual(ScriptError.Kind.empty, suspicionOf(aa, empty_facts).?.kind); } -test "parseVerdict: tolerant of prose around the JSON" { +test "parseVerdict: anchored to the marker, tolerant of prose around it" { var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); const aa = arena.allocator(); - const v = parseVerdict(aa, "Sure!\n{\"broken\": true, \"fields\": [\"comments\", \"\"], \"reason\": \"selector drift\"}").?; + const v = parseVerdict(aa, "The {comments} baseline says data was always present.\nVERDICT: {\"broken\": true, \"fields\": [\"comments\", \"\"], \"reason\": \"selector drift\"}").?; try std.testing.expect(v.broken); try std.testing.expectEqual(2, v.fields.len); try std.testing.expectEqualStrings("comments", v.fields[0].?); try std.testing.expectEqual(null, v.fields[1]); try std.testing.expectEqualStrings("selector drift", v.reason); - try std.testing.expectEqual(null, parseVerdict(aa, "no json here")); - try std.testing.expectEqual(null, parseVerdict(aa, "{\"fields\": []}")); + const fenced = parseVerdict(aa, "As instructed, VERDICT: first.\nVERDICT:\n```json\n{\"broken\": false}\n```").?; + try std.testing.expect(!fenced.broken); + + try std.testing.expectEqual(null, parseVerdict(aa, "{\"broken\": true, \"fields\": []}")); + try std.testing.expectEqual(null, parseVerdict(aa, "VERDICT: no json here")); + try std.testing.expectEqual(null, parseVerdict(aa, "VERDICT: {\"fields\": []}")); } test "savePrompt: save instructions followed by the rendered script skill" { From 738536719b4187136edb9e91f1f6d9fe47161bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 22 Jul 2026 15:40:11 +0200 Subject: [PATCH 14/15] mcp: add script replay and heal_commit tools --- src/agent/Agent.zig | 339 ++------------------- src/agent/Terminal.zig | 2 +- src/lightpanda.zig | 2 + src/mcp/Server.zig | 18 +- src/mcp/tools.zig | 464 +++++++++++++++++++++++++++++ src/{agent => script}/Baseline.zig | 0 src/script/Runtime.zig | 42 ++- src/script/heal.zig | 451 ++++++++++++++++++++++++++++ src/string.zig | 25 ++ 9 files changed, 1032 insertions(+), 311 deletions(-) rename src/{agent => script}/Baseline.zig (100%) create mode 100644 src/script/heal.zig diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index a3d4559381..7c82d50e94 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -29,7 +29,8 @@ const Command = lp.Command; const Schema = lp.Schema; const Recorder = lp.Recorder; const ScriptRuntime = lp.Runtime; -const Baseline = @import("Baseline.zig"); +const Baseline = lp.Baseline; + const Credentials = zenai.provider.Credentials; const App = @import("../App.zig"); @@ -1559,6 +1560,9 @@ const ScriptOutput = struct { } }; +const RunFacts = lp.heal.RunFacts; +const ScriptError = lp.heal.ScriptError; + /// `fatal` covers setup failures (unreadable file, runtime init, OOM) that a /// retry can't help. const ScriptRunOutcome = union(enum) { @@ -1567,47 +1571,6 @@ const ScriptRunOutcome = union(enum) { script_error: ScriptError, }; -/// What a completed run returned, as far as heal cares. -const Returned = union(enum) { - /// No `return`, or a value whose display form couldn't be computed. - none, - /// A value carrying data. - data, - /// A deep-empty value, carrying its capped display text. - empty: []const u8, -}; - -/// Facts about a run that completed without throwing — suspicion is judged by -/// the model, never here. Duped into the caller's arena — the runtime dies -/// with `runScriptOutcome`. -const RunFacts = struct { - returned: Returned, - extract_stats: []const ScriptRuntime.ExtractStat, - source: []const u8, -}; - -/// Both slices are duped into the caller's arena. `source` is the exact text -/// that ran, so a heal diagnoses what actually failed instead of re-reading a -/// possibly-changed file. -const ScriptError = struct { - kind: Kind, - /// Formatted error (line, stack) — or, for `empty`, what came back. - detail: []const u8, - source: []const u8, - /// For `dry_extracts`: the field names that were empty on every call - /// (null = a whole-array schema). The cure check requires each one to - /// come back with data before a heal may replace the file. - dry_fields: []const ?[]const u8 = &.{}, - - /// `empty` is a run that completed but returned a value with no data in - /// it; `dry_extracts` one whose return value had data, but where some - /// extract list field came back empty on every call. Both are the usual - /// symptom of a stale selector, which matches nothing instead of throwing. - /// Only heal treats them as failures; a plain replay still exits 0, since - /// an empty answer can be the right answer. - const Kind = enum { threw, empty, dry_extracts }; -}; - fn runScript(self: *Agent, path: []const u8) bool { var arena: std.heap.ArenaAllocator = .init(self.allocator); defer arena.deinit(); @@ -1666,102 +1629,26 @@ fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) Sc const result = runtime.runSource(content, path); self.terminal.endTool(); - const ok = switch (result catch |err| { + const run_result = result catch |err| { self.terminal.printError("Script failed: {s}", .{@errorName(err)}); return .fatal; - }) { - .err => |message| { - self.terminal.printError("{s}", .{message}); - // A Ctrl-C termination is not a script defect — never heal it. - if (self.cancel_requested.load(.acquire)) return .fatal; - // The message lives in the runtime's call arena and the source in - // script_arena; both are freed by the defers above — dupe first. - return .{ .script_error = .{ - .kind = .threw, - .detail = arena.dupe(u8, message) catch return .fatal, - .source = arena.dupe(u8, content) catch return .fatal, - } }; - }, - .ok => |ok| ok, }; - - // A script that printed nothing leaves no trace, so freeze the spinner into - // a green bullet (like /goto); one that printed already showed its result. - if (!output.emitted) self.terminal.printScriptDone("script", path); - const returned: Returned = if (ok.completion) |c| - (if (c.empty) .{ .empty = capDetail(arena, c.text) catch return .fatal } else .data) - else - .none; - return .{ .ok = .{ - .returned = returned, - .extract_stats = dupeExtractStats(arena, ok.extract_stats) catch return .fatal, - .source = arena.dupe(u8, content) catch return .fatal, - } }; -} - -fn dupeExtractStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) error{OutOfMemory}![]const ScriptRuntime.ExtractStat { - const out = try arena.alloc(ScriptRuntime.ExtractStat, stats.len); - for (stats, out) |stat, *o| { - o.* = .{ - .schema = try arena.dupe(u8, stat.schema), - .field = if (stat.field) |f| try arena.dupe(u8, f) else null, - .calls = stat.calls, - .empty = stat.empty, - }; - } - return out; -} - -/// Bound a value or schema echoed into a heal message; a degenerate empty-ish -/// result (hundreds of all-null rows) would otherwise bloat the LLM turn. -const detail_max_bytes: usize = 2048; - -/// `capOutput` at `detail_max_bytes`, always duped — `RunFacts` details must -/// outlive the runtime whose arena the text came from. -fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]const u8 { - return arena.dupe(u8, capOutput(arena, text, detail_max_bytes)); -} - -/// A finding worth a verdict, not yet confirmed: the return value was -/// deep-empty, or some extract field came back empty on every call — any field, -/// scalar or list, baseline or not. Whether that is breakage or legitimate -/// sparseness is the model's judgment, not encoded here. -fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?ScriptError { - switch (facts.returned) { - .empty => |text| return .{ - .kind = .empty, - .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, - .source = facts.source, - }, - .none, .data => {}, - } - return dryExtractsFinding(arena, facts.source, facts.extract_stats) catch return null; -} - -/// A `dry_extracts` finding with one detail line per extract field that came -/// back empty on every call, plus the field names for the cure check. Null when -/// no field was dry. -fn dryExtractsFinding(arena: std.mem.Allocator, source: []const u8, stats: []const ScriptRuntime.ExtractStat) !?ScriptError { - var aw: std.Io.Writer.Allocating = .init(arena); - var fields: std.ArrayList(?[]const u8) = .empty; - for (stats) |stat| { - if (stat.empty != stat.calls) continue; - if (fields.items.len == 0) { - try aw.writer.writeAll("some extracts came back empty on every call:\n"); - } - // `stat.field` already lives in `arena` (facts were duped into it). - try fields.append(arena, stat.field); - const schema = try capDetail(arena, stat.schema); - if (stat.field) |field| { - try aw.writer.print("- the \"{s}\" field in extract({s}) came back empty", .{ field, schema }); - } else { - try aw.writer.print("- extract({s}) returned no data", .{schema}); - } - if (stat.calls != 1) try aw.writer.print(" in all {d} calls", .{stat.calls}); - try aw.writer.writeAll("\n"); - } - if (fields.items.len == 0) return null; - return .{ .kind = .dry_extracts, .detail = aw.written(), .source = source, .dry_fields = fields.items }; + if (run_result == .err) { + self.terminal.printError("{s}", .{run_result.err}); + // A Ctrl-C termination is not a script defect — never heal it. + if (self.cancel_requested.load(.acquire)) return .fatal; + } else { + // A script that printed nothing leaves no trace, so freeze the spinner + // into a green bullet (like /goto); one that printed already showed its + // result. + if (!output.emitted) self.terminal.printScriptDone("script", path); + } + // `content` dies with script_arena; the outcome's source must outlive it. + const source = arena.dupe(u8, content) catch return .fatal; + return switch (lp.heal.classifyRun(arena, run_result, source) catch return .fatal) { + .facts => |facts| .{ .ok = facts }, + .script_error => |script_error| .{ .script_error = script_error }, + }; } /// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. @@ -1775,7 +1662,7 @@ fn runScriptWithHeal(self: *Agent, path: []const u8) bool { // the ones healing would act on. switch (self.runScriptOutcome(arena.allocator(), path)) { .fatal => return false, - .ok => |facts| if (suspicionOf(arena.allocator(), facts) == null) return true, + .ok => |facts| if (lp.heal.suspicionOf(arena.allocator(), facts) == null) return true, .script_error => {}, } self.terminal.printInfo("Script run failed or looks broken; retrying once before healing.", .{}); @@ -1791,39 +1678,6 @@ fn runScriptWithHeal(self: *Agent, path: []const u8) bool { const max_heal_attempts = 2; -fn buildHealDiagnoseMessage(arena: std.mem.Allocator, path: []const u8, source: []const u8, error_detail: []const u8) ![]const u8 { - return std.fmt.allocPrint(arena, - \\Replaying the saved script {s} failed. The browser session is still - \\at the failure state. - \\ - \\The script (its comments and structure carry the intent): - \\```js - \\{s} - \\``` - \\ - \\The error: - \\{s} - \\ - \\Diagnose the failure: inspect the live page (tree, findElement, - \\markdown) to see how the site differs from what the script expects, - \\then perform the corrected step(s) with tools to prove they work — - \\verify selectors against the live page, never guess. If the failing - \\step gated the rest of the script (a login, a navigation), carry on - \\far enough to show the script's goal is reachable again. - , .{ path, source, error_detail }); -} - -/// Heal synthesis instruction; rides on the regular save revision system prompt. -const heal_revision_prompt = - \\Fix the script so it replays successfully against the current site: the - \\error names what broke, and the diagnosis tool calls above that - \\succeeded against the live page show the repair. Keep - \\every step, selector, and output shape that still works unchanged. - \\Preserve the script's `//` intent comments; where you change a block, - \\update its comment so it still describes what the revised code does, and - \\add one for any block that lacks it. -; - /// Only a revision that passed validation in a fresh session replaces `path`; /// the original survives a failed heal. fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: ScriptError) bool { @@ -1855,10 +1709,10 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr while (attempt <= max_heal_attempts) : (attempt += 1) { self.terminal.printInfo("Healing {s} (attempt {d}/{d})", .{ path, attempt, max_heal_attempts }); - const diagnose = buildHealDiagnoseMessage(arena, path, source, error_detail) catch return self.healOom(); + const diagnose = lp.heal.buildDiagnoseMessage(arena, path, source, error_detail) catch return self.healOom(); if (!self.runTurn(.{ .prompt = diagnose, .capture_for_save = true, .label = "Heal" })) return false; - const revised = self.synthesizeScriptText(arena, "heal", path, source, heal_revision_prompt) orelse return false; + const revised = self.synthesizeScriptText(arena, "heal", path, source, lp.heal.heal_revision_prompt) orelse return false; save.writeContentFile(tmp_path, revised, .replace) catch |err| { self.terminal.printError("heal failed: could not write {s}: {s}", .{ tmp_path, @errorName(err) }); @@ -1874,7 +1728,7 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr switch (self.runScriptOutcome(arena, tmp_path)) { .ok => |facts| { - if (cureFailure(arena, first, facts) catch return self.healOom()) |failure| { + if (lp.heal.cureFailure(arena, first, facts) catch return self.healOom()) |failure| { self.terminal.printWarning("{s}", .{failure}); source = revised; error_detail = failure; @@ -1882,7 +1736,7 @@ fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: Scr // Refresh the baseline from the validation run — synthesis // may have copied the stale one from the broken script. // Best-effort: the validated revision is already on disk. - if (refreshedBaselineScript(arena, revised, facts.extract_stats)) |updated| { + if (lp.heal.refreshedBaselineScript(arena, revised, facts.extract_stats)) |updated| { save.writeContentFile(tmp_path, updated, .replace) catch {}; } keep_revision = true; @@ -1970,7 +1824,7 @@ fn parseVerdict(arena: std.mem.Allocator, raw: []const u8) ?Verdict { /// verdict call falls back to the raw facts — the heal prompt, defaulting to /// no, becomes the judgment of last resort. fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts: RunFacts) ?ScriptError { - const suspicion = suspicionOf(arena, facts) orelse return null; + const suspicion = lp.heal.suspicionOf(arena, facts) orelse return null; if (self.ai_client == null) return null; if (self.judgeSuspicion(arena, path, suspicion)) |verdict| { if (!verdict.broken) { @@ -2004,34 +1858,6 @@ fn confirmedDryFields(arena: std.mem.Allocator, judged: []const ?[]const u8, act return if (kept.items.len == 0) actual else kept.items; } -/// Null when the validation run cured the original finding; otherwise the -/// message fed to the next heal attempt. Running clean is not a cure on its -/// own — a revision that deletes the failing extract (or the `return`) also -/// runs clean. -fn cureFailure(arena: std.mem.Allocator, first: ScriptError, facts: RunFacts) error{OutOfMemory}!?[]const u8 { - switch (first.kind) { - .threw => return null, - .empty => return if (facts.returned == .data) - null - else - "The revised script ran, but still returns no data (or no longer returns anything) — the original returned a value.", - .dry_extracts => { - for (first.dry_fields) |dry| { - const cured = for (facts.extract_stats) |stat| { - if (ScriptRuntime.fieldEql(stat.field, dry) and stat.empty < stat.calls) break true; - } else false; - if (!cured) return try std.fmt.allocPrint(arena, "The revised script ran, but the \"{s}\" extract still came back empty on every call (or was removed) — keep it and fix its selector.", .{dry orelse ""}); - } - return null; - }, - } -} - -fn refreshedBaselineScript(arena: std.mem.Allocator, revised: []const u8, stats: []const ScriptRuntime.ExtractStat) ?[]const u8 { - const line = Baseline.serializeStats(arena, stats) catch return null; - return Baseline.withBaseline(arena, revised, line) catch null; -} - fn removeTempScript(self: *Agent, tmp_path: []const u8) void { std.fs.cwd().deleteFile(tmp_path) catch |err| switch (err) { // Covers exits before the first write. @@ -2065,9 +1891,9 @@ fn recordSlashToolCall( .arguments = if (args) |v| try zenai.json.dupeValue(ma, v) else null, }; - // capOutput returns its input unchanged under the cap; dupe so content + // capBytes returns its input unchanged under the cap; dupe so content // doesn't alias the caller's per-iteration arena. - const capped = capOutput(ma, result.text, tool_output_max_bytes); + const capped = string.capBytes(ma, result.text, tool_output_max_bytes); const content = if (capped.ptr == result.text.ptr) try ma.dupe(u8, capped) else capped; const tool_results = try ma.alloc(zenai.provider.ToolResult, 1); @@ -2342,16 +2168,6 @@ fn buildUserMessageParts( // the next request body) without bound. const tool_output_max_bytes: usize = 1 * 1024 * 1024; -/// Returns `output` unchanged when under `max_bytes`; a truncated-and-marked -/// copy in `allocator` otherwise (the bare prefix on OOM — never an error). -fn capOutput(allocator: std.mem.Allocator, output: []const u8, max_bytes: usize) []const u8 { - if (output.len <= max_bytes) return output; - const prefix = string.truncateUtf8(output, max_bytes); - var suffix_buf: [64]u8 = undefined; - const suffix = std.fmt.bufPrint(&suffix_buf, "\n...[truncated, original {d} bytes]", .{output.len}) catch return prefix; - return std.mem.concat(allocator, u8, &.{ prefix, suffix }) catch prefix; -} - fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) zenai.provider.Client.ToolHandler.Result { const self: *Agent = @ptrCast(@alignCast(ctx)); // Close any assistant text streamed this turn before the tool spinner shows. @@ -2369,7 +2185,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co defer self.terminal.spinner.setThinking(); const outcome: zenai.provider.Client.ToolHandler.Result = if (self.callTool(allocator, tool_name, arguments)) |result| - .{ .content = capOutput(allocator, result.text, tool_output_max_bytes), .is_error = result.is_error } + .{ .content = string.capBytes(allocator, result.text, tool_output_max_bytes), .is_error = result.is_error } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; @@ -2473,48 +2289,6 @@ test { _ = picker; } -test "cureFailure: running clean is not a cure" { - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const aa = arena.allocator(); - - const dry: ScriptError = .{ - .kind = .dry_extracts, - .detail = "", - .source = "", - .dry_fields = &.{ @as(?[]const u8, "comments"), null }, - }; - const cured_stats: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, - .{ .schema = "[]", .field = null, .calls = 1, .empty = 0 }, - }; - try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(.data, cured_stats))); - - // Fix-by-deletion: the dry field is simply gone from the revised run. - const deleted = (try cureFailure(aa, dry, testFacts(.data, cured_stats[1..]))).?; - try std.testing.expect(std.mem.indexOf(u8, deleted, "\"comments\"") != null); - - // Still dry counts as uncured. - const still_dry_stats: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 5 }, - cured_stats[1], - }; - try std.testing.expect((try cureFailure(aa, dry, testFacts(.data, still_dry_stats))) != null); - - // .empty is cured only by a data-carrying return. - const empty: ScriptError = .{ .kind = .empty, .detail = "", .source = "" }; - try std.testing.expectEqual(null, try cureFailure(aa, empty, testFacts(.data, &.{}))); - try std.testing.expect((try cureFailure(aa, empty, testFacts(.none, &.{}))) != null); - - // .threw needs nothing beyond running clean. - const threw: ScriptError = .{ .kind = .threw, .detail = "", .source = "" }; - try std.testing.expectEqual(null, try cureFailure(aa, threw, testFacts(.none, &.{}))); -} - -fn testFacts(returned: Returned, stats: []const ScriptRuntime.ExtractStat) RunFacts { - return .{ .returned = returned, .extract_stats = stats, .source = "" }; -} - test "confirmedDryFields drops hallucinated names, keeps the narrowing" { var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); @@ -2537,28 +2311,6 @@ test "confirmedDryFields drops hallucinated names, keeps the narrowing" { try std.testing.expectEqual(actual.ptr, (try confirmedDryFields(aa, &.{}, actual)).ptr); } -test "suspicionOf: any all-empty field is suspect, none is not" { - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - const aa = arena.allocator(); - - const sparse: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, - }; - try std.testing.expectEqual(null, suspicionOf(aa, testFacts(.data, sparse))); - - // Scalar all-empty is suspect too — judgment belongs to the model now. - const dry_scalar: []const ScriptRuntime.ExtractStat = &.{ - .{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }, - }; - const s = suspicionOf(aa, testFacts(.data, dry_scalar)).?; - try std.testing.expectEqual(ScriptError.Kind.dry_extracts, s.kind); - try std.testing.expectEqual(1, s.dry_fields.len); - - const empty_facts = testFacts(.{ .empty = "[]" }, &.{}); - try std.testing.expectEqual(ScriptError.Kind.empty, suspicionOf(aa, empty_facts).?.kind); -} - test "parseVerdict: anchored to the marker, tolerant of prose around it" { var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); @@ -2588,32 +2340,3 @@ test "savePrompt: save instructions followed by the rendered script skill" { try std.testing.expect(std.mem.indexOf(u8, revision, save_revision_note) != null); try std.testing.expect(std.mem.endsWith(u8, revision, lp.skill.text())); } - -test "capOutput: passes through when under cap" { - const ta = std.testing.allocator; - const out = capOutput(ta, "short", tool_output_max_bytes); - try std.testing.expectEqualStrings("short", out); -} - -// Boundary correctness lives in string.zig's `truncateUtf8` tests; here we only -// assert the agent-specific policy: an over-cap body keeps valid UTF-8 and gains -// the truncation marker. -test "capOutput: appends a marker when truncating" { - const ta = std.testing.allocator; - - // 3-byte Hangul codepoint (U+D55C '한' = 0xED 0x95 0x9C) straddling the cap. - const cap = tool_output_max_bytes; - const buf = try ta.alloc(u8, cap + 8); - defer ta.free(buf); - @memset(buf[0 .. cap - 1], 'a'); - buf[cap - 1] = 0xED; - buf[cap + 0] = 0x95; - buf[cap + 1] = 0x9C; - @memset(buf[cap + 2 ..], 'b'); - - const out = capOutput(ta, buf, tool_output_max_bytes); - defer if (out.ptr != buf.ptr) ta.free(out); - - try std.testing.expect(std.unicode.utf8ValidateSlice(out)); - try std.testing.expect(std.mem.indexOf(u8, out, "truncated") != null); -} diff --git a/src/agent/Terminal.zig b/src/agent/Terminal.zig index ed7e80c5a0..e6a10fe906 100644 --- a/src/agent/Terminal.zig +++ b/src/agent/Terminal.zig @@ -269,7 +269,7 @@ pub fn endAssistantStream(self: *Terminal) void { // Must exceed the downstream LLM-judge's snapshot window for full grounding // evidence. Does not cap the agent's own LLM, which gets up to -// tool_output_max_bytes (1 MiB) via Agent.zig:capOutput. Bypassed in REPL +// tool_output_max_bytes (1 MiB) via string.zig:capBytes. Bypassed in REPL // where the human can scroll. const max_result_display_len = 2000; diff --git a/src/lightpanda.zig b/src/lightpanda.zig index df0bd4e934..9fb13473e4 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -48,8 +48,10 @@ pub const HttpClient = @import("network/HttpClient.zig"); pub const mcp = @import("mcp.zig"); pub const Agent = @import("agent/Agent.zig"); pub const Command = @import("script/command.zig").Command; +pub const Baseline = @import("script/Baseline.zig"); pub const Recorder = @import("script/Recorder.zig"); pub const Runtime = @import("script/Runtime.zig"); +pub const heal = @import("script/heal.zig"); pub const Schema = @import("script/Schema.zig"); pub const skill = @import("script/skill.zig"); pub const cookies = @import("cookies.zig"); diff --git a/src/mcp/Server.zig b/src/mcp/Server.zig index 0455f7d26a..fa3e3e8502 100644 --- a/src/mcp/Server.zig +++ b/src/mcp/Server.zig @@ -120,6 +120,22 @@ pub fn createSession(self: *Self, id: []const u8) !*Session { return entry; } +/// Replace `entry`'s browsing session with a fresh one — pages, cookies and +/// node ids dropped — as heal validation requires. The default session +/// reloads the on-disk cookie file: that file is the clean baseline identity; +/// the in-memory failure-state cookies are what the fresh session discards. +pub fn restartSession(self: *Self, entry: *Session) !void { + entry.session = try entry.browser.newSession(entry.notification); + try entry.session.enableConsoleCapture(); + // Node IDs are session-scoped; drop them with the session they point into. + entry.node_registry.reset(); + if (entry.isDefault()) { + if (self.app.config.cookieFile()) |cookie_path| { + lp.cookies.loadFromFile(entry.session, cookie_path); + } + } +} + /// Switch to the multi-isolate discipline: park the default (which `Server.init` /// left entered) and require every use to bracket with `enterIsolate`. The HTTP /// transport calls this on its worker thread before serving anyone. @@ -220,7 +236,7 @@ pub fn handleInitialize(self: *Self, req: protocol.Request) !void { .tools = .{}, }, .serverInfo = .{ .name = "lightpanda", .version = "0.1.0" }, - .instructions = lp.tools.driver_guidance, + .instructions = lp.tools.driver_guidance ++ tools.script_lifecycle_note, }); } diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig index df3b7adc7a..9b79385cbf 100644 --- a/src/mcp/tools.zig +++ b/src/mcp/tools.zig @@ -4,6 +4,8 @@ const lp = @import("lightpanda"); const js = lp.js; const browser_tools = lp.tools; const BrowserTool = browser_tools.Tool; +const ScriptRuntime = lp.Runtime; +const string = @import("../string.zig"); const protocol = @import("protocol.zig"); const Server = @import("Server.zig"); @@ -55,7 +57,59 @@ const session_id_schema = browser_tools.minify( \\} ); +const replay_schema = browser_tools.minify( + \\{ + \\ "type": "object", + \\ "properties": { + \\ "path": { "type": "string", "description": "Relative path (no '..' segments) of the saved script to replay." }, + \\ "script": { "type": "string", "description": "Optional: script text to run instead of the file's contents - trial a candidate revision without writing it. `path` still names the run." } + \\ }, + \\ "required": ["path"] + \\} +); + +const heal_commit_schema = browser_tools.minify( + \\{ + \\ "type": "object", + \\ "properties": { + \\ "path": { "type": "string", "description": "Relative path (no '..' segments) of the broken script; replaced atomically only on cure." }, + \\ "script": { "type": "string", "description": "The full revised script. Keep $LP_* placeholders; never inline a resolved secret." }, + \\ "failure": { + \\ "type": "object", + \\ "description": "The `failure` object from the replay report you are healing, echoed back verbatim.", + \\ "properties": { + \\ "kind": { "type": "string", "enum": ["threw", "empty", "dry_extracts"] }, + \\ "detail": { "type": "string" }, + \\ "dry_fields": { "type": "array", "items": { "type": "string" }, "description": "For dry_extracts: the fields that must come back with data (\"\" = a whole-array extract). Deleting an extract is not a cure." } + \\ }, + \\ "required": ["kind"] + \\ } + \\ }, + \\ "required": ["path", "script", "failure"] + \\} +); + +/// Appended to the `initialize` instructions (`driver_guidance` is shared with +/// the standalone agent, which has no replay/heal tools — keep this MCP-only). +pub const script_lifecycle_note = + \\Script lifecycle: `save` a finished session as a script, `replay` it + \\any time for a token-free re-run, and when a replay reports it broken, + \\heal it — diagnose against the live session, then `heal_commit` a + \\revision (validated in a fresh session before it replaces the file). + \\ +; + const extra_tools = [_]McpTool{ + .{ + .name = "replay", + .description = "Replay a saved Lightpanda agent script (see `save`) and return a JSON run report. `status` is \"ok\" (ran, output carries data), \"suspicious\" (ran clean but the output looks dry — judge whether that is breakage or the page genuinely has no such data right now, weighing any `// lp:baseline` comment in `source` as evidence of what the fields held at save time), or \"failed\" (the script threw). The script's `console.*` output and returned value arrive in `console` (the returned value is the final line). On suspicious/failed the report carries the script `source`, a `failure` object and `guidance` for the heal flow: diagnose against the live session, then call `heal_commit`. Pass `script` to trial a candidate revision without writing it. The replay drives this session — the current page, cookies and node ids are replaced; re-inspect (tree) before reusing node ids.", + .inputSchema = replay_schema, + }, + .{ + .name = "heal_commit", + .description = "Commit a healed script: the revised `script` is validated by replaying it in a fresh session, and only a validated cure replaces the file at `path` — the original is untouched otherwise. Echo back the `failure` object from the replay report you are healing; the cure check is deterministic: `threw` needs a clean run, `empty` needs the return value to carry data, `dry_extracts` needs every listed field to come back with data (deleting the extract is not a cure). The response is a JSON heal report; on `cured: false` its `failure` says what is still wrong — diagnose further and try again. Afterwards the session is the fresh validation session at the script's end state (all prior node ids are stale).\n\n" ++ lp.heal.heal_revision_prompt ++ "\n\n" ++ browser_tools.save_synthesis_prompt ++ "\n\n" ++ browser_tools.save_script_rules, + .inputSchema = heal_commit_schema, + }, .{ .name = "save", .description = "Save the session as a reusable Lightpanda agent script. You hold the conversation, so synthesize the `script` yourself — `const page = new Page(); await page.goto(url);` then call the builtins you used as tools (extract, click, fill, …) as methods on `page` with the same object arguments. Keep `$LP_*` placeholders; never inline a resolved secret.\n\n" ++ browser_tools.save_synthesis_prompt ++ "\n\n" ++ browser_tools.save_script_rules, @@ -82,6 +136,8 @@ const all_tools = browser_tool_list ++ extra_tools; /// Tools that bypass the browser-tool dispatch and have their own handlers. const ExtraTool = enum { + replay, + heal_commit, save, session_new, session_list, @@ -104,6 +160,8 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque if (std.meta.stringToEnum(ExtraTool, call_params.name)) |tool| { return switch (tool) { + .replay => handleReplay(server, arena, id, call_params.arguments), + .heal_commit => handleHealCommit(server, arena, id, call_params.arguments), .save => handleSave(server, arena, id, call_params.arguments), .session_new => handleSessionNew(server, arena, id, call_params.arguments), .session_list => handleSessionList(server, arena, id), @@ -179,6 +237,204 @@ fn handleSave(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arg try sendToolResultText(server, id, msg, false); } +// Agent parity: the CLI reads scripts with the same bound. +const max_script_bytes = 10 * 1024 * 1024; + +/// Bound `RunReport.source` — a script is human-scale, but a synthesized one +/// with an embedded blob shouldn't balloon the report. +const source_max_bytes = 64 * 1024; + +/// Caps captured `console.*` output so a chatty script can't balloon the +/// report. +const ConsoleCollector = struct { + arena: std.mem.Allocator, + lines: std.ArrayList(lp.heal.ConsoleLine) = .empty, + bytes: usize = 0, + truncated: bool = false, + + const max_bytes = 16 * 1024; + + fn sink(self: *ConsoleCollector) ScriptRuntime.ConsoleSink { + return .{ .context = @ptrCast(self), .write = write }; + } + + fn write(context: *anyopaque, method: ScriptRuntime.ConsoleMethod, line: []const u8) void { + const self: *ConsoleCollector = @ptrCast(@alignCast(context)); + if (self.bytes >= max_bytes) { + self.truncated = true; + return; + } + // Scrub any resolved LP_* secret a script may have printed. + const scrubbed = browser_tools.reverseSubstituteEnvVars(self.arena, line) catch { + self.truncated = true; + return; + }; + const capped = string.capBytes(self.arena, scrubbed, max_bytes - self.bytes); + if (capped.ptr != scrubbed.ptr) self.truncated = true; + // A line that passed through both unchanged still aliases the + // runtime's per-call arena, which dies before the report is sent. + const text = if (capped.ptr == line.ptr) + self.arena.dupe(u8, capped) catch { + self.truncated = true; + return; + } + else + capped; + self.lines.append(self.arena, .{ .level = @tagName(method), .text = text }) catch { + self.truncated = true; + return; + }; + self.bytes += text.len; + } +}; + +fn runClassified(server: *Server, arena: std.mem.Allocator, path: []const u8, source: []const u8, collector: *ConsoleCollector) !lp.heal.Classified { + const active = server.active_session; + const runtime = try ScriptRuntime.init(server.allocator, server.app, active.session, &active.node_registry); + defer runtime.deinit(); + runtime.console_sink = collector.sink(); + const result = try runtime.runSource(source, path); + return lp.heal.classifyRun(arena, result, source); +} + +fn buildRunReport( + arena: std.mem.Allocator, + path: []const u8, + classified: lp.heal.Classified, + collector: *const ConsoleCollector, + with_guidance: bool, +) error{OutOfMemory}!lp.heal.RunReport { + var report: lp.heal.RunReport = .{ + .status = .ok, + .path = path, + .console = collector.lines.items, + .console_truncated = collector.truncated, + }; + switch (classified) { + .script_error => |script_error| { + report.status = .failed; + report.failure = try lp.heal.wireFailure(arena, script_error); + report.source = try scrubbedSource(arena, script_error.source); + if (with_guidance) report.guidance = lp.heal.replay_failed_guidance; + }, + .facts => |facts| { + report.returned = facts.returned; + report.extracts = facts.extract_stats; + if (lp.heal.suspicionOf(arena, facts)) |suspicion| { + report.status = .suspicious; + report.failure = try lp.heal.wireFailure(arena, suspicion); + report.source = try scrubbedSource(arena, facts.source); + if (with_guidance) report.guidance = lp.heal.replay_suspicious_guidance; + } + }, + } + return report; +} + +fn scrubbedSource(arena: std.mem.Allocator, source: []const u8) error{OutOfMemory}![]const u8 { + return string.capBytes(arena, try browser_tools.reverseSubstituteEnvVars(arena, source), source_max_bytes); +} + +fn sendReport(server: *Server, arena: std.mem.Allocator, id: std.json.Value, report: anytype) !void { + const json = std.json.Stringify.valueAlloc(arena, report, .{ .emit_null_optional_fields = false }) catch + return sendErrorContent(server, id, "out of memory"); + try sendToolResultText(server, id, json, false); +} + +fn handleReplay(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void { + const Args = struct { path: []const u8, script: ?[]const u8 = null }; + const args = browser_tools.parseArgs(Args, arena, arguments) catch { + return server.sendError(id, .InvalidParams, "expected { path: string, script?: string }"); + }; + if (!browser_tools.isPathSafe(args.path)) { + return sendErrorContent(server, id, "path must be relative and must not contain '..' segments"); + } + const source = args.script orelse std.fs.cwd().readFileAlloc(arena, args.path, max_script_bytes) catch |err| { + const msg = std.fmt.allocPrint(arena, "could not read {s}: {s}", .{ args.path, @errorName(err) }) catch "could not read script"; + return sendErrorContent(server, id, msg); + }; + + var collector: ConsoleCollector = .{ .arena = arena }; + const classified = runClassified(server, arena, args.path, source, &collector) catch |err| switch (err) { + error.OutOfMemory => return sendErrorContent(server, id, "out of memory"), + error.RuntimeInitFailed, error.TooManyContexts => return sendErrorContent(server, id, "could not initialize the script runtime"), + }; + // A failed script is still a successful replay: report it in-band, never + // as a tool error — the report is the answer. + const report = buildRunReport(arena, args.path, classified, &collector, true) catch + return sendErrorContent(server, id, "out of memory"); + return sendReport(server, arena, id, report); +} + +fn handleHealCommit(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void { + const Args = struct { path: []const u8, script: []const u8, failure: lp.heal.WireFailure }; + const args = browser_tools.parseArgs(Args, arena, arguments) catch { + return server.sendError(id, .InvalidParams, "expected { path: string, script: string, failure: { kind, detail?, dry_fields? } }"); + }; + if (!browser_tools.isPathSafe(args.path)) { + return sendErrorContent(server, id, "path must be relative and must not contain '..' segments"); + } + const first = lp.heal.scriptErrorFromWire(arena, args.failure) catch + return sendErrorContent(server, id, "out of memory"); + // The client never sees resolved secrets, but scrub as a safety net + // before running or persisting the candidate. + const script = browser_tools.reverseSubstituteEnvVars(arena, args.script) catch + return sendErrorContent(server, id, "out of memory"); + + // Validate in a fresh session so failure-state cookies and pages can't + // mask a still-broken script. + server.restartSession(server.active_session) catch |err| { + const msg = std.fmt.allocPrint(arena, "could not start a fresh session: {s}", .{@errorName(err)}) catch "could not start a fresh session"; + return sendErrorContent(server, id, msg); + }; + + var collector: ConsoleCollector = .{ .arena = arena }; + const classified = runClassified(server, arena, args.path, script, &collector) catch |err| switch (err) { + error.OutOfMemory => return sendErrorContent(server, id, "out of memory"), + error.RuntimeInitFailed, error.TooManyContexts => return sendErrorContent(server, id, "could not initialize the script runtime"), + }; + const run = buildRunReport(arena, args.path, classified, &collector, false) catch + return sendErrorContent(server, id, "out of memory"); + + var report: lp.heal.HealReport = .{ .cured = false, .committed = false, .run = run }; + switch (classified) { + .script_error => |script_error| report.failure = script_error.detail, + .facts => |facts| { + const residual = lp.heal.cureFailure(arena, first, facts) catch + return sendErrorContent(server, id, "out of memory"); + if (residual) |failure| { + report.failure = failure; + } else { + report.cured = true; + commitHealed(arena, args.path, script, facts.extract_stats, &report); + } + }, + } + return sendReport(server, arena, id, report); +} + +/// Write the validated revision next to `path` and atomically swap it in, +/// refreshing the `// lp:baseline` line from the validation run. Failures +/// land in the report (`cured` stays true, `committed` false). +fn commitHealed(arena: std.mem.Allocator, path: []const u8, script: []const u8, stats: []const ScriptRuntime.ExtractStat, report: *lp.heal.HealReport) void { + const final = lp.heal.refreshedBaselineScript(arena, script, stats) orelse script; + const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch { + report.failure = "validated, but out of memory writing the revision"; + return; + }; + writeScript(tmp_path, final) catch |err| { + std.fs.cwd().deleteFile(tmp_path) catch {}; + report.failure = std.fmt.allocPrint(arena, "validated, but writing {s} failed: {s}", .{ tmp_path, @errorName(err) }) catch null; + return; + }; + std.fs.cwd().rename(tmp_path, path) catch |err| { + // Deliberately keep the revision; the message points at it. + report.failure = std.fmt.allocPrint(arena, "validated, but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }) catch null; + return; + }; + report.committed = true; +} + /// The session tools require the HTTP transport's parked-isolate discipline: /// a second session means a second V8 isolate, only safe when isolates are /// entered around use. Over stdio (one permanently-entered isolate) they are @@ -937,6 +1193,214 @@ test "MCP - save writes the script to disk" { try std.testing.expectEqualStrings("const page = new Page();\nawait page.goto(\"https://example.com\");\n", written); } +fn testToolText(arena: std.mem.Allocator, response: []const u8) ![]const u8 { + const root = try std.json.parseFromSliceLeaky(std.json.Value, arena, std.mem.trim(u8, response, " \n"), .{}); + return root.object.get("result").?.object.get("content").?.array.items[0].object.get("text").?.string; +} + +fn testCall(server: *Server, out: *std.io.Writer.Allocating, name: []const u8, arguments: anytype) ![]const u8 { + const arena = testing.arena_allocator; + const args_json = try std.json.Stringify.valueAlloc(arena, arguments, .{}); + const msg = try std.fmt.allocPrint(arena, + \\{{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{{"name":"{s}","arguments":{s}}}}} + , .{ name, args_json }); + const start = out.written().len; + try router.handleMessage(server, arena, msg); + return testToolText(arena, out.written()[start..]); +} + +fn testCallReport(server: *Server, out: *std.io.Writer.Allocating, name: []const u8, arguments: anytype) !std.json.Value { + const text = try testCall(server, out, name, arguments); + return std.json.parseFromSliceLeaky(std.json.Value, testing.arena_allocator, text, .{}); +} + +const test_fixture_url = "http://localhost:9582/src/browser/tests/mcp_actions.html"; + +test "MCP - replay rejects unsafe path" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const msg = + \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"replay","arguments":{"path":"../evil.js"}}} + ; + try router.handleMessage(server, testing.arena_allocator, msg); + try testing.expect(std.mem.indexOf(u8, out.written(), "must be relative") != null); + try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":true") != null); +} + +test "MCP - replay: inline clean run reports ok" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "return [1];" }); + try testing.expectString("ok", report.object.get("status").?.string); + try testing.expectString("data", report.object.get("returned").?.string); + try testing.expectEqual(null, report.object.get("failure")); + try testing.expectEqual(null, report.object.get("guidance")); +} + +test "MCP - replay: throwing script reports failed with failure and guidance" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "throw new Error(\"boom\");" }); + try testing.expectString("failed", report.object.get("status").?.string); + const failure = report.object.get("failure").?.object; + try testing.expectString("threw", failure.get("kind").?.string); + try testing.expect(std.mem.indexOf(u8, failure.get("detail").?.string, "boom") != null); + try testing.expectString("throw new Error(\"boom\");", report.object.get("source").?.string); + try testing.expect(std.mem.indexOf(u8, report.object.get("guidance").?.string, "heal_commit") != null); +} + +test "MCP - replay: empty return reports suspicious" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "return [];" }); + try testing.expectString("suspicious", report.object.get("status").?.string); + try testing.expectString("empty", report.object.get("returned").?.string); + try testing.expectString("empty", report.object.get("failure").?.object.get("kind").?.string); + try testing.expect(std.mem.indexOf(u8, report.object.get("guidance").?.string, "lp:baseline") != null); +} + +test "MCP - replay: dry extract reports suspicious with dry_fields" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage(test_fixture_url, &out.writer); + defer server.deinit(); + + const script = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: ["#btn"], missing: [".no-such-thing"] }); + ; + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = script }); + try testing.expectString("suspicious", report.object.get("status").?.string); + const failure = report.object.get("failure").?.object; + try testing.expectString("dry_extracts", failure.get("kind").?.string); + const dry = failure.get("dry_fields").?.array.items; + try testing.expectEqual(1, dry.len); + try testing.expectString("missing", dry[0].string); + try testing.expectEqual(2, report.object.get("extracts").?.array.items.len); +} + +test "MCP - replay: console lines are captured in the report" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "console.log(\"hello\", 42);\nreturn [1];" }); + const console = report.object.get("console").?.array.items; + try testing.expectEqual(2, console.len); + try testing.expectString("log", console[0].object.get("level").?.string); + try testing.expectString("hello 42", console[0].object.get("text").?.string); + // The returned value is echoed as the final console line — how a replay + // hands its output to the client. + try testing.expectString("[1]", console[1].object.get("text").?.string); + try testing.expectEqual(false, report.object.get("console_truncated").?.bool); +} + +test "MCP - heal_commit: uncured candidate leaves the file untouched" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const path = "mcp-heal-uncured-test.js"; + std.fs.cwd().deleteFile(path) catch {}; + + const report = try testCallReport(server, &out, "heal_commit", .{ + .path = path, + .script = "return [];", + .failure = .{ .kind = "empty" }, + }); + try testing.expectEqual(false, report.object.get("cured").?.bool); + try testing.expectEqual(false, report.object.get("committed").?.bool); + try testing.expect(std.mem.indexOf(u8, report.object.get("failure").?.string, "no data") != null); + try testing.expectError(error.FileNotFound, std.fs.cwd().access(path, .{})); +} + +test "MCP - heal_commit: cure commits atomically and refreshes the baseline" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const path = "mcp-heal-cure-test.js"; + try std.fs.cwd().writeFile(.{ .sub_path = path, .data = "return [];\n" }); + defer std.fs.cwd().deleteFile(path) catch {}; + + const revised = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: ["#btn"] }); + ; + const report = try testCallReport(server, &out, "heal_commit", .{ + .path = path, + .script = revised, + .failure = .{ .kind = "empty" }, + }); + try testing.expectEqual(true, report.object.get("cured").?.bool); + try testing.expectEqual(true, report.object.get("committed").?.bool); + try testing.expectString("ok", report.object.get("run").?.object.get("status").?.string); + + const written = try std.fs.cwd().readFileAlloc(testing.arena_allocator, path, 4096); + try testing.expect(std.mem.startsWith(u8, written, "const page = new Page();")); + try testing.expect(std.mem.indexOf(u8, written, "// lp:baseline ") != null); + try testing.expect(std.mem.indexOf(u8, written, "\"btn\":{\"calls\":1,\"nonempty\":1}") != null); + try testing.expectError(error.FileNotFound, std.fs.cwd().access(path ++ ".heal.js", .{})); +} + +test "MCP - script lifecycle: save, replay broken, heal_commit, replay clean" { + defer testing.reset(); + var out: std.io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage(test_fixture_url, &out.writer); + defer server.deinit(); + + const path = "mcp-lifecycle-test.js"; + std.fs.cwd().deleteFile(path) catch {}; + defer std.fs.cwd().deleteFile(path) catch {}; + + const broken = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: [".no-such-btn"] }); + ; + const saved = try testCall(server, &out, "save", .{ .path = path, .script = broken }); + try testing.expect(std.mem.indexOf(u8, saved, "saved") != null); + + const run = try testCallReport(server, &out, "replay", .{ .path = path }); + try testing.expectString("suspicious", run.object.get("status").?.string); + const failure = run.object.get("failure").?; + + // The test plays the client model: fix the selector, echo the failure back. + const revised = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: ["#btn"] }); + ; + const healed = try testCallReport(server, &out, "heal_commit", .{ + .path = path, + .script = revised, + .failure = failure, + }); + try testing.expectEqual(true, healed.object.get("cured").?.bool); + try testing.expectEqual(true, healed.object.get("committed").?.bool); + + const rerun = try testCallReport(server, &out, "replay", .{ .path = path }); + try testing.expectString("ok", rerun.object.get("status").?.string); + try testing.expectString("data", rerun.object.get("returned").?.string); +} + test "MCP - tree rejects stale backendNodeId instead of dumping whole document" { defer testing.reset(); var out: std.io.Writer.Allocating = .init(testing.arena_allocator); diff --git a/src/agent/Baseline.zig b/src/script/Baseline.zig similarity index 100% rename from src/agent/Baseline.zig rename to src/script/Baseline.zig diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index e9140fc910..7827548b5a 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -42,6 +42,10 @@ console_data: [std.enums.values(ConsoleMethod).len]ConsoleData, /// clear the live spinner so script output starts on a clean line instead /// of colliding with the indicator; the line still goes to stdout/stderr. console_observer: ?ConsoleObserver = null, +/// When set, receives each `console.*` line instead of the process +/// stdout/stderr write — for embedders (MCP) whose stdout is a protocol +/// stream. The observer still fires first. +console_sink: ?ConsoleSink = null, /// In-flight async `goto`s; emptied before each `runSource` returns. pending_gotos: std.ArrayList(PendingGoto), /// Restarted per `runSource`; backs `PendingGoto.deadline_ms`. @@ -82,7 +86,7 @@ const PendingGoto = struct { } }; -const ConsoleMethod = enum { +pub const ConsoleMethod = enum { debug, @"error", info, @@ -107,6 +111,11 @@ pub const ConsoleObserver = struct { notify: *const fn (context: *anyopaque) void, }; +pub const ConsoleSink = struct { + context: *anyopaque, + write: *const fn (context: *anyopaque, method: ConsoleMethod, line: []const u8) void, +}; + pub const InitError = error{ OutOfMemory, RuntimeInitFailed, @@ -843,6 +852,7 @@ fn invokeConsole(self: *Runtime, method: ConsoleMethod, info: *const v8.Function fn writeConsoleLine(self: *Runtime, method: ConsoleMethod, line: []const u8) void { if (self.console_observer) |obs| obs.notify(obs.context); + if (self.console_sink) |sink| return sink.write(sink.context, method, line); var buf: [4096]u8 = undefined; var file = if (method.writesStderr()) std.fs.File.stderr() else std.fs.File.stdout(); var writer = file.writer(&buf); @@ -1604,6 +1614,36 @@ test "agent script runtime: console is available in agent context" { ); } +test "agent script runtime: console sink captures lines instead of the process streams" { + defer testing.reset(); + + var registry = CDPNode.Registry.init(testing.allocator); + defer registry.deinit(); + + const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); + defer runtime.deinit(); + + const Capture = struct { + buf: [256]u8 = undefined, + len: usize = 0, + + fn write(context: *anyopaque, method: ConsoleMethod, line: []const u8) void { + const capture: *@This() = @ptrCast(@alignCast(context)); + const out = std.fmt.bufPrint(capture.buf[capture.len..], "{s}:{s}\n", .{ @tagName(method), line }) catch return; + capture.len += out.len; + } + }; + var capture: Capture = .{}; + runtime.console_sink = .{ .context = @ptrCast(&capture), .write = Capture.write }; + + try runTestScript(runtime, + \\console.log("hello", 42); + \\console.warn("careful"); + ); + + try testing.expectString("log:hello 42\nwarn:careful\n", capture.buf[0..capture.len]); +} + test "agent script runtime: tool errors throw and stop execution" { defer testing.reset(); defer testing.test_session.closeAllPages(); diff --git a/src/script/heal.zig b/src/script/heal.zig new file mode 100644 index 0000000000..fd645e3b15 --- /dev/null +++ b/src/script/heal.zig @@ -0,0 +1,451 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Deterministic core of script self-heal, shared by the agent CLI and the +//! MCP server: breakage suspicion over a replay's facts, the cure check that +//! gates a healed script's commit, and the serializable run/heal reports. +//! Model judgment (verdict, diagnosis, revision) stays with the caller — the +//! agent's own LLM on the CLI path, the MCP client's model over MCP. + +const std = @import("std"); +const lp = @import("lightpanda"); +const ScriptRuntime = lp.Runtime; +const Baseline = lp.Baseline; +const string = @import("../string.zig"); + +/// What a completed run returned, as far as heal cares. +pub const Returned = union(enum) { + /// No `return`, or a value whose display form couldn't be computed. + none, + /// A value carrying data. + data, + /// A deep-empty value, carrying its capped display text. + empty: []const u8, +}; + +/// Facts about a run that completed without throwing — suspicion is judged by +/// the model, never here. Duped into the caller's arena — the runtime dies +/// with the run. +pub const RunFacts = struct { + returned: Returned, + extract_stats: []const ScriptRuntime.ExtractStat, + source: []const u8, +}; + +/// Both slices are duped into the caller's arena. `source` is the exact text +/// that ran, so a heal diagnoses what actually failed instead of re-reading a +/// possibly-changed file. +pub const ScriptError = struct { + kind: Kind, + /// Formatted error (line, stack) — or, for `empty`, what came back. + detail: []const u8, + source: []const u8, + /// For `dry_extracts`: the field names that were empty on every call + /// (null = a whole-array schema). The cure check requires each one to + /// come back with data before a heal may replace the file. + dry_fields: []const ?[]const u8 = &.{}, + + /// `empty` is a run that completed but returned a value with no data in + /// it; `dry_extracts` one whose return value had data, but where some + /// extract list field came back empty on every call. Both are the usual + /// symptom of a stale selector, which matches nothing instead of throwing. + /// Only heal treats them as failures; a plain replay still exits 0, since + /// an empty answer can be the right answer. + pub const Kind = enum { threw, empty, dry_extracts }; +}; + +pub const Classified = union(enum) { + facts: RunFacts, + script_error: ScriptError, +}; + +/// Map a run's raw result to facts or a `threw` finding. The error text and +/// stats are duped into `arena` — they live in the runtime's per-call arena — +/// but `source` is stored as given: the caller owns it and it must outlive +/// the outcome. Presentation (terminal output, cancellation policy) stays +/// with the caller. +pub fn classifyRun(arena: std.mem.Allocator, result: ScriptRuntime.RunResult, source: []const u8) error{OutOfMemory}!Classified { + switch (result) { + .err => |message| return .{ .script_error = .{ + .kind = .threw, + .detail = try arena.dupe(u8, message), + .source = source, + } }, + .ok => |ok| { + const returned: Returned = if (ok.completion) |c| + (if (c.empty) .{ .empty = try capDetail(arena, c.text) } else .data) + else + .none; + return .{ .facts = .{ + .returned = returned, + .extract_stats = try dupeExtractStats(arena, ok.extract_stats), + .source = source, + } }; + }, + } +} + +fn dupeExtractStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) error{OutOfMemory}![]const ScriptRuntime.ExtractStat { + const out = try arena.alloc(ScriptRuntime.ExtractStat, stats.len); + for (stats, out) |stat, *o| { + o.* = .{ + .schema = try arena.dupe(u8, stat.schema), + .field = if (stat.field) |f| try arena.dupe(u8, f) else null, + .calls = stat.calls, + .empty = stat.empty, + }; + } + return out; +} + +/// Bound a value or schema echoed into a heal message; a degenerate empty-ish +/// result (hundreds of all-null rows) would otherwise bloat the LLM turn. +const detail_max_bytes: usize = 2048; + +/// `string.capBytes` at `detail_max_bytes`, always duped — `RunFacts` details +/// must outlive the runtime whose arena the text came from. +fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]const u8 { + return arena.dupe(u8, string.capBytes(arena, text, detail_max_bytes)); +} + +/// A finding worth a verdict, not yet confirmed: the return value was +/// deep-empty, or some extract field came back empty on every call — any field, +/// scalar or list, baseline or not. Whether that is breakage or legitimate +/// sparseness is the model's judgment, not encoded here. +pub fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?ScriptError { + switch (facts.returned) { + .empty => |text| return .{ + .kind = .empty, + .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, + .source = facts.source, + }, + .none, .data => {}, + } + return dryExtractsFinding(arena, facts.source, facts.extract_stats) catch return null; +} + +/// A `dry_extracts` finding with one detail line per extract field that came +/// back empty on every call, plus the field names for the cure check. Null when +/// no field was dry. +fn dryExtractsFinding(arena: std.mem.Allocator, source: []const u8, stats: []const ScriptRuntime.ExtractStat) !?ScriptError { + var aw: std.Io.Writer.Allocating = .init(arena); + var fields: std.ArrayList(?[]const u8) = .empty; + for (stats) |stat| { + if (stat.empty != stat.calls) continue; + if (fields.items.len == 0) { + try aw.writer.writeAll("some extracts came back empty on every call:\n"); + } + // `stat.field` already lives in `arena` (facts were duped into it). + try fields.append(arena, stat.field); + const schema = try capDetail(arena, stat.schema); + if (stat.field) |field| { + try aw.writer.print("- the \"{s}\" field in extract({s}) came back empty", .{ field, schema }); + } else { + try aw.writer.print("- extract({s}) returned no data", .{schema}); + } + if (stat.calls != 1) try aw.writer.print(" in all {d} calls", .{stat.calls}); + try aw.writer.writeAll("\n"); + } + if (fields.items.len == 0) return null; + return .{ .kind = .dry_extracts, .detail = aw.written(), .source = source, .dry_fields = fields.items }; +} + +/// Null when the validation run cured the original finding; otherwise the +/// message fed to the next heal attempt. Running clean is not a cure on its +/// own — a revision that deletes the failing extract (or the `return`) also +/// runs clean. +pub fn cureFailure(arena: std.mem.Allocator, first: ScriptError, facts: RunFacts) error{OutOfMemory}!?[]const u8 { + switch (first.kind) { + .threw => return null, + .empty => return if (facts.returned == .data) + null + else + "The revised script ran, but still returns no data (or no longer returns anything) — the original returned a value.", + .dry_extracts => { + for (first.dry_fields) |dry| { + const cured = for (facts.extract_stats) |stat| { + if (ScriptRuntime.fieldEql(stat.field, dry) and stat.empty < stat.calls) break true; + } else false; + if (!cured) return try std.fmt.allocPrint(arena, "The revised script ran, but the \"{s}\" extract still came back empty on every call (or was removed) — keep it and fix its selector.", .{dry orelse ""}); + } + return null; + }, + } +} + +pub fn refreshedBaselineScript(arena: std.mem.Allocator, revised: []const u8, stats: []const ScriptRuntime.ExtractStat) ?[]const u8 { + const line = Baseline.serializeStats(arena, stats) catch return null; + return Baseline.withBaseline(arena, revised, line) catch null; +} + +pub fn buildDiagnoseMessage(arena: std.mem.Allocator, path: []const u8, source: []const u8, error_detail: []const u8) ![]const u8 { + return std.fmt.allocPrint(arena, + \\Replaying the saved script {s} failed. The browser session is still + \\at the failure state. + \\ + \\The script (its comments and structure carry the intent): + \\```js + \\{s} + \\``` + \\ + \\The error: + \\{s} + \\ + \\Diagnose the failure: inspect the live page (tree, findElement, + \\markdown) to see how the site differs from what the script expects, + \\then perform the corrected step(s) with tools to prove they work — + \\verify selectors against the live page, never guess. If the failing + \\step gated the rest of the script (a login, a navigation), carry on + \\far enough to show the script's goal is reachable again. + , .{ path, source, error_detail }); +} + +/// Heal synthesis instruction; rides on the regular save revision system prompt. +pub const heal_revision_prompt = + \\Fix the script so it replays successfully against the current site: the + \\error names what broke, and the diagnosis tool calls above that + \\succeeded against the live page show the repair. Keep + \\every step, selector, and output shape that still works unchanged. + \\Preserve the script's `//` intent comments; where you change a block, + \\update its comment so it still describes what the revised code does, and + \\add one for any block that lacks it. +; + +// Fixed next-step guidance embedded in MCP replay reports. Templates only — +// page-derived content rides in the report's data fields, never here. + +pub const replay_failed_guidance = + \\The replay failed and the session is still at the failure state. + \\Diagnose the failure: inspect the live page (tree, findElement, + \\markdown) to see how the site differs from what the script expects — + \\`source` carries the intent in its comments and structure — then + \\perform the corrected step(s) with tools to prove they work: verify + \\selectors against the live page, never guess. If the failing step gated + \\the rest of the script (a login, a navigation), carry on far enough to + \\show the script's goal is reachable again. Then call heal_commit with + \\the revised script and this report's `failure` object echoed back + \\verbatim. +; + +pub const replay_suspicious_guidance = + \\The replay completed without errors, but its output looks dry — decide + \\whether the script is broken (stale selectors after a site change) or + \\the result is legitimate (the page genuinely has no such data right + \\now). A ` +++ std.mem.trimEnd(u8, Baseline.marker, " ") ++ + \\` comment in `source`, when present, records how often + \\each output field carried data when the script was saved — weigh it as + \\evidence. If legitimate, stop: the empty answer is the answer. If + \\broken, diagnose against the live session (tree, findElement, + \\markdown), prove the corrected step(s) with tools, then call + \\heal_commit with the revised script and this report's `failure` object + \\echoed back verbatim. +; + +/// `failure` as it rides in reports and back through `heal_commit`: same +/// shape as the model-facing verdict wire, with "" standing in for a +/// whole-array extract (JSON has no place for null-in-a-string-array). +pub const WireFailure = struct { + kind: ScriptError.Kind, + detail: []const u8 = "", + dry_fields: []const []const u8 = &.{}, +}; + +pub const ConsoleLine = struct { + level: []const u8, + text: []const u8, +}; + +/// One replay, serializable: what ran, what came back, and — when the run +/// failed or looks dry — the finding and how to proceed. +pub const RunReport = struct { + status: Status, + path: []const u8, + returned: std.meta.Tag(Returned) = .none, + extracts: []const ScriptRuntime.ExtractStat = &.{}, + failure: ?WireFailure = null, + console: []const ConsoleLine = &.{}, + console_truncated: bool = false, + /// Scrubbed source that actually ran; set on suspicious/failed so the + /// client can diagnose without re-reading a possibly-changed file. + source: ?[]const u8 = null, + guidance: ?[]const u8 = null, + + pub const Status = enum { ok, suspicious, failed }; +}; + +/// One `heal_commit` validation, serializable. `failure` is the residual cure +/// failure (or the validation run's own error); null when cured. The script's +/// path rides in `run.path`. +pub const HealReport = struct { + cured: bool, + committed: bool, + failure: ?[]const u8 = null, + run: RunReport, +}; + +pub fn wireFailure(arena: std.mem.Allocator, e: ScriptError) error{OutOfMemory}!WireFailure { + const fields = try arena.alloc([]const u8, e.dry_fields.len); + for (e.dry_fields, fields) |f, *out| out.* = f orelse ""; + return .{ .kind = e.kind, .detail = e.detail, .dry_fields = fields }; +} + +pub fn scriptErrorFromWire(arena: std.mem.Allocator, w: WireFailure) error{OutOfMemory}!ScriptError { + const fields = try arena.alloc(?[]const u8, w.dry_fields.len); + for (w.dry_fields, fields) |f, *out| out.* = if (f.len == 0) null else f; + return .{ .kind = w.kind, .detail = w.detail, .source = "", .dry_fields = fields }; +} + +const testing = @import("../testing.zig"); + +fn testFacts(returned: Returned, stats: []const ScriptRuntime.ExtractStat) RunFacts { + return .{ .returned = returned, .extract_stats = stats, .source = "" }; +} + +test "cureFailure: running clean is not a cure" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const dry: ScriptError = .{ + .kind = .dry_extracts, + .detail = "", + .source = "", + .dry_fields = &.{ @as(?[]const u8, "comments"), null }, + }; + const cured_stats: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, + .{ .schema = "[]", .field = null, .calls = 1, .empty = 0 }, + }; + try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(.data, cured_stats))); + + // Fix-by-deletion: the dry field is simply gone from the revised run. + const deleted = (try cureFailure(aa, dry, testFacts(.data, cured_stats[1..]))).?; + try std.testing.expect(std.mem.indexOf(u8, deleted, "\"comments\"") != null); + + // Still dry counts as uncured. + const still_dry_stats: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 5 }, + cured_stats[1], + }; + try std.testing.expect((try cureFailure(aa, dry, testFacts(.data, still_dry_stats))) != null); + + // .empty is cured only by a data-carrying return. + const empty: ScriptError = .{ .kind = .empty, .detail = "", .source = "" }; + try std.testing.expectEqual(null, try cureFailure(aa, empty, testFacts(.data, &.{}))); + try std.testing.expect((try cureFailure(aa, empty, testFacts(.none, &.{}))) != null); + + // .threw needs nothing beyond running clean. + const threw: ScriptError = .{ .kind = .threw, .detail = "", .source = "" }; + try std.testing.expectEqual(null, try cureFailure(aa, threw, testFacts(.none, &.{}))); +} + +test "suspicionOf: any all-empty field is suspect, none is not" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const sparse: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, + }; + try std.testing.expectEqual(null, suspicionOf(aa, testFacts(.data, sparse))); + + // Scalar all-empty is suspect too — judgment belongs to the model now. + const dry_scalar: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }, + }; + const s = suspicionOf(aa, testFacts(.data, dry_scalar)).?; + try std.testing.expectEqual(ScriptError.Kind.dry_extracts, s.kind); + try std.testing.expectEqual(1, s.dry_fields.len); + + const empty_facts = testFacts(.{ .empty = "[]" }, &.{}); + try std.testing.expectEqual(ScriptError.Kind.empty, suspicionOf(aa, empty_facts).?.kind); +} + +test "classifyRun: maps err to threw, completion emptiness to returned" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const threw = try classifyRun(aa, .{ .err = "boom at line 2" }, "return 1;"); + try std.testing.expectEqual(ScriptError.Kind.threw, threw.script_error.kind); + try std.testing.expectEqualStrings("boom at line 2", threw.script_error.detail); + try std.testing.expectEqualStrings("return 1;", threw.script_error.source); + + const empty = try classifyRun(aa, .{ .ok = .{ + .completion = .{ .text = "[]", .empty = true }, + .extract_stats = &.{}, + } }, "return [];"); + try std.testing.expectEqualStrings("[]", empty.facts.returned.empty); + + const data = try classifyRun(aa, .{ .ok = .{ + .completion = .{ .text = "[1]", .empty = false }, + .extract_stats = &.{.{ .schema = "{}", .field = "a", .calls = 1, .empty = 0 }}, + } }, "return [1];"); + try std.testing.expectEqual(Returned.data, data.facts.returned); + try std.testing.expectEqual(1, data.facts.extract_stats.len); + + const none = try classifyRun(aa, .{ .ok = .{ .completion = null, .extract_stats = &.{} } }, "1;"); + try std.testing.expectEqual(Returned.none, none.facts.returned); +} + +test "wire failure round-trips, empty string standing in for null" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const original: ScriptError = .{ + .kind = .dry_extracts, + .detail = "dry", + .source = "return 1;", + .dry_fields = &.{ @as(?[]const u8, "comments"), null }, + }; + const wire = try wireFailure(aa, original); + try std.testing.expectEqualStrings("comments", wire.dry_fields[0]); + try std.testing.expectEqualStrings("", wire.dry_fields[1]); + + const back = try scriptErrorFromWire(aa, wire); + try std.testing.expectEqual(original.kind, back.kind); + try std.testing.expectEqualStrings("comments", back.dry_fields[0].?); + try std.testing.expectEqual(null, back.dry_fields[1]); +} + +test "RunReport serializes to the wire shape" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const report: RunReport = .{ + .status = .suspicious, + .path = "hn.js", + .returned = .data, + .extracts = &.{.{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }}, + .failure = .{ .kind = .dry_extracts, .detail = "dry", .dry_fields = &.{"title"} }, + .console = &.{.{ .level = "log", .text = "hello" }}, + }; + const json = try std.json.Stringify.valueAlloc(aa, report, .{ .emit_null_optional_fields = false }); + try testing.expectJson(.{ + .status = "suspicious", + .path = "hn.js", + .returned = "data", + .extracts = .{.{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }}, + .failure = .{ .kind = "dry_extracts", .detail = "dry", .dry_fields = .{"title"} }, + .console = .{.{ .level = "log", .text = "hello" }}, + .console_truncated = false, + }, json); +} diff --git a/src/string.zig b/src/string.zig index 181371114b..23c16c3969 100644 --- a/src/string.zig +++ b/src/string.zig @@ -409,6 +409,16 @@ pub fn truncateUtf8(bytes: []const u8, max_bytes: usize) []const u8 { return bytes[0..i]; } +/// Returns `bytes` unchanged when under `max_bytes`; a truncated-and-marked +/// copy in `allocator` otherwise (the bare prefix on OOM — never an error). +pub fn capBytes(allocator: std.mem.Allocator, bytes: []const u8, max_bytes: usize) []const u8 { + if (bytes.len <= max_bytes) return bytes; + const prefix = truncateUtf8(bytes, max_bytes); + var suffix_buf: [64]u8 = undefined; + const suffix = std.fmt.bufPrint(&suffix_buf, "\n...[truncated, original {d} bytes]", .{bytes.len}) catch return prefix; + return std.mem.concat(allocator, u8, &.{ prefix, suffix }) catch prefix; +} + // Discriminatory type that signals the bridge to use arena instead of call_arena // Use this for strings that need to persist beyond the current call // The caller can unwrap and store just the underlying .str field @@ -455,6 +465,21 @@ test "truncateUtf8" { try testing.expectEqual("\xFFx", truncateUtf8("\xFFx", 2)); } +test "capBytes: passes through when under cap" { + const out = capBytes(testing.allocator, "short", 16); + try testing.expectEqual("short", out); +} + +test "capBytes: appends a marker, keeps valid UTF-8" { + const ta = testing.allocator; + // 3-byte codepoint '한' (0xED 0x95 0x9C) straddling the cap. + const out = capBytes(ta, "aaaaaaa한bbb", 8); + defer ta.free(out); + try std.testing.expect(std.unicode.utf8ValidateSlice(out)); + try std.testing.expect(std.mem.startsWith(u8, out, "aaaaaaa")); + try std.testing.expect(std.mem.indexOf(u8, out, "truncated, original 13 bytes") != null); +} + test "String" { const other_short = try String.init(undefined, "other_short", .{}); const other_long = try String.init(testing.allocator, "other_long" ** 100, .{}); From f4297be1f94f863d3dc77bc55942b589a5e5177f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 22 Jul 2026 16:08:41 +0200 Subject: [PATCH 15/15] deps: update zenai dependency --- build.zig.zon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 725fbed4d7..4aeaf8f156 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -36,8 +36,8 @@ .hash = "sqlite3-3.53.2-DMxLWuAOAAA_Px0arJOIOaP4AKEu5prbsQgPMA35W1zz", }, .zenai = .{ - .url = "git+https://github.com/lightpanda-io/zenai.git#00c793fdd2797ae5fc9700c189d8617f167ae0dc", - .hash = "zenai-0.0.0-iOY_VGOnBQDaWE_Gn8cwhh0XXE8RZ8ESE7QtcITF-P73", + .url = "git+https://github.com/lightpanda-io/zenai.git#31da07c1fe9805267ab4dbe4ce41c0cc2d16a36a", + .hash = "zenai-0.0.0-iOY_VGnDBQDS-bBNsgXuHhwEMqTh9Q7bQuP0BKziOdrR", }, .isocline = .{ .url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47",