diff --git a/.antigravitycli/c9e31ac8-54b1-42f7-ad85-b50dcaa697ba.json b/.antigravitycli/c9e31ac8-54b1-42f7-ad85-b50dcaa697ba.json new file mode 120000 index 000000000..1fee1d537 --- /dev/null +++ b/.antigravitycli/c9e31ac8-54b1-42f7-ad85-b50dcaa697ba.json @@ -0,0 +1 @@ +/home/adria/.gemini/config/projects/c9e31ac8-54b1-42f7-ad85-b50dcaa697ba.json \ No newline at end of file diff --git a/Screenshot From 2026-07-03 21-36-34.png b/Screenshot From 2026-07-03 21-36-34.png new file mode 100644 index 000000000..e4b2551df Binary files /dev/null and b/Screenshot From 2026-07-03 21-36-34.png differ diff --git a/hacker-news-karma.js b/hacker-news-karma.js new file mode 100644 index 000000000..7203b6ae0 --- /dev/null +++ b/hacker-news-karma.js @@ -0,0 +1,9 @@ +const page = new Page(); +await page.goto("https://news.ycombinator.com/login"); +page.waitForSelector("input[name=\"acct\"]"); +page.fill("input[name=\"acct\"]", "$LP_HN_USERNAME"); +page.fill("input[name=\"pw\"]", "$LP_HN_PASSWORD"); +page.press("input[name=\"pw\"]", "Enter"); +await page.goto("https://news.ycombinator.com/user?id=$LP_HN_USERNAME"); +const { karma } = page.extract({ karma: "#hnmain table table tr:nth-child(3) td:nth-child(2)" }); +return { karma: parseInt(karma, 10) }; diff --git a/hacker-news.js b/hacker-news.js new file mode 100644 index 000000000..c2a3945d1 --- /dev/null +++ b/hacker-news.js @@ -0,0 +1,46 @@ +const page = new Page(); +await page.goto("https://news.ycombinator.com"); + +const { stories } = page.extract({ + stories: [{ + selector: "tr.athing", + limit: 5, + fields: { + id: { selector: "", attr: "id" }, + rank: ".rank", + title: ".titleline > a" + } + }] +}); + +const { scoreSpans } = page.extract({ + scoreSpans: [{ + selector: ".score", + fields: { + id: { selector: "", attr: "id" }, + points: "" + } + }] +}); + +const { authors } = page.extract({ + authors: [".hnuser"] +}); + +// .score id is "score_"; .hnuser entries are parallel to .score entries +const scoreMap = {}; +const authorMap = {}; +scoreSpans.forEach((s, i) => { + const storyId = s.id.replace("score_", ""); + scoreMap[storyId] = s.points; + authorMap[storyId] = authors[i] || null; +}); + +const results = stories.map(s => ({ + rank: s.rank.replace(".", "").trim(), + title: s.title, + author: authorMap[s.id] || null, + score: scoreMap[s.id] || null +})); + +return results; diff --git a/headers.txt b/headers.txt new file mode 100644 index 000000000..72411ffe2 --- /dev/null +++ b/headers.txt @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +content-length: 6006 +content-type: application/json +mcp-session-id: s1 + diff --git a/hn-agent-stories.zip b/hn-agent-stories.zip new file mode 100644 index 000000000..d429a4848 Binary files /dev/null and b/hn-agent-stories.zip differ diff --git a/hn-agent-stories/hn-karma.mp4 b/hn-agent-stories/hn-karma.mp4 new file mode 100644 index 000000000..9747a27ca Binary files /dev/null and b/hn-agent-stories/hn-karma.mp4 differ diff --git a/hn-agent-stories/hn-scrape.mp4 b/hn-agent-stories/hn-scrape.mp4 new file mode 100644 index 000000000..dd2629948 Binary files /dev/null and b/hn-agent-stories/hn-scrape.mp4 differ diff --git a/hn-agent-stories/hn-story-karma.mp4 b/hn-agent-stories/hn-story-karma.mp4 new file mode 100644 index 000000000..466b165dd Binary files /dev/null and b/hn-agent-stories/hn-story-karma.mp4 differ diff --git a/hn-agent-stories/hn-story-scrape.mp4 b/hn-agent-stories/hn-story-scrape.mp4 new file mode 100644 index 000000000..cf4083c43 Binary files /dev/null and b/hn-agent-stories/hn-story-scrape.mp4 differ diff --git a/hn-agent-stories/lightpanda b/hn-agent-stories/lightpanda new file mode 120000 index 000000000..049b6b6c8 --- /dev/null +++ b/hn-agent-stories/lightpanda @@ -0,0 +1 @@ +zig-out/bin/lightpanda \ No newline at end of file diff --git a/hn-agent-stories/make-stories.sh b/hn-agent-stories/make-stories.sh new file mode 100755 index 000000000..739153cfb --- /dev/null +++ b/hn-agent-stories/make-stories.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Build the two HN story videos. Each is: a 1s frozen lead-in frame, a step1 +# still, sped-up/normal clips, a step2 still, then closing clips. Everything is +# normalized to 1514x1032 @ 60fps CFR so the pieces concatenate seamlessly. +# +# The frozen lead-in is extracted to a PNG first (input-seek lands on the frame +# displayed at that timestamp) because these are sparse VFR screen recordings: +# a sub-second trim window inside the filtergraph can contain zero frames. +# +set -euo pipefail + +norm="scale=1514:1032,setsar=1,fps=60,format=yuv420p" + +# ============================================================================ +# Story 1 — from hn-scrape.mp4 +# ============================================================================ +ffmpeg -y -ss 16.5 -i hn-scrape.mp4 -frames:v 1 frame-scrape-16.5.png + +ffmpeg -y \ + -i hn-scrape.mp4 \ + -loop 1 -t 3 -i step1.png \ + -loop 1 -t 3 -i step2.png \ + -loop 1 -t 1 -i frame-scrape-16.5.png \ + -filter_complex "\ +[3:v]${norm}[frz];\ +[1:v]${norm}[img1];\ +[2:v]${norm}[img2];\ +[0:v]trim=16.5:20,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[a];\ +[0:v]trim=20:72,setpts=(PTS-STARTPTS)/5,fps=60,scale=1514:1032,setsar=1,format=yuv420p[b];\ +[0:v]trim=72:73,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[c];\ +[0:v]trim=94:105,setpts=(PTS-STARTPTS)/5,fps=60,scale=1514:1032,setsar=1,format=yuv420p[d];\ +[0:v]trim=105:112,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[e];\ +[0:v]trim=119.5:126.5,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[f];\ +[0:v]trim=start=134.5,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[g];\ +[frz][img1][a][b][c][d][e][img2][f][g]concat=n=10:v=1[out]" \ + -map "[out]" -fps_mode cfr -c:v libx264 -crf 20 -preset medium -pix_fmt yuv420p hn-story-scrape.mp4 + +# ============================================================================ +# Story 2 — from hn-karma.mp4 +# ============================================================================ +ffmpeg -y -ss 18 -i hn-karma.mp4 -frames:v 1 frame-karma-18.png + +ffmpeg -y \ + -i hn-karma.mp4 \ + -loop 1 -t 3 -i step1.png \ + -loop 1 -t 3 -i step2.png \ + -loop 1 -t 1 -i frame-karma-18.png \ + -filter_complex "\ +[3:v]${norm}[frz];\ +[1:v]${norm}[img1];\ +[2:v]${norm}[img2];\ +[0:v]trim=18:27,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[a];\ +[0:v]trim=27:57,setpts=(PTS-STARTPTS)/5,fps=60,scale=1514:1032,setsar=1,format=yuv420p[b];\ +[0:v]trim=67:75,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[c];\ +[0:v]trim=75:81,setpts=(PTS-STARTPTS)/5,fps=60,scale=1514:1032,setsar=1,format=yuv420p[d];\ +[0:v]trim=100:105,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[e];\ +[0:v]trim=105:114,setpts=(PTS-STARTPTS)/5,fps=60,scale=1514:1032,setsar=1,format=yuv420p[f];\ +[0:v]trim=114:117,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[g];\ +[0:v]trim=126:132,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[h];\ +[0:v]trim=138:146,setpts=PTS-STARTPTS,fps=60,scale=1514:1032,setsar=1,format=yuv420p[i];\ +[frz][img1][a][b][c][d][e][f][g][img2][h][i]concat=n=12:v=1[out]" \ + -map "[out]" -fps_mode cfr -c:v libx264 -crf 20 -preset medium -pix_fmt yuv420p hn-story-karma.mp4 + +echo "done:" +ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 hn-story-scrape.mp4 +ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 hn-story-karma.mp4 diff --git a/hn-agent-stories/step1.png b/hn-agent-stories/step1.png new file mode 100644 index 000000000..19367f2c4 Binary files /dev/null and b/hn-agent-stories/step1.png differ diff --git a/hn-agent-stories/step2.png b/hn-agent-stories/step2.png new file mode 100644 index 000000000..13a7937a0 Binary files /dev/null and b/hn-agent-stories/step2.png differ diff --git a/hn-comments.js b/hn-comments.js new file mode 100644 index 000000000..70f82f96d --- /dev/null +++ b/hn-comments.js @@ -0,0 +1,52 @@ +/* + * Summary of Reasoning: + * 1. Navigate to the Hacker News homepage to fetch the top 5 stories. + * 2. Extract story details along with their item IDs to construct individual comment page URLs. + * 3. Create independent Page objects for each story to load and extract their top 3 comments in parallel. + * 4. Merge the extracted comments with their respective stories and return the compiled results. + */ + +// Navigate to the Hacker News homepage +const homePage = new Page(); +await homePage.goto("https://news.ycombinator.com"); + +// Extract the top 5 stories and their item IDs +const { stories } = homePage.extract({ + stories: [{ + selector: ".athing", + limit: 5, + fields: { + id: { attr: "id" }, + title: ".titleline a", + link: { selector: ".titleline a", attr: "href" } + } + }] +}); + +// Open comment pages for the top 5 stories in parallel +const commentPages = stories.map(() => new Page()); +await Promise.all( + commentPages.map((page, i) => page.goto(`https://news.ycombinator.com/item?id=${stories[i].id}`)) +); + +// Extract the top 3 comments from each story page and combine the results +const results = commentPages.map((page, i) => { + const { comments } = page.extract({ + comments: [{ + selector: ".comment", + limit: 3, + fields: { + text: ".commtext" + } + }] + }); + + return { + title: stories[i].title, + link: stories[i].link, + discussionLink: `https://news.ycombinator.com/item?id=${stories[i].id}`, + comments: comments.map(c => c.text ? c.text.trim() : "") + }; +}); + +return results; diff --git a/hn-gemini.js b/hn-gemini.js new file mode 100644 index 000000000..6c22259e1 --- /dev/null +++ b/hn-gemini.js @@ -0,0 +1,46 @@ +const mainPage = new Page(); +await mainPage.goto("https://news.ycombinator.com"); + +const { stories } = mainPage.extract({ + stories: [ + { + selector: ".athing", + limit: 5, + fields: { + id: { selector: "", attr: "id" }, + title: ".titleline > a", + url: { selector: ".titleline > a", attr: "href" }, + }, + }, + ], +}); + +const detailPages = stories.map(() => new Page()); +await Promise.all( + detailPages.map((p, i) => + p.goto(`https://news.ycombinator.com/item?id=${stories[i].id}`), + ), +); + +const results = stories.map((story, i) => { + const p = detailPages[i]; + const { comments } = p.extract({ + comments: [ + { + selector: ".comtr", + limit: 3, + fields: { + user: ".hnuser", + text: ".commtext", + }, + }, + ], + }); + return { + title: story.title, + url: story.url, + comments: comments, + }; +}); + +return results; diff --git a/hn-pandascript-skill-refactor.js b/hn-pandascript-skill-refactor.js new file mode 100644 index 000000000..8de91016c --- /dev/null +++ b/hn-pandascript-skill-refactor.js @@ -0,0 +1,44 @@ +// Navigate to Hacker News homepage +const mainPage = new Page(); +await mainPage.goto("https://news.ycombinator.com/"); + +// Extract the top 5 stories and their unique item IDs +const { stories } = mainPage.extract({ + stories: [{ + selector: "tr.athing", + limit: 5, + fields: { + id: { selector: "", attr: "id" }, + title: ".titleline > a", + link: { selector: ".titleline > a", attr: "href" } + } + }] +}); + +// Create parallel page instances for each of the top 5 stories +const commentPages = stories.map(() => new Page()); + +// Navigate to all 5 comment pages concurrently +await Promise.all(commentPages.map((p, i) => p.goto(`https://news.ycombinator.com/item?id=${stories[i].id}`))); + +// Extract the top 3 comments from each story's comment page +const results = commentPages.map((p, i) => { + const { comments } = p.extract({ + comments: [{ + selector: "tr.comtr", + limit: 3, + fields: { + author: "a.hnuser", + text: ".commtext" + } + }] + }); + return { + title: stories[i].title, + link: stories[i].link, + comments: comments + }; +}); + +// Return the aggregated list of stories with their comments +return results; diff --git a/hn.js b/hn.js new file mode 100644 index 000000000..894b9cac4 --- /dev/null +++ b/hn.js @@ -0,0 +1,39 @@ +const page = new Page(); +await page.goto("https://news.ycombinator.com"); + +const { stories } = page.extract({ + stories: [{ + selector: ".athing", + limit: 5, + fields: { + rank: ".rank", + title: ".titleline > a", + url: { selector: ".titleline > a", attr: "href" }, + id: { selector: "", attr: "id" } + } + }] +}); + +const results = []; + +for (const story of stories) { + await page.goto(`https://news.ycombinator.com/item?id=${story.id}`); + const { comments } = page.extract({ + comments: [{ + selector: ".comtr", + limit: 3, + fields: { + user: ".hnuser", + text: ".commtext" + } + }] + }); + results.push({ + rank: story.rank, + title: story.title, + url: story.url, + comments + }); +} + +return results; diff --git a/mcp.pid b/mcp.pid new file mode 100644 index 000000000..e5a8923a9 --- /dev/null +++ b/mcp.pid @@ -0,0 +1 @@ +26444 diff --git a/src/Config.zig b/src/Config.zig index 7129f0082..c58afed6a 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -374,6 +374,7 @@ const Commands = cli.Builder(.{ .{ .name = "system_prompt", .type = ?[:0]const u8 }, .{ .name = "task", .type = ?[]const u8 }, .{ .name = "save", .type = ?[]const u8 }, + .{ .name = "heal", .type = bool }, .{ .name = "attach", .short = 'a', .type = []const u8, .multiple = true }, .{ .name = "verbosity", .type = ?AgentVerbosity }, .{ .name = "effort", .type = ?Effort }, diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index fcd62388a..c30bd30fb 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -29,6 +29,8 @@ const Command = lp.Command; const Schema = lp.Schema; const Recorder = lp.Recorder; const ScriptRuntime = lp.Runtime; +const Baseline = lp.Baseline; + const Credentials = zenai.provider.Credentials; const App = @import("../App.zig"); @@ -152,6 +154,7 @@ session: *lp.Session, node_registry: CDPNode.Registry, terminal: Terminal, save_buffer: Recorder, +baseline: Baseline, save_path: ?[]u8, script_runtime_mutex: std.Io.Mutex = .init, active_script_runtime: ?*ScriptRuntime = null, @@ -160,6 +163,7 @@ model: []u8, /// Per-turn reasoning budget for LLM turns. Mutable at runtime via `/effort`. effort: Config.Effort, script_file: ?[]const u8, +heal: bool, one_shot_task: ?[]const u8, one_shot_save: ?[]const u8, one_shot_attachments: ?[]const []const u8, @@ -225,6 +229,20 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent return error.InvalidFilename; } } + if (opts.heal) { + if (opts.script_file == null) { + log.fatal(.app, "conflicting flags", .{ + .hint = "--heal repairs a failing script run; pass a script positional", + }); + return error.ConflictingFlags; + } + if (opts.no_llm) { + log.fatal(.app, "conflicting flags", .{ + .hint = "--heal needs an LLM to repair the script; drop --no-llm", + }); + return error.ConflictingFlags; + } + } if (opts.no_llm and opts.provider != null) { log.warn(.app, "ignoring --provider", .{ .reason = "--no-llm takes precedence" }); } @@ -234,10 +252,12 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent const is_one_shot = opts.task != null; const will_repl = !is_one_shot and opts.script_file == null; + // One-shot --task and --heal both need a model before any interaction. + const needs_llm_upfront = is_one_shot or opts.heal; // Load remembered selection up front so a saved null provider can flip the // REPL into basic mode before resolution. Pure script runs need nothing. - const remembered: ?settings.Remembered = if (will_repl or is_one_shot) settings.loadRemembered(allocator) else null; + const remembered: ?settings.Remembered = if (will_repl or needs_llm_upfront) settings.loadRemembered(allocator) else null; defer if (remembered) |r| std.zon.parse.free(allocator, r); // A remembered null provider means the user disabled the LLM via @@ -250,7 +270,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent // null provider. Without it the REPL accepts natural language, so an absent // API key would only surface at the first non-slash-command line — too late. // Pure JavaScript script runs stay allowed: no REPL, no LLM. - const requires_llm = is_one_shot or (will_repl and !opts.no_llm and !remembered_no_llm); + const requires_llm = needs_llm_upfront or (will_repl and !opts.no_llm and !remembered_no_llm); // Skip resolve when no client is wanted — else resolveCredentials prints // "No API key detected" for a run that does not need one. @@ -332,12 +352,14 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent .node_registry = .init(allocator), .terminal = .init(allocator, history_paths, verbosity, will_repl), .save_buffer = .init(allocator), + .baseline = .init(allocator), .save_path = null, .conversation = .init(allocator, opts.system_prompt orelse default_system_prompt), .model = model, .effort = effort, .stream_enabled = stream_enabled, .script_file = opts.script_file, + .heal = opts.heal, .one_shot_task = opts.task, .one_shot_save = opts.save, .one_shot_attachments = if (opts.attach.items.len == 0) null else opts.attach.items, @@ -375,6 +397,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent pub fn deinit(self: *Agent) void { self.terminal.uninstallLogSink(); self.save_buffer.deinit(); + self.baseline.deinit(); if (self.save_path) |p| self.allocator.free(p); self.terminal.deinit(); self.conversation.deinit(); @@ -402,6 +425,8 @@ fn startSession(self: *Agent) !void { self.session = try self.browser.newSession(self.notification); self.session.cancel_hook = .{ .context = @ptrCast(self), .check = checkCancel }; try self.session.enableConsoleCapture(); + // Node IDs are session-scoped; drop them with the session they point into. + self.node_registry.reset(); } // Compile-time constant; projected once per process to avoid rebuilding per call. @@ -473,8 +498,14 @@ fn drainCancellation(self: *Agent, baseline: usize) error{UserCancelled} { /// The side effects of `drainCancellation` without surfacing the error, for /// void callers (e.g. `/save` synthesis) that just need to clean up. fn resetAfterCancel(self: *Agent, baseline: usize) void { - self.endStreamedText(); + self.clearCancelState(); self.conversation.rollback(baseline); +} + +/// Cancel cleanup that touches no conversation state, for turns whose message +/// cleanup is owned elsewhere (meta-turns roll back via their own defer). +fn clearCancelState(self: *Agent) void { + self.endStreamedText(); self.browser.env.cancelTerminate(); self.cancel_requested.store(false, .release); self.http_interrupt.reset(); @@ -540,6 +571,7 @@ pub fn run(self: *Agent) bool { return ok; } if (self.script_file) |path| { + if (self.heal) return self.runScriptWithHeal(path); return self.runScript(path); } self.runRepl(); @@ -804,6 +836,7 @@ fn handleUsage(self: *Agent) void { fn clearConversation(self: *Agent) void { self.conversation.rollback(0); self.save_buffer.reset(); + self.baseline.reset(); if (self.save_path) |p| self.allocator.free(p); self.save_path = null; self.total_usage = .{}; @@ -834,7 +867,54 @@ fn handleLoad(self: *Agent, rest: []const u8) void { self.terminal.printError("usage: /load ", .{}); return; } - _ = self.runScript(path); + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + while (true) { + // Nothing from a prior retry is read again. + _ = arena.reset(.retain_capacity); + const finding: ScriptError = switch (self.runAndJudge(arena.allocator(), path)) { + .fatal, .clean => return, + .broken => |f| f, + }; + if (self.ai_client == null) return; + switch (promptHeal(path, finding.kind)) { + .no => return, + .retry => {}, + .heal => { + _ = self.healLoop(arena.allocator(), path, finding); + return; + }, + } + } +} + +const HealChoice = enum { heal, retry, no }; + +/// Defaults to no: healing spends tokens and its validation step resets the +/// browser session. Retry is the antidote to a transient failure (a flaky +/// page load) that would otherwise send a correct script to the model. +fn promptHeal(path: []const u8, kind: ScriptError.Kind) HealChoice { + const symptom = switch (kind) { + .threw => "failed", + .empty => "ran but returned no data", + .dry_extracts => "ran but some extracts came back empty", + }; + var header_buf: [256]u8 = undefined; + const header = std.fmt.bufPrint(&header_buf, "{s} {s}. Heal it with the model?", .{ path, symptom }) catch + "Script failed. Heal it with the model?"; + const labels: []const [:0]const u8 = &.{ + "heal — diagnose and fix, then validate in a fresh session (drops page/cookies like /reset)", + "retry — run the script again, in case the failure was transient (no tokens)", + "no — leave the script and session as they are", + }; + // Blank line so the question doesn't run into the script's output dump. + std.debug.print("\n", .{}); + const idx = picker.promptNumberedChoice(header, labels, 2) catch return .no; + return switch (idx) { + 0 => .heal, + 1 => .retry, + else => .no, + }; } const api_keys_hint = settings.api_keys_hint; @@ -1077,7 +1157,7 @@ fn handleSave(self: *Agent, arena: std.mem.Allocator, rest: []const u8) void { null; defer if (new_save_path) |p| self.allocator.free(p); - save.writeContentFile(path, self.save_buffer.bytes(), mode) catch |err| { + save.writeContentFile(path, self.appendSessionBaseline(arena, self.save_buffer.bytes()), mode) catch |err| { self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) }); return; }; @@ -1091,6 +1171,14 @@ fn handleSave(self: *Agent, arena: std.mem.Allocator, rest: []const u8) void { self.terminal.printInfo("Saved {d} line(s) to {s}", .{ saved_lines, self.save_path.? }); } +/// `script` with the session's extract baseline as its trailing comment (stale +/// baseline lines stripped); the script itself on any failure — the baseline +/// is telemetry, never worth failing a save over. +fn appendSessionBaseline(self: *Agent, arena: std.mem.Allocator, script: []const u8) []const u8 { + const line = self.baseline.serialize(arena) catch return script; + return Baseline.withBaseline(arena, script, line) catch script; +} + fn promptSaveMode(self: *Agent, path: []const u8) ?save.Mode { var header_buf: [256]u8 = undefined; const header = std.fmt.bufPrint(&header_buf, "{s} already exists. Pick save mode:", .{path}) catch @@ -1118,15 +1206,81 @@ fn promptSaveMode(self: *Agent, path: []const u8) ?save.Mode { return modes[idx]; } -fn failSave(self: *Agent, reason: []const u8) void { - self.terminal.printError("save failed: {s}", .{reason}); +fn failSynthesis(self: *Agent, label: []const u8, reason: []const u8) ?[]const u8 { + self.terminal.printError("{s} failed: {s}", .{ label, reason }); + return null; } -/// Roll the in-flight save turn back out of the conversation, then report the -/// failure — so a doomed `/save` synthesis never leaks its messages into history. -fn abortSave(self: *Agent, baseline: usize, reason: []const u8) void { - self.conversation.rollback(baseline); - self.failSave(reason); +/// One out-of-conversation LLM turn — the transport shared by `/save`/heal +/// synthesis and the replay verdict: swap `system` in for this turn, send +/// `user_msg`, and return the response text duped into `arena`. The turn is +/// always rolled back out of history; null (with a `label`-tagged report) on +/// error, no text, or cancellation. +fn metaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: []const u8, user_msg: []const u8, max_tokens: i32, effort: Config.Effort) ?[]const u8 { + self.conversation.ensureSystemPrompt() catch return self.failSynthesis(label, "out of memory"); + + // Regular turns keep the driver prompt. (`messages[0]` is the system + // message — rollback and prune never touch it.) + const plain_system = self.conversation.messages.items[0].content; + self.conversation.messages.items[0].content = system; + defer self.conversation.messages.items[0].content = plain_system; + + const baseline = self.conversation.messages.items.len; + self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); + defer self.conversation.rollback(baseline); + + return self.sendMetaTurn(arena, label, &self.conversation.messages, self.conversation.arena.allocator(), max_tokens, effort); +} + +/// `metaTurn` minus the conversation: the turn sends `system` + `user_msg` +/// alone, for judgments that must come from the message itself (the replay +/// verdict) — session history would only add tokens and sway the verdict. +fn soloTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, system: []const u8, user_msg: []const u8, max_tokens: i32, effort: Config.Effort) ?[]const u8 { + var messages: std.ArrayList(zenai.provider.Message) = .empty; + messages.append(arena, .{ .role = .system, .content = system }) catch return self.failSynthesis(label, "out of memory"); + messages.append(arena, .{ .role = .user, .content = user_msg }) catch return self.failSynthesis(label, "out of memory"); + return self.sendMetaTurn(arena, label, &messages, arena, max_tokens, effort); +} + +fn sendMetaTurn(self: *Agent, arena: std.mem.Allocator, label: []const u8, messages: *std.ArrayList(zenai.provider.Message), message_arena: std.mem.Allocator, max_tokens: i32, effort: Config.Effort) ?[]const u8 { + const provider_client = self.ai_client.?; + self.http_interrupt.reset(); + self.terminal.spinner.start(); + var result = provider_client.runTools( + self.model, + messages, + self.allocator, + message_arena, + .{ .context = @ptrCast(self), .callFn = handleToolCall }, + .{ + .tools = &.{}, + .max_turns = 1, + .max_tokens = max_tokens, + .tool_choice = .none, + .effort = effort, + .cancel = .{ .context = @ptrCast(self), .checkFn = checkCancel }, + }, + ) catch |err| { + self.terminal.spinner.cancel(); + if (self.cancel_requested.load(.acquire)) { + self.clearCancelState(); + return null; + } + log.err(.app, "AI meta-turn error", .{ .label = label, .err = err }); + return self.failSynthesis(label, @errorName(err)); + }; + self.terminal.spinner.stop(); + defer result.deinit(); + self.total_usage.add(result.usage); + if (result.cancelled) { + self.clearCancelState(); + return null; + } + + // `result.text` lives in `message_arena`, which the caller may roll back + // on return; the dupe happens before that. + const raw = result.text orelse return self.failSynthesis(label, "the model returned no text"); + return arena.dupe(u8, raw) catch self.failSynthesis(label, "out of memory"); } /// Save synthesis warrants more reasoning than a normal turn. `.none` stays off @@ -1173,8 +1327,6 @@ fn saveOneShot(self: *Agent) void { /// LLM synthesis + write for an already-resolved destination. Shared by the /// interactive `/save` and one-shot `--save`. fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mode: save.Mode, prompt: ?[]const u8) void { - const provider_client = self.ai_client.?; - // Only update feeds the saved script back to the model; append stays // blind — the script is synthesized from this session alone and written // after the existing content. @@ -1186,67 +1338,10 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo else null; - self.conversation.ensureSystemPrompt() catch return self.failSave("out of memory"); - - // Swap the dedicated save_system_prompt in as the system prompt for this one turn; - // regular turns keep the driver prompt. (`messages[0]` is the system - // message — rollback and prune never touch it.) - const plain_system = self.conversation.messages.items[0].content; - self.conversation.messages.items[0].content = savePrompt(previous_script != null); - defer self.conversation.messages.items[0].content = plain_system; - - const ma = self.conversation.arena.allocator(); - const baseline = self.conversation.messages.items.len; - - const user_msg = self.buildSaveSynthesisMessage(ma, path, previous_script, prompt) catch return self.failSave("out of memory"); - self.conversation.messages.append(self.allocator, .{ .role = .user, .content = user_msg }) catch return self.failSave("out of memory"); + const script = self.synthesizeScriptText(arena, "save", path, previous_script, prompt) orelse return; - self.http_interrupt.reset(); - self.terminal.spinner.start(); - var result = provider_client.runTools( - self.model, - &self.conversation.messages, - self.allocator, - ma, - .{ .context = @ptrCast(self), .callFn = handleToolCall }, - .{ - .tools = &.{}, - .max_turns = 1, - .max_tokens = 8192, - .tool_choice = .none, - .effort = bumpedEffort(self.effort), - .cancel = .{ .context = @ptrCast(self), .checkFn = checkCancel }, - }, - ) catch |err| { - self.terminal.spinner.cancel(); - if (self.cancel_requested.load(.acquire)) { - self.resetAfterCancel(baseline); - return; - } - log.err(.app, "AI save synthesis error", .{ .err = err }); - return self.abortSave(baseline, @errorName(err)); - }; - self.terminal.spinner.stop(); - defer result.deinit(); - self.total_usage.add(result.usage); - - if (result.cancelled) { - self.resetAfterCancel(baseline); - return; - } - - const raw = result.text orelse return self.abortSave(baseline, "the model returned no script"); - - // `result.text` lives in the conversation arena, freed by the rollback - // below; copy into the command arena first (scrubbing may return its input - // as-is). - const owned = arena.dupe(u8, save.stripCodeFence(raw)) catch return self.abortSave(baseline, "out of memory"); - const script = browser_tools.reverseSubstituteEnvVars(arena, owned) catch return self.abortSave(baseline, "out of memory"); - - // The save turn is a meta-action; keep it out of the ongoing conversation. - self.conversation.rollback(baseline); - - save.writeContentFile(path, script, mode) catch |err| { + const content = self.appendSessionBaseline(arena, script); + save.writeContentFile(path, content, mode) catch |err| { self.terminal.printError("failed to save {s}: {s}", .{ path, @errorName(err) }); return; }; @@ -1256,6 +1351,20 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo self.terminal.printInfo("Saved synthesized script to {s}", .{path}); } +/// The synthesis turn shared by `/save` and `--heal`: hand the model the +/// conversation, the deterministic record of what ran, and the previous script +/// when revising, then return the scrubbed script text (arena-owned). Reports +/// the failure under `label` and returns null on any error or cancellation; +/// the synthesis turn never stays in history. +fn synthesizeScriptText(self: *Agent, arena: std.mem.Allocator, label: []const u8, path: []const u8, previous_script: ?[]const u8, prompt: ?[]const u8) ?[]const u8 { + const user_msg = self.buildSaveSynthesisMessage(self.conversation.arena.allocator(), path, previous_script, prompt) catch + return self.failSynthesis(label, "out of memory"); + const system = savePrompt(previous_script != null); + const raw = self.metaTurn(arena, label, system, user_msg, 8192, bumpedEffort(self.effort)) orelse return null; + return browser_tools.reverseSubstituteEnvVars(arena, save.stripCodeFence(raw)) catch + return self.failSynthesis(label, "out of memory"); +} + /// Persist `path` as the destination reused by a subsequent bare `/save`. fn rememberSavePath(self: *Agent, path: []const u8) void { if (self.save_path) |old| { @@ -1411,7 +1520,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool .tool_call => |t| t, else => return .{ .text = "internal: command has no tool mapping", .is_error = true }, }; - return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| .{ + return self.callTool(arena, tc.name(), tc.args) catch |err| .{ .text = switch (err) { error.OutOfMemory => "out of memory", error.FrameNotLoaded => "no page loaded — run /goto first", @@ -1421,6 +1530,17 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool }; } +/// `browser_tools.call` plus the session-baseline note owed on a successful +/// extract, shared by both dispatch paths. The note runs on the uncapped +/// result — a truncated one parses as malformed and records nothing. +fn callTool(self: *Agent, arena: std.mem.Allocator, tool_name: []const u8, args: ?std.json.Value) browser_tools.ToolError!browser_tools.ToolResult { + const result = try browser_tools.call(arena, self.session, &self.node_registry, tool_name, args); + if (!result.is_error and std.mem.eql(u8, tool_name, @tagName(BrowserTool.extract))) { + self.baseline.noteExtractResult(result.text) catch {}; + } + return result; +} + /// Data output (/extract, /evaluate, /markdown, /tree, …) → plain stdout on /// success so a caller can pipe it. Everything else routes through /// `printToolOutcome`, which lays down the green ● / red ● dot shared with the @@ -1462,18 +1582,55 @@ const ScriptOutput = struct { } }; +const RunFacts = lp.heal.RunFacts; +const ScriptError = lp.heal.ScriptError; + +/// `fatal` covers setup failures (unreadable file, runtime init, OOM) that a +/// retry can't help. +const ScriptRunOutcome = union(enum) { + ok: RunFacts, + fatal, + script_error: ScriptError, +}; + fn runScript(self: *Agent, path: []const u8) bool { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + return switch (self.runScriptOutcome(arena.allocator(), path)) { + .ok => true, + .fatal, .script_error => false, + }; +} + +/// Terminal `fatal`/`clean` states, or the `broken` finding to heal — the model +/// gets to judge a clean-but-suspicious run. Shared by the `/load` heal offer +/// and one-shot `--heal`. +const RunJudgement = union(enum) { + fatal, + clean, + broken: ScriptError, +}; + +fn runAndJudge(self: *Agent, arena: std.mem.Allocator, path: []const u8) RunJudgement { + return switch (self.runScriptOutcome(arena, path)) { + .fatal => .fatal, + .ok => |facts| if (self.judgedFinding(arena, path, facts)) |finding| .{ .broken = finding } else .clean, + .script_error => |script_error| .{ .broken = script_error }, + }; +} + +fn runScriptOutcome(self: *Agent, arena: std.mem.Allocator, path: []const u8) ScriptRunOutcome { var script_arena: std.heap.ArenaAllocator = .init(self.allocator); defer script_arena.deinit(); const content = std.Io.Dir.cwd().readFileAlloc(lp.io, path, script_arena.allocator(), .limited(10 * 1024 * 1024)) catch |err| { self.terminal.printError("Failed to read script '{s}': {s}", .{ path, @errorName(err) }); - return false; + return .fatal; }; const runtime = ScriptRuntime.init(self.allocator, self.browser.app, self.session, &self.node_registry) catch |err| { self.terminal.printError("Failed to initialize script runtime: {s}", .{@errorName(err)}); - return false; + return .fatal; }; defer runtime.deinit(); self.script_runtime_mutex.lockUncancelable(lp.io); @@ -1494,18 +1651,242 @@ fn runScript(self: *Agent, path: []const u8) bool { const result = runtime.runSource(content, path); self.terminal.endTool(); - if (result catch |err| { + const run_result = result catch |err| { self.terminal.printError("Script failed: {s}", .{@errorName(err)}); - return false; - }) |message| { - self.terminal.printError("{s}", .{message}); - return false; + return .fatal; + }; + if (run_result == .err) { + self.terminal.printError("{s}", .{run_result.err}); + // A Ctrl-C termination is not a script defect — never heal it. + if (self.cancel_requested.load(.acquire)) return .fatal; + } else { + // A script that printed nothing leaves no trace, so freeze the spinner + // into a green bullet (like /goto); one that printed already showed its + // result. + if (!output.emitted) self.terminal.printScriptDone("script", path); } + // `content` dies with script_arena; the outcome's source must outlive it. + const source = arena.dupe(u8, content) catch return .fatal; + return switch (lp.heal.classifyRun(arena, run_result, source) catch return .fatal) { + .facts => |facts| .{ .ok = facts }, + .script_error => |script_error| .{ .script_error = script_error }, + }; +} - // A script that printed nothing leaves no trace, so freeze the spinner into - // a green bullet (like /goto); one that printed already showed its result. - if (!output.emitted) self.terminal.printScriptDone("script", path); - return true; +/// One-shot `--heal` run; counterpart of the REPL's `/load` heal offer. +fn runScriptWithHeal(self: *Agent, path: []const u8) bool { + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + + // A re-run is free; a verdict turn and healing are not. The first pass + // gates on the local suspicion check alone — one retry filters a transient + // failure (a flaky page load), and the model judges the re-run's facts, + // the ones healing would act on. + switch (self.runScriptOutcome(arena.allocator(), path)) { + .fatal => return false, + .ok => |facts| if (lp.heal.suspicionOf(arena.allocator(), facts) == null) return true, + .script_error => {}, + } + self.terminal.printInfo("Script run failed or looks broken; retrying once before healing.", .{}); + const finding: ScriptError = switch (self.runAndJudge(arena.allocator(), path)) { + .fatal => return false, + .clean => return true, + .broken => |f| f, + }; + // Healing spends tokens; report the cost the way --task does. + defer self.printUsageSummary(); + return self.healLoop(arena.allocator(), path, finding); +} + +const max_heal_attempts = 2; + +/// Only a revision that passed validation in a fresh session replaces `path`; +/// the original survives a failed heal. +fn healLoop(self: *Agent, arena: std.mem.Allocator, path: []const u8, first: ScriptError) bool { + var source = first.source; + var error_detail = first.detail; + + const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch return self.healOom(); + + // Every exit leaves no stray revision behind; a failed rename is the one + // deliberate keep (its error message points at tmp_path). + var keep_revision = false; + defer if (!keep_revision) self.removeTempScript(tmp_path); + + // The recorder and baseline are /save's stream: heal must neither + // synthesize from the REPL's prior recordings nor leak its diagnose + // actions into a later /save. + var prior: Recorder = .init(self.allocator); + var prior_baseline: Baseline = .init(self.allocator); + std.mem.swap(Recorder, &self.save_buffer, &prior); + std.mem.swap(Baseline, &self.baseline, &prior_baseline); + defer { + std.mem.swap(Recorder, &self.save_buffer, &prior); + std.mem.swap(Baseline, &self.baseline, &prior_baseline); + prior.deinit(); + prior_baseline.deinit(); + } + + var attempt: usize = 1; + while (attempt <= max_heal_attempts) : (attempt += 1) { + self.terminal.printInfo("Healing {s} (attempt {d}/{d})", .{ path, attempt, max_heal_attempts }); + + const diagnose = lp.heal.buildDiagnoseMessage(arena, path, source, error_detail) catch return self.healOom(); + if (!self.runTurn(.{ .prompt = diagnose, .capture_for_save = true, .label = "Heal" })) return false; + + const revised = self.synthesizeScriptText(arena, "heal", path, source, lp.heal.heal_revision_prompt) orelse return false; + + save.writeContentFile(tmp_path, revised, .replace) catch |err| { + self.terminal.printError("heal failed: could not write {s}: {s}", .{ tmp_path, @errorName(err) }); + return false; + }; + + // Validate in a fresh session so failure-state cookies and pages can't + // mask a still-broken script. + self.startSession() catch |err| { + self.terminal.printError("heal failed: could not start a fresh session: {s}", .{@errorName(err)}); + return false; + }; + + switch (self.runScriptOutcome(arena, tmp_path)) { + .ok => |facts| { + if (lp.heal.cureFailure(arena, first, facts) catch return self.healOom()) |failure| { + self.terminal.printWarning("{s}", .{failure}); + source = revised; + error_detail = failure; + } else { + // Refresh the baseline from the validation run — synthesis + // may have copied the stale one from the broken script. + // Best-effort: the validated revision is already on disk. + if (lp.heal.refreshedBaselineScript(arena, revised, facts.extract_stats)) |updated| { + save.writeContentFile(tmp_path, updated, .replace) catch {}; + } + keep_revision = true; + const cwd = std.Io.Dir.cwd(); + cwd.rename(tmp_path, cwd, path, lp.io) catch |err| { + self.terminal.printError("healed script validated but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }); + return false; + }; + self.terminal.printInfo("Healed {s}: the revised script validated in a fresh session.", .{path}); + return true; + } + }, + .fatal => return false, + .script_error => |script_error| { + source = revised; + error_detail = script_error.detail; + }, + } + } + self.terminal.printError("heal gave up after {d} attempts; {s} is unchanged", .{ max_heal_attempts, path }); + return false; +} + +fn healOom(self: *Agent) bool { + self.terminal.printError("heal failed: out of memory", .{}); + return false; +} + +const verdict_system_prompt = + \\You judge replays of saved browser-automation scripts. Given the script + \\and facts about a run that completed without errors, decide whether the + \\empty output means the script is broken (stale selectors after a site + \\change) or legitimate (the page genuinely has no such data right now). + \\A ` +++ std.mem.trimEnd(u8, Baseline.marker, " ") ++ + \\` comment, when present, records how often each output + \\field carried data when the script was saved — weigh it as evidence. + \\Respond with your verdict on its own final line: + \\ +++ verdict_marker ++ + \\ {"broken": true|false, "fields": ["", ...], "reason": ""} + \\`fields` names the output fields you judge broken; use "" for the whole + \\return value. +; + +const verdict_marker = "VERDICT:"; + +const Verdict = struct { + broken: bool, + /// Output fields the model judged broken; null = the whole return value. + fields: []const ?[]const u8, + reason: []const u8, +}; + +/// One tool-free LLM turn deciding whether suspicious facts are breakage. +/// Null on transport/parse failure or cancellation — the caller falls back to +/// surfacing the facts instead of guessing. +fn judgeSuspicion(self: *Agent, arena: std.mem.Allocator, path: []const u8, suspicion: ScriptError) ?Verdict { + const user_msg = std.fmt.allocPrint(arena, + \\Replay of {s} completed without errors, but {s} + \\ + \\The script (its comments and structure carry the intent): + \\```js + \\{s} + \\``` + , .{ path, suspicion.detail, suspicion.source }) catch return null; + const raw = self.soloTurn(arena, "verdict", verdict_system_prompt, user_msg, 1024, self.effort) orelse return null; + return parseVerdict(arena, raw); +} + +fn parseVerdict(arena: std.mem.Allocator, raw: []const u8) ?Verdict { + const at = std.mem.lastIndexOf(u8, raw, verdict_marker) orelse return null; + const rest = raw[at + verdict_marker.len ..]; + const start = std.mem.indexOfScalar(u8, rest, '{') orelse return null; + const end = std.mem.lastIndexOfScalar(u8, rest, '}') orelse return null; + if (end < start) return null; + const Wire = struct { broken: bool, fields: []const []const u8 = &.{}, reason: []const u8 = "" }; + const wire = std.json.parseFromSliceLeaky(Wire, arena, rest[start .. end + 1], .{ .ignore_unknown_fields = true }) catch return null; + const fields = arena.alloc(?[]const u8, wire.fields.len) catch return null; + for (wire.fields, fields) |f, *out| out.* = if (f.len == 0) null else f; + return .{ .broken = wire.broken, .fields = fields, .reason = wire.reason }; +} + +/// The finding to heal, per the model's verdict on suspicious facts; null when +/// the run is fine (no suspicion, no model, or a not-broken verdict). A failed +/// verdict call falls back to the raw facts — the heal prompt, defaulting to +/// no, becomes the judgment of last resort. +fn judgedFinding(self: *Agent, arena: std.mem.Allocator, path: []const u8, facts: RunFacts) ?ScriptError { + const suspicion = lp.heal.suspicionOf(arena, facts) orelse return null; + if (self.ai_client == null) return null; + if (self.judgeSuspicion(arena, path, suspicion)) |verdict| { + if (!verdict.broken) { + self.terminal.printInfo("Output has empty fields, but the model judged the run consistent: {s}", .{verdict.reason}); + return null; + } + return .{ + .kind = suspicion.kind, + .detail = std.fmt.allocPrint(arena, "{s}\nVerdict: {s}", .{ suspicion.detail, verdict.reason }) catch return null, + .source = suspicion.source, + .dry_fields = confirmedDryFields(arena, verdict.fields, suspicion.dry_fields) catch return null, + }; + } + return suspicion; +} + +/// Verdict field names minus any that weren't actually dry — a hallucinated +/// name never matches an extract stat, so the cure check could never clear it. +/// Falls back to every dry field when none match. +fn confirmedDryFields(arena: std.mem.Allocator, judged: []const ?[]const u8, actual: []const ?[]const u8) error{OutOfMemory}![]const ?[]const u8 { + if (judged.len == 0) return actual; + var kept: std.ArrayList(?[]const u8) = .empty; + for (judged) |f| { + for (actual) |a| { + if (ScriptRuntime.fieldEql(f, a)) { + try kept.append(arena, f); + break; + } + } + } + return if (kept.items.len == 0) actual else kept.items; +} + +fn removeTempScript(self: *Agent, tmp_path: []const u8) void { + std.Io.Dir.cwd().deleteFile(lp.io, tmp_path) catch |err| switch (err) { + // Covers exits before the first write. + error.FileNotFound => {}, + else => self.terminal.printWarning("could not remove temp script {s}: {s}", .{ tmp_path, @errorName(err) }), + }; } /// Mirror a user-typed slash command into `self.conversation.messages` as if the @@ -1533,9 +1914,9 @@ fn recordSlashToolCall( .arguments = if (args) |v| try zenai.json.dupeValue(ma, v) else null, }; - // capToolOutput returns its input unchanged under the cap; dupe so content + // capBytes returns its input unchanged under the cap; dupe so content // doesn't alias the caller's per-iteration arena. - const capped = capToolOutput(ma, result.text); + const capped = string.capBytes(ma, result.text, tool_output_max_bytes); const content = if (capped.ptr == result.text.ptr) try ma.dupe(u8, capped) else capped; const tool_results = try ma.alloc(zenai.provider.ToolResult, 1); @@ -1811,14 +2192,6 @@ fn buildUserMessageParts( // the next request body) without bound. const tool_output_max_bytes: usize = 1 * 1024 * 1024; -fn capToolOutput(allocator: std.mem.Allocator, output: []const u8) []const u8 { - if (output.len <= tool_output_max_bytes) return output; - const prefix = string.truncateUtf8(output, tool_output_max_bytes); - var suffix_buf: [64]u8 = undefined; - const suffix = std.fmt.bufPrint(&suffix_buf, "\n...[truncated, original {d} bytes]", .{output.len}) catch return prefix; - return std.mem.concat(allocator, u8, &.{ prefix, suffix }) catch prefix; -} - fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) zenai.provider.Client.ToolHandler.Result { const self: *Agent = @ptrCast(@alignCast(ctx)); // Close any assistant text streamed this turn before the tool spinner shows. @@ -1835,8 +2208,8 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co self.terminal.spinner.setTool(tool_name, args_str); defer self.terminal.spinner.setThinking(); - const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments)) |result| - .{ .content = capToolOutput(allocator, result.text), .is_error = result.is_error } + const outcome: zenai.provider.Client.ToolHandler.Result = if (self.callTool(allocator, tool_name, arguments)) |result| + .{ .content = string.capBytes(allocator, result.text, tool_output_max_bytes), .is_error = result.is_error } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{@errorName(err)}) catch "Error: tool execution failed", .is_error = true }; @@ -1937,9 +2310,52 @@ fn completionModels(context: *anyopaque, _: std.mem.Allocator) []const []const u test { _ = save; _ = settings; + _ = Baseline; _ = picker; } +test "confirmedDryFields drops hallucinated names, keeps the narrowing" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const actual: []const ?[]const u8 = &.{ @as(?[]const u8, "comments"), @as(?[]const u8, "score"), null }; + + const narrowed = try confirmedDryFields(aa, &.{@as(?[]const u8, "comments")}, actual); + try std.testing.expectEqual(1, narrowed.len); + try std.testing.expectEqualStrings("comments", narrowed[0].?); + + const mixed = try confirmedDryFields(aa, &.{ @as(?[]const u8, "nonexistent"), @as(?[]const u8, "score") }, actual); + try std.testing.expectEqual(1, mixed.len); + try std.testing.expectEqualStrings("score", mixed[0].?); + + // All hallucinated → the full dry set, not an empty one. + const fallback = try confirmedDryFields(aa, &.{@as(?[]const u8, "bogus")}, actual); + try std.testing.expectEqual(3, fallback.len); + + try std.testing.expectEqual(actual.ptr, (try confirmedDryFields(aa, &.{}, actual)).ptr); +} + +test "parseVerdict: anchored to the marker, tolerant of prose around it" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const v = parseVerdict(aa, "The {comments} baseline says data was always present.\nVERDICT: {\"broken\": true, \"fields\": [\"comments\", \"\"], \"reason\": \"selector drift\"}").?; + try std.testing.expect(v.broken); + try std.testing.expectEqual(2, v.fields.len); + try std.testing.expectEqualStrings("comments", v.fields[0].?); + try std.testing.expectEqual(null, v.fields[1]); + try std.testing.expectEqualStrings("selector drift", v.reason); + + const fenced = parseVerdict(aa, "As instructed, VERDICT: first.\nVERDICT:\n```json\n{\"broken\": false}\n```").?; + try std.testing.expect(!fenced.broken); + + try std.testing.expectEqual(null, parseVerdict(aa, "{\"broken\": true, \"fields\": []}")); + try std.testing.expectEqual(null, parseVerdict(aa, "VERDICT: no json here")); + try std.testing.expectEqual(null, parseVerdict(aa, "VERDICT: {\"fields\": []}")); +} + test "savePrompt: save instructions followed by the rendered script skill" { const prompt = savePrompt(false); try std.testing.expect(std.mem.startsWith(u8, prompt, browser_tools.save_synthesis_prompt)); @@ -1949,32 +2365,3 @@ test "savePrompt: save instructions followed by the rendered script skill" { try std.testing.expect(std.mem.indexOf(u8, revision, save_revision_note) != null); try std.testing.expect(std.mem.endsWith(u8, revision, lp.skill.text())); } - -test "capToolOutput: passes through when under cap" { - const ta = std.testing.allocator; - const out = capToolOutput(ta, "short"); - try std.testing.expectEqualStrings("short", out); -} - -// Boundary correctness lives in string.zig's `truncateUtf8` tests; here we only -// assert the agent-specific policy: an over-cap body keeps valid UTF-8 and gains -// the truncation marker. -test "capToolOutput: appends a marker when truncating" { - const ta = std.testing.allocator; - - // 3-byte Hangul codepoint (U+D55C '한' = 0xED 0x95 0x9C) straddling the cap. - const cap = tool_output_max_bytes; - const buf = try ta.alloc(u8, cap + 8); - defer ta.free(buf); - @memset(buf[0 .. cap - 1], 'a'); - buf[cap - 1] = 0xED; - buf[cap + 0] = 0x95; - buf[cap + 1] = 0x9C; - @memset(buf[cap + 2 ..], 'b'); - - const out = capToolOutput(ta, buf); - defer if (out.ptr != buf.ptr) ta.free(out); - - try std.testing.expect(std.unicode.utf8ValidateSlice(out)); - try std.testing.expect(std.mem.indexOf(u8, out, "truncated") != null); -} diff --git a/src/agent/Terminal.zig b/src/agent/Terminal.zig index 044bc3a0d..f608b7f73 100644 --- a/src/agent/Terminal.zig +++ b/src/agent/Terminal.zig @@ -269,7 +269,7 @@ pub fn endAssistantStream(self: *Terminal) void { // Must exceed the downstream LLM-judge's snapshot window for full grounding // evidence. Does not cap the agent's own LLM, which gets up to -// tool_output_max_bytes (1 MiB) via Agent.zig:capToolOutput. Bypassed in REPL +// tool_output_max_bytes (1 MiB) via string.zig:capBytes. Bypassed in REPL // where the human can scroll. const max_result_display_len = 2000; diff --git a/src/help.zon b/src/help.zon index e88ede3b0..68cececc9 100644 --- a/src/help.zon +++ b/src/help.zon @@ -165,6 +165,7 @@ \\ {0s} agent --provider ollama --model qwen3.5:latest \\ {0s} agent --no-llm (basic slash-command-only REPL) \\ {0s} run script.js (replay a saved script; see `run`) + \\ {0s} agent script.js --heal (run it; LLM repairs it on failure) \\ {0s} agent --task "..." --save out.js (synthesize a replayable script) \\ \\Arguments: @@ -198,6 +199,12 @@ \\ default (Mistral defaults to none, as its default model rejects \\ effort). In the REPL, use /effort to change it. \\ Allowed values: none, minimal, low, medium, high, xhigh. + \\ --heal + \\ When the positional script fails, diagnose the failure with the + \\ LLM against the live page, revise the script, and validate the + \\ revision by re-running it in a fresh session. The script file is + \\ overwritten only when the revised version passes. Requires the + \\ script positional and an LLM provider. \\ --list-models \\ Print the model IDs usable with `agent` for --provider, one per \\ line, sorted, and exit. Auto-detects the provider from env when diff --git a/src/lightpanda.zig b/src/lightpanda.zig index 06ceb23d7..bd58a57d8 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -49,8 +49,10 @@ pub const HttpClient = @import("network/HttpClient.zig"); pub const mcp = @import("mcp.zig"); pub const Agent = @import("agent/Agent.zig"); pub const Command = @import("script/command.zig").Command; +pub const Baseline = @import("script/Baseline.zig"); pub const Recorder = @import("script/Recorder.zig"); pub const Runtime = @import("script/Runtime.zig"); +pub const heal = @import("script/heal.zig"); pub const Schema = @import("script/Schema.zig"); pub const skill = @import("script/skill.zig"); pub const cookies = @import("cookies.zig"); diff --git a/src/mcp/Server.zig b/src/mcp/Server.zig index a25c59aa9..fc564cc80 100644 --- a/src/mcp/Server.zig +++ b/src/mcp/Server.zig @@ -120,6 +120,22 @@ pub fn createSession(self: *Self, id: []const u8) !*Session { return entry; } +/// Replace `entry`'s browsing session with a fresh one — pages, cookies and +/// node ids dropped — as heal validation requires. The default session +/// reloads the on-disk cookie file: that file is the clean baseline identity; +/// the in-memory failure-state cookies are what the fresh session discards. +pub fn restartSession(self: *Self, entry: *Session) !void { + entry.session = try entry.browser.newSession(entry.notification); + try entry.session.enableConsoleCapture(); + // Node IDs are session-scoped; drop them with the session they point into. + entry.node_registry.reset(); + if (entry.isDefault()) { + if (self.app.config.cookieFile()) |cookie_path| { + lp.cookies.loadFromFile(entry.session, cookie_path); + } + } +} + /// Switch to the multi-isolate discipline: park the default (which `Server.init` /// left entered) and require every use to bracket with `enterIsolate`. The HTTP /// transport calls this on its worker thread before serving anyone. @@ -220,7 +236,7 @@ pub fn handleInitialize(self: *Self, req: protocol.Request) !void { .tools = .{}, }, .serverInfo = .{ .name = "lightpanda", .version = "0.1.0" }, - .instructions = lp.tools.driver_guidance, + .instructions = lp.tools.driver_guidance ++ tools.script_lifecycle_note, }); } diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig index 3ce55fc52..ea971b4cc 100644 --- a/src/mcp/tools.zig +++ b/src/mcp/tools.zig @@ -4,6 +4,8 @@ const lp = @import("lightpanda"); const js = lp.js; const browser_tools = lp.tools; const BrowserTool = browser_tools.Tool; +const ScriptRuntime = lp.Runtime; +const string = @import("../string.zig"); const protocol = @import("protocol.zig"); const Server = @import("Server.zig"); @@ -55,7 +57,59 @@ const session_id_schema = browser_tools.minify( \\} ); +const replay_schema = browser_tools.minify( + \\{ + \\ "type": "object", + \\ "properties": { + \\ "path": { "type": "string", "description": "Relative path (no '..' segments) of the saved script to replay." }, + \\ "script": { "type": "string", "description": "Optional: script text to run instead of the file's contents - trial a candidate revision without writing it. `path` still names the run." } + \\ }, + \\ "required": ["path"] + \\} +); + +const heal_commit_schema = browser_tools.minify( + \\{ + \\ "type": "object", + \\ "properties": { + \\ "path": { "type": "string", "description": "Relative path (no '..' segments) of the broken script; replaced atomically only on cure." }, + \\ "script": { "type": "string", "description": "The full revised script. Keep $LP_* placeholders; never inline a resolved secret." }, + \\ "failure": { + \\ "type": "object", + \\ "description": "The `failure` object from the replay report you are healing, echoed back verbatim.", + \\ "properties": { + \\ "kind": { "type": "string", "enum": ["threw", "empty", "dry_extracts"] }, + \\ "detail": { "type": "string" }, + \\ "dry_fields": { "type": "array", "items": { "type": "string" }, "description": "For dry_extracts: the fields that must come back with data (\"\" = a whole-array extract). Deleting an extract is not a cure." } + \\ }, + \\ "required": ["kind"] + \\ } + \\ }, + \\ "required": ["path", "script", "failure"] + \\} +); + +/// Appended to the `initialize` instructions (`driver_guidance` is shared with +/// the standalone agent, which has no replay/heal tools — keep this MCP-only). +pub const script_lifecycle_note = + \\Script lifecycle: `save` a finished session as a script, `replay` it + \\any time for a token-free re-run, and when a replay reports it broken, + \\heal it — diagnose against the live session, then `heal_commit` a + \\revision (validated in a fresh session before it replaces the file). + \\ +; + const extra_tools = [_]McpTool{ + .{ + .name = "replay", + .description = "Replay a saved Lightpanda agent script (see `save`) and return a JSON run report. `status` is \"ok\" (ran, output carries data), \"suspicious\" (ran clean but the output looks dry — judge whether that is breakage or the page genuinely has no such data right now, weighing any `// lp:baseline` comment in `source` as evidence of what the fields held at save time), or \"failed\" (the script threw). The script's `console.*` output and returned value arrive in `console` (the returned value is the final line). On suspicious/failed the report carries the script `source`, a `failure` object and `guidance` for the heal flow: diagnose against the live session, then call `heal_commit`. Pass `script` to trial a candidate revision without writing it. The replay drives this session — the current page, cookies and node ids are replaced; re-inspect (tree) before reusing node ids.", + .inputSchema = replay_schema, + }, + .{ + .name = "heal_commit", + .description = "Commit a healed script: the revised `script` is validated by replaying it in a fresh session, and only a validated cure replaces the file at `path` — the original is untouched otherwise. Echo back the `failure` object from the replay report you are healing; the cure check is deterministic: `threw` needs a clean run, `empty` needs the return value to carry data, `dry_extracts` needs every listed field to come back with data (deleting the extract is not a cure). The response is a JSON heal report; on `cured: false` its `failure` says what is still wrong — diagnose further and try again. Afterwards the session is the fresh validation session at the script's end state (all prior node ids are stale).\n\n" ++ lp.heal.heal_revision_prompt ++ "\n\n" ++ browser_tools.save_synthesis_prompt ++ "\n\n" ++ browser_tools.save_script_rules, + .inputSchema = heal_commit_schema, + }, .{ .name = "save", .description = "Save the session as a reusable Lightpanda agent script. You hold the conversation, so synthesize the `script` yourself — `const page = new Page(); await page.goto(url);` then call the builtins you used as tools (extract, click, fill, …) as methods on `page` with the same object arguments. Keep `$LP_*` placeholders; never inline a resolved secret.\n\n" ++ browser_tools.save_synthesis_prompt ++ "\n\n" ++ browser_tools.save_script_rules, @@ -82,6 +136,8 @@ const all_tools = browser_tool_list ++ extra_tools; /// Tools that bypass the browser-tool dispatch and have their own handlers. const ExtraTool = enum { + replay, + heal_commit, save, session_new, session_list, @@ -104,6 +160,8 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque if (std.meta.stringToEnum(ExtraTool, call_params.name)) |tool| { return switch (tool) { + .replay => handleReplay(server, arena, id, call_params.arguments), + .heal_commit => handleHealCommit(server, arena, id, call_params.arguments), .save => handleSave(server, arena, id, call_params.arguments), .session_new => handleSessionNew(server, arena, id, call_params.arguments), .session_list => handleSessionList(server, arena, id), @@ -179,6 +237,205 @@ fn handleSave(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arg try sendToolResultText(server, id, msg, false); } +// Agent parity: the CLI reads scripts with the same bound. +const max_script_bytes = 10 * 1024 * 1024; + +/// Bound `RunReport.source` — a script is human-scale, but a synthesized one +/// with an embedded blob shouldn't balloon the report. +const source_max_bytes = 64 * 1024; + +/// Caps captured `console.*` output so a chatty script can't balloon the +/// report. +const ConsoleCollector = struct { + arena: std.mem.Allocator, + lines: std.ArrayList(lp.heal.ConsoleLine) = .empty, + bytes: usize = 0, + truncated: bool = false, + + const max_bytes = 16 * 1024; + + fn sink(self: *ConsoleCollector) ScriptRuntime.ConsoleSink { + return .{ .context = @ptrCast(self), .write = write }; + } + + fn write(context: *anyopaque, method: ScriptRuntime.ConsoleMethod, line: []const u8) void { + const self: *ConsoleCollector = @ptrCast(@alignCast(context)); + if (self.bytes >= max_bytes) { + self.truncated = true; + return; + } + // Scrub any resolved LP_* secret a script may have printed. + const scrubbed = browser_tools.reverseSubstituteEnvVars(self.arena, line) catch { + self.truncated = true; + return; + }; + const capped = string.capBytes(self.arena, scrubbed, max_bytes - self.bytes); + if (capped.ptr != scrubbed.ptr) self.truncated = true; + // A line that passed through both unchanged still aliases the + // runtime's per-call arena, which dies before the report is sent. + const text = if (capped.ptr == line.ptr) + self.arena.dupe(u8, capped) catch { + self.truncated = true; + return; + } + else + capped; + self.lines.append(self.arena, .{ .level = @tagName(method), .text = text }) catch { + self.truncated = true; + return; + }; + self.bytes += text.len; + } +}; + +fn runClassified(server: *Server, arena: std.mem.Allocator, path: []const u8, source: []const u8, collector: *ConsoleCollector) !lp.heal.Classified { + const active = server.active_session; + const runtime = try ScriptRuntime.init(server.allocator, server.app, active.session, &active.node_registry); + defer runtime.deinit(); + runtime.console_sink = collector.sink(); + const result = try runtime.runSource(source, path); + return lp.heal.classifyRun(arena, result, source); +} + +fn buildRunReport( + arena: std.mem.Allocator, + path: []const u8, + classified: lp.heal.Classified, + collector: *const ConsoleCollector, + with_guidance: bool, +) error{OutOfMemory}!lp.heal.RunReport { + var report: lp.heal.RunReport = .{ + .status = .ok, + .path = path, + .console = collector.lines.items, + .console_truncated = collector.truncated, + }; + switch (classified) { + .script_error => |script_error| { + report.status = .failed; + report.failure = try lp.heal.wireFailure(arena, script_error); + report.source = try scrubbedSource(arena, script_error.source); + if (with_guidance) report.guidance = lp.heal.replay_failed_guidance; + }, + .facts => |facts| { + report.returned = facts.returned; + report.extracts = facts.extract_stats; + if (lp.heal.suspicionOf(arena, facts)) |suspicion| { + report.status = .suspicious; + report.failure = try lp.heal.wireFailure(arena, suspicion); + report.source = try scrubbedSource(arena, facts.source); + if (with_guidance) report.guidance = lp.heal.replay_suspicious_guidance; + } + }, + } + return report; +} + +fn scrubbedSource(arena: std.mem.Allocator, source: []const u8) error{OutOfMemory}![]const u8 { + return string.capBytes(arena, try browser_tools.reverseSubstituteEnvVars(arena, source), source_max_bytes); +} + +fn sendReport(server: *Server, arena: std.mem.Allocator, id: std.json.Value, report: anytype) !void { + const json = std.json.Stringify.valueAlloc(arena, report, .{ .emit_null_optional_fields = false }) catch + return sendErrorContent(server, id, "out of memory"); + try sendToolResultText(server, id, json, false); +} + +fn handleReplay(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void { + const Args = struct { path: []const u8, script: ?[]const u8 = null }; + const args = browser_tools.parseArgs(Args, arena, arguments) catch { + return server.sendError(id, .InvalidParams, "expected { path: string, script?: string }"); + }; + if (!browser_tools.isPathSafe(args.path)) { + return sendErrorContent(server, id, "path must be relative and must not contain '..' segments"); + } + const source = args.script orelse std.Io.Dir.cwd().readFileAlloc(lp.io, args.path, arena, .limited(max_script_bytes)) catch |err| { + const msg = std.fmt.allocPrint(arena, "could not read {s}: {s}", .{ args.path, @errorName(err) }) catch "could not read script"; + return sendErrorContent(server, id, msg); + }; + + var collector: ConsoleCollector = .{ .arena = arena }; + const classified = runClassified(server, arena, args.path, source, &collector) catch |err| switch (err) { + error.OutOfMemory => return sendErrorContent(server, id, "out of memory"), + error.RuntimeInitFailed, error.TooManyContexts => return sendErrorContent(server, id, "could not initialize the script runtime"), + }; + // A failed script is still a successful replay: report it in-band, never + // as a tool error — the report is the answer. + const report = buildRunReport(arena, args.path, classified, &collector, true) catch + return sendErrorContent(server, id, "out of memory"); + return sendReport(server, arena, id, report); +} + +fn handleHealCommit(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void { + const Args = struct { path: []const u8, script: []const u8, failure: lp.heal.WireFailure }; + const args = browser_tools.parseArgs(Args, arena, arguments) catch { + return server.sendError(id, .InvalidParams, "expected { path: string, script: string, failure: { kind, detail?, dry_fields? } }"); + }; + if (!browser_tools.isPathSafe(args.path)) { + return sendErrorContent(server, id, "path must be relative and must not contain '..' segments"); + } + const first = lp.heal.scriptErrorFromWire(arena, args.failure) catch + return sendErrorContent(server, id, "out of memory"); + // The client never sees resolved secrets, but scrub as a safety net + // before running or persisting the candidate. + const script = browser_tools.reverseSubstituteEnvVars(arena, args.script) catch + return sendErrorContent(server, id, "out of memory"); + + // Validate in a fresh session so failure-state cookies and pages can't + // mask a still-broken script. + server.restartSession(server.active_session) catch |err| { + const msg = std.fmt.allocPrint(arena, "could not start a fresh session: {s}", .{@errorName(err)}) catch "could not start a fresh session"; + return sendErrorContent(server, id, msg); + }; + + var collector: ConsoleCollector = .{ .arena = arena }; + const classified = runClassified(server, arena, args.path, script, &collector) catch |err| switch (err) { + error.OutOfMemory => return sendErrorContent(server, id, "out of memory"), + error.RuntimeInitFailed, error.TooManyContexts => return sendErrorContent(server, id, "could not initialize the script runtime"), + }; + const run = buildRunReport(arena, args.path, classified, &collector, false) catch + return sendErrorContent(server, id, "out of memory"); + + var report: lp.heal.HealReport = .{ .cured = false, .committed = false, .run = run }; + switch (classified) { + .script_error => |script_error| report.failure = script_error.detail, + .facts => |facts| { + const residual = lp.heal.cureFailure(arena, first, facts) catch + return sendErrorContent(server, id, "out of memory"); + if (residual) |failure| { + report.failure = failure; + } else { + report.cured = true; + commitHealed(arena, args.path, script, facts.extract_stats, &report); + } + }, + } + return sendReport(server, arena, id, report); +} + +/// Write the validated revision next to `path` and atomically swap it in, +/// refreshing the `// lp:baseline` line from the validation run. Failures +/// land in the report (`cured` stays true, `committed` false). +fn commitHealed(arena: std.mem.Allocator, path: []const u8, script: []const u8, stats: []const ScriptRuntime.ExtractStat, report: *lp.heal.HealReport) void { + const final = lp.heal.refreshedBaselineScript(arena, script, stats) orelse script; + const tmp_path = std.fmt.allocPrint(arena, "{s}.heal.js", .{path}) catch { + report.failure = "validated, but out of memory writing the revision"; + return; + }; + writeScript(tmp_path, final) catch |err| { + std.Io.Dir.cwd().deleteFile(lp.io, tmp_path) catch {}; + report.failure = std.fmt.allocPrint(arena, "validated, but writing {s} failed: {s}", .{ tmp_path, @errorName(err) }) catch null; + return; + }; + const cwd = std.Io.Dir.cwd(); + cwd.rename(tmp_path, cwd, path, lp.io) catch |err| { + // Deliberately keep the revision; the message points at it. + report.failure = std.fmt.allocPrint(arena, "validated, but replacing {s} failed: {s} (revision left at {s})", .{ path, @errorName(err), tmp_path }) catch null; + return; + }; + report.committed = true; +} + /// The session tools require the HTTP transport's parked-isolate discipline: /// a second session means a second V8 isolate, only safe when isolates are /// entered around use. Over stdio (one permanently-entered isolate) they are @@ -937,6 +1194,214 @@ test "MCP - save writes the script to disk" { try std.testing.expectEqualStrings("const page = new Page();\nawait page.goto(\"https://example.com\");\n", written); } +fn testToolText(arena: std.mem.Allocator, response: []const u8) ![]const u8 { + const root = try std.json.parseFromSliceLeaky(std.json.Value, arena, std.mem.trim(u8, response, " \n"), .{}); + return root.object.get("result").?.object.get("content").?.array.items[0].object.get("text").?.string; +} + +fn testCall(server: *Server, out: *std.Io.Writer.Allocating, name: []const u8, arguments: anytype) ![]const u8 { + const arena = testing.arena_allocator; + const args_json = try std.json.Stringify.valueAlloc(arena, arguments, .{}); + const msg = try std.fmt.allocPrint(arena, + \\{{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{{"name":"{s}","arguments":{s}}}}} + , .{ name, args_json }); + const start = out.written().len; + try router.handleMessage(server, arena, msg); + return testToolText(arena, out.written()[start..]); +} + +fn testCallReport(server: *Server, out: *std.Io.Writer.Allocating, name: []const u8, arguments: anytype) !std.json.Value { + const text = try testCall(server, out, name, arguments); + return std.json.parseFromSliceLeaky(std.json.Value, testing.arena_allocator, text, .{}); +} + +const test_fixture_url = "http://localhost:9582/src/browser/tests/mcp_actions.html"; + +test "MCP - replay rejects unsafe path" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const msg = + \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"replay","arguments":{"path":"../evil.js"}}} + ; + try router.handleMessage(server, testing.arena_allocator, msg); + try testing.expect(std.mem.indexOf(u8, out.written(), "must be relative") != null); + try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":true") != null); +} + +test "MCP - replay: inline clean run reports ok" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "return [1];" }); + try testing.expectString("ok", report.object.get("status").?.string); + try testing.expectString("data", report.object.get("returned").?.string); + try testing.expectEqual(null, report.object.get("failure")); + try testing.expectEqual(null, report.object.get("guidance")); +} + +test "MCP - replay: throwing script reports failed with failure and guidance" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "throw new Error(\"boom\");" }); + try testing.expectString("failed", report.object.get("status").?.string); + const failure = report.object.get("failure").?.object; + try testing.expectString("threw", failure.get("kind").?.string); + try testing.expect(std.mem.indexOf(u8, failure.get("detail").?.string, "boom") != null); + try testing.expectString("throw new Error(\"boom\");", report.object.get("source").?.string); + try testing.expect(std.mem.indexOf(u8, report.object.get("guidance").?.string, "heal_commit") != null); +} + +test "MCP - replay: empty return reports suspicious" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "return [];" }); + try testing.expectString("suspicious", report.object.get("status").?.string); + try testing.expectString("empty", report.object.get("returned").?.string); + try testing.expectString("empty", report.object.get("failure").?.object.get("kind").?.string); + try testing.expect(std.mem.indexOf(u8, report.object.get("guidance").?.string, "lp:baseline") != null); +} + +test "MCP - replay: dry extract reports suspicious with dry_fields" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage(test_fixture_url, &out.writer); + defer server.deinit(); + + const script = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: ["#btn"], missing: [".no-such-thing"] }); + ; + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = script }); + try testing.expectString("suspicious", report.object.get("status").?.string); + const failure = report.object.get("failure").?.object; + try testing.expectString("dry_extracts", failure.get("kind").?.string); + const dry = failure.get("dry_fields").?.array.items; + try testing.expectEqual(1, dry.len); + try testing.expectString("missing", dry[0].string); + try testing.expectEqual(2, report.object.get("extracts").?.array.items.len); +} + +test "MCP - replay: console lines are captured in the report" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const report = try testCallReport(server, &out, "replay", .{ .path = "t.js", .script = "console.log(\"hello\", 42);\nreturn [1];" }); + const console = report.object.get("console").?.array.items; + try testing.expectEqual(2, console.len); + try testing.expectString("log", console[0].object.get("level").?.string); + try testing.expectString("hello 42", console[0].object.get("text").?.string); + // The returned value is echoed as the final console line — how a replay + // hands its output to the client. + try testing.expectString("[1]", console[1].object.get("text").?.string); + try testing.expectEqual(false, report.object.get("console_truncated").?.bool); +} + +test "MCP - heal_commit: uncured candidate leaves the file untouched" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const path = "mcp-heal-uncured-test.js"; + std.Io.Dir.cwd().deleteFile(testing.io, path) catch {}; + + const report = try testCallReport(server, &out, "heal_commit", .{ + .path = path, + .script = "return [];", + .failure = .{ .kind = "empty" }, + }); + try testing.expectEqual(false, report.object.get("cured").?.bool); + try testing.expectEqual(false, report.object.get("committed").?.bool); + try testing.expect(std.mem.indexOf(u8, report.object.get("failure").?.string, "no data") != null); + try testing.expectError(error.FileNotFound, std.Io.Dir.cwd().access(testing.io, path, .{})); +} + +test "MCP - heal_commit: cure commits atomically and refreshes the baseline" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage("about:blank", &out.writer); + defer server.deinit(); + + const path = "mcp-heal-cure-test.js"; + try std.Io.Dir.cwd().writeFile(testing.io, .{ .sub_path = path, .data = "return [];\n" }); + defer std.Io.Dir.cwd().deleteFile(testing.io, path) catch {}; + + const revised = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: ["#btn"] }); + ; + const report = try testCallReport(server, &out, "heal_commit", .{ + .path = path, + .script = revised, + .failure = .{ .kind = "empty" }, + }); + try testing.expectEqual(true, report.object.get("cured").?.bool); + try testing.expectEqual(true, report.object.get("committed").?.bool); + try testing.expectString("ok", report.object.get("run").?.object.get("status").?.string); + + const written = try std.Io.Dir.cwd().readFileAlloc(testing.io, path, testing.arena_allocator, .limited(4096)); + try testing.expect(std.mem.startsWith(u8, written, "const page = new Page();")); + try testing.expect(std.mem.indexOf(u8, written, "// lp:baseline ") != null); + try testing.expect(std.mem.indexOf(u8, written, "\"btn\":{\"calls\":1,\"nonempty\":1}") != null); + try testing.expectError(error.FileNotFound, std.Io.Dir.cwd().access(testing.io, path ++ ".heal.js", .{})); +} + +test "MCP - script lifecycle: save, replay broken, heal_commit, replay clean" { + defer testing.reset(); + var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); + const server = try testLoadPage(test_fixture_url, &out.writer); + defer server.deinit(); + + const path = "mcp-lifecycle-test.js"; + std.Io.Dir.cwd().deleteFile(testing.io, path) catch {}; + defer std.Io.Dir.cwd().deleteFile(testing.io, path) catch {}; + + const broken = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: [".no-such-btn"] }); + ; + const saved = try testCall(server, &out, "save", .{ .path = path, .script = broken }); + try testing.expect(std.mem.indexOf(u8, saved, "saved") != null); + + const run = try testCallReport(server, &out, "replay", .{ .path = path }); + try testing.expectString("suspicious", run.object.get("status").?.string); + const failure = run.object.get("failure").?; + + // The test plays the client model: fix the selector, echo the failure back. + const revised = + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\return page.extract({ btn: ["#btn"] }); + ; + const healed = try testCallReport(server, &out, "heal_commit", .{ + .path = path, + .script = revised, + .failure = failure, + }); + try testing.expectEqual(true, healed.object.get("cured").?.bool); + try testing.expectEqual(true, healed.object.get("committed").?.bool); + + const rerun = try testCallReport(server, &out, "replay", .{ .path = path }); + try testing.expectString("ok", rerun.object.get("status").?.string); + try testing.expectString("data", rerun.object.get("returned").?.string); +} + test "MCP - tree rejects stale backendNodeId instead of dumping whole document" { defer testing.reset(); var out: std.Io.Writer.Allocating = .init(testing.arena_allocator); diff --git a/src/script/Baseline.zig b/src/script/Baseline.zig new file mode 100644 index 000000000..b43e02aec --- /dev/null +++ b/src/script/Baseline.zig @@ -0,0 +1,177 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Session-scoped record of what the agent's extract calls actually returned. +//! `/save` persists it into the script as a `// lp:baseline {...}` comment — +//! evidence for the replay verdict turn, which reads it in the script source +//! to judge whether empty output is breakage or record-time-normal sparseness. +//! Nothing parses it programmatically; the model is the only consumer. + +const std = @import("std"); +const lp = @import("lightpanda"); +const ScriptRuntime = lp.Runtime; + +const Baseline = @This(); + +pub const marker = "// lp:baseline "; + +pub const FieldStat = struct { + calls: u32 = 0, + nonempty: u32 = 0, +}; + +/// Keyed by top-level result field name; "" for a whole-array result. +pub const Fields = std.StringArrayHashMapUnmanaged(FieldStat); + +arena: std.heap.ArenaAllocator, +fields: Fields = .empty, + +pub fn init(allocator: std.mem.Allocator) Baseline { + return .{ .arena = .init(allocator) }; +} + +pub fn deinit(self: *Baseline) void { + self.arena.deinit(); +} + +pub fn reset(self: *Baseline) void { + self.fields = .empty; + _ = self.arena.reset(.retain_capacity); +} + +/// Tally one successful extract result (the tool's JSON output). Malformed +/// output records nothing — this is best-effort telemetry. +pub fn noteExtractResult(self: *Baseline, result_text: []const u8) error{OutOfMemory}!void { + var scratch_state: std.heap.ArenaAllocator = .init(self.arena.child_allocator); + defer scratch_state.deinit(); + const scratch = scratch_state.allocator(); + const parsed = std.json.parseFromSliceLeaky(std.json.Value, scratch, result_text, .{}) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return, + }; + for (try ScriptRuntime.classifyExtractFields(scratch, parsed)) |fc| { + try self.bump(fc.field orelse "", !fc.empty); + } +} + +fn bump(self: *Baseline, field: []const u8, nonempty: bool) error{OutOfMemory}!void { + const arena = self.arena.allocator(); + const gop = try self.fields.getOrPut(arena, field); + if (!gop.found_existing) { + // The probe key may live in caller scratch; own it. + gop.key_ptr.* = try arena.dupe(u8, field); + gop.value_ptr.* = .{}; + } + gop.value_ptr.calls += 1; + if (nonempty) gop.value_ptr.nonempty += 1; +} + +/// The session's baseline as a full `// lp:baseline {...}` line, or null when +/// no extract ran. +pub fn serialize(self: *const Baseline, arena: std.mem.Allocator) error{OutOfMemory}!?[]const u8 { + return fieldsToLine(arena, &self.fields); +} + +/// Baseline line from a script run's extract stats (per-field totals folded +/// across schemas), for refreshing a healed script from its validation run. +pub fn serializeStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) error{OutOfMemory}!?[]const u8 { + var fields: Fields = .empty; + for (stats) |stat| { + const gop = try fields.getOrPut(arena, stat.field orelse ""); + if (!gop.found_existing) gop.value_ptr.* = .{}; + gop.value_ptr.calls += stat.calls; + gop.value_ptr.nonempty += stat.calls - stat.empty; + } + return fieldsToLine(arena, &fields); +} + +fn fieldsToLine(arena: std.mem.Allocator, fields: *const Fields) error{OutOfMemory}!?[]const u8 { + if (fields.count() == 0) return null; + var fields_obj: std.json.ObjectMap = .empty; + var it = fields.iterator(); + while (it.next()) |entry| { + var stat_obj: std.json.ObjectMap = .empty; + try stat_obj.put(arena, "calls", .{ .integer = entry.value_ptr.calls }); + try stat_obj.put(arena, "nonempty", .{ .integer = entry.value_ptr.nonempty }); + try fields_obj.put(arena, entry.key_ptr.*, .{ .object = stat_obj }); + } + var root: std.json.ObjectMap = .empty; + try root.put(arena, "fields", .{ .object = fields_obj }); + const json = try std.json.Stringify.valueAlloc(arena, std.json.Value{ .object = root }, .{}); + return try std.mem.concat(arena, u8, &.{ marker, json }); +} + +/// `script` with any existing baseline lines dropped and `line` (a full +/// baseline line, or null) appended — synthesis may have copied a stale +/// baseline from the previous script verbatim. +pub fn withBaseline(arena: std.mem.Allocator, script: []const u8, line: ?[]const u8) ![]const u8 { + if (line == null and std.mem.indexOf(u8, script, marker) == null) return script; + var aw: std.Io.Writer.Allocating = .init(arena); + var lines = std.mem.splitScalar(u8, script, '\n'); + var pending_newline = false; + while (lines.next()) |script_line| { + if (std.mem.startsWith(u8, std.mem.trim(u8, script_line, " \t\r"), marker)) continue; + if (pending_newline) try aw.writer.writeByte('\n'); + try aw.writer.writeAll(script_line); + pending_newline = true; + } + if (line) |l| { + // A split of "a\nb\n" ends with an empty segment, so the writer is + // already newline-terminated in the common case. + if (aw.written().len != 0 and aw.written()[aw.written().len - 1] != '\n') { + try aw.writer.writeByte('\n'); + } + try aw.writer.writeAll(l); + try aw.writer.writeByte('\n'); + } + return aw.written(); +} + +test "baseline: note and serialize tally per-field data presence" { + var baseline: Baseline = .init(std.testing.allocator); + defer baseline.deinit(); + + try baseline.noteExtractResult("{\"stories\":[{\"title\":\"a\"}],\"empty_list\":[],\"scalar\":\"x\"}"); + try baseline.noteExtractResult("{\"stories\":[],\"empty_list\":[]}"); + // Malformed results record nothing. + try baseline.noteExtractResult("not json"); + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const line = (try baseline.serialize(arena.allocator())).?; + try std.testing.expectEqualStrings( + marker ++ "{\"fields\":{\"stories\":{\"calls\":2,\"nonempty\":1},\"empty_list\":{\"calls\":2,\"nonempty\":0},\"scalar\":{\"calls\":1,\"nonempty\":1}}}", + line, + ); +} + +test "baseline: withBaseline strips stale lines and appends" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const script = "const x = 1;\n// lp:baseline {\"fields\":{}}\nreturn x;\n"; + const out = try withBaseline(aa, script, marker ++ "{\"fields\":{\"a\":{\"calls\":1,\"nonempty\":1}}}"); + try std.testing.expectEqualStrings( + "const x = 1;\nreturn x;\n" ++ marker ++ "{\"fields\":{\"a\":{\"calls\":1,\"nonempty\":1}}}\n", + out, + ); + + const stripped = try withBaseline(aa, script, null); + try std.testing.expectEqualStrings("const x = 1;\nreturn x;\n", stripped); +} diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig index 79948ad83..d161ff2d8 100644 --- a/src/script/Runtime.zig +++ b/src/script/Runtime.zig @@ -42,12 +42,17 @@ console_data: [std.enums.values(ConsoleMethod).len]ConsoleData, /// clear the live spinner so script output starts on a clean line instead /// of colliding with the indicator; the line still goes to stdout/stderr. console_observer: ?ConsoleObserver = null, -/// When set, every console line writes here — stderr-bound methods included. -console_sink: ?*std.Io.Writer = null, +/// When set, receives each `console.*` line — stderr-bound methods included — +/// instead of the process stdout/stderr write, for embedders (MCP) whose +/// stdout is a protocol stream. The observer still fires first. +console_sink: ?ConsoleSink = null, /// In-flight async `goto`s; emptied before each `runSource` returns. pending_gotos: std.ArrayList(PendingGoto), /// Restarted per `runSource`; backs `PendingGoto.deadline_ms`. run_timer: std.Io.Timestamp, +/// Per-run tally of extract field emptiness. call_arena-backed, so it is +/// re-zeroed alongside the arena reset at the top of `runSource`. +extract_stats: std.ArrayList(ExtractStat) = .empty, /// The runtime installs exactly the recorded browser tools as script /// primitives — the same set the recorder writes — so every recorded call @@ -81,7 +86,7 @@ const PendingGoto = struct { } }; -const ConsoleMethod = enum { +pub const ConsoleMethod = enum { debug, @"error", info, @@ -106,6 +111,11 @@ pub const ConsoleObserver = struct { notify: *const fn (context: *anyopaque) void, }; +pub const ConsoleSink = struct { + context: *anyopaque, + write: *const fn (context: *anyopaque, method: ConsoleMethod, line: []const u8) void, +}; + pub const InitError = error{ OutOfMemory, RuntimeInitFailed, @@ -264,11 +274,81 @@ fn setObjectProperty( if (!out.has_value or !out.value) return error.RuntimeInitFailed; } -/// Run script source in the agent context. Returns null on success; on a JS -/// compile/runtime exception returns a formatted error allocated in this -/// runtime's call arena and valid until deinit or the next run. -pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!?[]const u8 { +/// The value a script ran to completion with. +pub const Completion = struct { + /// Display form of the returned value: objects and arrays as JSON. + text: []const u8, + /// True when the value carries no data — null, "", or an array/object all + /// of whose members are empty. + empty: bool, +}; + +/// One top-level field of a parsed extract result: an object yields an entry +/// per key, an array (a `__root` schema) the single null field. Slices +/// reference the parsed value. +pub const ExtractField = struct { + field: ?[]const u8, + empty: bool, +}; + +pub fn classifyExtractFields(arena: std.mem.Allocator, result: std.json.Value) error{OutOfMemory}![]const ExtractField { + switch (result) { + .array => { + const out = try arena.alloc(ExtractField, 1); + out[0] = .{ .field = null, .empty = jsonIsEmpty(result) }; + return out; + }, + .object => |obj| { + const out = try arena.alloc(ExtractField, obj.count()); + var it = obj.iterator(); + var i: usize = 0; + while (it.next()) |entry| : (i += 1) { + out[i] = .{ + .field = entry.key_ptr.*, + .empty = jsonIsEmpty(entry.value_ptr.*), + }; + } + return out; + }, + else => return &.{}, + } +} + +pub fn fieldEql(a: ?[]const u8, b: ?[]const u8) bool { + if (a) |x| return b != null and std.mem.eql(u8, x, b.?); + return b == null; +} + +/// One extract schema's top-level result field, tallied across the run. +pub const ExtractStat = struct { + /// Schema JSON as the script wrote it (array schemas are shown without the + /// internal `__root` wrapper). + schema: []const u8, + /// Top-level field of the result; null when the schema itself is a list. + field: ?[]const u8, + calls: u32, + empty: u32, +}; + +/// A script run's outcome. `err` is a formatted JS compile/runtime exception. +/// All slices are allocated in this runtime's call arena and valid until +/// deinit or the next run. +pub const RunResult = union(enum) { + ok: Ok, + err: []const u8, +}; + +pub const Ok = struct { + /// The value the script returned; null when it returned `undefined`. + completion: ?Completion, + /// Per-(schema, field) extract tallies for the run. + extract_stats: []const ExtractStat, +}; + +/// Run script source in the agent context. +pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!RunResult { _ = self.call_arena.reset(.retain_capacity); + self.extract_stats = .empty; self.run_timer = .now(lp.io, .boot); var hs: lp.js.HandleScope = undefined; @@ -276,7 +356,7 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError! defer hs.deinit(); const context: *const v8.Context = @ptrCast(v8.v8__Global__Get(&self.context, self.env.isolate.handle) orelse - return try self.dupeError("agent script context is not available")); + return .{ .err = try self.dupeError("agent script context is not available") }); v8.v8__Context__Enter(context); defer v8.v8__Context__Exit(context); @@ -291,7 +371,7 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError! // trailing expression no longer auto-prints — `await` and a script // completion value are mutually exclusive in JS.) const wrapped = std.fmt.allocPrint(self.call_arena.allocator(), "(async () => {{\n{s}\n}})()", .{source}) catch - return try self.dupeError("out of memory"); + return .{ .err = try self.dupeError("out of memory") }; const script_source = self.env.isolate.initStringHandle(wrapped); var origin: v8.ScriptOrigin = undefined; @@ -306,27 +386,33 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError! &compiler_source, v8.kNoCompileOptions, v8.kNoCacheNoReason, - ) orelse return try self.formatCaught(context, &try_catch, "compile failed"); + ) orelse return .{ .err = try self.formatCaught(context, &try_catch, "compile failed") }; - const completion = v8.v8__Script__Run(script, context) orelse - return try self.formatCaught(context, &try_catch, "script failed"); + const completion_promise = v8.v8__Script__Run(script, context) orelse + return .{ .err = try self.formatCaught(context, &try_catch, "script failed") }; // `goto` only *starts* a navigation, so the root Promise is usually still // pending; drive the in-flight ones to completion. - const root: *const v8.Promise = @ptrCast(completion); + const root: *const v8.Promise = @ptrCast(completion_promise); self.driveAsync(context, &try_catch, root); if (v8.v8__TryCatch__HasCaught(&try_catch)) { - return try self.formatCaught(context, &try_catch, "script failed"); + return .{ .err = try self.formatCaught(context, &try_catch, "script failed") }; } // A still-pending root means the script awaited something we can't settle // (no async navigation is in flight) — stay silent. const state = promiseState(root); - if (state != v8.kFulfilled and state != v8.kRejected) return null; - const completion_value = v8.v8__Promise__Result(root) orelse return null; - if (state == v8.kRejected) return try self.formatRejection(context, completion_value); - self.printCompletion(context, completion_value); - return null; + if (state != v8.kFulfilled and state != v8.kRejected) return self.runOk(null); + const completion_value = v8.v8__Promise__Result(root) orelse return self.runOk(null); + if (state == v8.kRejected) return .{ .err = try self.formatRejection(context, completion_value) }; + return self.runOk(try self.completion(context, completion_value)); +} + +fn runOk(self: *Runtime, completion_value: ?Completion) RunResult { + return .{ .ok = .{ + .completion = completion_value, + .extract_stats = self.extract_stats.items, + } }; } /// `v8__Promise__State` returns the `c_uint` `PromiseState`, but the `k*` @@ -355,15 +441,55 @@ fn errorMessage(self: *Runtime, context: *const v8.Context, reason: *const v8.Va } /// Echo a script's output — the value it `return`s from the async wrapper, so a -/// script ending in `return page.extract(...)` prints without `console.log`. -/// `undefined` — no `return`, or a bare trailing expression — stays silent. -fn printCompletion(self: *Runtime, context: *const v8.Context, value: *const v8.Value) void { - if (v8.v8__Value__IsUndefined(value)) return; +/// script ending in `return page.extract(...)` prints without `console.log` — +/// and hand it back to the caller. `undefined` — no `return`, or a bare +/// trailing expression — stays silent and yields null, as does a value whose +/// display form can't be computed. +fn completion(self: *Runtime, context: *const v8.Context, value: *const v8.Value) error{OutOfMemory}!?Completion { + if (v8.v8__Value__IsUndefined(value)) return null; - var arena_state: std.heap.ArenaAllocator = .init(self.allocator); - defer arena_state.deinit(); - const text = self.displayString(arena_state.allocator(), context, value) catch return; + const arena = self.call_arena.allocator(); + const text = self.displayString(arena, context, value) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.JsException => return null, + }; self.writeConsoleLine(.log, text); + return .{ .text = text, .empty = self.isEmptyValue(value, text) }; +} + +/// Whether a completion value carries no data. `text` is the value's display +/// form: JSON for objects and arrays (a fallback coercion — function, circular +/// reference — won't parse and counts as data), plain coercion otherwise. +fn isEmptyValue(self: *Runtime, value: *const v8.Value, text: []const u8) bool { + if (v8.v8__Value__IsNull(value)) return true; + if (v8.v8__Value__IsObject(value)) { + // Scratch arena: the parse tree is as large as the extract output and + // must not sit in the call arena until the next run. + var arena_state: std.heap.ArenaAllocator = .init(self.allocator); + defer arena_state.deinit(); + const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena_state.allocator(), text, .{}) catch return false; + return jsonIsEmpty(parsed); + } + return text.len == 0; +} + +/// Null, "", and arrays/objects all of whose members are empty carry no data; +/// numbers and booleans always do (0 and false are answers). +pub fn jsonIsEmpty(value: std.json.Value) bool { + switch (value) { + .null => return true, + .string => |s| return s.len == 0, + .array => |a| { + for (a.items) |item| if (!jsonIsEmpty(item)) return false; + return true; + }, + .object => |o| { + var it = o.iterator(); + while (it.next()) |entry| if (!jsonIsEmpty(entry.value_ptr.*)) return false; + return true; + }, + else => return false, + } } /// Unwrap a callback's info handle and its `External` payload (the `*T` passed @@ -465,10 +591,18 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf switch (result) { .ok => |text| switch (tool) { .extract => { - const normalized = self.normalizeExtractReturnJson(arena, text) catch |err| switch (err) { + const normalized = normalizeExtractReturnJson(arena, text) catch |err| switch (err) { error.OutOfMemory => return self.throwError("out of memory"), }; - self.setReturnJson(context, info, normalized); + // Recording happens after `callTool` returns, so a re-entrant + // extract (via a `toJSON` during argument marshalling) tallies + // inner-then-outer; the map is append-only. Failed extracts + // threw above and are intentionally not counted. + if (normalized.parsed) |parsed| { + self.recordExtractStats(arena, args, parsed) catch + return self.throwError("out of memory"); + } + self.setReturnJson(context, info, normalized.text); }, else => self.setReturnString(info, text), }, @@ -476,6 +610,37 @@ fn invoke(self: *Runtime, tool: BrowserTool, info: *const v8.FunctionCallbackInf } } +/// Tally each top-level field of an extract result; callers judge what the +/// tallies mean. +fn recordExtractStats(self: *Runtime, arena: std.mem.Allocator, args: ?std.json.Value, result: std.json.Value) error{OutOfMemory}!void { + const raw_schema = switch ((args orelse return).object.get("schema") orelse return) { + .string => |s| s, + else => return, + }; + const schema = stripExtractSchemaRoot(raw_schema); + for (try classifyExtractFields(arena, result)) |fc| { + try self.bumpExtractStat(schema, fc.field, fc.empty); + } +} + +fn bumpExtractStat(self: *Runtime, schema: []const u8, field: ?[]const u8, is_empty: bool) error{OutOfMemory}!void { + for (self.extract_stats.items) |*stat| { + if (!std.mem.eql(u8, stat.schema, schema)) continue; + if (!fieldEql(stat.field, field)) continue; + stat.calls += 1; + if (is_empty) stat.empty += 1; + return; + } + // `schema`/`field` live in invoke's per-call arena; the stat outlives it. + const arena = self.call_arena.allocator(); + try self.extract_stats.append(arena, .{ + .schema = try arena.dupe(u8, schema), + .field = if (field) |f| try arena.dupe(u8, f) else null, + .calls = 1, + .empty = @intFromBool(is_empty), + }); +} + /// Start the receiver Page's navigation and return a *pending* Promise of the /// page object; `driveAsync` settles it once the frame loads. This is what lets /// `Promise.all([a.goto(x), b.goto(y)])` fetch both in parallel. @@ -687,10 +852,7 @@ fn invokeConsole(self: *Runtime, method: ConsoleMethod, info: *const v8.Function fn writeConsoleLine(self: *Runtime, method: ConsoleMethod, line: []const u8) void { if (self.console_observer) |obs| obs.notify(obs.context); - if (self.console_sink) |sink| { - sink.print("{s}\n", .{line}) catch {}; - return; - } + if (self.console_sink) |sink| return sink.write(sink.context, method, line); var buf: [4096]u8 = undefined; const file = if (method.writesStderr()) std.Io.File.stderr() else std.Io.File.stdout(); var writer = file.writerStreaming(lp.io, &buf); @@ -817,10 +979,24 @@ fn extractSchemaString(arena: std.mem.Allocator, value: std.json.Value) error{Ou }; } +/// Key under which `normalizeExtractSchemaString` nests an array schema so the +/// walker always receives an object; wrap, unwrap, and display-strip must agree. +const extract_root_key = "__root"; + fn normalizeExtractSchemaString(arena: std.mem.Allocator, schema: []const u8) error{OutOfMemory}![]const u8 { const trimmed = std.mem.trim(u8, schema, &std.ascii.whitespace); if (trimmed.len == 0 or trimmed[0] != '[') return schema; - return try std.fmt.allocPrint(arena, "{{\"__root\":{s}}}", .{schema}); + return try std.fmt.allocPrint(arena, "{{\"" ++ extract_root_key ++ "\":{s}}}", .{schema}); +} + +/// Inverse of the wrapping above, for showing an array schema the way the +/// script wrote it. +fn stripExtractSchemaRoot(schema: []const u8) []const u8 { + const prefix = "{\"" ++ extract_root_key ++ "\":"; + if (std.mem.startsWith(u8, schema, prefix) and std.mem.endsWith(u8, schema, "}")) { + return schema[prefix.len .. schema.len - 1]; + } + return schema; } fn argJson( @@ -854,19 +1030,27 @@ fn objectWith(arena: std.mem.Allocator, key: []const u8, value: std.json.Value) /// Unwraps only the `__root` sentinel that `normalizeExtractSchemaString` injects /// for array schemas; a real single-field object schema keeps its shape. -fn normalizeExtractReturnJson(_: *Runtime, arena: std.mem.Allocator, value: []const u8) error{OutOfMemory}![]const u8 { - if (value.len == 0) return value; +/// `parsed` is the post-unwrap value, null when the raw text is empty or not +/// JSON — the stats hook classifies it without a second parse. +fn normalizeExtractReturnJson(arena: std.mem.Allocator, value: []const u8) error{OutOfMemory}!struct { + text: []const u8, + parsed: ?std.json.Value, +} { + if (value.len == 0) return .{ .text = value, .parsed = null }; const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, value, .{}) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, - else => return value, + else => return .{ .text = value, .parsed = null }, }; - if (parsed != .object or parsed.object.count() != 1) return value; - - var it = parsed.object.iterator(); - const entry = it.next() orelse return value; - if (!std.mem.eql(u8, entry.key_ptr.*, "__root")) return value; - return try std.json.Stringify.valueAlloc(arena, entry.value_ptr.*, .{}); + if (parsed == .object and parsed.object.count() == 1) { + var it = parsed.object.iterator(); + const entry = it.next().?; + if (std.mem.eql(u8, entry.key_ptr.*, extract_root_key)) return .{ + .text = try std.json.Stringify.valueAlloc(arena, entry.value_ptr.*, .{}), + .parsed = entry.value_ptr.*, + }; + } + return .{ .text = value, .parsed = parsed }; } fn setReturnString(self: *Runtime, info: *const v8.FunctionCallbackInfo, value: []const u8) void { @@ -985,12 +1169,20 @@ fn dupeError(self: *Runtime, message: []const u8) RunError![]const u8 { const testing = @import("../testing.zig"); fn runTestScript(runtime: *Runtime, source: []const u8) !void { - if (try runtime.runSource(source, "agent-runtime-test.js")) |message| { - std.debug.print("agent script failed:\n{s}\n", .{message}); - return error.AgentScriptFailed; + switch (try runtime.runSource(source, "agent-runtime-test.js")) { + .ok => {}, + .err => |message| { + std.debug.print("agent script failed:\n{s}\n", .{message}); + return error.AgentScriptFailed; + }, } } +fn appendConsoleLine(context: *anyopaque, _: ConsoleMethod, line: []const u8) void { + const aw: *std.Io.Writer.Allocating = @ptrCast(@alignCast(context)); + aw.writer.print("{s}\n", .{line}) catch {}; +} + fn terminateRuntimeSoon(runtime: *Runtime) void { lp.io.sleep(.fromMilliseconds(10), .awake) catch {}; runtime.terminate(); @@ -1027,7 +1219,7 @@ test "agent script runtime: Page must be called with new" { const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); defer runtime.deinit(); - const message = (try runtime.runSource("Page();", "agent-runtime-page-no-new.js")).?; + const message = (try runtime.runSource("Page();", "agent-runtime-page-no-new.js")).err; try testing.expect(std.mem.indexOf(u8, message, "must be called with new") != null); } @@ -1043,7 +1235,7 @@ test "agent script runtime: a method on an un-navigated page errors" { const message = (try runtime.runSource( \\const page = new Page(); \\page.extract({ btn: "#btn" }); - , "agent-runtime-not-navigated.js")).?; + , "agent-runtime-not-navigated.js")).err; try testing.expect(std.mem.indexOf(u8, message, "not navigated") != null); } @@ -1064,7 +1256,7 @@ test "agent script runtime: page.close stales the handle" { \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); \\page.close(); \\page.extract({ btn: "#btn" }); - , "agent-runtime-close.js")).?; + , "agent-runtime-close.js")).err; try testing.expect(std.mem.indexOf(u8, message, "closed") != null); } @@ -1337,7 +1529,7 @@ test "agent script runtime: primitives re-entered from argument callbacks stay i var sink: std.Io.Writer.Allocating = .init(testing.allocator); defer sink.deinit(); - runtime.console_sink = &sink.writer; + runtime.console_sink = .{ .context = @ptrCast(&sink), .write = appendConsoleLine }; try runTestScript(runtime, \\const page = new Page(); @@ -1367,8 +1559,8 @@ test "agent script runtime: terminate interrupts local JavaScript" { defer runtime.cancelTerminate(); defer thread.join(); - const message = try runtime.runSource("while (true) {}", "agent-runtime-terminate-test.js"); - try testing.expect(message != null); + const result = try runtime.runSource("while (true) {}", "agent-runtime-terminate-test.js"); + try testing.expect(result == .err); } test "agent script runtime: agent variables persist and page globals are isolated" { @@ -1427,7 +1619,7 @@ test "agent script runtime: console is available in agent context" { var sink: std.Io.Writer.Allocating = .init(testing.allocator); defer sink.deinit(); - runtime.console_sink = &sink.writer; + runtime.console_sink = .{ .context = @ptrCast(&sink), .write = appendConsoleLine }; try runTestScript(runtime, \\if (typeof console !== "object") throw new Error("missing console"); @@ -1437,6 +1629,36 @@ test "agent script runtime: console is available in agent context" { try testing.expectString("agent console ready\n", sink.written()); } +test "agent script runtime: console sink captures lines instead of the process streams" { + defer testing.reset(); + + var registry = CDPNode.Registry.init(testing.allocator); + defer registry.deinit(); + + const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); + defer runtime.deinit(); + + const Capture = struct { + buf: [256]u8 = undefined, + len: usize = 0, + + fn write(context: *anyopaque, method: ConsoleMethod, line: []const u8) void { + const capture: *@This() = @ptrCast(@alignCast(context)); + const out = std.fmt.bufPrint(capture.buf[capture.len..], "{s}:{s}\n", .{ @tagName(method), line }) catch return; + capture.len += out.len; + } + }; + var capture: Capture = .{}; + runtime.console_sink = .{ .context = @ptrCast(&capture), .write = Capture.write }; + + try runTestScript(runtime, + \\console.log("hello", 42); + \\console.warn("careful"); + ); + + try testing.expectString("log:hello 42\nwarn:careful\n", capture.buf[0..capture.len]); +} + test "agent script runtime: tool errors throw and stop execution" { defer testing.reset(); defer testing.test_session.closeAllPages(); @@ -1453,7 +1675,7 @@ test "agent script runtime: tool errors throw and stop execution" { \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); \\page.click({ selector: "#does-not-exist" }); \\globalThis.marker = "after"; - , "agent-runtime-failure.js")).?; + , "agent-runtime-failure.js")).err; try testing.expect(std.mem.indexOf(u8, message, "click") != null or std.mem.indexOf(u8, message, "NodeNotFound") != null or @@ -1508,7 +1730,7 @@ test "agent script runtime: builtin argument marshalling (positional + options)" { const message = (try runtime.runSource( \\await new Page().goto("http://localhost:9582/src/browser/tests/mcp_actions.html", { url: "http://other" }); - , "agent-runtime-conflict.js")).?; + , "agent-runtime-conflict.js")).err; try testing.expect(std.mem.indexOf(u8, message, "invalid arguments") != null); } @@ -1518,7 +1740,7 @@ test "agent script runtime: builtin argument marshalling (positional + options)" \\const page = new Page(); \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); \\page.click("#btn", "#extra"); - , "agent-runtime-arity.js")).?; + , "agent-runtime-arity.js")).err; try testing.expect(std.mem.indexOf(u8, message, "invalid arguments") != null); } } @@ -1534,7 +1756,7 @@ test "agent script runtime: top-level await runs in an async wrapper" { var sink: std.Io.Writer.Allocating = .init(testing.allocator); defer sink.deinit(); - runtime.console_sink = &sink.writer; + runtime.console_sink = .{ .context = @ptrCast(&sink), .write = appendConsoleLine }; // `await` on a non-goto promise resolves without touching the browser, and // a top-level `return` surfaces as the (otherwise un-echoed) result. @@ -1545,3 +1767,86 @@ test "agent script runtime: top-level await runs in an async wrapper" { ); try testing.expectString("42\n", sink.written()); } + +test "agent script runtime: completion value emptiness" { + defer testing.reset(); + + var registry = CDPNode.Registry.init(testing.allocator); + defer registry.deinit(); + + const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); + defer runtime.deinit(); + + const cases = [_]struct { source: []const u8, empty: bool }{ + // The shapes a stale extract selector produces: an empty list, or rows + // whose every field missed. + .{ .source = "return [];", .empty = true }, + .{ .source = "return {};", .empty = true }, + .{ .source = "return null;", .empty = true }, + .{ .source = "return { stories: [] };", .empty = true }, + .{ .source = "return [{ id: null, title: \"\" }];", .empty = true }, + .{ .source = "return [{ id: null, title: \"HN\" }];", .empty = false }, + .{ .source = "return \"text\";", .empty = false }, + .{ .source = "return 0;", .empty = false }, + .{ .source = "return false;", .empty = false }, + }; + for (cases) |case| { + const result = try runtime.runSource(case.source, "agent-runtime-emptiness.js"); + try testing.expectEqual(case.empty, result.ok.completion.?.empty); + } + + // No `return` — nothing to classify, nothing to heal. + const silent = try runtime.runSource("const x = 1;", "agent-runtime-emptiness.js"); + try testing.expectEqual(null, silent.ok.completion); +} + +test "agent script runtime: extract stats tally list-field emptiness" { + defer testing.reset(); + defer testing.test_session.closeAllPages(); + + var registry = CDPNode.Registry.init(testing.allocator); + defer registry.deinit(); + + const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, ®istry); + defer runtime.deinit(); + + const result = try runtime.runSource( + \\const page = new Page(); + \\await page.goto("http://localhost:9582/src/browser/tests/mcp_actions.html"); + \\page.extract({ items: [".no-such"] }); + \\page.extract({ items: [".no-such"] }); + \\page.extract({ btn: "#btn", buttons: ["button"] }); + \\page.extract([".no-such-root"]); + \\const other = new Page(); + \\await other.goto("http://localhost:9582/src/browser/tests/runner/runner1.html"); + \\page.extract({ sel: ["#sel0"] }); + \\other.extract({ sel: ["#sel0"] }); + , "agent-runtime-extract-stats.js"); + + const stats = result.ok.extract_stats; + try testing.expectEqual(5, stats.len); + + try testing.expectString("items", stats[0].field.?); + try testing.expectEqual(2, stats[0].calls); + try testing.expectEqual(2, stats[0].empty); + + try testing.expectString("btn", stats[1].field.?); + try testing.expectEqual(1, stats[1].calls); + try testing.expectEqual(0, stats[1].empty); + + try testing.expectString("buttons", stats[2].field.?); + try testing.expectEqual(1, stats[2].calls); + try testing.expectEqual(0, stats[2].empty); + + try testing.expectEqual(null, stats[3].field); + try testing.expectString("[\".no-such-root\"]", stats[3].schema); + try testing.expectEqual(1, stats[3].calls); + try testing.expectEqual(1, stats[3].empty); + + try testing.expectString("sel", stats[4].field.?); + try testing.expectEqual(2, stats[4].calls); + try testing.expectEqual(1, stats[4].empty); + + const rerun = try runtime.runSource("return 1;", "agent-runtime-extract-stats.js"); + try testing.expectEqual(0, rerun.ok.extract_stats.len); +} diff --git a/src/script/heal.zig b/src/script/heal.zig new file mode 100644 index 000000000..fd645e3b1 --- /dev/null +++ b/src/script/heal.zig @@ -0,0 +1,451 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Deterministic core of script self-heal, shared by the agent CLI and the +//! MCP server: breakage suspicion over a replay's facts, the cure check that +//! gates a healed script's commit, and the serializable run/heal reports. +//! Model judgment (verdict, diagnosis, revision) stays with the caller — the +//! agent's own LLM on the CLI path, the MCP client's model over MCP. + +const std = @import("std"); +const lp = @import("lightpanda"); +const ScriptRuntime = lp.Runtime; +const Baseline = lp.Baseline; +const string = @import("../string.zig"); + +/// What a completed run returned, as far as heal cares. +pub const Returned = union(enum) { + /// No `return`, or a value whose display form couldn't be computed. + none, + /// A value carrying data. + data, + /// A deep-empty value, carrying its capped display text. + empty: []const u8, +}; + +/// Facts about a run that completed without throwing — suspicion is judged by +/// the model, never here. Duped into the caller's arena — the runtime dies +/// with the run. +pub const RunFacts = struct { + returned: Returned, + extract_stats: []const ScriptRuntime.ExtractStat, + source: []const u8, +}; + +/// Both slices are duped into the caller's arena. `source` is the exact text +/// that ran, so a heal diagnoses what actually failed instead of re-reading a +/// possibly-changed file. +pub const ScriptError = struct { + kind: Kind, + /// Formatted error (line, stack) — or, for `empty`, what came back. + detail: []const u8, + source: []const u8, + /// For `dry_extracts`: the field names that were empty on every call + /// (null = a whole-array schema). The cure check requires each one to + /// come back with data before a heal may replace the file. + dry_fields: []const ?[]const u8 = &.{}, + + /// `empty` is a run that completed but returned a value with no data in + /// it; `dry_extracts` one whose return value had data, but where some + /// extract list field came back empty on every call. Both are the usual + /// symptom of a stale selector, which matches nothing instead of throwing. + /// Only heal treats them as failures; a plain replay still exits 0, since + /// an empty answer can be the right answer. + pub const Kind = enum { threw, empty, dry_extracts }; +}; + +pub const Classified = union(enum) { + facts: RunFacts, + script_error: ScriptError, +}; + +/// Map a run's raw result to facts or a `threw` finding. The error text and +/// stats are duped into `arena` — they live in the runtime's per-call arena — +/// but `source` is stored as given: the caller owns it and it must outlive +/// the outcome. Presentation (terminal output, cancellation policy) stays +/// with the caller. +pub fn classifyRun(arena: std.mem.Allocator, result: ScriptRuntime.RunResult, source: []const u8) error{OutOfMemory}!Classified { + switch (result) { + .err => |message| return .{ .script_error = .{ + .kind = .threw, + .detail = try arena.dupe(u8, message), + .source = source, + } }, + .ok => |ok| { + const returned: Returned = if (ok.completion) |c| + (if (c.empty) .{ .empty = try capDetail(arena, c.text) } else .data) + else + .none; + return .{ .facts = .{ + .returned = returned, + .extract_stats = try dupeExtractStats(arena, ok.extract_stats), + .source = source, + } }; + }, + } +} + +fn dupeExtractStats(arena: std.mem.Allocator, stats: []const ScriptRuntime.ExtractStat) error{OutOfMemory}![]const ScriptRuntime.ExtractStat { + const out = try arena.alloc(ScriptRuntime.ExtractStat, stats.len); + for (stats, out) |stat, *o| { + o.* = .{ + .schema = try arena.dupe(u8, stat.schema), + .field = if (stat.field) |f| try arena.dupe(u8, f) else null, + .calls = stat.calls, + .empty = stat.empty, + }; + } + return out; +} + +/// Bound a value or schema echoed into a heal message; a degenerate empty-ish +/// result (hundreds of all-null rows) would otherwise bloat the LLM turn. +const detail_max_bytes: usize = 2048; + +/// `string.capBytes` at `detail_max_bytes`, always duped — `RunFacts` details +/// must outlive the runtime whose arena the text came from. +fn capDetail(arena: std.mem.Allocator, text: []const u8) error{OutOfMemory}![]const u8 { + return arena.dupe(u8, string.capBytes(arena, text, detail_max_bytes)); +} + +/// A finding worth a verdict, not yet confirmed: the return value was +/// deep-empty, or some extract field came back empty on every call — any field, +/// scalar or list, baseline or not. Whether that is breakage or legitimate +/// sparseness is the model's judgment, not encoded here. +pub fn suspicionOf(arena: std.mem.Allocator, facts: RunFacts) ?ScriptError { + switch (facts.returned) { + .empty => |text| return .{ + .kind = .empty, + .detail = std.fmt.allocPrint(arena, "its return value carries no data: {s}", .{text}) catch return null, + .source = facts.source, + }, + .none, .data => {}, + } + return dryExtractsFinding(arena, facts.source, facts.extract_stats) catch return null; +} + +/// A `dry_extracts` finding with one detail line per extract field that came +/// back empty on every call, plus the field names for the cure check. Null when +/// no field was dry. +fn dryExtractsFinding(arena: std.mem.Allocator, source: []const u8, stats: []const ScriptRuntime.ExtractStat) !?ScriptError { + var aw: std.Io.Writer.Allocating = .init(arena); + var fields: std.ArrayList(?[]const u8) = .empty; + for (stats) |stat| { + if (stat.empty != stat.calls) continue; + if (fields.items.len == 0) { + try aw.writer.writeAll("some extracts came back empty on every call:\n"); + } + // `stat.field` already lives in `arena` (facts were duped into it). + try fields.append(arena, stat.field); + const schema = try capDetail(arena, stat.schema); + if (stat.field) |field| { + try aw.writer.print("- the \"{s}\" field in extract({s}) came back empty", .{ field, schema }); + } else { + try aw.writer.print("- extract({s}) returned no data", .{schema}); + } + if (stat.calls != 1) try aw.writer.print(" in all {d} calls", .{stat.calls}); + try aw.writer.writeAll("\n"); + } + if (fields.items.len == 0) return null; + return .{ .kind = .dry_extracts, .detail = aw.written(), .source = source, .dry_fields = fields.items }; +} + +/// Null when the validation run cured the original finding; otherwise the +/// message fed to the next heal attempt. Running clean is not a cure on its +/// own — a revision that deletes the failing extract (or the `return`) also +/// runs clean. +pub fn cureFailure(arena: std.mem.Allocator, first: ScriptError, facts: RunFacts) error{OutOfMemory}!?[]const u8 { + switch (first.kind) { + .threw => return null, + .empty => return if (facts.returned == .data) + null + else + "The revised script ran, but still returns no data (or no longer returns anything) — the original returned a value.", + .dry_extracts => { + for (first.dry_fields) |dry| { + const cured = for (facts.extract_stats) |stat| { + if (ScriptRuntime.fieldEql(stat.field, dry) and stat.empty < stat.calls) break true; + } else false; + if (!cured) return try std.fmt.allocPrint(arena, "The revised script ran, but the \"{s}\" extract still came back empty on every call (or was removed) — keep it and fix its selector.", .{dry orelse ""}); + } + return null; + }, + } +} + +pub fn refreshedBaselineScript(arena: std.mem.Allocator, revised: []const u8, stats: []const ScriptRuntime.ExtractStat) ?[]const u8 { + const line = Baseline.serializeStats(arena, stats) catch return null; + return Baseline.withBaseline(arena, revised, line) catch null; +} + +pub fn buildDiagnoseMessage(arena: std.mem.Allocator, path: []const u8, source: []const u8, error_detail: []const u8) ![]const u8 { + return std.fmt.allocPrint(arena, + \\Replaying the saved script {s} failed. The browser session is still + \\at the failure state. + \\ + \\The script (its comments and structure carry the intent): + \\```js + \\{s} + \\``` + \\ + \\The error: + \\{s} + \\ + \\Diagnose the failure: inspect the live page (tree, findElement, + \\markdown) to see how the site differs from what the script expects, + \\then perform the corrected step(s) with tools to prove they work — + \\verify selectors against the live page, never guess. If the failing + \\step gated the rest of the script (a login, a navigation), carry on + \\far enough to show the script's goal is reachable again. + , .{ path, source, error_detail }); +} + +/// Heal synthesis instruction; rides on the regular save revision system prompt. +pub const heal_revision_prompt = + \\Fix the script so it replays successfully against the current site: the + \\error names what broke, and the diagnosis tool calls above that + \\succeeded against the live page show the repair. Keep + \\every step, selector, and output shape that still works unchanged. + \\Preserve the script's `//` intent comments; where you change a block, + \\update its comment so it still describes what the revised code does, and + \\add one for any block that lacks it. +; + +// Fixed next-step guidance embedded in MCP replay reports. Templates only — +// page-derived content rides in the report's data fields, never here. + +pub const replay_failed_guidance = + \\The replay failed and the session is still at the failure state. + \\Diagnose the failure: inspect the live page (tree, findElement, + \\markdown) to see how the site differs from what the script expects — + \\`source` carries the intent in its comments and structure — then + \\perform the corrected step(s) with tools to prove they work: verify + \\selectors against the live page, never guess. If the failing step gated + \\the rest of the script (a login, a navigation), carry on far enough to + \\show the script's goal is reachable again. Then call heal_commit with + \\the revised script and this report's `failure` object echoed back + \\verbatim. +; + +pub const replay_suspicious_guidance = + \\The replay completed without errors, but its output looks dry — decide + \\whether the script is broken (stale selectors after a site change) or + \\the result is legitimate (the page genuinely has no such data right + \\now). A ` +++ std.mem.trimEnd(u8, Baseline.marker, " ") ++ + \\` comment in `source`, when present, records how often + \\each output field carried data when the script was saved — weigh it as + \\evidence. If legitimate, stop: the empty answer is the answer. If + \\broken, diagnose against the live session (tree, findElement, + \\markdown), prove the corrected step(s) with tools, then call + \\heal_commit with the revised script and this report's `failure` object + \\echoed back verbatim. +; + +/// `failure` as it rides in reports and back through `heal_commit`: same +/// shape as the model-facing verdict wire, with "" standing in for a +/// whole-array extract (JSON has no place for null-in-a-string-array). +pub const WireFailure = struct { + kind: ScriptError.Kind, + detail: []const u8 = "", + dry_fields: []const []const u8 = &.{}, +}; + +pub const ConsoleLine = struct { + level: []const u8, + text: []const u8, +}; + +/// One replay, serializable: what ran, what came back, and — when the run +/// failed or looks dry — the finding and how to proceed. +pub const RunReport = struct { + status: Status, + path: []const u8, + returned: std.meta.Tag(Returned) = .none, + extracts: []const ScriptRuntime.ExtractStat = &.{}, + failure: ?WireFailure = null, + console: []const ConsoleLine = &.{}, + console_truncated: bool = false, + /// Scrubbed source that actually ran; set on suspicious/failed so the + /// client can diagnose without re-reading a possibly-changed file. + source: ?[]const u8 = null, + guidance: ?[]const u8 = null, + + pub const Status = enum { ok, suspicious, failed }; +}; + +/// One `heal_commit` validation, serializable. `failure` is the residual cure +/// failure (or the validation run's own error); null when cured. The script's +/// path rides in `run.path`. +pub const HealReport = struct { + cured: bool, + committed: bool, + failure: ?[]const u8 = null, + run: RunReport, +}; + +pub fn wireFailure(arena: std.mem.Allocator, e: ScriptError) error{OutOfMemory}!WireFailure { + const fields = try arena.alloc([]const u8, e.dry_fields.len); + for (e.dry_fields, fields) |f, *out| out.* = f orelse ""; + return .{ .kind = e.kind, .detail = e.detail, .dry_fields = fields }; +} + +pub fn scriptErrorFromWire(arena: std.mem.Allocator, w: WireFailure) error{OutOfMemory}!ScriptError { + const fields = try arena.alloc(?[]const u8, w.dry_fields.len); + for (w.dry_fields, fields) |f, *out| out.* = if (f.len == 0) null else f; + return .{ .kind = w.kind, .detail = w.detail, .source = "", .dry_fields = fields }; +} + +const testing = @import("../testing.zig"); + +fn testFacts(returned: Returned, stats: []const ScriptRuntime.ExtractStat) RunFacts { + return .{ .returned = returned, .extract_stats = stats, .source = "" }; +} + +test "cureFailure: running clean is not a cure" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const dry: ScriptError = .{ + .kind = .dry_extracts, + .detail = "", + .source = "", + .dry_fields = &.{ @as(?[]const u8, "comments"), null }, + }; + const cured_stats: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, + .{ .schema = "[]", .field = null, .calls = 1, .empty = 0 }, + }; + try std.testing.expectEqual(null, try cureFailure(aa, dry, testFacts(.data, cured_stats))); + + // Fix-by-deletion: the dry field is simply gone from the revised run. + const deleted = (try cureFailure(aa, dry, testFacts(.data, cured_stats[1..]))).?; + try std.testing.expect(std.mem.indexOf(u8, deleted, "\"comments\"") != null); + + // Still dry counts as uncured. + const still_dry_stats: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 5 }, + cured_stats[1], + }; + try std.testing.expect((try cureFailure(aa, dry, testFacts(.data, still_dry_stats))) != null); + + // .empty is cured only by a data-carrying return. + const empty: ScriptError = .{ .kind = .empty, .detail = "", .source = "" }; + try std.testing.expectEqual(null, try cureFailure(aa, empty, testFacts(.data, &.{}))); + try std.testing.expect((try cureFailure(aa, empty, testFacts(.none, &.{}))) != null); + + // .threw needs nothing beyond running clean. + const threw: ScriptError = .{ .kind = .threw, .detail = "", .source = "" }; + try std.testing.expectEqual(null, try cureFailure(aa, threw, testFacts(.none, &.{}))); +} + +test "suspicionOf: any all-empty field is suspect, none is not" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const sparse: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "comments", .calls = 5, .empty = 2 }, + }; + try std.testing.expectEqual(null, suspicionOf(aa, testFacts(.data, sparse))); + + // Scalar all-empty is suspect too — judgment belongs to the model now. + const dry_scalar: []const ScriptRuntime.ExtractStat = &.{ + .{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }, + }; + const s = suspicionOf(aa, testFacts(.data, dry_scalar)).?; + try std.testing.expectEqual(ScriptError.Kind.dry_extracts, s.kind); + try std.testing.expectEqual(1, s.dry_fields.len); + + const empty_facts = testFacts(.{ .empty = "[]" }, &.{}); + try std.testing.expectEqual(ScriptError.Kind.empty, suspicionOf(aa, empty_facts).?.kind); +} + +test "classifyRun: maps err to threw, completion emptiness to returned" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const threw = try classifyRun(aa, .{ .err = "boom at line 2" }, "return 1;"); + try std.testing.expectEqual(ScriptError.Kind.threw, threw.script_error.kind); + try std.testing.expectEqualStrings("boom at line 2", threw.script_error.detail); + try std.testing.expectEqualStrings("return 1;", threw.script_error.source); + + const empty = try classifyRun(aa, .{ .ok = .{ + .completion = .{ .text = "[]", .empty = true }, + .extract_stats = &.{}, + } }, "return [];"); + try std.testing.expectEqualStrings("[]", empty.facts.returned.empty); + + const data = try classifyRun(aa, .{ .ok = .{ + .completion = .{ .text = "[1]", .empty = false }, + .extract_stats = &.{.{ .schema = "{}", .field = "a", .calls = 1, .empty = 0 }}, + } }, "return [1];"); + try std.testing.expectEqual(Returned.data, data.facts.returned); + try std.testing.expectEqual(1, data.facts.extract_stats.len); + + const none = try classifyRun(aa, .{ .ok = .{ .completion = null, .extract_stats = &.{} } }, "1;"); + try std.testing.expectEqual(Returned.none, none.facts.returned); +} + +test "wire failure round-trips, empty string standing in for null" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const original: ScriptError = .{ + .kind = .dry_extracts, + .detail = "dry", + .source = "return 1;", + .dry_fields = &.{ @as(?[]const u8, "comments"), null }, + }; + const wire = try wireFailure(aa, original); + try std.testing.expectEqualStrings("comments", wire.dry_fields[0]); + try std.testing.expectEqualStrings("", wire.dry_fields[1]); + + const back = try scriptErrorFromWire(aa, wire); + try std.testing.expectEqual(original.kind, back.kind); + try std.testing.expectEqualStrings("comments", back.dry_fields[0].?); + try std.testing.expectEqual(null, back.dry_fields[1]); +} + +test "RunReport serializes to the wire shape" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const aa = arena.allocator(); + + const report: RunReport = .{ + .status = .suspicious, + .path = "hn.js", + .returned = .data, + .extracts = &.{.{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }}, + .failure = .{ .kind = .dry_extracts, .detail = "dry", .dry_fields = &.{"title"} }, + .console = &.{.{ .level = "log", .text = "hello" }}, + }; + const json = try std.json.Stringify.valueAlloc(aa, report, .{ .emit_null_optional_fields = false }); + try testing.expectJson(.{ + .status = "suspicious", + .path = "hn.js", + .returned = "data", + .extracts = .{.{ .schema = "{}", .field = "title", .calls = 3, .empty = 3 }}, + .failure = .{ .kind = "dry_extracts", .detail = "dry", .dry_fields = .{"title"} }, + .console = .{.{ .level = "log", .text = "hello" }}, + .console_truncated = false, + }, json); +} diff --git a/src/string.zig b/src/string.zig index 437133ebf..b5850a8ab 100644 --- a/src/string.zig +++ b/src/string.zig @@ -422,6 +422,16 @@ pub fn truncateUtf8(bytes: []const u8, max_bytes: usize) []const u8 { return bytes[0..i]; } +/// Returns `bytes` unchanged when under `max_bytes`; a truncated-and-marked +/// copy in `allocator` otherwise (the bare prefix on OOM — never an error). +pub fn capBytes(allocator: std.mem.Allocator, bytes: []const u8, max_bytes: usize) []const u8 { + if (bytes.len <= max_bytes) return bytes; + const prefix = truncateUtf8(bytes, max_bytes); + var suffix_buf: [64]u8 = undefined; + const suffix = std.fmt.bufPrint(&suffix_buf, "\n...[truncated, original {d} bytes]", .{bytes.len}) catch return prefix; + return std.mem.concat(allocator, u8, &.{ prefix, suffix }) catch prefix; +} + /// Reinterprets `bytes` as Latin-1 (each byte one codepoint) and encodes it /// as UTF-8. For bytes that aren't valid UTF-8 but must become a valid UTF-8 /// string (JSON, filenames). @@ -502,6 +512,21 @@ test "truncateUtf8" { try testing.expectEqual("\xFFx", truncateUtf8("\xFFx", 2)); } +test "capBytes: passes through when under cap" { + const out = capBytes(testing.allocator, "short", 16); + try testing.expectEqual("short", out); +} + +test "capBytes: appends a marker, keeps valid UTF-8" { + const ta = testing.allocator; + // 3-byte codepoint '한' (0xED 0x95 0x9C) straddling the cap. + const out = capBytes(ta, "aaaaaaa한bbb", 8); + defer ta.free(out); + try std.testing.expect(std.unicode.utf8ValidateSlice(out)); + try std.testing.expect(std.mem.startsWith(u8, out, "aaaaaaa")); + try std.testing.expect(std.mem.indexOf(u8, out, "truncated, original 13 bytes") != null); +} + test "latin1ToUtf8" { const cases = [_]struct { in: []const u8, out: []const u8 }{ .{ .in = "caf\xE9.txt", .out = "café.txt" },