From 023242687593b23b21867817593581a499391d15 Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Fri, 14 Aug 2026 18:09:04 +0300 Subject: [PATCH 1/7] feat(llm): add support for Nemotron model and reasoning profile - Introduced the Nemotron 3.5 Lightning model with a new reasoning profile, `NEMOTRON_THINK_PROFILE`, which utilizes ChatML with an `enable_thinking` flag and prefills `` tags. - Updated the model profile detection logic to classify the Nemotron model correctly. - Enhanced grammar building to accommodate the unique structure of Nemotron's reasoning. - Expanded the local models catalog to include the new Nemotron model, ensuring it is properly integrated into the system. - Added tests to verify the functionality and correctness of the new model and its integration. This update enhances the system's capabilities by supporting an additional model, improving reasoning handling, and ensuring robust testing for new features. --- .pr-review-56 | 1 + AGENTS.md | 6 +- src/agent/profile-matrix.test.ts | 13 ++ src/cli/models-handlers.ts | 4 +- src/llm/grammar/build-grammar.test.ts | 20 +++ src/llm/index.ts | 1 + src/llm/model-profile.fixtures.ts | 197 ++++++++++++++++++++++++++ src/llm/model-profile.test.ts | 19 +++ src/llm/model-profile.ts | 31 +++- src/llm/profile-invariants.ts | 2 + src/local-llm/models-catalog.test.ts | 24 +++- src/local-llm/models-catalog.ts | 24 +++- 12 files changed, 326 insertions(+), 16 deletions(-) create mode 160000 .pr-review-56 diff --git a/.pr-review-56 b/.pr-review-56 new file mode 160000 index 00000000..507fff09 --- /dev/null +++ b/.pr-review-56 @@ -0,0 +1 @@ +Subproject commit 507fff0992de2fbeebe5e089a76b44607f896750 diff --git a/AGENTS.md b/AGENTS.md index 09b2ad11..f991bd77 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,9 +16,9 @@ This is the source-of-truth for automated contributors (LLM agents, codegen, etc 1. **Project ≠ Prompt.** Session state, compressed tool results, and world snapshots live outside the model; the prompt is always a small slice. 2. **Stable prefix.** The prompt is `buildStablePrefix` (persona + `### rules` + skill catalog under `### skills` + `### tools` + `### capabilities` + `### instructions`) followed by a **variable tail** in mutability order: `### loaded-skills` (optional) → `### loaded-tools` (optional) → `### profile` (optional) → `### memory-index` (optional) → `### session-facts` (optional) → `### recalled` (optional) → `### world` → `### conversation` → optional `### notice` → `### respond` (+ optional reasoning prefill). Only the stable-prefix bytes must stay stable within a session for KV-cache — this is what `cache_prompt + slot_id` on `llama-server` relies on. 3. **One inference per step.** No reasoning loops inside a single LLM call — the runtime drives the loop. A single inference always emits a JSON **array** of `1..N` tool calls (`[{tool, args}, ...]`); a "solo" step is just a length-1 array (`[{...}]`). `N` is capped by `agent.maxParallelToolCalls` (default 8, hard ceiling 16 in the grammar). See §"Parallel tool calls per step" for the rationale (GBNF first-token bias) and the executor pipeline. -4. **Grammar-constrained tool calls.** The sidecar sends a GBNF grammar with every completion request that must produce a tool call. The root collapsed to **array-only** (`root ::= tool-call-array`) so the model cannot fall into the single-object form via first-token bias even when it only needs one call. Reasoning-prelude profiles (`qwen-think`, `gemma4-think`) prepend a `...` / `<|channel>thought...` block to the array; the seam between the close sentinel and the leading `[` of the array routes through a dedicated **bounded** `prelude-trail-ws ::= ( [ \t\n\r] ){0,8}` rule rather than the global unbounded `ws`. This is the structural anti-degenerate-loop guard — small reasoning-capable models (Gemma 4 26B-A4B in particular) used to slide into a whitespace-only tail after a long reasoning block because the sampler could keep emitting newlines indefinitely. Pinned by [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts) "bounds the whitespace between the reasoning-close sentinel and the tool-call array". +4. **Grammar-constrained tool calls.** The sidecar sends a GBNF grammar with every completion request that must produce a tool call. The root collapsed to **array-only** (`root ::= tool-call-array`) so the model cannot fall into the single-object form via first-token bias even when it only needs one call. Reasoning-prelude profiles (`qwen-think`, `nemotron-think`, `gemma4-think`) prepend a `...` / `<|channel>thought...` block to the array; the seam between the close sentinel and the leading `[` of the array routes through a dedicated **bounded** `prelude-trail-ws ::= ( [ \t\n\r] ){0,8}` rule rather than the global unbounded `ws`. This is the structural anti-degenerate-loop guard — small reasoning-capable models (Gemma 4 26B-A4B in particular) used to slide into a whitespace-only tail after a long reasoning block because the sampler could keep emitting newlines indefinitely. Pinned by [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts) "bounds the whitespace between the reasoning-close sentinel and the tool-call array". - **Reasoning-open ownership differs by profile (`reasoningOpenEmittedByModel` in [src/llm/model-profile.ts](src/llm/model-profile.ts)).** `qwen-think` **prefills** its open tag at the end of the prompt (`### respond` → ``); the grammar prelude starts after the open tag (`think-prelude ::= think-body "" …`) and `step-executor` prepends the open tag back onto the completion before parsing. `gemma4-think` is the opposite: Gemma 4's QAT template treats a prefilled `<|channel>thought\n` as the *thinking-disabled* marker and immediately closes the channel, dumping its reasoning into `reply.text`. The fix is **native turn-framing** — the profile carries `turnFraming { systemOpen: "<|turn>system\n", turnClose: "\n", assistantOpen: "<|turn>model\n" }` and `reasoningEmittedByModel: true`. `buildStablePrefix` opens the prompt with `<|turn>system\n<|think|>\n### system…` (the `<|think|>` token must sit at the very top of a real system turn to activate the channel — a one-time gemma-only stable-prefix byte change / KV invalidation), `buildPrompt` ends the tail with `…### respond\nRespond now.\n\n\n<|turn>model\n` instead of a channel prefill, and the grammar prelude **includes the open sentinel** (`channel-prelude ::= "<|channel>thought\n" channel-body "" prelude-trail-ws`) so the model emits its own `<|channel>thought` block. `step-executor.normalizeContent` does **not** prepend the open tag for `reasoningEmittedByModel` profiles (it is already in `completion.content`), the stream parser runs with `preOpenedThink: false` so it detects the open tag live, and the repair prompt strips/re-appends `\n<|turn>model` rather than the bare open tag. `checkProfilePromptAligned` asserts a turn-framed prompt ends with the model-turn opener (not the open tag). Pinned by [src/agent/profile-matrix.test.ts](src/agent/profile-matrix.test.ts), [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts), [src/prompt/build-prompt.test.ts](src/prompt/build-prompt.test.ts), [src/llm/profile-invariants.test.ts](src/llm/profile-invariants.test.ts), [src/llm/grammar/stream-parser.test.ts](src/llm/grammar/stream-parser.test.ts). + **Reasoning-open ownership differs by profile (`reasoningOpenEmittedByModel` in [src/llm/model-profile.ts](src/llm/model-profile.ts)).** `qwen-think` **prefills** its open tag at the end of the prompt (`### respond` → ``); the grammar prelude starts after the open tag (`think-prelude ::= think-body "" …`) and `step-executor` prepends the open tag back onto the completion before parsing. `nemotron-think` uses the same ChatML `` prefill ownership as `qwen-think` (no `turnFraming`, no `reasoningEmittedByModel`) — Nemotron 3.5 Lightning's template ends generation at `<|im_start|>assistant\n\n`. `gemma4-think` is the opposite: Gemma 4's QAT template treats a prefilled `<|channel>thought\n` as the *thinking-disabled* marker and immediately closes the channel, dumping its reasoning into `reply.text`. The fix is **native turn-framing** — the profile carries `turnFraming { systemOpen: "<|turn>system\n", turnClose: "\n", assistantOpen: "<|turn>model\n" }` and `reasoningEmittedByModel: true`. `buildStablePrefix` opens the prompt with `<|turn>system\n<|think|>\n### system…` (the `<|think|>` token must sit at the very top of a real system turn to activate the channel — a one-time gemma-only stable-prefix byte change / KV invalidation), `buildPrompt` ends the tail with `…### respond\nRespond now.\n\n\n<|turn>model\n` instead of a channel prefill, and the grammar prelude **includes the open sentinel** (`channel-prelude ::= "<|channel>thought\n" channel-body "" prelude-trail-ws`) so the model emits its own `<|channel>thought` block. `step-executor.normalizeContent` does **not** prepend the open tag for `reasoningEmittedByModel` profiles (it is already in `completion.content`), the stream parser runs with `preOpenedThink: false` so it detects the open tag live, and the repair prompt strips/re-appends `\n<|turn>model` rather than the bare open tag. `checkProfilePromptAligned` asserts a turn-framed prompt ends with the model-turn opener (not the open tag). Pinned by [src/agent/profile-matrix.test.ts](src/agent/profile-matrix.test.ts), [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts), [src/prompt/build-prompt.test.ts](src/prompt/build-prompt.test.ts), [src/llm/profile-invariants.test.ts](src/llm/profile-invariants.test.ts), [src/llm/grammar/stream-parser.test.ts](src/llm/grammar/stream-parser.test.ts). 5. **No global singletons.** Dependencies are passed explicitly. `getConfig()` is the only exception. 6. **Session is multi-turn chat only.** A session is a long-lived chat: `user message → 0..N tool steps → reply` is a macro-turn, multiple turns share one `SessionState.turns[]`. Two terminals exist — `reply` ends the turn, `finish` ends the whole session. All three frontends (CLI `run`, TUI, sidecar) go through `runtime.runTurn` only; there is no one-shot goal mode. @@ -50,7 +50,7 @@ tool-call-array ::= "[" ws tool-call ( ws "," ws tool-call ){0,15} ws "]" **Why array-only.** The first iteration of this feature shipped with `root ::= tool-call | tool-call-array` so a solo step could keep the legacy `{tool, args}` shape. Production traces showed that small/medium models (Qwen3-30B-A3B-Instruct in particular) almost never picked the array branch even when their `` block reasoned about parallel reads — the GBNF sampler's first-token mass strongly favours `{` over `[`. Collapsing the root to `tool-call-array` removes that choice entirely: the model **must** start with `[`, which makes "one call vs many calls" a decision about array length instead of a first-token gamble. A solo step is now `[{...}]`. The legacy `parseToolCall` still accepts a bare `{tool, args}` for tests/replay scenarios, but `llama-server` will never emit one under the production grammar. -The hard upper bound on array length is **16** (grammar). The runtime soft cap is `agent.maxParallelToolCalls` (default `8`, env `ATOMIC_AGENT_MAX_PARALLEL_TOOL_CALLS`). Both reasoning profiles (`qwen-think`, `gemma4-think`) route the prelude into `tool-call-array`, so think-mode batches work the same way (see [src/llm/grammar/build-grammar.ts](src/llm/grammar/build-grammar.ts) and the matching invariant in [src/llm/profile-invariants.ts](src/llm/profile-invariants.ts)). +The hard upper bound on array length is **16** (grammar). The runtime soft cap is `agent.maxParallelToolCalls` (default `8`, env `ATOMIC_AGENT_MAX_PARALLEL_TOOL_CALLS`). All reasoning profiles (`qwen-think`, `nemotron-think`, `gemma4-think`) route the prelude into `tool-call-array`, so think-mode batches work the same way (see [src/llm/grammar/build-grammar.ts](src/llm/grammar/build-grammar.ts) and the matching invariant in [src/llm/profile-invariants.ts](src/llm/profile-invariants.ts)). The change to the array-only root **invalidates KV-cache** for any session that started under the old grammar — the stable prefix bytes change once, then stay stable. There is no hot migration path; restart with a fresh session pool. diff --git a/src/agent/profile-matrix.test.ts b/src/agent/profile-matrix.test.ts index 823e9590..3ec49a74 100644 --- a/src/agent/profile-matrix.test.ts +++ b/src/agent/profile-matrix.test.ts @@ -6,6 +6,7 @@ import { GEMMA4_PROPS, GPT_OSS_PROPS, LLAMA3_PROPS, + NEMOTRON_PROPS, QWEN3_PROPS, } from "../llm/model-profile.fixtures.js"; import { startTestHarness } from "../http/test-harness.js"; @@ -36,6 +37,18 @@ describe("profile matrix", () => { }); }); + it("streams nemotron inline reasoning through the full agent loop", async () => { + // Same open/close ownership as qwen: prompt prefills ``, the + // stream starts mid-body and closes with `` before the array. + await expectScenario({ + props: NEMOTRON_PROPS, + chunks: [ + 'inner thought[{"tool":"reply","args":{"text":"ok"}}]', + ], + expectReasoning: true, + }); + }); + it("streams gemma 4 channel reasoning through the full agent loop", async () => { // Turn-framed gemma: the model emits its OWN `<|channel>thought\n` opener // (the prompt no longer prefills it), reasons, then closes with diff --git a/src/cli/models-handlers.ts b/src/cli/models-handlers.ts index 80be86d0..c2155819 100644 --- a/src/cli/models-handlers.ts +++ b/src/cli/models-handlers.ts @@ -62,14 +62,14 @@ export async function runLocalModelsList(): Promise { const cfg = getConfig(); const dataDir = cfg.paths.localModelsDataDir; process.stdout.write( - "ID | FAMILY | SIZE | CONTEXT | DL | ACTIVE\n", + "ID | FAMILY | SIZE | CONTEXT | DL | ACTIVE\n", ); for (const m of LOCAL_MODELS_CATALOG) { const dl = isModelDownloaded(dataDir, m) ? "yes" : "no"; const active = cfg.localModels.managed.modelId === m.id && cfg.localModels.mode === "managed" ? "*" : " "; process.stdout.write( - `${m.id.padEnd(19)} | ${m.family.padEnd(6)} | ${m.sizeLabel.padEnd(6)} | ${m.contextLabel.padEnd(7)} | ${dl.padEnd(3)} | ${active}\n`, + `${m.id.padEnd(20)} | ${m.family.padEnd(8)} | ${m.sizeLabel.padEnd(6)} | ${m.contextLabel.padEnd(7)} | ${dl.padEnd(3)} | ${active}\n`, ); } return 0; diff --git a/src/llm/grammar/build-grammar.test.ts b/src/llm/grammar/build-grammar.test.ts index b45d5453..33eab9af 100644 --- a/src/llm/grammar/build-grammar.test.ts +++ b/src/llm/grammar/build-grammar.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { GEMMA4_THINK_PROFILE, + NEMOTRON_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "../model-profile.js"; @@ -23,6 +24,7 @@ describe("buildGrammar", () => { for (const profile of [ PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, + NEMOTRON_THINK_PROFILE, GEMMA4_THINK_PROFILE, ]) { const grammar = await buildGrammar(profile); @@ -42,6 +44,18 @@ describe("buildGrammar", () => { expect(grammar).toContain('think-fragment ::= [^<]+ | "<" [^/]'); }); + it("builds a nemotron think grammar with the same think prelude as qwen", async () => { + // Nemotron prefills `` in the prompt (like qwen), so the + // prelude must NOT force the open sentinel — only the close tag. + const grammar = await buildGrammar(NEMOTRON_THINK_PROFILE); + expect(grammar).toContain("root ::= think-prelude tool-call-array"); + expect(grammar).toContain( + 'think-prelude ::= think-body "" prelude-trail-ws', + ); + expect(grammar).toContain("prelude-trail-ws ::= ( [ \\t\\n\\r] ){0,8}"); + expect(grammar).not.toContain('think-prelude ::= ""'); + }); + it("builds a gemma 4 grammar with a channel prelude that forces the model-emitted open tag", async () => { const grammar = await buildGrammar(GEMMA4_THINK_PROFILE); expect(grammar).toContain("root ::= channel-prelude tool-call-array"); @@ -69,6 +83,12 @@ describe("buildGrammar", () => { const qwen = await buildGrammar(QWEN_THINK_PROFILE); expect(qwen).toContain("prelude-trail-ws ::= ( [ \\t\\n\\r] ){0,8}"); expect(qwen).not.toMatch(/^think-prelude ::= think-body "<\/think>" ws$/m); + + const nemotron = await buildGrammar(NEMOTRON_THINK_PROFILE); + expect(nemotron).toContain("prelude-trail-ws ::= ( [ \\t\\n\\r] ){0,8}"); + expect(nemotron).not.toMatch( + /^think-prelude ::= think-body "<\/think>" ws$/m, + ); }); it("does not introduce a prelude-trail-ws rule for plain-instruct profiles", async () => { diff --git a/src/llm/index.ts b/src/llm/index.ts index 775baf9a..c34a3780 100644 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -19,6 +19,7 @@ export { detectModelProfile, extractTotalSlots, GEMMA4_THINK_PROFILE, + NEMOTRON_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "./model-profile.js"; diff --git a/src/llm/model-profile.fixtures.ts b/src/llm/model-profile.fixtures.ts index d1403a09..bc9cbe93 100644 --- a/src/llm/model-profile.fixtures.ts +++ b/src/llm/model-profile.fixtures.ts @@ -444,3 +444,200 @@ export const GEMMA4_PROPS = { supports_preserve_reasoning: true, }, }; + +export const NEMOTRON_PROPS = { + model_alias: "nvidia-nemotron-3.5-lightning-30b-a3b", + chat_template: `{% macro render_extra_keys(json_dict, handled_keys) %} + {%- if json_dict is mapping %} + {%- for json_key in json_dict if json_key not in handled_keys %} + {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %} + {{- '\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '' }} + {%- else %} + {{-'\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '' }} + {%- endif %} + {%- endfor %} + {%- endif %} +{% endmacro %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %} +{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %} +{%- set ns = namespace(last_user_idx = -1) %} +{%- set loop_messages = messages %} +{%- for m in loop_messages %} + {%- if m["role"] == "user" %} + {%- set ns.last_user_idx = loop.index0 %} + {%- endif %} +{%- endfor %} +{%- if messages[0]["role"] == "system" %} + {%- set system_message = messages[0]["content"] %} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} + {%- set loop_messages = messages %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = [] %} +{%- endif %} +{%- set ns = namespace(last_user_idx = -1) %} +{%- for m in loop_messages %} + {%- if m["role"] == "user" %} + {%- set ns.last_user_idx = loop.index0 %} + {%- endif %} +{%- endfor %} +{%- if system_message is defined %} + {{- "<|im_start|>system\\n" + system_message }} +{%- else %} + {%- if tools is iterable and tools | length > 0 %} + {{- "<|im_start|>system\\n" }} + {%- endif %} +{%- endif %} +{%- if tools is iterable and tools | length > 0 %} + {%- if system_message is defined and system_message | length > 0 %} + {{- "\\n\\n" }} + {%- endif %} + {{- "# Tools\\n\\nYou have access to the following functions:\\n\\n" }} + {{- "" }} + {%- for tool in tools %} + {%- if tool.function is defined %} + {%- set tool = tool.function %} + {%- endif %} + {{- "\\n\\n" ~ tool.name ~ "" }} + {%- if tool.description is defined %} + {{- '\\n' ~ (tool.description | trim) ~ '' }} + {%- endif %} + {{- '\\n' }} + {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %} + {%- for param_name, param_fields in tool.parameters.properties|items %} + {{- '\\n' }} + {{- '\\n' ~ param_name ~ '' }} + {%- if param_fields.type is defined %} + {{- '\\n' ~ (param_fields.type | string) ~ '' }} + {%- endif %} + {%- if param_fields.description is defined %} + {{- '\\n' ~ (param_fields.description | trim) ~ '' }} + {%- endif %} + {%- if param_fields.enum is defined %} + {{- '\\n' ~ (param_fields.enum | tojson | safe) ~ '' }} + {%- endif %} + {%- set handled_keys = ['name', 'type', 'description', 'enum'] %} + {{- render_extra_keys(param_fields, handled_keys) }} + {{- '\\n' }} + {%- endfor %} + {%- endif %} + {% set handled_keys = ['type', 'properties', 'required'] %} + {{- render_extra_keys(tool.parameters, handled_keys) }} + {%- if tool.parameters is defined and tool.parameters.required is defined %} + {{- '\\n' ~ (tool.parameters.required | tojson | safe) ~ '' }} + {%- endif %} + {{- '\\n' }} + {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %} + {{- render_extra_keys(tool, handled_keys) }} + {{- '\\n' }} + {%- endfor %} + {{- "\\n" }} + {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }} +{%- endif %} +{%- if system_message is defined %} + {{- '<|im_end|>\\n' }} +{%- else %} + {%- if tools is iterable and tools | length > 0 %} + {{- '<|im_end|>\\n' }} + {%- endif %} +{%- endif %} +{%- for message in loop_messages %} + {%- if message.role == "assistant" %} + {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %} + {%- set content = "\\n" ~ message.reasoning_content ~ "" ~ (message.content | default('', true)) %} + {%- else %} + {%- set content = message.content | default('', true) %} + {%- if content is string -%} + {%- if '' not in content and '' not in content -%} + {%- set content = "" ~ content -%} + {%- endif -%} + {%- else -%} + {%- set content = content -%} + {%- endif -%} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %} + {{- '<|im_start|>assistant\\n' }} + {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %} + {%- if content is string and content | trim | length > 0 %} + {%- if include_content %} + {{- (content | trim) ~ '\\n' -}} + {%- else %} + {%- set c = (content | string) %} + {%- if '' in c %} + {%- set c = c.split('')[-1] %} + {%- elif '' in c %} + {%- set c = c.split('')[0] %} + {%- endif %} + {%- set c = "" ~ c %} + {%- if c | length > 0 %} + {{- c ~ '\\n' -}} + {%- endif %} + {%- endif %} + {%- else %} + {{- "" -}} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\\n\\n' -}} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\\n' -}} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value ~ '\\n\\n' -}} + {%- endfor %} + {%- endif %} + {{- '\\n\\n' -}} + {%- endfor %} + {{- '<|im_end|>\\n' }} + {%- else %} + {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %} + {{- '<|im_start|>assistant\\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\\n' }} + {%- else %} + {%- set c = (content | default('', true) | string) %} + {%- if '' in c and '' in c %} + {%- set c = "" ~ c.split('')[-1] %} + {%- endif %} + {%- set c = c | trim %} + {%- if c | length > 0 %} + {{- '<|im_start|>assistant\\n' ~ c ~ '<|im_end|>\\n' }} + {%- else %} + {{- '<|im_start|>assistant\\n<|im_end|>\\n' }} + {%- endif %} + {%- endif %} + {%- endif %} + {%- elif message.role == "user" or message.role == "system" %} + {{- '<|im_start|>' + message.role + '\\n' }} + {%- set content = message.content | string %} + {{- content }} + {{- '<|im_end|>\\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user\\n' }} + {%- endif %} + {{- '\\n' }} + {{- message.content }} + {{- '\\n\\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\\n' }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {%- if enable_thinking %} + {{- '<|im_start|>assistant\\n\\n' }} + {%- else %} + {{- '<|im_start|>assistant\\n' }} + {%- endif %} +{%- endif %}`, + chat_template_caps: { + supports_preserve_reasoning: true, + }, +}; diff --git a/src/llm/model-profile.test.ts b/src/llm/model-profile.test.ts index 172306d2..f22b1689 100644 --- a/src/llm/model-profile.test.ts +++ b/src/llm/model-profile.test.ts @@ -5,6 +5,7 @@ import { extractTotalSlots, detectVisionSupport, GEMMA4_THINK_PROFILE, + NEMOTRON_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "./model-profile.js"; @@ -12,6 +13,7 @@ import { GEMMA4_PROPS, GPT_OSS_PROPS, LLAMA3_PROPS, + NEMOTRON_PROPS, QWEN3_PROPS, } from "./model-profile.fixtures.js"; @@ -49,6 +51,23 @@ describe("detectModelProfile", () => { expect(detectModelProfile(GEMMA4_PROPS)).toEqual(GEMMA4_THINK_PROFILE); }); + it("detects nemotron think profile from ChatML + enable_thinking", () => { + expect(detectModelProfile(NEMOTRON_PROPS)).toEqual(NEMOTRON_THINK_PROFILE); + }); + + it("does not classify a non-nemotron ChatML think template as nemotron", () => { + // Same markers as Nemotron (ChatML + + enable_thinking) but the + // alias is not nemotron — must stay on the qwen path (or plain if the + // qwen alias hint also fails). Here the alias carries none of the + // qwen/nemotron hints, so the result is plain-instruct. + expect( + detectModelProfile({ + ...NEMOTRON_PROPS, + model_alias: "some-other-chatml-think-model", + }), + ).toEqual(PLAIN_INSTRUCT_PROFILE); + }); + it("falls back to plain profile for gpt-oss style templates", () => { expect(detectModelProfile(GPT_OSS_PROPS)).toEqual(PLAIN_INSTRUCT_PROFILE); }); diff --git a/src/llm/model-profile.ts b/src/llm/model-profile.ts index 1829f6ac..cd8d8a35 100644 --- a/src/llm/model-profile.ts +++ b/src/llm/model-profile.ts @@ -61,7 +61,7 @@ export interface ReasoningTurnFraming { } export interface TaggedReasoningModelProfile extends BaseModelProfile { - id: "qwen-think" | "gemma4-think"; + id: "qwen-think" | "gemma4-think" | "nemotron-think"; reasoningStyle: "think-tags" | "channel-tags"; reasoningOpenTag: string; reasoningCloseTag: string; @@ -130,6 +130,21 @@ export const QWEN_THINK_PROFILE: TaggedReasoningModelProfile = { vision: VISION_ABSENT, }; +/** + * Nemotron 3.5 Lightning uses ChatML with an `enable_thinking` flag and + * prefills `` at the generation point — same open/close ownership as + * `qwen-think`, without Gemma-style turn framing. + */ +export const NEMOTRON_THINK_PROFILE: TaggedReasoningModelProfile = { + id: "nemotron-think", + reasoningStyle: "think-tags", + reasoningOpenTag: "", + reasoningCloseTag: "", + requiresPromptThinkPrefix: true, + allowThinkPrelude: true, + vision: VISION_ABSENT, +}; + export const GEMMA4_THINK_PROFILE: TaggedReasoningModelProfile = { id: "gemma4-think", reasoningStyle: "channel-tags", @@ -225,6 +240,9 @@ function selectBaseProfile( if (looksLikeQwenThinkModel(modelAlias, templateLower, supportsPreserveReasoning)) { return QWEN_THINK_PROFILE; } + if (looksLikeNemotronThinkModel(modelAlias, templateLower)) { + return NEMOTRON_THINK_PROFILE; + } if (looksLikeGemma4ThinkModel(modelAlias, templateLower)) { return GEMMA4_THINK_PROFILE; } @@ -290,6 +308,17 @@ function looksLikeQwenThinkModel( return aliasHint && templateHint; } +function looksLikeNemotronThinkModel( + modelAlias: string, + templateLower: string, +): boolean { + const aliasHint = modelAlias.includes("nemotron"); + const templateHint = + templateLower.includes("") && + templateLower.includes("enable_thinking"); + return aliasHint && templateHint; +} + function looksLikeGemma4ThinkModel(modelAlias: string, templateLower: string): boolean { const aliasHint = modelAlias.includes("gemma"); const templateHint = diff --git a/src/llm/profile-invariants.ts b/src/llm/profile-invariants.ts index 9f96df3b..62d4e04a 100644 --- a/src/llm/profile-invariants.ts +++ b/src/llm/profile-invariants.ts @@ -1,5 +1,6 @@ import { GEMMA4_THINK_PROFILE, + NEMOTRON_THINK_PROFILE, QWEN_THINK_PROFILE, getReasoningTurnFraming, type ModelProfile, @@ -78,6 +79,7 @@ function escapeGrammarLiteral(text: string): string { function getKnownReasoningOpenTags(): string[] { return [ QWEN_THINK_PROFILE.reasoningOpenTag.trimEnd(), + NEMOTRON_THINK_PROFILE.reasoningOpenTag.trimEnd(), GEMMA4_THINK_PROFILE.reasoningOpenTag.trimEnd(), ]; } diff --git a/src/local-llm/models-catalog.test.ts b/src/local-llm/models-catalog.test.ts index 8e5792fc..e473cd5a 100644 --- a/src/local-llm/models-catalog.test.ts +++ b/src/local-llm/models-catalog.test.ts @@ -11,10 +11,10 @@ import { } from "./models-catalog.js"; describe("models-catalog", () => { - it("has exactly 9 Qwen+Gemma models with unique ids", () => { - expect(LOCAL_MODELS_CATALOG.length).toBe(9); + it("has exactly 10 Qwen+Gemma+Nemotron models with unique ids", () => { + expect(LOCAL_MODELS_CATALOG.length).toBe(10); const ids = new Set(LOCAL_MODELS_CATALOG.map((m) => m.id)); - expect(ids.size).toBe(9); + expect(ids.size).toBe(10); }); it("defaults to qwen-3.5-4b", () => { @@ -36,16 +36,26 @@ describe("models-catalog", () => { ).toThrow(/unknown local model id/); }); - it("marks every catalog entry as vision-capable with mmproj URL", () => { - expect(LOCAL_MODELS_CATALOG.length).toBeGreaterThan(0); - for (const def of LOCAL_MODELS_CATALOG) { - expect(def.supportsVision).toBe(true); + it("ships mmproj URL for every vision-capable catalog entry", () => { + const visionModels = LOCAL_MODELS_CATALOG.filter((m) => m.supportsVision); + expect(visionModels.length).toBeGreaterThan(0); + for (const def of visionModels) { expect(def.mmprojUrl).toMatch(/^https:\/\//); expect(def.mmprojFilename).toMatch(/\.gguf$/); expect(typeof def.mmprojFileSizeGb).toBe("number"); } }); + it("omits mmproj fields on text-only catalog entries", () => { + const textOnly = LOCAL_MODELS_CATALOG.filter((m) => !m.supportsVision); + expect(textOnly.map((m) => m.id)).toEqual(["nemotron-3.5-30b-a3b"]); + for (const def of textOnly) { + expect(def.mmprojUrl).toBeUndefined(); + expect(def.mmprojFilename).toBeUndefined(); + expect(def.mmprojFileSizeGb).toBeUndefined(); + } + }); + it("ensures mmproj URL points at the same HF repo as the GGUF weights", () => { for (const def of LOCAL_MODELS_CATALOG) { if (!def.mmprojUrl) continue; diff --git a/src/local-llm/models-catalog.ts b/src/local-llm/models-catalog.ts index 22d8e687..27e80477 100644 --- a/src/local-llm/models-catalog.ts +++ b/src/local-llm/models-catalog.ts @@ -1,5 +1,5 @@ /** - * Curated GGUF catalog (Qwen + Gemma only). URLs mirror atomic-hermes + * Curated GGUF catalog (Qwen + Gemma + Nemotron). URLs mirror atomic-hermes * desktop local LLM models; `family` replaces UI-only icon fields. */ @@ -12,7 +12,8 @@ export type LocalModelId = | "gemma-4-e4b" | "gemma-4-12b" | "gemma-4-26b-a4b" - | "gemma-4-31b"; + | "gemma-4-31b" + | "nemotron-3.5-30b-a3b"; /** * Memory-v2 phase 1B. Embedding model identifiers. A separate union @@ -42,7 +43,7 @@ export interface LocalModelDef { contextLabel: string; minRamGb: number; recommendedRamGb: number; - family: "qwen" | "gemma"; + family: "qwen" | "gemma" | "nemotron"; /** * Jinja file under `assets/ai-models/` passed to llama-server as * `--chat-template-file`, overriding the template baked into the GGUF. @@ -260,6 +261,23 @@ export const LOCAL_MODELS_CATALOG: readonly LocalModelDef[] = [ mmprojFilename: "mmproj-F16.gguf", mmprojFileSizeGb: 0.90, }, + { + id: "nemotron-3.5-30b-a3b", + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B GGUF", + filename: "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-AD-IQ4_NL.gguf", + huggingFaceUrl: + "https://huggingface.co/AtomicChat/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/resolve/main/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-AD-IQ4_NL.gguf", + fileSizeGb: 19.65, + sizeLabel: "19.7 GB", + description: "Hybrid Mamba2 MoE reasoning, imatrix-calibrated", + maxContextLength: 262_144, + contextLabel: "256K", + minRamGb: 24, + recommendedRamGb: 32, + family: "nemotron", + tag: "New", + supportsVision: false, + }, ]; export const DEFAULT_LLAMACPP_MODEL_ID: LocalModelId = "qwen-3.5-4b"; From 474f82061e7d5dc4f78e7a517c72d6ded7b09f71 Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Sun, 16 Aug 2026 21:58:21 +0300 Subject: [PATCH 2/7] feat(models-catalog): expand model catalog to include Muse and update tests - Removed the outdated output.txt file. - Updated the LOCAL_MODELS_CATALOG to include the new Muse model, increasing the total count of curated models to 12 (Qwen, Gemma, Nemotron, and Muse). - Adjusted related tests to reflect the new model count and ensure unique IDs across the catalog. This update enhances the model offerings and ensures the tests accurately represent the current state of the model catalog. --- output.txt | 1 - src/cli/models-command.test.ts | 16 ++++++++++---- src/local-llm/models-catalog.test.ts | 6 +++--- src/local-llm/models-catalog.ts | 31 ++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 12 deletions(-) delete mode 100644 output.txt diff --git a/output.txt b/output.txt deleted file mode 100644 index b5754e20..00000000 --- a/output.txt +++ /dev/null @@ -1 +0,0 @@ -ok \ No newline at end of file diff --git a/src/cli/models-command.test.ts b/src/cli/models-command.test.ts index ad44a087..f05edea7 100644 --- a/src/cli/models-command.test.ts +++ b/src/cli/models-command.test.ts @@ -53,10 +53,18 @@ describe("modelsCommand", () => { const code = await modelsCommand(["list"]); expect(code).toBe(0); const out = stdout(); - // One row per curated model in LOCAL_MODELS_CATALOG: 4 gemma + 6 qwen. - expect(out.split("\n").filter((l) => l.includes("qwen-") || l.includes("gemma-"))).toHaveLength( - 10, - ); + // One row per curated model in LOCAL_MODELS_CATALOG. + expect( + out + .split("\n") + .filter( + (l) => + l.includes("qwen-") || + l.includes("gemma-") || + l.includes("nemotron-") || + l.includes("muse-"), + ), + ).toHaveLength(12); }); it("pull with bad id exits 1", async () => { diff --git a/src/local-llm/models-catalog.test.ts b/src/local-llm/models-catalog.test.ts index b5a0718b..7d8ac3b9 100644 --- a/src/local-llm/models-catalog.test.ts +++ b/src/local-llm/models-catalog.test.ts @@ -11,10 +11,10 @@ import { } from "./models-catalog.js"; describe("models-catalog", () => { - it("has exactly 11 Qwen+Gemma+Nemotron models with unique ids", () => { - expect(LOCAL_MODELS_CATALOG.length).toBe(11); + it("has exactly 12 Qwen+Gemma+Nemotron+Muse models with unique ids", () => { + expect(LOCAL_MODELS_CATALOG.length).toBe(12); const ids = new Set(LOCAL_MODELS_CATALOG.map((m) => m.id)); - expect(ids.size).toBe(11); + expect(ids.size).toBe(12); }); it("defaults to qwen-3.5-4b", () => { diff --git a/src/local-llm/models-catalog.ts b/src/local-llm/models-catalog.ts index bef345d0..b1f75496 100644 --- a/src/local-llm/models-catalog.ts +++ b/src/local-llm/models-catalog.ts @@ -1,6 +1,7 @@ /** - * Curated GGUF catalog (Qwen + Gemma + Nemotron). URLs mirror atomic-hermes - * desktop local LLM models; `family` replaces UI-only icon fields. + * Curated GGUF catalog (Qwen + Gemma + Nemotron + Muse). URLs mirror + * atomic-hermes desktop local LLM models; `family` replaces UI-only + * icon fields. */ export type LocalModelId = @@ -14,7 +15,8 @@ export type LocalModelId = | "gemma-4-12b" | "gemma-4-26b-a4b" | "gemma-4-31b" - | "nemotron-3.5-30b-a3b"; + | "nemotron-3.5-30b-a3b" + | "muse-glimmer-30b"; /** * Memory-v2 phase 1B. Embedding model identifiers. A separate union @@ -44,7 +46,7 @@ export interface LocalModelDef { contextLabel: string; minRamGb: number; recommendedRamGb: number; - family: "qwen" | "gemma" | "nemotron"; + family: "qwen" | "gemma" | "nemotron" | "muse"; /** * Jinja file under `assets/ai-models/` passed to llama-server as * `--chat-template-file`, overriding the template baked into the GGUF. @@ -298,6 +300,27 @@ export const LOCAL_MODELS_CATALOG: readonly LocalModelDef[] = [ tag: "New", supportsVision: false, }, + { + id: "muse-glimmer-30b", + name: "Meta Muse Glimmer 30B GGUF", + filename: "Muse-Glimmer-30B-UD-Q4_K_XL.gguf", + huggingFaceUrl: + "https://huggingface.co/unsloth/Muse-Glimmer-30B-GGUF/resolve/main/Muse-Glimmer-30B-UD-Q4_K_XL.gguf", + fileSizeGb: 15.9, + sizeLabel: "15.9 GB", + description: "Multimodal 30B MoE (ATEM / Harmony)", + maxContextLength: 131_072, + contextLabel: "128K", + minRamGb: 20, + recommendedRamGb: 32, + family: "muse", + tag: "New", + supportsVision: true, + mmprojUrl: + "https://huggingface.co/unsloth/Muse-Glimmer-30B-GGUF/resolve/main/mmproj-Muse-Glimmer-30B-Q8_0.gguf", + mmprojFilename: "mmproj-Muse-Glimmer-30B-Q8_0.gguf", + mmprojFileSizeGb: 2.05, + }, ]; export const DEFAULT_LLAMACPP_MODEL_ID: LocalModelId = "qwen-3.5-4b"; From 18dd114701ede96b32be7cb3603e7bd62eafd765 Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Sun, 16 Aug 2026 22:13:17 +0300 Subject: [PATCH 3/7] fix: correct GitHub repository references for llama.cpp backend - Updated the GitHub repository references in `AGENTS.md`, `README.md`, and `backend-installer.ts` from `AtomicBot-ai/atopmic-llama-cpp-turboquant-nightly` to `AtomicBot-ai/atomic-llama-cpp-turboquant-nightly`. This change ensures consistency in the documentation and codebase regarding the source of the llama.cpp backend. --- AGENTS.md | 2 +- README.md | 2 +- src/local-llm/backend-installer.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b5bcb65c..117ed075 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -237,7 +237,7 @@ Symmetric **`EmbeddingProviderRegistry`** ([src/memory/embeddings/embedding-prov `atomic-agent` supports two modes for the llama-server backend (`config.llama.mode`): - `external` (default) — user runs `llama-server` out-of-band; runtime reads the URL from `config.llama.url` (env fallback `ATOMIC_AGENT_LLAMA_URL`). -- `managed` — `atomic-agent` downloads the llama.cpp binary from `AtomicBot-ai/atopmic-llama-cpp-turboquant-nightly` GitHub Releases into `/llamacpp/backend/` and GGUF models into `/llamacpp/models//`. The server is **not** spawned by the runtime; operators control lifecycle via `atomic-agent llama start|stop|status|update`. Managed start (TUI auto-start / `s`, CLI `models start`) auto-pulls a newer zip when `localModels.managed.autoUpdate` is true (default since config v38; check failures do not block start). +- `managed` — `atomic-agent` downloads the llama.cpp binary from `AtomicBot-ai/atomic-llama-cpp-turboquant-nightly` GitHub Releases into `/llamacpp/backend/` and GGUF models into `/llamacpp/models//`. The server is **not** spawned by the runtime; operators control lifecycle via `atomic-agent llama start|stop|status|update`. Managed start (TUI auto-start / `s`, CLI `models start`) auto-pulls a newer zip when `localModels.managed.autoUpdate` is true (default since config v38; check failures do not block start). **Invariant (preserved):** the agent runtime never starts a `llama-server` process. It only connects. Managed-mode lifecycle lives entirely in the `atomic-agent llama` CLI so runtime code paths stay single-mode. diff --git a/README.md b/README.md index c7fb5b07..c0e76471 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ The model chooses actions. Atomic Agent owns the loop, the state, the approvals, ### Built to Make Local Models Work -We run local models on our own TurboQuant `llama.cpp` ([`AtomicBot-ai/atopmic-llama-cpp-turboquant-nightly`](https://github.com/AtomicBot-ai/atopmic-llama-cpp-turboquant-nightly)): +We run local models on our own TurboQuant `llama.cpp` ([`AtomicBot-ai/atomic-llama-cpp-turboquant-nightly`](https://github.com/AtomicBot-ai/atomic-llama-cpp-turboquant-nightly)): - **TurboQuant KV-cache:** WHT-rotated low-bit quantization compresses the KV-cache up to ~6.4× versus F16, with a fused Metal decode kernel, so long-context sessions fit in far less memory. - **TurboQuant weights:** Lloyd-Max weight quantization with WHT rotation and fused Metal/Vulkan kernels keeps quality usable while small models fit on consumer hardware. diff --git a/src/local-llm/backend-installer.ts b/src/local-llm/backend-installer.ts index 813c823b..17b6730c 100644 --- a/src/local-llm/backend-installer.ts +++ b/src/local-llm/backend-installer.ts @@ -20,7 +20,7 @@ import { readBackendVersion, writeBackendVersion } from "./backend-version.js"; import { resolvePlatformAsset, UnsupportedPlatformError } from "./platform-assets.js"; import { resolveDownloadAsset } from "./windows-backend-variant.js"; -const GITHUB_REPO = "AtomicBot-ai/atopmic-llama-cpp-turboquant-nightly"; +const GITHUB_REPO = "AtomicBot-ai/atomic-llama-cpp-turboquant-nightly"; /** * Anonymous GitHub API allows ~60 req/h per IP. The Models tab polls From 826cfcdef4da9e967d7d26579e2bddee7e0984bd Mon Sep 17 00:00:00 2001 From: sosidudku1 Date: Thu, 20 Aug 2026 16:21:56 +0300 Subject: [PATCH 4/7] fix: address review blockers on PR #131 - staged backend download + atomic swap (never wipe a working install) - failed auto-update no longer aborts managed start - timeout on the GitHub releases check - update only on a genuinely newer release (no nightly downgrade/thrash) - drop the dead nemotron-think profile discriminant - stop advertising unwired ATEM/Harmony tool calling for Muse Glimmer - rebase config migration onto v41 and refresh AGENTS.md --- AGENTS.md | 8 +- src/agent/profile-matrix.test.ts | 7 +- src/cli/models-handlers.ts | 23 +- src/llm/grammar/build-grammar.test.ts | 20 - src/llm/index.ts | 1 - src/llm/model-profile.fixtures.ts | 70 ++++ src/llm/model-profile.test.ts | 39 +- src/llm/model-profile.ts | 26 +- src/llm/profile-invariants.ts | 2 - src/local-llm/backend-installer.test.ts | 363 +++++++++++++++++- src/local-llm/backend-installer.ts | 353 ++++++++--------- src/local-llm/backend-staging.ts | 267 +++++++++++++ src/local-llm/backend-version.ts | 32 +- src/local-llm/ensure-latest-backend.test.ts | 65 ++++ src/local-llm/ensure-latest-backend.ts | 56 ++- src/local-llm/models-catalog.test.ts | 35 +- src/local-llm/models-catalog.ts | 9 +- ...al-models-orchestrator-auto-update.test.ts | 138 +++++++ .../local-models/local-models-orchestrator.ts | 27 +- 19 files changed, 1260 insertions(+), 281 deletions(-) create mode 100644 src/local-llm/backend-staging.ts create mode 100644 src/tui/local-models/local-models-orchestrator-auto-update.test.ts diff --git a/AGENTS.md b/AGENTS.md index 883d7f58..8edf9529 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,9 +16,9 @@ This is the source-of-truth for automated contributors (LLM agents, codegen, etc 1. **Project ≠ Prompt.** Session state, compressed tool results, and world snapshots live outside the model; the prompt is always a small slice. 2. **Stable prefix.** The prompt is `buildStablePrefix` (persona + `### rules` + skill catalog under `### skills` + `### tools` + `### capabilities` + `### instructions`) followed by a **variable tail** in mutability order: `### loaded-skills` (optional) → `### loaded-tools` (optional) → `### profile` (optional) → `### memory-index` (optional) → `### session-facts` (optional) → `### recalled` (optional) → `### world` → `### conversation` → optional `### notice` (written by the no-progress loop detector and by mid-turn steering, composed in that order) → `### respond` (+ optional reasoning prefill). Only the stable-prefix bytes must stay stable within a session for KV-cache — this is what `cache_prompt + slot_id` on `llama-server` relies on. 3. **One inference per step.** No reasoning loops inside a single LLM call — the runtime drives the loop. A single inference always emits a JSON **array** of `1..N` tool calls (`[{tool, args}, ...]`); a "solo" step is just a length-1 array (`[{...}]`). `N` is capped by `agent.maxParallelToolCalls` (default 8, hard ceiling 16 in the grammar). See §"Parallel tool calls per step" for the rationale (GBNF first-token bias) and the executor pipeline. -4. **Grammar-constrained tool calls.** The sidecar sends a GBNF grammar with every completion request that must produce a tool call. The root collapsed to **array-only** (`root ::= tool-call-array`) so the model cannot fall into the single-object form via first-token bias even when it only needs one call. Reasoning-prelude profiles (`qwen-think`, `nemotron-think`, `gemma4-think`) prepend a `...` / `<|channel>thought...` block to the array; the seam between the close sentinel and the leading `[` of the array routes through a dedicated **bounded** `prelude-trail-ws ::= ( [ \t\n\r] ){0,8}` rule rather than the global unbounded `ws`. This is the structural anti-degenerate-loop guard — small reasoning-capable models (Gemma 4 26B-A4B in particular) used to slide into a whitespace-only tail after a long reasoning block because the sampler could keep emitting newlines indefinitely. Pinned by [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts) "bounds the whitespace between the reasoning-close sentinel and the tool-call array". +4. **Grammar-constrained tool calls.** The sidecar sends a GBNF grammar with every completion request that must produce a tool call. The root collapsed to **array-only** (`root ::= tool-call-array`) so the model cannot fall into the single-object form via first-token bias even when it only needs one call. Reasoning-prelude profiles (`qwen-think`, `gemma4-think`) prepend a `...` / `<|channel>thought...` block to the array; the seam between the close sentinel and the leading `[` of the array routes through a dedicated **bounded** `prelude-trail-ws ::= ( [ \t\n\r] ){0,8}` rule rather than the global unbounded `ws`. This is the structural anti-degenerate-loop guard — small reasoning-capable models (Gemma 4 26B-A4B in particular) used to slide into a whitespace-only tail after a long reasoning block because the sampler could keep emitting newlines indefinitely. Pinned by [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts) "bounds the whitespace between the reasoning-close sentinel and the tool-call array". - **Reasoning-open ownership differs by profile (`reasoningOpenEmittedByModel` in [src/llm/model-profile.ts](src/llm/model-profile.ts)).** `qwen-think` **prefills** its open tag at the end of the prompt (`### respond` → ``); the grammar prelude starts after the open tag (`think-prelude ::= think-body "" …`) and `step-executor` prepends the open tag back onto the completion before parsing. `nemotron-think` uses the same ChatML `` prefill ownership as `qwen-think` (no `turnFraming`, no `reasoningEmittedByModel`) — Nemotron 3.5 Lightning's template ends generation at `<|im_start|>assistant\n\n`. `gemma4-think` is the opposite: Gemma 4's QAT template treats a prefilled `<|channel>thought\n` as the *thinking-disabled* marker and immediately closes the channel, dumping its reasoning into `reply.text`. The fix is **native turn-framing** — the profile carries `turnFraming { systemOpen: "<|turn>system\n", turnClose: "\n", assistantOpen: "<|turn>model\n" }` and `reasoningEmittedByModel: true`. `buildStablePrefix` opens the prompt with `<|turn>system\n<|think|>\n### system…` (the `<|think|>` token must sit at the very top of a real system turn to activate the channel — a one-time gemma-only stable-prefix byte change / KV invalidation), `buildPrompt` ends the tail with `…### respond\nRespond now.\n\n\n<|turn>model\n` instead of a channel prefill, and the grammar prelude **includes the open sentinel** (`channel-prelude ::= "<|channel>thought\n" channel-body "" prelude-trail-ws`) so the model emits its own `<|channel>thought` block. `step-executor.normalizeContent` does **not** prepend the open tag for `reasoningEmittedByModel` profiles (it is already in `completion.content`), the stream parser runs with `preOpenedThink: false` so it detects the open tag live, and the repair prompt strips/re-appends `\n<|turn>model` rather than the bare open tag. `checkProfilePromptAligned` asserts a turn-framed prompt ends with the model-turn opener (not the open tag). Pinned by [src/agent/profile-matrix.test.ts](src/agent/profile-matrix.test.ts), [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts), [src/prompt/build-prompt.test.ts](src/prompt/build-prompt.test.ts), [src/llm/profile-invariants.test.ts](src/llm/profile-invariants.test.ts), [src/llm/grammar/stream-parser.test.ts](src/llm/grammar/stream-parser.test.ts). + **Reasoning-open ownership differs by profile (`reasoningOpenEmittedByModel` in [src/llm/model-profile.ts](src/llm/model-profile.ts)).** `qwen-think` **prefills** its open tag at the end of the prompt (`### respond` → ``); the grammar prelude starts after the open tag (`think-prelude ::= think-body "" …`) and `step-executor` prepends the open tag back onto the completion before parsing. Nemotron 3.5 Lightning shares that ownership exactly: its template ends generation at `<|im_start|>assistant\n\n`, so `selectBaseProfile` detects it separately (the alias hint differs) but returns `qwen-think` itself rather than a duplicate profile. `gemma4-think` is the opposite: Gemma 4's QAT template treats a prefilled `<|channel>thought\n` as the *thinking-disabled* marker and immediately closes the channel, dumping its reasoning into `reply.text`. The fix is **native turn-framing** — the profile carries `turnFraming { systemOpen: "<|turn>system\n", turnClose: "\n", assistantOpen: "<|turn>model\n" }` and `reasoningEmittedByModel: true`. `buildStablePrefix` opens the prompt with `<|turn>system\n<|think|>\n### system…` (the `<|think|>` token must sit at the very top of a real system turn to activate the channel — a one-time gemma-only stable-prefix byte change / KV invalidation), `buildPrompt` ends the tail with `…### respond\nRespond now.\n\n\n<|turn>model\n` instead of a channel prefill, and the grammar prelude **includes the open sentinel** (`channel-prelude ::= "<|channel>thought\n" channel-body "" prelude-trail-ws`) so the model emits its own `<|channel>thought` block. `step-executor.normalizeContent` does **not** prepend the open tag for `reasoningEmittedByModel` profiles (it is already in `completion.content`), the stream parser runs with `preOpenedThink: false` so it detects the open tag live, and the repair prompt strips/re-appends `\n<|turn>model` rather than the bare open tag. `checkProfilePromptAligned` asserts a turn-framed prompt ends with the model-turn opener (not the open tag). Pinned by [src/agent/profile-matrix.test.ts](src/agent/profile-matrix.test.ts), [src/llm/grammar/build-grammar.test.ts](src/llm/grammar/build-grammar.test.ts), [src/prompt/build-prompt.test.ts](src/prompt/build-prompt.test.ts), [src/llm/profile-invariants.test.ts](src/llm/profile-invariants.test.ts), [src/llm/grammar/stream-parser.test.ts](src/llm/grammar/stream-parser.test.ts). 5. **No global singletons.** Dependencies are passed explicitly. `getConfig()` is the only exception. 6. **Session is multi-turn chat only.** A session is a long-lived chat: `user message → 0..N tool steps → reply` is a macro-turn, multiple turns share one `SessionState.turns[]`. Two terminals exist — `reply` ends the turn, `finish` ends the whole session. All three frontends (CLI `run`, TUI, sidecar) go through `runtime.runTurn` only; there is no one-shot goal mode. @@ -50,7 +50,7 @@ tool-call-array ::= "[" ws tool-call ( ws "," ws tool-call ){0,15} ws "]" **Why array-only.** The first iteration of this feature shipped with `root ::= tool-call | tool-call-array` so a solo step could keep the legacy `{tool, args}` shape. Production traces showed that small/medium models (Qwen3-30B-A3B-Instruct in particular) almost never picked the array branch even when their `` block reasoned about parallel reads — the GBNF sampler's first-token mass strongly favours `{` over `[`. Collapsing the root to `tool-call-array` removes that choice entirely: the model **must** start with `[`, which makes "one call vs many calls" a decision about array length instead of a first-token gamble. A solo step is now `[{...}]`. The legacy `parseToolCall` still accepts a bare `{tool, args}` for tests/replay scenarios, but `llama-server` will never emit one under the production grammar. -The hard upper bound on array length is **16** (grammar). The runtime soft cap is `agent.maxParallelToolCalls` (default `8`, env `ATOMIC_AGENT_MAX_PARALLEL_TOOL_CALLS`). All reasoning profiles (`qwen-think`, `nemotron-think`, `gemma4-think`) route the prelude into `tool-call-array`, so think-mode batches work the same way (see [src/llm/grammar/build-grammar.ts](src/llm/grammar/build-grammar.ts) and the matching invariant in [src/llm/profile-invariants.ts](src/llm/profile-invariants.ts)). +The hard upper bound on array length is **16** (grammar). The runtime soft cap is `agent.maxParallelToolCalls` (default `8`, env `ATOMIC_AGENT_MAX_PARALLEL_TOOL_CALLS`). Both reasoning profiles (`qwen-think`, `gemma4-think`) route the prelude into `tool-call-array`, so think-mode batches work the same way (see [src/llm/grammar/build-grammar.ts](src/llm/grammar/build-grammar.ts) and the matching invariant in [src/llm/profile-invariants.ts](src/llm/profile-invariants.ts)). The change to the array-only root **invalidates KV-cache** for any session that started under the old grammar — the stable prefix bytes change once, then stay stable. There is no hot migration path; restart with a fresh session pool. @@ -276,7 +276,7 @@ Symmetric **`EmbeddingProviderRegistry`** ([src/memory/embeddings/embedding-prov `atomic-agent` supports two modes for the llama-server backend (`config.llama.mode`): - `external` (default) — user runs `llama-server` out-of-band; runtime reads the URL from `config.llama.url` (env fallback `ATOMIC_AGENT_LLAMA_URL`). -- `managed` — `atomic-agent` downloads the llama.cpp binary from `AtomicBot-ai/atomic-llama-cpp-turboquant-nightly` GitHub Releases into `/llamacpp/backend/` and GGUF models into `/llamacpp/models//`. The server is **not** spawned by the runtime; operators control lifecycle via `atomic-agent llama start|stop|status|update`. Managed start (TUI auto-start / `s`, CLI `models start`) auto-pulls a newer zip when `localModels.managed.autoUpdate` is true (default since config v38; check failures do not block start). +- `managed` — `atomic-agent` downloads the llama.cpp binary from `AtomicBot-ai/atomic-llama-cpp-turboquant-nightly` GitHub Releases into `/llamacpp/backend/` and GGUF models into `/llamacpp/models//`. The server is **not** spawned by the runtime; operators control lifecycle via `atomic-agent llama start|stop|status|update`. Managed start (TUI auto-start / `s`, CLI `models start`) auto-pulls a newer zip when `localModels.managed.autoUpdate` is true (default since config v41; a failed check or download never blocks start — the existing binary is used). **Invariant (preserved):** the agent runtime never starts a `llama-server` process. It only connects. Managed-mode lifecycle lives entirely in the `atomic-agent llama` CLI so runtime code paths stay single-mode. diff --git a/src/agent/profile-matrix.test.ts b/src/agent/profile-matrix.test.ts index 3ec49a74..f53cf338 100644 --- a/src/agent/profile-matrix.test.ts +++ b/src/agent/profile-matrix.test.ts @@ -38,8 +38,11 @@ describe("profile matrix", () => { }); it("streams nemotron inline reasoning through the full agent loop", async () => { - // Same open/close ownership as qwen: prompt prefills ``, the - // stream starts mid-body and closes with `` before the array. + // Nemotron shares the qwen think-tags profile (its alias is what needs a + // dedicated detector, not its behaviour): the prompt prefills ``, + // the stream starts mid-body and closes with `` before the array. + // Without the nemotron detection branch this props payload falls through + // to plain-instruct and emits no reasoning at all. await expectScenario({ props: NEMOTRON_PROPS, chunks: [ diff --git a/src/cli/models-handlers.ts b/src/cli/models-handlers.ts index 9570589a..e7490d51 100644 --- a/src/cli/models-handlers.ts +++ b/src/cli/models-handlers.ts @@ -237,6 +237,20 @@ export async function runLocalModelsStart(): Promise { process.stderr.write( `note: backend update check failed — starting current binary (${auto.error})\n`, ); + } else if (auto.action === "update_failed") { + if (process.stderr.isTTY) process.stderr.write("\n"); + if (!auto.backendUsable) { + // The daemon was stopped for the update and there is no binary + // left to fall back to — nothing can be started. + process.stderr.write( + `backend auto-update failed and no usable backend remains: ${auto.error}\n` + + `run 'atomic-agent models update' once connectivity is back.\n`, + ); + return 1; + } + process.stderr.write( + `note: backend update failed — starting current binary (${auto.error})\n`, + ); } } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -606,7 +620,14 @@ export async function runLocalModelsUpdate(): Promise { try { const { updateAvailable, latestTag, currentTag } = await checkForBackendUpdate(dataDir); if (!updateAvailable) { - process.stdout.write(`backend up to date (${latestTag})\n`); + // `latestTag` is null when no scanned release ships this + // platform's asset — nothing to compare against, so the install + // on disk stands. + process.stdout.write( + latestTag === null + ? `backend unchanged (no published release for this platform)\n` + : `backend up to date (${latestTag})\n`, + ); return 0; } process.stdout.write(`current: ${currentTag ?? "none"} → latest: ${latestTag}\n`); diff --git a/src/llm/grammar/build-grammar.test.ts b/src/llm/grammar/build-grammar.test.ts index 33eab9af..b45d5453 100644 --- a/src/llm/grammar/build-grammar.test.ts +++ b/src/llm/grammar/build-grammar.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { GEMMA4_THINK_PROFILE, - NEMOTRON_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "../model-profile.js"; @@ -24,7 +23,6 @@ describe("buildGrammar", () => { for (const profile of [ PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, - NEMOTRON_THINK_PROFILE, GEMMA4_THINK_PROFILE, ]) { const grammar = await buildGrammar(profile); @@ -44,18 +42,6 @@ describe("buildGrammar", () => { expect(grammar).toContain('think-fragment ::= [^<]+ | "<" [^/]'); }); - it("builds a nemotron think grammar with the same think prelude as qwen", async () => { - // Nemotron prefills `` in the prompt (like qwen), so the - // prelude must NOT force the open sentinel — only the close tag. - const grammar = await buildGrammar(NEMOTRON_THINK_PROFILE); - expect(grammar).toContain("root ::= think-prelude tool-call-array"); - expect(grammar).toContain( - 'think-prelude ::= think-body "" prelude-trail-ws', - ); - expect(grammar).toContain("prelude-trail-ws ::= ( [ \\t\\n\\r] ){0,8}"); - expect(grammar).not.toContain('think-prelude ::= ""'); - }); - it("builds a gemma 4 grammar with a channel prelude that forces the model-emitted open tag", async () => { const grammar = await buildGrammar(GEMMA4_THINK_PROFILE); expect(grammar).toContain("root ::= channel-prelude tool-call-array"); @@ -83,12 +69,6 @@ describe("buildGrammar", () => { const qwen = await buildGrammar(QWEN_THINK_PROFILE); expect(qwen).toContain("prelude-trail-ws ::= ( [ \\t\\n\\r] ){0,8}"); expect(qwen).not.toMatch(/^think-prelude ::= think-body "<\/think>" ws$/m); - - const nemotron = await buildGrammar(NEMOTRON_THINK_PROFILE); - expect(nemotron).toContain("prelude-trail-ws ::= ( [ \\t\\n\\r] ){0,8}"); - expect(nemotron).not.toMatch( - /^think-prelude ::= think-body "<\/think>" ws$/m, - ); }); it("does not introduce a prelude-trail-ws rule for plain-instruct profiles", async () => { diff --git a/src/llm/index.ts b/src/llm/index.ts index c34a3780..775baf9a 100644 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -19,7 +19,6 @@ export { detectModelProfile, extractTotalSlots, GEMMA4_THINK_PROFILE, - NEMOTRON_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "./model-profile.js"; diff --git a/src/llm/model-profile.fixtures.ts b/src/llm/model-profile.fixtures.ts index bc9cbe93..f12c5e08 100644 --- a/src/llm/model-profile.fixtures.ts +++ b/src/llm/model-profile.fixtures.ts @@ -641,3 +641,73 @@ export const NEMOTRON_PROPS = { supports_preserve_reasoning: true, }, }; + +/** + * Meta Muse Glimmer 30B as llama-server reports it. The alias is the + * catalog id verbatim — `daemon-lifecycle.ts` passes `model.id` to `-a`. + * + * The template is Harmony/ATEM channel framing, deliberately kept rich + * rather than stubbed: it carries `<|channel|>analysis` reasoning markers + * and native `<|start|>`/`<|end|>` tool framing. That is the point of the + * fixture — detection still falls through to `plain-instruct`, and it does + * so because the alias `muse-glimmer-30b` matches no hint in + * `selectBaseProfile` (not because the template is empty). A stub template + * would pass the same assertion for the wrong reason. + */ +export const MUSE_PROPS = { + model_alias: "muse-glimmer-30b", + chat_template: `{%- if messages[0].role == 'system' %} + {{- '<|start|>system<|message|>' + messages[0].content + '<|end|>' }} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set loop_messages = messages %} +{%- endif %} +{%- if tools is defined and tools | length > 0 %} + {{- '<|start|>developer<|message|># Tools\\n\\n' }} + {{- '## functions\\n\\nnamespace functions {\\n\\n' }} + {%- for tool in tools %} + {%- if tool.function is defined %} + {%- set tool = tool.function %} + {%- endif %} + {%- if tool.description is defined %} + {{- '// ' ~ (tool.description | trim) ~ '\\n' }} + {%- endif %} + {{- 'type ' ~ tool.name ~ ' = (_: ' }} + {{- (tool.parameters | tojson | safe) ~ ') => any;\\n\\n' }} + {%- endfor %} + {{- '} // namespace functions<|end|>' }} +{%- endif %} +{%- for message in loop_messages %} + {%- if message.role == 'assistant' %} + {%- if message.tool_calls is defined and message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '<|start|>assistant to=functions.' ~ tool_call.name }} + {{- '<|channel|>commentary json<|message|>' }} + {{- (tool_call.arguments | tojson | safe) ~ '<|call|>' }} + {%- endfor %} + {%- else %} + {%- set content = message.content | default('', true) | string %} + {%- if '<|channel|>analysis<|message|>' in content %} + {%- set content = content.split('<|end|>')[-1] %} + {%- endif %} + {{- '<|start|>assistant<|channel|>final<|message|>' }} + {{- content | trim ~ '<|end|>' }} + {%- endif %} + {%- elif message.role == 'tool' %} + {{- '<|start|>functions.' ~ message.name ~ ' to=assistant' }} + {{- '<|channel|>commentary<|message|>' ~ message.content ~ '<|end|>' }} + {%- else %} + {{- '<|start|>' ~ message.role ~ '<|message|>' }} + {{- (message.content | string) ~ '<|end|>' }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start|>assistant<|channel|>analysis<|message|>' }} +{%- endif %}`, + chat_template_caps: { + supports_preserve_reasoning: true, + }, +}; diff --git a/src/llm/model-profile.test.ts b/src/llm/model-profile.test.ts index f22b1689..04210571 100644 --- a/src/llm/model-profile.test.ts +++ b/src/llm/model-profile.test.ts @@ -5,7 +5,6 @@ import { extractTotalSlots, detectVisionSupport, GEMMA4_THINK_PROFILE, - NEMOTRON_THINK_PROFILE, PLAIN_INSTRUCT_PROFILE, QWEN_THINK_PROFILE, } from "./model-profile.js"; @@ -51,15 +50,22 @@ describe("detectModelProfile", () => { expect(detectModelProfile(GEMMA4_PROPS)).toEqual(GEMMA4_THINK_PROFILE); }); - it("detects nemotron think profile from ChatML + enable_thinking", () => { - expect(detectModelProfile(NEMOTRON_PROPS)).toEqual(NEMOTRON_THINK_PROFILE); + // Nemotron has no profile of its own: its ChatML template is qwen-shaped, + // so the dedicated detector deliberately maps onto QWEN_THINK_PROFILE. The + // contract under test is that the Nemotron template yields the think-tags + // reasoning profile at all — deleting the branch drops it to plain-instruct + // and silently kills the reasoning channel. + it("maps the nemotron ChatML + enable_thinking template onto the think-tags profile", () => { + const profile = detectModelProfile(NEMOTRON_PROPS); + expect(profile).toEqual(QWEN_THINK_PROFILE); + expect(profile.reasoningStyle).toBe("think-tags"); }); - it("does not classify a non-nemotron ChatML think template as nemotron", () => { - // Same markers as Nemotron (ChatML + + enable_thinking) but the - // alias is not nemotron — must stay on the qwen path (or plain if the - // qwen alias hint also fails). Here the alias carries none of the - // qwen/nemotron hints, so the result is plain-instruct. + // Pins the alias gate: the Nemotron detector must require a `nemotron` + // alias, not fire on the template markers alone. An alias carrying none of + // the qwen/qwq/deepseek-r1/nemotron hints must fall through to plain even + // though the template is a full ChatML + + enable_thinking match. + it("requires a nemotron alias — template markers alone do not classify", () => { expect( detectModelProfile({ ...NEMOTRON_PROPS, @@ -68,6 +74,23 @@ describe("detectModelProfile", () => { ).toEqual(PLAIN_INSTRUCT_PROFILE); }); + // Pins branch ordering. This alias satisfies BOTH gates (it contains + // "qwen" and "nemotron"), which is the only input where the order of the + // two branches is observable: whichever runs first decides. The qwen + // branch runs first, so the qwen gate must win. Both branches currently + // yield QWEN_THINK_PROFILE, so this is pinned on the gate that fired + // rather than on the returned object. + it("lets the qwen gate win when an alias matches both the qwen and nemotron hints", () => { + const alias = "qwen-nemotron-hybrid-think"; + // Guard: the alias really does trip both gates, so the assertion below + // is about ordering and not about one gate quietly failing to match. + expect(alias).toContain("qwen"); + expect(alias).toContain("nemotron"); + expect(detectModelProfile({ ...NEMOTRON_PROPS, model_alias: alias })).toEqual( + QWEN_THINK_PROFILE, + ); + }); + it("falls back to plain profile for gpt-oss style templates", () => { expect(detectModelProfile(GPT_OSS_PROPS)).toEqual(PLAIN_INSTRUCT_PROFILE); }); diff --git a/src/llm/model-profile.ts b/src/llm/model-profile.ts index cd8d8a35..aa3059a7 100644 --- a/src/llm/model-profile.ts +++ b/src/llm/model-profile.ts @@ -61,7 +61,7 @@ export interface ReasoningTurnFraming { } export interface TaggedReasoningModelProfile extends BaseModelProfile { - id: "qwen-think" | "gemma4-think" | "nemotron-think"; + id: "qwen-think" | "gemma4-think"; reasoningStyle: "think-tags" | "channel-tags"; reasoningOpenTag: string; reasoningCloseTag: string; @@ -130,21 +130,6 @@ export const QWEN_THINK_PROFILE: TaggedReasoningModelProfile = { vision: VISION_ABSENT, }; -/** - * Nemotron 3.5 Lightning uses ChatML with an `enable_thinking` flag and - * prefills `` at the generation point — same open/close ownership as - * `qwen-think`, without Gemma-style turn framing. - */ -export const NEMOTRON_THINK_PROFILE: TaggedReasoningModelProfile = { - id: "nemotron-think", - reasoningStyle: "think-tags", - reasoningOpenTag: "", - reasoningCloseTag: "", - requiresPromptThinkPrefix: true, - allowThinkPrelude: true, - vision: VISION_ABSENT, -}; - export const GEMMA4_THINK_PROFILE: TaggedReasoningModelProfile = { id: "gemma4-think", reasoningStyle: "channel-tags", @@ -240,8 +225,15 @@ function selectBaseProfile( if (looksLikeQwenThinkModel(modelAlias, templateLower, supportsPreserveReasoning)) { return QWEN_THINK_PROFILE; } + // Nemotron needs its own detector but not its own profile. Its ChatML + // template is qwen-shaped — ``/`` prefilled at the + // generation point, same open/close ownership, no turn framing — so the + // runtime behaviour is byte-for-byte `QWEN_THINK_PROFILE`. Only the alias + // gate differs: `looksLikeQwenThinkModel` requires a qwen/qwq/deepseek-r1 + // alias, which Nemotron's does not match, so without this branch it would + // fall through to `plain-instruct` and lose its reasoning channel. if (looksLikeNemotronThinkModel(modelAlias, templateLower)) { - return NEMOTRON_THINK_PROFILE; + return QWEN_THINK_PROFILE; } if (looksLikeGemma4ThinkModel(modelAlias, templateLower)) { return GEMMA4_THINK_PROFILE; diff --git a/src/llm/profile-invariants.ts b/src/llm/profile-invariants.ts index 62d4e04a..9f96df3b 100644 --- a/src/llm/profile-invariants.ts +++ b/src/llm/profile-invariants.ts @@ -1,6 +1,5 @@ import { GEMMA4_THINK_PROFILE, - NEMOTRON_THINK_PROFILE, QWEN_THINK_PROFILE, getReasoningTurnFraming, type ModelProfile, @@ -79,7 +78,6 @@ function escapeGrammarLiteral(text: string): string { function getKnownReasoningOpenTags(): string[] { return [ QWEN_THINK_PROFILE.reasoningOpenTag.trimEnd(), - NEMOTRON_THINK_PROFILE.reasoningOpenTag.trimEnd(), GEMMA4_THINK_PROFILE.reasoningOpenTag.trimEnd(), ]; } diff --git a/src/local-llm/backend-installer.test.ts b/src/local-llm/backend-installer.test.ts index 0f63e357..6a6822c7 100644 --- a/src/local-llm/backend-installer.test.ts +++ b/src/local-llm/backend-installer.test.ts @@ -1,13 +1,51 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import JSZip from "jszip"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { downloadBackend, isBackendDownloaded } from "./backend-installer.js"; +import { + checkForBackendUpdate, + downloadBackend, + isBackendDownloaded, + resetLatestReleaseCache, +} from "./backend-installer.js"; import { resolveServerBinPath } from "./backend-paths.js"; -import { readBackendVersion } from "./backend-version.js"; +import { readBackendVersion, writeBackendVersion } from "./backend-version.js"; + +/** Minimal GitHub releases-list payload for the macOS arm64 asset. */ +function releasesResponse( + releases: Array<{ + tag: string; + url?: string; + publishedAt?: string | null; + assetName?: string; + }>, +): Response { + return new Response( + JSON.stringify( + releases.map((r) => ({ + tag_name: r.tag, + published_at: r.publishedAt === undefined ? null : r.publishedAt, + assets: [ + { + name: r.assetName ?? "llama-turboquant-macos-arm64.zip", + browser_download_url: r.url ?? "https://example.com/asset.zip", + }, + ], + })), + ), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} describe("backend-installer", () => { let dir: string; @@ -16,10 +54,12 @@ describe("backend-installer", () => { beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "local-llm-be-")); prevFetch = globalThis.fetch; + resetLatestReleaseCache(); }); afterEach(() => { globalThis.fetch = prevFetch; + resetLatestReleaseCache(); rmSync(dir, { recursive: true, force: true }); }); @@ -180,6 +220,323 @@ describe("backend-installer", () => { } }); + it("keeps the working install when the download fails mid-flight", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + // Pre-existing, working install. + const backendDir = join(dir, "backend"); + mkdirSync(backendDir, { recursive: true }); + writeFileSync(join(backendDir, "llama-server"), "#!/bin/sh\necho old\n", { + mode: 0o755, + }); + writeBackendVersion(dir, { + tag: "turboquant-old", + downloadedAt: "2026-01-01T00:00:00.000Z", + asset: "llama-turboquant-macos-arm64.zip", + }); + + globalThis.fetch = vi.fn(async (url: string | URL) => { + const u = String(url); + if (u.includes("/releases")) { + return releasesResponse([ + { tag: "turboquant-new", publishedAt: "2026-02-01T00:00:00Z" }, + ]); + } + // Asset download dies part-way through, as a dropped connection does. + throw new Error("socket hang up"); + }) as typeof fetch; + + try { + await expect(downloadBackend(dir)).rejects.toThrow(/socket hang up/); + + const binPath = resolveServerBinPath(dir, "llama-server"); + expect(existsSync(binPath)).toBe(true); + expect(readFileSync(binPath, "utf-8").includes("echo old")).toBe(true); + expect(isBackendDownloaded(dir)).toBe(true); + // The version record must still describe the install that is live. + expect(readBackendVersion(dir)?.tag).toBe("turboquant-old"); + // No staging leftovers. + expect(existsSync(`${join(dir, "backend")}.next`)).toBe(false); + expect(existsSync(`${join(dir, "backend")}.old`)).toBe(false); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("keeps the working install when the archive has no server binary", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + const backendDir = join(dir, "backend"); + mkdirSync(backendDir, { recursive: true }); + writeFileSync(join(backendDir, "llama-server"), "#!/bin/sh\necho old\n", { + mode: 0o755, + }); + + // Well-formed zip, but it ships the wrong payload — the corrupt / + // mis-built release case. + const zip = new JSZip(); + zip.file("release-root/README.md", Buffer.from("no binary here")); + const zipBuf = await zip.generateAsync({ type: "nodebuffer" }); + + globalThis.fetch = vi.fn(async (url: string | URL) => { + const u = String(url); + if (u.includes("/releases")) { + return releasesResponse([ + { tag: "turboquant-broken", publishedAt: "2026-02-01T00:00:00Z" }, + ]); + } + return new Response(zipBuf, { + status: 200, + headers: { "content-length": String(zipBuf.length) }, + }); + }) as typeof fetch; + + try { + await expect(downloadBackend(dir)).rejects.toThrow(/not found after extract/); + const binPath = resolveServerBinPath(dir, "llama-server"); + expect(readFileSync(binPath, "utf-8").includes("echo old")).toBe(true); + expect(existsSync(`${join(dir, "backend")}.next`)).toBe(false); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("replaces a stale staging dir left by a previous crash", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + // Crash leftovers: a half-extracted `.next` carrying a foreign + // wrapper dir that would poison the flatten step, and a `.old`. + const stagingDir = join(dir, "backend.next"); + mkdirSync(join(stagingDir, "build", "bin"), { recursive: true }); + writeFileSync(join(stagingDir, "build", "bin", "llama-server"), "stale"); + mkdirSync(join(dir, "backend.old"), { recursive: true }); + + const zip = new JSZip(); + zip.file("llama-server", Buffer.from("#!/bin/sh\necho fresh\n")); + const zipBuf = await zip.generateAsync({ type: "nodebuffer" }); + + globalThis.fetch = vi.fn(async (url: string | URL) => { + const u = String(url); + if (u.includes("/releases")) { + return releasesResponse([ + { tag: "turboquant-fresh", publishedAt: "2026-02-01T00:00:00Z" }, + ]); + } + return new Response(zipBuf, { + status: 200, + headers: { "content-length": String(zipBuf.length) }, + }); + }) as typeof fetch; + + try { + await downloadBackend(dir); + const binPath = resolveServerBinPath(dir, "llama-server"); + expect(readFileSync(binPath, "utf-8").includes("echo fresh")).toBe(true); + expect(existsSync(join(dir, "backend", "build"))).toBe(false); + expect(existsSync(stagingDir)).toBe(false); + expect(existsSync(join(dir, "backend.old"))).toBe(false); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("records the release timestamp so later checks can order against it", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + const zip = new JSZip(); + zip.file("llama-server", Buffer.from("#!/bin/sh\necho ok\n")); + const zipBuf = await zip.generateAsync({ type: "nodebuffer" }); + + globalThis.fetch = vi.fn(async (url: string | URL) => { + const u = String(url); + if (u.includes("/releases")) { + return releasesResponse([ + { tag: "turboquant-dated", publishedAt: "2026-02-03T04:05:06Z" }, + ]); + } + return new Response(zipBuf, { + status: 200, + headers: { "content-length": String(zipBuf.length) }, + }); + }) as typeof fetch; + + try { + await downloadBackend(dir); + expect(readBackendVersion(dir)?.releasedAt).toBe("2026-02-03T04:05:06Z"); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("does not downgrade when a re-published older tag heads the list", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + writeBackendVersion(dir, { + tag: "turboquant-june", + downloadedAt: "2026-06-02T00:00:00.000Z", + asset: "llama-turboquant-macos-arm64.zip", + releasedAt: "2026-06-01T00:00:00Z", + }); + // A maintainer re-published the old January release, so GitHub's + // created_at ordering puts it first. Its own timestamp is still older. + globalThis.fetch = vi.fn(async () => + releasesResponse([ + { tag: "turboquant-january", publishedAt: "2026-01-01T00:00:00Z" }, + { tag: "turboquant-june", publishedAt: "2026-06-01T00:00:00Z" }, + ]), + ) as typeof fetch; + + try { + const check = await checkForBackendUpdate(dir); + expect(check.updateAvailable).toBe(false); + expect(check.latestTag).toBe("turboquant-june"); + expect(check.currentTag).toBe("turboquant-june"); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("does not downgrade when the newest available release predates the install", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + writeBackendVersion(dir, { + tag: "turboquant-june", + downloadedAt: "2026-06-02T00:00:00.000Z", + asset: "llama-turboquant-macos-arm64.zip", + releasedAt: "2026-06-01T00:00:00Z", + }); + // The June release was deleted from the repo; the newest one still + // listed is older than what this machine already runs. + globalThis.fetch = vi.fn(async () => + releasesResponse([ + { tag: "turboquant-may", publishedAt: "2026-05-01T00:00:00Z" }, + { tag: "turboquant-april", publishedAt: "2026-04-01T00:00:00Z" }, + ]), + ) as typeof fetch; + + try { + const check = await checkForBackendUpdate(dir); + expect(check.updateAvailable).toBe(false); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("still updates when the resolved release is genuinely newer", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + writeBackendVersion(dir, { + tag: "turboquant-june", + downloadedAt: "2026-06-02T00:00:00.000Z", + asset: "llama-turboquant-macos-arm64.zip", + releasedAt: "2026-06-01T00:00:00Z", + }); + globalThis.fetch = vi.fn(async () => + releasesResponse([ + { tag: "turboquant-july", publishedAt: "2026-07-01T00:00:00Z" }, + { tag: "turboquant-june", publishedAt: "2026-06-01T00:00:00Z" }, + ]), + ) as typeof fetch; + + try { + const check = await checkForBackendUpdate(dir); + expect(check.updateAvailable).toBe(true); + expect(check.latestTag).toBe("turboquant-july"); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("updates on a variant change even though the tag is unchanged", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("x64"); + // Installed the Vulkan build; the machine now warrants CUDA. Same + // tag, same timestamp — recency must not veto the variant re-pull. + writeBackendVersion(dir, { + tag: "turboquant-win", + downloadedAt: "2026-06-02T00:00:00.000Z", + asset: "llama-turboquant-windows-x64-cuda-13.3.zip", + releasedAt: "2026-06-01T00:00:00Z", + }); + globalThis.fetch = vi.fn(async () => + releasesResponse([ + { + tag: "turboquant-win", + publishedAt: "2026-06-01T00:00:00Z", + assetName: "llama-turboquant-windows-x64-vulkan.zip", + }, + ]), + ) as typeof fetch; + + try { + const check = await checkForBackendUpdate(dir); + expect(check.updateAvailable).toBe(true); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("treats a page-1 miss for this platform as 'no update', not an error", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + writeBackendVersion(dir, { + tag: "turboquant-installed", + downloadedAt: "2026-06-02T00:00:00.000Z", + asset: "llama-turboquant-macos-arm64.zip", + }); + // Page 1 is all Windows releases — the macOS asset fell off the end. + globalThis.fetch = vi.fn(async () => + releasesResponse([ + { + tag: "turboquant-windows-9", + publishedAt: "2026-07-01T00:00:00Z", + assetName: "llama-turboquant-windows-x64-vulkan.zip", + }, + ]), + ) as typeof fetch; + + try { + const check = await checkForBackendUpdate(dir); + expect(check.updateAvailable).toBe(false); + expect(check.latestTag).toBeNull(); + expect(check.currentTag).toBe("turboquant-installed"); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + + it("bounds the releases request with a timeout signal", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const archSpy = vi.spyOn(process, "arch", "get").mockReturnValue("arm64"); + let seenSignal: AbortSignal | undefined; + globalThis.fetch = vi.fn(async (_url: string | URL, init?: RequestInit) => { + seenSignal = init?.signal ?? undefined; + return releasesResponse([ + { tag: "turboquant-x", publishedAt: "2026-07-01T00:00:00Z" }, + ]); + }) as typeof fetch; + + try { + await checkForBackendUpdate(dir); + // A black-holed connection must not hang the start path until the + // OS TCP timeout, so the request has to carry an abort signal. + expect(seenSignal).toBeInstanceOf(AbortSignal); + expect(seenSignal?.aborted).toBe(false); + } finally { + platformSpy.mockRestore(); + archSpy.mockRestore(); + } + }); + it("isBackendDownloaded is false on unsupported platform (darwin x64)", () => { const platformSpy = vi .spyOn(process, "platform", "get") diff --git a/src/local-llm/backend-installer.ts b/src/local-llm/backend-installer.ts index 17b6730c..76271de8 100644 --- a/src/local-llm/backend-installer.ts +++ b/src/local-llm/backend-installer.ts @@ -1,22 +1,14 @@ -import { execSync } from "node:child_process"; -import { - chmodSync, - existsSync, - mkdirSync, - readdirSync, - readFileSync, - renameSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; -import { dirname, join, relative } from "node:path"; - -import JSZip from "jszip"; +import { existsSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; import { resolveBackendDir, resolveServerBinPath } from "./backend-paths.js"; +import { + extractBackendArchive, + rmDirQuiet, + swapInStagedBackend, +} from "./backend-staging.js"; import { downloadFile } from "./download-file.js"; -import { readBackendVersion, writeBackendVersion } from "./backend-version.js"; +import { readBackendVersion, writeBackendVersionAt } from "./backend-version.js"; import { resolvePlatformAsset, UnsupportedPlatformError } from "./platform-assets.js"; import { resolveDownloadAsset } from "./windows-backend-variant.js"; @@ -41,16 +33,36 @@ const RELEASE_CACHE_TTL_MS = 10 * 60_000; */ const RELEASES_PER_PAGE = 30; +/** + * Timeout for the small releases-list JSON request. With auto-update on + * this call sits on the critical path of every managed start, and a + * black-holed connection (captive portal, DNS sinkhole) would otherwise + * hang until the OS TCP timeout — ~130s on Linux — before the daemon + * even begins to boot. Aborting at 5s turns that into a normal check + * failure and the caller starts the binary already on disk. Only the + * JSON request is bounded; the multi-hundred-MB asset download keeps + * the caller's own `opts.signal`. + */ +const RELEASES_FETCH_TIMEOUT_MS = 5_000; + export type ReleaseAsset = { name: string; browser_download_url: string }; export interface LatestReleaseInfo { tag: string; assets: ReleaseAsset[]; + /** + * `published_at` (or `created_at` when a release was never published) + * as ISO-8601, or null when GitHub omitted both. Used to order this + * release against the installed one — the repo is a nightly, and its + * tags are not semver-sortable. + */ + releasedAt: string | null; } interface ReleaseCacheEntry { fetchedAt: number; - release: LatestReleaseInfo; + /** `null` = scanned successfully, no release carries our asset. */ + release: LatestReleaseInfo | null; } /** @@ -66,9 +78,13 @@ export function resetLatestReleaseCache(): void { releaseCache.clear(); } +/** + * Resolve the newest release that ships this platform's asset, or + * `null` when none of the scanned releases carries it. + */ export async function fetchLatestRelease(opts?: { force?: boolean; -}): Promise { +}): Promise { const { assetName } = resolveDownloadAsset(); const cached = releaseCache.get(assetName); if ( @@ -86,7 +102,10 @@ export async function fetchLatestRelease(opts?: { const token = (process.env.GITHUB_TOKEN || process.env.GH_TOKEN || "").trim(); if (token) headers.Authorization = `Bearer ${token}`; - const res = await fetch(url, { headers }); + const res = await fetch(url, { + headers, + signal: AbortSignal.timeout(RELEASES_FETCH_TIMEOUT_MS), + }); if (!res.ok) { if (res.status === 403 || res.status === 429) { throw new GithubRateLimitedError(res.status); @@ -96,26 +115,52 @@ export async function fetchLatestRelease(opts?: { const data = (await res.json()) as Array<{ tag_name: string; draft?: boolean; + published_at?: string | null; + created_at?: string | null; assets: Array<{ name: string; browser_download_url: string }>; }>; - // GitHub lists releases newest-first; pick the first (non-draft) whose - // assets include the asset for the current platform. - const match = (Array.isArray(data) ? data : []).find( + // GitHub lists releases newest-first *by creation*, which is not the + // same as newest-published: re-publishing or backfilling an old tag + // moves it. Collect every non-draft release carrying our platform + // asset and pick the one with the newest release timestamp, falling + // back to GitHub's own order when timestamps are missing. + const candidates = (Array.isArray(data) ? data : []).filter( (r) => !r.draft && (r.assets ?? []).some((a) => a.name === assetName), ); - if (!match) { - throw new Error( - `No release found containing asset ${assetName} (scanned ${RELEASES_PER_PAGE} releases)`, - ); + if (candidates.length === 0) { + // A rarely-built platform's newest asset can fall off page 1 of the + // list. That is "nothing to update to", not a hard error: throwing + // here would fail the check on every single start and, with the + // pre-staging installer, was the only thing standing between the + // user and a working binary already on disk. + releaseCache.set(assetName, { fetchedAt: Date.now(), release: null }); + return null; } + const match = candidates.reduce((best, cur) => + releaseTime(cur) > releaseTime(best) ? cur : best, + ); const release: LatestReleaseInfo = { tag: match.tag_name, assets: match.assets ?? [], + releasedAt: match.published_at ?? match.created_at ?? null, }; releaseCache.set(assetName, { fetchedAt: Date.now(), release }); return release; } +/** + * Sort key for release recency. `-Infinity` for a release with no + * usable timestamp so it never displaces a dated one; ties keep the + * earlier (GitHub-ordered) candidate because `reduce` only swaps on a + * strict improvement. + */ +function releaseTime(r: { published_at?: string | null; created_at?: string | null }): number { + const raw = r.published_at ?? r.created_at; + if (!raw) return -Infinity; + const t = Date.parse(raw); + return Number.isNaN(t) ? -Infinity : t; +} + export class GithubRateLimitedError extends Error { constructor(public readonly status: number) { super( @@ -127,110 +172,62 @@ export class GithubRateLimitedError extends Error { export async function checkForBackendUpdate( dataDir: string, -): Promise<{ updateAvailable: boolean; latestTag: string; currentTag: string | null }> { +): Promise<{ + updateAvailable: boolean; + latestTag: string | null; + currentTag: string | null; +}> { const current = readBackendVersion(dataDir); const release = await fetchLatestRelease(); // A variant mismatch counts as an update even at the same tag: a // Windows box that installed the Vulkan build before its NVIDIA driver // was present would otherwise keep running Vulkan (and offloading to - // whatever device Vulkan enumerates) forever. + // whatever device Vulkan enumerates) forever. This is a property of + // the local machine, not of release ordering, so it is checked before + // (and independently of) the recency comparison. const variantStale = current?.asset !== undefined && current.asset !== resolveDownloadAsset().assetName; + if (release === null) { + // Nothing resolvable to update *to* — keep whatever is installed. + return { + updateAvailable: false, + latestTag: null, + currentTag: current?.tag ?? null, + }; + } return { - updateAvailable: current?.tag !== release.tag || variantStale, + updateAvailable: variantStale || isNewerRelease(current, release), latestTag: release.tag, currentTag: current?.tag ?? null, }; } -function normalizeZipPath(entryName: string): string { - return entryName.replace(/\\/g, "/"); -} - -/** - * Recursively walk `root`, return the absolute path to the first file - * whose basename equals `name`. Used after zip extraction to find the - * `llama-server` binary regardless of the archive's internal nesting - * (some releases wrap it under `build/bin/`, others under a single - * top-level folder, others drop it at the root). - */ -function findFileByName(root: string, name: string): string | null { - const stack = [root]; - while (stack.length) { - const cur = stack.pop()!; - let entries: string[]; - try { - entries = readdirSync(cur); - } catch { - continue; - } - for (const entry of entries) { - const full = join(cur, entry); - let st; - try { - st = statSync(full); - } catch { - continue; - } - if (st.isDirectory()) { - stack.push(full); - } else if (entry === name) { - return full; - } - } - } - return null; -} - -/** - * Move every file in `from` (recursively) into `to`, flattening into - * siblings at `to`'s root. Used to promote `backend/build/bin/*` or - * `backend/release-root/*` up to `backend/` after extraction so the - * `llama-server` binary lives at the path `resolveServerBinPath` - * expects. Existing files at the destination are overwritten. - */ -function moveContentsFlat(from: string, to: string): void { - const walk = (dir: string): void => { - const entries = readdirSync(dir); - for (const entry of entries) { - const src = join(dir, entry); - const st = statSync(src); - if (st.isDirectory()) { - walk(src); - continue; - } - const dst = join(to, entry); - mkdirSync(dirname(dst), { recursive: true }); - try { - renameSync(src, dst); - } catch { - // Cross-device or other rename failure — fall back to copy+unlink. - writeFileSync(dst, readFileSync(src)); - try { - rmSync(src, { force: true }); - } catch { - /* ignore */ - } - } - } - }; - walk(from); -} - /** - * Return the first path segment under `backendRoot` leading to - * `fileInside` — e.g. for `backendRoot=/.../backend` and - * `fileInside=/.../backend/build/bin/llama-server` this returns - * `/.../backend/build`. Used after the flatten step to delete the - * now-empty wrapper tree. + * Is `release` genuinely newer than what is installed? + * + * A bare `current.tag !== release.tag` also fires when the resolved + * release is *older* — which happens for real on a nightly repo whose + * tags are not semver-sortable: re-publishing or backfilling a tag + * moves it, and every client would silently downgrade on next start. + * Two contending releases would additionally re-download hundreds of MB + * and bounce the daemon on *every* start. + * + * Release timestamps are the only defensible ordering available here, + * so when both sides carry one we require a strict increase. When + * either is missing — no install yet, or a version file written before + * `releasedAt` existed — we fall back to tag inequality so those users + * still converge onto the current build once. */ -function topLevelWrapper(backendRoot: string, fileInside: string): string | null { - const rel = relative(backendRoot, fileInside); - // `relative` yields platform-native separators: `/` on POSIX, `\` on - // Windows. Match either so the wrapper dir is cleaned up on both. - const firstSep = rel.search(/[/\\]/); - if (firstSep < 0) return null; - return join(backendRoot, rel.slice(0, firstSep)); +function isNewerRelease( + current: { tag: string; releasedAt?: string } | null, + release: LatestReleaseInfo, +): boolean { + if (current === null) return true; + if (current.tag === release.tag) return false; + const currentAt = current.releasedAt ? Date.parse(current.releasedAt) : NaN; + const releaseAt = release.releasedAt ? Date.parse(release.releasedAt) : NaN; + if (Number.isNaN(currentAt) || Number.isNaN(releaseAt)) return true; + return releaseAt > currentAt; } export function isBackendDownloaded(dataDir: string): boolean { @@ -253,6 +250,11 @@ export async function downloadBackend( // Always hit GitHub for an actual install so we don't grab a stale // tag from the snapshot cache. const release = await fetchLatestRelease({ force: true }); + if (release === null) { + throw new Error( + `No release found containing asset ${assetName} (scanned ${RELEASES_PER_PAGE} releases)`, + ); + } const asset = release.assets.find((a) => a.name === assetName); if (!asset) { const known = release.assets.map((a) => a.name).join(", "); @@ -262,94 +264,49 @@ export async function downloadBackend( } const backendDir = resolveBackendDir(dataDir); - // Wipe any leftovers from a previous failed download so the flatten - // step can't pick up stale `bin/` or `build/` wrappers. Done before - // mkdir so a missing dir is fine. - try { - rmSync(backendDir, { recursive: true, force: true }); - } catch { - /* ignore */ - } - mkdirSync(backendDir, { recursive: true }); - const archivePath = join(backendDir, assetName); - - await downloadFile(asset.browser_download_url, archivePath, { - onProgress: opts?.onProgress, - userAgent: "atomic-agent/local-llm-backend-download", - signal: opts?.signal, - }); - - const zip = await JSZip.loadAsync(readFileSync(archivePath)); - // Extract preserving the archive's internal layout. Flattening happens - // in a second pass so we can support any of: - // * `llama-server` (flat) - // * `release-root/llama-server` (single top folder) - // * `build/bin/llama-server` (nested) - for (const entry of Object.values(zip.files)) { - if (entry.dir) continue; - const rel = normalizeZipPath(entry.name); - if (!rel || rel.endsWith("/")) continue; - const out = join(backendDir, rel); - mkdirSync(dirname(out), { recursive: true }); - const buf = await entry.async("nodebuffer"); - writeFileSync(out, buf); - try { - chmodSync(out, 0o755); - } catch { - /* Windows may ignore chmod */ - } - } + // Download and extract into a sibling staging dir, never into the + // live one. The old code wiped `backend/` first and only then pulled + // several hundred MB, so a network drop, a Ctrl-C, a corrupt zip or a + // full disk left the user with no backend at all — and with + // auto-update this path now runs on every managed start, not just an + // explicit `models update`. Siblings (not tmpdir) so the final swap + // stays on one filesystem and can be a rename. + const stagingDir = `${backendDir}.next`; + const retiredDir = `${backendDir}.old`; + // A previous crash can leave either behind; both are scratch, and a + // stale `.next` would otherwise poison the flatten step with foreign + // `bin/` or `build/` wrappers. + rmDirQuiet(stagingDir); + rmDirQuiet(retiredDir); + mkdirSync(stagingDir, { recursive: true }); - const foundBin = findFileByName(backendDir, binaryName); - if (!foundBin) { - throw new Error( - `llama-server binary not found after extract (searched for ${binaryName} under ${backendDir})`, - ); - } - const expectedBin = resolveServerBinPath(dataDir, binaryName); - if (foundBin !== expectedBin) { - // Promote the binary's parent directory contents into `backendDir` - // so `llama-server` (and any sibling shared libs) sit at the path - // the daemon lifecycle expects, then remove the now-orphaned - // wrapper directories (e.g. `build/`, `release-root/`). - moveContentsFlat(dirname(foundBin), backendDir); - const topWrapper = topLevelWrapper(backendDir, foundBin); - if (topWrapper !== null) { - try { - rmSync(topWrapper, { recursive: true, force: true }); - } catch { - /* ignore — stray files, not fatal */ - } - } - } + try { + const archivePath = join(stagingDir, assetName); + await downloadFile(asset.browser_download_url, archivePath, { + onProgress: opts?.onProgress, + userAgent: "atomic-agent/local-llm-backend-download", + signal: opts?.signal, + }); - if (process.platform === "darwin") { - try { - execSync(`xattr -cr "${backendDir}"`, { timeout: 10_000 }); - } catch { - /* xattr may fail */ - } - } + await extractBackendArchive(archivePath, stagingDir, binaryName); - rmSync(archivePath, { force: true }); + // The version record lives inside `backend/`, so it is staged with + // the rest of the tree and rides in on the swap. It therefore never + // describes anything other than what is actually live, and a + // failure before this point leaves the old record untouched. + writeBackendVersionAt(stagingDir, { + tag: release.tag, + downloadedAt: new Date().toISOString(), + asset: assetName, + ...(release.releasedAt ? { releasedAt: release.releasedAt } : {}), + }); - if (!existsSync(expectedBin)) { - throw new Error( - `llama-server binary not found after extract + flatten (expected ${binaryName} at ${expectedBin}; ` + - `original location was ${relative(backendDir, foundBin)})`, - ); + swapInStagedBackend(backendDir, stagingDir, retiredDir); + } catch (err) { + // Leave the existing install exactly as it was. + rmDirQuiet(stagingDir); + throw err; } - try { - chmodSync(expectedBin, 0o755); - } catch { - /* Windows */ - } - - writeBackendVersion(dataDir, { - tag: release.tag, - downloadedAt: new Date().toISOString(), - asset: assetName, - }); return { ok: true, tag: release.tag }; } diff --git a/src/local-llm/backend-staging.ts b/src/local-llm/backend-staging.ts new file mode 100644 index 00000000..f8550eac --- /dev/null +++ b/src/local-llm/backend-staging.ts @@ -0,0 +1,267 @@ +import { execSync } from "node:child_process"; +import { + accessSync, + chmodSync, + constants, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, relative } from "node:path"; + +import JSZip from "jszip"; + +/** + * Filesystem side of a backend install: unpack an archive into a + * staging directory, normalise the layout, and swap it over the live + * one only once it is complete and usable. + * + * Kept separate from `backend-installer.ts` so the release-resolution + * logic there is not interleaved with extraction mechanics. + */ + +function normalizeZipPath(entryName: string): string { + return entryName.replace(/\\/g, "/"); +} + +/** + * Recursively walk `root`, return the absolute path to the first file + * whose basename equals `name`. Used after zip extraction to find the + * `llama-server` binary regardless of the archive's internal nesting + * (some releases wrap it under `build/bin/`, others under a single + * top-level folder, others drop it at the root). + */ +function findFileByName(root: string, name: string): string | null { + const stack = [root]; + while (stack.length) { + const cur = stack.pop()!; + let entries: string[]; + try { + entries = readdirSync(cur); + } catch { + continue; + } + for (const entry of entries) { + const full = join(cur, entry); + let st; + try { + st = statSync(full); + } catch { + continue; + } + if (st.isDirectory()) { + stack.push(full); + } else if (entry === name) { + return full; + } + } + } + return null; +} + +/** + * Move every file in `from` (recursively) into `to`, flattening into + * siblings at `to`'s root. Used to promote `backend/build/bin/*` or + * `backend/release-root/*` up to `backend/` after extraction so the + * `llama-server` binary lives at the path `resolveServerBinPath` + * expects. Existing files at the destination are overwritten. + */ +function moveContentsFlat(from: string, to: string): void { + const walk = (dir: string): void => { + const entries = readdirSync(dir); + for (const entry of entries) { + const src = join(dir, entry); + const st = statSync(src); + if (st.isDirectory()) { + walk(src); + continue; + } + const dst = join(to, entry); + mkdirSync(dirname(dst), { recursive: true }); + try { + renameSync(src, dst); + } catch { + // Cross-device or other rename failure — fall back to copy+unlink. + writeFileSync(dst, readFileSync(src)); + try { + rmSync(src, { force: true }); + } catch { + /* ignore */ + } + } + } + }; + walk(from); +} + +/** + * Return the first path segment under `backendRoot` leading to + * `fileInside` — e.g. for `backendRoot=/.../backend` and + * `fileInside=/.../backend/build/bin/llama-server` this returns + * `/.../backend/build`. Used after the flatten step to delete the + * now-empty wrapper tree. + */ +function topLevelWrapper(backendRoot: string, fileInside: string): string | null { + const rel = relative(backendRoot, fileInside); + // `relative` yields platform-native separators: `/` on POSIX, `\` on + // Windows. Match either so the wrapper dir is cleaned up on both. + const firstSep = rel.search(/[/\\]/); + if (firstSep < 0) return null; + return join(backendRoot, rel.slice(0, firstSep)); +} + +/** + * Remove `dir` if it exists, ignoring failures. Used for staging / + * rollback scratch dirs where a leftover is a nuisance, not a fault. + */ +export function rmDirQuiet(dir: string): void { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* ignore */ + } +} + +/** + * Is the staged tree a usable install? Guards the swap: only an + * extraction that actually produced an executable server binary at the + * path the daemon launches is allowed to replace a working one. + */ +function stagedBinaryUsable(binPath: string): boolean { + try { + const st = statSync(binPath); + if (!st.isFile() || st.size === 0) return false; + } catch { + return false; + } + if (process.platform === "win32") return true; + try { + accessSync(binPath, constants.X_OK); + return true; + } catch { + return false; + } +} + +/** + * Extract `archivePath` into `stagingDir` and normalise the layout so + * `binaryName` ends up at the staging root, executable. Throws when the + * archive does not yield a usable server binary — the caller is + * expected to discard the staging dir and keep the previous install. + */ +export async function extractBackendArchive( + archivePath: string, + stagingDir: string, + binaryName: string, +): Promise { + const zip = await JSZip.loadAsync(readFileSync(archivePath)); + // Extract preserving the archive's internal layout. Flattening + // happens in a second pass so we can support any of: + // * `llama-server` (flat) + // * `release-root/llama-server` (single top folder) + // * `build/bin/llama-server` (nested) + for (const entry of Object.values(zip.files)) { + if (entry.dir) continue; + const rel = normalizeZipPath(entry.name); + if (!rel || rel.endsWith("/")) continue; + const out = join(stagingDir, rel); + mkdirSync(dirname(out), { recursive: true }); + const buf = await entry.async("nodebuffer"); + writeFileSync(out, buf); + try { + chmodSync(out, 0o755); + } catch { + /* Windows may ignore chmod */ + } + } + + const foundBin = findFileByName(stagingDir, binaryName); + if (!foundBin) { + throw new Error( + `llama-server binary not found after extract (searched for ${binaryName} under ${stagingDir})`, + ); + } + const stagedBin = join(stagingDir, binaryName); + if (foundBin !== stagedBin) { + // Promote the binary's parent directory contents into the staging + // root so `llama-server` (and any sibling shared libs) sit at the + // path the daemon lifecycle expects once swapped in, then remove + // the now-orphaned wrapper dirs (e.g. `build/`, `release-root/`). + moveContentsFlat(dirname(foundBin), stagingDir); + const topWrapper = topLevelWrapper(stagingDir, foundBin); + if (topWrapper !== null) { + try { + rmSync(topWrapper, { recursive: true, force: true }); + } catch { + /* ignore — stray files, not fatal */ + } + } + } + + if (process.platform === "darwin") { + try { + execSync(`xattr -cr "${stagingDir}"`, { timeout: 10_000 }); + } catch { + /* xattr may fail */ + } + } + + rmSync(archivePath, { force: true }); + + if (!existsSync(stagedBin)) { + throw new Error( + `llama-server binary not found after extract + flatten (expected ${binaryName} at ${stagedBin}; ` + + `original location was ${relative(stagingDir, foundBin)})`, + ); + } + try { + chmodSync(stagedBin, 0o755); + } catch { + /* Windows */ + } + if (!stagedBinaryUsable(stagedBin)) { + throw new Error( + `staged llama-server at ${stagedBin} is not a usable executable — keeping the existing install`, + ); + } +} + +/** + * Replace `backendDir` with `stagingDir`. Two renames, which is the + * closest to atomic a directory swap gets on POSIX and Windows alike: + * the live dir is moved aside first so the second rename lands on a + * free name (`rename` onto a non-empty dir fails on both platforms). + * + * The window where neither dir is at the live path spans one rename of + * an already-materialised sibling — microseconds, and no I/O that can + * block on the network or the disk filling up. If the *second* rename + * still fails, the old install is rolled back so the caller is left + * with a working backend rather than none. + */ +export function swapInStagedBackend( + backendDir: string, + stagingDir: string, + retiredDir: string, +): void { + const hadLive = existsSync(backendDir); + if (hadLive) renameSync(backendDir, retiredDir); + try { + renameSync(stagingDir, backendDir); + } catch (err) { + if (hadLive) { + try { + renameSync(retiredDir, backendDir); + } catch { + /* rollback failed too — surface the original error */ + } + } + throw err; + } + rmDirQuiet(retiredDir); +} + diff --git a/src/local-llm/backend-version.ts b/src/local-llm/backend-version.ts index a06c624f..ac4d2ebb 100644 --- a/src/local-llm/backend-version.ts +++ b/src/local-llm/backend-version.ts @@ -1,5 +1,5 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname } from "node:path"; +import { dirname, join } from "node:path"; import { resolveVersionFilePath } from "./backend-paths.js"; @@ -16,6 +16,16 @@ export interface BackendVersionInfo { * installs predating this field. */ asset?: string; + /** + * `published_at` (falling back to `created_at`) of the GitHub release + * this install came from, ISO-8601. `checkForBackendUpdate` compares + * it against the resolved release so a re-published or backfilled + * older tag cannot present itself as an upgrade. Absent on installs + * predating this field, which is treated as "unknown, allow the + * tag-difference verdict to stand" so those users still get one more + * update. + */ + releasedAt?: string; } export function readBackendVersion(dataDir: string): BackendVersionInfo | null { @@ -28,7 +38,25 @@ export function readBackendVersion(dataDir: string): BackendVersionInfo | null { } export function writeBackendVersion(dataDir: string, info: BackendVersionInfo): void { - const p = resolveVersionFilePath(dataDir); + writeVersionFile(resolveVersionFilePath(dataDir), info); +} + +/** + * Write the version record into an arbitrary backend directory rather + * than the live one. The version file lives *inside* `backend/`, so a + * staged install must carry its own copy — writing it to the live path + * before the swap would describe a build that is not on disk yet, and + * writing it after would leave a window where the swapped-in binary is + * described by the previous tag. + */ +export function writeBackendVersionAt( + backendDir: string, + info: BackendVersionInfo, +): void { + writeVersionFile(join(backendDir, "backend-version.json"), info); +} + +function writeVersionFile(p: string, info: BackendVersionInfo): void { mkdirSync(dirname(p), { recursive: true }); writeFileSync(p, JSON.stringify(info, null, 2) + "\n"); } diff --git a/src/local-llm/ensure-latest-backend.test.ts b/src/local-llm/ensure-latest-backend.test.ts index 0d296749..41c67b8f 100644 --- a/src/local-llm/ensure-latest-backend.test.ts +++ b/src/local-llm/ensure-latest-backend.test.ts @@ -9,6 +9,7 @@ vi.mock("./backend-installer.js", async () => { ...actual, checkForBackendUpdate: vi.fn(), downloadBackend: vi.fn(), + isBackendDownloaded: vi.fn(), }; }); @@ -38,6 +39,7 @@ vi.mock("./session-registry.js", async () => { import { checkForBackendUpdate, downloadBackend, + isBackendDownloaded, } from "./backend-installer.js"; import { readRunningPid, @@ -54,6 +56,8 @@ describe("maybeAutoUpdateBackend", () => { vi.mocked(stopChatAndEmbeddingDaemons).mockReset(); vi.mocked(hasOtherLiveSessions).mockReset(); vi.mocked(hasOtherLiveSessions).mockReturnValue(false); + vi.mocked(isBackendDownloaded).mockReset(); + vi.mocked(isBackendDownloaded).mockReturnValue(true); }); it("is a no-op when autoUpdate is off", async () => { @@ -134,6 +138,67 @@ describe("maybeAutoUpdateBackend", () => { expect(downloadBackend).not.toHaveBeenCalled(); }); + it("folds a download failure into update_failed so start can continue", async () => { + vi.mocked(checkForBackendUpdate).mockResolvedValue({ + updateAvailable: true, + latestTag: "turboquant-new", + currentTag: "turboquant-old", + }); + vi.mocked(readRunningPid).mockReturnValue(4242); + vi.mocked(stopChatAndEmbeddingDaemons).mockResolvedValue(); + vi.mocked(downloadBackend).mockRejectedValue(new Error("socket hang up")); + // Staged install: the previous binary survives a failed download. + vi.mocked(isBackendDownloaded).mockReturnValue(true); + + // The daemon has already been stopped at this point, so throwing + // would leave the user with nothing running at all. + const result = await maybeAutoUpdateBackend("/tmp/data", { enabled: true }); + expect(result).toEqual({ + action: "update_failed", + error: "socket hang up", + backendUsable: true, + }); + expect(stopChatAndEmbeddingDaemons).toHaveBeenCalledWith("/tmp/data"); + }); + + it("reports backendUsable false when nothing is left to start", async () => { + vi.mocked(checkForBackendUpdate).mockResolvedValue({ + updateAvailable: true, + latestTag: "turboquant-new", + currentTag: null, + }); + vi.mocked(readRunningPid).mockReturnValue(null); + vi.mocked(downloadBackend).mockRejectedValue(new Error("disk full")); + vi.mocked(isBackendDownloaded).mockReturnValue(false); + + const result = await maybeAutoUpdateBackend("/tmp/data", { enabled: true }); + expect(result).toEqual({ + action: "update_failed", + error: "disk full", + backendUsable: false, + }); + }); + + it("folds a daemon-stop failure into update_failed rather than throwing", async () => { + vi.mocked(checkForBackendUpdate).mockResolvedValue({ + updateAvailable: true, + latestTag: "turboquant-new", + currentTag: "turboquant-old", + }); + vi.mocked(readRunningPid).mockReturnValue(4242); + vi.mocked(stopChatAndEmbeddingDaemons).mockRejectedValue( + new Error("kill EPERM"), + ); + + const result = await maybeAutoUpdateBackend("/tmp/data", { enabled: true }); + expect(result).toEqual({ + action: "update_failed", + error: "kill EPERM", + backendUsable: true, + }); + expect(downloadBackend).not.toHaveBeenCalled(); + }); + it("folds a GitHub check failure into check_failed so start can continue", async () => { vi.mocked(checkForBackendUpdate).mockRejectedValue( new Error("GitHub API rate-limited (HTTP 403)"), diff --git a/src/local-llm/ensure-latest-backend.ts b/src/local-llm/ensure-latest-backend.ts index 6a5610f2..f5bfaa1b 100644 --- a/src/local-llm/ensure-latest-backend.ts +++ b/src/local-llm/ensure-latest-backend.ts @@ -1,6 +1,7 @@ import { checkForBackendUpdate, downloadBackend, + isBackendDownloaded, } from "./backend-installer.js"; import type { DownloadProgressFn } from "./download-file.js"; import { @@ -14,14 +15,27 @@ export type AutoUpdateBackendResult = | { action: "current"; tag: string | null } | { action: "updated"; from: string | null; to: string } | { action: "deferred"; reason: "other_session" } - | { action: "check_failed"; error: string }; + | { action: "check_failed"; error: string } + /** + * The version check said "update", but stopping the daemon or + * downloading the replacement failed. `backendUsable` reports whether + * a server binary is still on disk: the staged installer keeps the + * previous install intact, so this is almost always true and the + * caller should start it. False means there is genuinely nothing to + * run and the caller must fail. + */ + | { action: "update_failed"; error: string; backendUsable: boolean }; /** * When `enabled`, pull a newer llama.cpp backend from GitHub Releases * before the managed daemon starts. Missing-backend first install is * still owned by the TUI/CLI start paths; this only upgrades an already - * installed zip. Check failures are fire-safe — the caller starts the - * current binary instead of aborting the turn. + * installed zip. Failures anywhere in the update are fire-safe: this + * never throws, and every non-fatal outcome leaves the caller free to + * start the binary already on disk instead of aborting the turn. That + * matters most *after* the daemon was stopped for the update — an + * exception there used to leave the user with nothing running, which is + * strictly worse than never having attempted the update. */ export async function maybeAutoUpdateBackend( dataDir: string, @@ -51,20 +65,28 @@ export async function maybeAutoUpdateBackend( // live pid. Stop both daemons first; the caller starts them after. // Skip the stop when another TUI/CLI session is live — killing their // model mid-chat is worse than sitting on an old tag until next solo start. - if (readRunningPid(dataDir) !== null) { - if (hasOtherLiveSessions(dataDir)) { - return { action: "deferred", reason: "other_session" }; + try { + if (readRunningPid(dataDir) !== null) { + if (hasOtherLiveSessions(dataDir)) { + return { action: "deferred", reason: "other_session" }; + } + await stopChatAndEmbeddingDaemons(dataDir); } - await stopChatAndEmbeddingDaemons(dataDir); - } - opts.onWillDownload?.(); - const downloaded = await downloadBackend(dataDir, { - onProgress: opts.onProgress, - }); - return { - action: "updated", - from: check.currentTag, - to: downloaded.tag, - }; + opts.onWillDownload?.(); + const downloaded = await downloadBackend(dataDir, { + onProgress: opts.onProgress, + }); + return { + action: "updated", + from: check.currentTag, + to: downloaded.tag, + }; + } catch (err) { + return { + action: "update_failed", + error: err instanceof Error ? err.message : String(err), + backendUsable: isBackendDownloaded(dataDir), + }; + } } diff --git a/src/local-llm/models-catalog.test.ts b/src/local-llm/models-catalog.test.ts index 7d8ac3b9..9afc93c6 100644 --- a/src/local-llm/models-catalog.test.ts +++ b/src/local-llm/models-catalog.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; +import { + detectModelProfile, + PLAIN_INSTRUCT_PROFILE, +} from "../llm/model-profile.js"; +import { MUSE_PROPS } from "../llm/model-profile.fixtures.js"; import { DEFAULT_EMBEDDING_MODEL_ID, DEFAULT_LLAMACPP_MODEL_ID, @@ -25,9 +30,16 @@ describe("models-catalog", () => { // no `` / `enable_thinking` markers. Because `--chat-template-file` // is what `/props.chat_template` reports back, it demoted the profile to // `plain-instruct` and deadlocked the grammar. See chat-templates.test.ts. - it("does not override the Qwen 3.5 chat template", () => { - expect(getLocalModelDef("qwen-3.5-4b").chatTemplateAsset).toBeUndefined(); - expect(getLocalModelDef("qwen-3.5-35b").chatTemplateAsset).toBeUndefined(); + // + // Catalog-wide rather than per-id: any entry that grows an override is + // exposed to the same failure mode, so adding one has to be a deliberate + // act that edits this test (and states why the override keeps every + // reasoning marker `detectModelProfile` keys on) — not a silent field. + it("ships no chat template override on any catalog entry", () => { + for (const def of LOCAL_MODELS_CATALOG) { + expect(def.chatTemplateAsset, `${def.id} must not override its chat template`) + .toBeUndefined(); + } }); it("throws on unknown id", () => { @@ -56,6 +68,23 @@ describe("models-catalog", () => { } }); + // Interim contract for Muse Glimmer. The catalog advertises multimodal + // (real: mmproj ships below) but NOT a native tool format, because there + // is none wired: the daemon passes `model.id` as the llama-server alias, + // and `muse-glimmer-30b` matches no alias hint in `selectBaseProfile`, so + // `/props` resolves to `plain-instruct` and tool calls run on the generic + // GBNF array grammar. This test is the tripwire: the day someone wires a + // native ATEM/Harmony profile, it fails and forces the description to be + // updated in the same commit instead of drifting into an over-promise. + it("resolves Muse Glimmer to plain-instruct, and says so in its description", () => { + expect(detectModelProfile(MUSE_PROPS)).toEqual(PLAIN_INSTRUCT_PROFILE); + + const muse = getLocalModelDef("muse-glimmer-30b"); + expect(muse.supportsVision).toBe(true); + expect(muse.description).not.toMatch(/atem|harmony/i); + expect(muse.description).toMatch(/generic tool calling/i); + }); + it("ensures mmproj URL points at the same HF repo as the GGUF weights", () => { for (const def of LOCAL_MODELS_CATALOG) { if (!def.mmprojUrl) continue; diff --git a/src/local-llm/models-catalog.ts b/src/local-llm/models-catalog.ts index b1f75496..02a31a84 100644 --- a/src/local-llm/models-catalog.ts +++ b/src/local-llm/models-catalog.ts @@ -308,7 +308,14 @@ export const LOCAL_MODELS_CATALOG: readonly LocalModelDef[] = [ "https://huggingface.co/unsloth/Muse-Glimmer-30B-GGUF/resolve/main/Muse-Glimmer-30B-UD-Q4_K_XL.gguf", fileSizeGb: 15.9, sizeLabel: "15.9 GB", - description: "Multimodal 30B MoE (ATEM / Harmony)", + // Vision is fully wired (mmproj below). The native ATEM/Harmony tool + // format is NOT: the daemon passes `model.id` as the llama-server + // alias (`daemon-lifecycle.ts`), and `muse-glimmer-30b` matches no + // alias hint in `selectBaseProfile`, so this resolves to + // `plain-instruct` and tool calls go through the generic GBNF array + // grammar. Pinned by `models-catalog.test.ts`; reword only when a + // native profile actually lands. + description: "Multimodal 30B MoE, generic tool calling", maxContextLength: 131_072, contextLabel: "128K", minRamGb: 20, diff --git a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts new file mode 100644 index 00000000..8d27b3ec --- /dev/null +++ b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts @@ -0,0 +1,138 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../../local-llm/index.js", async () => { + const actual = + await vi.importActual( + "../../local-llm/index.js", + ); + return { + ...actual, + getDaemonStatus: vi.fn(), + getEmbeddingDaemonStatus: vi.fn(), + startEmbeddingDaemon: vi.fn(), + stopEmbeddingDaemon: vi.fn(), + maybeAutoUpdateBackend: vi.fn(), + }; +}); + +import { getConfig, resetConfigCache } from "../../config/index.js"; +import * as localLlm from "../../local-llm/index.js"; +import { + resolveBackendDir, + resolveModelFilePath, + resolveServerBinPath, +} from "../../local-llm/index.js"; +import { resolvePlatformAsset } from "../../local-llm/platform-assets.js"; +import { persistUserLocalModelsConfig } from "../persist-user-local-models-config.js"; +import { LocalModelsOrchestrator } from "./local-models-orchestrator.js"; + +type Emitted = { type: string; line?: string }; + +/** + * The backend auto-update runs on every managed start, and it stops the + * daemon before downloading. If a post-check failure aborted the start, + * the user would be left with nothing running — strictly worse than not + * having the feature. These tests pin that a failed update still lets + * the existing binary start, and that the one genuinely fatal case + * (nothing usable left on disk) still stops. + */ +describe("LocalModelsOrchestrator backend auto-update", () => { + let stateDir: string; + + beforeEach(() => { + stateDir = mkdtempSync(join(tmpdir(), "local-models-au-")); + process.env.ATOMIC_AGENT_STATE_DIR = stateDir; + resetConfigCache(); + vi.mocked(localLlm.getDaemonStatus).mockReset(); + vi.mocked(localLlm.getEmbeddingDaemonStatus).mockReset(); + vi.mocked(localLlm.startEmbeddingDaemon).mockReset(); + vi.mocked(localLlm.stopEmbeddingDaemon).mockReset(); + vi.mocked(localLlm.maybeAutoUpdateBackend).mockReset(); + }); + + afterEach(() => { + delete process.env.ATOMIC_AGENT_STATE_DIR; + resetConfigCache(); + rmSync(stateDir, { recursive: true, force: true }); + }); + + it("starts the existing binary when the update failed but a backend remains", async () => { + const dataDir = prepareManagedInstall(); + vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ + action: "update_failed", + error: "socket hang up", + backendUsable: true, + }); + + const actions: Emitted[] = []; + const orchestrator = new LocalModelsOrchestrator({ + emit(a: unknown) { + actions.push(a as Emitted); + }, + }); + vi.spyOn(orchestrator, "startDaemon").mockResolvedValue(true); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + // No daemon adopted, so autoStartIfReady must reach startDaemon. + vi.mocked(localLlm.getDaemonStatus).mockResolvedValue({ + running: false, + healthy: false, + loading: false, + pid: null, + port: 19091, + }); + + await orchestrator.autoStartIfReady(); + + expect(orchestrator.startDaemon).toHaveBeenCalledTimes(1); + expect(actions.map((a) => a.line).filter(Boolean)).toContain( + "local-llm: backend update failed — starting current binary (socket hang up)", + ); + }); + + it("does not start when the update failed and no usable backend remains", async () => { + prepareManagedInstall(); + vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ + action: "update_failed", + error: "disk full", + backendUsable: false, + }); + + const actions: Emitted[] = []; + const orchestrator = new LocalModelsOrchestrator({ + emit(a: unknown) { + actions.push(a as Emitted); + }, + }); + vi.spyOn(orchestrator, "startDaemon").mockResolvedValue(true); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + + await orchestrator.autoStartIfReady(); + + expect(orchestrator.startDaemon).not.toHaveBeenCalled(); + expect(actions.map((a) => a.line).filter(Boolean)).toContain( + "local-llm: backend update failed and no usable backend remains — disk full", + ); + }); + + /** Managed mode with backend + chat model already on disk. */ + function prepareManagedInstall(): string { + const dataDir = getConfig().paths.localModelsDataDir; + const backendDir = resolveBackendDir(dataDir); + mkdirSync(backendDir, { recursive: true }); + const { binaryName } = resolvePlatformAsset(); + writeFileSync(resolveServerBinPath(dataDir, binaryName), ""); + const def = localLlm.getLocalModelDef("qwen-3.5-4b"); + mkdirSync(join(dataDir, "models", def.id), { recursive: true }); + writeFileSync(resolveModelFilePath(dataDir, def.id, def.filename), "stub"); + persistUserLocalModelsConfig({ + mode: "managed", + managed: { modelId: "qwen-3.5-4b" }, + }); + resetConfigCache(); + return dataDir; + } +}); diff --git a/src/tui/local-models/local-models-orchestrator.ts b/src/tui/local-models/local-models-orchestrator.ts index 2e51de98..11c4fc52 100644 --- a/src/tui/local-models/local-models-orchestrator.ts +++ b/src/tui/local-models/local-models-orchestrator.ts @@ -1508,8 +1508,14 @@ export class LocalModelsOrchestrator { /** * Check GitHub Releases and replace the llama.cpp zip when a newer - * tag exists. Check failures are fire-safe (returns true so start - * can use the current binary). Download errors return false. + * tag exists. Returns whether the caller may proceed to start. + * + * Every update failure is fire-safe: the installer stages the new + * build and only swaps it in once it is complete, so a failed check, + * stop or download leaves the previous binary on disk and start + * continues on it. `false` is reserved for the one case that cannot + * be papered over — the daemon was stopped for an update and no + * usable backend remains. */ private async applyBackendAutoUpdate(dataDir: string): Promise { try { @@ -1550,6 +1556,23 @@ export class LocalModelsOrchestrator { type: "runtime_info", line: `local-llm: backend update check failed — starting current binary (${result.error})`, }); + } else if (result.action === "update_failed") { + this.bus.emit({ + type: "local_models_pull_failed", + kind: "backend", + error: result.error, + }); + if (!result.backendUsable) { + this.bus.emit({ + type: "runtime_info", + line: `local-llm: backend update failed and no usable backend remains — ${result.error}`, + }); + return false; + } + this.bus.emit({ + type: "runtime_info", + line: `local-llm: backend update failed — starting current binary (${result.error})`, + }); } return true; } catch (e) { From 47021841fa8212dae2f79856b0d7e7fc30d6bd8a Mon Sep 17 00:00:00 2001 From: sosidudku1 Date: Thu, 20 Aug 2026 16:35:21 +0300 Subject: [PATCH 5/7] fix: single backend update check per start, plus sweep findings - autoStartIfReady no longer double-checks: startDaemon takes backendAlreadyChecked, the adopt branch keeps its own check - surface a deferred update in TUI and CLI instead of staying silent - correct the v41 migration narrative and the single-flight comment, both of which still described replaced behaviour --- src/cli/models-handlers.ts | 4 + src/config/config-schema.ts | 8 +- ...al-models-orchestrator-auto-update.test.ts | 74 ++++++++++++++++--- .../local-models/local-models-orchestrator.ts | 27 +++++-- 4 files changed, 93 insertions(+), 20 deletions(-) diff --git a/src/cli/models-handlers.ts b/src/cli/models-handlers.ts index e7490d51..00598898 100644 --- a/src/cli/models-handlers.ts +++ b/src/cli/models-handlers.ts @@ -237,6 +237,10 @@ export async function runLocalModelsStart(): Promise { process.stderr.write( `note: backend update check failed — starting current binary (${auto.error})\n`, ); + } else if (auto.action === "deferred") { + process.stderr.write( + "note: backend update deferred — another session is using the current binary\n", + ); } else if (auto.action === "update_failed") { if (process.stderr.isTTY) process.stderr.write("\n"); if (!auto.backendUsable) { diff --git a/src/config/config-schema.ts b/src/config/config-schema.ts index 0dd1fd35..cdf51614 100644 --- a/src/config/config-schema.ts +++ b/src/config/config-schema.ts @@ -1590,10 +1590,12 @@ export type RewriterGateMode = "heuristic" | "embedding" | "always"; * v32→v33 added the optional `analytics.*` block (anonymous PostHog * product analytics — opt-out via `analytics.enabled: false`). Older * files inherit `analytics: { enabled: true }` transparently. - * v37→v38 wired `localModels.managed.autoUpdate` (default `true`): + * v40→v41 wired `localModels.managed.autoUpdate` (default `true`): * managed start pulls a newer llama.cpp zip from GitHub Releases when - * one exists. Pre-v38 files that stored the unused `false` default - * migrate to `true`; an explicit `false` on a v38+ file is honoured. + * one exists. Pre-v41 the field was stored but never read, so a `false` + * there carried no meaning and is migrated to `true` — including one a + * user set deliberately, since the two are indistinguishable on disk. + * An explicit `false` on a v41+ file is honoured. * Older files are transparently upgraded by filling missing * blocks/fields from `USER_CONFIG_DEFAULTS`. Anything older than v5 * is not migrated: this is active development, callers delete their diff --git a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts index 8d27b3ec..8afaa9f6 100644 --- a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts +++ b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts @@ -93,7 +93,46 @@ describe("LocalModelsOrchestrator backend auto-update", () => { ); }); - it("does not start when the update failed and no usable backend remains", async () => { + // `autoStartIfReady` runs the update check itself and then delegates to + // `startDaemon`, which runs the same check — a TUI launch used to hit + // GitHub twice (against a ~60 req/h anonymous budget) and start two + // passes racing on the same `backend.next` staging dir. The flag below + // is what keeps it to one; these tests fail if it stops being passed. + it("checks for a backend update exactly once per start", async () => { + prepareManagedInstall(); + vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ + action: "current", + tag: "turboquant-07b9908", + }); + + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); + const startDaemon = vi + .spyOn(orchestrator, "startDaemon") + .mockResolvedValue(true); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + vi.mocked(localLlm.getDaemonStatus).mockResolvedValue({ + running: false, + healthy: false, + loading: false, + pid: null, + port: 19091, + }); + + await orchestrator.autoStartIfReady(); + + expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalledTimes(1); + expect(startDaemon).toHaveBeenCalledWith({ backendAlreadyChecked: true }); + }); + + // The flag must be opt-in: a bare `startDaemon()` (the `s` key, or a + // restart after a model switch) is the only update check on that path, + // so defaulting it to "already checked" would silently disable + // auto-update everywhere except TUI launch. + // Complements the flag test above: that one mocks `startDaemon`, so it + // only proves the flag is PASSED. This one calls the real method with + // the flag set and proves it is HONOURED — without it, the guard clause + // could be deleted and the pair would still look green. + it("skips the check when startDaemon is told the backend was checked", async () => { prepareManagedInstall(); vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ action: "update_failed", @@ -101,21 +140,32 @@ describe("LocalModelsOrchestrator backend auto-update", () => { backendUsable: false, }); - const actions: Emitted[] = []; - const orchestrator = new LocalModelsOrchestrator({ - emit(a: unknown) { - actions.push(a as Emitted); - }, + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + + // With the check skipped there is nothing to bail on, so the start + // proceeds past the point where `backendUsable: false` would stop it. + await orchestrator.startDaemon({ backendAlreadyChecked: true }); + + expect(localLlm.maybeAutoUpdateBackend).not.toHaveBeenCalled(); + }); + + it("still checks when startDaemon is invoked without the flag", async () => { + prepareManagedInstall(); + vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ + action: "update_failed", + error: "disk full", + backendUsable: false, }); - vi.spyOn(orchestrator, "startDaemon").mockResolvedValue(true); + + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); vi.spyOn(orchestrator, "refresh").mockResolvedValue(); - await orchestrator.autoStartIfReady(); + // `backendUsable: false` makes startDaemon bail before spawning, so + // this exercises the check without launching a real llama-server. + await expect(orchestrator.startDaemon()).resolves.toBe(false); - expect(orchestrator.startDaemon).not.toHaveBeenCalled(); - expect(actions.map((a) => a.line).filter(Boolean)).toContain( - "local-llm: backend update failed and no usable backend remains — disk full", - ); + expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalledTimes(1); }); /** Managed mode with backend + chat model already on disk. */ diff --git a/src/tui/local-models/local-models-orchestrator.ts b/src/tui/local-models/local-models-orchestrator.ts index 11c4fc52..a9c71549 100644 --- a/src/tui/local-models/local-models-orchestrator.ts +++ b/src/tui/local-models/local-models-orchestrator.ts @@ -118,8 +118,10 @@ export class LocalModelsOrchestrator { * `pullEmbeddingModel` `await this.pullBackend()` when the backend is * missing; without this a chat pull and an embedding pull started at the * same time would launch two concurrent `downloadBackend()` calls that - * wipe + extract into the same `/backend/` directory, corrupting - * the install. Concurrent callers share the same in-flight promise. + * race on the same `/backend.next` staging dir — each clears it + * before extracting, so the loser's tree is deleted mid-write and the + * winner can swap in a partial install. Concurrent callers share the + * same in-flight promise. */ private backendPullInFlight: Promise | null = null; /** @@ -802,7 +804,16 @@ export class LocalModelsOrchestrator { }); } - async startDaemon(): Promise { + /** + * @param opts.backendAlreadyChecked set by callers that ran + * `applyBackendAutoUpdate` themselves. Without it a TUI launch + * checks GitHub twice per start — two hits against the ~60 req/h + * anonymous budget, and two passes racing on the same + * `backend.next` staging dir. + */ + async startDaemon(opts?: { + backendAlreadyChecked?: boolean; + }): Promise { const cfg = getConfig(); if (cfg.localModels.mode !== "managed") { this.bus.emit({ @@ -838,7 +849,7 @@ export class LocalModelsOrchestrator { }); return false; } - if (!justPulledBackend) { + if (!justPulledBackend && !opts?.backendAlreadyChecked) { const updated = await this.applyBackendAutoUpdate(dataDir); if (!updated) return false; } @@ -1556,6 +1567,11 @@ export class LocalModelsOrchestrator { type: "runtime_info", line: `local-llm: backend update check failed — starting current binary (${result.error})`, }); + } else if (result.action === "deferred") { + this.bus.emit({ + type: "runtime_info", + line: `local-llm: backend update deferred — another session is using the current binary`, + }); } else if (result.action === "update_failed") { this.bus.emit({ type: "local_models_pull_failed", @@ -1624,7 +1640,8 @@ export class LocalModelsOrchestrator { await this.refresh(); return; } - await this.startDaemon(); + // The update check already ran above; `startDaemon` must not repeat it. + await this.startDaemon({ backendAlreadyChecked: true }); } /** From 76d78325f9c2f1b162c9d5c69757c6297023dd93 Mon Sep 17 00:00:00 2001 From: sosidudku1 Date: Thu, 20 Aug 2026 18:03:08 +0300 Subject: [PATCH 6/7] fix: start the daemon before the backend auto-update TUI auto-start ran the update check first, so the user got a rendered, typeable prompt with no model behind it while a 27-39 MB download ran. Start first; run one deferred pass afterwards, off the start path. The deferred pass must not stop the daemon it just started: hasOtherLiveSessions skips our own pid by design, so it would have reported 'no other sessions' for the very model the user is talking to. keepDaemonRunning makes that explicit and defers the swap to next start. Also bound the asset download with a timeout on both paths - the signal parameter existed but was never passed, so a stalled-open connection pinned the update for the life of the process. CLI 'models start' keeps updating before start: it is an explicit one-shot command. --- AGENTS.md | 2 +- src/cli/models-handlers.ts | 12 +++ src/local-llm/ensure-latest-backend.ts | 20 ++++- ...al-models-orchestrator-auto-update.test.ts | 73 +++++++++++++++++-- .../local-models/local-models-orchestrator.ts | 61 ++++++++++++++-- 5 files changed, 153 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8edf9529..21d53f48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -276,7 +276,7 @@ Symmetric **`EmbeddingProviderRegistry`** ([src/memory/embeddings/embedding-prov `atomic-agent` supports two modes for the llama-server backend (`config.llama.mode`): - `external` (default) — user runs `llama-server` out-of-band; runtime reads the URL from `config.llama.url` (env fallback `ATOMIC_AGENT_LLAMA_URL`). -- `managed` — `atomic-agent` downloads the llama.cpp binary from `AtomicBot-ai/atomic-llama-cpp-turboquant-nightly` GitHub Releases into `/llamacpp/backend/` and GGUF models into `/llamacpp/models//`. The server is **not** spawned by the runtime; operators control lifecycle via `atomic-agent llama start|stop|status|update`. Managed start (TUI auto-start / `s`, CLI `models start`) auto-pulls a newer zip when `localModels.managed.autoUpdate` is true (default since config v41; a failed check or download never blocks start — the existing binary is used). +- `managed` — `atomic-agent` downloads the llama.cpp binary from `AtomicBot-ai/atomic-llama-cpp-turboquant-nightly` GitHub Releases into `/llamacpp/backend/` and GGUF models into `/llamacpp/models//`. The server is **not** spawned by the runtime; operators control lifecycle via `atomic-agent llama start|stop|status|update`. Managed start auto-pulls a newer zip when `localModels.managed.autoUpdate` is true (default since config v41). A failed check or download never blocks start — the existing binary is used. The two entry points differ deliberately: **TUI auto-start** brings the daemon up first and runs the update afterwards, off the start path, so the user never faces a typeable prompt with no model behind it; that pass also refuses to stop the live daemon (`keepDaemonRunning`), so the swap lands on the next start. **CLI `models start`** is an explicit one-shot command, so it still updates before starting. Both bound the download with a timeout. **Invariant (preserved):** the agent runtime never starts a `llama-server` process. It only connects. Managed-mode lifecycle lives entirely in the `atomic-agent llama` CLI so runtime code paths stay single-mode. diff --git a/src/cli/models-handlers.ts b/src/cli/models-handlers.ts index 00598898..ef0f14cf 100644 --- a/src/cli/models-handlers.ts +++ b/src/cli/models-handlers.ts @@ -222,6 +222,11 @@ export async function runLocalModelsStart(): Promise { try { const auto = await maybeAutoUpdateBackend(dataDir, { enabled: cfg.localModels.managed.autoUpdate, + // Unlike the TUI, `models start` is an explicit one-shot command: + // updating before the daemon comes up is what the operator asked + // for. It still needs a deadline — a stalled-open connection would + // otherwise pin the command forever with a progress bar at 12%. + signal: AbortSignal.timeout(BACKEND_DOWNLOAD_TIMEOUT_MS), onProgress: (p: number, t: number, tot: number) => { const line = renderPullProgress("backend zip", p, t, tot); if (process.stderr.isTTY) process.stderr.write(`\r${line.padEnd(79)}`); @@ -381,6 +386,13 @@ function describeDeviceChoice( return resolved ?? configured; } +/** + * Deadline for the backend asset download. The zip is 27-39 MB, so this + * is generous for any working link; it exists because a stalled-but-open + * TCP connection never resolves on its own. + */ +const BACKEND_DOWNLOAD_TIMEOUT_MS = 10 * 60_000; + const DEVICE_ID_RE = /^[A-Za-z]+\d+$/; /** diff --git a/src/local-llm/ensure-latest-backend.ts b/src/local-llm/ensure-latest-backend.ts index f5bfaa1b..cbdd7aea 100644 --- a/src/local-llm/ensure-latest-backend.ts +++ b/src/local-llm/ensure-latest-backend.ts @@ -14,7 +14,7 @@ export type AutoUpdateBackendResult = | { action: "skipped" } | { action: "current"; tag: string | null } | { action: "updated"; from: string | null; to: string } - | { action: "deferred"; reason: "other_session" } + | { action: "deferred"; reason: "other_session" | "daemon_live" } | { action: "check_failed"; error: string } /** * The version check said "update", but stopping the daemon or @@ -43,6 +43,20 @@ export async function maybeAutoUpdateBackend( enabled: boolean; onProgress?: DownloadProgressFn; onWillDownload?: () => void; + /** + * Abort the (27-39 MB) asset download. Without one a stalled but + * open connection never resolves and the update hangs for the + * lifetime of the process. + */ + signal?: AbortSignal; + /** + * Never stop a running daemon to install the update. Set by the + * deferred pass that runs *after* start: there the live daemon is + * the one serving the user, and `hasOtherLiveSessions` cannot see + * it — it skips our own pid by design — so without this the + * background update would kill the model mid-turn. + */ + keepDaemonRunning?: boolean; }, ): Promise { if (!opts.enabled) return { action: "skipped" }; @@ -67,6 +81,9 @@ export async function maybeAutoUpdateBackend( // model mid-chat is worse than sitting on an old tag until next solo start. try { if (readRunningPid(dataDir) !== null) { + if (opts.keepDaemonRunning) { + return { action: "deferred", reason: "daemon_live" }; + } if (hasOtherLiveSessions(dataDir)) { return { action: "deferred", reason: "other_session" }; } @@ -76,6 +93,7 @@ export async function maybeAutoUpdateBackend( opts.onWillDownload?.(); const downloaded = await downloadBackend(dataDir, { onProgress: opts.onProgress, + signal: opts.signal, }); return { action: "updated", diff --git a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts index 8afaa9f6..9a253cf9 100644 --- a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts +++ b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts @@ -93,11 +93,10 @@ describe("LocalModelsOrchestrator backend auto-update", () => { ); }); - // `autoStartIfReady` runs the update check itself and then delegates to - // `startDaemon`, which runs the same check — a TUI launch used to hit - // GitHub twice (against a ~60 req/h anonymous budget) and start two - // passes racing on the same `backend.next` staging dir. The flag below - // is what keeps it to one; these tests fail if it stops being passed. + // `autoStartIfReady` starts the daemon and then runs one deferred + // update pass. A TUI launch used to hit GitHub twice (against a ~60 + // req/h anonymous budget) and race two passes on the same + // `backend.next` staging dir; the flag below is what keeps it to one. it("checks for a backend update exactly once per start", async () => { prepareManagedInstall(); vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ @@ -124,6 +123,70 @@ describe("LocalModelsOrchestrator backend auto-update", () => { expect(startDaemon).toHaveBeenCalledWith({ backendAlreadyChecked: true }); }); + // The whole point of the deferral: a rendered TUI the user can type + // into, with no model behind it, reads as a broken agent. The daemon + // must be up before the (27-39 MB, possibly stalled) download starts. + it("starts the daemon before checking for a backend update", async () => { + prepareManagedInstall(); + const order: string[] = []; + vi.mocked(localLlm.maybeAutoUpdateBackend).mockImplementation(async () => { + order.push("update"); + return { action: "current", tag: "turboquant-07b9908" }; + }); + + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); + vi.spyOn(orchestrator, "startDaemon").mockImplementation(async () => { + order.push("start"); + return true; + }); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + vi.mocked(localLlm.getDaemonStatus).mockResolvedValue({ + running: false, + healthy: false, + loading: false, + pid: null, + port: 19091, + }); + + await orchestrator.autoStartIfReady(); + await vi.waitFor(() => expect(order).toHaveLength(2)); + + expect(order).toEqual(["start", "update"]); + }); + + // `hasOtherLiveSessions` skips our own pid by design, so on the + // deferred pass it reports "no other sessions" for the very daemon we + // just started. Without `keepDaemonRunning` the background update + // would stop the model the user is talking to. + it("never stops the running daemon on the deferred pass", async () => { + prepareManagedInstall(); + vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ + action: "deferred", + reason: "daemon_live", + }); + + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); + vi.spyOn(orchestrator, "startDaemon").mockResolvedValue(true); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + vi.mocked(localLlm.getDaemonStatus).mockResolvedValue({ + running: false, + healthy: false, + loading: false, + pid: null, + port: 19091, + }); + + await orchestrator.autoStartIfReady(); + await vi.waitFor(() => + expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalled(), + ); + + expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ keepDaemonRunning: true }), + ); + }); + // The flag must be opt-in: a bare `startDaemon()` (the `s` key, or a // restart after a model switch) is the only update check on that path, // so defaulting it to "already checked" would silently disable diff --git a/src/tui/local-models/local-models-orchestrator.ts b/src/tui/local-models/local-models-orchestrator.ts index a9c71549..e2c7e51a 100644 --- a/src/tui/local-models/local-models-orchestrator.ts +++ b/src/tui/local-models/local-models-orchestrator.ts @@ -82,6 +82,15 @@ function describeDeviceChoice( } /** Log-tail poll cadence while the LLM logs tab is active. */ +/** + * Deadline for the backend asset download on the auto-update path. The + * zip is 27-39 MB, so this is generous for any working link; it exists + * because a stalled-but-open TCP connection otherwise never resolves + * and the update hangs for the life of the process. A timeout is + * reported like any other update failure and retried on the next start. + */ +const BACKEND_DOWNLOAD_TIMEOUT_MS = 10 * 60_000; + const LOGS_POLL_MS = 1000; /** Snapshot refresh cadence while the Models tab is idle. */ @@ -1528,10 +1537,18 @@ export class LocalModelsOrchestrator { * be papered over — the daemon was stopped for an update and no * usable backend remains. */ - private async applyBackendAutoUpdate(dataDir: string): Promise { + private async applyBackendAutoUpdate( + dataDir: string, + opts?: { keepDaemonRunning?: boolean }, + ): Promise { try { const result = await maybeAutoUpdateBackend(dataDir, { enabled: getConfig().localModels.managed.autoUpdate, + keepDaemonRunning: opts?.keepDaemonRunning, + // The zip is small (27-39 MB) but the link may not be. Without a + // deadline a stalled-open connection pins the download for the + // life of the process; the next start retries from scratch. + signal: AbortSignal.timeout(BACKEND_DOWNLOAD_TIMEOUT_MS), onWillDownload: () => { this.bus.emit({ type: "local_models_pull_started", @@ -1570,7 +1587,10 @@ export class LocalModelsOrchestrator { } else if (result.action === "deferred") { this.bus.emit({ type: "runtime_info", - line: `local-llm: backend update deferred — another session is using the current binary`, + line: + result.reason === "daemon_live" + ? "local-llm: backend update deferred — will install on next start" + : "local-llm: backend update deferred — another session is using the current binary", }); } else if (result.action === "update_failed") { this.bus.emit({ @@ -1610,9 +1630,16 @@ export class LocalModelsOrchestrator { * Called once at TUI startup. If the user is in managed mode AND the * backend + model are already on disk AND no daemon is currently * running, start the daemon so the user lands in a ready state - * without needing an extra keypress. When `autoUpdate` is on, a - * newer llama.cpp zip is pulled first even if a daemon is already - * running (it is stopped, replaced, then restarted). + * without needing an extra keypress. + * + * The backend auto-update runs **after** the daemon is up, never + * before. Checking first meant the user got a rendered TUI they + * could type into while no model was loaded — a prompt that silently + * does nothing reads as a broken agent, and the download (27-39 MB, + * unbounded on a stalled link) sat on that path. Starting first + * costs one session on the previous binary; the swap is picked up on + * the next start. A daemon already serving a model is left alone for + * the same reason. */ async autoStartIfReady(): Promise { const cfg = getConfig(); @@ -1626,8 +1653,6 @@ export class LocalModelsOrchestrator { if (!isBackendDownloaded(dataDir)) return; const def = getLocalModelDef(mid); if (!isModelDownloaded(dataDir, def)) return; - const autoOk = await this.applyBackendAutoUpdate(dataDir); - if (!autoOk) return; const running = await getDaemonStatus(dataDir, cfg.localModels.managed.port); if (running.running) { // Already started by a previous TUI session; adopt it. @@ -1638,10 +1663,30 @@ export class LocalModelsOrchestrator { // (see `ensureEmbeddingPaired`). await this.ensureEmbeddingPaired(); await this.refresh(); + this.scheduleBackendAutoUpdate(dataDir); return; } - // The update check already ran above; `startDaemon` must not repeat it. + // `startDaemon` owns the pre-start check on the path where the + // backend has to be replaced before a daemon exists; here it must + // not run one, because the deferred pass below owns it. await this.startDaemon({ backendAlreadyChecked: true }); + this.scheduleBackendAutoUpdate(dataDir); + } + + /** + * Run the backend auto-update once the daemon is serving, off the + * start path. Deliberately not awaited: the result only matters for + * the *next* start, so nothing the user is waiting on depends on it. + * `maybeAutoUpdateBackend` already defers when any daemon is live + * (`hasOtherLiveSessions` / `readRunningPid`), so the model we just + * started is not pulled out from under the user mid-turn. + */ + private scheduleBackendAutoUpdate(dataDir: string): void { + void this.applyBackendAutoUpdate(dataDir, { + keepDaemonRunning: true, + }).catch(() => { + /* applyBackendAutoUpdate already reports failures on the bus */ + }); } /** From ec5ad827c450bd0e37e8b99d2096012ddb14098b Mon Sep 17 00:00:00 2001 From: sosidudku1 Date: Thu, 20 Aug 2026 18:12:58 +0300 Subject: [PATCH 7/7] feat(tui): U toggles backend auto-update in the models panel The flag is on by default and drives a background download, but the only way to change it was hand-editing config.json - the CLI form rewrites the whole file. U flips it, the panel shows 'auto-update off' next to the backend tag, and the footer advertises the key. --- src/tui/components/local-models-panel.tsx | 3 +- .../local-models-key-bindings.test.ts | 47 +++++++++++++++++++ .../local-models/local-models-key-bindings.ts | 6 +++ ...al-models-orchestrator-auto-update.test.ts | 46 ++++++++++++++++++ .../local-models/local-models-orchestrator.ts | 20 ++++++++ .../local-models/local-models-panel-state.ts | 9 +++- src/tui/tui-app.tsx | 1 + src/tui/tui-command.ts | 2 + 8 files changed, 132 insertions(+), 2 deletions(-) diff --git a/src/tui/components/local-models-panel.tsx b/src/tui/components/local-models-panel.tsx index 358237ba..a6a71393 100644 --- a/src/tui/components/local-models-panel.tsx +++ b/src/tui/components/local-models-panel.tsx @@ -441,10 +441,11 @@ export function LocalModelsPanel({ data dir: {panel.dataDir} · backend{" "} {panel.backend.currentTag ?? "—"} {panel.backend.updateAvailable === true ? " (update available)" : ""} + {panel.backend.autoUpdate ? "" : " · auto-update off"} ) : null} - j/k move · Enter pull/activate (embedding: *row + Enter starts server) · g gguf · i info · d remove · s chat+embedding · E embeddings on/off · G gpu · B · r · L + j/k move · Enter pull/activate (embedding: *row + Enter starts server) · g gguf · i info · d remove · s chat+embedding · E embeddings on/off · G gpu · U auto-update · B · r · L ) : ( diff --git a/src/tui/local-models/local-models-key-bindings.test.ts b/src/tui/local-models/local-models-key-bindings.test.ts index 7f373d0f..726c7718 100644 --- a/src/tui/local-models/local-models-key-bindings.test.ts +++ b/src/tui/local-models/local-models-key-bindings.test.ts @@ -137,6 +137,53 @@ describe("handleLocalModelsTabKey — vision-aware Enter / g hotkey", () => { expect(onCycle).toHaveBeenCalledTimes(1); }); + // The flag is on by default and drives a background download, so it + // needs an in-TUI way out: the CLI equivalent rewrites the whole + // config file. Like `G`, it ignores the cursor row. + it("'U' toggles backend auto-update regardless of the cursor row type", () => { + const onToggle = vi.fn(); + const callbacks: TuiAppCallbacks = { + onApprovalDecision: vi.fn(), + onAbort: vi.fn(), + onQuit: vi.fn(), + onMessageSubmitted: vi.fn(), + onLocalModelsAutoUpdateToggleRequested: onToggle, + }; + const state = stateWithRow( + makeRow("gemma-4-e4b", { downloaded: true, mmprojStatus: "downloaded" }), + ); + const handled = handleLocalModelsTabKey("U", emptyKey({ shift: true }), { + state, + dispatch: vi.fn(), + callbacks, + }); + expect(handled).toBe(true); + expect(onToggle).toHaveBeenCalledTimes(1); + }); + + // Lowercase must not trigger it — `u` is unbound here, and silently + // flipping a background-download setting on a stray keypress is the + // kind of surprise the uppercase convention exists to prevent. + it("lowercase 'u' does not toggle backend auto-update", () => { + const onToggle = vi.fn(); + const callbacks: TuiAppCallbacks = { + onApprovalDecision: vi.fn(), + onAbort: vi.fn(), + onQuit: vi.fn(), + onMessageSubmitted: vi.fn(), + onLocalModelsAutoUpdateToggleRequested: onToggle, + }; + const state = stateWithRow( + makeRow("gemma-4-e4b", { downloaded: true, mmprojStatus: "downloaded" }), + ); + handleLocalModelsTabKey("u", emptyKey(), { + state, + dispatch: vi.fn(), + callbacks, + }); + expect(onToggle).not.toHaveBeenCalled(); + }); + it("Enter on a downloaded GGUF + missing mmproj row triggers mmproj-only pull", () => { const onPull = vi.fn(); const callbacks: TuiAppCallbacks = { diff --git a/src/tui/local-models/local-models-key-bindings.ts b/src/tui/local-models/local-models-key-bindings.ts index 8ea88eb8..52be5893 100644 --- a/src/tui/local-models/local-models-key-bindings.ts +++ b/src/tui/local-models/local-models-key-bindings.ts @@ -157,6 +157,12 @@ export function handleLocalModelsTabKey( callbacks.onLocalModelsDeviceCycleRequested?.(); return true; } + // `U` (uppercase) toggles backend auto-update, matching the `B`/`G` + // convention for panel-wide actions that ignore the cursor row. + if (input === "U") { + callbacks.onLocalModelsAutoUpdateToggleRequested?.(); + return true; + } if (input === "r") { callbacks.onLocalModelsRefreshRequested?.(); return true; diff --git a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts index 9a253cf9..c19c318c 100644 --- a/src/tui/local-models/local-models-orchestrator-auto-update.test.ts +++ b/src/tui/local-models/local-models-orchestrator-auto-update.test.ts @@ -231,6 +231,52 @@ describe("LocalModelsOrchestrator backend auto-update", () => { expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalledTimes(1); }); + // The key binding only proves an event fires; this proves the flag is + // actually written and read back, which is what `U` is for. + it("toggleBackendAutoUpdate flips the persisted flag both ways", async () => { + prepareManagedInstall(); + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + + expect(getConfig().localModels.managed.autoUpdate).toBe(true); + + await orchestrator.toggleBackendAutoUpdate(); + expect(getConfig().localModels.managed.autoUpdate).toBe(false); + + await orchestrator.toggleBackendAutoUpdate(); + expect(getConfig().localModels.managed.autoUpdate).toBe(true); + }); + + // Turning it off must actually stop the update, not just relabel it. + it("skips the update entirely once auto-update is toggled off", async () => { + prepareManagedInstall(); + const orchestrator = new LocalModelsOrchestrator({ emit() {} }); + vi.spyOn(orchestrator, "refresh").mockResolvedValue(); + await orchestrator.toggleBackendAutoUpdate(); + + vi.mocked(localLlm.maybeAutoUpdateBackend).mockResolvedValue({ + action: "skipped", + }); + vi.spyOn(orchestrator, "startDaemon").mockResolvedValue(true); + vi.mocked(localLlm.getDaemonStatus).mockResolvedValue({ + running: false, + healthy: false, + loading: false, + pid: null, + port: 19091, + }); + + await orchestrator.autoStartIfReady(); + await vi.waitFor(() => + expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalled(), + ); + + expect(localLlm.maybeAutoUpdateBackend).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ enabled: false }), + ); + }); + /** Managed mode with backend + chat model already on disk. */ function prepareManagedInstall(): string { const dataDir = getConfig().paths.localModelsDataDir; diff --git a/src/tui/local-models/local-models-orchestrator.ts b/src/tui/local-models/local-models-orchestrator.ts index e2c7e51a..87ac14c0 100644 --- a/src/tui/local-models/local-models-orchestrator.ts +++ b/src/tui/local-models/local-models-orchestrator.ts @@ -260,6 +260,7 @@ export class LocalModelsOrchestrator { currentTag: ver?.tag ?? null, latestTag, updateAvailable, + autoUpdate: cfg.localModels.managed.autoUpdate, }, daemon: { running: daemon.running, @@ -727,6 +728,25 @@ export class LocalModelsOrchestrator { * `auto → cpu → auto`. Does not restart the daemon — the operator * presses `s` to apply. */ + /** + * Flip `localModels.managed.autoUpdate`. The flag is on by default and + * governs a background download, so it needs a way out that is not + * "hand-edit config.json" — the CLI equivalent takes the whole file. + * Takes effect on the next start; nothing in flight is cancelled. + */ + async toggleBackendAutoUpdate(): Promise { + const next = !getConfig().localModels.managed.autoUpdate; + persistUserLocalModelsConfig({ managed: { autoUpdate: next } }); + resetConfigCache(); + this.bus.emit({ + type: "runtime_info", + line: next + ? "local-llm: backend auto-update on — a newer llama.cpp is fetched after start" + : "local-llm: backend auto-update off — update manually with 'B'", + }); + await this.refresh(); + } + async cycleManagedDevice(): Promise { const cfg = getConfig(); const dataDir = cfg.paths.localModelsDataDir; diff --git a/src/tui/local-models/local-models-panel-state.ts b/src/tui/local-models/local-models-panel-state.ts index d6329dd7..6e7fd13e 100644 --- a/src/tui/local-models/local-models-panel-state.ts +++ b/src/tui/local-models/local-models-panel-state.ts @@ -90,6 +90,8 @@ export interface LocalModelsBackendInfo { currentTag: string | null; latestTag: string | null; updateAvailable: boolean | null; + /** `localModels.managed.autoUpdate`; surfaced so `U` has visible state. */ + autoUpdate: boolean; } export interface LocalModelsDaemonInfo { @@ -207,7 +209,12 @@ export function createInitialLocalModelsPanelState(): LocalModelsPanelState { mode: "list", rows: [], cursor: 0, - backend: { currentTag: null, latestTag: null, updateAvailable: null }, + backend: { + currentTag: null, + latestTag: null, + updateAvailable: null, + autoUpdate: true, + }, daemon: { running: false, healthy: false, diff --git a/src/tui/tui-app.tsx b/src/tui/tui-app.tsx index 15e7fa35..b13c0291 100644 --- a/src/tui/tui-app.tsx +++ b/src/tui/tui-app.tsx @@ -170,6 +170,7 @@ export interface TuiAppCallbacks { onLocalModelsRefreshRequested?(): void; /** Cycle the managed daemon's GPU preference (auto → devices → cpu). */ onLocalModelsDeviceCycleRequested?(): void | Promise; + onLocalModelsAutoUpdateToggleRequested?(): void | Promise; onLocalModelsRemoveConfirmed?(modelId: import("../local-llm/index.js").LocalModelId): void; onLocalModelsStatusRequested?(): void | Promise; /** Ask the orchestrator to (re)start the llama-server daemon. */ diff --git a/src/tui/tui-command.ts b/src/tui/tui-command.ts index 3fc51a7e..35a20713 100644 --- a/src/tui/tui-command.ts +++ b/src/tui/tui-command.ts @@ -405,6 +405,8 @@ export async function tuiCommand(args: string[]): Promise { onLocalModelsRefreshRequested: () => void orchestrator.localModels.refresh(), onLocalModelsDeviceCycleRequested: () => void orchestrator.localModels.cycleManagedDevice(), + onLocalModelsAutoUpdateToggleRequested: () => + void orchestrator.localModels.toggleBackendAutoUpdate(), onLocalModelsRemoveConfirmed: (id) => void orchestrator.localModels.removeLocalModel(id), onLocalModelsStatusRequested: () => orchestrator.localModels.emitStatusLine(),