fix(harness): recover truncated/unterminated JSON tool requests

agent.py

@@ -849,6 +849,10 @@ async def stream_callback(chunk: str, total: str):
             rate_limiter_callback=(
                 self.rate_limiter_callback if not call_data["background"] else None
             ),
+            # Utility prompts (e.g. memory post-processing) treat an empty
+            # reply as a benign "nothing to do", so the main-turn
+            # empty-completion retry must not fire here.
+            a0_allow_empty_completion=True,
         )
 
         await extension.call_extensions_async(
@@ -1119,6 +1123,7 @@ async def process_llm_result_tools(self, llm_result: LLMResult):
             if (
                 extract_tools.extract_tool_request(llm_result.reasoning) is not None
                 or extract_tools.is_misformatted_tool_request(llm_result.reasoning)
+                or extract_tools.is_truncated_tool_request(llm_result.reasoning)
             ):
                 message = llm_result.reasoning
         if (
@@ -1127,13 +1132,14 @@ async def process_llm_result_tools(self, llm_result: LLMResult):
             and bool(message.strip())
             and extract_tools.extract_tool_request(message) is None
             and not extract_tools.is_misformatted_tool_request(message)
+            and not extract_tools.is_truncated_tool_request(message)
         ):
             return await self._execute_tool_request(
                 tool_name="response",
                 tool_args={"text": message},
                 message=message,
             )
-        return await self.process_tools(message)
+        return await self.process_tools(message, finish_reason=llm_result.finish_reason)
 
     async def _execute_tool_request(
         self,
@@ -1410,10 +1416,17 @@ def _clear_responses_pending_state(self) -> None:
             self.set_data(Agent.DATA_NAME_RESPONSES_STATE, state)
 
     @extension.extensible
-    async def process_tools(self, msg: str):
+    async def process_tools(self, msg: str, finish_reason: str = ""):
         # search for tool usage requests in agent message
         tool_request = extract_tools.extract_tool_request(msg)
 
+        recovered_embedded = False
+        if tool_request is None:
+            # narrowly scoped repair: accept exactly one valid tool request
+            # embedded in planning prose (DeepSeek V4 Flash thinking output)
+            tool_request = extract_tools.recover_embedded_tool_request(msg)
+            recovered_embedded = tool_request is not None
+
         raw_tool_name = ""
         tool_args = {}
 
@@ -1427,6 +1440,21 @@ async def process_tools(self, msg: str):
                 )
             except ValueError:
                 tool_request = None  # treat structural validation errors as misformat
+                recovered_embedded = False
+
+        if recovered_embedded:
+            # Corrective note: recovery succeeded, but teach the model to emit
+            # bare JSON next time. Deliberately NOT one of the prompts tracked
+            # by the unusable-response-loop guard - a recovered request is a
+            # usable response.
+            warning_msg = self.read_prompt("fw.msg_recovered_request.md")
+            wmsg = self.hist_add_warning(warning_msg)
+            PrintStyle(font_color="orange", padding=True).print(warning_msg)
+            self.context.log.log(
+                type="warning",
+                content=f"{self.agent_name}: Tool request embedded in prose was recovered.",
+                id=wmsg.id,
+            )
 
         if tool_request is not None:
             tool_name = raw_tool_name  # Initialize tool_name with raw_tool_name
@@ -1508,12 +1536,22 @@ async def process_tools(self, msg: str):
                     type="warning", content=f"{self.agent_name}: {error_detail}", id=wmsg.id
                 )
         else:
-            warning_msg_misformat = self.read_prompt("fw.msg_misformat.md")
-            wmsg = self.hist_add_warning(warning_msg_misformat)
-            PrintStyle(font_color="red", padding=True).print(warning_msg_misformat)
+            category = extract_tools.classify_tool_request_failure(msg)
+            reason = extract_tools.explain_tool_request_failure(msg, finish_reason)
+            if category == "truncated":
+                warning_msg = self.read_prompt("fw.msg_truncated_request.md")
+                log_reason = (
+                    f"truncated or unterminated JSON tool request; "
+                    f"reason: {reason}"
+                )
+            else:
+                warning_msg = self.read_prompt("fw.msg_misformat.md")
+                log_reason = f"no valid tool request found; reason: {reason}"
+            wmsg = self.hist_add_warning(warning_msg)
+            PrintStyle(font_color="red", padding=True).print(warning_msg)
             self.context.log.log(
                 type="warning",
-                content=f"{self.agent_name}: Message misformat, no valid tool request found.",
+                content=f"{self.agent_name}: Message misformat, {log_reason}.",
                 id=wmsg.id,
             )
 

extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py

@@ -21,6 +21,7 @@ def execute(self, data: dict | None = None, **kwargs):
             return
         if message not in {
             self.agent.read_prompt("fw.msg_misformat.md"),
+            self.agent.read_prompt("fw.msg_truncated_request.md"),
             self.agent.read_prompt("fw.msg_repeat.md"),
         }:
             return

helpers/__init__.py

@@ -0,0 +1,8 @@
+# This file must exist. Without it, `helpers` is imported as a namespace
+# package (no __file__), which makes test/module cleanup helpers that purge
+# `sys.modules` treat it as a stub and delete every loaded `helpers.*`
+# module. The subsequent re-imports then split extension-registry state
+# between stale and fresh module copies (observed as `Agent` instances
+# losing extension-initialized attributes such as `loop_data` depending on
+# test import order). Keeping `helpers` a regular package prevents that
+# entire pollution class.

helpers/extract_tools.py

@@ -1,4 +1,6 @@
 
+import json
+
 from .dirty_json import DirtyJson
 import regex, re
 from helpers.modules import load_classes_from_file, load_classes_from_folder # keep here for backwards compatibility
@@ -33,6 +35,176 @@ def extract_tool_request(content: str) -> dict[str, Any] | None:
     return request if request is not None and _is_tool_request(request) else None
 
 
+def recover_embedded_tool_request(content: str) -> dict[str, Any] | None:
+    """Recover exactly one valid tool request embedded in surrounding prose.
+
+    Strict extraction (extract_tool_request) stays unchanged; this is a
+    narrowly scoped repair for models (e.g. DeepSeek V4 Flash with thinking
+    enabled) that occasionally wrap a single valid JSON tool envelope in
+    planning prose. Returns None when zero or multiple *distinct* tool
+    requests are found, so arbitrary prose is never treated as a tool call.
+
+    Envelopes appearing inside quoted spans, inline code, or fenced code
+    blocks are examples being discussed, not requests being issued, and are
+    masked before scanning. Candidates must parse as strict JSON, so lenient
+    dirty-JSON forms (single-quoted keys, etc.) are not executable either.
+    """
+    if not content or not isinstance(content, str):
+        return None
+
+    content = content.strip()
+    if extract_tool_request(content) is not None:
+        return None  # strict path handles clean responses; recovery is failure-only
+
+    masked = _mask_non_executable_regions(content)
+    distinct: dict[str, dict[str, Any]] = {}
+    root_for_request = ""
+    for root in extract_json_root_strings(masked):
+        data = _parse_json_root_object_strict(root)
+        if data is None or not _is_tool_request(data):
+            continue
+        distinct.setdefault(json.dumps(data, sort_keys=True, default=str), data)
+        if len(distinct) > 1:
+            return None
+        root_for_request = root
+    request = next(iter(distinct.values()), None)
+    if request is None:
+        return None
+    if str(request.get("tool_name") or "") == "response" and _has_prose_around(
+        masked, root_for_request
+    ):
+        # A "response" envelope buried in deliberating prose is the model
+        # thinking out loud about how it *could* reply, not a completed task.
+        # Refuse recovery so the message takes the standard misformat path
+        # instead of ending the task. Operational tools keep being recovered:
+        # executing a valid operational envelope is the protocol's intent.
+        return None
+    return request
+
+
+_RESPONSE_PROSE_HEDGE_RE = re.compile(
+    r"\b(could|might|maybe|option|alternatively|but)\b", re.IGNORECASE
+)
+
+
+def _has_prose_around(masked_content: str, root: str) -> bool:
+    """Heuristic: True when substantial prose surrounds the extracted root.
+
+    Deliberately simple and deterministic: prose is substantial when the
+    non-whitespace text outside the JSON root exceeds 40 characters, or when
+    it contains hedging language ("could", "but", ...) that marks the
+    envelope as a possibility under discussion rather than the final answer.
+    Quoted/fenced spans are already blanked by _mask_non_executable_regions,
+    so examples under discussion do not count as prose.
+    """
+    prose = masked_content.replace(root, " ", 1)
+    if len("".join(prose.split())) > 40:
+        return True
+    return bool(_RESPONSE_PROSE_HEDGE_RE.search(prose))
+
+
+def _mask_non_executable_regions(content: str) -> str:
+    """Blank out fenced code blocks, inline code and top-level quoted spans.
+
+    Quoted/fenced regions in prose contain examples under discussion, not a
+    tool request the model is issuing. Masking them (to spaces, preserving
+    newlines and offsets) prevents the root scanner from starting a JSON
+    object inside them. Quotes inside an actual JSON object (depth > 0) are
+    left untouched, so a legitimate bare envelope survives masking.
+    """
+    chars = list(content)
+
+    for match in re.finditer(r"```.*?```", content, flags=re.DOTALL):
+        for index in range(match.start(), match.end()):
+            if chars[index] != "\n":
+                chars[index] = " "
+
+    depth = 0
+    quote: str | None = None
+    escaped = False
+    for index, char in enumerate(chars):
+        if quote:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == quote:
+                quote = None
+            if char != "\n":
+                chars[index] = " "
+            continue
+
+        if char in "{[":
+            depth += 1
+        elif char in "}]":
+            depth = max(0, depth - 1)
+        elif depth == 0 and char in ('"', "`"):
+            quote = char
+            chars[index] = " "
+
+    return "".join(chars)
+
+
+def _parse_json_root_object_strict(root: str) -> dict[str, Any] | None:
+    try:
+        data = json.loads(root)
+    except Exception:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+_TOOL_FAILURE_REASONS = {
+    "empty": "empty response from the model",
+    "prose": "plain prose with no JSON tool object",
+    "truncated": "truncated or unterminated JSON tool request",
+    "invalid_envelope": "JSON object without a valid tool_name/tool_args envelope",
+}
+
+
+def classify_tool_request_failure(content: Any) -> str:
+    """Single classification of an unusable model response.
+
+    Drives both the sanitized log reason (explain_tool_request_failure) and
+    the retry-prompt routing in Agent.process_tools, so the log label and the
+    reprompt choice can never diverge. Returns one of: "empty", "prose",
+    "truncated", "invalid_envelope".
+    """
+    if not isinstance(content, str) or not content.strip():
+        return "empty"
+    stripped = content.strip()
+    if "{" not in stripped:
+        return "prose"
+
+    roots = extract_json_root_strings(stripped)
+    if roots:
+        # A complete (non-tool) root followed by an unterminated fragment
+        # (odd quote count, or a new "{") is trailing truncation, not a
+        # clean-but-invalid envelope.
+        tail = stripped.split(roots[-1], 1)[1].lstrip(" \t\r\n,")
+        if tail and (tail.startswith("{") or tail.count('"') % 2 == 1):
+            return "truncated"
+        return "invalid_envelope"
+
+    return "truncated"
+
+
+def explain_tool_request_failure(content: str, finish_reason: str = "") -> str:
+    """Sanitized classification of why no tool request could be extracted.
+
+    Never includes response content; only a reason and the output length.
+    """
+    prefix = ""
+    if finish_reason == "length":
+        prefix = "provider truncated the response (finish_reason=length); "
+
+    category = classify_tool_request_failure(content)
+    reason = _TOOL_FAILURE_REASONS[category]
+    if category == "empty":
+        return f"{prefix}{reason}"
+    length = len(content.strip())
+    return f"{prefix}{reason} ({length} chars)"
+
+
 def is_misformatted_tool_request(content: str) -> bool:
     if not content or not isinstance(content, str):
         return False
@@ -75,6 +247,73 @@ def is_misformatted_tool_request(content: str) -> bool:
         )
     )
 
+def _json_scan_final_depth(content: str) -> int:
+    """Return the unclosed-brace depth after scanning JSON-ish text.
+
+    Tracks quote/escape state so braces inside strings do not count. Shared
+    by _json_root_object_balanced and is_truncated_tool_request so the two
+    scans cannot drift apart. Depth > 0 means the text is unterminated.
+    """
+    depth = 0
+    quote = None
+    escaped = False
+    for char in content:
+        if quote:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == quote:
+                quote = None
+            continue
+        if depth and char in ('"', "'", "`"):
+            quote = char
+        elif char == "{":
+            depth += 1
+        elif depth and char == "[":
+            depth += 1
+        elif depth and char in ("}", "]"):
+            depth -= 1
+    return depth
+
+
+def _json_root_object_balanced(content: str) -> bool:
+    return _json_scan_final_depth(content) == 0
+
+
+# Structural gate: an unterminated payload only counts as a truncated tool
+# request when it opens like a JSON tool envelope
+# ({"thoughts"/"headline"/"tool_name"/"tool_args": ...) or a Responses
+# function_call-style payload ({"type":"function", ...). A prose sentence
+# that merely mentions "tool"/"actions"/"function" after a stray "{" is
+# not a truncated request. "type" is included because in responses mode a
+# truncated function-call payload must take the repair-prompt path instead
+# of being shown to the user as a plain-text reply.
+_TRUNCATION_ENVELOPE_OPEN_RE = re.compile(
+    r'\{\s*"(thoughts|headline|tool_name|tool_args|type)"'
+)
+
+
+def is_truncated_tool_request(content: str) -> bool:
+    """Return True when content contains an unterminated JSON tool envelope.
+
+    Used by the harness to give a targeted retry prompt instead of a generic
+    misformat warning when providers cut a streaming/completion response
+    mid-object. The scan starts at the first envelope-shaped opening brace,
+    so a truncated payload wrapped in a ```json fence or preceded by prose
+    (or following a complete non-tool JSON object) is still recognized.
+    """
+    if not content or not isinstance(content, str):
+        return False
+    content = content.strip()
+    match = _TRUNCATION_ENVELOPE_OPEN_RE.search(content)
+    if not match:
+        return False
+    candidate = content[match.start():]
+    if candidate.endswith("}") and _json_root_object_balanced(candidate):
+        return False
+
+    return _json_scan_final_depth(candidate) > 0
 
 def normalize_tool_request(tool_request: Any) -> tuple[str, dict]:
     if not isinstance(tool_request, dict):
@@ -141,6 +380,9 @@ def extract_json_root_strings(content: str) -> list[str]:
         return []
 
     if content.lstrip().startswith("["):
+        # Blind spot, by design: content starting with "[" is treated as a
+        # JSON array and never scanned for object roots, so embedded-envelope
+        # recovery can never fire for it.
         return []
 
     roots: list[str] = []