From e9267d1a066731a3791a4f1d877fb88c54e06206 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 14:44:49 -0700 Subject: [PATCH 01/74] fix: keep Codex token usage when a turn fails The Codex adapter ran each turn with the SDK's AsyncTurnHandle.run(), which raises on a failed turn and is cancelled on a timeout. The token usage snapshots Codex had already sent for that turn were dropped with it, so a failed, timed-out or disconnected invocation reported no usage even though its model calls had spent tokens. The adapter now reads the turn's notification stream itself, records the last thread token usage snapshot, and hands the same stream to the SDK's turn collector. When the turn fails, times out or loses its connection, that snapshot goes into the failure output and is differenced against the runtime baseline like a successful turn. Usage covers the snapshots Codex sent before the failure. Codex sends a response's usage after its tool calls finish, so on a timeout the last response is not counted if it was still streaming or its tools were still running. Codex marks a context-window overflow by replacing the thread total with the window size, all other counters zero, and then fails the turn. The adapter skips that snapshot so it never reports the window size as spent tokens. A request rejected before its turn starts, such as non-text input, used to clear the usage baseline. The next successful invocation then reported no usage. Codex's thread totals do not move when no turn runs, so the adapter now keeps the baseline in that case. Signed-off-by: mnajafian-nv --- adapters/python/codex/README.md | 2 +- .../src/nemo_fabric_adapters/codex/adapter.py | 77 +++++- tests/adapters/test_codex_adapter.py | 261 +++++++++++++++--- 3 files changed, 301 insertions(+), 39 deletions(-) diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index b5fa351bc..20bc81995 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -79,7 +79,7 @@ The result includes the SDK's typed terminal response, turn status, token usage, timing, and completed thread items. It does not expose CLI commands, return codes, stdout, or stderr. -Normalized `usage` contains invocation-local token counts from the SDK's cumulative thread totals, including cached input tokens. Input tokens already include cached tokens (`input_tokens_include_cache=True`). Reused threads report the difference from the previous invocation rather than counting the thread total again. Missing, invalid, or reset counters remain unknown; a missing snapshot requires a new baseline before differences can be reported. The adapter preserves native usage in `output.usage`, does not infer cost, and retains available usage on unsuccessful results. If the SDK raises before returning usage, normalized usage remains unavailable. +Normalized `usage` contains invocation-local token counts from the SDK's cumulative thread totals, including cached input tokens. Input tokens already include cached tokens (`input_tokens_include_cache=True`). Reused threads report the difference from the previous invocation rather than counting the thread total again. Missing, invalid, or reset counters remain unknown; a missing snapshot requires a new baseline before differences can be reported. The adapter preserves native usage in `output.usage`, does not infer cost, and retains available usage on unsuccessful results. When a turn fails, times out, or loses its connection, usage covers the snapshots that Codex sent before the failure. Codex sends a response's usage after its tool calls finish, so on a timeout the last response is not counted if it was still streaming or its tools were still running. A failure before Codex sends any usage leaves usage unknown. When Codex exceeds the model context window, it replaces the thread total with the window size before it fails the turn; the adapter ignores that snapshot. A request rejected before its turn starts, such as non-text input, keeps the baseline for the next invocation. ## Configuration diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index e463332b7..86de0133f 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -17,6 +17,7 @@ import sys import tempfile import time +from collections.abc import AsyncIterator from dataclasses import asdict, dataclass, is_dataclass from enum import Enum from pathlib import Path @@ -30,6 +31,7 @@ JsonRpcError, Sandbox, TransportClosedError, + TurnResult, is_retryable_error, ) from openai_codex.generated.v2_all import ( @@ -38,7 +40,13 @@ McpServerOauthLoginCompletedNotification, McpServerOauthLoginResponse, SkillsExtraRootsSetResponse, + ThreadTokenUsage, + ThreadTokenUsageUpdatedNotification, ) + +# Private: the collector AsyncTurnHandle.run uses; recheck it on every openai-codex bump. +from openai_codex._run import _collect_async_turn_result +from openai_codex.models import Notification from openai_codex.types import Personality, ReasoningEffort, TurnStatus from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -1155,6 +1163,54 @@ def normalize_result( } +@dataclass +class _TurnUsage: + """Last thread token usage Codex reported for the running turn.""" + + snapshot: ThreadTokenUsage | None = None + + +def _fills_context_window(usage: ThreadTokenUsage) -> bool: + """Detect Codex's context-window marker, which replaces the counted thread totals. + + On a context-window overflow, Codex sets the thread total to the window size with + every other counter at zero and then fails the turn. + """ + total = usage.total + return ( + total.total_tokens == usage.model_context_window + and total.input_tokens == 0 + and total.output_tokens == 0 + ) + + +async def _turn_notifications( + handle: Any, usage: _TurnUsage +) -> AsyncIterator[Notification]: + stream = handle.stream() + try: + async for event in stream: + payload = event.payload + if ( + isinstance(payload, ThreadTokenUsageUpdatedNotification) + and payload.turn_id == handle.id + and not _fills_context_window(payload.token_usage) + ): + usage.snapshot = payload.token_usage + yield event + finally: + await stream.aclose() + + +async def _run_turn(handle: Any, usage: _TurnUsage) -> TurnResult: + """Collect the turn as ``AsyncTurnHandle.run`` does, keeping usage if the turn fails.""" + events = _turn_notifications(handle, usage) + try: + return await _collect_async_turn_result(events, turn_id=handle.id) + finally: + await events.aclose() + + async def _interrupt_turn(handle: Any) -> None: if handle is None: return @@ -1216,6 +1272,7 @@ async def _invoke_thread( """Run one turn and report whether the connected SDK transport remains usable.""" handle = None + usage = _TurnUsage() deadline = asyncio.timeout_at(invocation_deadline) try: async with deadline: @@ -1224,7 +1281,7 @@ async def _invoke_thread( effort=_reasoning_effort(config), output_schema=_output_schema(config), ) - result = await handle.run() + result = await _run_turn(handle, usage) return ( normalize_result( config, context, base_dir, thread_id=thread.id, result=result @@ -1234,12 +1291,19 @@ async def _invoke_thread( except TimeoutError as error: await _interrupt_turn(handle) if deadline.expired(): - return _failure("timeout", "Codex invocation timed out"), False - return sdk_failure(error), False + failure = _failure("timeout", "Codex invocation timed out") + else: + failure = sdk_failure(error) except CodexAdapterError: raise except (CodexError, RuntimeError, OSError) as error: - return sdk_failure(error), False + failure = sdk_failure(error) + # Usage covers the snapshots Codex sent before the failure. Codex sends a response's + # usage after its tool calls finish, so on a timeout the last response is not + # counted if it was still streaming or its tools were still running. + if usage.snapshot is not None: + failure["usage"] = _json_safe(usage.snapshot) + return failure, False def _relay_output( @@ -1458,6 +1522,7 @@ async def invoke( ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) + turn_requested = False try: request_prompt(request) _reasoning_effort(config) @@ -1477,6 +1542,7 @@ async def invoke( and relay_artifacts.expects_local_atif(relay.plugin_config) else None ) + turn_requested = True output, usable = await _invoke_thread( config, runtime_context, @@ -1520,7 +1586,8 @@ async def invoke( self._unusable = not usable if self._relay is not None: output = _relay_output(output, self._relay) - return _agent_run_result(output, self._usage_totals) + # A request rejected before its turn leaves Codex's thread totals unchanged. + return _agent_run_result(output, self._usage_totals if turn_requested else None) async def stop(self) -> None: client = self._client diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 32cb1bcfe..edb139212 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -23,7 +23,11 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.codex import adapter from openai_codex import AsyncCodex, AsyncThread, AsyncTurnHandle, JsonRpcError -from openai_codex.types import TurnStatus +from openai_codex import TransportClosedError +from openai_codex.generated.v2_all import ItemCompletedNotification +from openai_codex.generated.v2_all import ThreadTokenUsageUpdatedNotification +from openai_codex.generated.v2_all import TurnCompletedNotification +from openai_codex.models import Notification def lifecycle_start_payload(payload): @@ -137,27 +141,72 @@ def configure_mcp(payload, servers): payload["config"]["mcp"] = {"servers": servers} -def successful_result(response="done"): - return SimpleNamespace( - id="turn-1", - status=TurnStatus.completed, - error=None, - started_at=100, - completed_at=101, - duration_ms=1000, - final_response=response, - items=[ - { - "id": "item-1", - "type": "agentMessage", - "phase": "final_answer", - "text": response, - } - ], - usage={"total": {"inputTokens": 10, "outputTokens": 3}}, +def token_usage(input_tokens, output_tokens, *, cached_input_tokens=0): + """Return a Codex thread usage snapshot whose counters are cumulative.""" + total = { + "inputTokens": input_tokens, + "cachedInputTokens": cached_input_tokens, + "outputTokens": output_tokens, + "reasoningOutputTokens": 0, + "totalTokens": input_tokens + output_tokens, + } + return {"total": total, "last": total, "modelContextWindow": 272000} + + +def usage_event(usage): + return Notification( + method="thread/tokenUsage/updated", + payload=ThreadTokenUsageUpdatedNotification.model_validate( + {"threadId": "thread-123", "turnId": "turn-1", "tokenUsage": usage} + ), + ) + + +def turn_completed(error=None): + turn = { + "id": "turn-1", + "items": [], + "status": "completed" if error is None else "failed", + "startedAt": 100, + "completedAt": 101, + "durationMs": 1000, + } + if error is not None: + turn["error"] = {"message": error} + return Notification( + method="turn/completed", + payload=TurnCompletedNotification.model_validate( + {"threadId": "thread-123", "turn": turn} + ), ) +def turn_events(response="done", *, usage=None): + """Return the SDK notifications Codex sends for one completed turn.""" + events = [usage_event(token_usage(10, 3) if usage is None else usage)] + if response is not None: + events.append( + Notification( + method="item/completed", + payload=ItemCompletedNotification.model_validate( + { + "completedAtMs": 101000, + "threadId": "thread-123", + "turnId": "turn-1", + "item": { + "id": "item-1", + "type": "agentMessage", + "phase": "final_answer", + "text": response, + }, + } + ), + ) + ) + events.append(turn_completed()) + return events + + def test_agent_run_result_discards_oversized_token_count(): result = adapter._agent_run_result( {"response": "done", "usage": {"input_tokens": 1 << 64}} @@ -253,13 +302,46 @@ def test_reset_cumulative_counter_rebaselines_without_negative_usage(): assert resumed.usage.output_tokens == 2 -def mock_turn_handle(result=None): +def test_usage_keeps_reported_zeros(): + """Pins existing behavior: counters Codex reports as zero stay zero.""" + total = { + "inputTokens": 0, + "cachedInputTokens": 0, + "outputTokens": 0, + "totalTokens": 0, + } + + result = adapter._agent_run_result({"usage": {"total": total}}) + + assert result.usage.to_mapping() == { + "input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens_include_cache": True, + "output_tokens": 0, + "total_tokens": 0, + } + + +def mock_turn_handle(events=None): + """Stream notifications; an exception is raised and a callable is awaited.""" mock_handle = MagicMock(spec=AsyncTurnHandle) - outcome = successful_result() if result is None else result - if isinstance(outcome, BaseException): - mock_handle.run.side_effect = outcome - else: - mock_handle.run.return_value = outcome + mock_handle.id = "turn-1" + if events is None: + events = turn_events() + elif isinstance(events, BaseException): + events = [events] + mock_handle.events = events + + async def stream(): + for event in mock_handle.events: + if isinstance(event, BaseException): + raise event + if callable(event): + await event() + continue + yield event + + mock_handle.stream.side_effect = stream mock_handle.interrupted = False async def mark_interrupted(): @@ -269,10 +351,10 @@ async def mark_interrupted(): return mock_handle -def mock_thread(thread_id, result=None): +def mock_thread(thread_id, events=None): mock_sdk_thread = MagicMock(spec=AsyncThread) mock_sdk_thread.id = thread_id - mock_sdk_thread.handle = mock_turn_handle(result) + mock_sdk_thread.handle = mock_turn_handle(events) mock_sdk_thread.turn.return_value = mock_sdk_thread.handle return mock_sdk_thread @@ -1360,9 +1442,8 @@ async def write_atif(): ) write_task = asyncio.create_task(write_atif()) - return successful_result() - mock_sdk_thread.handle.run.side_effect = finish_turn + mock_sdk_thread.handle.events.insert(-1, finish_turn) mock_codex.next_thread = mock_sdk_thread runtime = adapter.CodexRuntime() @@ -1456,8 +1537,7 @@ def test_sdk_failure_preserves_jsonrpc_code(): def test_incomplete_sdk_turn_is_failed(codex_payload, mock_codex): - result = successful_result(response=None) - mock_codex.next_result = result + mock_codex.next_result = turn_events(response=None) output = invoke_once(codex_payload) @@ -1465,6 +1545,121 @@ def test_incomplete_sdk_turn_is_failed(codex_payload, mock_codex): assert output["turn_status"] == "completed" +async def invoke_turns(payload, mock_codex, *turns): + """Invoke one runtime once per (input, events) pair; None events mean no turn starts.""" + mock_codex.next_thread = mock_thread("thread-123") + mock_codex.next_thread.turn.side_effect = [ + mock_turn_handle(events) for _, events in turns if events is not None + ] + start_payload = dict(payload) + start_payload.pop("request") + runtime = adapter.CodexRuntime() + await runtime.start(lifecycle_start_payload(start_payload)) + results = [] + try: + for request_input, _ in turns: + payload["request"]["input"] = request_input + results.append(await runtime.invoke(*lifecycle_invocation(payload))) + finally: + await runtime.stop() + return results + + +@pytest.mark.parametrize( + ("failure", "code"), + [ + ([turn_completed(error="model request failed")], "codex_turn_failed"), + ([TimeoutError("SDK request timed out")], "codex_timed_out"), + ([TransportClosedError("closed")], "codex_connection_failed"), + ], + ids=["failed-turn", "sdk-timeout", "connection-closed"], +) +async def test_failed_invocation_keeps_usage_codex_reported_before_failure( + codex_payload, mock_codex, failure, code +): + _, failed = await invoke_turns( + codex_payload, + mock_codex, + ("Inspect.", turn_events(usage=token_usage(10, 3, cached_input_tokens=2))), + ( + "Continue.", + [usage_event(token_usage(40, 8, cached_input_tokens=6)), *failure], + ), + ) + + assert failed.status is AgentRunStatus.FAILED + assert failed.error.code == code + assert failed.usage.to_mapping() == { + "input_tokens": 30, + "cached_input_tokens": 4, + "input_tokens_include_cache": True, + "output_tokens": 5, + "total_tokens": 35, + } + assert failed.output["usage"]["total"]["inputTokens"] == 40 + + +async def test_context_window_overflow_reports_usage_before_codex_fills_the_window( + codex_payload, mock_codex +): + window = 272000 + filled = { + "inputTokens": 0, + "cachedInputTokens": 0, + "outputTokens": 0, + "reasoningOutputTokens": 0, + "totalTokens": window, + } + (failed,) = await invoke_turns( + codex_payload, + mock_codex, + ( + "Inspect.", + [ + usage_event(token_usage(40, 8)), + usage_event( + { + "total": filled, + "last": {**filled, "totalTokens": window - 48}, + "modelContextWindow": window, + } + ), + turn_completed(error="context window exceeded"), + ], + ), + ) + + assert failed.error.code == "codex_turn_failed" + assert failed.usage.to_mapping() == { + "input_tokens": 40, + "cached_input_tokens": 0, + "input_tokens_include_cache": True, + "output_tokens": 8, + "total_tokens": 48, + } + + +async def test_rejected_request_keeps_the_usage_baseline(codex_payload, mock_codex): + first, rejected, third = await invoke_turns( + codex_payload, + mock_codex, + ("Inspect.", turn_events(usage=token_usage(10, 3))), + ({"messages": []}, None), + ("Continue.", turn_events(usage=token_usage(25, 7))), + ) + + assert first.usage.input_tokens == 10 + assert rejected.error.code == "codex_invalid_request" + assert rejected.usage is None + assert third.usage.to_mapping() == { + "input_tokens": 15, + "cached_input_tokens": 0, + "input_tokens_include_cache": True, + "output_tokens": 4, + "total_tokens": 19, + } + + def test_custom_provider_requires_explicit_api_key_env(codex_payload, mock_codex): model = codex_payload["config"]["models"]["default"] model.update( @@ -1932,9 +2127,9 @@ async def block(): await asyncio.sleep(60) # An SDK timeout before Fabric's deadline is not invocation expiry. - mock_blocking_thread.handle.run.side_effect = ( + mock_blocking_thread.handle.events = [ block if deadline_expires else TimeoutError("SDK request timed out") - ) + ] mock_codex.next_thread = mock_blocking_thread codex_payload["runtime_context"]["deadline_millis"] = int( (time.time() + (0.05 if deadline_expires else 60)) * 1000 From 9d6bc218bf4debd918ea49ad44eb8e7789fba6e4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 15:28:16 -0700 Subject: [PATCH 02/74] fix: count every Deep Agents model call in token usage The Deep Agents adapter summed usage_metadata from the AI messages that reached graph state through the updates stream. Three cases came out wrong: - Deep Agents adds SummarizationMiddleware to every agent and subagent. Its summary call returns only text, so the summary model's message and usage never enter graph state, and an invocation that summarized under-reported input, output and total tokens. - When the invocation raised, including GraphRecursionError, the adapter returned no messages, so usage was None even though model calls had finished. - A count was summed over the calls that reported it and returned as known. ChatOpenAI also turns an omitted Chat Completions count into 0 and derives a missing total, so a partial provider report came out as measured zeros. The adapter now records usage from LangChain's chat-model callbacks, which fire for every chat-model call that runs in the adapter process: the main agent, declarative subagents and summarization. Agent Protocol subagents run remotely and are not included. A new recorder is created for each invocation, so earlier turns are never counted again. It outlives an exception, so a failed invocation still reports the fields that every counted call reported. Each token count and the cost are known only when every counted call in the invocation reported them, and an explicit zero stays a known zero. A request the provider rejects with a 4xx status (other than 408) contributes no usage. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the recorder, and only the final response's reported usage counts. When a response carries OpenAI-shaped raw usage, the raw counts decide what was reported: an omitted count stays unknown, the provider's total is used when present, and a derived total counts only when input and output were both reported. The graph tests run the real Deep Agents graph and ChatOpenAI against a scripted Chat Completions endpoint for summarization, parallel task subagents, 400, 408, 503 and timeout failures, the recursion limit, partial reports and explicit zeros. Separate ChatOpenAI.with_retry tests cover 429 then success and 503 then success. The existing usage tests now produce their messages through a LangChain chat-model call so the callbacks fire. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 28 +- .../deepagents/adapter.py | 254 +++++---- tests/adapters/test_deepagents.py | 491 +++++++++++++++--- tests/e2e/test_deepagents.py | 1 - tests/integrations/test_relay_scope_leak.py | 4 +- 5 files changed, 618 insertions(+), 160 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..78e94ce51 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -227,10 +227,30 @@ service. The normalized result includes the final response, buffered messages and per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and -preflight failures (a missing credential, an absent `deepagents` package, or an -invalid MCP server) fail runtime start before an invocation is accepted. +reports it), and errors. Configuration and preflight failures (a missing +credential, an absent `deepagents` package, or an invalid MCP server) fail +runtime start before an invocation is accepted. + +Usage sums every chat-model call that runs in the adapter process for the +invocation: the main agent, declarative subagents, and Deep Agents +summarization. Agent Protocol subagents (entries with `graph_id`) run in their +remote service, and their tokens are not included. A token count or cost is +reported only when every counted call reported it; otherwise it stays unknown. +This also holds for a failed invocation. A request the provider rejects with a +4xx status (other than 408) contributes no usage. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have done work without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; Fabric configures no such cache. + +When a response carries OpenAI-shaped usage in +`response_metadata.token_usage`, as `ChatOpenAI` responses without streaming +do, the raw counts decide what was reported: an omitted count stays unknown, +and the provider's total is used when present. Other providers and streamed +responses use LangChain's `usage_metadata`, which some integrations fill with 0 +for an omitted count. ## Runtime Lifecycle diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..37771f419 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -21,9 +21,12 @@ from pathlib import Path from typing import Any from typing import NamedTuple +from uuid import UUID from langchain.agents.middleware import AgentMiddleware +from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages import ToolMessage +from langchain_core.outputs import LLMResult from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentMcpServerConfig @@ -750,7 +753,7 @@ async def invoke( resumed=resumed, result_state=outcome.result_state, events=outcome.events or [], - turn_messages=outcome.turn_messages or [], + model_calls=outcome.model_calls or [], error=outcome.error, telemetry_runtime=telemetry_runtime, relay_artifacts=relay_artifacts, @@ -874,24 +877,26 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + model_usage = ModelCallUsage() try: - result_state, events, turn_messages = await invoke_compiled_agent( + result_state, events = await invoke_compiled_agent( self._agent, user_message, self._thread_id, - callbacks=callbacks, + callbacks=[*(callbacks or []), model_usage], ) except GraphRecursionError: return TurnOutcome( + model_calls=model_usage.calls(), error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome(model_calls=model_usage.calls(), error=_error_text(exc)) return TurnOutcome( result_state=result_state, events=events, - turn_messages=turn_messages, + model_calls=model_usage.calls(), ) def _telemetry_output( @@ -966,18 +971,65 @@ def _apply_callbacks( return config +class ModelCallUsage(BaseCallbackHandler): + """Record what each chat-model call of one invocation reports. + + LangChain fires these callbacks for every chat-model call in the run, + including calls whose response never enters graph state, such as the Deep + Agents summarization call. The records live on the handler, so calls that + finished before an invocation raised are still reported. + + A call is recorded as unreported when it starts, and its usage replaces that + record when it ends. A request the provider rejects with a 4xx status (other + than 408) contributes no usage and is dropped. A 5xx response, timeout, + cancellation or unfinished call stays unreported and makes the invocation's + counts and cost unknown, because the provider may have done work without + reporting usage. + """ + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._calls: dict[UUID, dict[str, Any]] = {} + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_end(self, response: LLMResult, *, run_id: UUID, **kwargs: Any) -> None: + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + self._calls[run_id] = {} if message is None else _message_to_dict(message) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + # Provider SDKs (OpenAI, Anthropic) raise errors carrying the HTTP status + # code when the provider answered with an error response. A 4xx other + # than 408 (request timeout) means the provider rejected the request. + status_code = getattr(error, "status_code", None) + if isinstance(status_code, int) and 400 <= status_code < 500: + if status_code != 408: + self._calls.pop(run_id, None) + + def calls(self) -> list[dict[str, Any]]: + return list(self._calls.values()) + + async def invoke_compiled_agent( agent: Any, user_message: str, thread_id: str | None, callbacks: list[Any] | None = None, -) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: - """Run a compiled agent and return state, events, and current-turn messages. - - On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. +) -> tuple[Any, list[dict[str, Any]]]: + """Run a compiled agent and return its final state and per-step events. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -992,12 +1044,10 @@ async def invoke_compiled_agent( config = config or None events: list[dict[str, Any]] = [] - turn_messages: list[Any] = [] final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose - # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is - # taken from the main graph only (empty namespace). + # namespace identifies the subgraph, so they appear in ``events``; the final + # ``values`` state is taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) if config @@ -1009,16 +1059,9 @@ async def invoke_compiled_agent( if namespace: event["subgraph"] = "/".join(str(part) for part in namespace) events.append(event) - for node_output in chunk.values(): - if isinstance(node_output, dict): - new = node_output.get("messages") - if isinstance(new, list): - turn_messages.extend(new) - elif new is not None: - turn_messages.append(new) elif mode == "values" and not namespace: final = chunk - return final, events, _dedup_messages(_messages_to_dicts(turn_messages)) + return final, events class Observability(NamedTuple): @@ -1034,7 +1077,7 @@ class TurnOutcome(NamedTuple): result_state: Any = None events: list[dict[str, Any]] | None = None - turn_messages: list[dict[str, Any]] | None = None + model_calls: list[dict[str, Any]] | None = None error: str | None = None error_code: str | None = None telemetry_error: str | None = None @@ -1151,7 +1194,7 @@ def normalize_output( resumed: bool, result_state: Any, events: list[dict[str, Any]], - turn_messages: list[dict[str, Any]], + model_calls: list[dict[str, Any]], error: str | None, telemetry_runtime: dict[str, Any] | None, relay_artifacts: list[dict[str, str]] | None, @@ -1160,9 +1203,9 @@ def normalize_output( ) -> dict[str, Any]: messages = _extract_messages(result_state) response = _final_response(messages) - # Aggregate usage/cost from this turn's messages only; the replayed final state - # replays prior-turn messages that must not be re-counted. - usage = _aggregate_usage(turn_messages) + # Usage comes from this invocation's model calls, not from the final state, + # which also replays earlier turns and omits internal calls like summarization. + usage = _aggregate_usage(model_calls) output: dict[str, Any] = { "harness": HARNESS, @@ -1210,26 +1253,6 @@ def _messages_to_dicts(raw: Any) -> list[dict[str, Any]]: return [item if isinstance(item, dict) else _message_to_dict(item) for item in raw] -def _dedup_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop duplicate messages by id, preserving first-seen order. - - With ``subgraphs=True`` the same message can surface in both a subgraph's - updates and the parent stream; de-duplicating by id keeps usage aggregation - from counting a message twice. Messages without an id are always kept. - """ - - seen: set[str] = set() - unique: list[dict[str, Any]] = [] - for message in messages: - message_id = message.get("id") - if isinstance(message_id, str): - if message_id in seen: - continue - seen.add(message_id) - unique.append(message) - return unique - - def _message_to_dict(message: Any) -> dict[str, Any]: role = ( getattr(message, "type", None) or getattr(message, "role", None) or "assistant" @@ -1259,56 +1282,103 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: return messages[-1].get("content") if messages else None -def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: +def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: + """Sum model-call usage; a count is known only when every call reported it. + + One call without a count makes the invocation's sum for that count a + partial value, so it stays unknown rather than under-reported. + """ + + if not calls: + return None totals: dict[str, Any] = {} - cost = 0.0 - has_cost = False - for message in messages: - usage = message.get("usage") if isinstance(message.get("usage"), dict) else {} - for source, target in ( - ("input_tokens", "prompt_tokens"), - ("output_tokens", "completion_tokens"), - ("total_tokens", "total_tokens"), - ): - value = usage.get(source) - if ( - isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - ): - totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. - candidate = ( - usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) - ) - if ( - isinstance(candidate, (int, float)) - and not isinstance(candidate, bool) - and math.isfinite(candidate) - and candidate >= 0 - ): - cost += float(candidate) - has_cost = True - if has_cost: - totals["cost"] = cost + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "total_tokens"), + ): + values = [_reported_tokens(call, source) for call in calls] + if all(value is not None for value in values): + totals[target] = sum(values) + costs = [_reported_cost(call) for call in calls] + if all(cost is not None for cost in costs): + totals["cost"] = math.fsum(costs) return totals or None -def _metadata_cost(message: dict[str, Any]) -> float | None: - metadata = message.get("response_metadata") - if isinstance(metadata, dict): - value = metadata.get("cost") - if ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - and value >= 0 - ): - return float(value) +# Raw OpenAI Chat Completions usage keys behind each LangChain usage counter. +_OPENAI_USAGE_KEYS = { + "input_tokens": "prompt_tokens", + "output_tokens": "completion_tokens", + "total_tokens": "total_tokens", +} + + +def _reported_tokens(call: dict[str, Any], name: str) -> int | None: + """Return one call's token count, or ``None`` when the provider omitted it. + + ``ChatOpenAI`` turns an omitted Chat Completions count into 0 and derives a + missing or zero total from input plus output. When a response carries + OpenAI-shaped raw usage in ``response_metadata.token_usage``, the raw counts + are used: an omitted count is unknown, and a derived total counts only when + both of its parts were reported. Other responses use ``usage_metadata``. + """ + + usage = call.get("usage") + value = usage.get(name) if isinstance(usage, dict) else None + if not _is_token_count(value): + return None + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if not isinstance(raw, dict): + return value + reported = raw.get(_OPENAI_USAGE_KEYS[name]) + if _is_token_count(reported): + return reported + if name == "total_tokens" and reported is None: + parts = (raw.get("prompt_tokens"), raw.get("completion_tokens")) + if all(_is_token_count(part) for part in parts): + return value return None +def _is_token_count(value: Any) -> bool: + return ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= (1 << 64) - 1 + ) + + +def _reported_cost(call: dict[str, Any]) -> float | None: + """Return a provider-reported cost for one call, or ``None`` when absent. + + Cost is not part of LangChain's UsageMetadata; some providers report it on the + usage or response metadata. + """ + + usage = call.get("usage") if isinstance(call.get("usage"), dict) else {} + metadata = call.get("response_metadata") + candidates = ( + usage.get("total_cost"), + usage.get("cost"), + metadata.get("cost") if isinstance(metadata, dict) else None, + ) + for value in candidates: + if value is not None: + return float(value) if _is_cost(value) else None + return None + + +def _is_cost(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + def _supported_kwargs(callable_obj: Any, kwargs: dict[str, Any]) -> dict[str, Any]: try: signature = inspect.signature(callable_obj) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..c916cb2a3 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -25,6 +25,9 @@ from unittest.mock import MagicMock import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI as InstalledChatOpenAI from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -34,6 +37,7 @@ from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import McpOAuth2Config from nemo_fabric_adapter_contract.models import McpServiceAccountConfig from nemo_fabric_adapter_contract.models import RuntimeContext @@ -49,6 +53,23 @@ def _activation(*diagnostics: dict[str, object]) -> types.SimpleNamespace: ROOT = Path(__file__).resolve().parents[2] +async def call_model(config: dict[str, Any] | None, message: AIMessage) -> AIMessage: + """Return ``message`` from a real LangChain chat-model call under ``config``. + + The call fires the run's LangChain callbacks the way a Deep Agents model call + does, which is where the adapter reads usage. Test stand-ins for the Relay + handler are not LangChain handlers, so they are left out. + """ + + callbacks = [ + callback + for callback in (config or {}).get("callbacks") or [] + if isinstance(callback, BaseCallbackHandler) + ] + model = GenericFakeChatModel(messages=iter([message])) + return await model.ainvoke("prompt", config={"callbacks": callbacks}) + + def test_descriptor_declares_supported_normalized_config(): descriptor = json.loads( (ROOT / "adapters/python/deepagents/deepagents.fabric-adapter.json").read_text( @@ -192,11 +213,17 @@ async def stream( recorder["subgraphs"] = subgraphs recorder["checkpointer"] = kwargs.get("checkpointer") user = inputs["messages"][-1]["content"] - ai = { - "role": "ai", - "content": f"reply to {user}", - "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, - } + ai = await call_model( + config, + AIMessage( + content=f"reply to {user}", + usage_metadata={ + "input_tokens": 5, + "output_tokens": 7, + "total_tokens": 12, + }, + ), + ) # subgraphs=True yields 3-tuples ``(namespace, mode, chunk)``; the main # graph has an empty namespace. ``updates`` carries the message produced # this turn; ``values`` is the full replayed state. @@ -1445,7 +1472,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_scope_or_relay_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1481,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + # The only run callback is the adapter's own model-usage recorder. + callbacks = fake_sdks["config"]["callbacks"] + assert [type(callback) for callback in callbacks] == [adapter.ModelCallUsage] assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2070,14 +2099,19 @@ async def test_cost_is_extracted_from_response_metadata( ): import deepagents - message = { - "role": "ai", - "content": "done", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.0025}, - } - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + message = await call_model( + config, + AIMessage( + content="done", + usage_metadata={ + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + }, + response_metadata={"cost": 0.0025}, + ), + ) yield ((), "updates", {"agent": {"messages": [message]}}) yield ((), "values", {"messages": [message]}) @@ -2108,6 +2142,23 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): assert adapter._aggregate_usage([{"usage": {"input_tokens": 1 << 64}}]) is None +@pytest.mark.parametrize( + ("costs", "expected"), + [ + pytest.param([0.0, 0.0], 0.0, id="explicit-zero"), + pytest.param([0.5, 0.25], 0.75, id="every-call"), + pytest.param([0.5, None], None, id="one-call-without-cost"), + ], +) +def test_aggregate_usage_reports_cost_only_when_every_call_did(costs, expected): + calls = [ + {"usage": {"input_tokens": 1}, "response_metadata": {"cost": cost}} + for cost in costs + ] + + assert adapter._aggregate_usage(calls).get("cost") == expected + + async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): @@ -2115,20 +2166,25 @@ async def test_replayed_state_usage_counts_current_turn_only( # cost must reflect only the message emitted this turn, not the replayed one. import deepagents - prior = { - "role": "ai", - "content": "prior", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.001}, - } - current = { - "role": "ai", - "content": "now", - "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, - "response_metadata": {"cost": 0.002}, - } + prior = AIMessage( + content="prior", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + response_metadata={"cost": 0.001}, + ) async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + current = await call_model( + config, + AIMessage( + content="now", + usage_metadata={ + "input_tokens": 2, + "output_tokens": 2, + "total_tokens": 4, + }, + response_metadata={"cost": 0.002}, + ), + ) # Only the current turn's message is emitted as an update... yield ((), "updates", {"agent": {"messages": [current]}}) # ...but the final state also replays the prior turn. @@ -2427,47 +2483,360 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): adapter._validated_deepagents_settings({"model": "untrusted-model"}) -async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. - import deepagents - - sub_ai = { - "role": "ai", - "content": "subagent work", - "id": "sub-1", - "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - } - main_ai = { - "role": "ai", - "content": "final", - "id": "main-1", - "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5}, +def chat_completion( + usage: dict[str, Any] | None, + *, + tool_calls: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Return an OpenAI Chat Completions response body with the given usage.""" + + message: dict[str, Any] = {"role": "assistant", "content": "done"} + if tool_calls: + message["content"] = None + message["tool_calls"] = tool_calls + body: dict[str, Any] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 0, + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", + } + ], } + if usage is not None: + body["usage"] = usage + return body - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): - assert subgraphs is True - # subagent step under a subgraph namespace, emitted twice (dedup by id) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - # main graph step + replayed final state - yield ((), "updates", {"agent": {"messages": [main_ai]}}) - yield ((), "values", {"messages": [main_ai]}) - agent = MagicMock() - agent.astream = astream - monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) +LS_TOOL_CALL = { + "id": "call-1", + "type": "function", + "function": {"name": "ls", "arguments": '{"path": "/"}'}, +} - output = await invoke_once(make_payload(tmp_path)) - # subagent (10/20/30) + main (2/3/5), the duplicate subagent message counted once - assert output["usage"] == { - "prompt_tokens": 12, - "completion_tokens": 23, - "total_tokens": 35, +@pytest.fixture(name="openai_endpoint") +def openai_endpoint_fixture(fake_sdks, monkeypatch) -> dict[str, Any]: + """Run the real Deep Agents graph and ChatOpenAI against a scripted endpoint. + + Tests append response bodies, ``httpx.Response`` objects, or exceptions to + raise to ``replies``; each model request takes the next one, and its JSON + body is kept in ``requests``. ``profile`` sets the chat model profile, which Deep Agents uses + to choose its summarization trigger. + """ + + import httpx + + for name in ( + "deepagents", + "deepagents.backends", + "deepagents.middleware", + "deepagents.middleware.subagents", + "langchain.chat_models", + "langgraph", + "langgraph.checkpoint", + "langgraph.checkpoint.sqlite", + "langgraph.checkpoint.sqlite.aio", + ): + monkeypatch.delitem(sys.modules, name, raising=False) + endpoint: dict[str, Any] = {"replies": [], "requests": [], "profile": None} + + def respond(request: httpx.Request) -> httpx.Response: + endpoint["requests"].append(json.loads(request.content)) + reply = endpoint["replies"].pop(0) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, httpx.Response): + return reply + return httpx.Response(200, json=reply) + + def build_chat_openai(**kwargs: Any) -> InstalledChatOpenAI: + return InstalledChatOpenAI( + **kwargs, + max_retries=0, + profile=endpoint["profile"], + http_async_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + + fake_sdks["chat_openai"].side_effect = build_chat_openai + return endpoint + + +async def run_invocations(payload: dict[str, Any], count: int) -> list[AgentRunResult]: + runtime = adapter.DeepAgentsRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return [ + await runtime.invoke(*lifecycle_invocation(payload)) for _ in range(count) + ] + finally: + await runtime.stop() + + +async def test_summarization_model_call_usage_is_counted( + tmp_path, make_payload, openai_endpoint +): + # Deep Agents summarizes older history with a separate model call whose response + # never enters graph state. Its usage belongs to the invocation that made it. + openai_endpoint["profile"] = {"max_input_tokens": 1000} + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1500, "completion_tokens": 10, "total_tokens": 1510} + ), + # Second invocation: the summary call, then the agent's model call. + chat_completion( + {"prompt_tokens": 1400, "completion_tokens": 60, "total_tokens": 1460} + ), + chat_completion( + {"prompt_tokens": 300, "completion_tokens": 5, "total_tokens": 305} + ), + ] + payload = make_payload(tmp_path) + payload["request"]["input"] = "word " * 6000 + + first, second = await run_invocations(payload, 2) + + summary_request, agent_request = openai_endpoint["requests"][1:] + assert "tools" not in summary_request + assert "tools" in agent_request + assert first.usage == AgentUsage( + input_tokens=1500, output_tokens=10, total_tokens=1510 + ) + assert second.status is AgentRunStatus.SUCCEEDED + assert second.usage == AgentUsage( + input_tokens=1700, output_tokens=65, total_tokens=1765 + ) + + +async def test_subagent_model_calls_are_counted( + tmp_path, make_payload, openai_endpoint +): + # Two parallel ``task`` calls run the general-purpose subagent twice; each + # subagent's model call belongs to this invocation. + task_calls = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo the word {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + tool_calls=task_calls, + ), + chat_completion( + {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20} + ), + chat_completion( + {"prompt_tokens": 100, "completion_tokens": 100, "total_tokens": 200} + ), + chat_completion( + {"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000} + ), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert len(openai_endpoint["requests"]) == 4 + subgraphs = { + event["subgraph"] for event in result.output["events"] if "subgraph" in event } - # the subgraph step is recorded with its namespace label - assert any(evt.get("subgraph") == "task:researcher" for evt in output["events"]) + assert len(subgraphs) == 2 + assert result.usage == AgentUsage( + input_tokens=1111, output_tokens=1111, total_tokens=2222 + ) + + +async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( + tmp_path, make_payload, openai_endpoint +): + tool_call_reply = chat_completion( + {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + tool_calls=[LS_TOOL_CALL], + ) + openai_endpoint["replies"] += [tool_call_reply] * 20 + payload = make_payload(tmp_path) + payload["config"]["runtime"] = {"max_turns": 6} + + (result,) = await run_invocations(payload, 1) + + calls = len(openai_endpoint["requests"]) + assert calls > 1 + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "deepagents_recursion_limit_reached" + assert result.usage == AgentUsage( + input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + ) + + +def provider_failure(kind: str) -> Any: + """Return a scripted endpoint reply that fails a model call.""" + + import httpx + + if kind == "timeout": + return httpx.ReadTimeout("provider did not answer in time") + return httpx.Response(int(kind), json={"error": {"message": "request failed"}}) + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + # A 4xx rejection other than 408 spends nothing, so the finished call's + # usage stays known. + pytest.param( + "400", + AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + id="400-rejection", + ), + # The provider may have done work without reporting usage. + pytest.param("408", None, id="408-request-timeout"), + pytest.param("503", None, id="503-server-error"), + pytest.param("timeout", None, id="read-timeout"), + ], +) +async def test_failed_model_call_usage_follows_provider_response( + tmp_path, make_payload, openai_endpoint, failure, expected +): + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128}, + tool_calls=[LS_TOOL_CALL], + ), + provider_failure(failure), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == expected + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + pytest.param( + "429", + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32}, + id="429-then-success", + ), + pytest.param("503", None, id="503-then-success"), + ], +) +async def test_retried_model_call_usage_follows_failed_attempt( + fake_sdks, openai_endpoint, failure, expected +): + # LangChain's with_retry runs each attempt as its own chat-model call, as the + # Deep Agents summarization model does. + openai_endpoint["replies"] += [ + provider_failure(failure), + chat_completion( + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32} + ), + ] + model = fake_sdks["chat_openai"]( + model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + api_key="test123", + base_url="https://integrate.api.nvidia.com/v1", + ) + model_usage = adapter.ModelCallUsage() + + await model.with_retry(stop_after_attempt=2, wait_exponential_jitter=False).ainvoke( + "summarize", config={"callbacks": [model_usage]} + ) + + assert adapter._aggregate_usage(model_usage.calls()) == expected + + +async def test_recorder_keeps_a_cancelled_call_unreported(): + # Recorder unit test: a cancelled call ends without on_llm_end and, when + # LangChain gathers it, without on_llm_error; its start record keeps the + # invocation's counts unknown. + model_usage = adapter.ModelCallUsage() + await call_model( + {"callbacks": [model_usage]}, + AIMessage( + content="done", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ), + ) + model_usage.on_chat_model_start({}, [], run_id=uuid.uuid4()) + + assert adapter._aggregate_usage(model_usage.calls()) is None + + +@pytest.mark.parametrize( + ("usages", "expected"), + [ + pytest.param( + [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], + AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + id="explicit-zero", + ), + pytest.param( + [{"prompt_tokens": 100}], + AgentUsage(input_tokens=100), + id="input-only", + ), + pytest.param( + [{"completion_tokens": 5}], + AgentUsage(output_tokens=5), + id="output-only", + ), + pytest.param( + [{"prompt_tokens": 100, "completion_tokens": 5}], + AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + id="total-omitted", + ), + pytest.param( + [{"completion_tokens": 5, "total_tokens": 105}], + AgentUsage(output_tokens=5, total_tokens=105), + id="provider-total-with-input-omitted", + ), + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], + AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + id="provider-total-zero-kept", + ), + pytest.param( + [ + {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105}, + None, + ], + None, + id="one-call-without-usage", + ), + ], +) +async def test_partial_usage_reports_keep_missing_counts_unknown( + tmp_path, make_payload, openai_endpoint, usages, expected +): + # ChatOpenAI turns an omitted count into 0; a count is known only when every + # model call in the invocation reported it. + openai_endpoint["replies"] += [ + *(chat_completion(usage, tool_calls=[LS_TOOL_CALL]) for usage in usages[:-1]), + chat_completion(usages[-1]), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == expected async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..08155598c 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -486,7 +486,6 @@ async def test_deepagents_builtin_subagent_delegation(): call["args"]["subagent_type"] == "general-purpose" for call in task_calls ), result.to_mapping() assert any("subgraph" in event for event in output["events"]), result.to_mapping() - # delegated steps are folded into this turn's usage aggregation assert output["resumed"] is False, result.to_mapping() diff --git a/tests/integrations/test_relay_scope_leak.py b/tests/integrations/test_relay_scope_leak.py index 9dabfdcd5..5d5dbc58a 100644 --- a/tests/integrations/test_relay_scope_leak.py +++ b/tests/integrations/test_relay_scope_leak.py @@ -168,7 +168,7 @@ async def test_uuid_request_id_seeds_real_relay_parent( monkeypatch, make_runtime_context ): async def fake_invoke(agent, user_message, thread_id, callbacks=None): - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) @@ -210,7 +210,7 @@ async def test_overlapping_turns_stay_telemetry_clean( async def fake_invoke(agent, user_message, thread_id, callbacks=None): if callbacks: await _overlapping_chain_runs(callbacks[0]) - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) recording_scope = _RecordingScope() From 5de490480561c05c6bc2f7b7b3d6d439735262ff Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:18 -0700 Subject: [PATCH 03/74] feat: add what_ran provenance to run results A RunResult says which agent and adapter ran, but not which Fabric version, models or MCP servers were behind it, so a rerun weeks later cannot tell what changed. Add an optional, versioned what_ran object. Each fact holds a value, or a state and a reason, plus how Fabric learned it: its source, collector and method. Core fills the facts it holds before it sends the request: its own version, the adapter ID, the requested models, the configured MCP servers with secret-free endpoints, and the environment provider. Core reads no adapter report yet, so adapter-reported facts read unsupported with reason key_not_declared. On the OpenShell host path, the host relabels the task core's facts as task_core, records its own version as host_core_version and sets the environment provider to openshell. A task result without a readable what_ran keeps what_ran absent. The Rust and Python readers apply the same structural checks, tested against one shared vector file. A malformed what_ran reads as absent and never fails the enclosing result. Core gains a direct dependency on sha2 0.10, already in the lockfile, for the string length rule. Rust code that builds RunResult with a struct literal must add the what_ran field. Python's RunResult.to_mapping() now always includes it. Signed-off-by: mnajafian-nv --- Cargo.lock | 1 + Cargo.toml | 4 + crates/fabric-core/Cargo.toml | 1 + crates/fabric-core/src/endpoint.rs | 232 +++++ crates/fabric-core/src/lib.rs | 2 + crates/fabric-core/src/runtime.rs | 59 ++ crates/fabric-core/src/runtime/openshell.rs | 102 ++ crates/fabric-core/src/what_ran.rs | 984 ++++++++++++++++++ schemas/sdk/run-result.schema.json | 512 +++++++++ .../src/nemo_fabric/types.py | 141 ++- tests/fixtures/what_ran/reader-vectors.json | 809 ++++++++++++++ tests/python/test_native_sdk.py | 4 + tests/python/test_sdk_contract.py | 74 ++ 13 files changed, 2924 insertions(+), 1 deletion(-) create mode 100644 crates/fabric-core/src/endpoint.rs create mode 100644 crates/fabric-core/src/what_ran.rs create mode 100644 tests/fixtures/what_ran/reader-vectors.json diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..9cdefbdd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,10 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# SHA-256 for the `what_ran` string length rule, which ends a cut value with a +# digest prefix. Std has no SHA-256, and `sha2` 0.10 is already in the lockfile +# through the OpenShell provider. +sha2 = { version = "0.10.9", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/src/endpoint.rs b/crates/fabric-core/src/endpoint.rs new file mode 100644 index 000000000..cddabac29 --- /dev/null +++ b/crates/fabric-core/src/endpoint.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Secret-free identities for configured endpoints. + +use std::net::Ipv6Addr; + +/// The endpoint identity of `url`: its lowercase scheme and host, and its +/// port as a number. An omitted port becomes the scheme's default, 80 for +/// `http` and `ws` and 443 for `https` and `wss`; other schemes keep a port +/// only when one is written. An IPv6 host keeps its brackets in canonical, +/// compressed lowercase form, such as `[2001:db8::1]`. The user info, path, +/// query and fragment never enter it, because a hosted service can carry a +/// key in any of them. +/// +/// `None` when `url` has no `scheme://host`, when an `@` follows the +/// authority (a password holding an unencoded `/`, `\`, `?` or `#` ends +/// the authority early, so the host cannot be told apart from the user +/// info), when the host holds a character other than ASCII letters, digits, +/// `-`, `.`, `_` and `~`, when a bracketed host is not an IPv6 address +/// (a scoped address with a zone ID is rejected too), or when the port is +/// not a number below 65,536. +pub(crate) fn endpoint_identity(url: &str) -> Option { + let (scheme, rest) = url.trim().split_once("://")?; + if !is_scheme(scheme) { + return None; + } + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + if tail.contains('@') { + return None; + } + let host_port = authority + .rsplit_once('@') + .map_or(authority, |(_, host_port)| host_port); + let (host, port) = split_host_port(host_port)?; + let scheme = scheme.to_ascii_lowercase(); + let mut identity = format!("{scheme}://{host}"); + if let Some(port) = port.or_else(|| default_port(&scheme)) { + identity.push_str(&format!(":{port}")); + } + Some(identity) +} + +/// Whether `scheme` is a URL scheme: a letter, then letters, digits, `+`, +/// `-` or `.`. +fn is_scheme(scheme: &str) -> bool { + let mut characters = scheme.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic()) + && characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '-' | '.') + }) +} + +/// Split `host[:port]` into the lowercase host, with a bracketed IPv6 +/// address in canonical form, and the port. `None` when the host is empty, +/// holds a character no host name holds, or is bracketed but not an IPv6 +/// address, or when the port is not a number below 65,536. An empty port +/// reads as no port. +fn split_host_port(host_port: &str) -> Option<(String, Option)> { + let (host, port) = match host_port.strip_prefix('[') { + Some(bracketed) => { + let (address, after) = bracketed.split_once(']')?; + let address: Ipv6Addr = address.parse().ok()?; + let port = match after { + "" => None, + after => Some(after.strip_prefix(':')?), + }; + (format!("[{address}]"), port) + } + None => { + let (host, port) = match host_port.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (host_port, None), + }; + let host_character = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '.' | '_' | '~') + }; + if host.is_empty() || !host.chars().all(host_character) { + return None; + } + (host.to_ascii_lowercase(), port) + } + }; + match port.filter(|port| !port.is_empty()) { + None => Some((host, None)), + Some(port) if port.bytes().all(|byte| byte.is_ascii_digit()) => { + port.parse::().ok().map(|port| (host, Some(port))) + } + Some(_) => None, + } +} + +fn default_port(scheme: &str) -> Option { + match scheme { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +/// The file name of a `stdio` server's executable: the part after the last +/// `/` or `\\`. +/// +/// `None` when the executable holds an `=` or any whitespace anywhere, or +/// when the file name is empty or holds a character other than ASCII +/// letters, digits, `.`, `_`, `+` and `-`. The plan keeps arguments in their +/// own field, so whitespace in the executable field means arguments may +/// have been written into it, and an argument can put a secret after a `/` +/// (`npx --token a/SECRET`). A leading `NAME=value` can do the same +/// (`TOKEN=a/SECRET server`). Both checks run before the folder is dropped. +/// A path with a space in a folder is rejected too. This is a conservative +/// extractor, not a shell parser. +pub(crate) fn executable_name(executable: &str) -> Option { + if executable.contains('=') || executable.chars().any(char::is_whitespace) { + return None; + } + let name = executable.rsplit(['/', '\\']).next().unwrap_or_default(); + let plain = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '+' | '-') + }; + (!name.is_empty() && name.chars().all(plain)).then(|| name.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_identity_keeps_scheme_host_and_port_only() { + for (url, identity) in [ + ( + "HTTP://user:pw@10.0.0.5:8000/v1/?key=abc#frag", + "http://10.0.0.5:8000", + ), + ("https://API.example.com/", "https://api.example.com:443"), + ("http://[::1]/v1", "http://[::1]:80"), + ( + "https://[2001:DB8:0:0::1]:8443/", + "https://[2001:db8::1]:8443", + ), + ("https://example.com", "https://example.com:443"), + ("https://example.com:443", "https://example.com:443"), + ( + "https://actions.zapier.com/mcp/sk-ak-SECRET/sse", + "https://actions.zapier.com:443", + ), + ("https://u%40x:p%2Fq@host:8443/v1", "https://host:8443"), + ("https://host:/v1", "https://host:443"), + ("https://host:0443/v1", "https://host:443"), + ("WSS://Host.Example/s", "wss://host.example:443"), + ("grpc://host:9000", "grpc://host:9000"), + ] { + assert_eq!(endpoint_identity(url).as_deref(), Some(identity), "{url}"); + } + } + + /// A URL whose host cannot be told apart from its user info, or that is + /// not a URL, has no identity, so no part of a password becomes a host. + #[test] + fn endpoint_identity_rejects_ambiguous_or_invalid_urls() { + for url in [ + "https://user:123/SECRET@host/x", + "https://u:p#SECRET@host/x", + "https://u:p?SECRET@host/x", + "https://user:pw@host/p@x", + r"https://api.example.com\SECRET@secret-host", + "https://ärzte.example/mcp", + "https://server.example.com/@owner/server/mcp", + "https://u:p/x", + "https://host:99999", + "https://host:8a", + "https://", + "1http://host", + "http://ho st/", + "http://[::1/", + "http://[]/", + "https://[SECRET]/mcp", + "http://[::::]/", + "http://[fe80::1%25eth0]/", + "http://[fe80::1%eth0]/", + "http://[::1]x/", + "localhost:8000", + "not a url", + ] { + assert_eq!(endpoint_identity(url), None, "{url}"); + } + } + + #[test] + fn executable_name_is_the_file_name_without_its_folder() { + for (executable, name) in [ + ("db-mcp", Some("db-mcp")), + ("/home/alice/.venv/bin/python", Some("python")), + (r"C:\tools\npx.cmd", Some("npx.cmd")), + (r"C:\MCP\server.exe", Some("server.exe")), + ] { + assert_eq!(executable_name(executable).as_deref(), name, "{executable}"); + } + } + + /// An `=` or whitespace anywhere rejects the executable before its + /// folder is dropped, so no part of an assignment or an argument + /// becomes the name. + #[test] + fn executable_name_rejects_assignments_and_unplain_names() { + for executable in [ + "OPENAI_API_KEY=SECRET npx", + "TOKEN=prefix/SECRET", + r"TOKEN=prefix\SECRET", + "TOKEN=https://SECRET", + "/opt/server --token=SECRET", + "npx --api-key SECRET", + "npx -y @org/server-github --token SECRET", + "/opt/My Tools/db server", + "npx --token a/SECRET", + r"npx --token a\SECRET", + r"C:\Program Files\MCP\server.exe", + "/opt/My Tools/db-server", + "db-mcp\t", + " db-mcp", + "'quoted server'", + "/opt/tools/ärzte", + "", + "/opt/tools/", + ] { + assert_eq!(executable_name(executable), None, "{executable}"); + } + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index af3493a60..98853977d 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -8,11 +8,13 @@ pub mod agent_config; pub mod agent_execution; pub mod config; pub mod doctor; +mod endpoint; mod environment; pub mod error; pub mod runtime; pub mod schema; mod task_command; +pub mod what_ran; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 3c3463596..893914ff9 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -32,6 +32,7 @@ use crate::config::{ }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; +use crate::what_ran::WhatRan; mod openshell; use openshell::OpenShellRuntimeAdapter; @@ -152,6 +153,15 @@ pub struct RunResult { /// Adapter-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// What executed for this invocation, and how NeMo Fabric learned each + /// fact. Absent when the producing core predates it, or when this core + /// finds it malformed or of another schema or version. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::what_ran::deserialize_lenient" + )] + pub what_ran: Option, } /// Runtime completion status. @@ -1913,6 +1923,11 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let what_ran = crate::what_ran::collect_before_request( + plan, + adapter_id(plan), + &runtime.environment.provider, + ); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -2167,6 +2182,7 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, + what_ran: Some(what_ran), }) } @@ -4418,6 +4434,7 @@ for line in sys.stdin: ], events: Vec::new(), metadata: BTreeMap::new(), + what_ran: None, }; let encoded = serde_json::to_value(&result).expect("serialize SDK run result"); @@ -5442,6 +5459,48 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_result_carries_what_ran_without_changing_the_outcome() { + let run_result_schema = + crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let validator = jsonschema::validator_for(&run_result_schema).expect("valid schema"); + let not_declared = serde_json::json!({ + "state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report" + }); + for (mode, status) in [ + ("success", RunStatus::Succeeded), + ("adapter_reported_failure", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, status); + let result_json = serde_json::to_value(&result).expect("result JSON"); + assert!(validator.is_valid(&result_json), "{result_json}"); + let what_ran = &result_json["what_ran"]; + assert_eq!(what_ran["schema"], "nemo.fabric.what_ran"); + assert_eq!(what_ran["version"], "v1alpha1"); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "acme.fabric.local-host"); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!(what_ran["harness"]["name"], not_declared); + assert_eq!(what_ran["harness"]["version"], not_declared); + assert_eq!(what_ran["models"], serde_json::json!({})); + assert_eq!(what_ran["mcp"], serde_json::json!({"servers": {}})); + assert_eq!( + what_ran["environment"], + serde_json::json!({"provider": "local"}) + ); + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..b8e7f5e80 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -433,6 +433,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + adopt_task_what_ran(&mut result, &runtime.environment); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -516,6 +517,16 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { } } +/// Attribute the task core's `what_ran` to the task core, this host's +/// version and this host's environment provider. A task result without a +/// readable `what_ran`, from an older task core or one this host cannot +/// decode, keeps `what_ran` absent and is otherwise unchanged. +fn adopt_task_what_ran(result: &mut RunResult, environment: &EnvironmentHandle) { + if let Some(what_ran) = result.what_ran.as_mut() { + crate::what_ran::attribute_to_task_core(what_ran, &environment.provider); + } +} + pub(super) fn release(environment: &EnvironmentHandle) -> Result<()> { let key = resource_key(environment)?; let previous = { @@ -847,6 +858,97 @@ mod tests { } } + /// A task result as the task core returns it, with `what_ran` from the + /// shared reader vectors when given. + fn task_result(what_ran: Option) -> RunResult { + let mut result = serde_json::json!({ + "agent_name": "agent", "harness": "nvidia.fabric.claude", "adapter_kind": "python", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "output": "done" + }); + if let Some(what_ran) = what_ran { + result["what_ran"] = what_ran; + } + serde_json::from_value(result).expect("a task result decodes whatever what_ran holds") + } + + fn task_what_ran() -> Value { + let vectors: Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors["base"].clone() + } + + fn facts_collected_by(value: &Value, collector: &str) -> usize { + match value { + Value::Object(map) if map.contains_key("source") => { + usize::from(map.get("collected_by") == Some(&Value::from(collector))) + } + Value::Object(map) => map + .values() + .map(|value| facts_collected_by(value, collector)) + .sum(), + _ => 0, + } + } + + #[test] + fn task_what_ran_is_attributed_to_the_task_core_and_this_host() { + for task_version in [crate::version(), "0.5.0"] { + let mut what_ran = task_what_ran(); + what_ran["fabric"]["core_version"]["value"] = Value::from(task_version); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + assert_task_facts_are_attributed(&what_ran, task_version); + } + } + + fn assert_task_facts_are_attributed(what_ran: &Value, task_version: &str) { + let host_version = serde_json::json!({"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}); + let task_core_version = serde_json::json!({"value": task_version, "source": "observed", + "collected_by": "task_core", "method": "crate_version"}); + assert_eq!(what_ran["fabric"]["core_version"], task_core_version); + assert_eq!(what_ran["fabric"]["host_core_version"], host_version); + assert_eq!( + facts_collected_by(what_ran, "host_core"), + 1, + "only host_core_version is the host's fact" + ); + assert_eq!(what_ran["environment"]["provider"], "openshell"); + assert_eq!( + what_ran["models"]["default"]["requested"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["mcp"]["servers"]["docs"]["endpoint"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["harness"]["version"]["collected_by"], "adapter", + "adapter facts keep their collector" + ); + } + + #[test] + fn task_result_without_readable_what_ran_is_kept_without_it() { + let mut undecodable = task_what_ran(); + undecodable["harness"]["version"]["state"] = Value::from("withheld"); + for what_ran in [None, Some(undecodable)] { + let mut result = task_result(what_ran); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!(result.what_ran, None); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs new file mode 100644 index 000000000..ef5686ee4 --- /dev/null +++ b/crates/fabric-core/src/what_ran.rs @@ -0,0 +1,984 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What executed for one invocation, and how NeMo Fabric learned each fact. +//! +//! [`WhatRan`] rides on [`RunResult`](crate::RunResult). Each fact is a +//! [`Fact`]: a value, or a [`FactState`] with a reason, and in both cases +//! the [`FactSource`], the component that collected it and the method it +//! used. A few names fixed by the run plan stay plain strings: the adapter +//! ID, each model role's provider, the environment provider, and each MCP +//! server's origin and transport. +//! +//! A string value longer than 256 characters, in a string fact or as an +//! element of a string-list fact, keeps its first 240 characters, then `~`, +//! then the first 15 hex digits of its SHA-256. Map keys and the plain names +//! above are never cut. +//! +//! `what_ran` is evidence. It never decides which runs are compared. +//! +//! Compatibility rules for readers: +//! +//! - `source` and `state` are closed sets. A new value changes +//! [`WHAT_RAN_VERSION`]. +//! - `collected_by`, `method` and `reason` are open. Readers accept values +//! they do not know. +//! - Readers accept unknown properties. Adding an optional property keeps +//! [`WHAT_RAN_VERSION`]; removing one or changing its meaning changes it. +//! The Rust types drop unknown properties when they decode, so a host +//! that relays a task core's result drops them too. +//! - A fact holds `value`, or `state` and `reason`, never both. A +//! `RunResult` reader that finds a fact breaking this, any other value +//! the schema rejects, or another schema or version reads `what_ran` as +//! absent and still decodes the result. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::de::{DeserializeOwned, Error as _}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; +use crate::endpoint::{endpoint_identity, executable_name}; + +/// Schema name written into every [`WhatRan`]. +pub const WHAT_RAN_SCHEMA: &str = "nemo.fabric.what_ran"; + +/// Contract version written into every [`WhatRan`]. +pub const WHAT_RAN_VERSION: &str = "v1alpha1"; + +/// How NeMo Fabric learned a fact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactSource { + /// NeMo Fabric core saw it itself. + Observed, + /// The adapter, the harness, an MCP server or a remote service said so. + Reported, + /// It comes from the resolved run plan. + Configured, +} + +/// Why a fact has no value. A collector picks the first state that matches, +/// in declaration order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactState { + /// The fact cannot exist for this run. + NotApplicable, + /// The adapter or environment provider has no source for the fact. + Unsupported, + /// A source exists, but it failed, timed out, returned an invalid report + /// or never returned. + CollectionFailed, + /// NeMo Fabric holds no value and no evidence for the other states. + Unavailable, +} + +/// Pattern of the open codes `collected_by`, `method` and `reason`. +const CODE_PATTERN: &str = "^[a-z0-9_.:#+/-]{1,128}$"; + +/// Who collected a fact and how. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FactOrigin { + /// How NeMo Fabric learned the fact. + pub source: FactSource, + /// Component that collected the fact, such as `host_core`, `task_core` + /// or `adapter`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub collected_by: String, + /// How the collector got the fact, such as `crate_version`, + /// `resolved_plan` or `adapter_report`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub method: String, +} + +/// A fact with a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars( + rename = "KnownFact_for_{T}", + extend("not" = {"anyOf": [{"required": ["state"]}, {"required": ["reason"]}]}) +)] +pub struct KnownFact { + /// The fact itself: a string, a boolean or a list of strings. + pub value: T, + /// Who collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// A fact without a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("not" = {"required": ["value"]}))] +pub struct MissingFact { + /// Why the fact has no value. + pub state: FactState, + /// Short code that explains the state, such as `key_not_declared`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// Who would have collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// One fact: a `value`, or a `state` with a `reason`, never both. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +#[schemars(rename = "Fact_for_{T}")] +pub enum Fact { + /// The fact has a value. + Known(KnownFact), + /// The fact has no value. + Missing(MissingFact), +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact { + /// Decode a fact, rejecting one that holds `value` together with + /// `state` or `reason`, or an open code outside its pattern. Unknown + /// properties are ignored. + fn deserialize>(deserializer: D) -> Result { + let fact = serde_json::Map::::deserialize(deserializer)?; + let fact = if fact.contains_key("value") { + if fact.contains_key("state") || fact.contains_key("reason") { + return Err(D::Error::custom( + "a fact holds `value`, or `state` and `reason`, never both", + )); + } + KnownFact::deserialize(Value::Object(fact)).map(Self::Known) + } else { + MissingFact::deserialize(Value::Object(fact)).map(Self::Missing) + } + .map_err(D::Error::custom)?; + let origin = fact.origin(); + let reason = match &fact { + Self::Known(_) => None, + Self::Missing(missing) => Some(missing.reason.as_str()), + }; + if [origin.collected_by.as_str(), origin.method.as_str()] + .into_iter() + .chain(reason) + .all(is_code) + { + Ok(fact) + } else { + Err(D::Error::custom(format!( + "`collected_by`, `method` and `reason` must match {CODE_PATTERN}" + ))) + } + } +} + +/// Whether `code` matches [`CODE_PATTERN`]. +fn is_code(code: &str) -> bool { + (1..=128).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'.' | b':' | b'#' | b'+' | b'/' | b'-') + }) +} + +/// What executed for one invocation. +/// +/// Decoding a `WhatRan` directly checks the shape of every section and +/// fact. `RunResult` also checks the schema name and version, and that +/// every `plan` MCP entry names its transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct WhatRan { + /// Schema name, `nemo.fabric.what_ran`. + #[schemars(extend("const" = WHAT_RAN_SCHEMA))] + pub schema: String, + /// Contract version, `v1alpha1`. + #[schemars(extend("const" = WHAT_RAN_VERSION))] + pub version: String, + /// NeMo Fabric core that ran the adapter. + pub fabric: FabricFacts, + /// Adapter that ran. + pub adapter: AdapterFacts, + /// Harness behind the adapter. + pub harness: HarnessFacts, + /// Model facts keyed by the role names of the plan's `models` settings. + pub models: BTreeMap, + /// MCP server facts. + pub mcp: McpFacts, + /// Environment the runtime ran in. + pub environment: EnvironmentFacts, +} + +/// NeMo Fabric core facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FabricFacts { + /// Version of the core that ran the adapter. + pub core_version: Fact, + /// Version of the host core that relayed the result from a task core + /// in another environment. `not_applicable` for a local run. + pub host_core_version: Fact, +} + +/// Adapter facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AdapterFacts { + /// Adapter ID from the plan, when an adapter was resolved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Version of the adapter package, as the adapter reports it. + pub package_version: Fact, +} + +/// Harness facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct HarnessFacts { + /// Name of the harness program that ran, as the adapter reports it. + /// Core never derives it from the adapter ID. + pub name: Fact, + /// Version of the harness program that ran, as the adapter reports it. + pub version: Fact, +} + +/// Facts about one model role. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelFacts { + /// Provider from the plan. + pub provider: String, + /// Model the plan requested. + pub requested: Fact, + /// Model that served, as the harness or provider reports it. + pub served: Fact, +} + +/// MCP facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpFacts { + /// Servers keyed by name. Empty when the plan configures no MCP server + /// and the harness reports none. + pub servers: BTreeMap, +} + +/// Facts about one MCP server. +/// +/// An entry with origin `harness` is a server the harness reports and the +/// plan does not configure. It has the same shape: its `endpoint` reads +/// `not_applicable` with reason `not_configured`, its `transport` may be +/// absent, and runtime facts the adapter did not collect read +/// `unavailable` with reason `not_collected`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend( + "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, + "then" = {"required": ["transport"], "properties": {"transport": {"type": "string"}}} +))] +pub struct McpServerFacts { + /// Where the entry comes from: `plan` for a server the plan configures, + /// `harness` for one only the harness reports. + pub origin: String, + /// Transport, such as `stdio` or `streamable-http`. Always present for + /// a `plan` entry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport: Option, + /// Configured endpoint without secrets: the lowercase scheme and host + /// and the port for a network server, or the executable's file name for + /// a `stdio` server. + pub endpoint: Fact, + /// What the harness learned when it connected, as the adapter reports it. + pub runtime: McpRuntimeFacts, +} + +/// What the harness learned about one MCP server when it connected. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpRuntimeFacts { + /// Whether the harness connected to the server. + pub connected: Fact, + /// MCP protocol version negotiated with the server. + pub protocol_version: Fact, + /// Name the server reports for itself. + pub server_name: Fact, + /// Version the server reports for itself. + pub server_version: Fact, + /// Raw tool names the server lists, each bounded by the string length + /// rule. + pub tool_names: Fact>, +} + +/// Environment facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentFacts { + /// Environment provider, such as `local`. + pub provider: String, +} + +const HOST_CORE: &str = "host_core"; +const TASK_CORE: &str = "task_core"; +const ADAPTER: &str = "adapter"; +const ORIGIN_PLAN: &str = "plan"; +const METHOD_CRATE_VERSION: &str = "crate_version"; +const METHOD_RESOLVED_PLAN: &str = "resolved_plan"; +const METHOD_ADAPTER_REPORT: &str = "adapter_report"; +const REASON_KEY_NOT_DECLARED: &str = "key_not_declared"; +const REASON_LOCAL_RUN: &str = "local_run"; +const REASON_NOT_CONFIGURED: &str = "not_configured"; +const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; +const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; + +/// Longest string value kept whole, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Characters a longer string value keeps before `~` and its digest prefix. +const TRUNCATED_PREFIX_CHARS: usize = 240; +/// Hex digits of the SHA-256 that end a truncated string value. +const TRUNCATED_DIGEST_HEX_DIGITS: usize = 15; +const _: () = assert!(TRUNCATED_PREFIX_CHARS + 1 + TRUNCATED_DIGEST_HEX_DIGITS == MAX_VALUE_CHARS); + +impl Fact { + /// A known string fact, with the string length rule applied. + fn text(value: &str, source: FactSource, collected_by: &str, method: &str) -> Self { + Self::Known(KnownFact { + value: bounded_text(value), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } +} + +impl Fact { + fn origin(&self) -> &FactOrigin { + match self { + Self::Known(known) => &known.origin, + Self::Missing(missing) => &missing.origin, + } + } + + fn origin_mut(&mut self) -> &mut FactOrigin { + match self { + Self::Known(known) => &mut known.origin, + Self::Missing(missing) => &mut missing.origin, + } + } + + fn missing( + state: FactState, + reason: &str, + source: FactSource, + collected_by: &str, + method: &str, + ) -> Self { + Self::Missing(MissingFact { + state, + reason: reason.to_string(), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } + + /// An adapter-reported fact while core reads no adapter report key. + fn report_key_not_declared() -> Self { + Self::missing( + FactState::Unsupported, + REASON_KEY_NOT_DECLARED, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) + } +} + +impl WhatRan { + /// The origin of every fact, so a host can relabel the facts a task + /// core collected. + fn origins_mut(&mut self) -> Vec<&mut FactOrigin> { + let Self { + fabric, + adapter, + harness, + models, + mcp, + .. + } = self; + let mut origins = vec![ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + adapter.package_version.origin_mut(), + harness.name.origin_mut(), + harness.version.origin_mut(), + ]; + for model in models.values_mut() { + origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); + } + for server in mcp.servers.values_mut() { + let runtime = &mut server.runtime; + origins.extend([ + server.endpoint.origin_mut(), + runtime.connected.origin_mut(), + runtime.protocol_version.origin_mut(), + runtime.server_name.origin_mut(), + runtime.server_version.origin_mut(), + runtime.tool_names.origin_mut(), + ]); + } + origins + } + + /// Whether every `plan` MCP entry names its transport. Only a `harness` + /// entry may leave it out. + fn plan_servers_have_transports(&self) -> bool { + self.mcp + .servers + .values() + .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) + } +} + +/// Collect the facts the core that runs the adapter knows before it sends +/// the request: its own version and the plan's adapter ID, models, MCP +/// servers and environment provider. +/// +/// Core reads no adapter report key yet, so every adapter-reported fact +/// reads `unsupported` with reason `key_not_declared`. The collecting core +/// runs the adapter in a local environment, so `host_core_version` reads +/// `not_applicable`. A host that relays the result from a task core calls +/// [`attribute_to_task_core`]. +pub(crate) fn collect_before_request( + plan: &RunPlan, + adapter_id: Option, + environment_provider: &str, +) -> WhatRan { + WhatRan { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: FabricFacts { + core_version: core_version_fact(), + host_core_version: Fact::missing( + FactState::NotApplicable, + REASON_LOCAL_RUN, + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ), + }, + adapter: AdapterFacts { + id: adapter_id, + package_version: Fact::report_key_not_declared(), + }, + harness: HarnessFacts { + name: Fact::report_key_not_declared(), + version: Fact::report_key_not_declared(), + }, + models: plan + .config + .models + .iter() + .map(|(role, model)| (role.clone(), model_facts(model))) + .collect(), + mcp: McpFacts { + servers: plan + .capability_plan + .mcp_servers + .iter() + .map(|(name, server)| (name.clone(), mcp_server_facts(server))) + .collect(), + }, + environment: EnvironmentFacts { + provider: environment_provider.to_string(), + }, + } +} + +fn core_version_fact() -> Fact { + Fact::text( + crate::version(), + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ) +} + +fn model_facts(model: &ModelConfig) -> ModelFacts { + ModelFacts { + provider: model.provider.clone(), + requested: Fact::text( + &model.model, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + served: Fact::report_key_not_declared(), + } +} + +fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { + McpServerFacts { + origin: ORIGIN_PLAN.to_string(), + transport: Some(server.transport.as_str().to_string()), + endpoint: configured_endpoint(server.transport, &server.url), + runtime: McpRuntimeFacts { + connected: Fact::report_key_not_declared(), + protocol_version: Fact::report_key_not_declared(), + server_name: Fact::report_key_not_declared(), + server_version: Fact::report_key_not_declared(), + tool_names: Fact::report_key_not_declared(), + }, + } +} + +/// The configured endpoint of one MCP server, without secrets. `url` holds +/// the server URL, or the command for a `stdio` server. +/// +/// An empty `url` reads `unavailable` with reason `not_configured`. A value +/// core cannot reduce to its safe form reads `collection_failed`, and the +/// raw value never enters the fact. +fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { + let missing = |state, reason| { + Fact::missing( + state, + reason, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ) + }; + if url.trim().is_empty() { + return missing(FactState::Unavailable, REASON_NOT_CONFIGURED); + } + let endpoint = match transport { + McpTransport::Stdio => executable_name(url).ok_or(REASON_EXECUTABLE_UNPARSED), + McpTransport::Sse | McpTransport::StreamableHttp => { + endpoint_identity(url).ok_or(REASON_ENDPOINT_UNPARSED) + } + }; + match endpoint { + Ok(endpoint) => Fact::text( + &endpoint, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + Err(reason) => missing(FactState::CollectionFailed, reason), + } +} + +/// Attribute a task core's `what_ran` on the host that relayed its result. +/// +/// The task core runs in a projected `local` environment and labels its +/// own facts `host_core`. The host relabels them `task_core`, records its +/// own version as `host_core_version`, and replaces the environment +/// provider with its own, such as `openshell`. +pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { + for origin in what_ran.origins_mut() { + if origin.collected_by == HOST_CORE { + origin.collected_by = TASK_CORE.to_string(); + } + } + what_ran.fabric.host_core_version = core_version_fact(); + what_ran.environment.provider = environment_provider.to_string(); +} + +/// `text` whole when it has at most 256 characters. A longer string keeps +/// its first 240 characters, then `~`, then the first 15 hex digits of the +/// whole string's SHA-256, so two long values collide only on a 60-bit +/// hash collision. Characters are Unicode scalar values, and the digest +/// covers the UTF-8 bytes. +fn bounded_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut bounded: String = text.chars().take(TRUNCATED_PREFIX_CHARS).collect(); + bounded.push('~'); + let digest_hex: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + bounded.push_str(&digest_hex[..TRUNCATED_DIGEST_HEX_DIGITS]); + bounded +} + +/// Decode `RunResult.what_ran`, reading it as absent when it breaks the +/// schema or names another schema or version, so the rest of the result +/// still decodes. +pub(crate) fn deserialize_lenient<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value + .and_then(|value| serde_json::from_value::(value).ok()) + .filter(|what_ran| { + what_ran.schema == WHAT_RAN_SCHEMA + && what_ran.version == WHAT_RAN_VERSION + && what_ran.plan_servers_have_transports() + })) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::json; + + use super::*; + use crate::config::{ + FabricConfig, ResolveContext, resolve_run_plan_from_config_with_adapter_directories, + }; + + fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") + } + + fn resolve_plan(config: Value) -> RunPlan { + let root = repository_root(); + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + resolve_run_plan_from_config_with_adapter_directories( + config, + ResolveContext::new(&root), + &[root.join("adapters")], + ) + .expect("resolve plan") + } + + fn config_for(adapter_id: &str) -> Value { + json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "what-ran-agent"}, + "harness": {"adapter_id": adapter_id}, + "runtime": {} + }) + } + + /// A Claude Code plan whose MCP servers hold a secret in every place a + /// configured server can: URL user info, path, query and fragment, + /// headers, and a `stdio` command's folder, arguments and environment. + fn plan_with_secrets() -> RunPlan { + let mut config = config_for("nvidia.fabric.claude"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + }); + config["mcp"] = json!({"servers": { + "github": { + "transport": "streamable-http", + "url": "https://user:SECRET_PASSWORD@API.GitHubCopilot.com/mcp/SECRET_PATH/?token=SECRET_QUERY#SECRET_FRAGMENT", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + }, + "files": { + "transport": "stdio", + "url": "/home/SECRET_USER/.venv/bin/files-mcp", + "args": ["--api-key", "SECRET_ARGUMENT"], + "env": {"FILES_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + } + }}); + resolve_plan(config) + } + + fn collect(plan: &RunPlan) -> Value { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + serde_json::to_value(collect_before_request(plan, adapter_id, "local")) + .expect("serialize what_ran") + } + + fn not_declared() -> Value { + json!({"state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + fn configured(value: &str) -> Value { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + } + + fn mcp_server(transport: &str, endpoint: Value) -> Value { + let missing = not_declared(); + json!({ + "origin": "plan", + "transport": transport, + "endpoint": endpoint, + "runtime": { + "connected": missing, + "protocol_version": missing, + "server_name": missing, + "server_version": missing, + "tool_names": missing + } + }) + } + + /// Every JSON object that carries `source` is a fact. + fn facts(value: &Value) -> Vec<&serde_json::Map> { + match value { + Value::Object(map) if map.contains_key("source") => vec![map], + Value::Object(map) => map.values().flat_map(facts).collect(), + _ => Vec::new(), + } + } + + #[test] + fn collected_facts_read_the_plan_and_mark_adapter_facts_key_not_declared() { + let what_ran = collect(&plan_with_secrets()); + let not_declared = not_declared(); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "nvidia.fabric.claude", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": {"default": { + "provider": "anthropic", + "requested": configured("claude-sonnet-4-5"), + "served": not_declared + }}, + "mcp": {"servers": { + "events": mcp_server("sse", configured("http://localhost:8931")), + "files": mcp_server("stdio", configured("files-mcp")), + "github": mcp_server( + "streamable-http", + configured("https://api.githubcopilot.com:443"), + ) + }}, + "environment": {"provider": "local"} + }) + ); + } + + #[test] + fn collected_facts_hold_no_configured_secret() { + let text = serde_json::to_string(&collect(&plan_with_secrets())).expect("JSON"); + + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn every_fact_has_a_value_or_a_state_with_a_reason_never_both() { + let what_ran = collect(&plan_with_secrets()); + let facts = facts(&what_ran); + + // 2 fabric + 1 adapter + 2 harness + 2 model + 3 servers x (endpoint + 5 runtime). + assert_eq!(facts.len(), 25); + for fact in facts { + assert_ne!( + fact.contains_key("value"), + fact.contains_key("state"), + "{fact:?}" + ); + assert_eq!( + fact.contains_key("state"), + fact.contains_key("reason"), + "{fact:?}" + ); + for key in ["collected_by", "method"] { + assert!(fact.get(key).is_some_and(Value::is_string), "{fact:?}"); + } + } + } + + #[test] + fn plan_without_models_or_mcp_servers_reads_empty_maps() { + let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); + + assert_eq!(what_ran["models"], json!({})); + assert_eq!(what_ran["mcp"], json!({"servers": {}})); + } + + #[test] + fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { + let mut config = config_for("nvidia.fabric.claude"); + config["mcp"] = json!({"servers": { + "arguments": { + "transport": "stdio", + "url": "npx --token a/SECRET", + "exposure": "harness_native" + }, + "spaced": { + "transport": "stdio", + "url": "/opt/My Tools/db-server", + "exposure": "harness_native" + } + }}); + + let what_ran = collect(&resolve_plan(config)); + + let unparsed = json!({"state": "collection_failed", "reason": "executable_unparsed", + "source": "configured", "collected_by": "host_core", "method": "resolved_plan"}); + for name in ["arguments", "spaced"] { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"], unparsed, + "{name}" + ); + } + let text = serde_json::to_string(&what_ran).expect("JSON"); + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn unparsed_or_empty_endpoint_reads_a_state_without_the_raw_value() { + for (transport, url, state, reason) in [ + ( + McpTransport::StreamableHttp, + "https://u:p/SECRET@host/x", + "collection_failed", + "endpoint_unparsed", + ), + ( + McpTransport::Stdio, + "TOKEN=prefix/SECRET", + "collection_failed", + "executable_unparsed", + ), + (McpTransport::Sse, " ", "unavailable", "not_configured"), + (McpTransport::Stdio, "", "unavailable", "not_configured"), + ] { + let endpoint = serde_json::to_value(configured_endpoint(transport, url)) + .expect("serialize endpoint"); + + assert_eq!( + endpoint, + json!({"state": state, "reason": reason, "source": "configured", + "collected_by": "host_core", "method": "resolved_plan"}), + "{url}" + ); + } + } + + #[test] + fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { + let exact = "é".repeat(256); + assert_eq!(bounded_text(&exact), exact); + + // Expected digests: the first 15 hex digits of `sha256sum` over the + // whole value's UTF-8 bytes. + for (long, prefix, digest) in [ + (format!("{}a", "m".repeat(300)), "m", "96858eedeb3c8e6"), + ("é".repeat(300), "é", "7250b66610f8b7d"), + ] { + let bounded = bounded_text(&long); + + assert_eq!(bounded, format!("{}~{digest}", prefix.repeat(240))); + assert_eq!(bounded.chars().count(), 256); + } + } + + fn run_result_with(what_ran: Value) -> crate::RunResult { + serde_json::from_value(json!({ + "agent_name": "agent", "harness": "nvidia.fabric.codex", "adapter_kind": "python", + "runtime_id": "runtime-1", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "what_ran": what_ran + })) + .expect("the result decodes whatever what_ran holds") + } + + /// The reader vectors the Python SDK tests share, with each case's + /// patch applied to the base object. + fn reader_vectors(group: &str) -> Vec<(String, Value)> { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors[group] + .as_array() + .expect("vector group") + .iter() + .map(|case| { + let mut what_ran = vectors["base"].clone(); + for operation in case["patch"].as_array().expect("patch") { + apply_patch(&mut what_ran, operation); + } + (case["name"].as_str().expect("name").to_string(), what_ran) + }) + .collect() + } + + fn apply_patch(target: &mut Value, operation: &Value) { + let path: Vec<&str> = operation["path"] + .as_array() + .expect("path") + .iter() + .map(|key| key.as_str().expect("key")) + .collect(); + let Some((last, parents)) = path.split_last() else { + *target = operation["value"].clone(); + return; + }; + let parent = parents + .iter() + .fold(target, |value, key| &mut value[*key]) + .as_object_mut() + .expect("patch parent is an object"); + if operation.get("remove").is_some() { + parent.remove(*last); + } else { + parent.insert((*last).to_string(), operation["value"].clone()); + } + } + + fn what_ran_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let mut what_ran_schema = schema["properties"]["what_ran"].clone(); + what_ran_schema["$defs"] = schema["$defs"].clone(); + jsonschema::validator_for(&what_ran_schema).expect("valid what_ran schema") + } + + #[test] + fn reader_and_schema_keep_every_valid_vector() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("valid") { + assert!(validator.is_valid(&what_ran), "{name}"); + let decoded = run_result_with(what_ran.clone()).what_ran; + assert!(decoded.is_some(), "{name}"); + } + } + + #[test] + fn reader_reads_every_invalid_vector_as_absent_and_schema_rejects_it() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("invalid") { + assert!(!validator.is_valid(&what_ran), "{name}"); + assert_eq!(run_result_with(what_ran).what_ran, None, "{name}"); + } + assert_eq!(run_result_with(Value::Null).what_ran, None); + } + + #[test] + fn collected_facts_match_the_schema() { + let what_ran = collect(&plan_with_secrets()); + + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn run_result_accepts_unknown_properties_and_open_codes() { + let (_, newer) = reader_vectors("valid") + .into_iter() + .find(|(name, _)| name == "unknown properties and vendor codes") + .expect("vector"); + + let what_ran = run_result_with(newer) + .what_ran + .expect("unknown properties never fail what_ran"); + + assert_eq!( + what_ran.fabric.core_version.origin().collected_by, + "acme.collector" + ); + } +} diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 492ef7136..72a10b9bf 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -1,5 +1,25 @@ { "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, "AdapterKind": { "description": "Adapter implementation kind.", "oneOf": [ @@ -82,6 +102,19 @@ ], "type": "object" }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, "ErrorInfo": { "description": "Normalized error metadata.", "properties": { @@ -195,6 +228,421 @@ ], "type": "object" }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none.", + "type": "object" + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, its `transport` may be\nabsent, and runtime facts the adapter did not collect read\n`unavailable` with reason `not_collected`.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Always present for\na `plan` entry.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, "RunStatus": { "description": "Runtime completion status.", "oneOf": [ @@ -322,6 +770,59 @@ } }, "type": "object" + }, + "WhatRan": { + "description": "What executed for one invocation.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact. `RunResult` also checks the schema name and version, and that\nevery `plan` MCP entry names its transport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.", + "type": "object" + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + } + }, + "required": [ + "schema", + "version", + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -411,6 +912,17 @@ } ], "description": "Normalized invocation usage when reported by the adapter target." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "What executed for this invocation, and how NeMo Fabric learned each\nfact. Absent when the producing core predates it, or when this core\nfinds it malformed or of another schema or version." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 7ef06a715..0f23621a6 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -6,6 +6,8 @@ from __future__ import annotations import math +import re +from collections.abc import Callable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -1963,6 +1965,133 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +_WHAT_RAN_SCHEMA = "nemo.fabric.what_ran" +_WHAT_RAN_VERSION = "v1alpha1" +_FACT_SOURCES = frozenset({"observed", "reported", "configured"}) +_FACT_STATES = frozenset( + {"not_applicable", "unsupported", "collection_failed", "unavailable"} +) +_FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) + + +def _is_flag(value: Any) -> bool: + return isinstance(value, bool) + + +def _is_text_list(value: Any) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _is_code(value: Any) -> bool: + return isinstance(value, str) and _FACT_CODE.fullmatch(value) is not None + + +def _is_fact(fact: Any, is_value: Callable[[Any], bool]) -> bool: + """Whether ``fact`` holds ``value``, or ``state`` and ``reason``, never both.""" + + if not isinstance(fact, Mapping): + return False + source = fact.get("source") + if not (isinstance(source, str) and source in _FACT_SOURCES): + return False + if not (_is_code(fact.get("collected_by")) and _is_code(fact.get("method"))): + return False + if "value" in fact: + return "state" not in fact and "reason" not in fact and is_value(fact["value"]) + state = fact.get("state") + return ( + isinstance(state, str) + and state in _FACT_STATES + and _is_code(fact.get("reason")) + ) + + +def _has_facts(section: Any, facts: Mapping[str, Callable[[Any], bool]]) -> bool: + return isinstance(section, Mapping) and all( + _is_fact(section.get(name), is_value) for name, is_value in facts.items() + ) + + +def _is_model_entry(model: Any) -> bool: + return _has_facts( + model, {"requested": _is_text, "served": _is_text} + ) and isinstance(model.get("provider"), str) + + +def _is_mcp_server_entry(server: Any) -> bool: + if not _has_facts(server, {"endpoint": _is_text}): + return False + origin = server.get("origin") + transport = server.get("transport") + # Only a server the harness reports may leave out its transport. + transport_ok = isinstance(transport, str) or ( + transport is None and origin != "plan" + ) + runtime_facts = { + "connected": _is_flag, + "protocol_version": _is_text, + "server_name": _is_text, + "server_version": _is_text, + "tool_names": _is_text_list, + } + return ( + isinstance(origin, str) + and transport_ok + and _has_facts(server.get("runtime"), runtime_facts) + ) + + +def _is_readable_what_ran(value: Any) -> bool: + """Whether ``value`` is a ``what_ran`` object this SDK reads. + + Mirrors the Rust core's reader: the schema and version match, every + section and fact is well formed, and unknown properties are ignored. + """ + + if not isinstance(value, Mapping): + return False + if ( + value.get("schema") != _WHAT_RAN_SCHEMA + or value.get("version") != _WHAT_RAN_VERSION + ): + return False + adapter = value.get("adapter") + models = value.get("models") + mcp = value.get("mcp") + servers = mcp.get("servers") if isinstance(mcp, Mapping) else None + environment = value.get("environment") + return ( + _has_facts( + value.get("fabric"), + {"core_version": _is_text, "host_core_version": _is_text}, + ) + and _has_facts(adapter, {"package_version": _is_text}) + and (adapter.get("id") is None or isinstance(adapter.get("id"), str)) + and _has_facts(value.get("harness"), {"name": _is_text, "version": _is_text}) + and isinstance(models, Mapping) + and all(_is_model_entry(model) for model in models.values()) + and isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and isinstance(environment, Mapping) + and isinstance(environment.get("provider"), str) + ) + + +def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: + """Return ``what_ran``, or ``None`` when this SDK cannot read it. + + As in the Rust core, a value that is absent, not a JSON object, names + another schema or version, or breaks the section's structure reads as + absent instead of failing the result. + """ + + return value if _is_readable_what_ran(value) else None + + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -1987,6 +2116,13 @@ class RunResult(FabricMapping): telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. metadata: Result-specific structured details. + what_ran: What executed for this invocation, schema + ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. + Each fact holds ``value``, or ``state`` and ``reason``, plus + ``source``, ``collected_by`` and ``method``. ``None`` when the + producing core predates it, or when the value is not a + ``what_ran`` object of that schema and version with every section + and fact well formed, as the Rust core reads it. """ agent_name: str @@ -2004,6 +2140,7 @@ class RunResult(FabricMapping): telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] metadata: Mapping[str, Any] + what_ran: Mapping[str, Any] | None _fields = frozenset( { "agent_name", @@ -2021,9 +2158,10 @@ class RunResult(FabricMapping): "telemetry", "events", "metadata", + "what_ran", } ) - _json_fields = frozenset({"metadata"}) + _json_fields = frozenset({"metadata", "what_ran"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -2059,6 +2197,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: FabricEvent.from_mapping(event) for event in data.get("events", []) ) data["metadata"] = _mapping(data.get("metadata", {}), "result metadata") + data["what_ran"] = _readable_what_ran(data.get("what_ran")) raw_output = data.get("output") if isinstance(raw_output, RunOutput): data["output"] = raw_output diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json new file mode 100644 index 000000000..eb21c95e6 --- /dev/null +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -0,0 +1,809 @@ +{ + "description": "Shared what_ran reader vectors. Each case applies its patch to base: a path with value sets that member (an empty path replaces the whole object), and a path with remove deletes it. Rust and Python readers keep every valid case and read every invalid case as absent; the generated run-result schema accepts and rejects the same cases.", + "base": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + }, + "host_core_version": { + "state": "not_applicable", + "reason": "local_run", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + "adapter": { + "id": "nvidia.fabric.claude", + "package_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "harness": { + "name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "models": { + "default": { + "provider": "anthropic", + "requested": { + "value": "claude-sonnet-4-5", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "served": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "mcp": { + "servers": { + "docs": { + "origin": "plan", + "transport": "streamable-http", + "endpoint": { + "value": "https://docs.example.com:443", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "files": { + "origin": "plan", + "transport": "stdio", + "endpoint": { + "value": "files-mcp", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + } + } + }, + "environment": { + "provider": "local" + } + }, + "valid": [ + { + "name": "local run as core collects it", + "patch": [] + }, + { + "name": "known values of every value type", + "patch": [ + { + "path": [ + "harness", + "version" + ], + "value": { + "value": "2.1.0", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + "fetch" + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "task core result relayed by an OpenShell host", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "task_core" + }, + { + "path": [ + "fabric", + "host_core_version" + ], + "value": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + { + "path": [ + "environment", + "provider" + ], + "value": "openshell" + } + ] + }, + { + "name": "harness-origin MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "plugin" + ], + "value": { + "origin": "harness", + "endpoint": { + "state": "not_applicable", + "reason": "not_configured", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "protocol_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_name": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "tool_names": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + } + } + } + } + ] + }, + { + "name": "unknown properties and vendor codes", + "patch": [ + { + "path": [ + "workspace" + ], + "value": { + "base_commit": { + "value": "abc" + } + } + }, + { + "path": [ + "fabric", + "core_version", + "confidence" + ], + "value": "high" + }, + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "acme.collector" + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "acme:withheld#1" + }, + { + "path": [ + "harness", + "version", + "method" + ], + "value": "acme.lookup/v2#names" + } + ] + }, + { + "name": "no adapter ID, models or MCP servers", + "patch": [ + { + "path": [ + "adapter", + "id" + ], + "remove": true + }, + { + "path": [ + "models" + ], + "value": {} + }, + { + "path": [ + "mcp", + "servers" + ], + "value": {} + } + ] + }, + { + "name": "128-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + } + ], + "invalid": [ + { + "name": "value with state", + "patch": [ + { + "path": [ + "harness", + "version", + "value" + ], + "value": "2.1.0" + } + ] + }, + { + "name": "value with reason only", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "reason" + ], + "value": "local_run" + } + ] + }, + { + "name": "neither value nor state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "remove": true + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "state without reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "unknown state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "value": "withheld" + } + ] + }, + { + "name": "unknown source", + "patch": [ + { + "path": [ + "harness", + "version", + "source" + ], + "value": "guessed" + } + ] + }, + { + "name": "number for a string value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": 6 + } + ] + }, + { + "name": "null value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": null + } + ] + }, + { + "name": "string for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": "true", + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "non-string tool name", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + 1 + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "method outside the code pattern", + "patch": [ + { + "path": [ + "models", + "default", + "requested", + "method" + ], + "value": "models.Default.model" + } + ] + }, + { + "name": "empty reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "" + } + ] + }, + { + "name": "129-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "non-ASCII code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "réason" + } + ] + }, + { + "name": "code with a trailing newline", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "key_not_declared\n" + } + ] + }, + { + "name": "number for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": 1, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "missing collected_by", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "remove": true + } + ] + }, + { + "name": "missing section", + "patch": [ + { + "path": [ + "harness" + ], + "remove": true + } + ] + }, + { + "name": "missing harness name", + "patch": [ + { + "path": [ + "harness", + "name" + ], + "remove": true + } + ] + }, + { + "name": "fact that is not an object", + "patch": [ + { + "path": [ + "adapter", + "package_version" + ], + "value": "1.0.0" + } + ] + }, + { + "name": "model without provider", + "patch": [ + { + "path": [ + "models", + "default", + "provider" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry with a null transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "value": null + } + ] + }, + { + "name": "MCP entry without runtime facts", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime" + ], + "remove": true + } + ] + }, + { + "name": "non-string environment provider", + "patch": [ + { + "path": [ + "environment", + "provider" + ], + "value": null + } + ] + }, + { + "name": "another schema", + "patch": [ + { + "path": [ + "schema" + ], + "value": "acme.what_ran" + } + ] + }, + { + "name": "another version", + "patch": [ + { + "path": [ + "version" + ], + "value": "v1beta1" + } + ] + }, + { + "name": "missing version", + "patch": [ + { + "path": [ + "version" + ], + "remove": true + } + ] + }, + { + "name": "string instead of an object", + "patch": [ + { + "path": [], + "value": "not an object" + } + ] + }, + { + "name": "array instead of an object", + "patch": [ + { + "path": [], + "value": [ + "fact" + ] + } + ] + } + ] +} diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index f571159b7..d58d1b5bd 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -418,6 +418,10 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert result["adapter_kind"] == "python" assert result["metadata"]["adapter_runner"] == "persistent_local_host" assert result["output"]["received"] == "hello native" + assert result.what_ran["fabric"]["core_version"]["value"] == native.version() + assert result.what_ran["adapter"]["id"] == "test.fabric.hermes_shim" + assert result.what_ran["harness"]["version"]["reason"] == "key_not_declared" + assert result.what_ran["environment"] == {"provider": "local"} assert result.output["native_mcp_servers"] == ["github"] assert any(artifact.name == "stdout" for artifact in result.artifacts.artifacts) assert first["status"] == "succeeded" diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index b71bea32d..403fb9fa8 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -8,6 +8,7 @@ import asyncio import json import threading +from copy import deepcopy from inspect import signature from pathlib import Path from typing import Any @@ -2001,6 +2002,79 @@ def test_run_result_wraps_normalized_usage(): assert result.usage.metadata == {"provider": "test"} +_WHAT_RAN_VECTORS = json.loads( + ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "what_ran" + / "reader-vectors.json" + ).read_text(encoding="utf-8") +) + + +def _what_ran_vector(case: dict[str, Any]) -> Any: + """Apply one shared reader vector's patch to the base ``what_ran``.""" + + what_ran = deepcopy(_WHAT_RAN_VECTORS["base"]) + for operation in case["patch"]: + path = operation["path"] + if not path: + what_ran = deepcopy(operation["value"]) + continue + *parents, last = path + parent = what_ran + for key in parents: + parent = parent[key] + if operation.get("remove"): + del parent[last] + else: + parent[last] = deepcopy(operation["value"]) + return what_ran + + +def test_run_result_exposes_what_ran_as_a_detached_json_mapping(): + what_ran = _what_ran_vector(_WHAT_RAN_VECTORS["valid"][0]) + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + detached = result.what_ran + detached["fabric"]["core_version"]["value"] = "mutated" + + assert result.what_ran == what_ran + assert result["what_ran"] == what_ran + assert result.to_mapping()["what_ran"] == what_ran + assert "what_ran" not in result.extra_fields + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["valid"], ids=lambda case: case["name"] +) +def test_run_result_keeps_valid_what_ran_vector(case): + what_ran = _what_ran_vector(case) + + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + assert result.what_ran == what_ran + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["invalid"], ids=lambda case: case["name"] +) +def test_run_result_reads_invalid_what_ran_vector_as_none(case): + result = RunResult.from_mapping(_run_result(what_ran=_what_ran_vector(case))) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + assert result.status == "succeeded" + + +@pytest.mark.parametrize("mapping", [_run_result(), _run_result(what_ran=None)]) +def test_run_result_without_what_ran_reads_none(mapping): + result = RunResult.from_mapping(mapping) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + + @pytest.mark.parametrize( "field", ["input_tokens", "cached_input_tokens", "output_tokens", "total_tokens"] ) From 4f16bfff9fbd64ff3389c6b0e00cb79411612070 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:18 -0700 Subject: [PATCH 04/74] docs: describe RunResult.what_ran in the Python SDK guide Explain the fact envelope, the four no-value states, the endpoint rule, the OpenShell attribution and the structural read, and keep the integration skill's RunResult table in step with the guide. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 74 +++++++++++++++++++ .../references/results-and-errors.md | 10 +++ 2 files changed, 84 insertions(+) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index adafb55f8..fa23c2a6c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -967,6 +967,7 @@ Important fields: | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation, and how NeMo Fabric learned each fact. | | `runtime_id`, `invocation_id`, `request_id` | IDs for correlation across runtimes, logs, telemetry, and artifacts. | These are structured correlation fields, not interchangeable metadata: @@ -996,6 +997,79 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +### What Ran + +`what_ran` records what executed for the invocation, as schema +`nemo.fabric.what_ran` version `v1alpha1`. Each fact is an object with a +`value`, or with a `state` and a `reason` when it has no value. Both forms +carry `source`, `collected_by`, and `method`: + +- `source` is `observed` when NeMo Fabric core saw the fact itself, `reported` + when the adapter, harness, an MCP server, or a remote service said so, and + `configured` when it comes from the resolved plan. +- `state` is `not_applicable` when the fact cannot exist for this run, + `unsupported` when there is no source for it, `collection_failed` when the + source failed, and `unavailable` when NeMo Fabric holds no value and no + evidence for the other states. +- `collected_by`, `method`, and `reason` are open codes that match + `^[a-z0-9_.:#+/-]{1,128}$`. Accept values you do not recognize. + `collected_by` is `task_core` for facts a core inside an OpenShell sandbox + collected. `method` names the technique, such as `crate_version`, + `resolved_plan`, or `adapter_report`. + +| Part | Facts | +| --- | --- | +| `fabric` | `core_version` of the core that ran the adapter. `host_core_version` is `not_applicable` with reason `local_run` in a local run, and the host core's version in an OpenShell run. | +| `adapter` | `id` from the plan, and the reported `package_version`. | +| `harness` | The reported `name` and `version` of the harness program. | +| `models.` | `provider` and the `requested` model from the plan, and the reported `served` model. | +| `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | +| `environment` | The environment `provider`, such as `local` or `openshell`. | + +NeMo Fabric reads no adapter-reported facts yet, so every reported fact has +state `unsupported` with reason `key_not_declared` and method +`adapter_report`. + +Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` +describes a server the harness reports but the plan does not configure. It has +the same shape, except that `transport` can be absent and `endpoint` has state +`not_applicable` with reason `not_configured`. + +`endpoint` keeps no secret. A network server's endpoint is its lowercase scheme +and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 +for `https` and `wss`, so `https://example.com` and `https://example.com:443` +give the same endpoint. An IPv6 host keeps its brackets, in compressed lowercase +form such as `[2001:db8::1]`. A `stdio` server's endpoint is the file name of +its executable, such as `server.exe` for `C:\MCP\server.exe`. +User info, paths, queries, fragments, headers, arguments, and environment values +are never copied. + +An endpoint NeMo Fabric cannot reduce safely has state `collection_failed` with +reason `endpoint_unparsed` or `executable_unparsed`. Examples are a URL with an +`@` after the host, a bracketed host that is not an IPv6 address, and an +executable that contains `=` or any whitespace, even in a folder name. An empty +URL or executable has state `unavailable` with reason `not_configured`. + +A string value longer than 256 characters keeps its first 240 characters, then +`~`, then the first 15 hex digits of its SHA-256. The rule applies to each +element of a list value too. + +`what_ran` is `None` when the core that produced the result predates it. The +Rust core and the Python SDK apply the same structural checks. A `what_ran` +that is not an object, names another schema or version, or has a malformed +section or fact reads as `None`, and the rest of the result is unchanged. A +fact with both `value` and `state` is malformed. Unknown properties are +ignored. Treat `what_ran` as evidence, not as a key for grouping runs. + +```python +what_ran = result.what_ran +if what_ran is not None: + core = what_ran["fabric"]["core_version"] + print(core.get("value"), core["source"]) + for role, model in what_ran["models"].items(): + print(role, model["requested"].get("value"), model["served"].get("state")) +``` + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..b9b51edfc 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -21,6 +21,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation: core version, adapter ID, requested models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | Handle a result with this status-first pattern: @@ -46,6 +47,15 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +Read `what_ran` as evidence of what executed, never as a grouping key. Each +fact holds a `value`, or a `state` and `reason` when it has no value, plus +`source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and +`reason` codes you do not recognize. Adapter-reported facts, such as the +harness version and the served model, currently read state `unsupported` with +reason `key_not_declared`. A malformed `what_ran` reads as `None` without +failing the result. Refer to +[What Ran](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#what-ran). + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one From 26f06b06464132e1ec72ced8db0b32d308b8f6d5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:19 -0700 Subject: [PATCH 05/74] docs: regenerate API references for what_ran Regenerate the Rust library reference with Rust 1.94.0 for the new what_ran module and the RunResult field, and the Python types reference for RunResult.what_ran. Only the pages this change affects are included. Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 2 + .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 1 + .../runtime/struct-runresult.mdx | 6 +- .../what-ran/constant-what-ran-schema.mdx | 15 ++ .../what-ran/constant-what-ran-version.mdx | 15 ++ .../nemo-fabric-core/what-ran/enum-fact.mdx | 109 +++++++++++++++ .../what-ran/enum-factsource.mdx | 123 +++++++++++++++++ .../what-ran/enum-factstate.mdx | 130 ++++++++++++++++++ .../nemo-fabric-core/what-ran/index.mdx | 52 +++++++ .../what-ran/struct-adapterfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-environmentfacts.mdx | 99 +++++++++++++ .../what-ran/struct-fabricfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-factorigin.mdx | 107 ++++++++++++++ .../what-ran/struct-harnessfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-knownfact.mdx | 103 ++++++++++++++ .../what-ran/struct-mcpfacts.mdx | 99 +++++++++++++ .../what-ran/struct-mcpruntimefacts.mdx | 115 ++++++++++++++++ .../what-ran/struct-mcpserverfacts.mdx | 113 +++++++++++++++ .../what-ran/struct-missingfact.mdx | 107 ++++++++++++++ .../what-ran/struct-modelfacts.mdx | 107 ++++++++++++++ .../what-ran/struct-whatran.mdx | 129 +++++++++++++++++ 22 files changed, 1741 insertions(+), 2 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index ea149fc81..7664714b5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1443,6 +1443,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. - `metadata`: Result-specific structured details. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every section and fact well formed, as the Rust core reads it. @@ -1467,6 +1468,7 @@ The mapping exposes the following typed fields: | `telemetry` | `Sequence[TelemetryRef]` | | `events` | `Sequence[FabricEvent]` | | `metadata` | `Mapping[str, Any]` | +| `what_ran` | `Mapping[str, Any] \| None` | ### method `__init__` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..3a81be390 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 08ae7ee42..6a951bfe9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -152,6 +152,7 @@ Core config and runtime contract for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. +- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NeMo Fabric learned each fact. ## Functions diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..9dfc3f1be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -75,6 +75,10 @@ NeMo Fabric lifecycle/progress events emitted during the run. Adapter-specific metadata. +### `what_ran: Option` + +What executed for this invocation, and how NeMo Fabric learned each fact. Absent when the producing core predates it, or when this core finds it malformed or of another schema or version. + ## Trait Implementations ### `impl Clone for RunResult` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx new file mode 100644 index 000000000..06068c300 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_SCHEMA" +sidebar-title: "WHAT_RAN_SCHEMA" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" +description: "Schema name written into every `WhatRan`." +position: 16 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo.fabric.what_ran\";"}} />
+ +Schema name written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx new file mode 100644 index 000000000..01dc53369 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_VERSION" +sidebar-title: "WHAT_RAN_VERSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" +description: "Contract version written into every `WhatRan`." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"v1alpha1\";"}} />
+ +Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx new file mode 100644 index 000000000..55c94c912 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -0,0 +1,109 @@ +--- +title: "Enum Fact" +sidebar-title: "Fact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" +description: "One fact: a `value`, or a `state` with a `reason`, never both." +position: 13 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
KnownFact<T>),\n    Missing(MissingFact),\n}"}} />
+ +One fact: a `value`, or a `state` with a `reason`, never both. + +## Variants + +### `Known(KnownFact)` + +
KnownFact<T>)"}} />
+ +The fact has a value. + +### `Missing(MissingFact)` + +
MissingFact)"}} />
+ +The fact has no value. + +## Trait Implementations + +### `impl Clone for Fact` + +
Clone> Clone for Fact<T>"}} />
+ +#### `clone` + +
clone(&self) -> Fact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for Fact` + +
Debug> Debug for Fact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact` + +
DeserializeOwned> Deserialize<'de> for Fact<T>"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a fact, rejecting one that holds `value` together with `state` or `reason`, or an open code outside its pattern. Unknown properties are ignored. + +### `impl JsonSchema for Factwhere T: JsonSchema,` + +
Fact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for Fact` + +
PartialEq> PartialEq for Fact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &Fact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for Factwhere T: Serialize,` + +
Serialize for Fact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for Fact` + +
Eq> Eq for Fact<T>"}} />
+ +### `impl StructuralPartialEq for Fact` + +
StructuralPartialEq for Fact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx new file mode 100644 index 000000000..66a4e30c5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -0,0 +1,123 @@ +--- +title: "Enum Fact Source" +sidebar-title: "FactSource" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" +description: "How NeMo Fabric learned a fact." +position: 14 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactSource { + Observed, + Reported, + Configured, +} +``` + +How NeMo Fabric learned a fact. + +## Variants + +### `Observed` + +
+ +NeMo Fabric core saw it itself. + +### `Reported` + +
+ +The adapter, the harness, an MCP server or a remote service said so. + +### `Configured` + +
+ +It comes from the resolved run plan. + +## Trait Implementations + +### `impl Clone for FactSource` + +
Clone for FactSource"}} />
+ +#### `clone` + +
clone(&self) -> FactSource"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactSource` + +
Debug for FactSource"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactSource` + +
Deserialize<'de> for FactSource"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactSource` + +
FactSource"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactSource` + +
PartialEq for FactSource"}} />
+ +#### `eq` + +
eq(&self, other: &FactSource) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactSource` + +
Serialize for FactSource"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactSource` + +
Copy for FactSource"}} />
+ +### `impl Eq for FactSource` + +
Eq for FactSource"}} />
+ +### `impl StructuralPartialEq for FactSource` + +
StructuralPartialEq for FactSource"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx new file mode 100644 index 000000000..9fd1a13cf --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -0,0 +1,130 @@ +--- +title: "Enum Fact State" +sidebar-title: "FactState" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" +description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." +position: 15 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactState { + NotApplicable, + Unsupported, + CollectionFailed, + Unavailable, +} +``` + +Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Variants + +### `NotApplicable` + +
+ +The fact cannot exist for this run. + +### `Unsupported` + +
+ +The adapter or environment provider has no source for the fact. + +### `CollectionFailed` + +
+ +A source exists, but it failed, timed out, returned an invalid report or never returned. + +### `Unavailable` + +
+ +NeMo Fabric holds no value and no evidence for the other states. + +## Trait Implementations + +### `impl Clone for FactState` + +
Clone for FactState"}} />
+ +#### `clone` + +
clone(&self) -> FactState"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactState` + +
Debug for FactState"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactState` + +
Deserialize<'de> for FactState"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactState` + +
FactState"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactState` + +
PartialEq for FactState"}} />
+ +#### `eq` + +
eq(&self, other: &FactState) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactState` + +
Serialize for FactState"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactState` + +
Copy for FactState"}} />
+ +### `impl Eq for FactState` + +
Eq for FactState"}} />
+ +### `impl StructuralPartialEq for FactState` + +
StructuralPartialEq for FactState"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx new file mode 100644 index 000000000..18ba5a872 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -0,0 +1,52 @@ +--- +title: "Module what_ran" +sidebar-title: "what_ran" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran" +description: "What executed for one invocation, and how NeMo Fabric learned each fact." +position: 137 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +What executed for one invocation, and how NeMo Fabric learned each fact. + +[`WhatRan`](struct-whatran.mdx) rides on [`RunResult`](../runtime/struct-runresult.mdx). Each fact is a [`Fact`](enum-fact.mdx): a value, or a [`FactState`](enum-factstate.mdx) with a reason, and in both cases the [`FactSource`](enum-factsource.mdx), the component that collected it and the method it used. A few names fixed by the run plan stay plain strings: the adapter ID, each model role's provider, the environment provider, and each MCP server's origin and transport. + +A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. + +`what_ran` is evidence. It never decides which runs are compared. + +Compatibility rules for readers: + +- `source` and `state` are closed sets. A new value changes [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx). +- `collected_by`, `method` and `reason` are open. Readers accept values they do not know. +- Readers accept unknown properties. Adding an optional property keeps [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx); removing one or changing its meaning changes it. The Rust types drop unknown properties when they decode, so a host that relays a task core's result drops them too. +- A fact holds `value`, or `state` and `reason`, never both. A `RunResult` reader that finds a fact breaking this, any other value the schema rejects, or another schema or version reads `what_ran` as absent and still decodes the result. + +## Structs + +- [AdapterFacts](struct-adapterfacts.mdx): Adapter facts. +- [EnvironmentFacts](struct-environmentfacts.mdx): Environment facts. +- [FabricFacts](struct-fabricfacts.mdx): NeMo Fabric core facts. +- [FactOrigin](struct-factorigin.mdx): Who collected a fact and how. +- [HarnessFacts](struct-harnessfacts.mdx): Harness facts. +- [KnownFact](struct-knownfact.mdx): A fact with a value. +- [McpFacts](struct-mcpfacts.mdx): MCP facts. +- [McpRuntimeFacts](struct-mcpruntimefacts.mdx): What the harness learned about one MCP server when it connected. +- [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. +- [MissingFact](struct-missingfact.mdx): A fact without a value. +- [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [WhatRan](struct-whatran.mdx): What executed for one invocation. + +## Enums + +- [Fact](enum-fact.mdx): One fact: a `value`, or a `state` with a `reason`, never both. +- [FactSource](enum-factsource.mdx): How NeMo Fabric learned a fact. +- [FactState](enum-factstate.mdx): Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Constants + +- [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). +- [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx new file mode 100644 index 000000000..dcd1f6145 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Adapter Facts" +sidebar-title: "AdapterFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-adapterfacts" +description: "Adapter facts." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<String>,\n    pub package_version: Fact<String>,\n}"}} />
+ +Adapter facts. + +## Fields + +### `id: Option` + +Adapter ID from the plan, when an adapter was resolved. + +### `package_version: Fact` + +Version of the adapter package, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for AdapterFacts` + +
Clone for AdapterFacts"}} />
+ +#### `clone` + +
clone(&self) -> AdapterFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterFacts` + +
Debug for AdapterFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterFacts` + +
Deserialize<'de> for AdapterFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterFacts` + +
AdapterFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterFacts` + +
PartialEq for AdapterFacts"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterFacts` + +
Serialize for AdapterFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AdapterFacts` + +
Eq for AdapterFacts"}} />
+ +### `impl StructuralPartialEq for AdapterFacts` + +
StructuralPartialEq for AdapterFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx new file mode 100644 index 000000000..f4b0a06b0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Environment Facts" +sidebar-title: "EnvironmentFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-environmentfacts" +description: "Environment facts." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n}"}} />
+ +Environment facts. + +## Fields + +### `provider: String` + +Environment provider, such as `local`. + +## Trait Implementations + +### `impl Clone for EnvironmentFacts` + +
Clone for EnvironmentFacts"}} />
+ +#### `clone` + +
clone(&self) -> EnvironmentFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for EnvironmentFacts` + +
Debug for EnvironmentFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for EnvironmentFacts` + +
Deserialize<'de> for EnvironmentFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for EnvironmentFacts` + +
EnvironmentFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for EnvironmentFacts` + +
PartialEq for EnvironmentFacts"}} />
+ +#### `eq` + +
eq(&self, other: &EnvironmentFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for EnvironmentFacts` + +
Serialize for EnvironmentFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for EnvironmentFacts` + +
Eq for EnvironmentFacts"}} />
+ +### `impl StructuralPartialEq for EnvironmentFacts` + +
StructuralPartialEq for EnvironmentFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx new file mode 100644 index 000000000..eea91521c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Fabric Facts" +sidebar-title: "FabricFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-fabricfacts" +description: "NeMo Fabric core facts." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub host_core_version: Fact<String>,\n}"}} />
+ +NeMo Fabric core facts. + +## Fields + +### `core_version: Fact` + +Version of the core that ran the adapter. + +### `host_core_version: Fact` + +Version of the host core that relayed the result from a task core in another environment. `not_applicable` for a local run. + +## Trait Implementations + +### `impl Clone for FabricFacts` + +
Clone for FabricFacts"}} />
+ +#### `clone` + +
clone(&self) -> FabricFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FabricFacts` + +
Debug for FabricFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FabricFacts` + +
Deserialize<'de> for FabricFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FabricFacts` + +
FabricFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FabricFacts` + +
PartialEq for FabricFacts"}} />
+ +#### `eq` + +
eq(&self, other: &FabricFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FabricFacts` + +
Serialize for FabricFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FabricFacts` + +
Eq for FabricFacts"}} />
+ +### `impl StructuralPartialEq for FabricFacts` + +
StructuralPartialEq for FabricFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx new file mode 100644 index 000000000..513cb6af9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Fact Origin" +sidebar-title: "FactOrigin" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-factorigin" +description: "Who collected a fact and how." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactSource,\n    pub collected_by: String,\n    pub method: String,\n}"}} />
+ +Who collected a fact and how. + +## Fields + +### `source: FactSource` + +How NeMo Fabric learned the fact. + +### `collected_by: String` + +Component that collected the fact, such as `host_core`, `task_core` or `adapter`. + +### `method: String` + +How the collector got the fact, such as `crate_version`, `resolved_plan` or `adapter_report`. + +## Trait Implementations + +### `impl Clone for FactOrigin` + +
Clone for FactOrigin"}} />
+ +#### `clone` + +
clone(&self) -> FactOrigin"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactOrigin` + +
Debug for FactOrigin"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactOrigin` + +
Deserialize<'de> for FactOrigin"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactOrigin` + +
FactOrigin"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactOrigin` + +
PartialEq for FactOrigin"}} />
+ +#### `eq` + +
eq(&self, other: &FactOrigin) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactOrigin` + +
Serialize for FactOrigin"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FactOrigin` + +
Eq for FactOrigin"}} />
+ +### `impl StructuralPartialEq for FactOrigin` + +
StructuralPartialEq for FactOrigin"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx new file mode 100644 index 000000000..a683a0d5f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Harness Facts" +sidebar-title: "HarnessFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-harnessfacts" +description: "Harness facts." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub version: Fact<String>,\n}"}} />
+ +Harness facts. + +## Fields + +### `name: Fact` + +Name of the harness program that ran, as the adapter reports it. Core never derives it from the adapter ID. + +### `version: Fact` + +Version of the harness program that ran, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for HarnessFacts` + +
Clone for HarnessFacts"}} />
+ +#### `clone` + +
clone(&self) -> HarnessFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HarnessFacts` + +
Debug for HarnessFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HarnessFacts` + +
Deserialize<'de> for HarnessFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HarnessFacts` + +
HarnessFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HarnessFacts` + +
PartialEq for HarnessFacts"}} />
+ +#### `eq` + +
eq(&self, other: &HarnessFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HarnessFacts` + +
Serialize for HarnessFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for HarnessFacts` + +
Eq for HarnessFacts"}} />
+ +### `impl StructuralPartialEq for HarnessFacts` + +
StructuralPartialEq for HarnessFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx new file mode 100644 index 000000000..8e559977a --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Known Fact" +sidebar-title: "KnownFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-knownfact" +description: "A fact with a value." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactOrigin,\n}"}} />
+ +A fact with a value. + +## Fields + +### `value: T` + +The fact itself: a string, a boolean or a list of strings. + +### `origin: FactOrigin` + +Who collected the fact and how. + +## Trait Implementations + +### `impl Clone for KnownFact` + +
Clone> Clone for KnownFact<T>"}} />
+ +#### `clone` + +
clone(&self) -> KnownFact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for KnownFact` + +
Debug> Debug for KnownFact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T> Deserialize<'de> for KnownFactwhere T: Deserialize<'de>,` + +
Deserialize<'de> for KnownFact<T>where\n    T: Deserialize<'de>,"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for KnownFactwhere T: JsonSchema,` + +
KnownFact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for KnownFact` + +
PartialEq> PartialEq for KnownFact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &KnownFact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for KnownFactwhere T: Serialize,` + +
Serialize for KnownFact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for KnownFact` + +
Eq> Eq for KnownFact<T>"}} />
+ +### `impl StructuralPartialEq for KnownFact` + +
StructuralPartialEq for KnownFact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx new file mode 100644 index 000000000..c5ddca739 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct McpFacts" +sidebar-title: "McpFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpfacts" +description: "MCP facts." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
BTreeMap<String, McpServerFacts>,\n}"}} />
+ +MCP facts. + +## Fields + +### `servers: BTreeMap` + +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. + +## Trait Implementations + +### `impl Clone for McpFacts` + +
Clone for McpFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpFacts` + +
Debug for McpFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpFacts` + +
Deserialize<'de> for McpFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpFacts` + +
McpFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpFacts` + +
PartialEq for McpFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpFacts` + +
Serialize for McpFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpFacts` + +
Eq for McpFacts"}} />
+ +### `impl StructuralPartialEq for McpFacts` + +
StructuralPartialEq for McpFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx new file mode 100644 index 000000000..afa1bece5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct McpRuntime Facts" +sidebar-title: "McpRuntimeFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpruntimefacts" +description: "What the harness learned about one MCP server when it connected." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n}"}} />
+ +What the harness learned about one MCP server when it connected. + +## Fields + +### `connected: Fact` + +Whether the harness connected to the server. + +### `protocol_version: Fact` + +MCP protocol version negotiated with the server. + +### `server_name: Fact` + +Name the server reports for itself. + +### `server_version: Fact` + +Version the server reports for itself. + +### `tool_names: Fact>` + +Raw tool names the server lists, each bounded by the string length rule. + +## Trait Implementations + +### `impl Clone for McpRuntimeFacts` + +
Clone for McpRuntimeFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpRuntimeFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpRuntimeFacts` + +
Debug for McpRuntimeFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpRuntimeFacts` + +
Deserialize<'de> for McpRuntimeFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpRuntimeFacts` + +
McpRuntimeFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpRuntimeFacts` + +
PartialEq for McpRuntimeFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpRuntimeFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpRuntimeFacts` + +
Serialize for McpRuntimeFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpRuntimeFacts` + +
Eq for McpRuntimeFacts"}} />
+ +### `impl StructuralPartialEq for McpRuntimeFacts` + +
StructuralPartialEq for McpRuntimeFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx new file mode 100644 index 000000000..19730218c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct McpServer Facts" +sidebar-title: "McpServerFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpserverfacts" +description: "Facts about one MCP server." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub transport: Option<String>,\n    pub endpoint: Fact<String>,\n    pub runtime: McpRuntimeFacts,\n}"}} />
+ +Facts about one MCP server. + +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, its `transport` may be absent, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. + +## Fields + +### `origin: String` + +Where the entry comes from: `plan` for a server the plan configures, `harness` for one only the harness reports. + +### `transport: Option` + +Transport, such as `stdio` or `streamable-http`. Always present for a `plan` entry. + +### `endpoint: Fact` + +Configured endpoint without secrets: the lowercase scheme and host and the port for a network server, or the executable's file name for a `stdio` server. + +### `runtime: McpRuntimeFacts` + +What the harness learned when it connected, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for McpServerFacts` + +
Clone for McpServerFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpServerFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpServerFacts` + +
Debug for McpServerFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpServerFacts` + +
Deserialize<'de> for McpServerFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpServerFacts` + +
McpServerFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpServerFacts` + +
PartialEq for McpServerFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpServerFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpServerFacts` + +
Serialize for McpServerFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpServerFacts` + +
Eq for McpServerFacts"}} />
+ +### `impl StructuralPartialEq for McpServerFacts` + +
StructuralPartialEq for McpServerFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx new file mode 100644 index 000000000..04a2c23d3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Missing Fact" +sidebar-title: "MissingFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-missingfact" +description: "A fact without a value." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactState,\n    pub reason: String,\n    pub origin: FactOrigin,\n}"}} />
+ +A fact without a value. + +## Fields + +### `state: FactState` + +Why the fact has no value. + +### `reason: String` + +Short code that explains the state, such as `key_not_declared`. + +### `origin: FactOrigin` + +Who would have collected the fact and how. + +## Trait Implementations + +### `impl Clone for MissingFact` + +
Clone for MissingFact"}} />
+ +#### `clone` + +
clone(&self) -> MissingFact"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for MissingFact` + +
Debug for MissingFact"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for MissingFact` + +
Deserialize<'de> for MissingFact"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for MissingFact` + +
MissingFact"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for MissingFact` + +
PartialEq for MissingFact"}} />
+ +#### `eq` + +
eq(&self, other: &MissingFact) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for MissingFact` + +
Serialize for MissingFact"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for MissingFact` + +
Eq for MissingFact"}} />
+ +### `impl StructuralPartialEq for MissingFact` + +
StructuralPartialEq for MissingFact"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx new file mode 100644 index 000000000..e42753677 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Model Facts" +sidebar-title: "ModelFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-modelfacts" +description: "Facts about one model role." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub requested: Fact<String>,\n    pub served: Fact<String>,\n}"}} />
+ +Facts about one model role. + +## Fields + +### `provider: String` + +Provider from the plan. + +### `requested: Fact` + +Model the plan requested. + +### `served: Fact` + +Model that served, as the harness or provider reports it. + +## Trait Implementations + +### `impl Clone for ModelFacts` + +
Clone for ModelFacts"}} />
+ +#### `clone` + +
clone(&self) -> ModelFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelFacts` + +
Debug for ModelFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelFacts` + +
Deserialize<'de> for ModelFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelFacts` + +
ModelFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelFacts` + +
PartialEq for ModelFacts"}} />
+ +#### `eq` + +
eq(&self, other: &ModelFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelFacts` + +
Serialize for ModelFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelFacts` + +
Eq for ModelFacts"}} />
+ +### `impl StructuralPartialEq for ModelFacts` + +
StructuralPartialEq for ModelFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx new file mode 100644 index 000000000..d25aaa76f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -0,0 +1,129 @@ +--- +title: "Struct WhatRan" +sidebar-title: "WhatRan" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" +description: "What executed for one invocation." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub version: String,\n    pub fabric: FabricFacts,\n    pub adapter: AdapterFacts,\n    pub harness: HarnessFacts,\n    pub models: BTreeMap<String, ModelFacts>,\n    pub mcp: McpFacts,\n    pub environment: EnvironmentFacts,\n}"}} />
+ +What executed for one invocation. + +Decoding a `WhatRan` directly checks the shape of every section and fact. `RunResult` also checks the schema name and version, and that every `plan` MCP entry names its transport. + +## Fields + +### `schema: String` + +Schema name, `nemo.fabric.what_ran`. + +### `version: String` + +Contract version, `v1alpha1`. + +### `fabric: FabricFacts` + +NeMo Fabric core that ran the adapter. + +### `adapter: AdapterFacts` + +Adapter that ran. + +### `harness: HarnessFacts` + +Harness behind the adapter. + +### `models: BTreeMap` + +Model facts keyed by the role names of the plan's `models` settings. + +### `mcp: McpFacts` + +MCP server facts. + +### `environment: EnvironmentFacts` + +Environment the runtime ran in. + +## Trait Implementations + +### `impl Clone for WhatRan` + +
Clone for WhatRan"}} />
+ +#### `clone` + +
clone(&self) -> WhatRan"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for WhatRan` + +
Debug for WhatRan"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for WhatRan` + +
Deserialize<'de> for WhatRan"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for WhatRan` + +
WhatRan"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for WhatRan` + +
PartialEq for WhatRan"}} />
+ +#### `eq` + +
eq(&self, other: &WhatRan) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for WhatRan` + +
Serialize for WhatRan"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for WhatRan` + +
Eq for WhatRan"}} />
+ +### `impl StructuralPartialEq for WhatRan` + +
StructuralPartialEq for WhatRan"}} />
From 638cb2eef7e70d0c20f1f53df51af2cf5b458f47 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:16 -0700 Subject: [PATCH 06/74] feat: bound what_ran size and entry counts A plan with many model roles or MCP servers, or very long names, could make what_ran arbitrarily large, and every RunResult carries it. Keep it bounded without ever failing a run. Core keeps the first 16 model roles and 32 MCP servers in sorted key order, and drops entries whose key is longer than 128 characters. While the compact JSON of what_ran is larger than 128 KiB, it drops MCP servers from the end of the sorted order, then model roles. If plan identifiers alone still exceed the budget, what_ran becomes a minimal object with only the schema name, the version and what_ran_truncated. A marker next to each map counts the dropped entries and names the last rule applied. The OpenShell host applies the same bounds again after it attributes the task core's facts, so they hold for a task core of any version. The six what_ran sections become optional in the Rust type so the minimal object can be represented. Every section is still required unless what_ran_truncated is present, and no section may be null. The Rust reader, the generated schema and the Python reader check this against the shared vectors. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 77 ++ crates/fabric-core/src/runtime/openshell.rs | 47 ++ crates/fabric-core/src/what_ran.rs | 657 ++++++++++++++++-- schemas/sdk/run-result.schema.json | 69 +- .../src/nemo_fabric/types.py | 117 +++- tests/fixtures/what_ran/reader-vectors.json | 214 ++++++ 6 files changed, 1089 insertions(+), 92 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 893914ff9..1b5493545 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5501,6 +5501,83 @@ for line in sys.stdin: } } + #[test] + fn local_host_what_ran_bounds_keep_the_status_and_error() { + let model = |provider: &str| -> crate::config::ModelConfig { + serde_json::from_value(serde_json::json!({"provider": provider, "model": "m"})) + .expect("model config") + }; + // One `default` role keeps the adapter's single-role check satisfied. + let roles = || { + std::iter::once("default".to_string()) + .chain((0..16).map(|index| format!("role-{index:02}"))) + }; + for mode in ["success", "adapter_reported_failure"] { + let (root, plan) = local_host_plan(mode); + let baseline = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let mut accepting_models = plan.clone(); + accepting_models + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .config + .accepts + .push(crate::config::AdapterConfigField::Models); + + let mut past_the_cap = accepting_models.clone(); + past_the_cap.config.models = roles().map(|role| (role, model("openai"))).collect(); + let mut past_the_budget = accepting_models.clone(); + past_the_budget.config.models = roles() + .map(|role| (role, model(&"p".repeat(9_000)))) + .collect(); + let mut minimal = plan.clone(); + minimal + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .adapter_id = "a".repeat(140_000); + + for (case, bounded_plan) in [ + ("entry cap", past_the_cap), + ("byte budget", past_the_budget), + ("minimal", minimal), + ] { + let result = + run_plan(&bounded_plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, baseline.status, "{case}"); + assert_eq!( + serde_json::to_value(&result.error).expect("error JSON"), + serde_json::to_value(&baseline.error).expect("error JSON"), + "{case}" + ); + let what_ran = result.what_ran.expect("what_ran"); + let marker = |marker: Option| { + marker.map(|marker| (marker.reason, marker.dropped)) + }; + match case { + "entry cap" => assert_eq!( + marker(what_ran.models_truncated), + Some(("entry_cap".to_string(), 1)) + ), + "byte budget" => { + let (reason, dropped) = + marker(what_ran.models_truncated).expect("models marker"); + assert_eq!(reason, "byte_budget"); + assert!(dropped > 1, "{dropped}"); + } + _ => assert_eq!( + marker(what_ran.what_ran_truncated), + Some(("byte_budget".to_string(), 1)) + ), + } + } + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index b8e7f5e80..9d3bf6fb9 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -934,6 +934,53 @@ mod tests { ); } + #[test] + fn task_what_ran_over_the_byte_budget_becomes_minimal_on_this_host() { + let minimal = serde_json::json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let mut oversized = task_what_ran(); + oversized["adapter"]["id"] = Value::from("a".repeat(140_000)); + for what_ran in [oversized, minimal.clone()] { + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!( + serde_json::to_value(result.what_ran).expect("what_ran JSON"), + minimal + ); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + + #[test] + fn task_what_ran_past_the_entry_caps_is_bounded_on_this_host() { + let mut what_ran = task_what_ran(); + let server = what_ran["mcp"]["servers"]["docs"].clone(); + what_ran["mcp"]["servers"] = (0..35) + .map(|index| (format!("server-{index:02}"), server.clone())) + .collect::>() + .into(); + what_ran["mcp"]["servers_truncated"] = + serde_json::json!({"reason": "key_too_long", "dropped": 2}); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + let expected: Vec = (0..32).map(|index| format!("server-{index:02}")).collect(); + assert_eq!(servers.keys().cloned().collect::>(), expected); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + serde_json::json!({"reason": "entry_cap", "dropped": 5}) + ); + } + #[test] fn task_result_without_readable_what_ran_is_kept_without_it() { let mut undecodable = task_what_ran(); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index ef5686ee4..27dbda17f 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -15,6 +15,16 @@ //! then the first 15 hex digits of its SHA-256. Map keys and the plain names //! above are never cut. //! +//! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in +//! sorted key order. An entry whose key is longer than 128 characters is +//! dropped. While the compact JSON of the whole object is larger than +//! 128 KiB, MCP server entries are dropped from the end of the sorted order, +//! then model roles. Dropped entries are counted in a [`TruncatedEntries`] +//! marker next to the map: `models_truncated` and `mcp.servers_truncated`. +//! If the object is still too large with no MCP server and no model role, +//! it is replaced by a minimal one that holds only the schema name, the +//! version and the `what_ran_truncated` marker. +//! //! `what_ran` is evidence. It never decides which runs are compared. //! //! Compatibility rules for readers: @@ -180,12 +190,66 @@ fn is_code(code: &str) -> bool { }) } +/// Entries core dropped to keep `what_ran` within its bounds: next to a +/// map for the entries dropped from it, or as `what_ran_truncated` for the +/// whole object. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct TruncatedEntries { + /// Why the entries were dropped: `key_too_long` for a key longer than + /// 128 characters, `entry_cap` for an entry past the map's cap, or + /// `byte_budget` for an entry dropped to keep the whole object within + /// 128 KiB. One marker covers a map: when more than one rule dropped + /// entries from it, `reason` names the last rule applied, in that + /// order, and `dropped` counts them all. An open code like a fact's + /// `reason`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// How many entries were dropped. A JSON integer: the Rust and Python + /// readers reject `1.0`, although a JSON Schema validator may accept it. + #[schemars(range(min = 1, max = u64::MAX))] + pub dropped: u64, +} + +impl<'de> Deserialize<'de> for TruncatedEntries { + /// Decode a marker, rejecting a `reason` outside the code pattern or a + /// `dropped` count of zero. Unknown properties are ignored. + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Marker { + reason: String, + dropped: u64, + } + let Marker { reason, dropped } = Marker::deserialize(deserializer)?; + if !is_code(&reason) { + return Err(D::Error::custom(format!( + "a truncation `reason` must match {CODE_PATTERN}" + ))); + } + if dropped == 0 { + return Err(D::Error::custom( + "a truncation `dropped` count is at least 1", + )); + } + Ok(Self { reason, dropped }) + } +} + /// What executed for one invocation. /// +/// Every section is present, unless `what_ran_truncated` is present: then +/// the object is a minimal one that the byte budget left with only its +/// schema name and version, and every section may be absent. +/// /// Decoding a `WhatRan` directly checks the shape of every section and -/// fact. `RunResult` also checks the schema name and version, and that -/// every `plan` MCP entry names its transport. +/// fact, and that no section is `null`. `RunResult` also checks the schema +/// name and version, that every section is present unless +/// `what_ran_truncated` is, and that every `plan` MCP entry names its +/// transport. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("anyOf" = [ + {"required": ["what_ran_truncated"]}, + {"required": ["fabric", "adapter", "harness", "models", "mcp", "environment"]} +]))] pub struct WhatRan { /// Schema name, `nemo.fabric.what_ran`. #[schemars(extend("const" = WHAT_RAN_SCHEMA))] @@ -194,17 +258,83 @@ pub struct WhatRan { #[schemars(extend("const" = WHAT_RAN_VERSION))] pub version: String, /// NeMo Fabric core that ran the adapter. - pub fabric: FabricFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "FabricFacts")] + pub fabric: Option, /// Adapter that ran. - pub adapter: AdapterFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "AdapterFacts")] + pub adapter: Option, /// Harness behind the adapter. - pub harness: HarnessFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "HarnessFacts")] + pub harness: Option, /// Model facts keyed by the role names of the plan's `models` settings. - pub models: BTreeMap, + /// At most 16 roles, in sorted key order, each key at most 128 + /// characters. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "BTreeMap")] + pub models: Option>, + /// Model roles dropped from `models`. Absent when none was dropped, + /// never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub models_truncated: Option, /// MCP server facts. - pub mcp: McpFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "McpFacts")] + pub mcp: Option, /// Environment the runtime ran in. - pub environment: EnvironmentFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "EnvironmentFacts")] + pub environment: Option, + /// Present only on a minimal object: every section was dropped to keep + /// the object within 128 KiB, with reason `byte_budget` and a dropped + /// count of 1. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub what_ran_truncated: Option, +} + +/// Decode a property that may be absent but is never `null`. +fn present<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) } /// NeMo Fabric core facts. @@ -252,17 +382,28 @@ pub struct ModelFacts { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct McpFacts { /// Servers keyed by name. Empty when the plan configures no MCP server - /// and the harness reports none. + /// and the harness reports none. At most 32 servers, in sorted key + /// order, each key at most 128 characters. pub servers: BTreeMap, + /// Servers dropped from `servers`. Absent when none was dropped, never + /// `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub servers_truncated: Option, } /// Facts about one MCP server. /// /// An entry with origin `harness` is a server the harness reports and the /// plan does not configure. It has the same shape: its `endpoint` reads -/// `not_applicable` with reason `not_configured`, its `transport` may be -/// absent, and runtime facts the adapter did not collect read -/// `unavailable` with reason `not_collected`. +/// `not_applicable` with reason `not_configured`, and runtime facts the +/// adapter did not collect read `unavailable` with reason `not_collected`. +/// Only a `plan` entry must name its transport. An entry of any other +/// origin may leave it out. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[schemars(extend( "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, @@ -272,8 +413,8 @@ pub struct McpServerFacts { /// Where the entry comes from: `plan` for a server the plan configures, /// `harness` for one only the harness reports. pub origin: String, - /// Transport, such as `stdio` or `streamable-http`. Always present for - /// a `plan` entry. + /// Transport, such as `stdio` or `streamable-http`. Required for a + /// `plan` entry, and optional for any other origin. #[serde(default, skip_serializing_if = "Option::is_none")] pub transport: Option, /// Configured endpoint without secrets: the lowercase scheme and host @@ -319,6 +460,18 @@ const REASON_LOCAL_RUN: &str = "local_run"; const REASON_NOT_CONFIGURED: &str = "not_configured"; const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; +const REASON_ENTRY_CAP: &str = "entry_cap"; +const REASON_KEY_TOO_LONG: &str = "key_too_long"; +const REASON_BYTE_BUDGET: &str = "byte_budget"; + +/// Most model roles `what_ran` keeps. +const MAX_MODEL_ROLES: usize = 16; +/// Most MCP servers `what_ran` keeps. +const MAX_MCP_SERVERS: usize = 32; +/// Longest map key `what_ran` keeps, in characters. +const MAX_KEY_CHARS: usize = 128; +/// Largest compact JSON of a whole `what_ran`, in bytes. +const MAX_WHAT_RAN_BYTES: usize = 128 * 1024; /// Longest string value kept whole, in characters. const MAX_VALUE_CHARS: usize = 256; @@ -399,17 +552,23 @@ impl WhatRan { mcp, .. } = self; - let mut origins = vec![ - fabric.core_version.origin_mut(), - fabric.host_core_version.origin_mut(), - adapter.package_version.origin_mut(), - harness.name.origin_mut(), - harness.version.origin_mut(), - ]; - for model in models.values_mut() { + let mut origins = Vec::new(); + if let Some(fabric) = fabric { + origins.extend([ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + ]); + } + if let Some(adapter) = adapter { + origins.push(adapter.package_version.origin_mut()); + } + if let Some(harness) = harness { + origins.extend([harness.name.origin_mut(), harness.version.origin_mut()]); + } + for model in models.iter_mut().flat_map(BTreeMap::values_mut) { origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); } - for server in mcp.servers.values_mut() { + for server in mcp.iter_mut().flat_map(|mcp| mcp.servers.values_mut()) { let runtime = &mut server.runtime; origins.extend([ server.endpoint.origin_mut(), @@ -423,19 +582,95 @@ impl WhatRan { origins } - /// Whether every `plan` MCP entry names its transport. Only a `harness` - /// entry may leave it out. + /// Keep the compact JSON of the whole object within 128 KiB. While it + /// is larger, drop the last MCP server in sorted key order, then the + /// last model role, counting each in its map's marker with reason + /// `byte_budget`. With neither left and the object still too large, + /// replace it with [`WhatRan::minimal`]. + fn apply_byte_budget(&mut self) { + while self.serialized_len() > MAX_WHAT_RAN_BYTES { + if let Some(mcp) = self.mcp.as_mut() + && mcp.servers.pop_last().is_some() + { + mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); + } else if let Some(models) = self.models.as_mut() + && models.pop_last().is_some() + { + self.models_truncated = Some(one_more_byte_budget_drop(&self.models_truncated)); + } else { + *self = Self::minimal(); + return; + } + } + } + + /// Apply the entry caps and the key rule to an object another core + /// collected, adding what they drop to the existing markers. + fn apply_entry_caps(&mut self) { + if let Some(models) = self.models.take() { + let (kept, dropped) = bounded_entries(&models, MAX_MODEL_ROLES, ModelFacts::clone); + self.models = Some(kept); + self.models_truncated = merged_marker(self.models_truncated.take(), dropped); + } + if let Some(mcp) = self.mcp.as_mut() { + let (kept, dropped) = + bounded_entries(&mcp.servers, MAX_MCP_SERVERS, McpServerFacts::clone); + mcp.servers = kept; + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); + } + } + + /// The object left when every section had to go: the schema name, the + /// version and a `what_ran_truncated` marker. + fn minimal() -> Self { + Self { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: None, + adapter: None, + harness: None, + models: None, + models_truncated: None, + mcp: None, + environment: None, + what_ran_truncated: Some(TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: 1, + }), + } + } + + fn serialized_len(&self) -> usize { + serde_json::to_vec(self) + .expect("what_ran has only string keys and is always JSON serializable") + .len() + } + + /// Whether every `plan` MCP entry names its transport. An entry of any + /// other origin may leave it out. fn plan_servers_have_transports(&self) -> bool { self.mcp - .servers - .values() + .iter() + .flat_map(|mcp| mcp.servers.values()) .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) } + + /// Whether every section is present, or the object is a minimal one. + fn has_every_section_unless_truncated(&self) -> bool { + self.what_ran_truncated.is_some() + || (self.fabric.is_some() + && self.adapter.is_some() + && self.harness.is_some() + && self.models.is_some() + && self.mcp.is_some() + && self.environment.is_some()) + } } /// Collect the facts the core that runs the adapter knows before it sends /// the request: its own version and the plan's adapter ID, models, MCP -/// servers and environment provider. +/// servers and environment provider. The model and MCP server maps follow +/// the entry caps and the byte budget, and collection never fails. /// /// Core reads no adapter report key yet, so every adapter-reported fact /// reads `unsupported` with reason `key_not_declared`. The collecting core @@ -447,10 +682,17 @@ pub(crate) fn collect_before_request( adapter_id: Option, environment_provider: &str, ) -> WhatRan { - WhatRan { + let (models, models_truncated) = + bounded_entries(&plan.config.models, MAX_MODEL_ROLES, model_facts); + let (servers, servers_truncated) = bounded_entries( + &plan.capability_plan.mcp_servers, + MAX_MCP_SERVERS, + mcp_server_facts, + ); + let mut what_ran = WhatRan { schema: WHAT_RAN_SCHEMA.to_string(), version: WHAT_RAN_VERSION.to_string(), - fabric: FabricFacts { + fabric: Some(FabricFacts { core_version: core_version_fact(), host_core_version: Fact::missing( FactState::NotApplicable, @@ -459,32 +701,90 @@ pub(crate) fn collect_before_request( HOST_CORE, METHOD_CRATE_VERSION, ), - }, - adapter: AdapterFacts { + }), + adapter: Some(AdapterFacts { id: adapter_id, package_version: Fact::report_key_not_declared(), - }, - harness: HarnessFacts { + }), + harness: Some(HarnessFacts { name: Fact::report_key_not_declared(), version: Fact::report_key_not_declared(), - }, - models: plan - .config - .models - .iter() - .map(|(role, model)| (role.clone(), model_facts(model))) - .collect(), - mcp: McpFacts { - servers: plan - .capability_plan - .mcp_servers - .iter() - .map(|(name, server)| (name.clone(), mcp_server_facts(server))) - .collect(), - }, - environment: EnvironmentFacts { + }), + models: Some(models), + models_truncated, + mcp: Some(McpFacts { + servers, + servers_truncated, + }), + environment: Some(EnvironmentFacts { provider: environment_provider.to_string(), - }, + }), + what_ran_truncated: None, + }; + what_ran.apply_byte_budget(); + what_ran +} + +/// The facts of at most `cap` entries, taken in sorted key order, and a +/// marker that counts the dropped entries. +/// +/// An entry whose key is longer than 128 characters is dropped and takes no +/// place under the cap. The marker reason is `entry_cap` when the cap +/// dropped an entry, and `key_too_long` when only long keys were dropped. +/// [`BTreeMap`] iteration gives the sorted key order, whatever order the +/// plan's entries were written in. +fn bounded_entries( + entries: &BTreeMap, + cap: usize, + facts: impl Fn(&P) -> F, +) -> (BTreeMap, Option) { + let mut kept = BTreeMap::new(); + let (mut key_too_long, mut over_cap) = (0_u64, 0_u64); + for (key, entry) in entries { + if key.chars().count() > MAX_KEY_CHARS { + key_too_long += 1; + } else if kept.len() == cap { + over_cap += 1; + } else { + kept.insert(key.clone(), facts(entry)); + } + } + let reason = if over_cap > 0 { + REASON_ENTRY_CAP + } else { + REASON_KEY_TOO_LONG + }; + let dropped = key_too_long + over_cap; + let truncated = (dropped > 0).then(|| TruncatedEntries { + reason: reason.to_string(), + dropped, + }); + (kept, truncated) +} + +/// `marker` with one more dropped entry and reason `byte_budget`. +fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { + TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: marker + .as_ref() + .map_or(0, |marker| marker.dropped) + .saturating_add(1), + } +} + +/// An `earlier` marker followed by a `later` one: the later reason, and the +/// two counts added. A task core can send any count, so the sum saturates. +fn merged_marker( + earlier: Option, + later: Option, +) -> Option { + match (earlier, later) { + (Some(earlier), Some(later)) => Some(TruncatedEntries { + reason: later.reason, + dropped: earlier.dropped.saturating_add(later.dropped), + }), + (earlier, later) => later.or(earlier), } } @@ -566,15 +866,24 @@ fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { /// The task core runs in a projected `local` environment and labels its /// own facts `host_core`. The host relabels them `task_core`, records its /// own version as `host_core_version`, and replaces the environment -/// provider with its own, such as `openshell`. +/// provider with its own, such as `openshell`. A minimal object stays +/// minimal. The task core may run another Fabric version, and the host's +/// own facts can grow the object, so the entry caps, the key rule and the +/// byte budget apply again. pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { for origin in what_ran.origins_mut() { if origin.collected_by == HOST_CORE { origin.collected_by = TASK_CORE.to_string(); } } - what_ran.fabric.host_core_version = core_version_fact(); - what_ran.environment.provider = environment_provider.to_string(); + if let Some(fabric) = what_ran.fabric.as_mut() { + fabric.host_core_version = core_version_fact(); + } + if let Some(environment) = what_ran.environment.as_mut() { + environment.provider = environment_provider.to_string(); + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); } /// `text` whole when it has at most 256 characters. A longer string keeps @@ -609,6 +918,7 @@ where .filter(|what_ran| { what_ran.schema == WHAT_RAN_SCHEMA && what_ran.version == WHAT_RAN_VERSION + && what_ran.has_every_section_unless_truncated() && what_ran.plan_servers_have_transports() })) } @@ -791,6 +1101,238 @@ mod tests { } } + /// `plan_with_secrets` with `models` and MCP servers under the given + /// keys, each a copy of its `default` model or its `files` server. + fn plan_with_entries(model_roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_secrets(); + let model = plan.config.models["default"].clone(); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.config.models = model_roles + .iter() + .map(|role| (role.clone(), model.clone())) + .collect(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + plan + } + + fn numbered_keys(prefix: &str, count: usize) -> Vec { + (0..count) + .map(|index| format!("{prefix}-{index:02}")) + .collect() + } + + fn map_keys(map: &Value) -> Vec { + map.as_object().expect("map").keys().cloned().collect() + } + + #[test] + fn models_past_16_roles_keep_the_first_in_sorted_order_and_count_the_rest() { + let mut roles = numbered_keys("role", 17); + roles.reverse(); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn mcp_servers_past_32_keep_the_first_in_sorted_order_and_count_the_rest() { + let what_ran = collect(&plan_with_entries(&[], &numbered_keys("server", 35))); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + numbered_keys("server", 32) + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + assert!(what_ran.get("models_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { + let longest = "é".repeat(128); + let too_long = "é".repeat(129); + let mut roles = numbered_keys("role", 15); + roles.extend([longest.clone(), too_long.clone()]); + let mut expected_roles = numbered_keys("role", 15); + expected_roles.push(longest.clone()); + + let what_ran = collect(&plan_with_entries(&roles, &[longest, too_long])); + + assert_eq!(map_keys(&what_ran["models"]), expected_roles); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), ["é".repeat(128)]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn a_long_key_and_the_cap_together_count_every_dropped_entry_as_entry_cap() { + let mut roles = numbered_keys("role", 17); + roles.push("r".repeat(129)); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 2}) + ); + } + + #[test] + fn retained_entries_do_not_depend_on_the_order_the_plan_lists_them() { + let roles = numbered_keys("role", 20); + let servers = numbered_keys("server", 40); + let mut reversed_roles = roles.clone(); + reversed_roles.reverse(); + let mut reversed_servers = servers.clone(); + reversed_servers.reverse(); + + assert_eq!( + collect(&plan_with_entries(&roles, &servers)), + collect(&plan_with_entries(&reversed_roles, &reversed_servers)) + ); + } + + /// A plan whose 16 model providers take most of the 128 KiB budget. + fn plan_near_the_byte_budget(server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(&numbered_keys("role", 16), server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + plan + } + + fn json_len(value: &Value) -> usize { + serde_json::to_vec(value).expect("JSON").len() + } + + #[test] + fn mcp_servers_past_the_byte_budget_are_dropped_from_the_end_of_sorted_order() { + let servers = numbered_keys("server", 32); + + let what_ran = collect(&plan_near_the_byte_budget(&servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = map_keys(&what_ran["mcp"]["servers"]); + assert!(!kept.is_empty() && kept.len() < servers.len(), "{kept:?}"); + assert_eq!(kept, servers[..kept.len()]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": servers.len() - kept.len()}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next server in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&[], &servers)); + let mut one_more = what_ran.clone(); + let next = &servers[kept.len()]; + one_more["mcp"]["servers"][next] = full["mcp"]["servers"][next].clone(); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn the_byte_budget_counts_servers_the_entry_cap_already_dropped() { + let within_cap = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let budget_dropped = within_cap["mcp"]["servers_truncated"]["dropped"] + .as_u64() + .expect("dropped count"); + + let what_ran = collect(&plan_near_the_byte_budget(&numbered_keys("server", 35))); + + assert_eq!(what_ran["mcp"]["servers"], within_cap["mcp"]["servers"]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": budget_dropped + 3}) + ); + } + + /// A plan with the given model roles and MCP servers whose providers + /// together exceed the 128 KiB budget. + fn plan_over_the_byte_budget(roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(roles, server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(9_000); + } + plan + } + + #[test] + fn model_roles_are_dropped_for_the_byte_budget_after_every_mcp_server() { + let roles = numbered_keys("role", 18); + let servers = numbered_keys("server", 3); + + let what_ran = collect(&plan_over_the_byte_budget(&roles, &servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + assert_eq!(what_ran["mcp"]["servers"], json!({})); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + let kept = map_keys(&what_ran["models"]); + assert!(!kept.is_empty() && kept.len() < 16, "{kept:?}"); + assert_eq!(kept, roles[..kept.len()]); + // Two roles past the cap, then every role the budget dropped. + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "byte_budget", "dropped": roles.len() - kept.len()}) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next role in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&roles, &[])); + let mut one_more = what_ran.clone(); + let next = &roles[kept.len()]; + one_more["models"][next] = full["models"][next].clone(); + one_more["models"][next]["provider"] = json!("p".repeat(9_000)); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn an_object_over_the_byte_budget_without_maps_to_drop_becomes_minimal() { + let plan = plan_with_entries(&numbered_keys("role", 2), &numbered_keys("server", 2)); + let adapter_id = "a".repeat(140_000); + + let what_ran = + serde_json::to_value(collect_before_request(&plan, Some(adapter_id), "local")) + .expect("serialize what_ran"); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + let decoded = run_result_with(what_ran) + .what_ran + .expect("a minimal what_ran reads as present"); + assert_eq!(decoded.fabric, None); + } + #[test] fn plan_without_models_or_mcp_servers_reads_empty_maps() { let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); @@ -977,7 +1519,12 @@ mod tests { .expect("unknown properties never fail what_ran"); assert_eq!( - what_ran.fabric.core_version.origin().collected_by, + what_ran + .fabric + .expect("fabric section") + .core_version + .origin() + .collected_by, "acme.collector" ); } diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 72a10b9bf..008750fb9 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -490,8 +490,12 @@ "additionalProperties": { "$ref": "#/$defs/McpServerFacts" }, - "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none.", + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." } }, "required": [ @@ -533,7 +537,7 @@ "type": "object" }, "McpServerFacts": { - "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, its `transport` may be\nabsent, and runtime facts the adapter did not collect read\n`unavailable` with reason `not_collected`.", + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", "if": { "properties": { "origin": { @@ -555,7 +559,7 @@ "description": "What the harness learned when it connected, as the adapter reports it." }, "transport": { - "description": "Transport, such as `stdio` or `streamable-http`. Always present for\na `plan` entry.", + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", "type": [ "string", "null" @@ -771,8 +775,47 @@ }, "type": "object" }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, "WhatRan": { - "description": "What executed for one invocation.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact. `RunResult` also checks the schema name and version, and that\nevery `plan` MCP entry names its transport.", + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", "properties": { "adapter": { "$ref": "#/$defs/AdapterFacts", @@ -798,9 +841,13 @@ "additionalProperties": { "$ref": "#/$defs/ModelFacts" }, - "description": "Model facts keyed by the role names of the plan's `models` settings.", + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", "type": "object" }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, "schema": { "const": "nemo.fabric.what_ran", "description": "Schema name, `nemo.fabric.what_ran`.", @@ -810,17 +857,15 @@ "const": "v1alpha1", "description": "Contract version, `v1alpha1`.", "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." } }, "required": [ "schema", - "version", - "fabric", - "adapter", - "harness", - "models", - "mcp", - "environment" + "version" ], "type": "object" } diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0f23621a6..ebaf128e9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1972,6 +1972,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: {"not_applicable", "unsupported", "collection_failed", "unavailable"} ) _FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") +_MAX_U64 = 2**64 - 1 def _is_text(value: Any) -> bool: @@ -2027,7 +2028,8 @@ def _is_mcp_server_entry(server: Any) -> bool: return False origin = server.get("origin") transport = server.get("transport") - # Only a server the harness reports may leave out its transport. + # A `plan` entry requires `transport`. An entry of any other origin, + # including an origin this SDK does not know, may leave it out. transport_ok = isinstance(transport, str) or ( transport is None and origin != "plan" ) @@ -2045,11 +2047,84 @@ def _is_mcp_server_entry(server: Any) -> bool: ) +def _is_truncation_marker(marker: Any) -> bool: + """Whether ``marker`` counts the entries dropped from a map or section.""" + + if not isinstance(marker, Mapping): + return False + dropped = marker.get("dropped") + return ( + _is_code(marker.get("reason")) + and isinstance(dropped, int) + and not isinstance(dropped, bool) + and 1 <= dropped <= _MAX_U64 + ) + + +def _has_valid_marker(section: Mapping[str, Any], name: str) -> bool: + """Whether marker ``name`` is absent from ``section`` or well formed. + + A marker is never ``null``. + """ + + return name not in section or _is_truncation_marker(section[name]) + + +def _is_fabric_section(fabric: Any) -> bool: + return _has_facts(fabric, {"core_version": _is_text, "host_core_version": _is_text}) + + +def _is_adapter_section(adapter: Any) -> bool: + return _has_facts(adapter, {"package_version": _is_text}) and ( + adapter.get("id") is None or isinstance(adapter.get("id"), str) + ) + + +def _is_harness_section(harness: Any) -> bool: + return _has_facts(harness, {"name": _is_text, "version": _is_text}) + + +def _is_models_section(models: Any) -> bool: + return isinstance(models, Mapping) and all( + _is_model_entry(model) for model in models.values() + ) + + +def _is_mcp_section(mcp: Any) -> bool: + if not isinstance(mcp, Mapping): + return False + servers = mcp.get("servers") + return ( + isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and _has_valid_marker(mcp, "servers_truncated") + ) + + +def _is_environment_section(environment: Any) -> bool: + return isinstance(environment, Mapping) and isinstance( + environment.get("provider"), str + ) + + +_WHAT_RAN_SECTIONS: Mapping[str, Callable[[Any], bool]] = { + "fabric": _is_fabric_section, + "adapter": _is_adapter_section, + "harness": _is_harness_section, + "models": _is_models_section, + "mcp": _is_mcp_section, + "environment": _is_environment_section, +} + + def _is_readable_what_ran(value: Any) -> bool: """Whether ``value`` is a ``what_ran`` object this SDK reads. Mirrors the Rust core's reader: the schema and version match, every - section and fact is well formed, and unknown properties are ignored. + present section and fact is well formed, no section is ``null``, and + unknown properties are ignored. Every section is required unless + ``what_ran_truncated`` is present, which marks a minimal object that + the byte budget left with only its schema name and version. """ if not isinstance(value, Mapping): @@ -2059,26 +2134,16 @@ def _is_readable_what_ran(value: Any) -> bool: or value.get("version") != _WHAT_RAN_VERSION ): return False - adapter = value.get("adapter") - models = value.get("models") - mcp = value.get("mcp") - servers = mcp.get("servers") if isinstance(mcp, Mapping) else None - environment = value.get("environment") - return ( - _has_facts( - value.get("fabric"), - {"core_version": _is_text, "host_core_version": _is_text}, - ) - and _has_facts(adapter, {"package_version": _is_text}) - and (adapter.get("id") is None or isinstance(adapter.get("id"), str)) - and _has_facts(value.get("harness"), {"name": _is_text, "version": _is_text}) - and isinstance(models, Mapping) - and all(_is_model_entry(model) for model in models.values()) - and isinstance(servers, Mapping) - and all(_is_mcp_server_entry(server) for server in servers.values()) - and isinstance(environment, Mapping) - and isinstance(environment.get("provider"), str) - ) + minimal = "what_ran_truncated" in value + if minimal and not _is_truncation_marker(value["what_ran_truncated"]): + return False + for name, is_section in _WHAT_RAN_SECTIONS.items(): + if name in value: + if not is_section(value[name]): + return False + elif not minimal: + return False + return _has_valid_marker(value, "models_truncated") def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: @@ -2119,10 +2184,12 @@ class RunResult(FabricMapping): what_ran: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus - ``source``, ``collected_by`` and ``method``. ``None`` when the + ``source``, ``collected_by`` and ``method``. A minimal object + that the byte budget left holds only ``schema``, ``version`` and + ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a - ``what_ran`` object of that schema and version with every section - and fact well formed, as the Rust core reads it. + ``what_ran`` object of that schema and version with every present + section and fact well formed, as the Rust core reads it. """ agent_name: str diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json index eb21c95e6..8866d76fe 100644 --- a/tests/fixtures/what_ran/reader-vectors.json +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -393,6 +393,60 @@ "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] + }, + { + "name": "truncation markers next to models and MCP servers", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + }, + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 3 + } + } + ] + }, + { + "name": "vendor truncation reason", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "acme.quota", + "dropped": 2 + } + } + ] + }, + { + "name": "minimal section left by the byte budget", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + } + } + } + ] } ], "invalid": [ @@ -804,6 +858,166 @@ ] } ] + }, + { + "name": "truncation marker without a dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap" + } + } + ] + }, + { + "name": "truncation marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 0 + } + } + ] + }, + { + "name": "truncation marker with a string dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": "1" + } + } + ] + }, + { + "name": "truncation reason outside the code pattern", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "Entry Cap", + "dropped": 1 + } + } + ] + }, + { + "name": "truncation marker that is not an object", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": "entry_cap" + } + ] + }, + { + "name": "schema and version only, without the minimal marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1" + } + } + ] + }, + { + "name": "minimal section with a null marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": null + } + } + ] + }, + { + "name": "minimal section with a zero dropped count", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 0 + } + } + } + ] + }, + { + "name": "minimal section with a malformed section", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + }, + "harness": "codex" + } + } + ] + }, + { + "name": "null section", + "patch": [ + { + "path": [ + "harness" + ], + "value": null + } + ] + }, + { + "name": "null models_truncated", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": null + } + ] + }, + { + "name": "null servers_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": null + } + ] } ] } From 6dea00aad281a751bacd554064a427873dda5f2b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:17 -0700 Subject: [PATCH 07/74] docs: describe what_ran bounds in the Python SDK guide Explain the entry caps, the key length rule, the byte budget, the truncation markers and the minimal object, and tell readers to check what_ran_truncated before reading a section. Keep the integration skill's RunResult notes in step. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 22 ++++++++++++++++--- .../references/results-and-errors.md | 4 +++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index fa23c2a6c..cfb12c1ef 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1032,8 +1032,9 @@ state `unsupported` with reason `key_not_declared` and method Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` describes a server the harness reports but the plan does not configure. It has -the same shape, except that `transport` can be absent and `endpoint` has state -`not_applicable` with reason `not_configured`. +the same shape, except that `endpoint` has state `not_applicable` with reason +`not_configured`. A `plan` entry always has `transport`. An entry of any other +origin can leave it out. `endpoint` keeps no secret. A network server's endpoint is its lowercase scheme and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 @@ -1054,6 +1055,21 @@ A string value longer than 256 characters keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. The rule applies to each element of a list value too. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in +sorted key order. An entry whose key is longer than 128 characters is dropped. +While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers +are dropped from the end of the sorted order, and then model roles. A map that +lost entries has a marker next to it, `models_truncated` or +`mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or +`byte_budget`) and the `dropped` count. Each map has one marker: when more than +one rule dropped entries, `reason` names the last rule applied, in that order, +and `dropped` counts every dropped entry. `dropped` is a JSON integer. The Rust +core and the Python SDK reject `1.0`, although a JSON Schema validator may +accept it. If `what_ran` is still too large with no MCP server and no model +role left, it holds only `schema`, `version`, and `what_ran_truncated` with +reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you +read a section. Dropping entries never changes the run's status or error. + `what_ran` is `None` when the core that produced the result predates it. The Rust core and the Python SDK apply the same structural checks. A `what_ran` that is not an object, names another schema or version, or has a malformed @@ -1063,7 +1079,7 @@ ignored. Treat `what_ran` as evidence, not as a key for grouping runs. ```python what_ran = result.what_ran -if what_ran is not None: +if what_ran is not None and "what_ran_truncated" not in what_ran: core = what_ran["fabric"]["core_version"] print(core.get("value"), core["source"]) for role, model in what_ran["models"].items(): diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index b9b51edfc..3793b558a 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -53,7 +53,9 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, currently read state `unsupported` with reason `key_not_declared`. A malformed `what_ran` reads as `None` without -failing the result. Refer to +failing the result. A `what_ran` over its 128 KiB budget can hold only +`schema`, `version`, and `what_ran_truncated`, so check for that marker before +reading a section. Refer to [What Ran](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#what-ran). ## Correlation IDs From e3a287652b04b8fd23fe75712bc753294bc11a91 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:18 -0700 Subject: [PATCH 08/74] docs: regenerate API references for what_ran bounds Regenerate the Rust library reference pages of the what_ran module for TruncatedEntries and the optional sections, and update the Python types reference for RunResult.what_ran. Only the pages this change affects are included. Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 2 +- .../what-ran/constant-what-ran-schema.mdx | 2 +- .../what-ran/constant-what-ran-version.mdx | 2 +- .../nemo-fabric-core/what-ran/enum-fact.mdx | 2 +- .../what-ran/enum-factsource.mdx | 2 +- .../what-ran/enum-factstate.mdx | 2 +- .../nemo-fabric-core/what-ran/index.mdx | 3 + .../what-ran/struct-mcpfacts.mdx | 8 +- .../what-ran/struct-mcpserverfacts.mdx | 4 +- .../what-ran/struct-truncatedentries.mdx | 105 ++++++++++++++++++ .../what-ran/struct-whatran.mdx | 30 +++-- 11 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 7664714b5..159aa7afd 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1443,7 +1443,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. - `metadata`: Result-specific structured details. - - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every section and fact well formed, as the Rust core reads it. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. A minimal object that the byte budget left holds only ``schema``, ``version`` and ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every present section and fact well formed, as the Rust core reads it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx index 06068c300..3b95258d3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_SCHEMA" sidebar-title: "WHAT_RAN_SCHEMA" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" description: "Schema name written into every `WhatRan`." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx index 01dc53369..f517163e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_VERSION" sidebar-title: "WHAT_RAN_VERSION" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" description: "Contract version written into every `WhatRan`." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx index 55c94c912..027e467d6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -3,7 +3,7 @@ title: "Enum Fact" sidebar-title: "Fact" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" description: "One fact: a `value`, or a `state` with a `reason`, never both." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx index 66a4e30c5..fde659009 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -3,7 +3,7 @@ title: "Enum Fact Source" sidebar-title: "FactSource" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" description: "How NeMo Fabric learned a fact." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx index 9fd1a13cf..b91461c16 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -3,7 +3,7 @@ title: "Enum Fact State" sidebar-title: "FactState" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index 18ba5a872..b99298410 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -16,6 +16,8 @@ What executed for one invocation, and how NeMo Fabric learned each fact. A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. + `what_ran` is evidence. It never decides which runs are compared. Compatibility rules for readers: @@ -38,6 +40,7 @@ Compatibility rules for readers: - [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. - [MissingFact](struct-missingfact.mdx): A fact without a value. - [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [TruncatedEntries](struct-truncatedentries.mdx): Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. - [WhatRan](struct-whatran.mdx): What executed for one invocation. ## Enums diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx index c5ddca739..a5628947d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, McpServerFacts>,\n}"}} />
+
BTreeMap<String, McpServerFacts>,\n    pub servers_truncated: Option<TruncatedEntries>,\n}"}} />
MCP facts. @@ -18,7 +18,11 @@ MCP facts. ### `servers: BTreeMap` -Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. At most 32 servers, in sorted key order, each key at most 128 characters. + +### `servers_truncated: Option` + +Servers dropped from `servers`. Absent when none was dropped, never `null`. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx index 19730218c..ad6c67371 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Facts about one MCP server. -An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, its `transport` may be absent, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. Only a `plan` entry must name its transport. An entry of any other origin may leave it out. ## Fields @@ -24,7 +24,7 @@ Where the entry comes from: `plan` for a server the plan configures, `harness` f ### `transport: Option` -Transport, such as `stdio` or `streamable-http`. Always present for a `plan` entry. +Transport, such as `stdio` or `streamable-http`. Required for a `plan` entry, and optional for any other origin. ### `endpoint: Fact` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx new file mode 100644 index 000000000..062b10bd0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx @@ -0,0 +1,105 @@ +--- +title: "Struct Truncated Entries" +sidebar-title: "TruncatedEntries" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-truncatedentries" +description: "Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub dropped: u64,\n}"}} />
+ +Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. + +## Fields + +### `reason: String` + +Why the entries were dropped: `key_too_long` for a key longer than 128 characters, `entry_cap` for an entry past the map's cap, or `byte_budget` for an entry dropped to keep the whole object within 128 KiB. One marker covers a map: when more than one rule dropped entries from it, `reason` names the last rule applied, in that order, and `dropped` counts them all. An open code like a fact's `reason`. + +### `dropped: u64` + +How many entries were dropped. A JSON integer: the Rust and Python readers reject `1.0`, although a JSON Schema validator may accept it. + +## Trait Implementations + +### `impl Clone for TruncatedEntries` + +
Clone for TruncatedEntries"}} />
+ +#### `clone` + +
clone(&self) -> TruncatedEntries"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TruncatedEntries` + +
Debug for TruncatedEntries"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TruncatedEntries` + +
Deserialize<'de> for TruncatedEntries"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a marker, rejecting a `reason` outside the code pattern or a `dropped` count of zero. Unknown properties are ignored. + +### `impl JsonSchema for TruncatedEntries` + +
TruncatedEntries"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TruncatedEntries` + +
PartialEq for TruncatedEntries"}} />
+ +#### `eq` + +
eq(&self, other: &TruncatedEntries) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TruncatedEntries` + +
Serialize for TruncatedEntries"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TruncatedEntries` + +
Eq for TruncatedEntries"}} />
+ +### `impl StructuralPartialEq for TruncatedEntries` + +
StructuralPartialEq for TruncatedEntries"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx index d25aaa76f..2873628a8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -3,18 +3,20 @@ title: "Struct WhatRan" sidebar-title: "WhatRan" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" description: "What executed for one invocation." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub version: String,\n    pub fabric: FabricFacts,\n    pub adapter: AdapterFacts,\n    pub harness: HarnessFacts,\n    pub models: BTreeMap<String, ModelFacts>,\n    pub mcp: McpFacts,\n    pub environment: EnvironmentFacts,\n}"}} />
+
String,\n    pub version: String,\n    pub fabric: Option<FabricFacts>,\n    pub adapter: Option<AdapterFacts>,\n    pub harness: Option<HarnessFacts>,\n    pub models: Option<BTreeMap<String, ModelFacts>>,\n    pub models_truncated: Option<TruncatedEntries>,\n    pub mcp: Option<McpFacts>,\n    pub environment: Option<EnvironmentFacts>,\n    pub what_ran_truncated: Option<TruncatedEntries>,\n}"}} />
What executed for one invocation. -Decoding a `WhatRan` directly checks the shape of every section and fact. `RunResult` also checks the schema name and version, and that every `plan` MCP entry names its transport. +Every section is present, unless `what_ran_truncated` is present: then the object is a minimal one that the byte budget left with only its schema name and version, and every section may be absent. + +Decoding a `WhatRan` directly checks the shape of every section and fact, and that no section is `null`. `RunResult` also checks the schema name and version, that every section is present unless `what_ran_truncated` is, and that every `plan` MCP entry names its transport. ## Fields @@ -26,30 +28,38 @@ Schema name, `nemo.fabric.what_ran`. Contract version, `v1alpha1`. -### `fabric: FabricFacts` +### `fabric: Option` NeMo Fabric core that ran the adapter. -### `adapter: AdapterFacts` +### `adapter: Option` Adapter that ran. -### `harness: HarnessFacts` +### `harness: Option` Harness behind the adapter. -### `models: BTreeMap` +### `models: Option>` + +Model facts keyed by the role names of the plan's `models` settings. At most 16 roles, in sorted key order, each key at most 128 characters. + +### `models_truncated: Option` -Model facts keyed by the role names of the plan's `models` settings. +Model roles dropped from `models`. Absent when none was dropped, never `null`. -### `mcp: McpFacts` +### `mcp: Option` MCP server facts. -### `environment: EnvironmentFacts` +### `environment: Option` Environment the runtime ran in. +### `what_ran_truncated: Option` + +Present only on a minimal object: every section was dropped to keep the object within 128 KiB, with reason `byte_budget` and a dropped count of 1. + ## Trait Implementations ### `impl Clone for WhatRan` From cdc0636a647d562a02a72110f9921b8943d3bd04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:00:16 -0700 Subject: [PATCH 09/74] docs: name NeMo Fabric in the Deep Agents usage limits Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 78e94ce51..66a44eece 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -243,7 +243,7 @@ unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; Fabric configures no such cache. +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming From f076c101323b4c424bfe8af0dbbc07241a11bdde Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 22:51:03 -0700 Subject: [PATCH 10/74] test: cover what_ran on a realistic config Run a local host invocation with two model roles and three MCP servers (npx with arguments, streamable-http and sse) and no environment section, and assert the exact what_ran a user gets, including the origin of every unsupported adapter-reported fact and that no configured secret appears. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 97 +++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 1b5493545..d05ddfc1e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5501,6 +5501,103 @@ for line in sys.stdin: } } + /// The `what_ran` a user gets from a normal benchmark config: two model + /// roles and three MCP servers on an adapter that accepts both, and no + /// `environment` section, so the provider defaults to `local`. + #[test] + fn local_host_what_ran_reports_a_realistic_config() { + use serde_json::json; + let (root, plan) = local_host_plan("success"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["config"] = json!({"accepts": ["models", "mcp"]}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let mut config = serde_json::to_value(&plan.config).expect("config JSON"); + config + .as_object_mut() + .expect("config") + .remove("environment"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}, + "summarizer": {"provider": "nvidia", "model": "nvidia/nemotron-nano-9b-v2"}, + }); + config["mcp"] = json!({"servers": { + "filesystem": { + "transport": "stdio", + "url": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/SECRET_ARGUMENT"], + "env": {"FS_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + }, + "search": { + "transport": "streamable-http", + "url": "https://mcp.example.com/mcp?api_key=SECRET_QUERY", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + } + }}); + let config: crate::config::FabricConfig = + serde_json::from_value(config).expect("typed config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) + .expect("resolve realistic plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let configured = |value: &str| { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + }; + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let model = |provider: &str, requested: &str| { + json!({"provider": provider, "requested": configured(requested), + "served": not_declared}) + }; + let server = |transport: &str, endpoint: &str| { + json!({"origin": "plan", "transport": transport, "endpoint": configured(endpoint), + "runtime": {"connected": not_declared, "protocol_version": not_declared, + "server_name": not_declared, "server_version": not_declared, + "tool_names": not_declared}}) + }; + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "acme.fabric.local-host", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": { + "default": model("anthropic", "anthropic/claude-sonnet-4-5"), + "summarizer": model("nvidia", "nvidia/nemotron-nano-9b-v2") + }, + "mcp": {"servers": { + "events": server("sse", "http://localhost:8931"), + "filesystem": server("stdio", "npx"), + "search": server("streamable-http", "https://mcp.example.com:443") + }}, + "environment": {"provider": "local"} + }) + ); + assert!(!what_ran.to_string().contains("SECRET"), "{what_ran}"); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_what_ran_bounds_keep_the_status_and_error() { let model = |provider: &str| -> crate::config::ModelConfig { From d64832dfaa38e3cc80030736d8134a77beb6e2fe Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 22:51:09 -0700 Subject: [PATCH 11/74] test: cover what_ran for each bundled descriptor and at the entry caps Resolve a plan from each of the 16 bundled descriptors with a model and, where the adapter accepts MCP, three servers, and check that what_ran passes the schema, carries no truncation marker, and reads every adapter-reported fact as unsupported/key_not_declared. Also show that 16 roles and 32 servers with 128-character keys fit the byte budget, and that stdio placeholders read executable_unparsed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 150 +++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 27dbda17f..9773a9756 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1341,6 +1341,143 @@ mod tests { assert_eq!(what_ran["mcp"], json!({"servers": {}})); } + /// Every adapter-reported fact in `what_ran`, by its JSON path. + fn reported_facts(value: &Value, path: &str) -> Vec<(String, Value)> { + match value { + Value::Object(map) if map.get("source") == Some(&json!("reported")) => { + vec![(path.to_string(), value.clone())] + } + Value::Object(map) => map + .iter() + .flat_map(|(key, value)| reported_facts(value, &format!("{path}.{key}"))) + .collect(), + _ => Vec::new(), + } + } + + #[test] + fn a_plan_from_each_bundled_descriptor_collects_plan_facts_and_undeclared_adapter_facts() { + let catalog_path = repository_root().join( + "sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json", + ); + let catalog: Value = + serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) + .expect("catalog JSON"); + let adapters = catalog["adapters"].as_object().expect("catalog adapters"); + assert_eq!(adapters.len(), 16); + let validator = what_ran_validator(); + for (adapter_id, entry) in adapters { + let descriptor = &entry["descriptor"]; + let mut config = config_for(adapter_id); + for setting in descriptor["settings_schema"]["required"] + .as_array() + .into_iter() + .flatten() + { + let setting = setting.as_str().expect("setting name"); + config["harness"]["settings"][setting] = json!("https://agent.example.com"); + } + let accepts = |field: &str| { + descriptor["config"]["accepts"] + .as_array() + .is_some_and(|accepts| accepts.contains(&json!(field))) + }; + // A model each adapter's model schema accepts: its fixed provider + // if it has one, and every connection field the schema names. + let model_fields = &descriptor["model_schema"]["properties"]; + let provider = model_fields["provider"]["const"] + .as_str() + .unwrap_or("nvidia"); + config["models"] = + json!({"default": {"provider": provider, "model": "nvidia/nemotron"}}); + for (field, value) in [ + ("base_url", "https://inference.example.com/v1"), + ("api_key_env", "MODEL_API_KEY"), + ] { + if model_fields.get(field).is_some() { + config["models"]["default"][field] = json!(value); + } + } + let accepts_mcp = accepts("mcp"); + if accepts_mcp { + config["mcp"] = json!({"servers": { + "events": {"transport": "sse", "url": "http://localhost:8931/sse", + "exposure": "harness_native"}, + "filesystem": {"transport": "stdio", "url": "npx", + "args": ["-y", "@scope/server-filesystem"], + "exposure": "harness_native"}, + "search": {"transport": "streamable-http", "url": "https://mcp.example.com/mcp", + "exposure": "harness_native"} + }}); + } + + // Core discovers the repository's adapters as its bundled set. + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + let plan = crate::config::resolve_run_plan_from_config( + config, + ResolveContext::new(repository_root()), + ) + .unwrap_or_else(|error| panic!("{adapter_id}: {error}")); + let what_ran = collect(&plan); + + assert!(validator.is_valid(&what_ran), "{adapter_id}: {what_ran}"); + assert_eq!(what_ran["adapter"]["id"], json!(adapter_id)); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("nvidia/nemotron") + ); + let servers = if accepts_mcp { 3 } else { 0 }; + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]).len(), + servers, + "{adapter_id}" + ); + // package version, harness name and version, served, 5 per server. + let reported = reported_facts(&what_ran, "what_ran"); + assert_eq!(reported.len(), 4 + 5 * servers, "{adapter_id}"); + for (path, fact) in reported { + assert_eq!(fact, not_declared(), "{adapter_id} {path}"); + } + for marker in [ + &what_ran["models_truncated"], + &what_ran["mcp"]["servers_truncated"], + &what_ran["what_ran_truncated"], + ] { + assert!(marker.is_null(), "{adapter_id}: {what_ran}"); + } + assert!(json_len(&what_ran) <= 128 * 1024, "{adapter_id}"); + } + } + + #[test] + fn both_entry_caps_with_the_longest_keys_and_values_fit_the_byte_budget() { + let longest_key = |prefix: &str, index: usize| { + let key = format!("{prefix}-{index:02}-"); + format!("{key}{}", "k".repeat(128 - key.len())) + }; + let roles: Vec = (0..16).map(|index| longest_key("role", index)).collect(); + let servers: Vec = (0..32).map(|index| longest_key("server", index)).collect(); + let mut plan = plan_with_entries(&roles, &servers); + for model in plan.config.models.values_mut() { + model.model = "m".repeat(256); + } + + let what_ran = collect(&plan); + + assert_eq!(map_keys(&what_ran["models"]), roles); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), servers); + assert!(what_ran.get("models_truncated").is_none(), "{what_ran}"); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{what_ran}" + ); + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + } + #[test] fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { let mut config = config_for("nvidia.fabric.claude"); @@ -1386,6 +1523,19 @@ mod tests { "collection_failed", "executable_unparsed", ), + // Placeholders a harness expands later, not a file name. + ( + McpTransport::Stdio, + "$MCP_COMMAND", + "collection_failed", + "executable_unparsed", + ), + ( + McpTransport::Stdio, + "/opt/mcp/server-{env:MCP_VARIANT}", + "collection_failed", + "executable_unparsed", + ), (McpTransport::Sse, " ", "unavailable", "not_configured"), (McpTransport::Stdio, "", "unavailable", "not_configured"), ] { From 8eba8b5c8b755544847a0ae30ec17b88a8736ca9 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:33:43 -0700 Subject: [PATCH 12/74] test: loosen the catalog count and rename the what_ran size test Loop over whatever the catalog lists without pinning its size, so adding or removing a bundled adapter doesn't break the test. Name the size test after what it builds: ASCII keys and model names at their maximum length. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 9773a9756..d21df1d3c 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1364,7 +1364,7 @@ mod tests { serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) .expect("catalog JSON"); let adapters = catalog["adapters"].as_object().expect("catalog adapters"); - assert_eq!(adapters.len(), 16); + assert!(!adapters.is_empty()); let validator = what_ran_validator(); for (adapter_id, entry) in adapters { let descriptor = &entry["descriptor"]; @@ -1454,7 +1454,7 @@ mod tests { } #[test] - fn both_entry_caps_with_the_longest_keys_and_values_fit_the_byte_budget() { + fn both_entry_caps_with_max_length_ascii_keys_and_models_fit_the_byte_budget() { let longest_key = |prefix: &str, index: usize| { let key = format!("{prefix}-{index:02}-"); format!("{key}{}", "k".repeat(128 - key.len())) From 07ecc566cd3f8a4e3fc05057e21c2a31b84c7c6d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:30:54 -0700 Subject: [PATCH 13/74] feat: read adapter report keys into what_ran Adapters had no way to report their own facts, so the adapter package version, the harness name and version, the served model and the MCP runtime facts always read unsupported with reason key_not_declared. Core now reads nemo_fabric.versions.v1 and nemo_fabric.mcp.v1 from a result's extensions after the descriptor's extension check, when the descriptor's run_result schema names the key under properties. Only adapter-owned facts with source reported and collector adapter are kept; a malformed or forged fact reads invalid_report, an omitted one fact_omitted, an omitted key key_omitted, and a key over 64 KiB too_large. A reported server the plan does not configure becomes a harness entry. tool_names keeps 64 sorted names and counts the rest in a new tool_names_truncated marker, added to the schema and both readers. Ingestion never changes the run's status or error. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 78 +- crates/fabric-core/src/what_ran.rs | 756 +++++++++++++++++- schemas/sdk/run-result.schema.json | 6 +- .../src/nemo_fabric/types.py | 4 +- tests/fixtures/what_ran/reader-vectors.json | 51 ++ 5 files changed, 873 insertions(+), 22 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d05ddfc1e..17c4fee03 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1923,7 +1923,7 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; - let what_ran = crate::what_ran::collect_before_request( + let mut what_ran = crate::what_ran::collect_before_request( plan, adapter_id(plan), &runtime.environment.provider, @@ -2034,15 +2034,16 @@ fn run_local_host_invocation_with_timeout( fabric_invocation, ) = exchange_result?; - let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { - lifecycle_error( - operation, - &runtime.runtime_id, - "invalid_agent_run_result", - format!("adapter returned an invalid AgentRunResult: {error}"), - &stderr, - ) - })?; + let mut agent_result: AgentRunResult = + serde_json::from_value(adapter_output).map_err(|error| { + lifecycle_error( + operation, + &runtime.runtime_id, + "invalid_agent_run_result", + format!("adapter returned an invalid AgentRunResult: {error}"), + &stderr, + ) + })?; agent_result.validate().map_err(|error| { lifecycle_error( operation, @@ -2053,6 +2054,7 @@ fn run_local_host_invocation_with_timeout( ) })?; validate_agent_run_result_extensions(&agent_result, plan.adapter_descriptor.as_ref())?; + crate::what_ran::ingest_adapter_reports(&mut what_ran, plan, &mut agent_result.extensions); if !agent_result.artifacts.is_empty() && artifacts.root.is_none() { return Err(lifecycle_error( operation, @@ -4156,6 +4158,25 @@ for line in sys.stdin: "extensions": {"source": "fake-host"}, }, } + elif MODE == "versions_report": + fact = lambda value, method: { + "value": value, "source": "reported", "collected_by": "adapter", + "method": method, + } + result = { + "status": "succeeded", + "output": output, + "extensions": { + "trace_id": "trace-1", + "nemo_fabric.versions.v1": { + "adapter": {"package_version": fact("1.2.3", "package_metadata")}, + "harness": { + "name": fact("fake-harness", "harness_api"), + "version": fact("4.5.6", "harness_api"), + }, + }, + }, + } elif MODE == "typed_result": result = { "status": "succeeded", @@ -5598,6 +5619,43 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + /// An adapter that declares the versions key and reports it gets its + /// facts into `what_ran`, and the key leaves `metadata.adapter`. + #[test] + fn local_host_what_ran_ingests_a_declared_versions_report() { + use serde_json::json; + let (root, plan) = local_host_plan("versions_report"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["extension_schemas"]["run_result"]["properties"]["nemo_fabric.versions.v1"] = + json!({"type": "object"}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let plan = resolve_run_plan_from_config(plan.config, ResolveContext::new(&root)) + .expect("resolve plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.metadata["adapter"], json!({"trace_id": "trace-1"})); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let reported = |value: &str, method: &str| { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + }; + assert_eq!( + what_ran["adapter"]["package_version"], + reported("1.2.3", "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported("fake-harness", "harness_api"), + "version": reported("4.5.6", "harness_api")}) + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_what_ran_bounds_keep_the_status_and_error() { let model = |provider: &str| -> crate::config::ModelConfig { diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index d21df1d3c..f9be07a25 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -25,6 +25,12 @@ //! it is replaced by a minimal one that holds only the schema name, the //! version and the `what_ran_truncated` marker. //! +//! An adapter reports its own facts through the report keys +//! [`VERSIONS_REPORT_KEY`] and [`MCP_REPORT_KEY`] in its result's +//! extensions, once its descriptor declares them. A reported list keeps at +//! most 64 elements in sorted order, counted in a marker beside it, such as +//! `tool_names_truncated`. +//! //! `what_ran` is evidence. It never decides which runs are compared. //! //! Compatibility rules for readers: @@ -50,6 +56,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; +use crate::adapter_contract::AdapterExtensionPoint; use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; use crate::endpoint::{endpoint_identity, executable_name}; @@ -437,8 +444,17 @@ pub struct McpRuntimeFacts { /// Version the server reports for itself. pub server_version: Fact, /// Raw tool names the server lists, each bounded by the string length - /// rule. + /// rule. At most 64 names, in sorted order. pub tool_names: Fact>, + /// Tool names dropped from `tool_names`, by the adapter's helper or by + /// core. Absent when none was dropped, never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub tool_names_truncated: Option, } /// Environment facts. @@ -530,13 +546,7 @@ impl Fact { /// An adapter-reported fact while core reads no adapter report key. fn report_key_not_declared() -> Self { - Self::missing( - FactState::Unsupported, - REASON_KEY_NOT_DECLARED, - FactSource::Reported, - ADAPTER, - METHOD_ADAPTER_REPORT, - ) + reported_missing(FactState::Unsupported, REASON_KEY_NOT_DECLARED) } } @@ -672,8 +682,9 @@ impl WhatRan { /// servers and environment provider. The model and MCP server maps follow /// the entry caps and the byte budget, and collection never fails. /// -/// Core reads no adapter report key yet, so every adapter-reported fact -/// reads `unsupported` with reason `key_not_declared`. The collecting core +/// Every adapter-reported fact reads `unsupported` with reason +/// `key_not_declared` until [`ingest_adapter_reports`] reads the report keys +/// the result carries. The collecting core /// runs the adapter in a local environment, so `host_core_version` reads /// `not_applicable`. A host that relays the result from a task core calls /// [`attribute_to_task_core`]. @@ -821,6 +832,7 @@ fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { server_name: Fact::report_key_not_declared(), server_version: Fact::report_key_not_declared(), tool_names: Fact::report_key_not_declared(), + tool_names_truncated: None, }, } } @@ -886,6 +898,280 @@ pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provide what_ran.apply_byte_budget(); } +/// Report key for the adapter package version, the harness name and version, +/// and the served model of each role. +pub const VERSIONS_REPORT_KEY: &str = "nemo_fabric.versions.v1"; +/// Report key for the `runtime` block of each MCP server, and for servers +/// the harness reports that the plan does not configure. +pub const MCP_REPORT_KEY: &str = "nemo_fabric.mcp.v1"; + +/// Largest compact JSON of one report key, in bytes. +const MAX_REPORT_BYTES: usize = 64 * 1024; +/// Most elements a list fact keeps. +const MAX_LIST_ENTRIES: usize = 64; +const ORIGIN_HARNESS: &str = "harness"; +const REASON_KEY_OMITTED: &str = "key_omitted"; +const REASON_FACT_OMITTED: &str = "fact_omitted"; +const REASON_INVALID_REPORT: &str = "invalid_report"; +const REASON_TOO_LARGE: &str = "too_large"; + +/// Move the adapter's declared report keys from a result's extensions into +/// `what_ran`. +/// +/// Core calls this after the descriptor's extension schema accepted +/// `extensions`, so ingestion never rejects a result. A key is declared +/// only when the descriptor's `run_result` extension schema names it +/// literally under `properties`; the facts of an undeclared key keep +/// `unsupported` with reason `key_not_declared`, and the key stays in the +/// extensions. A declared key that is absent reads `unavailable` with +/// reason `key_omitted`, and one that is not an object reads +/// `collection_failed` with reason `invalid_report` and stays in the +/// extensions. A declared object is removed from the extensions. Past +/// 64 KiB of compact JSON, every fact of that key reads `collection_failed` +/// with reason `too_large`. +/// +/// Inside a report, a key may hold only adapter-owned facts, each with +/// source `reported` and collector `adapter`: the adapter package version, +/// the harness name and version, each plan role's served model, each MCP +/// server's `runtime` block with its `tool_names_truncated` marker, and a +/// `harness` server's transport. A malformed or forged fact reads +/// `collection_failed` with reason `invalid_report`, and an omitted one +/// reads `unavailable` with reason `fact_omitted`. Core ignores every other +/// property, so a report never changes a fact core owns. The string length +/// rule, the 64-entry cap on `tool_names`, the entry caps and the byte +/// budget then apply to the result. +pub(crate) fn ingest_adapter_reports( + what_ran: &mut WhatRan, + plan: &RunPlan, + extensions: &mut BTreeMap, +) { + let run_result_schema = plan.adapter_descriptor.as_ref().and_then(|adapter| { + adapter + .descriptor + .extension_schemas + .get(&AdapterExtensionPoint::RunResult) + }); + for key in [VERSIONS_REPORT_KEY, MCP_REPORT_KEY] { + if !declares_report_key(run_result_schema, key) { + continue; + } + let removed = match extensions.get(key) { + Some(Value::Object(_)) => extensions.remove(key), + _ => None, + }; + let report = match &removed { + Some(Value::Object(object)) if compact_json_len(object) > MAX_REPORT_BYTES => { + Report::missing(FactState::CollectionFailed, REASON_TOO_LARGE) + } + Some(Value::Object(object)) => Report::Object(object), + _ if extensions.contains_key(key) => { + Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT) + } + _ => Report::missing(FactState::Unavailable, REASON_KEY_OMITTED), + }; + if key == VERSIONS_REPORT_KEY { + apply_versions_report(what_ran, &report); + } else { + apply_mcp_report(what_ran, plan, &report); + } + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); +} + +fn compact_json_len(object: &serde_json::Map) -> usize { + serde_json::to_vec(object) + .expect("a JSON object is always serializable") + .len() +} + +/// Whether a `run_result` extension schema names `key` literally under +/// `properties`. Core never resolves `$ref`, `allOf` or patterns. +fn declares_report_key(schema: Option<&serde_json::Map>, key: &str) -> bool { + schema + .and_then(|schema| schema.get("properties")) + .and_then(Value::as_object) + .is_some_and(|properties| properties.contains_key(key)) +} + +/// One object of an adapter report, or the state every fact under it reads +/// when the object is missing or malformed. +enum Report<'a> { + Object(&'a serde_json::Map), + Missing(MissingFact), +} + +impl<'a> Report<'a> { + fn missing(state: FactState, reason: &str) -> Self { + match reported_missing::<()>(state, reason) { + Fact::Missing(missing) => Self::Missing(missing), + Fact::Known(_) => unreachable!("reported_missing builds a missing fact"), + } + } + + /// The object under `key`: `fact_omitted` when absent, `invalid_report` + /// when not an object. + fn child(&self, key: &str) -> Report<'a> { + match self { + Self::Object(object) => match object.get(key) { + Some(Value::Object(child)) => Report::Object(child), + Some(_) => Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + None => Report::missing(FactState::Unavailable, REASON_FACT_OMITTED), + }, + Self::Missing(missing) => Report::Missing(missing.clone()), + } + } + + /// The reported fact under `key`, kept only when it decodes and names + /// source `reported` and collector `adapter`. + fn fact(&self, key: &str) -> Fact { + let value = match self { + Self::Missing(missing) => return Fact::Missing(missing.clone()), + Self::Object(object) => object.get(key), + }; + match value.map(Fact::::deserialize) { + None => reported_missing(FactState::Unavailable, REASON_FACT_OMITTED), + Some(Ok(fact)) + if fact.origin().source == FactSource::Reported + && fact.origin().collected_by == ADAPTER => + { + fact + } + Some(_) => reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + } + } +} + +/// A reported fact without a value, collected by the adapter's report. +fn reported_missing(state: FactState, reason: &str) -> Fact { + Fact::missing( + state, + reason, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) +} + +fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { + if let Some(adapter) = what_ran.adapter.as_mut() { + adapter.package_version = bounded_string(report.child("adapter").fact("package_version")); + } + if let Some(harness) = what_ran.harness.as_mut() { + let reported = report.child("harness"); + harness.name = bounded_string(reported.fact("name")); + harness.version = bounded_string(reported.fact("version")); + } + let models = report.child("models"); + for (role, model) in what_ran.models.iter_mut().flatten() { + model.served = bounded_string(models.child(role).fact("served")); + } +} + +/// Fill each plan server's `runtime` block, and add a `harness` entry for +/// each reported server the plan does not configure. +fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { + let Some(mcp) = what_ran.mcp.as_mut() else { + return; + }; + let servers = report.child("servers"); + for (name, server) in &mut mcp.servers { + server.runtime = runtime_facts(&servers.child(name).child("runtime")); + } + let Report::Object(reported) = servers else { + return; + }; + let plan_servers = &plan.capability_plan.mcp_servers; + for (name, entry) in reported { + if plan_servers.contains_key(name) { + continue; + } + let entry_report = servers.child(name); + mcp.servers.insert( + name.clone(), + McpServerFacts { + origin: ORIGIN_HARNESS.to_string(), + transport: entry + .get("transport") + .and_then(Value::as_str) + .map(str::to_lowercase), + endpoint: Fact::missing( + FactState::NotApplicable, + REASON_NOT_CONFIGURED, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + runtime: runtime_facts(&entry_report.child("runtime")), + }, + ); + } +} + +/// One server's `runtime` block from its report. `tool_names` keeps at most +/// 64 names in sorted order, and its marker adds core's drops to the +/// helper's. +fn runtime_facts(report: &Report<'_>) -> McpRuntimeFacts { + let (mut tool_names, core_marker) = bounded_list(report.fact("tool_names")); + let mut helper_marker = None; + if let (Fact::Known(_), Report::Object(object)) = (&tool_names, report) + && let Some(marker) = object.get("tool_names_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + tool_names = reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + let tool_names_truncated = match tool_names { + Fact::Known(_) => merged_marker(helper_marker, core_marker), + Fact::Missing(_) => None, + }; + McpRuntimeFacts { + connected: report.fact("connected"), + protocol_version: bounded_string(report.fact("protocol_version")), + server_name: bounded_string(report.fact("server_name")), + server_version: bounded_string(report.fact("server_version")), + tool_names, + tool_names_truncated, + } +} + +/// `fact` with the string length rule applied to its value. +fn bounded_string(fact: Fact) -> Fact { + match fact { + Fact::Known(known) => Fact::Known(KnownFact { + value: bounded_text(&known.value), + origin: known.origin, + }), + missing => missing, + } +} + +/// `fact` with each element bounded by the string length rule, sorted by +/// code point, and cut to 64 elements, with a marker that counts the cut. +fn bounded_list(fact: Fact>) -> (Fact>, Option) { + let Fact::Known(known) = fact else { + return (fact, None); + }; + let mut value: Vec = known.value.iter().map(|item| bounded_text(item)).collect(); + value.sort(); + let dropped = value.len().saturating_sub(MAX_LIST_ENTRIES); + value.truncate(MAX_LIST_ENTRIES); + let marker = (dropped > 0).then(|| TruncatedEntries { + reason: REASON_ENTRY_CAP.to_string(), + dropped: dropped as u64, + }); + ( + Fact::Known(KnownFact { + value, + origin: known.origin, + }), + marker, + ) +} + /// `text` whole when it has at most 256 characters. A longer string keeps /// its first 240 characters, then `~`, then the first 15 hex digits of the /// whole string's SHA-256, so two long values collide only on a 60-bit @@ -1678,4 +1964,454 @@ mod tests { "acme.collector" ); } + + /// The secrets plan with a descriptor that declares `keys` in its + /// `run_result` extension schema. + fn plan_declaring(keys: &[&str]) -> RunPlan { + let mut plan = plan_with_secrets(); + let properties: serde_json::Map = keys + .iter() + .map(|key| (key.to_string(), json!({"type": "object"}))) + .collect(); + let schema = json!({"type": "object", "properties": properties}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema object").clone(), + ); + plan + } + + fn reported(value: Value, method: &str) -> Value { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + } + + fn report_state(state: &str, reason: &str) -> Value { + json!({"state": state, "reason": reason, "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + /// Ingest `extensions` into the facts collected for `plan`, and return + /// `what_ran` with the extensions core leaves on the result. + fn ingest(plan: &RunPlan, extensions: Value) -> (Value, Value) { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + let mut what_ran = collect_before_request(plan, adapter_id, "local"); + let mut extensions: BTreeMap = + serde_json::from_value(extensions).expect("extensions map"); + ingest_adapter_reports(&mut what_ran, plan, &mut extensions); + let what_ran = serde_json::to_value(what_ran).expect("serialize what_ran"); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + ( + what_ran, + serde_json::to_value(extensions).expect("extensions JSON"), + ) + } + + fn versions_report() -> Value { + json!({ + "adapter": {"package_version": reported(json!("0.6.0"), "package_metadata")}, + "harness": { + "name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message") + }, + "models": {"default": {"served": reported(json!("claude-sonnet-4-5-20250929"), + "harness_api")}} + }) + } + + #[test] + fn a_declared_versions_report_fills_the_adapter_facts_and_leaves_the_extensions() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + + let (what_ran, extensions) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: versions_report(), "trace_id": "trace-1"}), + ); + + assert_eq!( + what_ran["adapter"]["package_version"], + reported(json!("0.6.0"), "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message")}) + ); + assert_eq!( + what_ran["models"]["default"]["served"], + reported(json!("claude-sonnet-4-5-20250929"), "harness_api") + ); + assert_eq!(extensions, json!({"trace_id": "trace-1"})); + // The MCP key is not declared, so its facts keep `key_not_declared`. + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + not_declared() + ); + } + + #[test] + fn an_undeclared_report_key_stays_in_the_extensions_and_reads_key_not_declared() { + let plan = plan_with_secrets(); + let extensions = json!({VERSIONS_REPORT_KEY: versions_report()}); + + let (what_ran, kept) = ingest(&plan, extensions.clone()); + + assert_eq!(kept, extensions); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, ¬_declared()); + } + } + + #[test] + fn a_report_key_declared_only_through_a_reference_is_not_declared() { + let mut plan = plan_with_secrets(); + let schema = json!({"type": "object", "allOf": [{"properties": { + VERSIONS_REPORT_KEY: {"type": "object"}}}]}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema").clone(), + ); + + let (what_ran, kept) = ingest(&plan, json!({VERSIONS_REPORT_KEY: versions_report()})); + + assert_eq!(what_ran["harness"]["version"], not_declared()); + assert_eq!(kept, json!({VERSIONS_REPORT_KEY: versions_report()})); + } + + #[test] + fn an_omitted_non_object_or_oversized_report_key_marks_every_fact_of_that_key() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY, MCP_REPORT_KEY]); + let oversized = json!({"harness": {"version": reported( + json!("v".repeat(MAX_REPORT_BYTES)), "harness_api")}}); + for (extensions, state, reason, kept) in [ + (json!({}), "unavailable", "key_omitted", json!({})), + ( + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + "collection_failed", + "invalid_report", + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + ), + ( + json!({VERSIONS_REPORT_KEY: oversized, MCP_REPORT_KEY: {"servers": { + "github": {"runtime": {"server_name": reported( + json!("s".repeat(MAX_REPORT_BYTES)), "harness_status")}}}}}), + "collection_failed", + "too_large", + json!({}), + ), + ] { + let (what_ran, extensions) = ingest(&plan, extensions); + + let expected = report_state(state, reason); + assert_eq!(extensions, kept, "{reason}"); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, &expected, "{reason}"); + } + for server in what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .values() + { + for fact in server["runtime"].as_object().expect("runtime").values() { + assert_eq!(fact, &expected, "{reason}"); + } + } + } + } + + #[test] + fn a_report_at_64_kib_is_read_and_one_byte_more_is_too_large() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let with_padding = |padding: usize| { + let mut report = versions_report(); + report["padding"] = json!(""); + let base = serde_json::to_vec(&report).expect("JSON").len(); + report["padding"] = json!("p".repeat(padding - base)); + report + }; + + let (at_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES)}), + ); + let (past_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES + 1)}), + ); + + assert_eq!( + at_limit["harness"]["version"], + reported(json!("2.1.294"), "harness_init_message") + ); + assert_eq!( + past_limit["harness"]["version"], + report_state("collection_failed", "too_large") + ); + } + + #[test] + fn the_origin_rule_rejects_forged_or_malformed_facts_and_keeps_the_rest_of_the_report() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let forged = json!({"value": "9.9.9", "source": "observed", "collected_by": "adapter", + "method": "harness_api"}); + let other_collector = json!({"value": "9.9.9", "source": "reported", + "collected_by": "task_core", "method": "harness_api"}); + let report = json!({ + "fabric": {"core_version": reported(json!("0.0.0-forged"), "crate_version")}, + "adapter": {"id": "forged.adapter", + "package_version": {"value": "0.6.0", "state": "unavailable", + "reason": "both", "source": "reported", "collected_by": "adapter", + "method": "package_metadata"}}, + "harness": {"name": forged, "version": other_collector}, + "models": { + "default": {"served": reported(json!(42), "harness_api"), + "requested": reported(json!("forged"), "harness_api"), + "provider": "forged"}, + "unplanned": {"served": reported(json!("other"), "harness_api")} + } + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(what_ran["adapter"]["package_version"], invalid); + assert_eq!(what_ran["harness"]["name"], invalid); + assert_eq!(what_ran["harness"]["version"], invalid); + assert_eq!(what_ran["models"]["default"]["served"], invalid); + // Facts core owns keep core's values, and roles stay the plan's. + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "nvidia.fabric.claude"); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("claude-sonnet-4-5") + ); + assert_eq!(what_ran["models"]["default"]["provider"], "anthropic"); + assert_eq!(map_keys(&what_ran["models"]), ["default"]); + } + + #[test] + fn omitted_facts_read_fact_omitted_and_a_reported_state_is_kept() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let no_source = json!({"state": "unsupported", "reason": "no_source", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let report = json!({ + "harness": {"name": no_source, "version": no_source}, + "models": "not an object" + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["adapter"]["package_version"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!(what_ran["harness"]["name"], no_source); + assert_eq!( + what_ran["models"]["default"]["served"], + report_state("collection_failed", "invalid_report") + ); + } + + #[test] + fn reported_strings_follow_the_length_rule() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let long = format!("{}a", "m".repeat(300)); + let mut report = versions_report(); + report["harness"]["version"] = reported(json!(long), "harness_api"); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["harness"]["version"]["value"], + format!("{}~96858eedeb3c8e6", "m".repeat(240)) + ); + } + + fn tool_names(count: usize) -> Vec { + // Reverse order, so core's sort decides which names stay. + (0..count) + .rev() + .map(|index| format!("tool-{index:02}")) + .collect() + } + + #[test] + fn mcp_runtime_facts_fill_plan_servers_and_cap_tool_names_at_64() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let runtime = |names: Vec, marker: Option| { + let mut runtime = json!({ + "connected": reported(json!(true), "harness_status"), + "protocol_version": reported(json!("2025-06-18"), "mcp_initialize"), + "server_name": reported(json!("github-mcp"), "harness_status"), + "server_version": reported(json!("1.2.3"), "harness_status"), + "tool_names": reported(json!(names), "harness_status") + }); + if let Some(marker) = marker { + runtime["tool_names_truncated"] = marker; + } + runtime + }; + let report = json!({"servers": { + "github": {"transport": "stdio", "endpoint": configured("forged"), + "runtime": runtime(tool_names(65), None)}, + "events": {"runtime": runtime(tool_names(70), + Some(json!({"reason": "byte_budget", "dropped": 3})))}, + "files": {"runtime": runtime(tool_names(64), None)} + }}); + + let (what_ran, extensions) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!(extensions, json!({})); + let servers = &what_ran["mcp"]["servers"]; + let github = &servers["github"]; + let kept: Vec = tool_names(64).into_iter().rev().collect(); + assert_eq!(github["runtime"]["tool_names"]["value"], json!(kept)); + assert_eq!( + github["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert_eq!( + servers["events"]["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 9}) + ); + assert!(servers["files"]["runtime"]["tool_names_truncated"].is_null()); + // A plan entry keeps its own origin, transport and endpoint. + assert_eq!(github["origin"], "plan"); + assert_eq!(github["transport"], "streamable-http"); + assert_eq!( + github["endpoint"], + configured("https://api.githubcopilot.com:443") + ); + assert_eq!( + github["runtime"]["connected"], + reported(json!(true), "harness_status") + ); + } + + #[test] + fn mcp_report_adds_harness_entries_and_marks_omitted_plan_servers() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let not_collected = json!({"state": "unavailable", "reason": "not_collected", + "source": "reported", "collected_by": "adapter", + "method": "harness_startup_status"}); + let long_name = "h".repeat(129); + let report = json!({"servers": { + "plugin-docs": {"transport": "STDIO", "runtime": { + "connected": reported(json!(true), "harness_startup_status"), + "protocol_version": not_collected, + "server_name": not_collected, + "server_version": not_collected, + "tool_names": not_collected, + "tool_names_truncated": {"reason": "entry_cap", "dropped": 2} + }}, + "broken": "not an object", + long_name: {"runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!( + map_keys(servers), + ["broken", "events", "files", "github", "plugin-docs"] + ); + let harness_entry = &servers["plugin-docs"]; + assert_eq!(harness_entry["origin"], "harness"); + assert_eq!(harness_entry["transport"], "stdio"); + assert_eq!( + harness_entry["endpoint"], + json!({"state": "not_applicable", "reason": "not_configured", + "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + ); + assert_eq!(harness_entry["runtime"]["server_name"], not_collected); + // A marker beside a list without a value is dropped. + assert!(harness_entry["runtime"]["tool_names_truncated"].is_null()); + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(servers["broken"]["runtime"]["connected"], invalid); + assert!(servers["broken"].get("transport").is_none()); + assert_eq!( + servers["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + } + + #[test] + fn a_malformed_helper_marker_invalidates_tool_names() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": {"github": {"runtime": { + "tool_names": reported(json!(["a"]), "harness_status"), + "tool_names_truncated": {"reason": "entry_cap", "dropped": 0} + }}}}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let runtime = &what_ran["mcp"]["servers"]["github"]["runtime"]; + assert_eq!( + runtime["tool_names"], + report_state("collection_failed", "invalid_report") + ); + assert!(runtime["tool_names_truncated"].is_null()); + } + + #[test] + fn a_plan_server_dropped_by_the_entry_cap_never_returns_as_a_harness_entry() { + let names = numbered_keys("server", 33); + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let template = plan.capability_plan.mcp_servers["events"].clone(); + plan.capability_plan.mcp_servers = names + .iter() + .map(|name| (name.clone(), template.clone())) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"runtime": { + "connected": reported(json!(true), "harness_status")}}), + ) + }) + .collect(); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: {"servers": servers}})); + + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + assert_eq!(servers.len(), 32); + assert!(servers.values().all(|server| server["origin"] == "plan")); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } } diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 008750fb9..867799101 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -524,7 +524,11 @@ }, "tool_names": { "$ref": "#/$defs/Fact_for_Array_of_string", - "description": "Raw tool names the server lists, each bounded by the string length\nrule." + "description": "Raw tool names the server lists, each bounded by the string length\nrule. At most 64 names, in sorted order." + }, + "tool_names_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Tool names dropped from `tool_names`, by the adapter's helper or by\ncore. Absent when none was dropped, never `null`." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index ebaf128e9..3e40fc608 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2040,10 +2040,12 @@ def _is_mcp_server_entry(server: Any) -> bool: "server_version": _is_text, "tool_names": _is_text_list, } + runtime = server.get("runtime") return ( isinstance(origin, str) and transport_ok - and _has_facts(server.get("runtime"), runtime_facts) + and _has_facts(runtime, runtime_facts) + and _has_valid_marker(runtime, "tool_names_truncated") ) diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json index 8866d76fe..e3d931c17 100644 --- a/tests/fixtures/what_ran/reader-vectors.json +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -447,6 +447,24 @@ } } ] + }, + { + "name": "tool names marker inside an MCP runtime block", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + } + ] } ], "invalid": [ @@ -1018,6 +1036,39 @@ "value": null } ] + }, + { + "name": "tool names marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 0 + } + } + ] + }, + { + "name": "null tool_names_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": null + } + ] } ] } From 4a7425b971779c921977c9aaf1a2a0e167182329 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:52:14 -0700 Subject: [PATCH 14/74] feat: report adapter and harness versions from six adapters A what_ran could not say which harness build ran. Six adapters now declare nemo_fabric.versions.v1 and report their package version and the harness name and version from a source that names the build that ran: - Claude Code: claude_code_version from the CLI's init message. - Codex: the thread's cli_version from one bounded thread/read request. - Hermes: get_version_info(), a blocking call that runs git, on a daemon thread with a 5 second limit. - OpenClaw: the --version check, or an attached gateway's /startupz version, both already read at startup. - Deep Agents and mini-swe-agent: package metadata, accepted only when the distribution installed the module the adapter loaded. A shared helper builds the report, applies core's string length rule, and runs blocking calls with at most one outstanding probe per runtime; stopping a runtime never waits for it. A source that times out, fails or answers nothing reads a state with a reason, never a substitute value. The adapter catalog is regenerated for the six changed descriptors. Signed-off-by: mnajafian-nv --- .../python/claude/claude.fabric-adapter.json | 8 + .../nemo_fabric_adapters/claude/adapter.py | 89 ++++++-- .../python/codex/codex.fabric-adapter.json | 8 + .../src/nemo_fabric_adapters/codex/adapter.py | 63 ++++- .../nemo_fabric_adapters/common/provenance.py | 215 ++++++++++++++++++ .../deepagents/deepagents.fabric-adapter.json | 10 + .../deepagents/adapter.py | 27 +++ .../python/hermes/hermes.fabric-adapter.json | 8 + .../nemo_fabric_adapters/hermes/adapter.py | 68 ++++++ .../mini-swe-agent.fabric-adapter.json | 10 + .../mini_swe_agent/adapter.py | 26 +++ .../openclaw/openclaw.fabric-adapter.json | 10 + .../nemo_fabric_adapters/openclaw/adapter.py | 34 ++- .../claude/claude.fabric-adapter.json | 8 + .../adapters/codex/codex.fabric-adapter.json | 8 + .../deepagents/deepagents.fabric-adapter.json | 10 + .../hermes/hermes.fabric-adapter.json | 8 + crates/fabric-core/src/config.rs | 4 + crates/fabric-core/src/what_ran.rs | 4 +- .../nemo_fabric_adapter_catalog/catalog.json | 78 ++++++- .../test_adapters_common_provenance.py | 168 ++++++++++++++ tests/adapters/test_claude_adapter.py | 104 +++++++++ tests/adapters/test_codex_adapter.py | 62 +++++ tests/adapters/test_hermes_adapter.py | 155 +++++++++++++ tests/adapters/test_openclaw.py | 21 ++ tests/e2e/test_deepagents.py | 20 ++ tests/e2e/test_mini_swe_agent.py | 27 +++ tests/fixtures/claude_streams/api-error.jsonl | 3 + tests/fixtures/claude_streams/normal.jsonl | 3 + 29 files changed, 1231 insertions(+), 28 deletions(-) create mode 100644 adapters/python/common/src/nemo_fabric_adapters/common/provenance.py create mode 100644 tests/adapters/test_adapters_common_provenance.py create mode 100644 tests/fixtures/claude_streams/api-error.jsonl create mode 100644 tests/fixtures/claude_streams/normal.jsonl diff --git a/adapters/python/claude/claude.fabric-adapter.json b/adapters/python/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/adapters/python/claude/claude.fabric-adapter.json +++ b/adapters/python/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 458a58cde..ec9238886 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -30,6 +30,7 @@ from claude_agent_sdk import Message from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage +from claude_agent_sdk import SystemMessage from claude_agent_sdk import HookMatcher from claude_agent_sdk._errors import MessageParseError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -43,6 +44,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import relay_artifacts from nemo_fabric_adapters.common import relay_gateway from nemo_fabric_adapters.common import relay_hooks @@ -209,16 +211,21 @@ def _settings(config: AgentConfig) -> dict[str, Any]: return config.harness.settings if config.harness is not None else {} +def _selected_model_role(config: AgentConfig) -> str: + """The model role Claude Code runs: ``default``, or the sole configured role.""" + + if "default" in config.models: + return "default" + if len(config.models) == 1: + return next(iter(config.models)) + raise AdapterConfigError( + "claude_invalid_configuration", + "Claude requires a default model or exactly one model", + ) + + def _selected_model_config(config: AgentConfig) -> AgentModelConfig: - model = config.models.get("default") - if model is None and len(config.models) == 1: - model = next(iter(config.models.values())) - if model is None: - raise AdapterConfigError( - "claude_invalid_configuration", - "Claude requires a default model or exactly one model", - ) - return model + return config.models[_selected_model_role(config)] def _resolve_path(base_dir: str, value: str | Path) -> Path: @@ -825,6 +832,27 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-claude" +_HARNESS_NAME = "claude-code" +_METHOD_INIT_MESSAGE = "harness_init_message" + + +def init_message_version(messages: list[Message]) -> dict[str, Any] | None: + """The Claude Code version the latest init message names, as a fact. + + ``None`` when no init message arrived. An init message without a version + reads ``unavailable`` with reason ``empty_response``. + """ + + for message in reversed(messages): + if isinstance(message, SystemMessage) and message.subtype == "init": + version = message.data.get("claude_code_version") + if isinstance(version, str) and version: + return provenance.known_fact(version, _METHOD_INIT_MESSAGE) + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_INIT_MESSAGE + ) + return None # Claude Code's result usage keys and the detailed input counters they fill. # Anthropic's input_tokens excludes cache reads and cache writes, and its # output_tokens includes thinking, which Anthropic bills as output. @@ -871,7 +899,9 @@ def _detailed_token_usage(usage: dict[str, Any]) -> dict[str, int | bool]: return counters -def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: +def _agent_run_result( + output: dict[str, Any], versions: dict[str, Any] | None = None +) -> AgentRunResult: normalized = dict(output) failed = bool(normalized.pop("failed", False)) reported_error = normalized.pop("error", None) @@ -914,6 +944,7 @@ def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: output=normalized, error=error, usage=agent_usage, + extensions={provenance.VERSIONS_REPORT_KEY: versions} if versions else {}, ) @@ -1068,6 +1099,10 @@ def __init__(self) -> None: self._gateway_process: subprocess.Popen[Any] | None = None self._mcp_config_path: Path | None = None self._unusable = False + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_INIT_MESSAGE + ) async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1130,12 +1165,14 @@ async def invoke( "claude_runtime_mismatch", "Claude invocation does not match the connected runtime", ) + messages: list[Message] = [] if self._unusable: return _agent_run_result( _failure( "claude_runtime_unavailable", "Claude runtime cannot accept another invocation after a runtime failure", - ) + ), + self._versions_report(agent_config, messages), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1157,6 +1194,7 @@ async def invoke( client, prompt, _remaining_timeout(invocation_deadline), + messages, ) if ( output.get("completed") @@ -1183,22 +1221,43 @@ async def invoke( ), relay, artifacts=[], - ) + ), + self._versions_report(agent_config, messages), ) if self._relay is not None: output = _relay_output(output, self._relay) - return _agent_run_result(output) + return _agent_run_result(output, self._versions_report(agent_config, messages)) + + def _versions_report( + self, config: AgentConfig, messages: list[Message] + ) -> dict[str, Any]: + """Report this runtime's versions, refreshed from any init message.""" + + version = init_message_version(messages) + if version is not None: + self._harness_version = version + return provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_INIT_MESSAGE), + harness_version=self._harness_version, + served_models={role: provenance.no_source_fact() for role in config.models}, + ) async def _run_query( self, client: ClaudeSDKClient, prompt: str, remaining_timeout: float, + messages: list[Message] | None = None, ) -> dict[str, Any]: - """Run one SDK query and normalize its terminal result.""" + """Run one SDK query and normalize its terminal result. - messages: list[Message] = [] + Every message other than the terminal result is appended to + ``messages``, so the caller can read this invocation's messages. + """ + + messages = [] if messages is None else messages result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: diff --git a/adapters/python/codex/codex.fabric-adapter.json b/adapters/python/codex/codex.fabric-adapter.json index 03819ee12..3879c64f8 100644 --- a/adapters/python/codex/codex.fabric-adapter.json +++ b/adapters/python/codex/codex.fabric-adapter.json @@ -94,6 +94,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index e463332b7..be0143a43 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -59,9 +59,13 @@ import nemo_fabric_adapters.common.relay_artifacts as relay_artifacts import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance INTERRUPT_TIMEOUT_SECONDS = 5.0 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-codex" +_HARNESS_NAME = "codex" +_METHOD_HARNESS_API = "harness_api" CLOCK_RESOLUTION_SECONDS = time.get_clock_info("monotonic").resolution SANDBOXES = { "read-only": Sandbox.read_only, @@ -1078,8 +1082,48 @@ def _normalize_usage( return AgentUsage(**tokens, input_tokens_include_cache=True) +async def read_cli_version( + thread: Any, timeout: float = provenance.PROBE_TIMEOUT_SECONDS +) -> dict[str, Any]: + """The version of the Codex CLI that created ``thread``, from its thread data. + + One bounded ``thread/read`` request: a timeout reads ``collection_failed`` + with reason ``timeout``, and a failed request reads reason ``no_result``. + """ + + try: + async with asyncio.timeout(timeout): + response = await thread.read() + except TimeoutError: + return provenance.missing_fact("collection_failed", "timeout", _METHOD_HARNESS_API) + except (CodexError, RuntimeError, OSError, ValueError): + # ValueError covers a response the SDK's models reject. + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + version = getattr(getattr(response, "thread", None), "cli_version", None) + if not isinstance(version, str) or not version: + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + +def versions_report(config: AgentConfig, harness_version: dict[str, Any]) -> dict[str, Any]: + """Codex's versions report. Codex knows only the configured model.""" + + return provenance.versions_report( + package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), + harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_HARNESS_API), + harness_version=harness_version, + served_models={role: provenance.no_source_fact() for role in config.models}, + ) + + def _agent_run_result( - output: dict[str, Any], cumulative_totals: dict[str, int] | None = None + output: dict[str, Any], + cumulative_totals: dict[str, int] | None = None, + versions: dict[str, Any] | None = None, ) -> AgentRunResult: normalized = dict(output) failed = bool(normalized.pop("failed", False)) @@ -1105,6 +1149,7 @@ def _agent_run_result( output=normalized, error=error, usage=agent_usage, + extensions={provenance.VERSIONS_REPORT_KEY: versions} if versions else {}, ) @@ -1338,6 +1383,9 @@ def __init__(self) -> None: self._mcp_authentication_checked = False self._unusable = False self._usage_totals: dict[str, int] = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1401,6 +1449,7 @@ async def start(self, payload: dict[str, Any]) -> None: base_dir, relay=relay, ) + harness_version = await read_cli_version(thread) except CodexAdapterError as error: await self._cleanup_failed_start() raise _as_lifecycle_error(error) from error @@ -1423,6 +1472,7 @@ async def start(self, payload: dict[str, Any]) -> None: self._fabric_runtime_id = fabric_runtime_id self._thread = thread self._usage_totals = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = harness_version async def invoke( self, @@ -1454,7 +1504,8 @@ async def invoke( _failure( "codex_runtime_unavailable", "Codex runtime cannot accept another invocation after a runtime failure", - ) + ), + versions=versions_report(config, self._harness_version), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1508,7 +1559,9 @@ async def invoke( ) failure["usage"] = output.get("usage") return _agent_run_result( - _relay_output(failure, relay, artifacts=[]), self._usage_totals + _relay_output(failure, relay, artifacts=[]), + self._usage_totals, + versions_report(config, self._harness_version), ) except AdapterRelayError as error: output = adapter_failure(error) @@ -1520,7 +1573,9 @@ async def invoke( self._unusable = not usable if self._relay is not None: output = _relay_output(output, self._relay) - return _agent_run_result(output, self._usage_totals) + return _agent_run_result( + output, self._usage_totals, versions_report(config, self._harness_version) + ) async def stop(self) -> None: client = self._client diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py new file mode 100644 index 000000000..43d41933d --- /dev/null +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the ``nemo_fabric.versions.v1`` report an adapter returns with each result. + +NeMo Fabric core reads the report from ``AgentRunResult.extensions`` when the +adapter's descriptor names the key under its ``run_result`` extension schema, +and moves its facts into ``RunResult.what_ran``. Every fact is reported by the +adapter, so each carries ``source`` ``reported`` and ``collected_by`` +``adapter``. A fact holds ``value``, or ``state`` and ``reason``, never both. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.metadata +import sys +import threading +from collections.abc import Callable +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from typing import TypeVar + +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +PROBE_TIMEOUT_SECONDS = 5.0 + +METHOD_ADAPTER_REPORT = "adapter_report" +METHOD_PACKAGE_METADATA = "package_metadata" + +_MAX_VALUE_CHARS = 256 +_TRUNCATED_PREFIX_CHARS = 240 +_TRUNCATED_DIGEST_HEX_DIGITS = 15 +_MAX_MODEL_ROLES = 16 +_MAX_KEY_CHARS = 128 + +T = TypeVar("T") + + +def known_fact(value: str, method: str) -> dict[str, Any]: + """A reported fact with a value, bounded by the string length rule. + + A lone surrogate, which JSON text from a harness can carry, becomes ``?`` + first, because core rejects a result that holds one. + """ + + sanitized = value.encode("utf-8", "replace").decode("utf-8") + return { + "value": bounded_text(sanitized), + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing_fact( + state: str, reason: str, method: str = METHOD_ADAPTER_REPORT +) -> dict[str, Any]: + """A reported fact without a value.""" + + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def no_source_fact() -> dict[str, Any]: + """A fact this adapter has no source for.""" + + return missing_fact("unsupported", "no_source") + + +def bounded_text(text: str) -> str: + """Apply NeMo Fabric core's string length rule. + + A string of at most 256 characters is kept whole. A longer one keeps its + first 240 characters, then ``~``, then the first 15 hex digits of the + SHA-256 of its UTF-8 bytes, as core does. + """ + + if len(text) <= _MAX_VALUE_CHARS: + return text + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + return ( + f"{text[:_TRUNCATED_PREFIX_CHARS]}~{digest[:_TRUNCATED_DIGEST_HEX_DIGITS]}" + ) + + +def package_version_fact(distribution: str) -> dict[str, Any]: + """The installed version of ``distribution``, read from its package metadata.""" + + try: + version = importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + if not version: + return missing_fact("unavailable", "empty_response", METHOD_PACKAGE_METADATA) + return known_fact(version, METHOD_PACKAGE_METADATA) + + +def loaded_module_version_fact(distribution: str, module_name: str) -> dict[str, Any]: + """The version of ``distribution`` when it installed the loaded ``module_name``. + + Another copy of the module can shadow the installed package, for example a + source checkout earlier on ``sys.path``. The version is reported only when + the distribution's file list holds the file the module was loaded from; + otherwise it reads ``unavailable`` with reason ``not_attributed``. A module + that is not loaded reads ``not_collected``. + """ + + module = sys.modules.get(module_name) + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + try: + installed = importlib.metadata.distribution(distribution) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + loaded = Path(module_file).resolve() + if not any( + Path(str(installed.locate_file(file))).resolve() == loaded + for file in installed.files or () + if file.name == loaded.name + ): + return missing_fact("unavailable", "not_attributed", METHOD_PACKAGE_METADATA) + version = installed.version + if not version: + return missing_fact("unavailable", "empty_response", METHOD_PACKAGE_METADATA) + return known_fact(version, METHOD_PACKAGE_METADATA) + + +def versions_report( + *, + package_version: Mapping[str, Any], + harness_name: Mapping[str, Any], + harness_version: Mapping[str, Any], + served_models: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """The ``nemo_fabric.versions.v1`` report object. + + ``served_models`` maps each plan role to its served-model fact. Like core, + the report keeps at most 16 roles in sorted order and skips a role longer + than 128 characters, so the object stays far below core's 64 KiB limit + for one report key. + """ + + roles = sorted(role for role in served_models if len(role) <= _MAX_KEY_CHARS) + return { + "adapter": {"package_version": dict(package_version)}, + "harness": {"name": dict(harness_name), "version": dict(harness_version)}, + "models": { + role: {"served": dict(served_models[role])} + for role in roles[:_MAX_MODEL_ROLES] + }, + } + + +class BlockingProbe: + """Run a runtime's blocking version calls off the event loop, one at a time. + + A blocking call cannot be cancelled, so it runs on a daemon thread and the + caller waits at most ``timeout`` seconds for it. While an earlier call + still runs, no new thread starts and the call times out at once, so a + runtime never has more than one outstanding probe. Stopping the runtime + never waits for the thread, and a daemon thread never delays process exit. + """ + + def __init__(self) -> None: + self._running = threading.Event() + + async def call( + self, function: Callable[[], T], *, timeout: float = PROBE_TIMEOUT_SECONDS + ) -> T: + """Return ``function()``, or raise ``TimeoutError`` past ``timeout``. + + An exception that ``function`` raises reaches the caller. + """ + + if self._running.is_set(): + raise TimeoutError("an earlier version probe is still running") + loop = asyncio.get_running_loop() + outcome: asyncio.Future[T] = loop.create_future() + + def deliver(result: T | None, error: Exception | None) -> None: + if outcome.done(): + return + if error is not None: + outcome.set_exception(error) + else: + outcome.set_result(result) # type: ignore[arg-type] + + def run() -> None: + result: T | None = None + error: Exception | None = None + try: + result = function() + except Exception as raised: + error = raised + finally: + self._running.clear() + try: + loop.call_soon_threadsafe(deliver, result, error) + except RuntimeError: + # The event loop closed while the probe ran; nobody waits for it. + return + + self._running.set() + threading.Thread(target=run, name="nemo-fabric-version-probe", daemon=True).start() + # On timeout, wait_for cancels only this future. The thread keeps + # running, and its late outcome is dropped. + return await asyncio.wait_for(outcome, timeout) diff --git a/adapters/python/deepagents/deepagents.fabric-adapter.json b/adapters/python/deepagents/deepagents.fabric-adapter.json index 12eb69898..796fbbfb2 100644 --- a/adapters/python/deepagents/deepagents.fabric-adapter.json +++ b/adapters/python/deepagents/deepagents.fabric-adapter.json @@ -195,6 +195,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..721e31221 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -36,6 +36,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils HARNESS = "deepagents" @@ -587,6 +588,11 @@ def _gated_subagents( return gated +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-deepagents" +_HARNESS_DISTRIBUTION = "deepagents" +_HARNESS_MODULE = "deepagents" + + class DeepAgentsRuntime: """One compiled Deep Agents graph and checkpointer owned by a Fabric runtime.""" @@ -609,6 +615,9 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -669,6 +678,9 @@ async def start(self, payload: dict[str, Any]) -> None: from deepagents import create_deep_agent + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) agent = create_deep_agent( **_supported_kwargs(create_deep_agent, agent_kwargs) ) @@ -799,6 +811,21 @@ async def invoke( if tokens or cost is not None else None ), + extensions={ + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact( + _PACKAGE_DISTRIBUTION + ), + harness_name=provenance.known_fact( + _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in start_payload["config"].models + }, + ) + }, ) async def _invoke_with_telemetry( diff --git a/adapters/python/hermes/hermes.fabric-adapter.json b/adapters/python/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/adapters/python/hermes/hermes.fabric-adapter.json +++ b/adapters/python/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 653304993..0f8fe2e18 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -15,6 +15,7 @@ import logging import os import sys +from collections.abc import Callable from contextlib import redirect_stdout from io import StringIO from pathlib import Path @@ -30,6 +31,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.hermes import configuration from nemo_fabric_adapters.hermes import telemetry @@ -37,9 +39,51 @@ # The actual internal hermes default as of v0.21 is `sys.maxsize` DEFAULT_MAX_ITERATIONS: int = 90 LOGGER = logging.getLogger(__name__) +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-hermes" +_HARNESS_NAME = "hermes-agent" +_METHOD_HARNESS_API = "harness_api" hermes_mcp_server_config = configuration.hermes_mcp_server_config +def hermes_version() -> str: + """The running Hermes Agent's version from its version call. + + ``get_version_info()`` can run several ``git`` commands, so callers run it + through a :class:`provenance.BlockingProbe`. + """ + + from hermes_cli.version_info import get_version_info + + return get_version_info().display_version + + +async def probe_hermes_version( + probe: provenance.BlockingProbe, + version_call: Callable[[], str] = hermes_version, + timeout: float = provenance.PROBE_TIMEOUT_SECONDS, +) -> dict[str, Any]: + """Hermes's version as a reported fact, bounded by the probe time limit.""" + + try: + version = await probe.call(version_call, timeout=timeout) + except TimeoutError: + return provenance.missing_fact("collection_failed", "timeout", _METHOD_HARNESS_API) + except ImportError: + # A Hermes Agent older than its version call. + return provenance.missing_fact("unsupported", "no_source", _METHOD_HARNESS_API) + except Exception: + # Provenance never fails a run, whatever Hermes's version call raises. + LOGGER.exception("Hermes version call failed") + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + if not isinstance(version, str) or not version or version == "unknown": + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + def main() -> None: """Serve the persistent local-host lifecycle protocol.""" @@ -66,6 +110,10 @@ def __init__(self) -> None: self._relay_plugin_config: dict[str, Any] | None = None self._relay_plugin_config_path: Path | None = None self._active_invoke_task: asyncio.Task[tuple[dict[str, Any], str]] | None = None + self._version_probe = provenance.BlockingProbe() + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -151,6 +199,7 @@ async def start(self, payload: dict[str, Any]) -> None: from hermes_state import SessionDB from run_agent import AIAgent + self._harness_version = await probe_hermes_version(self._version_probe) with redirect_stdout(StringIO()): if relay_enabled: common_utils.reject_ambient_relay_plugin_config() @@ -368,10 +417,29 @@ def clear_active_invoke_task( code="hermes_reported_failure", message="Hermes did not complete the invocation", ) + if self._harness_version.get("reason") == "timeout": + # Pick up a version call that finished after its limit. While it + # still runs, the probe starts no new one. + self._harness_version = await probe_hermes_version(self._version_probe) return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, error=error, + extensions={ + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact( + _PACKAGE_DISTRIBUTION + ), + harness_name=provenance.known_fact( + _HARNESS_NAME, _METHOD_HARNESS_API + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in agent_config.models + }, + ) + }, ) async def _authenticate_mcp_servers(self) -> None: diff --git a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json index 70a6e0929..3b38b405d 100644 --- a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json +++ b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json @@ -39,6 +39,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py index 15df4f5c4..148e924d0 100644 --- a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py +++ b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py @@ -12,6 +12,7 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils SYSTEM_TEMPLATE = "{{system_instruction}}" @@ -25,6 +26,9 @@ your final response, so include it in the same command. """ _UNREADABLE = object() +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-mini-swe-agent" +_HARNESS_DISTRIBUTION = "mini-swe-agent" +_HARNESS_MODULE = "minisweagent" def _selected_model(config: contract.AgentConfig) -> contract.AgentModelConfig: @@ -46,6 +50,10 @@ def __init__(self) -> None: self._relay_plugin = self._relay_scope = self._relay_scope_type = None self._relay_plugin_config: dict[str, Any] | None = None self._telemetry_quarantine: str | None = None + self._model_roles: list[str] = [] + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] @@ -61,6 +69,10 @@ async def start(self, payload: dict[str, Any]) -> None: ) from nemo_fabric_adapters.mini_swe_agent.agents import RetainingDefaultAgent + self._model_roles = list(config.models) + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) runtime_context = contract.RuntimeContext.from_mapping( payload.get("runtime_context") ) @@ -179,6 +191,20 @@ async def invoke( ), output=output, error=error, + extensions={ + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact( + _PACKAGE_DISTRIBUTION + ), + harness_name=provenance.known_fact( + _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() for role in self._model_roles + }, + ) + }, ) async def _run_agent(self, task: str) -> dict[str, Any]: diff --git a/adapters/python/openclaw/openclaw.fabric-adapter.json b/adapters/python/openclaw/openclaw.fabric-adapter.json index 38ec42f4b..9c4d4e844 100644 --- a/adapters/python/openclaw/openclaw.fabric-adapter.json +++ b/adapters/python/openclaw/openclaw.fabric-adapter.json @@ -112,6 +112,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index 75d99c9fa..719bf2abb 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -32,6 +32,7 @@ from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle from nemo_fabric_adapters.common import openai_chat +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils from nemo_fabric_adapters.openclaw import _windows_job @@ -50,6 +51,12 @@ MAX_PORT_ATTEMPTS = 20 MAX_DERIVED_PORT_OFFSET = 110 MAX_TCP_PORT = 65_535 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openclaw" +_HARNESS_NAME = "openclaw" +# The local gateway's version comes from `openclaw --version`; an attached +# gateway's from its own startup endpoint. +_METHOD_VERSION_PROBE = "harness_probe" +_METHOD_GATEWAY_API = "harness_api" logger = logging.getLogger(__name__) @@ -759,6 +766,9 @@ def __init__(self) -> None: self._previous_signal_handlers: dict[int, Any] = {} self._signal_handler: Any = None self._windows_job: int | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_VERSION_PROBE + ) async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: config: contract.AgentConfig = payload["config"] @@ -806,6 +816,7 @@ async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: command, "--version", env=child_env, timeout=startup_timeout ) version = _validate_openclaw_version(version_output) + self._harness_version = provenance.known_fact(version, _METHOD_VERSION_PROBE) port = _select_port(settings) @@ -1029,7 +1040,7 @@ async def _connect_service_runtime( ) client = self._client_for(settings, token) try: - await self._probe_gateway(client, gateway_url, agent_id) + version = await self._probe_gateway(client, gateway_url, agent_id) except BaseException: await client.aclose() raise @@ -1038,6 +1049,7 @@ async def _connect_service_runtime( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact(version, _METHOD_GATEWAY_API) async def _attach_service( self, @@ -1120,6 +1132,9 @@ async def _attach_service( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact( + openclaw_version, _METHOD_GATEWAY_API + ) return self._service_info( gateway_url, agent_id, @@ -1370,6 +1385,7 @@ async def invoke( retryable=error.response.status_code >= 500 or error.response.status_code == 429, ), + extensions=self._versions_extensions(), ) except httpx.RequestError as error: raise lifecycle.LifecycleError( @@ -1386,8 +1402,24 @@ async def invoke( status=contract.AgentRunStatus.SUCCEEDED, output={"response": text}, usage=usage, + extensions=self._versions_extensions(), ) + def _versions_extensions(self) -> dict[str, Any]: + """The versions report, with the gateway version read at startup.""" + + models = self._config.models if self._config is not None else {} + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), + harness_name=provenance.known_fact( + _HARNESS_NAME, self._harness_version["method"] + ), + harness_version=self._harness_version, + served_models={role: provenance.no_source_fact() for role in models}, + ) + } + async def stop(self) -> None: cancellation: asyncio.CancelledError | None = None self._stopping = True diff --git a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json index 03819ee12..3879c64f8 100644 --- a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json @@ -94,6 +94,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json index 12eb69898..796fbbfb2 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json @@ -195,6 +195,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 667fe951a..a452c7f94 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -6865,6 +6865,10 @@ mod tests { fn invocation_extensions_are_fail_closed_and_schema_validated() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + // Start from a descriptor that declares no result extension schema. + descriptor + .extension_schemas + .remove(&AdapterExtensionPoint::RunResult); let request = AgentRunRequest { input: serde_json::json!("review"), relay_session_root: None, diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index f9be07a25..a10a21695 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -2059,7 +2059,7 @@ mod tests { #[test] fn an_undeclared_report_key_stays_in_the_extensions_and_reads_key_not_declared() { - let plan = plan_with_secrets(); + let plan = plan_declaring(&[]); let extensions = json!({VERSIONS_REPORT_KEY: versions_report()}); let (what_ran, kept) = ingest(&plan, extensions.clone()); @@ -2077,7 +2077,7 @@ mod tests { #[test] fn a_report_key_declared_only_through_a_reference_is_not_declared() { - let mut plan = plan_with_secrets(); + let mut plan = plan_declaring(&[]); let schema = json!({"type": "object", "allOf": [{"properties": { VERSIONS_REPORT_KEY: {"type": "object"}}}]}); plan.adapter_descriptor diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index 4c15e1112..14d79212e 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -42,6 +42,16 @@ }, "type": "object" }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -196,7 +206,7 @@ "name": "nemo-fabric-adapters-claude", "version": "0.6.0" }, - "sha256": "cded65f1e927f3b7154f79b3eb6312a5fa34551265ffe6df1ec271e55de9c407", + "sha256": "027fa0b3d396e93dce27d5848320165c27a52c909955f774f2a4372728d549d0", "source": "adapters/python/claude/claude.fabric-adapter.json" }, "nvidia.fabric.cline": { @@ -377,6 +387,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "model_schema": { @@ -530,7 +550,7 @@ "name": "nemo-fabric-adapters-codex", "version": "0.6.0" }, - "sha256": "f66fb66d3a366977619bc9e88a4d926e0b71bbc28ac965b9a416bf2559106ce5", + "sha256": "019e21098570ba1a5725e7b6c2efe5f34b84e688cd38ce432b0562e444672cbd", "source": "adapters/python/codex/codex.fabric-adapter.json" }, "nvidia.fabric.droid": { @@ -697,6 +717,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "requirements": {}, @@ -781,7 +811,7 @@ "name": "nemo-fabric-adapters-hermes", "version": "0.6.0" }, - "sha256": "e482a89dda834dc5bfdbcb7f6041ad5e457a73918d74d8a8215f0171b8947465", + "sha256": "75969f9ab81e0211c0248cec9d3193ef90dcf075af38265819f9d84adcb7f0c0", "source": "adapters/python/hermes/hermes.fabric-adapter.json" }, "nvidia.fabric.kilo": { @@ -907,6 +937,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1164,7 +1206,7 @@ "name": "nemo-fabric-adapters-deepagents", "version": "0.6.0" }, - "sha256": "819b1f9747af83011c98d5889c56dac9a735008b25e93bbc94c1cfde3050a44e", + "sha256": "1fb843947fc2383a7ef92946f0ba889b22557f4f7427e50bb17345f5f6ed206c", "source": "adapters/python/deepagents/deepagents.fabric-adapter.json" }, "nvidia.fabric.mini-swe-agent": { @@ -1192,6 +1234,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1270,7 +1324,7 @@ "name": "nemo-fabric-adapters-mini-swe-agent", "version": "0.6.0" }, - "sha256": "40ee9364117fd6d4d1fe4e9a822ecd7b9022a5f0f8053d8b550d065e3726df37", + "sha256": "e78d5665059801baafa9012fdb252c0ab110f065a47089996e14e280e385be77", "source": "adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json" }, "nvidia.fabric.nooa": { @@ -1490,6 +1544,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1646,7 +1712,7 @@ "name": "nemo-fabric-adapters-openclaw", "version": "0.6.0" }, - "sha256": "5ab8b29595214e214214b051b60bc314dc65f07add60a73283f434a62e3f7402", + "sha256": "eef2a0677ed569f2433a47886fd86bf05febb7e5ffa2d67b08b2ab2f14f206e0", "source": "adapters/python/openclaw/openclaw.fabric-adapter.json" }, "nvidia.fabric.opencode": { diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py new file mode 100644 index 000000000..2b3627f09 --- /dev/null +++ b/tests/adapters/test_adapters_common_provenance.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import json +import sys +import threading +import time +import types +from pathlib import Path + +import deepagents +import minisweagent +import pytest +from nemo_fabric_adapters.common import provenance + + +def known(value: str, method: str = "package_metadata") -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing(state: str, reason: str, method: str = "package_metadata") -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +# The same vectors as core's string length test in `what_ran.rs`. +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("é" * 256, "é" * 256), + ("m" * 300 + "a", "m" * 240 + "~96858eedeb3c8e6"), + ("é" * 300, "é" * 240 + "~7250b66610f8b7d"), + ], +) +def test_bounded_text_matches_core_string_length_rule(text, expected): + assert provenance.bounded_text(text) == expected + + +def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): + fact = provenance.known_fact("claude-\ud800", "harness_api") + + assert fact["value"] == "claude-?" + json.dumps(fact).encode("utf-8") + + +def test_package_version_fact_reads_installed_metadata(): + assert provenance.package_version_fact("nemo-fabric-adapters-common") == known( + "0.6.0" + ) + assert provenance.package_version_fact("nemo-fabric-no-such-package") == missing( + "unavailable", "not_collected" + ) + + +@pytest.mark.parametrize( + ("distribution", "module", "expected"), + [ + ("deepagents", deepagents.__name__, known("0.7.9")), + ("mini-swe-agent", minisweagent.__name__, known("2.4.6")), + # The distribution exists but did not install the loaded module. + ("mini-swe-agent", deepagents.__name__, missing("unavailable", "not_attributed")), + ("deepagents", "nemo_fabric_not_loaded", missing("unavailable", "not_collected")), + ], +) +def test_loaded_module_version_fact_accepts_only_the_module_the_package_installed( + distribution, module, expected +): + assert provenance.loaded_module_version_fact(distribution, module) == expected + + +def test_loaded_module_version_fact_rejects_a_shadowing_source_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + shadow = types.ModuleType("deepagents") + shadow.__file__ = str(tmp_path / "deepagents" / "__init__.py") + monkeypatch.setitem(sys.modules, "deepagents", shadow) + + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == missing( + "unavailable", "not_attributed" + ) + + +def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): + served = {f"role-{index:02}": provenance.no_source_fact() for index in range(17)} + served["r" * 129] = provenance.no_source_fact() + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.known_fact("harness", "harness_api"), + harness_version=provenance.known_fact("v" * 300, "harness_api"), + served_models=served, + ) + + assert list(report["models"]) == [f"role-{index:02}" for index in range(16)] + assert report["harness"]["version"]["value"] == provenance.bounded_text("v" * 300) + assert report["adapter"]["package_version"] == known("0.6.0") + + +def probe_threads() -> int: + return sum( + thread.name == "nemo-fabric-version-probe" for thread in threading.enumerate() + ) + + +async def test_blocking_probe_bounds_a_function_that_never_returns(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "1.0.0" + + probe = provenance.BlockingProbe() + ticks = 0 + + async def count_ticks() -> None: + nonlocal ticks + while True: + ticks += 1 + await asyncio.sleep(0.01) + + ticker = asyncio.create_task(count_ticks()) + started = time.monotonic() + try: + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=0.2) + elapsed = time.monotonic() - started + # A second call while the first still runs starts no new thread. + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=5) + assert time.monotonic() - started < 1 + assert calls == ["called"] + assert probe_threads() == 1 + finally: + ticker.cancel() + release.set() + + assert 0.2 <= elapsed < 1 + # The event loop kept running while the function blocked its thread. + assert ticks >= 5 + while probe_threads(): + await asyncio.sleep(0.01) + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + +async def test_blocking_probe_returns_the_result_or_the_raised_error(): + probe = provenance.BlockingProbe() + + def fail() -> str: + raise RuntimeError("no version") + + assert await probe.call(lambda: "1.2.3") == "1.2.3" + with pytest.raises(RuntimeError, match="no version"): + await probe.call(fail) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index e7b17060f..f5a176b47 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import json import os import sys @@ -31,6 +32,7 @@ from claude_agent_sdk import SystemMessage from claude_agent_sdk import TextBlock from claude_agent_sdk._errors import MessageParseError +from claude_agent_sdk._internal.message_parser import parse_message from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult @@ -53,6 +55,8 @@ } DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +CLAUDE_STREAMS = ROOT / "tests" / "fixtures" / "claude_streams" def lifecycle_invocation( @@ -258,6 +262,12 @@ def test_claude_descriptor_is_narrow_and_versioned(): }, "additionalProperties": False, }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + }, }, "config": { "accepts": [ @@ -2123,3 +2133,97 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.ClaudeRuntime, config_loader=AgentConfig.from_mapping ) + + +def stream_messages(name: str) -> list[Message]: + """Parse a saved Claude Code stream with the SDK's own message parser.""" + + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [parse_message(json.loads(line)) for line in lines] + + +async def versions_reports( + claude_payload: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + streams: list[list[Message]], +) -> list[dict[str, Any]]: + """Run one invocation per stream and return each result's versions report.""" + + install_scripted_client(monkeypatch, streams) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + reports = [] + for _ in streams: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + reports.append(result.extensions[VERSIONS_REPORT_KEY]) + await runtime.stop() + return reports + + +def reported(value: str, method: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +async def test_claude_reports_the_cli_version_from_its_init_message( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [stream_messages("normal.jsonl"), [running_total_result(0.1)]], + ) + + package_version = importlib.metadata.version("nemo-fabric-adapters-claude") + for report in reports: + # A later invocation without an init message keeps the runtime's version. + assert report["adapter"] == { + "package_version": reported(package_version, "package_metadata") + } + assert report["harness"] == { + "name": reported("claude-code", "harness_init_message"), + "version": reported("2.1.294", "harness_init_message"), + } + + +async def test_claude_versions_report_matches_the_descriptor( + claude_payload, monkeypatch +): + install_scripted_client(monkeypatch, [stream_messages("normal.jsonl")]) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + descriptor = json.loads( + (ROOT / "adapters/python/claude/claude.fabric-adapter.json").read_text( + encoding="utf-8" + ) + ) + + jsonschema.validate( + result.extensions, descriptor["extension_schemas"]["run_result"] + ) + + +async def test_claude_without_an_init_message_reads_the_version_as_not_collected( + claude_payload, monkeypatch +): + [report] = await versions_reports( + claude_payload, monkeypatch, [[running_total_result(0.1)]] + ) + + assert report["harness"]["version"] == { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message", + } diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 32cb1bcfe..144309aa1 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2103,3 +2103,65 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.CodexRuntime, config_loader=AgentConfig.from_mapping ) + + +def _version_fact(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_codex_reads_the_cli_version_from_a_real_app_server(tmp_path: Path): + import importlib.metadata + + from openai_codex.client import CodexConfig + + env = {"CODEX_HOME": str(tmp_path), "HOME": str(tmp_path), "PATH": os.environ["PATH"]} + async with AsyncCodex(config=CodexConfig(cwd=str(tmp_path), env=env)) as codex: + thread = await codex.thread_start(cwd=str(tmp_path), model="gpt-test") + + fact = await adapter.read_cli_version(thread) + + assert fact == { + "value": importlib.metadata.version("openai-codex-cli-bin"), + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def _never_settles() -> None: + await asyncio.Event().wait() + + +@pytest.mark.parametrize( + ("read", "expected"), + [ + pytest.param( + AsyncMock(side_effect=_never_settles), + _version_fact("collection_failed", "timeout"), + id="never-settles", + ), + pytest.param( + AsyncMock(side_effect=JsonRpcError(-32603, "internal error")), + _version_fact("collection_failed", "no_result"), + id="rpc-error", + ), + pytest.param( + AsyncMock(return_value=SimpleNamespace(thread=SimpleNamespace(cli_version=""))), + _version_fact("unavailable", "empty_response"), + id="empty", + ), + ], +) +async def test_codex_cli_version_read_degrades_to_a_state(read, expected): + thread = MagicMock(spec=AsyncThread) + thread.read = read + started = time.monotonic() + + assert await adapter.read_cli_version(thread, timeout=0.1) == expected + assert time.monotonic() - started < 1 diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 9a466f0b1..420ca7018 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1657,3 +1657,158 @@ def test_main_serves_persistent_runtime(monkeypatch): adapter.HermesRuntime, config_loader=AgentConfig.from_mapping, ) + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +def _version_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_hermes_reports_the_version_its_version_call_returns( + monkeypatch, tmp_path: Path +): + from hermes_cli.version_info import get_version_info + + mock_ai_agent_type = MagicMock(spec=AIAgent, return_value=MagicMock(spec=AIAgent)) + monkeypatch.setattr( + mock_ai_agent_type.__init__.__func__, + "__signature__", + inspect.signature(AIAgent.__init__), + raising=False, + ) + hermes_config = ModuleType("hermes_cli.config") + hermes_config.load_config = lambda: {} # type: ignore[attr-defined] + hermes_plugins = ModuleType("hermes_cli.plugins") + hermes_plugins.discover_plugins = lambda force=False: None # type: ignore[attr-defined] + hermes_state = ModuleType("hermes_state") + hermes_state.SessionDB = MagicMock(spec=SessionDB) # type: ignore[attr-defined] + run_agent = ModuleType("run_agent") + run_agent.AIAgent = mock_ai_agent_type # type: ignore[attr-defined] + # Hermes's own version call stays real. + monkeypatch.setitem(sys.modules, "hermes_cli.config", hermes_config) + monkeypatch.setitem(sys.modules, "hermes_cli.plugins", hermes_plugins) + monkeypatch.setitem(sys.modules, "hermes_state", hermes_state) + monkeypatch.setitem(sys.modules, "run_agent", run_agent) + monkeypatch.setenv("TEST_API_KEY", "secret") + mock_turn = MagicMock( + return_value=( + {"final_response": "done", "completed": True, "messages": []}, + "", + ) + ) + monkeypatch.setattr(adapter, "_invoke_hermes_turn", mock_turn) + monkeypatch.setattr( + configuration, "resolve_hermes_toolsets", lambda *_args: None + ) + runtime_context = _runtime_context( + runtime_id="runtime-version", + workspace=str(tmp_path), + artifact_root=str(tmp_path / "artifacts"), + ) + payload = { + "agent_name": "hermes-version", + "base_dir": str(tmp_path), + "runtime_context": runtime_context.to_mapping(), + "config": _agent_config( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "test-provider", + "model": "test-model", + "api_key_env": "TEST_API_KEY", + } + }, + } + ), + } + + runtime = adapter.HermesRuntime() + await runtime.start(payload) + try: + result = await runtime.invoke( + *_invocation( + { + "runtime_context": runtime_context.to_mapping(), + "request": {"input": "complete the task"}, + } + ) + ) + finally: + await runtime.stop() + + report = result.extensions["nemo_fabric.versions.v1"] + assert report["harness"] == { + "name": _reported("hermes-agent"), + "version": _reported(get_version_info().display_version), + } + assert report["adapter"]["package_version"]["value"] == "0.6.0" + + +def _raise_import_error() -> str: + raise ImportError("hermes_cli.version_info") + + +def _raise_runtime_error() -> str: + raise RuntimeError("malformed install stamp") + + +@pytest.mark.parametrize( + ("version_call", "expected"), + [ + pytest.param(lambda: "0.21.6+3", _reported("0.21.6+3"), id="version"), + pytest.param( + lambda: "unknown", + _version_state("unavailable", "empty_response"), + id="unknown", + ), + pytest.param( + _raise_import_error, + _version_state("unsupported", "no_source"), + id="no-version-call", + ), + pytest.param( + _raise_runtime_error, + _version_state("collection_failed", "no_result"), + id="raised", + ), + ], +) +async def test_hermes_version_probe_maps_each_outcome(version_call, expected): + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe, version_call) == expected + + +async def test_hermes_version_probe_times_out_and_starts_no_second_call(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "0.21.6" + + probe = adapter.provenance.BlockingProbe() + try: + first = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + second = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + finally: + release.set() + + assert first == second == _version_state("collection_failed", "timeout") + assert calls == ["called"] diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index 031624bf2..9c31c94c4 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -7,6 +7,7 @@ import asyncio import ctypes +import importlib.metadata import json import os import shutil @@ -287,6 +288,20 @@ async def test_openclaw_runtime_generates_config_invokes_and_cleans_up( assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} assert result.usage.total_tokens == 5 + assert result.extensions["nemo_fabric.versions.v1"]["harness"] == { + "name": { + "value": "openclaw", + "source": "reported", + "collected_by": "adapter", + "method": "harness_probe", + }, + "version": { + "value": "2026.9.4", + "source": "reported", + "collected_by": "adapter", + "method": "harness_probe", + }, + } assert request == { "model": "openclaw/default", "messages": [ @@ -1473,6 +1488,12 @@ async def test_openclaw_plan_doctor_and_run_without_credentials( assert report.status == "pass" assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} + # Core moved the adapter's versions report into what_ran. + assert "adapter" not in result.metadata + assert result.what_ran["harness"]["version"]["value"] == "2026.9.4" + assert result.what_ran["adapter"]["package_version"]["value"] == ( + importlib.metadata.version("nemo-fabric-adapters-openclaw") + ) @pytest.mark.skipif( diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..ce96f6511 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -9,6 +9,7 @@ from __future__ import annotations +import importlib.metadata import importlib.util import json import os @@ -53,6 +54,25 @@ async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): assert first["output"]["resumed"] is False, results assert second["output"]["resumed"] is True, results assert "user_count=2" in second["output"]["response"], results + # Core read the adapter's versions report into what_ran. + for result in (first, second): + assert result.what_ran["harness"] == { + "name": _reported("deepagents"), + "version": _reported(importlib.metadata.version("deepagents")), + }, results + assert result.what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-deepagents") + ) + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") diff --git a/tests/e2e/test_mini_swe_agent.py b/tests/e2e/test_mini_swe_agent.py index 8e03b7ccf..f557de688 100644 --- a/tests/e2e/test_mini_swe_agent.py +++ b/tests/e2e/test_mini_swe_agent.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import sys import warnings from pathlib import Path @@ -92,6 +93,32 @@ async def test_mini_swe_agent_plans_diagnoses_and_runs( assert result.status == "succeeded", result.to_mapping() assert result.output["output"] == "done\n" assert result.output["usage"]["api_calls"] == 1 + what_ran = result.what_ran + assert what_ran["harness"] == { + "name": _reported("mini-swe-agent"), + "version": _reported(importlib.metadata.version("mini-swe-agent")), + } + assert what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-mini-swe-agent") + ) + assert what_ran["models"]["default"]["served"] == { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + assert what_ran["models"]["default"]["requested"]["value"] == "gpt-4o-mini" + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("nemo_relay") diff --git a/tests/fixtures/claude_streams/api-error.jsonl b/tests/fixtures/claude_streams/api-error.jsonl new file mode 100644 index 000000000..655684b79 --- /dev/null +++ b/tests/fixtures/claude_streams/api-error.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"cef05a1a-6da4-4dc1-a94f-2b6c26dd46a4","cwd":"/workspace"} +{"type":"assistant","message":{"diagnostics":null,"id":"68fbbc6b-0c2f-4556-94cd-f68c2aff4a43","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"output_tokens_details":null,"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null,"fallback_credit":null},"content":[{"type":"text","text":"API Error: 400 mock bad request"}],"context_management":null},"parent_tool_use_id":null,"session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","uuid":"72126e34-8d7a-444b-8961-470bea8bb512","timestamp":"2026-10-20T05:19:34.225Z","error":"unknown","is_api_error_message":true} +{"duration_api_ms":0,"stop_reason":"stop_sequence","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","total_cost_usd":0,"usage":{"output_tokens_details":{"thinking_tokens":0},"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{},"permission_denials":[],"terminal_reason":"api_error","is_error":true,"num_turns":1,"subtype":"success","result":"API Error: 400 mock bad request","type":"result","duration_ms":187,"uuid":"28f8d366-742e-4db2-a7eb-8bb34104492c"} diff --git a/tests/fixtures/claude_streams/normal.jsonl b/tests/fixtures/claude_streams/normal.jsonl new file mode 100644 index 000000000..341eca7fa --- /dev/null +++ b/tests/fixtures/claude_streams/normal.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"6fbe3966-4850-48f8-89db-81c3f98a7bb9","cwd":"/workspace"} +{"type":"assistant","message":{"id":"msg_mock01","type":"message","role":"assistant","model":"claude-served-snapshot-20991231","content":[{"type":"text","text":"hello"}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1},"context_management":null},"parent_tool_use_id":null,"session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","uuid":"6614faf3-67b5-46ce-b5e0-acc00afd4c6e","timestamp":"2026-10-20T05:18:01.090Z"} +{"duration_api_ms":50,"stop_reason":"end_turn","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","total_cost_usd":6e-05,"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-sonnet-4-5":{"inputTokens":10,"outputTokens":2,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":6e-05,"contextWindow":200000,"maxOutputTokens":32000,"thinkingTokens":0,"canonicalModel":"claude-sonnet-4-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","is_error":false,"num_turns":1,"subtype":"success","result":"hello","type":"result","duration_ms":163,"uuid":"5c77634b-91b6-4d74-958a-b17cde293d82"} From c76d9fbb0f891d004ee4f9ba821028c97973704f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:53:42 -0700 Subject: [PATCH 15/74] feat: report the model that served each Claude Code invocation what_ran.models..served stayed unsupported for Claude Code even though each assistant message names the model that answered. The Claude Code adapter now reports, for the role it runs (default, or the sole configured role), the one distinct model named by this invocation's main-thread assistant messages. Messages with a parent_tool_use_id belong to subagents and are skipped, and so are Claude Code's API-error messages. Two distinct models read not_attributed, none reads empty_response, a failed stream reads collection_failed, and every invocation starts from its own messages. Tests feed saved Claude Code CLI streams through the SDK's parser. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/claude/adapter.py | 72 ++++++++- tests/adapters/test_claude_adapter.py | 138 ++++++++++++++++++ 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index ec9238886..2a1f68480 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -15,11 +15,13 @@ import subprocess from dataclasses import asdict from dataclasses import dataclass +from dataclasses import field from dataclasses import is_dataclass from hashlib import sha256 from pathlib import Path from typing import Any +from claude_agent_sdk import AssistantMessage from claude_agent_sdk import ClaudeAgentOptions from claude_agent_sdk import ClaudeSDKClient from claude_agent_sdk import ClaudeSDKError @@ -835,6 +837,50 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-claude" _HARNESS_NAME = "claude-code" _METHOD_INIT_MESSAGE = "harness_init_message" +_METHOD_HARNESS_API = "harness_api" +# Claude Code names its own API-error messages with this model. +_SYNTHETIC_MODEL = "" + + +@dataclass +class InvocationMessages: + """One invocation's messages, and why collecting them failed, if it did.""" + + items: list[Message] = field(default_factory=list) + failure: str | None = None + + +def served_model_fact(messages: InvocationMessages | None) -> dict[str, Any]: + """The model that served this invocation's main thread, as a fact. + + Only assistant messages without a ``parent_tool_use_id`` count, so a + subagent's model never decides it, and Claude Code's ```` + API-error messages are skipped. One distinct model is reported. Two or + more read ``not_attributed``, and none reads ``empty_response``. A stream + that failed reads ``collection_failed``, and an invocation that never + queried Claude Code reads ``not_collected``. + """ + + if messages is None: + return provenance.missing_fact("unavailable", "not_collected", _METHOD_HARNESS_API) + if messages.failure is not None: + return provenance.missing_fact( + "collection_failed", messages.failure, _METHOD_HARNESS_API + ) + models = { + message.model + for message in messages.items + if isinstance(message, AssistantMessage) + and message.parent_tool_use_id is None + and isinstance(message.model, str) + and message.model + and message.model != _SYNTHETIC_MODEL + } + if not models: + return provenance.missing_fact("unavailable", "empty_response", _METHOD_HARNESS_API) + if len(models) > 1: + return provenance.missing_fact("unavailable", "not_attributed", _METHOD_HARNESS_API) + return provenance.known_fact(models.pop(), _METHOD_HARNESS_API) def init_message_version(messages: list[Message]) -> dict[str, Any] | None: @@ -1165,7 +1211,7 @@ async def invoke( "claude_runtime_mismatch", "Claude invocation does not match the connected runtime", ) - messages: list[Message] = [] + messages: InvocationMessages | None = None if self._unusable: return _agent_run_result( _failure( @@ -1183,6 +1229,7 @@ async def invoke( except ClaudeSDKError as error: output = sdk_failure(error) else: + messages = InvocationMessages() relay = self._relay atif_before = ( relay_artifacts.snapshot_atif_files(relay.plugin_config) @@ -1230,18 +1277,24 @@ async def invoke( return _agent_run_result(output, self._versions_report(agent_config, messages)) def _versions_report( - self, config: AgentConfig, messages: list[Message] + self, config: AgentConfig, messages: InvocationMessages | None ) -> dict[str, Any]: - """Report this runtime's versions, refreshed from any init message.""" + """Report this runtime's versions and this invocation's served model. + + The harness version is refreshed from any init message. The served + model belongs to the role Claude Code runs; another role has no source. + """ - version = init_message_version(messages) + version = init_message_version(messages.items if messages else []) if version is not None: self._harness_version = version + served = {role: provenance.no_source_fact() for role in config.models} + served[_selected_model_role(config)] = served_model_fact(messages) return provenance.versions_report( package_version=self._package_version, harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_INIT_MESSAGE), harness_version=self._harness_version, - served_models={role: provenance.no_source_fact() for role in config.models}, + served_models=served, ) async def _run_query( @@ -1249,15 +1302,16 @@ async def _run_query( client: ClaudeSDKClient, prompt: str, remaining_timeout: float, - messages: list[Message] | None = None, + collected: InvocationMessages | None = None, ) -> dict[str, Any]: """Run one SDK query and normalize its terminal result. Every message other than the terminal result is appended to - ``messages``, so the caller can read this invocation's messages. + ``collected``, which also records why the stream failed, if it did. """ - messages = [] if messages is None else messages + collected = InvocationMessages() if collected is None else collected + messages = collected.items result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: @@ -1270,6 +1324,7 @@ async def _run_query( messages.append(message) except (TimeoutError, ClaudeSDKError) as error: self._unusable = True + collected.failure = "timeout" if deadline.expired() else "no_result" await self._interrupt_failed_invocation() output = ( _failure("timeout", "Claude invocation timed out") @@ -1287,6 +1342,7 @@ async def _run_query( else: if result is None: self._unusable = True + collected.failure = "no_result" output = _failure( "claude_missing_result", "Claude returned no terminal result" ) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index f5a176b47..7ac4ecb54 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -2227,3 +2227,141 @@ async def test_claude_without_an_init_message_reads_the_version_as_not_collected "collected_by": "adapter", "method": "harness_init_message", } + + +def wire_lines(name: str) -> list[dict[str, Any]]: + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines] + + +def assistant_line(model: str, parent_tool_use_id: str | None = None) -> dict[str, Any]: + """Claude Code's wire shape of an assistant message, from the saved stream.""" + + [line] = [line for line in wire_lines("normal.jsonl") if line["type"] == "assistant"] + line["message"]["model"] = model + line["parent_tool_use_id"] = parent_tool_use_id + return line + + +def parsed_stream(*middle: dict[str, Any]) -> list[Message]: + """The saved init and result lines around ``middle``, through the SDK parser.""" + + init, _, result = wire_lines("normal.jsonl") + return [parse_message(line) for line in (init, *middle, result)] + + +def served_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +SERVED = "claude-served-snapshot-20991231" + + +@pytest.mark.parametrize( + ("stream", "expected"), + [ + pytest.param( + lambda: stream_messages("normal.jsonl"), + reported(SERVED, "harness_api"), + id="one-main-thread-model", + ), + pytest.param( + lambda: parsed_stream( + assistant_line("claude-a"), assistant_line("claude-b") + ), + served_state("unavailable", "not_attributed"), + id="two-main-thread-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), + assistant_line("claude-haiku-child", parent_tool_use_id="toolu_01"), + ), + reported(SERVED, "harness_api"), + id="main-and-child-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), assistant_line("") + ), + reported(SERVED, "harness_api"), + id="real-and-synthetic", + ), + pytest.param( + lambda: stream_messages("api-error.jsonl"), + served_state("unavailable", "empty_response"), + id="synthetic-only", + ), + pytest.param( + lambda: parsed_stream(), + served_state("unavailable", "empty_response"), + id="no-assistant-message", + ), + ], +) +async def test_claude_reports_the_main_thread_served_model( + claude_payload, monkeypatch, stream, expected +): + [report] = await versions_reports(claude_payload, monkeypatch, [stream()]) + + assert report["models"] == {"default": {"served": expected}} + + +async def test_claude_reports_the_served_model_under_a_sole_non_default_role( + claude_payload, monkeypatch +): + claude_payload["config"]["models"] = { + "primary": claude_payload["config"]["models"].pop("default") + } + + [report] = await versions_reports( + claude_payload, monkeypatch, [stream_messages("normal.jsonl")] + ) + + assert report["models"] == {"primary": {"served": reported(SERVED, "harness_api")}} + + +async def test_claude_served_model_is_isolated_per_invocation( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [ + parsed_stream(assistant_line("claude-a")), + parsed_stream(assistant_line("claude-b")), + stream_messages("api-error.jsonl"), + ], + ) + + assert [report["models"]["default"]["served"] for report in reports] == [ + reported("claude-a", "harness_api"), + reported("claude-b", "harness_api"), + served_state("unavailable", "empty_response"), + ] + + +async def test_claude_served_model_reads_collection_failed_when_the_stream_fails( + claude_payload, monkeypatch +): + async def respond(_client: MagicMock) -> AsyncIterator[Message]: + yield parse_message(assistant_line(SERVED)) + raise CLIConnectionError("stream closed") + + install_fake_client(monkeypatch, respond) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + + assert result.extensions[VERSIONS_REPORT_KEY]["models"]["default"]["served"] == ( + served_state("collection_failed", "no_result") + ) From 68b4d0fb04ebfe178bea1241638f00fd87868f7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:59:13 -0700 Subject: [PATCH 16/74] docs: describe adapter-reported versions in what_ran The SDK guide and the integration reference said every adapter-reported fact reads key_not_declared. They now name the six adapters that report harness versions, the Claude Code served-model rule, the states a report key can produce and the tool_names cap. The adapter tutorial and the build-adapter skill show adapter authors how to declare and fill nemo_fabric.versions.v1. Signed-off-by: mnajafian-nv --- docs/adapter-contract/tutorials/results.md | 2 ++ docs/sdk/python.mdx | 35 +++++++++++++++---- skills/nemo-fabric-build-adapter/SKILL.md | 2 ++ .../references/results-and-errors.md | 8 +++-- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 7e8f6646b..1aa9fcce6 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,6 +79,8 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. + NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. **Success Check**: `invoke` returns one typed `AgentRunResult` whose explicit diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index cfb12c1ef..2e540e7bf 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1026,12 +1026,31 @@ carry `source`, `collected_by`, and `method`: | `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | | `environment` | The environment `provider`, such as `local` or `openshell`. | -NeMo Fabric reads no adapter-reported facts yet, so every reported fact has -state `unsupported` with reason `key_not_declared` and method -`adapter_report`. - -Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` -describes a server the harness reports but the plan does not configure. It has +An adapter reports its facts through the result keys `nemo_fabric.versions.v1` +(package version, harness name and version, and served models) and +`nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. +The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and +mini-SWE-agent adapters report their package version and their harness name +and version. The Claude Code adapter also reports the served model of the role +it runs: the one model that this invocation's main-thread assistant messages +name, skipping subagent messages and Claude Code's `` API-error +messages. Two models give `unavailable` with reason `not_attributed`, and none +gives `unavailable` with reason `empty_response`. No bundled adapter reports +MCP runtime facts yet. + +A reported fact whose key the adapter does not declare has state `unsupported` +with reason `key_not_declared` and method `adapter_report`. A fact the adapter +has no source for has reason `no_source`. A declared key that the result omits +gives `unavailable` with reason `key_omitted`, and a fact the key omits gives +`fact_omitted`. A malformed fact, or one that does not name source `reported` +and collector `adapter`, gives `collection_failed` with reason +`invalid_report`. A key larger than 64 KiB of compact JSON gives +`collection_failed` with reason `too_large`. None of these changes the run's +status or error. + +An MCP entry has `origin` `plan` for a server the plan configures. An entry +with `origin` `harness` describes a server the harness reports but the plan +does not configure. It has the same shape, except that `endpoint` has state `not_applicable` with reason `not_configured`. A `plan` entry always has `transport`. An entry of any other origin can leave it out. @@ -1053,7 +1072,9 @@ URL or executable has state `unavailable` with reason `not_configured`. A string value longer than 256 characters keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. The rule applies to each -element of a list value too. +element of a list value too. `tool_names` keeps at most 64 names in sorted +order, and a `tool_names_truncated` marker in the `runtime` block counts the +names the adapter or NeMo Fabric dropped. `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in sorted key order. An entry whose key is longer than 128 characters is dropped. diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 503448cba..15705843c 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,6 +230,8 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. + Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then prove another runtime cannot observe that context. Refer to the diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 3793b558a..71b4994c7 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -21,7 +21,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | -| `what_ran` | What executed for this invocation: core version, adapter ID, requested models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | +| `what_ran` | What executed for this invocation: core version, adapter ID and package version, harness name and version, requested and served models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | Handle a result with this status-first pattern: @@ -51,8 +51,10 @@ Read `what_ran` as evidence of what executed, never as a grouping key. Each fact holds a `value`, or a `state` and `reason` when it has no value, plus `source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and `reason` codes you do not recognize. Adapter-reported facts, such as the -harness version and the served model, currently read state `unsupported` with -reason `key_not_declared`. A malformed `what_ran` reads as `None` without +harness version and the served model, read state `unsupported` with reason +`key_not_declared` when the adapter does not report them. The Claude Code, +Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report +the harness version, and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without failing the result. A `what_ran` over its 128 KiB budget can hold only `schema`, `version`, and `what_ran_truncated`, so check for that marker before reading a section. Refer to From 81044a63d0bc3c1afcf2ebfd1be55cf2d72649b1 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:59:14 -0700 Subject: [PATCH 17/74] docs: regenerate API references for adapter report keys Regenerated with just docs for the new report-key constants and the tool_names_truncated field. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 66 ++++++++++++++++++- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 8 +-- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../what-ran/constant-mcp-report-key.mdx | 15 +++++ .../what-ran/constant-versions-report-key.mdx | 15 +++++ .../what-ran/constant-what-ran-schema.mdx | 2 +- .../what-ran/constant-what-ran-version.mdx | 2 +- .../nemo-fabric-core/what-ran/index.mdx | 4 ++ .../what-ran/struct-mcpruntimefacts.mdx | 8 ++- 47 files changed, 151 insertions(+), 49 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..1a5cbd1a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..b89e437a4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..3048bc293 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..595a1fd29 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..b5f664a1a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 75900423a..975080702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,70 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + +### `EnvironmentInUse` + +
+ +A runtime lifecycle operation is still active or has an uncertain outcome. + +#### Fields + +##### `environment_id: String` + +Environment whose runtime operation is unresolved. + +##### `runtime_id: String` + +Runtime identity, or `unconfirmed` if the start response was lost. + +### `EnvironmentReleaseInProgress` + +
+ +An environment release is already in progress. + +#### Fields + +##### `environment_id: String` + +Environment whose lifecycle is being released or detached. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..5acac20bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..ce8835391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 9dfc3f1be..1ee527133 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,14 +2,14 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -63,9 +63,9 @@ Normalized invocation usage when reported by the adapter target. Artifacts produced or collected by this run. -### `telemetry: Option` +### `telemetry: Vec` -Telemetry reference when available. +Ordered telemetry references when available. ### `events: Vec` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index cd66e4025..94338baeb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx new file mode 100644 index 000000000..612f219bf --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant MCP_REPORT_KEY" +sidebar-title: "MCP_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-mcp-report-key" +description: "Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.mcp.v1\";"}} />
+ +Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx new file mode 100644 index 000000000..b2938f6b4 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant VERSIONS_REPORT_KEY" +sidebar-title: "VERSIONS_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-versions-report-key" +description: "Report key for the adapter package version, the harness name and version, and the served model of each role." +position: 18 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.versions.v1\";"}} />
+ +Report key for the adapter package version, the harness name and version, and the served model of each role. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx index 3b95258d3..ede3a0ea9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_SCHEMA" sidebar-title: "WHAT_RAN_SCHEMA" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" description: "Schema name written into every `WhatRan`." -position: 17 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx index f517163e1..e25ef5233 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_VERSION" sidebar-title: "WHAT_RAN_VERSION" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" description: "Contract version written into every `WhatRan`." -position: 18 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index b99298410..a8c31143e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -18,6 +18,8 @@ A string value longer than 256 characters, in a string fact or as an element of `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. +An adapter reports its own facts through the report keys [`VERSIONS_REPORT_KEY`](constant-versions-report-key.mdx) and [`MCP_REPORT_KEY`](constant-mcp-report-key.mdx) in its result's extensions, once its descriptor declares them. A reported list keeps at most 64 elements in sorted order, counted in a marker beside it, such as `tool_names_truncated`. + `what_ran` is evidence. It never decides which runs are compared. Compatibility rules for readers: @@ -51,5 +53,7 @@ Compatibility rules for readers: ## Constants +- [MCP_REPORT_KEY](constant-mcp-report-key.mdx): Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. +- [VERSIONS_REPORT_KEY](constant-versions-report-key.mdx): Report key for the adapter package version, the harness name and version, and the served model of each role. - [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). - [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx index afa1bece5..1dca67789 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n}"}} />
+
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n    pub tool_names_truncated: Option<TruncatedEntries>,\n}"}} />
What the harness learned about one MCP server when it connected. @@ -34,7 +34,11 @@ Version the server reports for itself. ### `tool_names: Fact>` -Raw tool names the server lists, each bounded by the string length rule. +Raw tool names the server lists, each bounded by the string length rule. At most 64 names, in sorted order. + +### `tool_names_truncated: Option` + +Tool names dropped from `tool_names`, by the adapter's helper or by core. Absent when none was dropped, never `null`. ## Trait Implementations From 57b64f35411976bade2fb3791ff9ad9eccb7d285 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 02:49:52 -0700 Subject: [PATCH 18/74] feat: report the package version from the other Python adapters NOOA, NOOA Bench, OpenHands and Remote Agent left adapter.package_version at key_not_declared, so a run with them could not say which adapter build ran. Each now declares nemo_fabric.versions.v1 and attaches its installed package version to every result. These adapters have no harness version or served-model source yet, so those facts read unsupported with reason no_source. The catalog is regenerated. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/common/provenance.py | 18 ++++ .../nooa/nooa-bench.fabric-adapter.json | 10 ++ adapters/python/nooa/nooa.fabric-adapter.json | 10 ++ .../src/nemo_fabric_adapters/nooa/adapter.py | 18 ++++ .../nooa/bench_adapter.py | 18 ++++ .../openhands/openhands.fabric-adapter.json | 10 ++ .../nemo_fabric_adapters/openhands/adapter.py | 18 ++++ .../remote-agent.fabric-adapter.json | 10 ++ .../remote_agent/adapter.py | 18 ++++ .../nemo_fabric_adapter_catalog/catalog.json | 56 ++++++++++- .../adapters/test_adapter_package_versions.py | 93 +++++++++++++++++++ .../test_adapters_common_provenance.py | 13 +++ 12 files changed, 288 insertions(+), 4 deletions(-) create mode 100644 tests/adapters/test_adapter_package_versions.py diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py index 43d41933d..b16a213b9 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -18,6 +18,7 @@ import sys import threading from collections.abc import Callable +from collections.abc import Iterable from collections.abc import Mapping from pathlib import Path from typing import Any @@ -159,6 +160,23 @@ def versions_report( } +def package_version_report( + distribution: str, model_roles: Iterable[str] +) -> dict[str, Any]: + """The report of an adapter that has a source only for its package version. + + The harness name and version and each role's served model read + ``unsupported`` with reason ``no_source``. + """ + + return versions_report( + package_version=package_version_fact(distribution), + harness_name=no_source_fact(), + harness_version=no_source_fact(), + served_models={role: no_source_fact() for role in model_roles}, + ) + + class BlockingProbe: """Run a runtime's blocking version calls off the event loop, one at a time. diff --git a/adapters/python/nooa/nooa-bench.fabric-adapter.json b/adapters/python/nooa/nooa-bench.fabric-adapter.json index 8ef2b3f71..529e391d3 100644 --- a/adapters/python/nooa/nooa-bench.fabric-adapter.json +++ b/adapters/python/nooa/nooa-bench.fabric-adapter.json @@ -35,6 +35,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/nooa/nooa.fabric-adapter.json b/adapters/python/nooa/nooa.fabric-adapter.json index 475c144c2..5b280f880 100644 --- a/adapters/python/nooa/nooa.fabric-adapter.json +++ b/adapters/python/nooa/nooa.fabric-adapter.json @@ -36,6 +36,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py index e4725af2c..670e075fd 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py @@ -26,12 +26,15 @@ from nemo_fabric_adapter_contract.models import AgentRunStatus from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa" MODE = "interactive_agent" @@ -522,6 +525,7 @@ def __init__(self) -> None: self._target: InteractiveAgentTarget | None = None self._models: dict[str, Any] = {} self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: if self._target is not None: @@ -531,6 +535,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) factory = _load_factory(_factory_ref(config)) models: dict[str, Any] = {} target: InteractiveAgentTarget | None = None @@ -586,6 +591,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if self._target is None or self._runtime_id is None: raise lifecycle.LifecycleError( diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py index ff103932b..a4175e94a 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py @@ -21,12 +21,15 @@ from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa-bench" MODE = "bench_agent" @@ -201,6 +204,7 @@ def __init__(self) -> None: self._workspace: Path | None = None self._instruction: str | None = None self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: if self._bench_agent_factory is not None: @@ -210,6 +214,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) role = _selected_model_role(config) models: dict[str, Any] = {} bench_agent_factory: Callable[..., Any] | None = None @@ -260,6 +265,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if ( self._bench_agent_factory is None diff --git a/adapters/python/openhands/openhands.fabric-adapter.json b/adapters/python/openhands/openhands.fabric-adapter.json index 1c3af5253..d0eb80c6e 100644 --- a/adapters/python/openhands/openhands.fabric-adapter.json +++ b/adapters/python/openhands/openhands.fabric-adapter.json @@ -39,6 +39,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py index 7ec53266a..646192bb5 100644 --- a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py +++ b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py @@ -16,8 +16,11 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openhands" + HARNESS = "OpenHands" DEFAULT_MAX_ITERATIONS = 500 TOOL_ALIASES = { @@ -333,9 +336,11 @@ def __init__(self) -> None: self._conversation: Any = None self._profile_store_temp: tempfile.TemporaryDirectory[str] | None = None self._runtime_id: str | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload.get("runtime_context")) base_dir = common_utils.base_dir(payload) instruction = common_instructions.system_instruction( @@ -401,6 +406,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if ( self._api is None diff --git a/adapters/python/remote-agent/remote-agent.fabric-adapter.json b/adapters/python/remote-agent/remote-agent.fabric-adapter.json index a6563914a..46e8cf458 100644 --- a/adapters/python/remote-agent/remote-agent.fabric-adapter.json +++ b/adapters/python/remote-agent/remote-agent.fabric-adapter.json @@ -69,6 +69,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py index 89e82dd71..9425fec36 100644 --- a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py +++ b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py @@ -16,9 +16,12 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import openai_chat from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-remote-agent" + DEFAULT_API_TYPE = "openai-responses" DEFAULT_ANTHROPIC_MAX_TOKENS = 4096 @@ -127,9 +130,11 @@ def __init__(self) -> None: self._api_type = DEFAULT_API_TYPE self._messages: list[dict[str, str]] = [] self._relay_streaming = False + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload["runtime_context"]) settings = config.harness.settings if config.harness is not None else {} self._api_type = settings.get("api_type", DEFAULT_API_TYPE) @@ -210,6 +215,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if self._client is None or self._config is None: raise lifecycle.LifecycleError( diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index 14d79212e..bbcb85c6c 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -1348,6 +1348,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1421,7 +1433,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "80f632e09ea3279e7283a58790e53516d693162bc7742f2e932b8e7453373042", + "sha256": "c7905de4f20a376eac20061c9f5c49f6be6d8eb40c83497c919c3a53173e30fd", "source": "adapters/python/nooa/nooa.fabric-adapter.json" }, "nvidia.fabric.nooa.bench-agent": { @@ -1443,6 +1455,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1513,7 +1537,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "87e09e21e71d7b7bb081c357f1cbc2278a9b467ba76cc4db77d230713d126b20", + "sha256": "a70774d0926488f0a7f03bbf1a1f3d2962a939c7433795cc8cd618839b8c938e", "source": "adapters/python/nooa/nooa-bench.fabric-adapter.json" }, "nvidia.fabric.openclaw": { @@ -1874,6 +1898,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1945,7 +1981,7 @@ "name": "nemo-fabric-adapters-openhands", "version": "0.6.0" }, - "sha256": "062ef09ba5e82dfdf0b4374632a8b39b7ff975160b0c6f62e68d2f4ac6f0b59e", + "sha256": "9f621c3c5308bce2104b7d17ff71e3c401b1823429aa9f70bac1053eaa420f21", "source": "adapters/python/openhands/openhands.fabric-adapter.json" }, "nvidia.fabric.pi": { @@ -2303,6 +2339,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -2414,7 +2462,7 @@ "name": "nemo-fabric-adapters-remote-agent", "version": "0.6.0" }, - "sha256": "39870c056b7aa15b8706b6cabf9190029f6c9165c5c9ce4b427f5f18bd7ba70a", + "sha256": "e5da937cefdfddf59720e9e04fbc22d3abd0f7418eaab99ffaeee8c0c3bc681b", "source": "adapters/python/remote-agent/remote-agent.fabric-adapter.json" } }, diff --git a/tests/adapters/test_adapter_package_versions.py b/tests/adapters/test_adapter_package_versions.py new file mode 100644 index 000000000..b3f0e339a --- /dev/null +++ b/tests/adapters/test_adapter_package_versions.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Python adapters without a harness version source still report their package version.""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +import jsonschema +import pytest +from nemo_fabric_adapter_contract.models import AgentRunRequest +from nemo_fabric_adapter_contract.models import AgentRunResult +from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import RuntimeContext + +ROOT = Path(__file__).resolve().parents[2] +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +NO_SOURCE = { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", +} + + +@pytest.mark.parametrize( + ("module", "runtime_type", "distribution", "descriptor"), + [ + ( + "nemo_fabric_adapters.nooa.adapter", + "NooaRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.nooa.bench_adapter", + "BenchRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa-bench.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.openhands.adapter", + "OpenHandsRuntime", + "nemo-fabric-adapters-openhands", + "adapters/python/openhands/openhands.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.remote_agent.adapter", + "RemoteAgentRuntime", + "nemo-fabric-adapters-remote-agent", + "adapters/python/remote-agent/remote-agent.fabric-adapter.json", + ), + ], +) +async def test_adapter_reports_its_package_version_on_each_result( + module, runtime_type, distribution, descriptor, monkeypatch +): + runtime = getattr(pytest.importorskip(module), runtime_type)() + runtime._model_roles = ["default"] + returned = AgentRunResult( + status=AgentRunStatus.FAILED, + output={"response": None}, + error={"code": "target_failed", "message": "failed"}, + ) + monkeypatch.setattr(runtime, "_invoke", AsyncMock(return_value=returned)) + + result = await runtime.invoke( + AgentRunRequest(input="hello"), MagicMock(spec=RuntimeContext) + ) + + assert result.status is AgentRunStatus.FAILED + assert result.extensions[VERSIONS_REPORT_KEY] == { + "adapter": { + "package_version": { + "value": importlib.metadata.version(distribution), + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } + }, + "harness": {"name": NO_SOURCE, "version": NO_SOURCE}, + "models": {"default": {"served": NO_SOURCE}}, + } + schema = json.loads((ROOT / descriptor).read_text(encoding="utf-8")) + jsonschema.validate( + result.to_mapping()["extensions"], schema["extension_schemas"]["run_result"] + ) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index 2b3627f09..fada78a86 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -166,3 +166,16 @@ def fail() -> str: assert await probe.call(lambda: "1.2.3") == "1.2.3" with pytest.raises(RuntimeError, match="no version"): await probe.call(fail) + + +def test_package_version_report_marks_harness_and_served_models_no_source(): + report = provenance.package_version_report( + "nemo-fabric-adapters-common", ["judge", "default"] + ) + + no_source = missing("unsupported", "no_source", "adapter_report") + assert report == { + "adapter": {"package_version": known("0.6.0")}, + "harness": {"name": no_source, "version": no_source}, + "models": {"default": {"served": no_source}, "judge": {"served": no_source}}, + } From a6a7290c4afec5c5a861f9525976b3ddc72deb1f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 02:50:00 -0700 Subject: [PATCH 19/74] feat: report the package version from the TypeScript adapters Cline, Droid, Kilo Code, OpenCode, Pi and Qwen Code left adapter.package_version at key_not_declared. A shared withPackageVersion wrapper in the TypeScript common package reads the adapter's own package.json synchronously at start and adds nemo_fabric.versions.v1 to every result, with the harness and served model facts unsupported/no_source until their sources land. Each CLI wraps its runtime, each descriptor declares the key, and the catalog is regenerated. A catalog test checks that all bundled descriptors declare the key. Signed-off-by: mnajafian-nv --- .../cline/cline.fabric-adapter.json | 1 + adapters/typescript/cline/src/cli.ts | 4 +- adapters/typescript/common/src/index.ts | 8 ++ adapters/typescript/common/src/provenance.ts | 98 +++++++++++++++++++ .../common/test/provenance.test.mjs | 72 ++++++++++++++ .../droid/droid.fabric-adapter.json | 1 + adapters/typescript/droid/src/cli.ts | 4 +- .../typescript/kilo/kilo.fabric-adapter.json | 10 ++ adapters/typescript/kilo/src/cli.ts | 4 +- .../opencode/opencode.fabric-adapter.json | 10 ++ adapters/typescript/opencode/src/cli.ts | 4 +- adapters/typescript/pi/pi.fabric-adapter.json | 1 + adapters/typescript/pi/src/cli.ts | 4 +- adapters/typescript/pi/test/runtime.test.mjs | 1 + .../typescript/qwen/qwen.fabric-adapter.json | 10 ++ adapters/typescript/qwen/src/cli.ts | 4 +- .../nemo_fabric_adapter_catalog/catalog.json | 57 +++++++++-- tests/adapters/test_opencode_adapter.py | 8 ++ tests/adapters/test_pi_adapter.py | 1 + tests/python/test_adapter_catalog.py | 14 +++ 20 files changed, 298 insertions(+), 18 deletions(-) create mode 100644 adapters/typescript/common/src/provenance.ts create mode 100644 adapters/typescript/common/test/provenance.test.mjs diff --git a/adapters/typescript/cline/cline.fabric-adapter.json b/adapters/typescript/cline/cline.fabric-adapter.json index 49c72f67c..e349f8f5d 100644 --- a/adapters/typescript/cline/cline.fabric-adapter.json +++ b/adapters/typescript/cline/cline.fabric-adapter.json @@ -42,6 +42,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "session_id": {"type": "string", "minLength": 1}, "finish_reason": { "type": "string", diff --git a/adapters/typescript/cline/src/cli.ts b/adapters/typescript/cline/src/cli.ts index 2d5a12025..79b2a3bd2 100644 --- a/adapters/typescript/cline/src/cli.ts +++ b/adapters/typescript/cline/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ ClineSdkSessionFactory }, { ClineAdapterRuntime }] = await Promise.all([ import("./cline-sdk.js"), import("./runtime.js"), ]); - return new ClineAdapterRuntime(new ClineSdkSessionFactory()); + return withPackageVersion(new ClineAdapterRuntime(new ClineSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/common/src/index.ts b/adapters/typescript/common/src/index.ts index e207ca41a..94cee2995 100644 --- a/adapters/typescript/common/src/index.ts +++ b/adapters/typescript/common/src/index.ts @@ -15,3 +15,11 @@ export { } from "./lifecycle.js"; export { insecureHttpModelHosts, modelEndpointAllowed } from "./model-endpoint.js"; + +export { + VERSIONS_REPORT_KEY, + boundedText, + packageVersionFact, + packageVersionReport, + withPackageVersion, +} from "./provenance.js"; diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts new file mode 100644 index 000000000..d0a4ef35d --- /dev/null +++ b/adapters/typescript/common/src/provenance.ts @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Reports an adapter's own package version in the `nemo_fabric.versions.v1` +// result extension. NeMo Fabric core reads the key when the adapter descriptor +// names it under `extension_schemas.run_result`, and moves it into +// `RunResult.what_ran`. + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +import type { AgentRunRequest, AgentRunResult, JsonObject, RuntimeContext } from "nemo-fabric-adapter-contract"; + +import type { AdapterRuntime, AdapterStartInput } from "./lifecycle.js"; + +export const VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1"; + +const MAX_VALUE_CHARS = 256; +const TRUNCATED_PREFIX_CHARS = 240; +const TRUNCATED_DIGEST_HEX_DIGITS = 15; +const MAX_MODEL_ROLES = 16; +const MAX_KEY_CHARS = 128; + +/** + * Apply NeMo Fabric core's string length rule, counting Unicode scalar values. + * A lone surrogate becomes `?` first, because core rejects a result that holds + * one. + */ +export function boundedText(text: string): string { + const chars = Array.from(text, (char) => { + const code = char.codePointAt(0) ?? 0; + return code >= 0xd800 && code <= 0xdfff ? "?" : char; + }); + if (chars.length <= MAX_VALUE_CHARS) return chars.join(""); + const digest = createHash("sha256").update(chars.join(""), "utf8").digest("hex"); + return `${chars.slice(0, TRUNCATED_PREFIX_CHARS).join("")}~${digest.slice(0, TRUNCATED_DIGEST_HEX_DIGITS)}`; +} + +function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { + return {state, reason, source: "reported", collected_by: "adapter", method}; +} + +/** + * The `version` field of the adapter's own `package.json`, read synchronously. + * A file that can't be read or parsed reads `unavailable` with reason + * `not_collected`, and an empty version reads `empty_response`. + */ +export function packageVersionFact(packageJson: URL): JsonObject { + let version: unknown; + try { + version = (JSON.parse(readFileSync(packageJson, "utf8")) as {version?: unknown}).version; + } catch { + return missingFact("unavailable", "not_collected", "package_metadata"); + } + if (typeof version !== "string" || version.length === 0) { + return missingFact("unavailable", "empty_response", "package_metadata"); + } + return {value: boundedText(version), source: "reported", collected_by: "adapter", method: "package_metadata"}; +} + +/** + * The report of an adapter that has a source only for its package version. + * The harness name and version and each role's served model read + * `unsupported` with reason `no_source`. Like core, it keeps at most 16 roles + * in sorted order and skips a role longer than 128 characters. + */ +export function packageVersionReport(packageVersion: JsonObject, modelRoles: readonly string[]): JsonObject { + const noSource = () => missingFact("unsupported", "no_source"); + const roles = modelRoles.filter((role) => Array.from(role).length <= MAX_KEY_CHARS).sort().slice(0, MAX_MODEL_ROLES); + return { + adapter: {package_version: packageVersion}, + harness: {name: noSource(), version: noSource()}, + models: Object.fromEntries(roles.map((role) => [role, {served: noSource()}])), + }; +} + +/** + * Wrap `runtime` so each result reports the adapter package version read from + * `packageJson` when the runtime starts. Results and errors are otherwise + * unchanged. + */ +export function withPackageVersion(runtime: AdapterRuntime, packageJson: URL): AdapterRuntime { + let report: JsonObject | undefined; + return { + async start(input: AdapterStartInput): Promise { + report = packageVersionReport(packageVersionFact(packageJson), Object.keys(input.config.models ?? {})); + await runtime.start(input); + }, + async invoke(request: AgentRunRequest, context: RuntimeContext): Promise { + const result = await runtime.invoke(request, context); + if (report === undefined) return result; + return {...result, extensions: {...(result.extensions ?? {}), [VERSIONS_REPORT_KEY]: report}}; + }, + stop(): Promise { + return runtime.stop(); + }, + }; +} diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs new file mode 100644 index 000000000..178e305f1 --- /dev/null +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; + +import { + VERSIONS_REPORT_KEY, + boundedText, + packageVersionFact, + withPackageVersion, +} from "../dist/index.js"; + +const noSource = {state: "unsupported", reason: "no_source", source: "reported", collected_by: "adapter", method: "adapter_report"}; + +// The same vectors as core's string length test in `what_ran.rs`. +test("bounded text matches core's string length rule", () => { + assert.equal(boundedText("é".repeat(256)), "é".repeat(256)); + assert.equal(boundedText(`${"m".repeat(300)}a`), `${"m".repeat(240)}~96858eedeb3c8e6`); + assert.equal(boundedText("é".repeat(300)), `${"é".repeat(240)}~7250b66610f8b7d`); + assert.equal(boundedText("v\ud800"), "v?"); +}); + +test("each TypeScript adapter reads its own package version", async () => { + for (const adapter of ["cline", "droid", "kilo", "opencode", "pi", "qwen"]) { + const packageJson = new URL(`../../${adapter}/package.json`, import.meta.url); + const {version} = JSON.parse(await readFile(packageJson, "utf8")); + assert.deepEqual(packageVersionFact(packageJson), { + value: version, source: "reported", collected_by: "adapter", method: "package_metadata", + }, adapter); + } +}); + +test("an unreadable package.json reads not_collected", () => { + assert.deepEqual(packageVersionFact(new URL("file:///nonexistent/package.json")), { + state: "unavailable", reason: "not_collected", source: "reported", collected_by: "adapter", method: "package_metadata", + }); +}); + +test("the wrapper adds the versions report and keeps the result", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const calls = []; + const runtime = withPackageVersion({ + async start(input) { calls.push(["start", input.agentName]); }, + async invoke() { return {status: "succeeded", output: "done", extensions: {session_id: "s-1"}}; }, + async stop() { calls.push(["stop"]); }, + }, packageJson); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {default: {}, judge: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + await runtime.stop(); + + assert.deepEqual(calls, [["start", "agent"], ["stop"]]); + assert.equal(result.status, "succeeded"); + assert.equal(result.output, "done"); + assert.equal(result.extensions.session_id, "s-1"); + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY], { + adapter: {package_version: {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}}, + harness: {name: noSource, version: noSource}, + models: {default: {served: noSource}, judge: {served: noSource}}, + }); + } finally { + await rm(directory, {recursive: true, force: true}); + } +}); diff --git a/adapters/typescript/droid/droid.fabric-adapter.json b/adapters/typescript/droid/droid.fabric-adapter.json index 0c11ce867..b037aab74 100644 --- a/adapters/typescript/droid/droid.fabric-adapter.json +++ b/adapters/typescript/droid/droid.fabric-adapter.json @@ -40,6 +40,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "session_id": {"type": "string", "minLength": 1}, "finish_reason": { "type": "string", diff --git a/adapters/typescript/droid/src/cli.ts b/adapters/typescript/droid/src/cli.ts index fe522cdac..42c063af5 100644 --- a/adapters/typescript/droid/src/cli.ts +++ b/adapters/typescript/droid/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ DroidSdkSessionFactory }, { DroidAdapterRuntime }] = await Promise.all([ import("./droid-sdk.js"), import("./runtime.js"), ]); - return new DroidAdapterRuntime(new DroidSdkSessionFactory()); + return withPackageVersion(new DroidAdapterRuntime(new DroidSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/kilo/kilo.fabric-adapter.json b/adapters/typescript/kilo/kilo.fabric-adapter.json index 5ee7eb9d2..c58c8f99e 100644 --- a/adapters/typescript/kilo/kilo.fabric-adapter.json +++ b/adapters/typescript/kilo/kilo.fabric-adapter.json @@ -44,5 +44,15 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} } diff --git a/adapters/typescript/kilo/src/cli.ts b/adapters/typescript/kilo/src/cli.ts index 05bb8e687..fdbc9a378 100644 --- a/adapters/typescript/kilo/src/cli.ts +++ b/adapters/typescript/kilo/src/cli.ts @@ -2,11 +2,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{KiloSdkSessionFactory}, {KiloAdapterRuntime}] = await Promise.all([ import("./kilo-sdk.js"), import("./runtime.js"), ]); - return new KiloAdapterRuntime(new KiloSdkSessionFactory()); + return withPackageVersion(new KiloAdapterRuntime(new KiloSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/opencode/opencode.fabric-adapter.json b/adapters/typescript/opencode/opencode.fabric-adapter.json index 3558508b7..c102cc824 100644 --- a/adapters/typescript/opencode/opencode.fabric-adapter.json +++ b/adapters/typescript/opencode/opencode.fabric-adapter.json @@ -62,6 +62,16 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": { "streaming": false, "cancellation": false, diff --git a/adapters/typescript/opencode/src/cli.ts b/adapters/typescript/opencode/src/cli.ts index e53dbdb6c..4fd730268 100644 --- a/adapters/typescript/opencode/src/cli.ts +++ b/adapters/typescript/opencode/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedBunVersion } from "./bun-version.js"; @@ -30,5 +30,5 @@ await serve(async () => { import("./opencode-sdk.js"), import("./runtime.js"), ]); - return new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()); + return withPackageVersion(new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/pi.fabric-adapter.json b/adapters/typescript/pi/pi.fabric-adapter.json index db31bde6c..5fdd440bc 100644 --- a/adapters/typescript/pi/pi.fabric-adapter.json +++ b/adapters/typescript/pi/pi.fabric-adapter.json @@ -97,6 +97,7 @@ "run_result": { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"} }, diff --git a/adapters/typescript/pi/src/cli.ts b/adapters/typescript/pi/src/cli.ts index b495c41e9..74d1d9421 100644 --- a/adapters/typescript/pi/src/cli.ts +++ b/adapters/typescript/pi/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedNodeVersion } from "./node-version.js"; @@ -22,5 +22,5 @@ await serve(async () => { import("./pi-sdk.js"), import("./runtime.js"), ]); - return new PiAdapterRuntime(new PiSdkSessionFactory()); + return withPackageVersion(new PiAdapterRuntime(new PiSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/test/runtime.test.mjs b/adapters/typescript/pi/test/runtime.test.mjs index 1726e6201..581485bb1 100644 --- a/adapters/typescript/pi/test/runtime.test.mjs +++ b/adapters/typescript/pi/test/runtime.test.mjs @@ -42,6 +42,7 @@ test("declares the cumulative Pi turn count in every run result", async () => { assert.deepEqual(descriptor.extension_schemas.run_result, { type: "object", properties: { + "nemo_fabric.versions.v1": { type: "object" }, pi_turn_count: { type: "integer", minimum: 0 }, pi_turn_started: { type: "boolean" }, }, diff --git a/adapters/typescript/qwen/qwen.fabric-adapter.json b/adapters/typescript/qwen/qwen.fabric-adapter.json index 19ea58016..cd2c3cf97 100644 --- a/adapters/typescript/qwen/qwen.fabric-adapter.json +++ b/adapters/typescript/qwen/qwen.fabric-adapter.json @@ -44,5 +44,15 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} } diff --git a/adapters/typescript/qwen/src/cli.ts b/adapters/typescript/qwen/src/cli.ts index 66a8d7064..8f5291ca4 100644 --- a/adapters/typescript/qwen/src/cli.ts +++ b/adapters/typescript/qwen/src/cli.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; const [major, minor] = process.versions.node.split(".").map(Number); if (major === undefined || minor === undefined || major < 22 || (major === 22 && minor < 19)) { @@ -14,5 +14,5 @@ await serve(async () => { import("./qwen-sdk.js"), import("./runtime.js"), ]); - return new QwenAdapterRuntime(new QwenSdkSessionFactory()); + return withPackageVersion(new QwenAdapterRuntime(new QwenSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index bbcb85c6c..7bbfe3eac 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -269,6 +269,9 @@ ], "type": "object" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -346,7 +349,7 @@ "name": "nemo-fabric-adapters-cline", "version": "0.6.0" }, - "sha256": "4c695e93660eafc004482c1b1e5ac951c33dad22bdb3752791f184e02735e684", + "sha256": "c7e93c832ce8e88401fa2548577cb04a95b2431dd3068692a4e79211074f5646", "source": "adapters/typescript/cline/cline.fabric-adapter.json" }, "nvidia.fabric.codex": { @@ -596,6 +599,9 @@ ], "type": "string" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -680,7 +686,7 @@ "name": "nemo-fabric-adapters-droid", "version": "0.6.0" }, - "sha256": "6d1b3e67bf4d7b803c292057a27aa6c1c272f1160118cf0bb349ee273b067f14", + "sha256": "5aae42b43dbe867ef43444f3796e392143e652be714de649ee75d351d1a5a65f", "source": "adapters/typescript/droid/droid.fabric-adapter.json" }, "nvidia.fabric.hermes": { @@ -842,6 +848,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -911,7 +929,7 @@ "name": "nemo-fabric-adapters-kilo", "version": "0.6.0" }, - "sha256": "608aeacfa0cfa5ff7ccd28e3f66cadd53f82780bc882b8577e7e8687dfd3dd07", + "sha256": "a7e4124a236408686856225f908813a3197efde6bb932a0ac2eb45e0462fe6d1", "source": "adapters/typescript/kilo/kilo.fabric-adapter.json" }, "nvidia.fabric.langchain.deepagents": { @@ -1764,6 +1782,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "allOf": [ @@ -1865,7 +1895,7 @@ "name": "nemo-fabric-adapters-opencode", "version": "0.6.0" }, - "sha256": "86175ea45671c92601bf1a132fc0fe91f3de82f19ce8bf4249e9993125c32641", + "sha256": "498d0c0fa0e4f835711d4990b571969e2aebf70ccf0a31fe1260a5ebc0a0d850", "source": "adapters/typescript/opencode/opencode.fabric-adapter.json" }, "nvidia.fabric.openhands": { @@ -2091,6 +2121,9 @@ "run_result": { "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "pi_turn_count": { "minimum": 0, "type": "integer" @@ -2203,7 +2236,7 @@ "name": "nemo-fabric-adapters-pi", "version": "0.6.0" }, - "sha256": "3cb85c7a0e52e49827e889631da96ca7936c7260d52aac8b52835484a0cafad9", + "sha256": "413cb592ffe6d4724dfa0e4cd35ad8be1474acde8cd0cf33ce8073d666db0a9a", "source": "adapters/typescript/pi/pi.fabric-adapter.json" }, "nvidia.fabric.qwen": { @@ -2234,6 +2267,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -2313,7 +2358,7 @@ "name": "nemo-fabric-adapters-qwen", "version": "0.6.0" }, - "sha256": "ffd6ed76db07ac79e04edc6cdc183729b1e569cb67015503b93aea50f951b3d2", + "sha256": "691728864183e7bffdfcfc6eb520743b14684f69a404256a9b13f239f7f6ed13", "source": "adapters/typescript/qwen/qwen.fabric-adapter.json" }, "nvidia.fabric.remote-agent": { diff --git a/tests/adapters/test_opencode_adapter.py b/tests/adapters/test_opencode_adapter.py index 109955f72..d32b97a50 100644 --- a/tests/adapters/test_opencode_adapter.py +++ b/tests/adapters/test_opencode_adapter.py @@ -143,6 +143,14 @@ def test_opencode_descriptor_declares_the_minimum_supported_surface(): }, "additionalProperties": False, }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + } + }, "capabilities": { "streaming": False, "cancellation": False, diff --git a/tests/adapters/test_pi_adapter.py b/tests/adapters/test_pi_adapter.py index c5fca8ec5..ac344918c 100644 --- a/tests/adapters/test_pi_adapter.py +++ b/tests/adapters/test_pi_adapter.py @@ -77,6 +77,7 @@ def test_pi_descriptor_declares_the_supported_surface(): assert descriptor["extension_schemas"]["run_result"] == { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"}, }, diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 3288ddc26..50c431c57 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -84,3 +84,17 @@ def test_invalid_resources_fail_clearly( monkeypatch.setattr(catalog, "files", Mock(return_value=resource)) with pytest.raises(error): catalog.get_adapter_descriptor("not.a.bundled.id") + + +def test_every_bundled_descriptor_declares_the_versions_report(bundle): + adapters = bundle["adapters"] + + assert adapters + for adapter_id, entry in adapters.items(): + run_result = entry["descriptor"]["extension_schemas"]["run_result"] + assert run_result["properties"]["nemo_fabric.versions.v1"] == { + "type": "object" + }, adapter_id + assert "nemo_fabric.versions.v1" not in run_result.get("required", []), ( + adapter_id + ) From b852eacacea65056dcf55d02af3806d8e1468ca2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 02:50:09 -0700 Subject: [PATCH 20/74] docs: note that every bundled adapter reports its package version The SDK guide and the integration reference named only the six adapters with a harness version source. The TypeScript common README documents withPackageVersion. Signed-off-by: mnajafian-nv --- adapters/typescript/common/README.md | 2 ++ docs/sdk/python.mdx | 7 ++++--- .../nemo-fabric-integrate/references/results-and-errors.md | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index ed30c0d93..2da8912a4 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -46,3 +46,5 @@ contract. Published package manifests use the registry version instead. ## Model Endpoint Policy `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. + +`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2e540e7bf..6f5a3c383 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1029,9 +1029,10 @@ carry `source`, `collected_by`, and `method`: An adapter reports its facts through the result keys `nemo_fabric.versions.v1` (package version, harness name and version, and served models) and `nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. -The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and -mini-SWE-agent adapters report their package version and their harness name -and version. The Claude Code adapter also reports the served model of the role +Every bundled adapter reports its package version. The Claude Code, Codex, +Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters also report +their harness name and version. The other adapters report those facts with +state `unsupported` and reason `no_source`. The Claude Code adapter also reports the served model of the role it runs: the one model that this invocation's main-thread assistant messages name, skipping subagent messages and Claude Code's `` API-error messages. Two models give `unavailable` with reason `not_attributed`, and none diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 71b4994c7..9fe5bd040 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -52,9 +52,10 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, read state `unsupported` with reason -`key_not_declared` when the adapter does not report them. The Claude Code, -Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report -the harness version, and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without +`key_not_declared` when the adapter does not report them. Every bundled +adapter reports its package version. The Claude Code, Codex, Hermes Agent, +OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, +and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without failing the result. A `what_ran` over its 128 KiB budget can hold only `schema`, `version`, and `what_ran_truncated`, so check for that marker before reading a section. Refer to From 98ba43b14316a47a8709c44380c394ec16f48264 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:21:00 -0700 Subject: [PATCH 21/74] fix: declare the versions report in the Harbor SWE-bench descriptors The Claude and Hermes adapters attach nemo_fabric.versions.v1 to every result, and core rejects a result extension that the descriptor's run_result schema does not declare. The Harbor SWE-bench copies of both descriptors lacked the block, so runs through them failed. A new test finds every checked-in copy of a bundled descriptor and checks that it declares the key. Signed-off-by: mnajafian-nv --- .../claude/claude.fabric-adapter.json | 8 ++++ .../hermes/hermes.fabric-adapter.json | 8 ++++ tests/python/test_adapter_catalog.py | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 50c431c57..2142ebad3 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -98,3 +98,50 @@ def test_every_bundled_descriptor_declares_the_versions_report(bundle): assert "nemo_fabric.versions.v1" not in run_result.get("required", []), ( adapter_id ) + + +SKIPPED_DIRECTORIES = {"node_modules", "target", "build", "dist"} + + +def _checked_in_descriptors(repo_root: Path): + for directory, subdirectories, filenames in os.walk(repo_root): + subdirectories[:] = [ + name + for name in subdirectories + if not name.startswith(".") and name not in SKIPPED_DIRECTORIES + ] + for filename in filenames: + if filename.endswith(".fabric-adapter.json"): + yield Path(directory) / filename + + +def test_every_copy_of_a_bundled_descriptor_declares_the_versions_report( + bundle, repo_root: Path +): + # Bundled adapters always attach the versions report, and core fails a run + # whose descriptor does not declare it, so copies used by examples and the + # CLI presets must declare it too. + bundled_ids = set(bundle["adapters"]) + bundled_modules = { + entry["descriptor"]["runner"]["module"] + for entry in bundle["adapters"].values() + if "module" in entry["descriptor"]["runner"] + } + sources = { + (repo_root / entry["source"]).resolve() for entry in bundle["adapters"].values() + } + + copies = [] + for path in _checked_in_descriptors(repo_root): + descriptor = json.loads(path.read_bytes()) + if ( + descriptor.get("adapter_id") in bundled_ids + or descriptor.get("runner", {}).get("module") in bundled_modules + ): + copies.append(path) + run_result = descriptor.get("extension_schemas", {}).get("run_result", {}) + assert run_result.get("properties", {}).get("nemo_fabric.versions.v1") == { + "type": "object" + }, path + + assert {path.resolve() for path in copies} - sources From 2d43065237ae3a941dc46b221aadd150b7691c68 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:28:08 -0700 Subject: [PATCH 22/74] fix: report the Hermes version that hermes --version prints display_version drops the commit and the dirty marker of a source checkout, so two different Hermes builds could report the same version. Report derived_version, which Hermes's own --version output uses. Signed-off-by: mnajafian-nv --- .../src/nemo_fabric_adapters/hermes/adapter.py | 8 +++++--- tests/adapters/test_hermes_adapter.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 0f8fe2e18..4b978dc13 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -48,13 +48,15 @@ def hermes_version() -> str: """The running Hermes Agent's version from its version call. - ``get_version_info()`` can run several ``git`` commands, so callers run it - through a :class:`provenance.BlockingProbe`. + Returns the same ``derived_version`` that ``hermes --version`` prints, which + keeps the commit and dirty marker of a source checkout. ``get_version_info()`` + can run several ``git`` commands, so callers run it through a + :class:`provenance.BlockingProbe`. """ from hermes_cli.version_info import get_version_info - return get_version_info().display_version + return get_version_info().derived_version async def probe_hermes_version( diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 420ca7018..959895ce6 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1754,7 +1754,7 @@ async def test_hermes_reports_the_version_its_version_call_returns( report = result.extensions["nemo_fabric.versions.v1"] assert report["harness"] == { "name": _reported("hermes-agent"), - "version": _reported(get_version_info().display_version), + "version": _reported(get_version_info().derived_version), } assert report["adapter"]["package_version"]["value"] == "0.6.0" @@ -1812,3 +1812,13 @@ def blocking() -> str: assert first == second == _version_state("collection_failed", "timeout") assert calls == ["called"] + + +def test_hermes_version_keeps_the_commit_and_dirty_marker(monkeypatch): + version_info = ModuleType("hermes_cli.version_info") + version_info.get_version_info = lambda: MagicMock( # type: ignore[attr-defined] + derived_version="0.21.6+3.gabc1234.dirty", display_version="0.21.6+3" + ) + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_info) + + assert adapter.hermes_version() == "0.21.6+3.gabc1234.dirty" From a021cdf9f63c2ecf7636c671cea10ac457de74c6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:28:40 -0700 Subject: [PATCH 23/74] docs: keep only the API reference pages this branch changes The regenerated references also picked up changes that come from the base branch, such as new FabricError variants, the telemetry field type and page positions. Restore those pages so the branch carries only the what_ran pages for the new report keys. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 66 +------------------ .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 8 +-- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- 41 files changed, 45 insertions(+), 109 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 1a5cbd1a3..52a3b6f57 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 129 +position: 125 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index b89e437a4..26a286a0c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 130 +position: 126 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 3048bc293..2c89a0e39 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 131 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 595a1fd29..fa7255361 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 132 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index b5f664a1a..ea299ed94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 133 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 975080702..75900423a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,70 +497,6 @@ Environment provider. Adapter kind. -### `UnsupportedEnvironmentRelease` - -
- -An environment provider does not implement release. - -#### Fields - -##### `provider: String` - -Environment provider. - -### `EnvironmentProviderOperation` - -
- -An environment provider process could not complete an operation. - -#### Fields - -##### `provider: String` - -Environment provider. - -##### `operation: String` - -Lifecycle operation. - -##### `code: String` - -Stable provider or host error code. - -##### `message: String` - -Human-readable message that excludes credentials and raw diagnostics. - -### `EnvironmentInUse` - -
- -A runtime lifecycle operation is still active or has an uncertain outcome. - -#### Fields - -##### `environment_id: String` - -Environment whose runtime operation is unresolved. - -##### `runtime_id: String` - -Runtime identity, or `unconfirmed` if the start response was lost. - -### `EnvironmentReleaseInProgress` - -
- -An environment release is already in progress. - -#### Fields - -##### `environment_id: String` - -Environment whose lifecycle is being released or detached. - ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index cdaccd776..eb802320b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 24 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index 83505d3b2..cca22a793 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 25 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index 5e337a0e0..f2d130b85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 26 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index eb3f8fe51..1c3f839ae 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 27 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index ee69f77c3..86fddd94c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 28 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index d02adc8bf..a3a933cdf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 29 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index f7acd827c..97954ce13 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 30 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index 310cf8983..c5050a301 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 31 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index 494089eb6..d8ff25825 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 33 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index 1fa28e020..b81268897 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 34 +position: 32 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index 28b005c5d..e1471de73 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 35 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index 49e08a5bb..a59f0067c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 37 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index 72ca1b83e..b356d2db0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 39 +position: 36 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 331c08b3c..55a5a47cb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 40 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 5acac20bd..547b47760 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 43 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index ce8835391..71febb037 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 44 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 67102fa82..55d47378d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. +Workspace visible to the harness runtime. ### `artifacts: Option` -Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. +Artifact root visible to the harness runtime. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 376449e6a..9468739a0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 6 +position: 5 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5ec70a8ad..5330e632a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 7 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 1df9629e6..3f3ac0d42 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 8 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 96e965402..026b2de72 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 9 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index f3dbf5424..94b2e70ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 10 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 53b0a6972..845afed45 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 11 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index b5a3b7fcb..49b4c9c49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 12 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index a178c8113..8ad3827d5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 13 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 4a7f2e2f9..2ed1e5c62 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 14 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index d0200756d..8dcc6950f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 15 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1ee527133..9dfc3f1be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,14 +2,14 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 16 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -63,9 +63,9 @@ Normalized invocation usage when reported by the adapter target. Artifacts produced or collected by this run. -### `telemetry: Vec` +### `telemetry: Option` -Ordered telemetry references when available. +Telemetry reference when available. ### `events: Vec` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 625527702..28af565b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 18 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index 0671b565e..b0392902d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 19 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 890abc95b..1525ed89d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 20 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 864be9b27..6f16e387b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 17 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index dead906b7..8c36ff5de 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 21 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index 4a249074a..e28440353 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 22 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 94338baeb..cd66e4025 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 23 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 12f8f9ebd714033e0f7f699fe257be5a2a1d5a72 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:29:46 -0700 Subject: [PATCH 24/74] fix: let the version probe recover when its thread cannot start BlockingProbe marked a probe as running before starting its thread. If the start raised, nothing cleared the mark, and every later probe in the runtime timed out at once. Clear the mark and re-raise. Signed-off-by: mnajafian-nv --- .../src/nemo_fabric_adapters/common/provenance.py | 10 +++++++++- tests/adapters/test_adapters_common_provenance.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py index b16a213b9..29095edd5 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -227,7 +227,15 @@ def run() -> None: return self._running.set() - threading.Thread(target=run, name="nemo-fabric-version-probe", daemon=True).start() + try: + threading.Thread( + target=run, name="nemo-fabric-version-probe", daemon=True + ).start() + except Exception: + # No thread runs to clear the flag, so clear it here or every later + # call would time out at once. + self._running.clear() + raise # On timeout, wait_for cancels only this future. The thread keeps # running, and its late outcome is dropped. return await asyncio.wait_for(outcome, timeout) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index fada78a86..3e9f00453 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -168,6 +168,20 @@ def fail() -> str: await probe.call(fail) +async def test_blocking_probe_recovers_when_its_thread_cannot_start(monkeypatch): + probe = provenance.BlockingProbe() + + def refuse_to_start(_thread: threading.Thread) -> None: + raise RuntimeError("can't start new thread") + + with monkeypatch.context() as patch: + patch.setattr(threading.Thread, "start", refuse_to_start) + with pytest.raises(RuntimeError, match="can't start new thread"): + await probe.call(lambda: "1.0.0", timeout=1) + + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + def test_package_version_report_marks_harness_and_served_models_no_source(): report = provenance.package_version_report( "nemo-fabric-adapters-common", ["judge", "default"] From bd48fc02be0c5d18884313f092dfb8d58fc58a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:35:17 -0700 Subject: [PATCH 25/74] fix: report hard-coded harness names with the adapter_report method Each adapter knows its harness name as a constant, so no collection technique stands behind it. The names reused the version's method, such as harness_init_message or package_metadata, which claimed a source the name never came from. Report them with adapter_report, as the design requires for an adapter fact with no technique behind it. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/claude/adapter.py | 4 +++- .../src/nemo_fabric_adapters/codex/adapter.py | 2 +- .../deepagents/adapter.py | 2 +- .../nemo_fabric_adapters/hermes/adapter.py | 2 +- .../mini_swe_agent/adapter.py | 2 +- .../nemo_fabric_adapters/openclaw/adapter.py | 2 +- tests/adapters/test_claude_adapter.py | 2 +- tests/adapters/test_codex_adapter.py | 19 +++++++++++++++++++ tests/adapters/test_hermes_adapter.py | 2 +- tests/adapters/test_openclaw.py | 2 +- tests/e2e/test_deepagents.py | 2 +- tests/e2e/test_mini_swe_agent.py | 2 +- 12 files changed, 32 insertions(+), 11 deletions(-) diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 2a1f68480..c334b3427 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -1292,7 +1292,9 @@ def _versions_report( served[_selected_model_role(config)] = served_model_fact(messages) return provenance.versions_report( package_version=self._package_version, - harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_INIT_MESSAGE), + harness_name=provenance.known_fact( + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT + ), harness_version=self._harness_version, served_models=served, ) diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index be0143a43..87868c54d 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -1114,7 +1114,7 @@ def versions_report(config: AgentConfig, harness_version: dict[str, Any]) -> dic return provenance.versions_report( package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), - harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_HARNESS_API), + harness_name=provenance.known_fact(_HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT), harness_version=harness_version, served_models={role: provenance.no_source_fact() for role in config.models}, ) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 721e31221..65774bc6e 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -817,7 +817,7 @@ async def invoke( _PACKAGE_DISTRIBUTION ), harness_name=provenance.known_fact( - _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + _HARNESS_DISTRIBUTION, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={ diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 4b978dc13..0bbc07589 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -433,7 +433,7 @@ def clear_active_invoke_task( _PACKAGE_DISTRIBUTION ), harness_name=provenance.known_fact( - _HARNESS_NAME, _METHOD_HARNESS_API + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={ diff --git a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py index 148e924d0..57b6c0c9a 100644 --- a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py +++ b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py @@ -197,7 +197,7 @@ async def invoke( _PACKAGE_DISTRIBUTION ), harness_name=provenance.known_fact( - _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + _HARNESS_DISTRIBUTION, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={ diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index 719bf2abb..1b1ddf207 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -1413,7 +1413,7 @@ def _versions_extensions(self) -> dict[str, Any]: provenance.VERSIONS_REPORT_KEY: provenance.versions_report( package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), harness_name=provenance.known_fact( - _HARNESS_NAME, self._harness_version["method"] + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={role: provenance.no_source_fact() for role in models}, diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index 7ac4ecb54..cd40aa1dd 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -2187,7 +2187,7 @@ async def test_claude_reports_the_cli_version_from_its_init_message( "package_version": reported(package_version, "package_metadata") } assert report["harness"] == { - "name": reported("claude-code", "harness_init_message"), + "name": reported("claude-code", "adapter_report"), "version": reported("2.1.294", "harness_init_message"), } diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 144309aa1..01ad47577 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2165,3 +2165,22 @@ async def test_codex_cli_version_read_degrades_to_a_state(read, expected): assert await adapter.read_cli_version(thread, timeout=0.1) == expected assert time.monotonic() - started < 1 + + +def test_codex_versions_report_names_the_harness_as_an_adapter_report(): + config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "gpt-test"}}} + ) + harness_version = _version_fact("unavailable", "not_collected") + + report = adapter.versions_report(config, harness_version) + + assert report["harness"] == { + "name": { + "value": "codex", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + "version": harness_version, + } diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 959895ce6..6b9892e73 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1753,7 +1753,7 @@ async def test_hermes_reports_the_version_its_version_call_returns( report = result.extensions["nemo_fabric.versions.v1"] assert report["harness"] == { - "name": _reported("hermes-agent"), + "name": {**_reported("hermes-agent"), "method": "adapter_report"}, "version": _reported(get_version_info().derived_version), } assert report["adapter"]["package_version"]["value"] == "0.6.0" diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index 9c31c94c4..bc1ebe23f 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -293,7 +293,7 @@ async def test_openclaw_runtime_generates_config_invokes_and_cleans_up( "value": "openclaw", "source": "reported", "collected_by": "adapter", - "method": "harness_probe", + "method": "adapter_report", }, "version": { "value": "2026.9.4", diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index ce96f6511..81ea30aee 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -57,7 +57,7 @@ async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): # Core read the adapter's versions report into what_ran. for result in (first, second): assert result.what_ran["harness"] == { - "name": _reported("deepagents"), + "name": {**_reported("deepagents"), "method": "adapter_report"}, "version": _reported(importlib.metadata.version("deepagents")), }, results assert result.what_ran["adapter"]["package_version"] == _reported( diff --git a/tests/e2e/test_mini_swe_agent.py b/tests/e2e/test_mini_swe_agent.py index f557de688..cfac06e35 100644 --- a/tests/e2e/test_mini_swe_agent.py +++ b/tests/e2e/test_mini_swe_agent.py @@ -95,7 +95,7 @@ async def test_mini_swe_agent_plans_diagnoses_and_runs( assert result.output["usage"]["api_calls"] == 1 what_ran = result.what_ran assert what_ran["harness"] == { - "name": _reported("mini-swe-agent"), + "name": {**_reported("mini-swe-agent"), "method": "adapter_report"}, "version": _reported(importlib.metadata.version("mini-swe-agent")), } assert what_ran["adapter"]["package_version"] == _reported( From bc4d30fbd1095fd1c02e03ea781d4924d0ef1da5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:37:50 -0700 Subject: [PATCH 26/74] feat: report Cline's served model as not attributed The Cline SDK's result names a model, but nothing yet shows that it is the model that served the invocation, so the design reads Cline's served model as unavailable with reason not_attributed rather than unsupported. packageVersionReport and withPackageVersion take an optional served-model fact for one role, and the Cline entry point passes it for the role Cline runs. Signed-off-by: mnajafian-nv --- adapters/typescript/cline/src/cli.ts | 17 ++++++-- adapters/typescript/cline/src/cline-sdk.ts | 16 +++++-- .../typescript/cline/test/cline-sdk.test.mjs | 10 ++++- adapters/typescript/common/README.md | 2 +- adapters/typescript/common/src/index.ts | 2 + adapters/typescript/common/src/provenance.ts | 43 ++++++++++++++----- .../common/test/provenance.test.mjs | 37 ++++++++++++++++ docs/sdk/python.mdx | 5 ++- .../references/results-and-errors.md | 3 +- 9 files changed, 115 insertions(+), 20 deletions(-) diff --git a/adapters/typescript/cline/src/cli.ts b/adapters/typescript/cline/src/cli.ts index 79b2a3bd2..60f1237a8 100644 --- a/adapters/typescript/cline/src/cli.ts +++ b/adapters/typescript/cline/src/cli.ts @@ -2,12 +2,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; +import { missingFact, serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { - const [{ ClineSdkSessionFactory }, { ClineAdapterRuntime }] = await Promise.all([ + const [{ ClineSdkSessionFactory, selectedModelRole }, { ClineAdapterRuntime }] = await Promise.all([ import("./cline-sdk.js"), import("./runtime.js"), ]); - return withPackageVersion(new ClineAdapterRuntime(new ClineSdkSessionFactory()), new URL("../package.json", import.meta.url)); + // The SDK's result names a model, but nothing yet shows that it is the model + // that served this invocation, so the selected role reads not_attributed. + return withPackageVersion( + new ClineAdapterRuntime(new ClineSdkSessionFactory()), + new URL("../package.json", import.meta.url), + (input) => { + const role = selectedModelRole(input.config); + return role === undefined + ? undefined + : { role, fact: missingFact("unavailable", "not_attributed", "harness_api") }; + }, + ); }); diff --git a/adapters/typescript/cline/src/cline-sdk.ts b/adapters/typescript/cline/src/cline-sdk.ts index 5c8ca5e5f..e9a99e14d 100644 --- a/adapters/typescript/cline/src/cline-sdk.ts +++ b/adapters/typescript/cline/src/cline-sdk.ts @@ -134,12 +134,22 @@ async function loadClineSdk(): Promise { return sdk; } +/** + * The model role Cline runs: `default`, or the only configured role. Returns + * `undefined` when no role or several roles without `default` are configured. + */ +export function selectedModelRole(config: AgentConfig): string | undefined { + const roles = Object.keys(config.models ?? {}); + if (roles.includes("default")) return "default"; + return roles.length === 1 ? roles[0] : undefined; +} + function selectModel(config: AgentConfig): AgentModelConfig { - const entries = Object.entries(config.models ?? {}); - if (entries.length === 0) { + if (Object.keys(config.models ?? {}).length === 0) { throw new LifecycleError("cline_model_required", "The Cline adapter requires one configured model"); } - const selected = config.models?.default ?? (entries.length === 1 ? entries[0]?.[1] : undefined); + const role = selectedModelRole(config); + const selected = role === undefined ? undefined : config.models?.[role]; if (selected === undefined) { throw new LifecycleError( "cline_model_ambiguous", diff --git a/adapters/typescript/cline/test/cline-sdk.test.mjs b/adapters/typescript/cline/test/cline-sdk.test.mjs index 2aac26163..b5a44a327 100644 --- a/adapters/typescript/cline/test/cline-sdk.test.mjs +++ b/adapters/typescript/cline/test/cline-sdk.test.mjs @@ -7,7 +7,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; -import { ClineSdkSessionFactory } from "../dist/cline-sdk.js"; +import { ClineSdkSessionFactory, selectedModelRole } from "../dist/cline-sdk.js"; function input(overrides = {}) { return { @@ -444,3 +444,11 @@ test("rejects unsupported MCP authentication and tool filters", async () => { (error) => error.code === "cline_mcp_blocklist_unsupported", ); }); + +test("the selected model role is default, or the only configured role", () => { + assert.equal(selectedModelRole({ models: { default: {}, judge: {} } }), "default"); + assert.equal(selectedModelRole({ models: { primary: {} } }), "primary"); + assert.equal(selectedModelRole({ models: { primary: {}, judge: {} } }), undefined); + assert.equal(selectedModelRole({ models: {} }), undefined); + assert.equal(selectedModelRole({}), undefined); +}); diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 2da8912a4..700ae129c 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -47,4 +47,4 @@ contract. Published package manifests use the registry version instead. `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. -`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. +`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional third argument, `servedModel(input)`, returns the `role` and `fact` of one role's served model from the start input, such as `missingFact("unavailable", "not_attributed", "harness_api")`; every other role keeps `no_source`. diff --git a/adapters/typescript/common/src/index.ts b/adapters/typescript/common/src/index.ts index 94cee2995..7d8d3f909 100644 --- a/adapters/typescript/common/src/index.ts +++ b/adapters/typescript/common/src/index.ts @@ -19,7 +19,9 @@ export { insecureHttpModelHosts, modelEndpointAllowed } from "./model-endpoint.j export { VERSIONS_REPORT_KEY, boundedText, + missingFact, packageVersionFact, packageVersionReport, withPackageVersion, + type ServedModelFact, } from "./provenance.js"; diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts index d0a4ef35d..8e084519d 100644 --- a/adapters/typescript/common/src/provenance.ts +++ b/adapters/typescript/common/src/provenance.ts @@ -36,10 +36,17 @@ export function boundedText(text: string): string { return `${chars.slice(0, TRUNCATED_PREFIX_CHARS).join("")}~${digest.slice(0, TRUNCATED_DIGEST_HEX_DIGITS)}`; } -function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { +/** A reported fact without a value. */ +export function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { return {state, reason, source: "reported", collected_by: "adapter", method}; } +/** The served-model fact an adapter reports for one model role. */ +export interface ServedModelFact { + role: string; + fact: JsonObject; +} + /** * The `version` field of the adapter's own `package.json`, read synchronously. * A file that can't be read or parsed reads `unavailable` with reason @@ -60,30 +67,46 @@ export function packageVersionFact(packageJson: URL): JsonObject { /** * The report of an adapter that has a source only for its package version. - * The harness name and version and each role's served model read - * `unsupported` with reason `no_source`. Like core, it keeps at most 16 roles - * in sorted order and skips a role longer than 128 characters. + * The harness name and version read `unsupported` with reason `no_source`, and + * so does each role's served model, except the role that `servedModel` names, + * which gets its fact. Like core, it keeps at most 16 roles in sorted order + * and skips a role longer than 128 characters. */ -export function packageVersionReport(packageVersion: JsonObject, modelRoles: readonly string[]): JsonObject { +export function packageVersionReport( + packageVersion: JsonObject, + modelRoles: readonly string[], + servedModel?: ServedModelFact, +): JsonObject { const noSource = () => missingFact("unsupported", "no_source"); const roles = modelRoles.filter((role) => Array.from(role).length <= MAX_KEY_CHARS).sort().slice(0, MAX_MODEL_ROLES); return { adapter: {package_version: packageVersion}, harness: {name: noSource(), version: noSource()}, - models: Object.fromEntries(roles.map((role) => [role, {served: noSource()}])), + models: Object.fromEntries( + roles.map((role) => [role, {served: role === servedModel?.role ? {...servedModel.fact} : noSource()}]), + ), }; } /** * Wrap `runtime` so each result reports the adapter package version read from - * `packageJson` when the runtime starts. Results and errors are otherwise - * unchanged. + * `packageJson` when the runtime starts. `servedModel`, when given, picks the + * served-model fact of one role from the start input. Results and errors are + * otherwise unchanged. */ -export function withPackageVersion(runtime: AdapterRuntime, packageJson: URL): AdapterRuntime { +export function withPackageVersion( + runtime: AdapterRuntime, + packageJson: URL, + servedModel?: (input: AdapterStartInput) => ServedModelFact | undefined, +): AdapterRuntime { let report: JsonObject | undefined; return { async start(input: AdapterStartInput): Promise { - report = packageVersionReport(packageVersionFact(packageJson), Object.keys(input.config.models ?? {})); + report = packageVersionReport( + packageVersionFact(packageJson), + Object.keys(input.config.models ?? {}), + servedModel?.(input), + ); await runtime.start(input); }, async invoke(request: AgentRunRequest, context: RuntimeContext): Promise { diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 178e305f1..9746b3a97 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -11,7 +11,9 @@ import { pathToFileURL } from "node:url"; import { VERSIONS_REPORT_KEY, boundedText, + missingFact, packageVersionFact, + packageVersionReport, withPackageVersion, } from "../dist/index.js"; @@ -70,3 +72,38 @@ test("the wrapper adds the versions report and keeps the result", async () => { await rm(directory, {recursive: true, force: true}); } }); + +test("a served-model override reaches only the role it names", () => { + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + assert.deepEqual(packageVersionReport(packageVersion, ["judge", "default"], {role: "default", fact: notAttributed}).models, { + default: {served: notAttributed}, + judge: {served: noSource}, + }); + // A role that is not configured adds no entry. + assert.deepEqual(packageVersionReport(packageVersion, ["default"], {role: "absent", fact: notAttributed}).models, { + default: {served: noSource}, + }); +}); + +test("the wrapper picks the served-model role from the start input", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + const runtime = withPackageVersion({ + async start() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + async stop() {}, + }, packageJson, (input) => ({role: Object.keys(input.config.models)[0], fact: notAttributed})); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {primary: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY].models, {primary: {served: notAttributed}}); + } finally { + await rm(directory, {recursive: true, force: true}); + } +}); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6f5a3c383..f181293e2 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1036,7 +1036,10 @@ state `unsupported` and reason `no_source`. The Claude Code adapter also reports it runs: the one model that this invocation's main-thread assistant messages name, skipping subagent messages and Claude Code's `` API-error messages. Two models give `unavailable` with reason `not_attributed`, and none -gives `unavailable` with reason `empty_response`. No bundled adapter reports +gives `unavailable` with reason `empty_response`. The Cline adapter reports the +served model of the role it runs as `unavailable` with reason `not_attributed`, +because nothing yet shows that the model its SDK names served this invocation. +No bundled adapter reports MCP runtime facts yet. A reported fact whose key the adapter does not declare has state `unsupported` diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 9fe5bd040..08979aabf 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,8 @@ harness version and the served model, read state `unsupported` with reason `key_not_declared` when the adapter does not report them. Every bundled adapter reports its package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, -and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without +and the Claude Code adapter reports the served model. The Cline adapter reports +the served model as `unavailable` with reason `not_attributed`. A malformed `what_ran` reads as `None` without failing the result. A `what_ran` over its 128 KiB budget can hold only `schema`, `version`, and `what_ran_truncated`, so check for that marker before reading a section. Refer to From ea23194210898ada982b2711ff545a4102dd1ed3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:38:02 -0700 Subject: [PATCH 27/74] docs: narrow when an adapter fact reads key_not_declared The integrate skill said key_not_declared applies whenever an adapter does not report a fact. Core uses it only when the descriptor does not declare the report key; a declared key that the result omits reads key_omitted, and a declared key that omits the fact reads fact_omitted. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 08979aabf..6abc28ea5 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -52,7 +52,10 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, read state `unsupported` with reason -`key_not_declared` when the adapter does not report them. Every bundled +`key_not_declared` when the adapter descriptor does not declare their report +key. When the descriptor declares the key, a result that omits the key reads +`unavailable` with reason `key_omitted`, and a key that omits the fact reads +`unavailable` with reason `fact_omitted`. Every bundled adapter reports its package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, and the Claude Code adapter reports the served model. The Cline adapter reports From fbddfd8b169217fb36c51a19e3b925c97b2ca955 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 04:54:35 -0700 Subject: [PATCH 28/74] fix: read no_source for Hermes only when its version call is missing The probe mapped any ImportError to no_source, including one raised inside get_version_info(), and labeled the fact harness_api. Only a failed import of the version call now reads no_source, through the shared no_source fact with method adapter_report. An ImportError inside the call reads collection_failed with reason no_result like any other error. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/hermes/adapter.py | 18 +++++-- tests/adapters/test_hermes_adapter.py | 50 +++++++++++++++---- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 0bbc07589..b051b2138 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -45,16 +45,25 @@ hermes_mcp_server_config = configuration.hermes_mcp_server_config +class HermesVersionCallMissingError(Exception): + """The installed Hermes Agent predates its version call.""" + + def hermes_version() -> str: """The running Hermes Agent's version from its version call. Returns the same ``derived_version`` that ``hermes --version`` prints, which keeps the commit and dirty marker of a source checkout. ``get_version_info()`` can run several ``git`` commands, so callers run it through a - :class:`provenance.BlockingProbe`. + :class:`provenance.BlockingProbe`. Raises + :class:`HermesVersionCallMissingError` only when the version call itself + can't be imported; an error raised inside the call propagates unchanged. """ - from hermes_cli.version_info import get_version_info + try: + from hermes_cli.version_info import get_version_info + except ImportError as error: + raise HermesVersionCallMissingError from error return get_version_info().derived_version @@ -70,9 +79,8 @@ async def probe_hermes_version( version = await probe.call(version_call, timeout=timeout) except TimeoutError: return provenance.missing_fact("collection_failed", "timeout", _METHOD_HARNESS_API) - except ImportError: - # A Hermes Agent older than its version call. - return provenance.missing_fact("unsupported", "no_source", _METHOD_HARNESS_API) + except HermesVersionCallMissingError: + return provenance.no_source_fact() except Exception: # Provenance never fails a run, whatever Hermes's version call raises. LOGGER.exception("Hermes version call failed") diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 6b9892e73..04bfe3336 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1759,10 +1759,6 @@ async def test_hermes_reports_the_version_its_version_call_returns( assert report["adapter"]["package_version"]["value"] == "0.6.0" -def _raise_import_error() -> str: - raise ImportError("hermes_cli.version_info") - - def _raise_runtime_error() -> str: raise RuntimeError("malformed install stamp") @@ -1776,11 +1772,6 @@ def _raise_runtime_error() -> str: _version_state("unavailable", "empty_response"), id="unknown", ), - pytest.param( - _raise_import_error, - _version_state("unsupported", "no_source"), - id="no-version-call", - ), pytest.param( _raise_runtime_error, _version_state("collection_failed", "no_result"), @@ -1794,6 +1785,47 @@ async def test_hermes_version_probe_maps_each_outcome(version_call, expected): assert await adapter.probe_hermes_version(probe, version_call) == expected +def _version_module_raising_import_error() -> ModuleType: + version_info = ModuleType("hermes_cli.version_info") + + def get_version_info() -> object: + raise ImportError("a module the version call needs") + + version_info.get_version_info = get_version_info # type: ignore[attr-defined] + return version_info + + +@pytest.mark.parametrize( + ("version_module", "expected"), + [ + # None in sys.modules makes the import itself raise ImportError. + pytest.param( + None, + { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + id="no-version-call", + ), + pytest.param( + _version_module_raising_import_error(), + _version_state("collection_failed", "no_result"), + id="import-error-inside-the-call", + ), + ], +) +async def test_hermes_version_probe_reads_no_source_only_without_a_version_call( + monkeypatch, version_module, expected +): + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_module) + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe) == expected + + async def test_hermes_version_probe_times_out_and_starts_no_second_call(): release = threading.Event() calls: list[str] = [] From 0dca1d2df257a5d8295dd68c5640660d6011df17 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 04:56:09 -0700 Subject: [PATCH 29/74] test: require descriptor copies to match their bundled source byte for byte The copy check read every file on disk, skipped build, dist and target folders, and looked only for the versions key. It now lists tracked descriptors with git ls-files, walking the tree only outside a git checkout, matches copies by adapter ID, runner module or runner script, and requires each to equal its source byte for byte, as the CLI's packaged-descriptor test already does for its own copies. Signed-off-by: mnajafian-nv --- tests/python/test_adapter_catalog.py | 90 ++++++++++++++++++---------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 2142ebad3..3b464fefe 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -5,6 +5,7 @@ import json import os +import subprocess from importlib.resources import files from pathlib import Path from unittest.mock import Mock @@ -100,48 +101,75 @@ def test_every_bundled_descriptor_declares_the_versions_report(bundle): ) -SKIPPED_DIRECTORIES = {"node_modules", "target", "build", "dist"} - - -def _checked_in_descriptors(repo_root: Path): +def _descriptors_on_disk(repo_root: Path) -> list[Path]: + found = [] for directory, subdirectories, filenames in os.walk(repo_root): + # Installed dependencies are not part of the source tree. subdirectories[:] = [ - name - for name in subdirectories - if not name.startswith(".") and name not in SKIPPED_DIRECTORIES + name for name in subdirectories if name not in {".git", "node_modules"} ] - for filename in filenames: - if filename.endswith(".fabric-adapter.json"): - yield Path(directory) / filename + found.extend( + Path(directory) / filename + for filename in filenames + if filename.endswith(".fabric-adapter.json") + ) + return found + +def _checked_in_descriptors(repo_root: Path) -> list[Path]: + """Every tracked descriptor, or every descriptor on disk outside a git checkout.""" -def test_every_copy_of_a_bundled_descriptor_declares_the_versions_report( + try: + listed = subprocess.run( + ["git", "-C", str(repo_root), "ls-files", "-z", "*.fabric-adapter.json"], + capture_output=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError): + return _descriptors_on_disk(repo_root) + return [ + repo_root / name + for name in listed.decode("utf-8").split("\0") + if name and (repo_root / name).is_file() + ] + + +def test_every_checked_in_copy_of_a_bundled_descriptor_matches_its_source( bundle, repo_root: Path ): # Bundled adapters always attach the versions report, and core fails a run - # whose descriptor does not declare it, so copies used by examples and the - # CLI presets must declare it too. - bundled_ids = set(bundle["adapters"]) - bundled_modules = { - entry["descriptor"]["runner"]["module"] - for entry in bundle["adapters"].values() - if "module" in entry["descriptor"]["runner"] - } - sources = { - (repo_root / entry["source"]).resolve() for entry in bundle["adapters"].values() - } + # whose result carries an undeclared key, so a copy used by an example or a + # CLI preset must stay byte-identical to the descriptor it copies, as the + # CLI's packaged-descriptor test already requires for its own copies. + sources_by_id: dict[str, Path] = {} + sources_by_module: dict[str, Path] = {} + sources_by_script: dict[str, list[Path]] = {} + for adapter_id, entry in bundle["adapters"].items(): + source = repo_root / entry["source"] + runner = entry["descriptor"]["runner"] + sources_by_id[adapter_id] = source + if "module" in runner: + sources_by_module[runner["module"]] = source + if "script" in runner: + sources_by_script.setdefault(runner["script"], []).append(source) copies = [] for path in _checked_in_descriptors(repo_root): descriptor = json.loads(path.read_bytes()) - if ( - descriptor.get("adapter_id") in bundled_ids - or descriptor.get("runner", {}).get("module") in bundled_modules - ): - copies.append(path) - run_result = descriptor.get("extension_schemas", {}).get("run_result", {}) - assert run_result.get("properties", {}).get("nemo_fabric.versions.v1") == { - "type": "object" - }, path + runner = descriptor.get("runner", {}) + if descriptor.get("adapter_id") in sources_by_id: + candidates = [sources_by_id[descriptor["adapter_id"]]] + elif runner.get("module") in sources_by_module: + candidates = [sources_by_module[runner["module"]]] + elif runner.get("script") in sources_by_script: + # Several TypeScript adapters share one script path. + candidates = sources_by_script[runner["script"]] + else: + continue + copies.append(path) + assert any(path.read_bytes() == source.read_bytes() for source in candidates), ( + f"{path} drifted from {[str(source) for source in candidates]}" + ) + sources = {path.resolve() for path in sources_by_id.values()} assert {path.resolve() for path in copies} - sources From 14ff8c6a7ac8fb689ca4a86a0dae1cb2221c2aa7 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 04:56:37 -0700 Subject: [PATCH 30/74] docs: document the served-model argument of the TypeScript report helpers The common package README described withPackageVersion's callback but not packageVersionReport's third parameter, the ServedModelFact type, or missingFact, which the Cline adapter uses to build its fact. Signed-off-by: mnajafian-nv --- adapters/typescript/common/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 700ae129c..93cb2803a 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -47,4 +47,8 @@ contract. Published package manifests use the registry version instead. `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. -`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional third argument, `servedModel(input)`, returns the `role` and `fact` of one role's served model from the start input, such as `missingFact("unavailable", "not_attributed", "harness_api")`; every other role keeps `no_source`. +`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `no_source` for every role. + +`ServedModelFact` is `{role: string; fact: JsonObject}`: the served-model fact for one model role. Build a fact without a value with `missingFact(state, reason, method)`, for example `missingFact("unavailable", "not_attributed", "harness_api")`; `method` defaults to `adapter_report`. + +`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. From 4531f8aa1def7519bf31b2e70b2e01a96b494d52 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:56:01 -0700 Subject: [PATCH 31/74] fix: keep plan MCP servers ahead of harness entries under the caps The entry cap kept the first 32 servers in sorted order whatever their origin, and the byte budget dropped the last one. A report naming servers that sort early could therefore push plan servers, and the endpoint facts core owns for them, out of what_ran. Plan servers now take their places first and harness entries fill only the places left, and the byte budget drops harness entries before plan servers. Both cases are counted in servers_truncated as before. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 188 +++++++++++++++++- .../nemo-fabric-core/what-ran/index.mdx | 2 +- docs/sdk/python.mdx | 9 +- 3 files changed, 185 insertions(+), 14 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index a10a21695..fdddf0ec7 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -16,9 +16,11 @@ //! above are never cut. //! //! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in -//! sorted key order. An entry whose key is longer than 128 characters is -//! dropped. While the compact JSON of the whole object is larger than -//! 128 KiB, MCP server entries are dropped from the end of the sorted order, +//! sorted key order. `plan` servers take their places first, and servers +//! only the harness reports fill the places left. An entry whose key is +//! longer than 128 characters is dropped. While the compact JSON of the +//! whole object is larger than 128 KiB, MCP server entries are dropped from +//! the end of the sorted order, `harness` entries before `plan` entries, //! then model roles. Dropped entries are counted in a [`TruncatedEntries`] //! marker next to the map: `models_truncated` and `mcp.servers_truncated`. //! If the object is still too large with no MCP server and no model role, @@ -593,14 +595,15 @@ impl WhatRan { } /// Keep the compact JSON of the whole object within 128 KiB. While it - /// is larger, drop the last MCP server in sorted key order, then the - /// last model role, counting each in its map's marker with reason - /// `byte_budget`. With neither left and the object still too large, - /// replace it with [`WhatRan::minimal`]. + /// is larger, drop the last MCP server in sorted key order, a server of + /// any other origin before a `plan` server, then the last model role, + /// counting each in its map's marker with reason `byte_budget`. With + /// neither left and the object still too large, replace it with + /// [`WhatRan::minimal`]. fn apply_byte_budget(&mut self) { while self.serialized_len() > MAX_WHAT_RAN_BYTES { if let Some(mcp) = self.mcp.as_mut() - && mcp.servers.pop_last().is_some() + && pop_last_server(&mut mcp.servers).is_some() { mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); } else if let Some(models) = self.models.as_mut() @@ -623,8 +626,7 @@ impl WhatRan { self.models_truncated = merged_marker(self.models_truncated.take(), dropped); } if let Some(mcp) = self.mcp.as_mut() { - let (kept, dropped) = - bounded_entries(&mcp.servers, MAX_MCP_SERVERS, McpServerFacts::clone); + let (kept, dropped) = bounded_servers(&mcp.servers); mcp.servers = kept; mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); } @@ -773,6 +775,48 @@ fn bounded_entries( (kept, truncated) } +/// At most 32 MCP servers, `plan` servers first: the cap and the key rule +/// of [`bounded_entries`] apply to the `plan` servers, and servers of any +/// other origin, such as `harness`, fill only the places left, so a report +/// never pushes out a server the plan configures. One marker counts every +/// dropped server. +fn bounded_servers( + servers: &BTreeMap, +) -> (BTreeMap, Option) { + let (plan, others): (BTreeMap<_, _>, BTreeMap<_, _>) = servers + .iter() + .map(|(name, server)| (name.clone(), server.clone())) + .partition(|(_, server)| server.origin == ORIGIN_PLAN); + let (mut kept, plan_dropped) = bounded_entries(&plan, MAX_MCP_SERVERS, McpServerFacts::clone); + let (others_kept, others_dropped) = + bounded_entries(&others, MAX_MCP_SERVERS - kept.len(), McpServerFacts::clone); + kept.extend(others_kept); + let dropped = match (plan_dropped, others_dropped) { + (Some(plan), Some(others)) => Some(TruncatedEntries { + reason: if plan.reason == REASON_ENTRY_CAP { + plan.reason + } else { + others.reason + }, + dropped: plan.dropped.saturating_add(others.dropped), + }), + (plan, others) => plan.or(others), + }; + (kept, dropped) +} + +/// Remove the last server in sorted key order whose origin is not `plan`, +/// or the last server when every one is a `plan` server. +fn pop_last_server(servers: &mut BTreeMap) -> Option { + let name = servers + .iter() + .rev() + .find(|(_, server)| server.origin != ORIGIN_PLAN) + .or_else(|| servers.last_key_value()) + .map(|(name, _)| name.clone())?; + servers.remove(&name) +} + /// `marker` with one more dropped entry and reason `byte_budget`. fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { TruncatedEntries { @@ -2414,4 +2458,128 @@ mod tests { json!({"reason": "entry_cap", "dropped": 1}) ); } + + /// A plan declaring the MCP report key, with MCP servers under the given + /// names and, when `heavy_roles` is set, 16 model roles whose providers + /// take most of the 128 KiB budget. + fn mcp_plan(server_names: &[String], heavy_roles: bool) -> RunPlan { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + if heavy_roles { + let model = plan.config.models["default"].clone(); + plan.config.models = numbered_keys("role", 16) + .into_iter() + .map(|role| (role, model.clone())) + .collect(); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + } + plan + } + + /// An MCP report naming each server in `names` with a `runtime` block + /// that lists `tool_count` long tool names. + fn mcp_report(names: &[String], tool_count: usize) -> Value { + let tools: Vec = (0..tool_count) + .map(|index| format!("{index:03}{}", "t".repeat(250))) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"transport": "stdio", "runtime": { + "connected": reported(json!(true), "harness_status"), + "tool_names": reported(json!(tools), "harness_api")}}), + ) + }) + .collect(); + json!({MCP_REPORT_KEY: {"servers": servers}}) + } + + fn origins(what_ran: &Value) -> Vec<(String, String)> { + what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .iter() + .map(|(name, server)| { + ( + name.clone(), + server["origin"].as_str().expect("origin").to_string(), + ) + }) + .collect() + } + + #[test] + fn harness_entries_fill_only_the_places_plan_servers_leave_under_the_cap() { + let plan_names = numbered_keys("server", 30); + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), mcp_report(&harness_names, 0)); + + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "harness") + .map(|(name, _)| name.clone()) + .collect(); + assert_eq!(plan_kept, plan_names); + assert_eq!(harness_kept, harness_names[..2]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + // Every kept plan server keeps the endpoint core configured. + for name in &plan_names { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"]["source"], + "configured" + ); + } + } + + #[test] + fn the_byte_budget_drops_harness_entries_before_plan_servers() { + // The plan servers that fit beside 16 heavy model roles. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + assert!(!plan_names.is_empty()); + // Harness names sort before every plan server. Each entry is about + // 10 KiB, so the five cannot all fit, and the report stays under its + // own 64 KiB limit. + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), mcp_report(&harness_names, 40)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept = kept.len() - plan_kept.len(); + assert_eq!(plan_kept, plan_names); + assert!(harness_kept < harness_names.len(), "{kept:?}"); + assert!( + json_len(&mcp_report(&harness_names, 40)[MCP_REPORT_KEY]) < 64 * 1024, + "the report must be ingested, not marked too_large" + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": harness_names.len() - harness_kept}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + } } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index a8c31143e..4c475c9d6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -16,7 +16,7 @@ What executed for one invocation, and how NeMo Fabric learned each fact. A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. -`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. `plan` servers take their places first, and servers only the harness reports fill the places left. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, `harness` entries before `plan` entries, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. An adapter reports its own facts through the report keys [`VERSIONS_REPORT_KEY`](constant-versions-report-key.mdx) and [`MCP_REPORT_KEY`](constant-mcp-report-key.mdx) in its result's extensions, once its descriptor declares them. A reported list keeps at most 64 elements in sorted order, counted in a marker beside it, such as `tool_names_truncated`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f181293e2..bf743891c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1081,9 +1081,12 @@ order, and a `tool_names_truncated` marker in the `runtime` block counts the names the adapter or NeMo Fabric dropped. `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in -sorted key order. An entry whose key is longer than 128 characters is dropped. -While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers -are dropped from the end of the sorted order, and then model roles. A map that +sorted key order. Servers the plan configures take their places first, and +servers only the harness reports fill the places left. An entry whose key is +longer than 128 characters is dropped. While the whole `what_ran` is larger +than 128 KiB of compact JSON, MCP servers are dropped from the end of the +sorted order, harness-reported servers before plan servers, and then model +roles. A map that lost entries has a marker next to it, `models_truncated` or `mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or `byte_budget`) and the `dropped` count. Each map has one marker: when more than From c9d6fad1f292d90488ad82432092d6e3331934e3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:56:32 -0700 Subject: [PATCH 32/74] docs: name the state together with each what_ran reason code The adapter tutorial said core marks a fact invalid_report or too_large, which reads as a state. Core sets state collection_failed and uses those codes as the reason. Name the state with its reason there, and in the SDK guide and common TypeScript README sentences that gave only a reason. Signed-off-by: mnajafian-nv --- adapters/typescript/common/README.md | 4 ++-- docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 93cb2803a..acaacab2d 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -47,8 +47,8 @@ contract. Published package manifests use the registry version instead. `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. -`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `no_source` for every role. +`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `unsupported` with reason `no_source` for every role. `ServedModelFact` is `{role: string; fact: JsonObject}`: the served-model fact for one model role. Build a fact without a value with `missingFact(state, reason, method)`, for example `missingFact("unavailable", "not_attributed", "harness_api")`; `method` defaults to `adapter_report`. -`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. +`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `unsupported` with reason `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 1aa9fcce6..cfa6d58d7 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index bf743891c..2e62492ae 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1044,9 +1044,9 @@ MCP runtime facts yet. A reported fact whose key the adapter does not declare has state `unsupported` with reason `key_not_declared` and method `adapter_report`. A fact the adapter -has no source for has reason `no_source`. A declared key that the result omits -gives `unavailable` with reason `key_omitted`, and a fact the key omits gives -`fact_omitted`. A malformed fact, or one that does not name source `reported` +has no source for has state `unsupported` with reason `no_source`. A declared +key that the result omits gives `unavailable` with reason `key_omitted`, and a +fact the key omits gives `unavailable` with reason `fact_omitted`. A malformed fact, or one that does not name source `reported` and collector `adapter`, gives `collection_failed` with reason `invalid_report`. A key larger than 64 KiB of compact JSON gives `collection_failed` with reason `too_large`. None of these changes the run's From b35c3ff24b32d14d6d722bb15dcac325a2efd363 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:57:45 -0700 Subject: [PATCH 33/74] test: read expected versions in the provenance tests from package metadata The helper tests compared reported versions with literal package and harness versions, so a routine version bump of Fabric, Deep Agents or mini-SWE-agent would fail them. Read the expected values from the installed package metadata; literal versions remain only as test input. Signed-off-by: mnajafian-nv --- .../adapters/test_adapters_common_provenance.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index 3e9f00453..d9dda1acf 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import json import sys import threading @@ -58,7 +59,7 @@ def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): def test_package_version_fact_reads_installed_metadata(): assert provenance.package_version_fact("nemo-fabric-adapters-common") == known( - "0.6.0" + importlib.metadata.version("nemo-fabric-adapters-common") ) assert provenance.package_version_fact("nemo-fabric-no-such-package") == missing( "unavailable", "not_collected" @@ -68,8 +69,12 @@ def test_package_version_fact_reads_installed_metadata(): @pytest.mark.parametrize( ("distribution", "module", "expected"), [ - ("deepagents", deepagents.__name__, known("0.7.9")), - ("mini-swe-agent", minisweagent.__name__, known("2.4.6")), + ("deepagents", deepagents.__name__, known(importlib.metadata.version("deepagents"))), + ( + "mini-swe-agent", + minisweagent.__name__, + known(importlib.metadata.version("mini-swe-agent")), + ), # The distribution exists but did not install the loaded module. ("mini-swe-agent", deepagents.__name__, missing("unavailable", "not_attributed")), ("deepagents", "nemo_fabric_not_loaded", missing("unavailable", "not_collected")), @@ -189,7 +194,11 @@ def test_package_version_report_marks_harness_and_served_models_no_source(): no_source = missing("unsupported", "no_source", "adapter_report") assert report == { - "adapter": {"package_version": known("0.6.0")}, + "adapter": { + "package_version": known( + importlib.metadata.version("nemo-fabric-adapters-common") + ) + }, "harness": {"name": no_source, "version": no_source}, "models": {"default": {"served": no_source}, "judge": {"served": no_source}}, } From b047245d9db0d1a212d47e4eb3c63aa310a61419 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:57:45 -0700 Subject: [PATCH 34/74] test: read the Hermes adapter version from package metadata The Hermes versions-report test expected the literal 0.6.0, which fails after a normal version bump. Read it from the installed package metadata as the OpenClaw and end-to-end tests do. Signed-off-by: mnajafian-nv --- tests/adapters/test_hermes_adapter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 04bfe3336..c200cc0ce 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import importlib.util import inspect import json @@ -1756,7 +1757,9 @@ async def test_hermes_reports_the_version_its_version_call_returns( "name": {**_reported("hermes-agent"), "method": "adapter_report"}, "version": _reported(get_version_info().derived_version), } - assert report["adapter"]["package_version"]["value"] == "0.6.0" + assert report["adapter"]["package_version"]["value"] == importlib.metadata.version( + "nemo-fabric-adapters-hermes" + ) def _raise_runtime_error() -> str: From d35a1e94fa8ca20289ba9d3e805ac894ba89f227 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:59:15 -0700 Subject: [PATCH 35/74] test: fix the common package check and the Windows probe timing The common TypeScript package now ships dist/provenance.js and its declarations, so its pack check lists them. On Windows the monotonic clock ticks about every 15.6 ms, and the version-probe timeout test measured 0.187 s for a 0.2 s timeout, so its lower bound now allows that resolution. Signed-off-by: mnajafian-nv --- adapters/typescript/scripts/check-package.mjs | 2 ++ tests/adapters/test_adapters_common_provenance.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/adapters/typescript/scripts/check-package.mjs b/adapters/typescript/scripts/check-package.mjs index ef6c77b2f..f6a861645 100644 --- a/adapters/typescript/scripts/check-package.mjs +++ b/adapters/typescript/scripts/check-package.mjs @@ -28,6 +28,8 @@ const expectedByPackage = { "dist/lifecycle.js", "dist/model-endpoint.d.ts", "dist/model-endpoint.js", + "dist/provenance.d.ts", + "dist/provenance.js", "package.json", ], "nemo-fabric-adapters-cline": [ diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index d9dda1acf..ef2c67d18 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -154,7 +154,9 @@ async def count_ticks() -> None: ticker.cancel() release.set() - assert 0.2 <= elapsed < 1 + # Windows' monotonic clock ticks about every 15.6 ms, so the measured wait + # can read slightly under the 0.2 s timeout. + assert 0.15 <= elapsed < 1 # The event loop kept running while the function blocked its thread. assert ticks >= 5 while probe_threads(): From 57a98746848e288cea047f7d7fcb8346f4752f25 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:22:49 -0700 Subject: [PATCH 36/74] fix: order TypeScript report roles by code point like core packageVersionReport sorted roles with Array.prototype.sort, which compares UTF-16 code units. Core and the Python helper order by code point, which is UTF-8 byte order, so past the 16-role cap a role above U+FFFF and one in U+E000 to U+FFFF swapped places. The TypeScript helper then kept a role core drops, and a served-model override on the role core keeps was lost. Sort by code point, and check all three implementations against one shared role-order fixture. Signed-off-by: mnajafian-nv --- adapters/typescript/common/src/provenance.ts | 22 +++++++++- .../common/test/provenance.test.mjs | 12 ++++++ crates/fabric-core/src/what_ran.rs | 18 +++++++++ .../test_adapters_common_provenance.py | 21 ++++++++++ tests/fixtures/what_ran/role-order.json | 40 +++++++++++++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/what_ran/role-order.json diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts index 8e084519d..1b24e3fb5 100644 --- a/adapters/typescript/common/src/provenance.ts +++ b/adapters/typescript/common/src/provenance.ts @@ -65,6 +65,23 @@ export function packageVersionFact(packageJson: URL): JsonObject { return {value: boundedText(version), source: "reported", collected_by: "adapter", method: "package_metadata"}; } +/** + * Order two strings by Unicode code point, which is the UTF-8 byte order that + * NeMo Fabric core and the Python helper use. `Array.prototype.sort` compares + * UTF-16 code units, which puts a character above U+FFFF before one in + * U+E000 to U+FFFF. + */ +function compareCodePoints(left: string, right: string): number { + const leftPoints = Array.from(left, (char) => char.codePointAt(0) ?? 0); + const rightPoints = Array.from(right, (char) => char.codePointAt(0) ?? 0); + for (const [index, point] of leftPoints.entries()) { + const other = rightPoints[index]; + if (other === undefined) return 1; + if (point !== other) return point - other; + } + return leftPoints.length - rightPoints.length; +} + /** * The report of an adapter that has a source only for its package version. * The harness name and version read `unsupported` with reason `no_source`, and @@ -78,7 +95,10 @@ export function packageVersionReport( servedModel?: ServedModelFact, ): JsonObject { const noSource = () => missingFact("unsupported", "no_source"); - const roles = modelRoles.filter((role) => Array.from(role).length <= MAX_KEY_CHARS).sort().slice(0, MAX_MODEL_ROLES); + const roles = modelRoles + .filter((role) => Array.from(role).length <= MAX_KEY_CHARS) + .sort(compareCodePoints) + .slice(0, MAX_MODEL_ROLES); return { adapter: {package_version: packageVersion}, harness: {name: noSource(), version: noSource()}, diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 9746b3a97..187886cd8 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -107,3 +107,15 @@ test("the wrapper picks the served-model role from the start input", async () => await rm(directory, {recursive: true, force: true}); } }); + +test("the report keeps the roles core keeps, in code point order", async () => { + // The vectors Rust core and the Python helper also read. + const vectors = JSON.parse(await readFile(new URL("../../../../tests/fixtures/what_ran/role-order.json", import.meta.url), "utf8")); + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + const report = packageVersionReport(packageVersion, vectors.roles, {role: "\u{E000}", fact: notAttributed}); + + assert.deepEqual(Object.keys(report.models), vectors.kept); + assert.deepEqual(report.models["\u{E000}"], {served: notAttributed}); +}); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index fdddf0ec7..1603c7e39 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1490,6 +1490,24 @@ mod tests { assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); } + #[test] + fn model_roles_past_the_cap_keep_the_code_point_order_the_helpers_share() { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/role-order.json" + )) + .expect("role order JSON"); + let roles: Vec = serde_json::from_value(vectors["roles"].clone()).expect("roles"); + let expected: Vec = serde_json::from_value(vectors["kept"].clone()).expect("kept"); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), expected); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } + #[test] fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { let longest = "é".repeat(128); diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index ef2c67d18..b649ee10f 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -114,6 +114,27 @@ def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): assert report["adapter"]["package_version"] == known("0.6.0") + +def test_versions_report_keeps_the_roles_core_keeps_in_code_point_order(): + vectors = json.loads( + ( + Path(__file__).resolve().parents[1] / "fixtures/what_ran/role-order.json" + ).read_text(encoding="utf-8") + ) + override = missing("unavailable", "not_attributed", "harness_api") + served = {role: provenance.no_source_fact() for role in vectors["roles"]} + served["\ue000"] = override + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.no_source_fact(), + harness_version=provenance.no_source_fact(), + served_models=served, + ) + + assert list(report["models"]) == vectors["kept"] + assert report["models"]["\ue000"] == {"served": override} + def probe_threads() -> int: return sum( thread.name == "nemo-fabric-version-probe" for thread in threading.enumerate() diff --git a/tests/fixtures/what_ran/role-order.json b/tests/fixtures/what_ran/role-order.json new file mode 100644 index 000000000..012e9ea4a --- /dev/null +++ b/tests/fixtures/what_ran/role-order.json @@ -0,0 +1,40 @@ +{ + "description": "Model roles past the 16-role cap. Rust core, the Python helper and the TypeScript helper sort role keys by Unicode code point (UTF-8 byte order) and keep the first 16, so all three keep the same roles. U+10000 sorts after U+E000 by code point but before it by UTF-16 code unit.", + "roles": [ + "\ud800\udc00", + "\ue000", + "role-14", + "role-13", + "role-12", + "role-11", + "role-10", + "role-09", + "role-08", + "role-07", + "role-06", + "role-05", + "role-04", + "role-03", + "role-02", + "role-01", + "role-00" + ], + "kept": [ + "role-00", + "role-01", + "role-02", + "role-03", + "role-04", + "role-05", + "role-06", + "role-07", + "role-08", + "role-09", + "role-10", + "role-11", + "role-12", + "role-13", + "role-14", + "\ue000" + ] +} From 9af928bb79d5f1f149a194a39c7ed6807aee002d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:23:04 -0700 Subject: [PATCH 37/74] docs: say when an undeclared report key fails the run The guide said none of the ingestion outcomes changes a run's status, right after describing undeclared keys. A result that carries a key the descriptor's run_result schema rejects fails before ingestion and returns no RunResult. Say so, and limit the status sentence to a returned run. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2e62492ae..e9a6bfcb9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1042,15 +1042,18 @@ because nothing yet shows that the model its SDK names served this invocation. No bundled adapter reports MCP runtime facts yet. -A reported fact whose key the adapter does not declare has state `unsupported` -with reason `key_not_declared` and method `adapter_report`. A fact the adapter -has no source for has state `unsupported` with reason `no_source`. A declared -key that the result omits gives `unavailable` with reason `key_omitted`, and a -fact the key omits gives `unavailable` with reason `fact_omitted`. A malformed fact, or one that does not name source `reported` +When the descriptor does not declare a report key, ingestion leaves the +corresponding facts `unsupported` with reason `key_not_declared`. +Result-extension validation runs first: if the schema rejects an emitted +undeclared key, the run fails before ingestion and no `RunResult` is returned. +A fact the adapter has no source for has state `unsupported` with reason +`no_source`. A declared key that the result omits gives `unavailable` with +reason `key_omitted`, and a fact the key omits gives `unavailable` with reason +`fact_omitted`. A malformed fact, or one that does not name source `reported` and collector `adapter`, gives `collection_failed` with reason `invalid_report`. A key larger than 64 KiB of compact JSON gives -`collection_failed` with reason `too_large`. None of these changes the run's -status or error. +`collection_failed` with reason `too_large`. These ingestion outcomes do not +change a returned run's status or error. An MCP entry has `origin` `plan` for a server the plan configures. An entry with `origin` `harness` describes a server the harness reports but the plan From 3023e80cc42be616ff91354f9489b6134885acc2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:52:14 -0700 Subject: [PATCH 38/74] fix: add a report's harness server drops to core's servers marker An adapter's helper can drop MCP servers from its report to keep the key under 64 KiB, and count them in the report's servers_truncated marker. Core ignored that marker, so harness-only servers the helper dropped vanished from what_ran without a count. Core now adds the marker's count, less the plan servers the report omits (those keep their entries), to its own marker. A malformed marker makes the servers report read invalid_report, as a malformed tool_names marker does. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 131 ++++++++++++++++++++++++++++- docs/sdk/python.mdx | 15 ++-- 2 files changed, 136 insertions(+), 10 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 1603c7e39..2724f85d1 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -977,8 +977,9 @@ const REASON_TOO_LARGE: &str = "too_large"; /// Inside a report, a key may hold only adapter-owned facts, each with /// source `reported` and collector `adapter`: the adapter package version, /// the harness name and version, each plan role's served model, each MCP -/// server's `runtime` block with its `tool_names_truncated` marker, and a -/// `harness` server's transport. A malformed or forged fact reads +/// server's `runtime` block with its `tool_names_truncated` marker, a +/// `harness` server's transport, and the `servers_truncated` marker beside +/// `servers`, whose count core adds to its own only for `harness` entries. A malformed or forged fact reads /// `collection_failed` with reason `invalid_report`, and an omitted one /// reads `unavailable` with reason `fact_omitted`. Core ignores every other /// property, so a report never changes a fact core owns. The string length @@ -1114,18 +1115,53 @@ fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { /// Fill each plan server's `runtime` block, and add a `harness` entry for /// each reported server the plan does not configure. +/// +/// The report's `servers_truncated` marker counts servers the adapter's +/// helper dropped from the report. A plan server the helper dropped keeps +/// its entry, with facts that read `fact_omitted`, so core adds to its own +/// marker only the count left after the plan servers the report omits: +/// the `harness` entries that disappear from `what_ran`. A malformed marker +/// makes the whole `servers` report read `invalid_report`, as a malformed +/// `tool_names_truncated` does for `tool_names`. fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { let Some(mcp) = what_ran.mcp.as_mut() else { return; }; - let servers = report.child("servers"); + let mut servers = report.child("servers"); + let mut helper_marker = None; + if let (Report::Object(_), Report::Object(object)) = (&servers, report) + && let Some(marker) = object.get("servers_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + servers = Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + let plan_servers = &plan.capability_plan.mcp_servers; + if let (Some(marker), Report::Object(reported)) = (helper_marker, &servers) { + let plan_servers_omitted = plan_servers + .keys() + .filter(|name| !reported.contains_key(*name)) + .count() as u64; + let harness_dropped = marker.dropped.saturating_sub(plan_servers_omitted); + if harness_dropped > 0 { + mcp.servers_truncated = merged_marker( + mcp.servers_truncated.take(), + Some(TruncatedEntries { + reason: marker.reason, + dropped: harness_dropped, + }), + ); + } + } for (name, server) in &mut mcp.servers { server.runtime = runtime_facts(&servers.child(name).child("runtime")); } let Report::Object(reported) = servers else { return; }; - let plan_servers = &plan.capability_plan.mcp_servers; for (name, entry) in reported { if plan_servers.contains_key(name) { continue; @@ -2600,4 +2636,91 @@ mod tests { ); assert_eq!(map_keys(&what_ran["models"]).len(), 16); } + + fn connected_runtime() -> Value { + json!({"runtime": {"connected": reported(json!(true), "harness_status")}}) + } + + #[test] + fn a_helper_servers_marker_adds_its_harness_drops_to_core_marker() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // Every plan server is reported, so all three helper drops were + // harness entries. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 3} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github", "plugin-docs"] + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn a_helper_servers_marker_never_counts_plan_servers_the_report_omits() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // The helper dropped `github`, a plan server that keeps its entry, + // and two harness entries. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 3} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 2}) + ); + + // A marker that counts only omitted plan servers adds nothing. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + } + + #[test] + fn a_malformed_helper_servers_marker_invalidates_the_servers_report() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + for marker in [ + json!({"reason": "byte_budget", "dropped": 0}), + json!({"reason": "byte_budget"}), + json!("three"), + Value::Null, + ] { + let report = json!({ + "servers": {"events": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": marker + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!(map_keys(servers), ["events", "files", "github"], "{marker}"); + assert_eq!( + servers["events"]["runtime"]["connected"], + report_state("collection_failed", "invalid_report"), + "{marker}" + ); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{marker}" + ); + } + } } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index e9a6bfcb9..3c1761b3e 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1089,15 +1089,18 @@ servers only the harness reports fill the places left. An entry whose key is longer than 128 characters is dropped. While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers are dropped from the end of the sorted order, harness-reported servers before plan servers, and then model -roles. A map that -lost entries has a marker next to it, `models_truncated` or +roles. A map that lost entries has a marker next to it, `models_truncated` or `mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or `byte_budget`) and the `dropped` count. Each map has one marker: when more than one rule dropped entries, `reason` names the last rule applied, in that order, -and `dropped` counts every dropped entry. `dropped` is a JSON integer. The Rust -core and the Python SDK reject `1.0`, although a JSON Schema validator may -accept it. If `what_ran` is still too large with no MCP server and no model -role left, it holds only `schema`, `version`, and `what_ran_truncated` with +and `dropped` counts every dropped entry. When the adapter's report drops +servers to fit its own size limit, `mcp.servers_truncated` also counts the ones +only the harness reported, because those disappear; a plan server the report +drops keeps its entry, and its runtime facts read `unavailable` with reason +`fact_omitted`. `dropped` is a JSON integer. The Rust core and the Python SDK +reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is +still too large with no MCP server and no model role left, it holds only +`schema`, `version`, and `what_ran_truncated` with reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you read a section. Dropping entries never changes the run's status or error. From 29cf0fd22502338baad902140043c98f66f2c6ce Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:54:56 -0700 Subject: [PATCH 39/74] test: show an older core accepts descriptors that declare the versions key The design requires that a core without report-key ingestion still accept the new descriptors. The test loads every bundled descriptor and runs the result-extension check that predates ingestion, with and without the versions report, and checks that the key stays an optional object property. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 113 +++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index a452c7f94..cfce6639e 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -6930,6 +6930,119 @@ mod tests { .expect("declared result extension"); } + /// Every bundled descriptor under `adapters/`. + fn bundled_descriptor_paths() -> Vec { + let mut paths = Vec::new(); + for language in ["python", "typescript"] { + let root = repository_root().join("adapters").join(language); + for adapter in fs::read_dir(&root).expect("adapter directory") { + let adapter = adapter.expect("adapter entry").path(); + let Ok(files) = fs::read_dir(&adapter) else { + continue; + }; + for file in files { + let file = file.expect("adapter file").path(); + if file + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fabric-adapter.json")) + { + paths.push(file); + } + } + } + } + paths.sort(); + paths + } + + // The descriptor loader and the result-extension check below predate + // report-key ingestion and run unchanged in an older core, which treats + // `nemo_fabric.versions.v1` as an ordinary optional object property. + #[test] + fn a_descriptor_declaring_the_versions_key_passes_the_pre_ingestion_checks() { + let paths = bundled_descriptor_paths(); + assert_eq!(paths.len(), 16, "{paths:?}"); + let report = serde_json::json!({ + "adapter": {"package_version": {"value": "0.6.0", "source": "reported", + "collected_by": "adapter", "method": "package_metadata"}} + }); + for path in paths { + let descriptor = load_adapter_descriptor(&path).expect("descriptor loads"); + let run_result = &descriptor.extension_schemas[&AdapterExtensionPoint::RunResult]; + assert_eq!( + run_result["properties"]["nemo_fabric.versions.v1"], + serde_json::json!({"type": "object"}), + "{}", + path.display() + ); + // Core skips an empty extensions block, so check optionality on + // the schema too: a required key would reject an older adapter's + // result that carries other keys but not this one. + assert!( + !run_result + .get("required") + .and_then(serde_json::Value::as_array) + .is_some_and( + |required| required.contains(&serde_json::json!("nemo_fabric.versions.v1")) + ), + "{}", + path.display() + ); + let base = required_result_extensions(run_result); + let resolved = resolved_adapter(path.clone(), descriptor); + let mut with_report = base.clone(); + with_report.insert("nemo_fabric.versions.v1".to_string(), report.clone()); + for extensions in [base, with_report] { + let result: AgentRunResult = serde_json::from_value(serde_json::json!({ + "status": "succeeded", + "output": "done", + "extensions": extensions + })) + .expect("agent result"); + validate_agent_run_result_extensions(&result, Some(&resolved)).unwrap_or_else( + |error| { + panic!( + "{} rejected {:?}: {error}", + path.display(), + result.extensions + ) + }, + ); + } + } + } + + /// The smallest values for the keys other than the versions key that a + /// `run_result` schema requires, such as a session ID. + fn required_result_extensions( + schema: &serde_json::Map, + ) -> serde_json::Map { + let required = schema + .get("required") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + required + .iter() + .map(|key| key.as_str().expect("required key")) + // A descriptor that required the versions key would break an + // older core's results that omit it, so the base never adds it. + .filter(|key| *key != "nemo_fabric.versions.v1") + .map(|key| { + let property = &schema["properties"][key]; + let value = match (property.get("enum"), property["type"].as_str()) { + (Some(values), _) => values[0].clone(), + (None, Some("string")) => serde_json::json!("value"), + (None, Some("integer")) => serde_json::json!(0), + (None, Some("boolean")) => serde_json::json!(false), + (None, other) => panic!("no sample for {key}: {other:?}"), + }; + (key.to_string(), value) + }) + .collect() + } + #[test] fn workflow_settings_are_fail_closed() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); From 5696458868ce73146e2eefa33d6dbb7a414d16d3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:56:48 -0700 Subject: [PATCH 40/74] test: read the string length vectors from one shared fixture Rust core, the Python helper and the TypeScript helper each kept their own copy of the three string length vectors, so a change to one copy could leave the others agreeing with a stale rule. All three now read tests/fixtures/what_ran/string-length.json, next to the role-order fixture. The lone-surrogate case stays in the helper tests because a JSON fixture can't carry it into Rust. Signed-off-by: mnajafian-nv --- .../common/test/provenance.test.mjs | 15 +++++---- crates/fabric-core/src/what_ran.rs | 28 ++++++++-------- .../test_adapters_common_provenance.py | 33 ++++++++++--------- tests/fixtures/what_ran/string-length.json | 17 ++++++++++ 4 files changed, 58 insertions(+), 35 deletions(-) create mode 100644 tests/fixtures/what_ran/string-length.json diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 187886cd8..bb38d8cd5 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -19,11 +19,14 @@ import { const noSource = {state: "unsupported", reason: "no_source", source: "reported", collected_by: "adapter", method: "adapter_report"}; -// The same vectors as core's string length test in `what_ran.rs`. -test("bounded text matches core's string length rule", () => { - assert.equal(boundedText("é".repeat(256)), "é".repeat(256)); - assert.equal(boundedText(`${"m".repeat(300)}a`), `${"m".repeat(240)}~96858eedeb3c8e6`); - assert.equal(boundedText("é".repeat(300)), `${"é".repeat(240)}~7250b66610f8b7d`); +const fixtures = new URL("../../../../tests/fixtures/what_ran/", import.meta.url); + +test("bounded text matches core's string length rule", async () => { + // The vectors Rust core and the Python helper also read. + const {cases} = JSON.parse(await readFile(new URL("string-length.json", fixtures), "utf8")); + assert.equal(cases.length, 3); + for (const {text, expected} of cases) assert.equal(boundedText(text), expected); + // JSON fixtures can't carry a lone surrogate into Rust, so this case stays here. assert.equal(boundedText("v\ud800"), "v?"); }); @@ -110,7 +113,7 @@ test("the wrapper picks the served-model role from the start input", async () => test("the report keeps the roles core keeps, in code point order", async () => { // The vectors Rust core and the Python helper also read. - const vectors = JSON.parse(await readFile(new URL("../../../../tests/fixtures/what_ran/role-order.json", import.meta.url), "utf8")); + const vectors = JSON.parse(await readFile(new URL("role-order.json", fixtures), "utf8")); const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 2724f85d1..a8ec1873f 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1937,19 +1937,21 @@ mod tests { #[test] fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { - let exact = "é".repeat(256); - assert_eq!(bounded_text(&exact), exact); - - // Expected digests: the first 15 hex digits of `sha256sum` over the - // whole value's UTF-8 bytes. - for (long, prefix, digest) in [ - (format!("{}a", "m".repeat(300)), "m", "96858eedeb3c8e6"), - ("é".repeat(300), "é", "7250b66610f8b7d"), - ] { - let bounded = bounded_text(&long); - - assert_eq!(bounded, format!("{}~{digest}", prefix.repeat(240))); - assert_eq!(bounded.chars().count(), 256); + // The vectors the Python and TypeScript helpers also read. Expected + // digests are the first 15 hex digits of `sha256sum` over the whole + // value's UTF-8 bytes. + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/string-length.json" + )) + .expect("string length JSON"); + let cases = vectors["cases"].as_array().expect("cases"); + assert_eq!(cases.len(), 3); + for case in cases { + let text = case["text"].as_str().expect("text"); + let bounded = bounded_text(text); + + assert_eq!(bounded, case["expected"].as_str().expect("expected")); + assert_eq!(bounded.chars().count(), text.chars().count().min(256)); } } diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index b649ee10f..4f0e899f7 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -37,17 +37,22 @@ def missing(state: str, reason: str, method: str = "package_metadata") -> dict[s } -# The same vectors as core's string length test in `what_ran.rs`. -@pytest.mark.parametrize( - ("text", "expected"), - [ - ("é" * 256, "é" * 256), - ("m" * 300 + "a", "m" * 240 + "~96858eedeb3c8e6"), - ("é" * 300, "é" * 240 + "~7250b66610f8b7d"), - ], -) -def test_bounded_text_matches_core_string_length_rule(text, expected): - assert provenance.bounded_text(text) == expected +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/what_ran" + + +def string_length_cases() -> list[dict[str, str]]: + """The string length vectors that core and the TypeScript helper also read.""" + + vectors = json.loads((FIXTURES / "string-length.json").read_text(encoding="utf-8")) + return vectors["cases"] + + +def test_bounded_text_matches_core_string_length_rule(): + cases = string_length_cases() + + assert len(cases) == 3 + for case in cases: + assert provenance.bounded_text(case["text"]) == case["expected"] def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): @@ -116,11 +121,7 @@ def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): def test_versions_report_keeps_the_roles_core_keeps_in_code_point_order(): - vectors = json.loads( - ( - Path(__file__).resolve().parents[1] / "fixtures/what_ran/role-order.json" - ).read_text(encoding="utf-8") - ) + vectors = json.loads((FIXTURES / "role-order.json").read_text(encoding="utf-8")) override = missing("unavailable", "not_attributed", "harness_api") served = {role: provenance.no_source_fact() for role in vectors["roles"]} served["\ue000"] = override diff --git a/tests/fixtures/what_ran/string-length.json b/tests/fixtures/what_ran/string-length.json new file mode 100644 index 000000000..893fc1ae1 --- /dev/null +++ b/tests/fixtures/what_ran/string-length.json @@ -0,0 +1,17 @@ +{ + "description": "String length rule vectors that Rust core, the Python helper and the TypeScript helper share. A value of at most 256 Unicode scalar values is kept whole. A longer one keeps its first 240, then ~, then the first 15 hex digits of the SHA-256 of its UTF-8 bytes.", + "cases": [ + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé" + }, + { + "text": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmma", + "expected": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm~96858eedeb3c8e6" + }, + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé~7250b66610f8b7d" + } + ] +} From 64feb12af801e3b208d486af5ae3c1fdd803872e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:01:29 -0700 Subject: [PATCH 41/74] test: check versions report validation under three run_result schemas Core keeps its strict result-extension check ahead of ingestion. The runtime test now runs the same emitted versions report with no run_result schema and with a schema that rejects the key, both of which fail the invocation with the exact existing errors, and with a schema that accepts the key without naming it, which succeeds, keeps the key in metadata.adapter and leaves the facts key_not_declared. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 82 ++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 17c4fee03..8c017798e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5620,22 +5620,41 @@ for line in sys.stdin: } /// An adapter that declares the versions key and reports it gets its - /// facts into `what_ran`, and the key leaves `metadata.adapter`. + /// facts into `what_ran`, and the key leaves `metadata.adapter`. The + /// existing result-extension check runs first: a descriptor with no + /// `run_result` schema, or one that rejects the key, fails the + /// invocation, and one that accepts the key without naming it leaves + /// the facts `key_not_declared`. #[test] fn local_host_what_ran_ingests_a_declared_versions_report() { use serde_json::json; let (root, plan) = local_host_plan("versions_report"); let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); - let mut descriptor: Value = + let original: Value = serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) .expect("descriptor JSON"); - descriptor["extension_schemas"]["run_result"]["properties"]["nemo_fabric.versions.v1"] = - json!({"type": "object"}); - fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); - let plan = resolve_run_plan_from_config(plan.config, ResolveContext::new(&root)) - .expect("resolve plan"); + // Run the same emitted report under the given `run_result` schema. + let run_with = |run_result: Option| { + let mut descriptor = original.clone(); + let schemas = descriptor["extension_schemas"] + .as_object_mut() + .expect("extension schemas"); + match run_result { + Some(schema) => schemas.insert("run_result".to_string(), schema), + None => schemas.remove("run_result"), + }; + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let plan = + resolve_run_plan_from_config(plan.config.clone(), ResolveContext::new(&root)) + .expect("resolve plan"); + run_plan(&plan, RunRequest::text("provenance")) + }; + let declared = json!({"type": "object", "properties": { + "trace_id": {"type": "string"}, + "nemo_fabric.versions.v1": {"type": "object"} + }, "additionalProperties": false}); - let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let result = run_with(Some(declared)).expect("result"); assert_eq!(result.status, RunStatus::Succeeded); assert_eq!(result.metadata["adapter"], json!({"trace_id": "trace-1"})); @@ -5653,6 +5672,53 @@ for line in sys.stdin: json!({"name": reported("fake-harness", "harness_api"), "version": reported("4.5.6", "harness_api")}) ); + + // No `run_result` schema, and one that rejects the undeclared key, + // both fail the invocation before ingestion. + let error = run_with(None).expect_err("no run_result schema"); + assert!( + matches!( + &error, + FabricError::AdapterCompatibility { field, reason, .. } + if field == "result.extensions" + && reason + == "the adapter descriptor does not declare an extension schema for run_result" + ), + "{error:?}" + ); + let rejecting = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}, + "additionalProperties": false}); + let error = run_with(Some(rejecting)).expect_err("rejecting run_result schema"); + assert!( + matches!( + &error, + FabricError::InvalidAdapterExtension { extension_path, reason, .. } + if extension_path == "result.extensions.nemo_fabric.versions.v1" + && reason == "is not declared by the adapter extension schema" + ), + "{error:?}" + ); + + // A schema that accepts the key without naming it keeps the run and + // the key in `metadata.adapter`, and ingestion leaves the facts + // `key_not_declared`. + let permissive = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}}); + + let result = run_with(Some(permissive)).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!( + result.metadata["adapter"]["nemo_fabric.versions.v1"]["harness"]["version"], + reported("4.5.6", "harness_api") + ); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!( + what_ran["harness"], + json!({"name": not_declared, "version": not_declared}) + ); let _ = fs::remove_dir_all(root); } From e16b42b41a099dd0201f4174016d76986ad0b837 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:01:55 -0700 Subject: [PATCH 42/74] docs: explain the descriptor-copy migration for the versions report Core validates result extensions before it ingests report keys, so an external copy of a bundled descriptor whose run_result schema rejects nemo_fabric.versions.v1 fails every invocation once the adapter implementation is upgraded. The SDK guide and the integrate skill now say how to update such a copy, and the adapter tutorial and the adapter authoring skill say to declare the key before emitting it. Signed-off-by: mnajafian-nv --- docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 8 ++++++++ skills/nemo-fabric-build-adapter/SKILL.md | 2 +- .../references/results-and-errors.md | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index cfa6d58d7..ea89dce17 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3c1761b3e..444e2abac 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1046,6 +1046,14 @@ When the descriptor does not declare a report key, ingestion leaves the corresponding facts `unsupported` with reason `key_not_declared`. Result-extension validation runs first: if the schema rejects an emitted undeclared key, the run fails before ingestion and no `RunResult` is returned. + +If you maintain an external descriptor copy for a bundled adapter, update its +`run_result` extension schema before upgrading the adapter implementation. Add +`"nemo_fabric.versions.v1": {"type": "object"}` under `properties`, preserving +existing properties and requirements, and do not add the key to `required`. +Without a schema that accepts the report, invocation fails during +result-extension validation. + A fact the adapter has no source for has state `unsupported` with reason `no_source`. A declared key that the result omits gives `unavailable` with reason `key_omitted`, and a fact the key omits gives `unavailable` with reason diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 15705843c..a8541ec8a 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,7 +230,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 6abc28ea5..9af5d92d2 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,7 @@ harness version and the served model, read state `unsupported` with reason `key_not_declared` when the adapter descriptor does not declare their report key. When the descriptor declares the key, a result that omits the key reads `unavailable` with reason `key_omitted`, and a key that omits the fact reads -`unavailable` with reason `fact_omitted`. Every bundled +`unavailable` with reason `fact_omitted`. If you maintain an external descriptor copy for a bundled adapter, update its `run_result` extension schema before upgrading the adapter implementation. Add `"nemo_fabric.versions.v1": {"type": "object"}` under `properties`, preserving existing properties and requirements, and do not add the key to `required`. Without a schema that accepts the report, invocation fails during result-extension validation. Every bundled adapter reports its package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, and the Claude Code adapter reports the served model. The Cline adapter reports From b6de51bdc02f9ddb8cc9da77b9e2c8018cfb05c1 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:22:56 -0700 Subject: [PATCH 43/74] fix: report unknown Codex usage when a turn never completes After a timeout, a closed connection, or a stream that ended early, the adapter reported the turn's last usage snapshot as the invocation's complete usage. Codex sends a response's usage after its tool calls finish, so without turn/completed a response may still have been running and its tokens are missing from that snapshot. The adapter now records whether Codex sent turn/completed for the turn. Without it, normalized usage is unknown and the last snapshot stays in output.usage. A turn that Codex completed with a failed status keeps its usage, because every response of that turn has ended. After these failures the runtime accepts no further invocations, so the interrupted turn's usage never reaches a later invocation. Signed-off-by: mnajafian-nv --- adapters/python/codex/README.md | 2 +- .../src/nemo_fabric_adapters/codex/adapter.py | 20 +++++-- tests/adapters/test_codex_adapter.py | 56 ++++++++++++++----- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 20bc81995..4555aca58 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -79,7 +79,7 @@ The result includes the SDK's typed terminal response, turn status, token usage, timing, and completed thread items. It does not expose CLI commands, return codes, stdout, or stderr. -Normalized `usage` contains invocation-local token counts from the SDK's cumulative thread totals, including cached input tokens. Input tokens already include cached tokens (`input_tokens_include_cache=True`). Reused threads report the difference from the previous invocation rather than counting the thread total again. Missing, invalid, or reset counters remain unknown; a missing snapshot requires a new baseline before differences can be reported. The adapter preserves native usage in `output.usage`, does not infer cost, and retains available usage on unsuccessful results. When a turn fails, times out, or loses its connection, usage covers the snapshots that Codex sent before the failure. Codex sends a response's usage after its tool calls finish, so on a timeout the last response is not counted if it was still streaming or its tools were still running. A failure before Codex sends any usage leaves usage unknown. When Codex exceeds the model context window, it replaces the thread total with the window size before it fails the turn; the adapter ignores that snapshot. A request rejected before its turn starts, such as non-text input, keeps the baseline for the next invocation. +Normalized `usage` contains invocation-local token counts from the SDK's cumulative thread totals, including cached input tokens. Input tokens already include cached tokens (`input_tokens_include_cache=True`). Reused threads report the difference from the previous invocation rather than counting the thread total again. Missing, invalid, or reset counters remain unknown; a missing snapshot requires a new baseline before differences can be reported. The adapter preserves native usage in `output.usage`, does not infer cost, and retains available usage on unsuccessful results. When Codex completes a turn with a failed status, usage covers the snapshots that Codex sent before the failure. Codex sends a response's usage after its tool calls finish, so when a turn times out, loses its connection, or ends without Codex completing it, a response may still have been running and normalized usage is unknown. The last snapshot stays in `output.usage`, and the runtime accepts no further invocations, so that turn's usage never moves into a later invocation. A failure before Codex sends any usage leaves usage unknown. When Codex exceeds the model context window, it replaces the thread total with the window size before it fails the turn; the adapter ignores that snapshot. A request rejected before its turn starts, such as non-text input, keeps the baseline for the next invocation. ## Configuration diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index 86de0133f..4b143323f 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -42,6 +42,7 @@ SkillsExtraRootsSetResponse, ThreadTokenUsage, ThreadTokenUsageUpdatedNotification, + TurnCompletedNotification, ) # Private: the collector AsyncTurnHandle.run uses; recheck it on every openai-codex bump. @@ -1091,6 +1092,7 @@ def _agent_run_result( ) -> AgentRunResult: normalized = dict(output) failed = bool(normalized.pop("failed", False)) + usage_complete = normalized.pop("usage_complete", True) reported_error = normalized.pop("error", None) error = None if failed: @@ -1107,7 +1109,9 @@ def _agent_run_result( if cumulative_totals is None else cumulative_totals ) - agent_usage = _normalize_usage(normalized.get("usage"), baseline) + agent_usage = _normalize_usage( + normalized.get("usage") if usage_complete else None, baseline + ) return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=normalized, @@ -1168,6 +1172,8 @@ class _TurnUsage: """Last thread token usage Codex reported for the running turn.""" snapshot: ThreadTokenUsage | None = None + # Codex ends every response of a turn before it sends turn/completed. + turn_completed: bool = False def _fills_context_window(usage: ThreadTokenUsage) -> bool: @@ -1197,6 +1203,11 @@ async def _turn_notifications( and not _fills_context_window(payload.token_usage) ): usage.snapshot = payload.token_usage + elif ( + isinstance(payload, TurnCompletedNotification) + and payload.turn.id == handle.id + ): + usage.turn_completed = True yield event finally: await stream.aclose() @@ -1298,11 +1309,12 @@ async def _invoke_thread( raise except (CodexError, RuntimeError, OSError) as error: failure = sdk_failure(error) - # Usage covers the snapshots Codex sent before the failure. Codex sends a response's - # usage after its tool calls finish, so on a timeout the last response is not - # counted if it was still streaming or its tools were still running. if usage.snapshot is not None: failure["usage"] = _json_safe(usage.snapshot) + # Codex sends a response's usage after its tool calls finish. Without + # turn/completed, a response may still have been running, so the last + # snapshot is not the invocation's complete total. + failure["usage_complete"] = usage.turn_completed return failure, False diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index edb139212..bad10f26c 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -1565,17 +1565,8 @@ async def invoke_turns(payload, mock_codex, *turns): return results -@pytest.mark.parametrize( - ("failure", "code"), - [ - ([turn_completed(error="model request failed")], "codex_turn_failed"), - ([TimeoutError("SDK request timed out")], "codex_timed_out"), - ([TransportClosedError("closed")], "codex_connection_failed"), - ], - ids=["failed-turn", "sdk-timeout", "connection-closed"], -) -async def test_failed_invocation_keeps_usage_codex_reported_before_failure( - codex_payload, mock_codex, failure, code +async def test_failed_turn_keeps_usage_codex_reported_before_turn_completed( + codex_payload, mock_codex ): _, failed = await invoke_turns( codex_payload, @@ -1583,12 +1574,15 @@ async def test_failed_invocation_keeps_usage_codex_reported_before_failure( ("Inspect.", turn_events(usage=token_usage(10, 3, cached_input_tokens=2))), ( "Continue.", - [usage_event(token_usage(40, 8, cached_input_tokens=6)), *failure], + [ + usage_event(token_usage(40, 8, cached_input_tokens=6)), + turn_completed(error="model request failed"), + ], ), ) assert failed.status is AgentRunStatus.FAILED - assert failed.error.code == code + assert failed.error.code == "codex_turn_failed" assert failed.usage.to_mapping() == { "input_tokens": 30, "cached_input_tokens": 4, @@ -1597,6 +1591,42 @@ async def test_failed_invocation_keeps_usage_codex_reported_before_failure( "total_tokens": 35, } assert failed.output["usage"]["total"]["inputTokens"] == 40 + assert "usage_complete" not in failed.output + + +@pytest.mark.parametrize( + ("failure", "code"), + [ + (TimeoutError("SDK request timed out"), "codex_timed_out"), + (TransportClosedError("closed"), "codex_connection_failed"), + (None, "codex_turn_failed"), + ], + ids=["sdk-timeout", "connection-closed", "stream-ended-before-turn-completed"], +) +async def test_turn_without_turn_completed_reports_unknown_usage( + codex_payload, mock_codex, failure, code +): + """A response may still have been running, so the last snapshot is not complete.""" + unsettled = [usage_event(token_usage(40, 8, cached_input_tokens=6))] + if failure is not None: + unsettled.append(failure) + first, failed, after = await invoke_turns( + codex_payload, + mock_codex, + ("Inspect.", turn_events(usage=token_usage(10, 3, cached_input_tokens=2))), + ("Continue.", unsettled), + ("Again.", turn_events(usage=token_usage(60, 10, cached_input_tokens=6))), + ) + + assert first.usage.input_tokens == 10 + assert failed.status is AgentRunStatus.FAILED + assert failed.error.code == code + assert failed.usage is None + assert failed.output["usage"]["total"]["inputTokens"] == 40 + assert "usage_complete" not in failed.output + # The interrupted turn's usage never reaches a later invocation. + assert after.error.code == "codex_runtime_unavailable" + assert after.usage is None async def test_context_window_overflow_reports_usage_before_codex_fills_the_window( From 4585ac7ee03f0d055f71fc8dd46eef91778fbd97 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:51:15 -0700 Subject: [PATCH 44/74] fix: report Deep Agents cached input tokens LangChain carries cache reads per model call in usage_metadata.input_token_details.cache_read, and OpenAI raw usage carries them in prompt_tokens_details.cached_tokens, but the adapter never reported cached_input_tokens. Sum cached input per call with the same rule as the other counts: one call without a cache count leaves the sum unknown, and a reported 0 stays 0. Raw OpenAI usage is read first because ChatOpenAI renames the cache key for priority and flex service tiers. Report input_tokens_include_cache=True whenever input tokens are known. LangChain's UsageMetadata.input_tokens counts every input token type; langchain-anthropic adds cache reads and writes back into it, and OpenAI's prompt_tokens already includes cached tokens. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 7 ++ .../deepagents/adapter.py | 30 +++++ tests/adapters/test_deepagents.py | 113 ++++++++++++++++-- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 66a44eece..9e41e8f53 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -252,6 +252,13 @@ and the provider's total is used when present. Other providers and streamed responses use LangChain's `usage_metadata`, which some integrations fill with 0 for an omitted count. +Cached input tokens come from `prompt_tokens_details.cached_tokens` in +OpenAI-shaped raw usage and from `usage_metadata.input_token_details.cache_read` +otherwise. They follow the same rule: one call without a cache count leaves the +invocation's cached count unknown, and a reported 0 stays 0. When input tokens +are known, the adapter reports that they include cached tokens, because +LangChain's `UsageMetadata.input_tokens` counts every input token type. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 37771f419..ba304d6b3 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -770,6 +770,7 @@ async def invoke( "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), + "cached_input_tokens": usage.get("cached_tokens"), } tokens = { name: value @@ -778,6 +779,11 @@ async def invoke( and not isinstance(value, bool) and 0 <= value <= (1 << 64) - 1 } + if "input_tokens" in tokens: + # LangChain's UsageMetadata defines input_tokens as the sum of every + # input token type, cache reads included; OpenAI's raw prompt_tokens + # includes cached tokens too. + tokens["input_tokens_include_cache"] = True cost = usage.get("cost") if ( not isinstance(cost, (int, float)) @@ -1300,6 +1306,9 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: values = [_reported_tokens(call, source) for call in calls] if all(value is not None for value in values): totals[target] = sum(values) + cached = [_reported_cached_tokens(call) for call in calls] + if all(value is not None for value in cached): + totals["cached_tokens"] = sum(cached) costs = [_reported_cost(call) for call in calls] if all(cost is not None for cost in costs): totals["cost"] = math.fsum(costs) @@ -1342,6 +1351,27 @@ def _reported_tokens(call: dict[str, Any], name: str) -> int | None: return None +def _reported_cached_tokens(call: dict[str, Any]) -> int | None: + """Return one call's cache-read input tokens, or ``None`` when not reported. + + OpenAI-shaped raw usage reports them as ``prompt_tokens_details.cached_tokens``; + ``ChatOpenAI`` renames that key for priority and flex service tiers, so the raw + count is read when present. Other responses use LangChain's + ``usage_metadata.input_token_details.cache_read``. + """ + + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if isinstance(raw, dict): + details, key = raw.get("prompt_tokens_details"), "cached_tokens" + else: + usage = call.get("usage") + details = usage.get("input_token_details") if isinstance(usage, dict) else None + key = "cache_read" + value = details.get(key) if isinstance(details, dict) else None + return value if _is_token_count(value) else None + + def _is_token_count(value: Any) -> bool: return ( isinstance(value, int) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index c916cb2a3..5268f3537 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2604,11 +2604,17 @@ async def test_summarization_model_call_usage_is_counted( assert "tools" not in summary_request assert "tools" in agent_request assert first.usage == AgentUsage( - input_tokens=1500, output_tokens=10, total_tokens=1510 + input_tokens=1500, + output_tokens=10, + total_tokens=1510, + input_tokens_include_cache=True, ) assert second.status is AgentRunStatus.SUCCEEDED assert second.usage == AgentUsage( - input_tokens=1700, output_tokens=65, total_tokens=1765 + input_tokens=1700, + output_tokens=65, + total_tokens=1765, + input_tokens_include_cache=True, ) @@ -2658,7 +2664,10 @@ async def test_subagent_model_calls_are_counted( } assert len(subgraphs) == 2 assert result.usage == AgentUsage( - input_tokens=1111, output_tokens=1111, total_tokens=2222 + input_tokens=1111, + output_tokens=1111, + total_tokens=2222, + input_tokens_include_cache=True, ) @@ -2680,7 +2689,10 @@ async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( assert result.status is AgentRunStatus.FAILED assert result.error.code == "deepagents_recursion_limit_reached" assert result.usage == AgentUsage( - input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + input_tokens=10 * calls, + output_tokens=calls, + total_tokens=11 * calls, + input_tokens_include_cache=True, ) @@ -2701,7 +2713,12 @@ def provider_failure(kind: str) -> Any: # usage stays known. pytest.param( "400", - AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + AgentUsage( + input_tokens=120, + output_tokens=8, + total_tokens=128, + input_tokens_include_cache=True, + ), id="400-rejection", ), # The provider may have done work without reporting usage. @@ -2785,12 +2802,17 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): [ pytest.param( [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], - AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + AgentUsage( + input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + ), id="explicit-zero", ), pytest.param( [{"prompt_tokens": 100}], - AgentUsage(input_tokens=100), + AgentUsage(input_tokens=100, input_tokens_include_cache=True), id="input-only", ), pytest.param( @@ -2800,7 +2822,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 100, "completion_tokens": 5}], - AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + AgentUsage( + input_tokens=100, + output_tokens=5, + total_tokens=105, + input_tokens_include_cache=True, + ), id="total-omitted", ), pytest.param( @@ -2810,7 +2837,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], - AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + AgentUsage( + input_tokens=7, + output_tokens=5, + total_tokens=0, + input_tokens_include_cache=True, + ), id="provider-total-zero-kept", ), pytest.param( @@ -2839,6 +2871,69 @@ async def test_partial_usage_reports_keep_missing_counts_unknown( assert result.usage == expected +@pytest.mark.parametrize( + ("cached", "expected"), + [ + pytest.param([40, 60], 100, id="every-call-reports-cache"), + pytest.param([40, None], None, id="one-call-omits-cache"), + pytest.param([0, 0], 0, id="explicit-zero"), + ], +) +async def test_cached_input_tokens_are_known_only_when_every_call_reported_them( + tmp_path, make_payload, openai_endpoint, cached, expected +): + def usage(cached_tokens: int | None) -> dict[str, Any]: + reported: dict[str, Any] = { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + } + if cached_tokens is not None: + reported["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return reported + + openai_endpoint["replies"] += [ + chat_completion(usage(cached[0]), tool_calls=[LS_TOOL_CALL]), + chat_completion(usage(cached[1])), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == AgentUsage( + input_tokens=200, + cached_input_tokens=expected, + input_tokens_include_cache=True, + output_tokens=10, + total_tokens=210, + ) + + +@pytest.mark.parametrize( + ("cache_read", "expected"), + [ + pytest.param(30, 30, id="reported"), + pytest.param(0, 0, id="explicit-zero"), + pytest.param(None, None, id="omitted"), + ], +) +def test_anthropic_cache_reads_are_counted_as_included_input(cache_read, expected): + # langchain-anthropic adds cache reads and writes to input_tokens and keeps an + # omitted cache count out of input_token_details. + from anthropic.types import Usage + from langchain_anthropic.chat_models import _create_usage_metadata + + usage_metadata = _create_usage_metadata( + Usage(input_tokens=10, output_tokens=5, cache_read_input_tokens=cache_read) + ) + totals = adapter._aggregate_usage( + [adapter._message_to_dict(AIMessage("done", usage_metadata=usage_metadata))] + ) + + assert totals.get("cached_tokens") == expected + assert totals["prompt_tokens"] == 10 + (cache_read or 0) + + async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): # A misconfigured MCP server must fail loudly, not be silently dropped. payload = make_payload(tmp_path) From 1ab2640f4f30fa870b62779bd716971a19c88553 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:52:15 -0700 Subject: [PATCH 45/74] fix: count only the report's harness-only server drops in core's marker Core subtracted every plan server missing from an MCP report from the report's servers_truncated count, assuming the helper had dropped them. A plan server can be missing for other reasons, so a harness-only drop could vanish from the count. The report marker now counts only servers outside the adapter's start configuration by definition, since a helper knows its configured servers, and core adds the count directly. A plan server the helper drops keeps its entry with fact_omitted facts. No bundled adapter writes the MCP report yet. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 106 +++++++++++++++------ docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 9 +- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 4 files changed, 82 insertions(+), 37 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index a8ec1873f..6377252ec 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -979,7 +979,7 @@ const REASON_TOO_LARGE: &str = "too_large"; /// the harness name and version, each plan role's served model, each MCP /// server's `runtime` block with its `tool_names_truncated` marker, a /// `harness` server's transport, and the `servers_truncated` marker beside -/// `servers`, whose count core adds to its own only for `harness` entries. A malformed or forged fact reads +/// `servers`, which counts only harness-only servers the helper dropped. A malformed or forged fact reads /// `collection_failed` with reason `invalid_report`, and an omitted one /// reads `unavailable` with reason `fact_omitted`. Core ignores every other /// property, so a report never changes a fact core owns. The string length @@ -1116,11 +1116,12 @@ fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { /// Fill each plan server's `runtime` block, and add a `harness` entry for /// each reported server the plan does not configure. /// -/// The report's `servers_truncated` marker counts servers the adapter's -/// helper dropped from the report. A plan server the helper dropped keeps -/// its entry, with facts that read `fact_omitted`, so core adds to its own -/// marker only the count left after the plan servers the report omits: -/// the `harness` entries that disappear from `what_ran`. A malformed marker +/// The report's `servers_truncated` marker counts only the harness-only +/// servers the adapter's helper dropped from the report: servers outside +/// the `mcp.servers` of the adapter's start configuration, which would +/// otherwise disappear from `what_ran`. A plan server the helper dropped +/// keeps its entry, with runtime facts that read `fact_omitted`, and is not +/// counted. Core adds the marker's count to its own. A malformed marker /// makes the whole `servers` report read `invalid_report`, as a malformed /// `tool_names_truncated` does for `tool_names`. fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { @@ -1139,23 +1140,10 @@ fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) } } } - let plan_servers = &plan.capability_plan.mcp_servers; - if let (Some(marker), Report::Object(reported)) = (helper_marker, &servers) { - let plan_servers_omitted = plan_servers - .keys() - .filter(|name| !reported.contains_key(*name)) - .count() as u64; - let harness_dropped = marker.dropped.saturating_sub(plan_servers_omitted); - if harness_dropped > 0 { - mcp.servers_truncated = merged_marker( - mcp.servers_truncated.take(), - Some(TruncatedEntries { - reason: marker.reason, - dropped: harness_dropped, - }), - ); - } + if let Report::Object(_) = servers { + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), helper_marker); } + let plan_servers = &plan.capability_plan.mcp_servers; for (name, server) in &mut mcp.servers { server.runtime = runtime_facts(&servers.child(name).child("runtime")); } @@ -2667,33 +2655,89 @@ mod tests { } #[test] - fn a_helper_servers_marker_never_counts_plan_servers_the_report_omits() { + fn a_helper_servers_marker_counts_harness_drops_whatever_plan_servers_the_report_omits() { let plan = plan_declaring(&[MCP_REPORT_KEY]); - // The helper dropped `github`, a plan server that keeps its entry, - // and two harness entries. + // `github` was never reported, and the helper dropped the harness-only + // `plugin-docs` for its byte budget. let report = json!({ "servers": {"events": connected_runtime(), "files": connected_runtime()}, - "servers_truncated": {"reason": "byte_budget", "dropped": 3} + "servers_truncated": {"reason": "byte_budget", "dropped": 1} }); let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github"] + ); assert_eq!( what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], report_state("unavailable", "fact_omitted") ); assert_eq!( what_ran["mcp"]["servers_truncated"], - json!({"reason": "byte_budget", "dropped": 2}) + json!({"reason": "byte_budget", "dropped": 1}) ); + } - // A marker that counts only omitted plan servers adds nothing. + #[test] + fn a_helper_servers_marker_adds_to_the_marker_core_already_holds() { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + // Core drops this plan server for its long key when it collects. + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan + .mcp_servers + .insert("s".repeat(129), server); let report = json!({ - "servers": {"events": connected_runtime(), "files": connected_runtime()}, - "servers_truncated": {"reason": "byte_budget", "dropped": 1} + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 2} }); + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); - assert!(what_ran["mcp"].get("servers_truncated").is_none()); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn core_entry_cap_and_byte_budget_drops_follow_a_helper_marker() { + // Entry cap: 30 plan servers leave two places for five harness entries. + let plan_names = numbered_keys("server", 30); + let mut report = mcp_report(&numbered_keys("aa", 5), 0); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 2}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), report); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 5}) + ); + + // Byte budget: harness entries that do not fit add to the helper's + // count. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + let harness_names = numbered_keys("aa", 5); + let mut report = mcp_report(&harness_names, 40); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 4}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), report); + + let harness_kept = origins(&what_ran) + .iter() + .filter(|(_, origin)| origin == "harness") + .count(); + assert!(harness_kept < harness_names.len()); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", + "dropped": 4 + harness_names.len() - harness_kept}) + ); } #[test] diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index ea89dce17..3da1c7241 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 444e2abac..6f572a084 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1102,10 +1102,11 @@ roles. A map that lost entries has a marker next to it, `models_truncated` or `byte_budget`) and the `dropped` count. Each map has one marker: when more than one rule dropped entries, `reason` names the last rule applied, in that order, and `dropped` counts every dropped entry. When the adapter's report drops -servers to fit its own size limit, `mcp.servers_truncated` also counts the ones -only the harness reported, because those disappear; a plan server the report -drops keeps its entry, and its runtime facts read `unavailable` with reason -`fact_omitted`. `dropped` is a JSON integer. The Rust core and the Python SDK +servers to fit its own size limit, its `servers_truncated` marker counts only +the servers the harness reported outside the plan, because those disappear, and +`mcp.servers_truncated` adds that count. A plan server the report drops keeps +its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +`dropped` is a JSON integer. The Rust core and the Python SDK reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is still too large with no MCP server and no model role left, it holds only `schema`, `version`, and `what_ran_truncated` with diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index a8541ec8a..409393ae0 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,7 +230,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From 7501f47c1fecb87525c69202f9d747ded1304dd6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:10:02 -0700 Subject: [PATCH 46/74] docs: define report server drops against the full capability plan The helper rule counted harness-only drops against the adapter's start configuration, which holds only native MCP servers, while core classifies servers against the full capability plan. Every lifecycle start payload already carries that plan, so the rule now uses it, and the rustdoc, SDK guide, adapter tutorial and adapter authoring skill share one wording. Core behavior is unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 12 ++++++------ docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 11 ++++++----- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 6377252ec..29dd2a3c8 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1116,12 +1116,12 @@ fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { /// Fill each plan server's `runtime` block, and add a `harness` entry for /// each reported server the plan does not configure. /// -/// The report's `servers_truncated` marker counts only the harness-only -/// servers the adapter's helper dropped from the report: servers outside -/// the `mcp.servers` of the adapter's start configuration, which would -/// otherwise disappear from `what_ran`. A plan server the helper dropped -/// keeps its entry, with runtime facts that read `fact_omitted`, and is not -/// counted. Core adds the marker's count to its own. A malformed marker +/// When a helper drops server entries to keep `nemo_fabric.mcp.v1` under +/// 64 KiB, the report's `servers_truncated` counts only dropped names absent +/// from the full capability plan supplied at lifecycle start. Core adds that +/// count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its +/// entry, with runtime facts marked `unavailable` for reason `fact_omitted`. +/// A malformed marker /// makes the whole `servers` report read `invalid_report`, as a malformed /// `tool_names_truncated` does for `tool_names`. fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 3da1c7241..47e6895c4 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop server entries to keep `nemo_fabric.mcp.v1` under 64 KiB, `servers_truncated` counts only dropped names absent from the full capability plan supplied at lifecycle start. Core adds that count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its entry, with runtime facts marked `unavailable` for reason `fact_omitted`. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6f572a084..def66e8d3 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1101,11 +1101,12 @@ roles. A map that lost entries has a marker next to it, `models_truncated` or `mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or `byte_budget`) and the `dropped` count. Each map has one marker: when more than one rule dropped entries, `reason` names the last rule applied, in that order, -and `dropped` counts every dropped entry. When the adapter's report drops -servers to fit its own size limit, its `servers_truncated` marker counts only -the servers the harness reported outside the plan, because those disappear, and -`mcp.servers_truncated` adds that count. A plan server the report drops keeps -its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +and `dropped` counts every dropped entry. When an adapter drops server entries +to keep `nemo_fabric.mcp.v1` under 64 KiB, the report's `servers_truncated` +counts only dropped names absent from the full capability plan supplied at +lifecycle start. NeMo Fabric adds that count to +`what_ran.mcp.servers_truncated`. A dropped plan server keeps its entry, with +runtime facts marked `unavailable` for reason `fact_omitted`. `dropped` is a JSON integer. The Rust core and the Python SDK reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is still too large with no MCP server and no model role left, it holds only diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 409393ae0..d1b9cbcb8 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,7 +230,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`. If you drop server entries to keep `nemo_fabric.mcp.v1` under 64 KiB, `servers_truncated` counts only dropped names absent from the full capability plan supplied at lifecycle start. Core adds that count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its entry, with runtime facts marked `unavailable` for reason `fact_omitted`. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From fee4c9287832d1fd24664bfa1848ac784af66bad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:30:54 -0700 Subject: [PATCH 47/74] fix: report zero Deep Agents usage when no model call ran The adapter drops a model call that the provider rejects with a 4xx status other than 408, because the provider spent no tokens on it. When every call was rejected, or no call started, the call list was empty and the usage was reported as unknown, although the invocation provably spent no tokens. The adapter now reports 0 for every token count and the cost in that case. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 15 +++++----- .../deepagents/adapter.py | 6 ++-- tests/adapters/test_deepagents.py | 29 +++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 9e41e8f53..e88b41c04 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -237,13 +237,14 @@ summarization. Agent Protocol subagents (entries with `graph_id`) run in their remote service, and their tokens are not included. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage. A 5xx response, timeout, -cancellation or unfinished call leaves the invocation's counts and cost -unknown, because the provider may have done work without reporting usage. -OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries -to success is not visible to the adapter, and only the final response's -reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; NeMo Fabric configures no such cache. +4xx status (other than 408) contributes no usage, so an invocation that made no +model call, or whose every call was rejected, reports 0 tokens and 0 cost. A +5xx response, timeout, cancellation or unfinished call leaves the invocation's +counts and cost unknown, because the provider may have done work without +reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx +the SDK retries to success is not visible to the adapter, and only the final +response's reported usage counts. Usage replayed from a configured LangChain +model cache would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index ba304d6b3..e5561f215 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1292,11 +1292,11 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: """Sum model-call usage; a count is known only when every call reported it. One call without a count makes the invocation's sum for that count a - partial value, so it stays unknown rather than under-reported. + partial value, so it stays unknown rather than under-reported. Without a + call, because none started or the provider rejected every one, the + invocation spent nothing, so every count and the cost are 0. """ - if not calls: - return None totals: dict[str, Any] = {} for source, target in ( ("input_tokens", "prompt_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 5268f3537..6cdb0f466 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2744,6 +2744,35 @@ async def test_failed_model_call_usage_follows_provider_response( assert result.usage == expected +async def test_rejected_only_model_call_reports_zero_usage( + tmp_path, make_payload, openai_endpoint +): + # The provider rejected the only model call, so the invocation spent nothing. + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == AgentUsage( + input_tokens=0, + cached_input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + cost_usd=0.0, + ) + + +def test_aggregate_usage_without_model_calls_is_zero(): + assert adapter._aggregate_usage([]) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "cost": 0.0, + } + + @pytest.mark.parametrize( ("failure", "expected"), [ From 2ee128d7485f15db1a5b0c66f98cbfdb04b30277 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:31:49 -0700 Subject: [PATCH 48/74] fix: report unknown Deep Agents usage with an Agent Protocol subagent Deep Agents routes a subagent spec with graph_id to its async subagent middleware, which starts and updates tasks on a remote Agent Protocol server. Those model calls never reach the adapter's callbacks, so the reported counts left them out while appearing complete. The adapter blocked such subagents only when a tools policy was configured. When the configured subagents include one with graph_id, the adapter now reports the invocation's token counts and cost as unknown. The local sums stay in output.usage. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 24 ++++++++-------- .../deepagents/adapter.py | 25 ++++++++++++++++- tests/adapters/test_deepagents.py | 28 +++++++++++++++++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index e88b41c04..56f93989d 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -234,17 +234,19 @@ runtime start before an invocation is accepted. Usage sums every chat-model call that runs in the adapter process for the invocation: the main agent, declarative subagents, and Deep Agents summarization. Agent Protocol subagents (entries with `graph_id`) run in their -remote service, and their tokens are not included. A token count or cost is -reported only when every counted call reported it; otherwise it stays unknown. -This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage, so an invocation that made no -model call, or whose every call was rejected, reports 0 tokens and 0 cost. A -5xx response, timeout, cancellation or unfinished call leaves the invocation's -counts and cost unknown, because the provider may have done work without -reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx -the SDK retries to success is not visible to the adapter, and only the final -response's reported usage counts. Usage replayed from a configured LangChain -model cache would be counted as reported; NeMo Fabric configures no such cache. +remote service, where the adapter cannot count their calls, so a runtime +configured with one reports unknown token counts and cost. A token count or +cost is reported only when every counted call reported it; otherwise it stays +unknown. This also holds for a failed invocation. A request the provider +rejects with a 4xx status (other than 408) contributes no usage, so an +invocation that made no model call, or whose every call was rejected, reports 0 +tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call +leaves the invocation's counts and cost unknown, because the provider may have +done work without reporting usage. OpenAI SDK retries happen inside one +LangChain call, so a 5xx the SDK retries to success is not visible to the +adapter, and only the final response's reported usage counts. Usage replayed +from a configured LangChain model cache would be counted as reported; NeMo +Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index e5561f215..38a0eacbf 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -541,6 +541,18 @@ def _validated_deepagents_settings(extra: Any) -> dict[str, Any]: return dict(extra) +def _has_agent_protocol_subagent(subagents: Any) -> bool: + """Whether ``create_deep_agent`` runs a subagent on an Agent Protocol server. + + Deep Agents routes a subagent spec with ``graph_id`` to its async subagent + middleware, which starts and updates tasks on a remote server. + """ + return isinstance(subagents, list) and any( + isinstance(subagent, dict) and "graph_id" in subagent + for subagent in subagents + ) + + def _gate_subagent( subagent: dict[str, Any], enabled: set[str] | None, blocked: set[str] ) -> dict[str, Any]: @@ -612,6 +624,7 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._has_remote_subagent = False async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -662,6 +675,9 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._has_remote_subagent = _has_agent_protocol_subagent( + agent_kwargs.get("subagents") + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -765,7 +781,14 @@ async def invoke( failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} + # An Agent Protocol subagent's model calls run on its remote server and + # never reach this process, so the local sums may undercount the + # invocation and its counts and cost stay unknown. + usage = ( + raw_usage + if isinstance(raw_usage, dict) and not self._has_remote_subagent + else {} + ) token_fields = { "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 6cdb0f466..873126845 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2763,6 +2763,34 @@ async def test_rejected_only_model_call_reports_zero_usage( ) +async def test_agent_protocol_subagent_makes_usage_unknown( + tmp_path, make_payload, openai_endpoint +): + # An Agent Protocol subagent runs its model calls on a remote server, which + # the adapter cannot count, so local sums would undercount the invocation. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append( + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128} + ) + ) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From 06de4c0281439b64a9d05340ebca771278fe5743 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:32:07 -0700 Subject: [PATCH 49/74] docs: say a failed Deep Agents call may have been billed The usage notes said that after a 5xx response, timeout, or cancelled call the provider may have done work. The reason the counts stay unknown is narrower: the provider may have billed the call without reporting usage. The README, the recorder docstring, and a test comment now say that. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 4 ++-- tests/adapters/test_deepagents.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 56f93989d..c1a5a8c98 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -242,7 +242,7 @@ rejects with a 4xx status (other than 408) contributes no usage, so an invocation that made no model call, or whose every call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have -done work without reporting usage. OpenAI SDK retries happen inside one +billed the call without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache would be counted as reported; NeMo diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 38a0eacbf..f886155c9 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1012,8 +1012,8 @@ class ModelCallUsage(BaseCallbackHandler): record when it ends. A request the provider rejects with a 4xx status (other than 408) contributes no usage and is dropped. A 5xx response, timeout, cancellation or unfinished call stays unreported and makes the invocation's - counts and cost unknown, because the provider may have done work without - reporting usage. + counts and cost unknown, because the provider may have billed the call + without reporting usage. """ run_inline = True diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 873126845..eec895d6a 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2721,7 +2721,7 @@ def provider_failure(kind: str) -> Any: ), id="400-rejection", ), - # The provider may have done work without reporting usage. + # The provider may have billed the call without reporting usage. pytest.param("408", None, id="408-request-timeout"), pytest.param("503", None, id="503-server-error"), pytest.param("timeout", None, id="read-timeout"), From e535051318613392a86fcb75cd4a469cedd21b18 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:38:36 -0700 Subject: [PATCH 50/74] style: format the Deep Agents remote subagent check Apply ruff format to the check added in the previous commit. Signed-off-by: mnajafian-nv --- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index f886155c9..9a51bd817 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -548,8 +548,7 @@ def _has_agent_protocol_subagent(subagents: Any) -> bool: middleware, which starts and updates tasks on a remote server. """ return isinstance(subagents, list) and any( - isinstance(subagent, dict) and "graph_id" in subagent - for subagent in subagents + isinstance(subagent, dict) and "graph_id" in subagent for subagent in subagents ) From cb42bfe7d37d76beb93d1c0d8aa7f6bf61edda53 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:45:54 -0700 Subject: [PATCH 51/74] test: keep zero observed Deep Agents calls unknown with a remote subagent The adapter reports 0 when its callbacks observed no billed model call. That 0 holds for the invocation only when every model call runs through those callbacks. Deep Agents runs declarative subagents and summarization inside the graph, where the run config's callbacks reach them, but an Agent Protocol subagent (graph_id) runs on a remote server that the callbacks never see. The adapter already reports unknown usage for a runtime with such a subagent; this test pins that for an invocation whose only observed call was rejected, and the docstring and README state the condition. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 18 +++++++------- .../deepagents/adapter.py | 5 +++- tests/adapters/test_deepagents.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index c1a5a8c98..6340b4ce3 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -238,15 +238,15 @@ remote service, where the adapter cannot count their calls, so a runtime configured with one reports unknown token counts and cost. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider -rejects with a 4xx status (other than 408) contributes no usage, so an -invocation that made no model call, or whose every call was rejected, reports 0 -tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call -leaves the invocation's counts and cost unknown, because the provider may have -billed the call without reporting usage. OpenAI SDK retries happen inside one -LangChain call, so a 5xx the SDK retries to success is not visible to the -adapter, and only the final response's reported usage counts. Usage replayed -from a configured LangChain model cache would be counted as reported; NeMo -Fabric configures no such cache. +rejects with a 4xx status (other than 408) contributes no usage, so without an +Agent Protocol subagent an invocation that made no model call, or whose every +call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have billed the call without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 9a51bd817..bfda396dc 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1316,7 +1316,10 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: One call without a count makes the invocation's sum for that count a partial value, so it stays unknown rather than under-reported. Without a call, because none started or the provider rejected every one, the - invocation spent nothing, so every count and the cost are 0. + observed calls spent nothing, so every count and the cost are 0. That 0 + covers the invocation only when every model call runs through these + callbacks; the caller makes usage unknown when an Agent Protocol subagent + can run models on a remote server. """ totals: dict[str, Any] = {} diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index eec895d6a..816ccb1ec 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2791,6 +2791,30 @@ async def test_agent_protocol_subagent_makes_usage_unknown( assert result.usage is None +async def test_rejected_only_call_with_agent_protocol_subagent_is_unknown( + tmp_path, make_payload, openai_endpoint +): + # Zero observed calls prove nothing when a remote subagent can run models + # that the adapter's callbacks never see. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From ce744150ea1e26be5b6b8a42a26d95b8a533d77f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:04:44 -0700 Subject: [PATCH 52/74] fix: preserve empty OpenShell environment arguments Signed-off-by: mnajafian-nv --- examples/openshell/run-demo.sh | 4 ++- tests/scripts/test_openshell_launcher.py | 44 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/openshell/run-demo.sh b/examples/openshell/run-demo.sh index 56d2b534f..4fe36156e 100644 --- a/examples/openshell/run-demo.sh +++ b/examples/openshell/run-demo.sh @@ -272,9 +272,11 @@ run_consumer() { if [[ "${EXAMPLE_MODE}" == caller_owned || "${EXAMPLE_MODE}" == both ]]; then CALLER_SANDBOX_ATTEMPTED=1 + # Guard empty arrays for Bash 3.2's set -u behavior without adding empty args. "${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox create --name "${SANDBOX_NAME}" --from "${RUNTIME_IMAGE}" \ - --policy "${POLICY_FILE}" "${MODEL_ENV_ARGS[@]}" "${SERVER_ENV_ARGS[@]}" \ + --policy "${POLICY_FILE}" ${MODEL_ENV_ARGS[@]+"${MODEL_ENV_ARGS[@]}"} \ + ${SERVER_ENV_ARGS[@]+"${SERVER_ENV_ARGS[@]}"} \ --detach --no-tty -- fabric-agent-ctl serve SANDBOX_JSON="$("${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox get "${SANDBOX_NAME}" --output json)" diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 317b8052d..15ab68caa 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shlex import shutil import socket import subprocess @@ -172,7 +173,22 @@ def _run(launcher, workload, env): ) -def test_generic_workload_gets_ownership_and_consumer_args(launcher): +@pytest.mark.parametrize( + ("model_env_args", "server_env_args"), + [ + pytest.param([], None, id="default-server-env"), + pytest.param([], [], id="no-env"), + pytest.param(["--env", "MODEL_HINT=words with spaces"], [], id="no-server-env"), + pytest.param( + ["--env", "MODEL_HINT=words with spaces"], + ["--env", "SERVER_HINT=words with spaces"], + id="quoted-env", + ), + ], +) +def test_generic_workload_gets_ownership_and_consumer_args( + launcher, model_env_args, server_env_args +): _, root, _ = launcher workload = root / "third-workload" workload.mkdir() @@ -185,6 +201,12 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): stage_workload_image() { :; } run_workload_consumer() { local mode="$1"; shift; uv run --no-sync consumer "$mode" "$@"; } """ + + f"MODEL_ENV_ARGS=({shlex.join(model_env_args)})\n" + + ( + f"SERVER_ENV_ARGS=({shlex.join(server_env_args)})\n" + if server_env_args is not None + else "" + ) ) completed = _run(launcher, hook, {"OPENSHELL_EXAMPLE_MODE": "both"}) @@ -195,6 +217,26 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): assert all(call["args"][-2:] == ["--input", "words with spaces"] for call in calls) assert all(call["provider"].endswith("/fabric-environment-openshell") for call in calls) assert not (root / "sandbox-state.json").exists() + create = next( + operation["args"] + for operation in _operations(root) + if operation["command"] == "openshell" + and "sandbox" in operation["args"] + and "create" in operation["args"] + ) + expected_server_env = ( + server_env_args + if server_env_args is not None + else [ + "--env", + "ADAPTER_PYTHON=/opt/fabric/bin/python", + "--env", + "TMPDIR=/sandbox/tmp", + ] + ) + assert create[create.index("--policy") + 2 : create.index("--detach")] == ( + model_env_args + expected_server_env + ) @pytest.mark.parametrize("mode", ["caller_owned", "fabric_owned"]) From 790f3693aaebf9465bc224bd76da41fb3d81d420 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:10:57 -0700 Subject: [PATCH 53/74] fix: avoid premature Relay gateway startup failures Signed-off-by: mnajafian-nv --- adapters/python/claude/README.md | 2 +- adapters/python/codex/README.md | 4 +- adapters/python/common/README.md | 14 +++++ .../common/relay_gateway.py | 21 +++++-- tests/_utils/utils.py | 22 +++++-- tests/adapters/test_adapaters_common_utils.py | 4 +- .../test_adapters_common_relay_gateway.py | 63 +++++++++++++++++++ 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index a6b8ac271..bfa13847c 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -169,7 +169,7 @@ config.enable_relay( ``` For each Relay-enabled Claude runtime, NeMo Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it with the runtime. NeMo Fabric passes the +waits up to 30 seconds for its health endpoint, and stops it with the runtime. NeMo Fabric passes the gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`, and passes the selected explicit model endpoint to the gateway as its Anthropic upstream. It also stages a runtime-scoped Claude diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 3cf1de779..ba7eab1b7 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,7 +154,9 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. +separately. Gateway readiness has a 30-second budget covering configuration +validation and required worker initialization. Startup fails on early process exit +or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 17e537cd5..fb8400337 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -122,6 +122,20 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Relay Gateway Readiness + +The Claude and Codex adapters wait up to 30 seconds for the supervised Relay +gateway's HTTP health endpoint. This budget includes configuration validation +and required worker initialization. Early process exit fails startup immediately; +a gateway that never becomes healthy fails when the budget expires and is stopped. +The adapters retain its log for diagnosis. Local Python hosts separately enforce +a 90-second adapter startup limit. + +Relay 0.10.3 includes an +[upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). +Use that release to obtain the fix on 0.10 installations. Fabric continues to +accept the supported `>=0.9,<0.11` CLI range. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 45cf04faa..959f54fa7 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -20,7 +20,10 @@ from nemo_fabric_adapters.common.relay_version import is_supported_relay_version -RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +# Readiness includes plugin preflight and worker startup before health is served. +# This operational budget is separate from Relay's inner request timeouts and +# Fabric's outer adapter-start limit. +RELAY_STARTUP_TIMEOUT_SECONDS = 30.0 RELAY_STOP_TIMEOUT_SECONDS = 5.0 RELAY_VERSION_TIMEOUT_SECONDS = 5.0 @@ -110,24 +113,32 @@ def wait_for_relay_gateway( process: subprocess.Popen[Any], health_url: str, *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, + timeout: float = RELAY_STARTUP_TIMEOUT_SECONDS, ) -> None: """Wait until the Relay health endpoint succeeds or startup fails.""" deadline = time.monotonic() + timeout - while time.monotonic() < deadline: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break returncode = process.poll() if returncode is not None: raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) try: - with urllib.request.urlopen(health_url, timeout=1) as response: + with urllib.request.urlopen( + health_url, timeout=min(1.0, remaining) + ) as response: if 200 <= response.status < 300: return except (OSError, urllib.error.URLError): pass - time.sleep(0.1) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.1, remaining)) raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 9b7ca63be..aa5521ae8 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -22,11 +22,17 @@ def assert_atof_skill_selection( output: Mapping[str, Any], expected_skill: str | None ) -> None: records = atof_records(output) - tool_records = [record for record in records if record["category"] == "tool"] + tool_records = [ + record + for record in records + if record["kind"] == "scope" and record["category"] == "tool" + ] llm_end_records = [ record for record in records - if record["category"] == "llm" and record["scope_category"] == "end" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "end" ] serialized_tool_records = json.dumps(tool_records).replace("\\\\", "/") serialized_skill_calls = json.dumps(tool_records + llm_end_records).replace( @@ -55,7 +61,9 @@ def assert_atof_model(output: Mapping[str, Any], expected_model: str) -> None: llm_starts = [ record for record in events - if record["category"] == "llm" and record["scope_category"] == "start" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "start" ] assert llm_starts, events assert {record["data"]["content"]["model"] for record in llm_starts} == { @@ -81,7 +89,9 @@ def assert_semantic_relay_artifacts( llm_starts = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "start" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "start" ] assert llm_starts, events assert all( @@ -92,7 +102,9 @@ def assert_semantic_relay_artifacts( llm_ends = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "end" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "end" ] assert llm_ends, events assert any(_relay_event_total_tokens(event) > 0 for event in llm_ends) diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index d1805f12b..0be298052 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -1332,8 +1332,10 @@ async def test_relay_initializes_v4_atof_atif_config( } common_utils.validate_relay_observability(plugin_config) assert plugin.validate_exact(plugin_config)["config"]["diagnostics"] == [] + # Activation includes discovered host plugins, so compare the same layers. + expected_diagnostics = plugin.validate(plugin_config)["config"]["diagnostics"] async with plugin.activate(plugin_config) as activation: - assert activation.report["config"]["diagnostics"] == [] + assert activation.report["config"]["diagnostics"] == expected_diagnostics def test_relay_validates_all_v4_otlp_fields(): diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index c307254ae..e478f85de 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -271,3 +271,66 @@ def test_stop_relay_gateway_is_idempotent_for_exited_process(): process.terminate.assert_not_called() process.kill.assert_not_called() + + +@pytest.fixture(name="gateway_clock") +def gateway_clock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + mock_clock = MagicMock(spec=relay_gateway.time) + mock_clock.monotonic.return_value = 0.0 + + def advance(seconds: float) -> None: + mock_clock.monotonic.return_value += seconds + + mock_clock.sleep.side_effect = advance + monkeypatch.setattr(relay_gateway, "time", mock_clock) + return mock_clock + + +def test_wait_for_relay_gateway_allows_preflight_before_worker_readiness( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += timeout + if gateway_clock.monotonic() < 14.0: + raise relay_gateway.urllib.error.URLError("still initializing") + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + assert 14.0 <= gateway_clock.monotonic() < 30.0 + + +@pytest.mark.parametrize( + ("probe_elapsed", "expected_sleep"), [(0.25, None), (0.20, 0.05)] +) +def test_wait_for_relay_gateway_limits_each_probe_to_remaining_startup_budget( + monkeypatch, gateway_clock, probe_elapsed, expected_sleep +): + process = MagicMock() + process.poll.return_value = None + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += probe_elapsed + raise relay_gateway.urllib.error.URLError("not ready") + + mock_probe = MagicMock(side_effect=probe_health) + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", mock_probe) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + mock_probe.assert_called_once_with("http://127.0.0.1:43210/healthz", timeout=0.25) + if expected_sleep is None: + gateway_clock.sleep.assert_not_called() + else: + gateway_clock.sleep.assert_called_once_with(pytest.approx(expected_sleep)) + assert gateway_clock.monotonic() == pytest.approx(0.25) From 5db49175c27b37f704350cfbfa6fc51bdba1fd0e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:41:46 -0700 Subject: [PATCH 54/74] docs: use the NeMo Fabric product name Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index fb8400337..919bdd233 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -133,7 +133,7 @@ a 90-second adapter startup limit. Relay 0.10.3 includes an [upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). -Use that release to obtain the fix on 0.10 installations. Fabric continues to +Use that release to obtain the fix on 0.10 installations. NeMo Fabric continues to accept the supported `>=0.9,<0.11` CLI range. ## Relay Request Correlation From 1272ea326b185a96d5349ab09ee3b5181f8dc20e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:54:14 -0700 Subject: [PATCH 55/74] docs: use NeMo Fabric naming in Codex usage guidance Signed-off-by: mnajafian-nv --- adapters/python/codex/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index ad377cee9..99a682dc4 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -85,7 +85,7 @@ A completed failed turn keeps accounted usage. Without `turn/completed`, complet The adapter requests `experimentalRawEvents` when starting its thread through the SDK's generic request API. This observational flag and the private turn collector are qualified against `openai-codex==0.154.0` and its matching app-server. Raw `usageMetadata.metadata` preserves the provider's field presence before Codex zero-fills optional details. If no raw report arrives, the adapter does not certify a cumulative snapshot as complete usage. Recheck these boundaries when either pinned component changes. -An actual subagent dispatch leaves that invocation's descendant coverage unverified. This gap does not invalidate independent later parent turns. Raw reports for another turn are ignored, and returned results remain frozen. Automatic child completion requests no new turn (`trigger_turn=false`); Codex can still start pending work when a durable sleep is outstanding. Unused delegation capabilities do not create gaps. Model calls hidden inside external tools and unreported internal attempts remain outside this notification boundary. The qualified accounting boundary is the parent turn initiated by the Fabric invocation. The SDK can also join an already-active regular turn, and its response does not expose a joined-turn flag. Independently running parent turns that can be joined are outside this qualified ownership boundary. An exact `turnId` filters other turns; it does not establish who initiated a pre-existing turn. The reports do not identify each served model, so the adapter supplies no per-model allocation or provider cost. Core does not price this evidence using the requested model. +An actual subagent dispatch leaves that invocation's descendant coverage unverified. This gap does not invalidate independent later parent turns. Raw reports for another turn are ignored, and returned results remain frozen. Automatic child completion requests no new turn (`trigger_turn=false`); Codex can still start pending work when a durable sleep is outstanding. Unused delegation capabilities do not create gaps. Model calls hidden inside external tools and unreported internal attempts remain outside this notification boundary. The qualified accounting boundary is the parent turn initiated by the NeMo Fabric invocation. The SDK can also join an already-active regular turn, and its response does not expose a joined-turn flag. Independently running parent turns that can be joined are outside this qualified ownership boundary. An exact `turnId` filters other turns; it does not establish who initiated a pre-existing turn. The reports do not identify each served model, so the adapter supplies no per-model allocation or provider cost. Core does not price this evidence using the requested model. ## Configuration From cbb06993ec9db38d0b7e2dfdd072c94eecfd16ae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:59:52 -0700 Subject: [PATCH 56/74] fix: validate recovered usage and optional pricing rates Signed-off-by: mnajafian-nv --- .../generated/usage-accounting-vectors.ts | 180 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 7 +- crates/fabric-core/src/usage_normalization.rs | 23 +++ docs/adapter-contract/tutorials/results.md | 2 +- .../nemo_fabric.types.md | 1 + docs/sdk/python.mdx | 3 - schemas/SCHEMA.md | 9 +- .../src/nemo_fabric/types.py | 7 +- tests/adapters/test_deepagents_accounting.py | 4 +- tests/docs/test_python_api_docs.py | 4 +- .../usage_accounting/conformance.json | 180 ++++++++++++++++++ tests/python/test_model_usage.py | 22 ++- 12 files changed, 426 insertions(+), 16 deletions(-) diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts index 05a913834..cbc90c193 100644 --- a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -493,6 +493,186 @@ export const accountingVectors = [ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] satisfies { name: string; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..b03484eee 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2324,7 +2324,7 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(crate::model_usage::normalize_model_usage) .transpose() { Ok(usage) => usage.flatten(), @@ -6114,7 +6114,7 @@ for line in sys.stdin: #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { - let (_root, plan) = local_host_plan("accounted_usage_result"); + let (root, plan) = local_host_plan("accounted_usage_result"); let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); let usage = result.model_usage.expect("supported accounting"); @@ -6129,6 +6129,7 @@ for line in sys.stdin: assert_eq!(usage.cost.estimated.observed_nano_usd, None); assert!(usage.cost.pricing.is_none()); assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); } #[test] @@ -6688,7 +6689,7 @@ for line in sys.stdin: let diagnostic = serde_json::to_string(events[0]).unwrap(); assert!(!diagnostic.contains("private-accounting-identity")); assert!(!diagnostic.contains("request totals")); - let _ = root; + let _ = fs::remove_dir_all(root); } } } diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..f8924207e 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -348,6 +348,7 @@ pub fn describe_model_usage(usage: &AgentUsage) -> Result Result Result input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + // An unconvertible aggregate must not erase a qualified model/request subset. // Contradictions and overflow stay errors; these reasons only lack a usable // normalized observation. Recovery never certifies a complete invocation. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..f361e00ba 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -108,7 +108,7 @@ A model aggregate is not a single request. No adapter should fabricate call boundaries, hide execution features or change retry settings to make usage look complete. Costs computed from a price table never become `cost_usd`. -## 2. Separate Failure Classes +### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 57ba50589..83cdfab2b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2634,6 +2634,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - `artifacts`: Normalized artifact manifest. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..f55fe8ba9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1194,9 +1194,6 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -The bundled Deep Agents adapter supplies this evidence for its documented collection scope. - - `result.model_usage` is present when the adapter supplies the declared `nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently nullable counts: input includes cache reads and writes, output includes diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 6e90b6cec..8f43fc0a8 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,9 +40,10 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 72b8ed64d..bea167f4a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2587,8 +2587,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: raise FabricConfigError("invalid applied pricing fingerprint") rates = _mapping(data["rates"], "pricing rates") - if set(rates) != {"input", "output", "cache_read", "cache_write"}: + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) for key, value in rates.items(): if value is not None or key in {"input", "output"}: _usage_counter(value, "pricing rate") @@ -2902,6 +2905,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index 5ba450330..dccaf29a3 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -365,8 +365,8 @@ async def test_non_model_failure_keeps_completed_call_usage(make_model): @pytest.fixture(name="make_runtime") async def make_runtime_fixture(tmp_path, monkeypatch): runtimes = [] - os.environ["SCRIPTED_MODEL_KEY"] = "unused-scripted" - os.environ.pop("OPENAI_PROXY", None) + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) async def make(replies, *, identity="runtime", profile=None, barrier=None): ledger = [] diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 73a2c51ac..6fbcbfec2 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -177,7 +177,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: for module, names in functions.items(): page = REFERENCE_DIR / f"{module}.md" documented = { - match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } assert names <= documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") @@ -274,7 +274,7 @@ def test_generated_signatures_preserve_python_callable_semantics(): page = REFERENCE_DIR / f"{module_name}.md" rendered = { match.group(1): match.group(2) - for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } for function_name in function_names: function = getattr(module, function_name) diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json index dc9d5cf47..b5e026d61 100644 --- a/tests/fixtures/usage_accounting/conformance.json +++ b/tests/fixtures/usage_accounting/conformance.json @@ -484,5 +484,185 @@ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 93d31dd63..fb018ffaa 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -10,7 +10,7 @@ from nemo_fabric.accounting import summarize_usage from nemo_fabric.errors import FabricConfigError -from nemo_fabric.types import ModelUsage, PerModelUsage, TokenCounts +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts def test_per_model_complete_counts_have_the_public_token_types(): @@ -472,3 +472,23 @@ def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} usage["models"] = [per_model_usage] assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + assert parsed.rates == { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } From 559ce0a558bb9faaa258b7ec17b1ac870b9d3248 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:05:43 -0700 Subject: [PATCH 57/74] fix: preserve fractional cost precision across usage reducers Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 38 ++++++++++++++++++- .../usage_accounting/reducer-cases.json | 36 ++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 726b5aff0..cb8f0dd9f 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -44,6 +44,42 @@ function validate(record: UsageObservation): void { } } +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { const reported: ReportedUsage = {}; const observed: ReportedUsage = {}; @@ -58,7 +94,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage let value: number | undefined; if (values.length) { if (key === "cost_usd") { - const total = values.reduce((sum, item) => sum + item, 0); + const total = sumCosts(values); if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; } else if (key === "peak_request_input_tokens") { value = values.reduce((maximum, item) => Math.max(maximum, item)); diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index e79697c63..cba63db2e 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -437,5 +437,41 @@ "observed_input": 100, "exhaustive": true } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } } ] From 2ae6b9ebfd847ad47ffa05695d7a33c6630ece23 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:32:12 -0700 Subject: [PATCH 58/74] test: respect NOOA Python support in provenance cases Signed-off-by: mnajafian-nv --- tests/adapters/test_adapter_provenance.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/adapters/test_adapter_provenance.py b/tests/adapters/test_adapter_provenance.py index a210150e1..74e2537c8 100644 --- a/tests/adapters/test_adapter_provenance.py +++ b/tests/adapters/test_adapter_provenance.py @@ -7,6 +7,7 @@ import asyncio import importlib +import sys from types import SimpleNamespace from unittest.mock import AsyncMock @@ -22,6 +23,12 @@ from nemo_fabric_adapters.common import provenance +_NOOA_PYTHON = pytest.mark.skipif( + not ((3, 12) <= sys.version_info[:2] < (3, 14)), + reason="NOOA requires Python 3.12 or 3.13", +) + + @pytest.mark.parametrize( ("module", "runtime_type"), [ @@ -30,8 +37,8 @@ ("deepagents", "DeepAgentsRuntime"), ("hermes", "HermesRuntime"), ("mini_swe_agent", "MiniSweAgentRuntime"), - ("nooa", "NooaRuntime"), - ("nooa.bench", "BenchRuntime"), + pytest.param("nooa", "NooaRuntime", marks=_NOOA_PYTHON), + pytest.param("nooa.bench", "BenchRuntime", marks=_NOOA_PYTHON), ("openhands", "OpenHandsRuntime"), ("remote_agent", "RemoteAgentRuntime"), ], From a6318d4dede15289bbf31485296c6bec8bc524b0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:24:17 -0700 Subject: [PATCH 59/74] chore: check packaged usage schema declarations Keep packaged adapter declarations in the canonical usage-schema synchronization and drift check. Prevent the check from passing with a stale packaged schema. Signed-off-by: mnajafian-nv --- scripts/contracts/generate_usage_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py index 49bb52bef..3b5cbd19c 100644 --- a/scripts/contracts/generate_usage_accounting.py +++ b/scripts/contracts/generate_usage_accounting.py @@ -22,6 +22,7 @@ def synchronize(*, check: bool) -> None: descriptors = [ *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), ] for path in descriptors: original = path.read_text() From 630820f93384bf5ce9729edd1e2c9c18d63f1999 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:44:48 -0700 Subject: [PATCH 60/74] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 93b00ecdbf82a618e8b6947692b9ea333f59f38b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 17:23:32 -0700 Subject: [PATCH 61/74] fix: expose omitted optional SDK usage attributes Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 16 +++++++- .../src/nemo_fabric/types.py | 12 +++++- tests/python/test_model_usage.py | 39 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 83cdfab2b..b930369db 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2067,7 +2067,6 @@ The mapping exposes the following typed fields: | `normalization_version` | `str` | | `assumption_policy` | `str` | | `bundled_catalog_sha256` | `str` | -| `user_catalog_sha256` | `str \| None` | ### method `__init__` @@ -2088,6 +2087,13 @@ Return an immutable view of preserved extension fields. +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + --- ### classmethod `from_mapping` @@ -2475,7 +2481,6 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | | `model` | `str` | -| `provider` | `str \| None` | | `input_tokens` | `int \| None` | | `cached_input_tokens` | `int \| None` | | `cache_write_input_tokens` | `int \| None` | @@ -2504,6 +2509,13 @@ Return an immutable view of preserved extension fields. +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + --- ### classmethod `from_mapping` diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f5a52cc4c..2dfcd259b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2527,7 +2527,6 @@ class PricingBasis(FabricMapping): normalization_version: str assumption_policy: str bundled_catalog_sha256: str - user_catalog_sha256: str | None _fields = frozenset( { "basis", @@ -2539,6 +2538,11 @@ class PricingBasis(FabricMapping): } ) + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if set(data).difference(cls._fields) or any( @@ -2796,7 +2800,6 @@ class PerModelUsage(FabricMapping): """ model: str - provider: str | None input_tokens: int | None cached_input_tokens: int | None cache_write_input_tokens: int | None @@ -2809,6 +2812,11 @@ class PerModelUsage(FabricMapping): {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} ) + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["model"] = _required_text(data.get("model"), "model") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 805263033..4c731b546 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -496,3 +496,42 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): assert json.loads(json.dumps(rates)) == expected rates["input"] = 999 assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source From 101bce6206a4ce6663e1fc876f99c35404de6fe8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 20:53:44 -0700 Subject: [PATCH 62/74] fix: reject Relay exit during successful health probe Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 6 ++++-- .../nemo_fabric_adapters/common/relay_gateway.py | 5 +++++ .../adapters/test_adapters_common_relay_gateway.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 919bdd233..df9e8aa93 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -126,8 +126,10 @@ runtime. The Claude and Codex adapters wait up to 30 seconds for the supervised Relay gateway's HTTP health endpoint. This budget includes configuration validation -and required worker initialization. Early process exit fails startup immediately; -a gateway that never becomes healthy fails when the budget expires and is stopped. +and required worker initialization. Process exit is checked before each health +probe and again before accepting a successful response. An exit during an +unsuccessful probe is detected at the next poll. A gateway that never becomes +healthy fails when the budget expires and is stopped. The adapters retain its log for diagnosis. Local Python hosts separately enforce a 90-second adapter startup limit. diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 959f54fa7..1a57c5fe4 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -132,6 +132,11 @@ def wait_for_relay_gateway( health_url, timeout=min(1.0, remaining) ) as response: if 200 <= response.status < 300: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index e478f85de..fbd9ca893 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -239,6 +239,19 @@ def test_wait_for_relay_gateway_reports_early_exit(): ) +def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch): + process = MagicMock() + process.poll.side_effect = [None, 17] + response = MagicMock() + response.__enter__.return_value.status = 200 + monkeypatch.setattr( + relay_gateway.urllib.request, "urlopen", MagicMock(return_value=response) + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2014cbe68af1259c837ab3da256883ff3dcd4a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:28:53 -0700 Subject: [PATCH 63/74] fix: exclude unknown keys from request usage Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 7 ++++++- adapter-contract/typescript/test/usage-reducer.test.js | 8 ++++++++ adapters/python/deepagents/README.md | 7 +++++++ docs/sdk/python.mdx | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index cb8f0dd9f..16c8cc5ae 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -144,7 +144,12 @@ function aligned(records: readonly UsageObservation[]): boolean { function requestUsage(record: UsageObservation): ReportedUsage { // Retain uncertain literal counts in model observations, not complete calls. - const fields = { ...record.reported }; + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; return fields; } diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index e83f1ad8b..de7fbdfb2 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -46,3 +46,11 @@ test("scoped request gaps retain independent fields without certifying cache", ( assert.equal(model.requests[0].gap, undefined); assert.equal(model.requests[0].tokens_aligned, true); }); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -241,6 +241,13 @@ The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimension observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and reasoning conventions. An omitted count stays unknown; a reported zero stays zero. +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + For adapter-created OpenAI-compatible clients, the collector observes sync and async calls and their HTTP attempts without changing requests or retries. Normal reported calls stay known. An actual failed or unfinished attempt without usable diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f55fe8ba9..d47a7bdb1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1229,7 +1229,7 @@ accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. `cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; +amounts use integer nano-USD. Catalog estimates are not available; `cost.estimated` has no amounts and no typed pricing setting is exposed. The stable result shape retains optional pricing-basis fields for persisted evidence. From c038b361008beab1d78f90289de03fc1519a59f2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:57:07 -0700 Subject: [PATCH 64/74] fix: validate reported MCP transport facts Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 916efce53..12f0dcc91 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1162,7 +1162,8 @@ fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) transport: entry .get("transport") .and_then(Value::as_str) - .map(str::to_lowercase), + .map(str::to_lowercase) + .filter(|transport| is_code(transport)), endpoint: Fact::missing( FactState::NotApplicable, REASON_NOT_CONFIGURED, @@ -2457,6 +2458,22 @@ mod tests { ); } + #[test] + fn long_reported_transport_is_discarded_without_dropping_its_server() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": { + "plugin-docs": {"transport": "X".repeat(60 * 1024), "runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert!(what_ran["mcp"]["servers"]["plugin-docs"]["transport"].is_null()); + assert_eq!( + what_ran["mcp"]["servers"]["plugin-docs"]["origin"], + "harness" + ); + } + #[test] fn a_malformed_helper_marker_invalidates_tool_names() { let plan = plan_declaring(&[MCP_REPORT_KEY]); From 835598987388966bcd8195f711b4824dba92858e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:56:59 -0700 Subject: [PATCH 65/74] fix: avoid repeating a timed-out Hermes version probe Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/hermes/adapter.py | 4 --- tests/adapters/test_hermes_adapter.py | 30 ++++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 54ae775d7..0f280a386 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -458,10 +458,6 @@ def clear_active_invoke_task( code="hermes_reported_failure", message="Hermes did not complete the invocation", ) - if self._harness_version.get("reason") == "timeout": - # Pick up a version call that finished after its limit. While it - # still runs, the probe starts no new one. - self._harness_version = await probe_hermes_version(self._version_probe) return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index c200cc0ce..3bd123d22 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -17,7 +17,7 @@ from io import StringIO from pathlib import Path from types import ModuleType -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from nemo_fabric_adapter_contract.models import AgentConfig @@ -1849,6 +1849,34 @@ def blocking() -> str: assert calls == ["called"] +async def test_timeout_version_probe_is_not_repeated_after_turn(monkeypatch): + runtime = adapter.HermesRuntime() + runtime._started = True + runtime._runtime_id = "runtime-version-timeout" + runtime._agent_config = _agent_config( + {"models": {"default": {"provider": "test", "model": "test-model"}}} + ) + runtime._model_config = runtime._agent_config.models["default"] + runtime._agent = MagicMock() + runtime._mcp_authentication_checked = True + runtime._harness_version = _version_state("collection_failed", "timeout") + monkeypatch.setattr( + adapter, + "_invoke_hermes_turn", + lambda **_kwargs: ({"completed": True, "final_response": "done"}, ""), + ) + probe = AsyncMock(return_value=_version_state("collection_failed", "timeout")) + monkeypatch.setattr(adapter, "probe_hermes_version", probe) + + context = _runtime_context(runtime_id=runtime._runtime_id) + for message in ("first", "second"): + result = await runtime.invoke(AgentRunRequest(input=message), context) + assert result.extensions["nemo_fabric.versions.v1"]["harness"]["version"] == ( + _version_state("collection_failed", "timeout") + ) + probe.assert_not_awaited() + + def test_hermes_version_keeps_the_commit_and_dirty_marker(monkeypatch): version_info = ModuleType("hermes_cli.version_info") version_info.get_version_info = lambda: MagicMock( # type: ignore[attr-defined] From 16c5dbcb0849b48908542a5aab37849e9dda3918 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 23:34:24 -0700 Subject: [PATCH 66/74] docs: name NeMo Fabric in Codex usage scope Signed-off-by: mnajafian-nv --- docs/integrations/harness/codex.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/harness/codex.mdx b/docs/integrations/harness/codex.mdx index d048ed483..ffd025573 100644 --- a/docs/integrations/harness/codex.mdx +++ b/docs/integrations/harness/codex.mdx @@ -146,7 +146,7 @@ The adapter requests the observational `experimentalRawEvents` thread flag throu Actual subagent dispatch creates a descendant coverage gap for its initiating invocation. Independent later parent turns retain their own reported usage. Raw reports from a different turn are ignored, and earlier returned results remain frozen. Automatic child completion requests no new turn (`trigger_turn=false`); Codex can still start pending work when a durable sleep is outstanding. Unused delegation capabilities do not create gaps. Unreported internal attempts and model calls inside external tools are outside the notification boundary. -The qualified scope is a parent turn initiated by the Fabric invocation. The SDK can join an already-active regular turn, and its response does not expose whether a turn was joined. Independently running parent turns that can be joined are outside this qualified ownership boundary. An exact `turnId` does not prove who initiated a pre-existing turn. +The qualified scope is a parent turn initiated by the NeMo Fabric invocation. The SDK can join an already-active regular turn, and its response does not expose whether a turn was joined. Independently running parent turns that can be joined are outside this qualified ownership boundary. An exact `turnId` does not prove who initiated a pre-existing turn. Raw reports preserve request boundaries but provide no served-model allocation. Core does not estimate cost from the requested model. The adapter reports no provider cost. From 11f72a48ccae09da27dd424bd8879c5815b6f65f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 00:03:45 -0700 Subject: [PATCH 67/74] fix: enforce Relay gateway startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..957b2f967 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,24 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline(monkeypatch, gateway_clock): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From e3a2355b585e1eb231a7395cb1f4e7945c76d12b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 01:41:33 -0700 Subject: [PATCH 68/74] fix: classify empty usage coverage as unverified Signed-off-by: mnajafian-nv --- .../python/src/nemo_fabric_adapter_contract/usage.py | 2 +- adapter-contract/typescript/src/usage.ts | 2 +- adapter-contract/typescript/test/usage-reducer.test.js | 4 ++++ tests/fixtures/usage_accounting/reducer-cases.json | 10 ++++++++++ tests/python/test_usage_reducer_parity.py | 3 +++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py index d28998d36..f296d5d21 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -79,7 +79,7 @@ def _reduce( if total is not None and not causes: complete[key] = total elif not causes: - causes.add(GapReason.MISSING_USAGE) + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) for cause in sorted(causes): gaps[cause].append(dimension) for flag, counter in ( diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 16c8cc5ae..bf75221c4 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -111,7 +111,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage if (values.length !== records.length) causes.add("missing_usage"); if (values.length && value === undefined) causes.add("scope_unverified"); if (value !== undefined && !causes.size) reported[key] = value; - else if (!causes.size) causes.add("missing_usage"); + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); for (const reason of causes) gap(reason, key); } for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index de7fbdfb2..4c6e7040c 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -12,6 +12,10 @@ for (const vector of cases) { const usage = usageFromObservations(vector.records); const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } const actual = field === "exhaustive" ? accounting.models_exhaustive : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index cba63db2e..39b4c4959 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -1,4 +1,14 @@ [ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, { "name": "complete_calls", "records": [ diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..ccdf1628c 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -51,6 +51,9 @@ def test_python_reducer_matches_shared_independent_expectations(reduced): for vector, usage in zip(VECTORS, reduced, strict=True): accounting = usage["extensions"][KEY] for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue if field == "exhaustive": actual = accounting["models_exhaustive"] elif field.startswith("observed_"): From a8f6fd807b9a2dc5cd3ae3f1b01538aec4820a7b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:05:52 -0700 Subject: [PATCH 69/74] test: account for unverified empty usage coverage Signed-off-by: mnajafian-nv --- tests/adapters/test_deepagents_accounting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index dccaf29a3..4b674815a 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -210,7 +210,8 @@ async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( dispatched = operation in {"start_async_task", "update_async_task"} assert result.input_tokens == (100 if foreground_call and not dispatched else None) reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} - assert ("scope_unverified" in reasons) is dispatched + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) if foreground_call: assert result.extensions[KEY]["observed"]["input_tokens"] == 100 elif dispatched: From 83d8adace3ca3dd886c8769ba1cdc1a01d3a7c3b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:55:34 -0700 Subject: [PATCH 70/74] docs: keep adapter usage guidance within current contract Signed-off-by: mnajafian-nv --- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 064429506..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -231,7 +231,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts and price them. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From ebcfe97c48bcc90e93f2ae74078f1b82e9049132 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:26:13 -0700 Subject: [PATCH 71/74] test: run OpenShell launcher checks with system Bash Use the system Bash path so macOS tests exercise Bash 3.2 even when PATH contains a newer Bash. This keeps the empty-array regression meaningful in CI. Signed-off-by: mnajafian-nv --- tests/scripts/test_openshell_launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 15ab68caa..f54157496 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -165,7 +165,7 @@ def _operations(root: Path) -> list[dict]: def _run(launcher, workload, env): script, _, base_env = launcher return subprocess.run( - ["bash", str(script), str(workload), "--input", "words with spaces"], + ["/bin/bash", str(script), str(workload), "--input", "words with spaces"], env={**base_env, **env}, text=True, capture_output=True, @@ -385,7 +385,7 @@ def test_invalid_platform_or_mode_stops_before_building(launcher, environment, m "stage_workload_image() { :; }\nrun_workload_consumer() { :; }\n" ) completed = subprocess.run( - ["bash", str(script), str(workload)], + ["/bin/bash", str(script), str(workload)], env={**base_env, **environment}, text=True, capture_output=True, From 83b5e76b8b5f3629e7ef9215bd5be505e3a13298 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 15:07:21 -0700 Subject: [PATCH 72/74] fix: reject Relay readiness after the startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..86c23ede2 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,26 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2149bd1a0e98d996f64065724c3658e6ea43f06d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 16:29:34 -0700 Subject: [PATCH 73/74] fix: bound Pi Relay readiness checks Signed-off-by: mnajafian-nv --- adapters/typescript/pi/src/relay-gateway.ts | 31 +++++++-- adapters/typescript/pi/test/relay.test.mjs | 71 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/adapters/typescript/pi/src/relay-gateway.ts b/adapters/typescript/pi/src/relay-gateway.ts index 163bcac09..99446bf14 100644 --- a/adapters/typescript/pi/src/relay-gateway.ts +++ b/adapters/typescript/pi/src/relay-gateway.ts @@ -147,26 +147,43 @@ export async function waitForRelayGateway( const fetchRequest = options.fetch ?? fetch; const pollIntervalMs = options.pollIntervalMs ?? 100; const deadline = performance.now() + timeoutMs; - while (performance.now() < deadline) { + while (true) { + const remaining = deadline - performance.now(); + if (remaining <= 0) { + break; + } if (processExited(child)) { throw new RelayGatewayError( `NeMo Relay gateway exited with status ${child.exitCode ?? child.signalCode} before becoming ready`, ); } + let ready = false; try { const response = await fetchRequest(healthUrl, { method: "HEAD", - signal: AbortSignal.timeout(1_000), + signal: AbortSignal.timeout(Math.min(1_000, Math.ceil(remaining))), }); - const ready = response.ok; await response.body?.cancel(); - if (ready) { - return; - } + ready = response.ok; } catch { // Retry until the gateway exits or the readiness deadline expires. } - await delay(Math.min(pollIntervalMs, Math.max(0, deadline - performance.now()))); + if (ready) { + if (processExited(child)) { + throw new RelayGatewayError( + `NeMo Relay gateway exited with status ${child.exitCode ?? child.signalCode} before becoming ready`, + ); + } + if (performance.now() >= deadline) { + break; + } + return; + } + const remainingAfterProbe = deadline - performance.now(); + if (remainingAfterProbe <= 0) { + break; + } + await delay(Math.min(pollIntervalMs, remainingAfterProbe)); } throw new RelayGatewayError(`NeMo Relay gateway did not become ready at ${healthUrl}`); } diff --git a/adapters/typescript/pi/test/relay.test.mjs b/adapters/typescript/pi/test/relay.test.mjs index ee0d541d5..b77e3c459 100644 --- a/adapters/typescript/pi/test/relay.test.mjs +++ b/adapters/typescript/pi/test/relay.test.mjs @@ -844,6 +844,77 @@ test("drains unsuccessful and successful Relay health responses", async () => { assert.deepEqual(cancelled, [0, 1]); }); +test("rejects a gateway exit during a successful health probe", async (t) => { + for (const boundary of ["fetch", "body cancellation"]) { + await t.test(boundary, async () => { + const child = new MockChild(); + await assert.rejects( + waitForRelayGateway(child, "http://127.0.0.1:41000/healthz", { + async fetch() { + if (boundary === "fetch") child.exitCode = 17; + return { + ok: true, + body: { + async cancel() { + if (boundary === "body cancellation") child.exitCode = 17; + }, + }, + }; + }, + }), + /exited with status 17 before becoming ready/u, + ); + }); + } +}); + +test("rejects a successful health probe at the readiness deadline", async (t) => { + for (const boundary of ["fetch", "body cancellation"]) { + await t.test(boundary, async (subtest) => { + let now = 0; + subtest.mock.method(performance, "now", () => now); + await assert.rejects( + waitForRelayGateway(new MockChild(), "http://127.0.0.1:41000/healthz", { + timeoutMs: 25, + async fetch() { + if (boundary === "fetch") now = 25; + return { + ok: true, + body: { + async cancel() { + if (boundary === "body cancellation") now = 25; + }, + }, + }; + }, + }), + /did not become ready/u, + ); + }); + } +}); + +test("limits a health probe to the remaining readiness budget", async (t) => { + let now = 0; + t.mock.method(performance, "now", () => now); + const timeouts = []; + t.mock.method(AbortSignal, "timeout", (milliseconds) => { + timeouts.push(milliseconds); + return new AbortController().signal; + }); + await assert.rejects( + waitForRelayGateway(new MockChild(), "http://127.0.0.1:41000/healthz", { + timeoutMs: 25, + async fetch() { + now = 25; + throw new Error("still initializing"); + }, + }), + /did not become ready/u, + ); + assert.deepEqual(timeouts, [25]); +}); + test("escalates gateway shutdown from terminate to kill and is safe after exit", async () => { const mockChild = new MockChild(); mockChild.kill = function (signal) { From a1d552799517a68a3bea1c136305a4ab7ab6fa06 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sun, 25 Oct 2026 22:28:15 -0700 Subject: [PATCH 74/74] fix: reject positive usage with exhaustive empty models Signed-off-by: mnajafian-nv --- crates/fabric-core/src/model_usage.rs | 54 +++++++++++++++++++ crates/fabric-core/src/usage_accounting.rs | 30 +++++++++-- .../src/nemo_fabric/types.py | 11 ++++ tests/python/test_model_usage.py | 48 +++++++++++++++++ 4 files changed, 138 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs index 9d9aaff80..ca2e708e0 100644 --- a/crates/fabric-core/src/model_usage.rs +++ b/crates/fabric-core/src/model_usage.rs @@ -313,6 +313,60 @@ mod accounting_tests { } } + #[test] + fn exhaustive_empty_model_list_rejects_positive_tokens() { + for observed in [false, true] { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] { + let tokens = json!({field: 1}); + let mut usage: AgentUsage = + serde_json::from_value(if observed || field == "cache_write_input_tokens" { + json!({}) + } else { + tokens.clone() + }) + .unwrap(); + let mut accounting = json!({"models_exhaustive": true, "models": [], "gaps": []}); + if observed { + accounting["observed"] = tokens; + } else if field == "cache_write_input_tokens" { + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), tokens); + } + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + let error = normalize_model_usage(&usage).unwrap_err(); + assert!( + error.to_string().contains("exhaustive empty model list"), + "{field}, observed={observed}: {error}" + ); + } + } + } + + #[test] + fn exhaustive_empty_model_list_preserves_unknown_and_zero_work() { + for tokens in [ + json!({}), + json!({"input_tokens": 0, "cached_input_tokens": 0, "input_tokens_include_cache": true, "output_tokens": 0, "total_tokens": 0}), + ] { + let result = normalize( + tokens.clone(), + json!({"models_exhaustive": true, "models": [], "gaps": [], "observed": tokens}), + ); + assert_eq!(result["models_exhaustive"], true); + assert!(result.get("models").is_none()); + assert_eq!(result["input_tokens"], tokens["input_tokens"]); + } + } + #[test] fn legacy_usage_does_not_gain_a_completeness_guarantee() { let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); diff --git a/crates/fabric-core/src/usage_accounting.rs b/crates/fabric-core/src/usage_accounting.rs index c9cfa2fa6..aaf275911 100644 --- a/crates/fabric-core/src/usage_accounting.rs +++ b/crates/fabric-core/src/usage_accounting.rs @@ -216,11 +216,31 @@ impl UsageAccounting { }; let evidence: Self = serde_json::from_value(value.clone()) .map_err(|error| AccountingError(error.to_string()))?; - validate_record( - &reported_usage(usage)?, - evidence.observed.as_ref(), - &evidence.gaps, - )?; + let reported = reported_usage(usage)?; + validate_record(&reported, evidence.observed.as_ref(), &evidence.gaps)?; + if evidence.models_exhaustive + && evidence.models.is_empty() + && std::iter::once(&reported) + .chain(evidence.observed.as_ref()) + .any(|counts| { + [ + counts.input_tokens, + counts.cached_input_tokens, + counts.cache_write_input_tokens, + counts.output_tokens, + counts.reasoning_tokens, + counts.total_tokens, + counts.peak_request_input_tokens, + ] + .into_iter() + .flatten() + .any(|count| count > 0) + }) + { + return Err(AccountingError( + "exhaustive empty model list conflicts with positive tokens".into(), + )); + } let mut identities = BTreeSet::new(); for model in &evidence.models { if model.model.trim().is_empty() diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 2dfcd259b..e8cb52224 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2863,6 +2863,17 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if not isinstance(models, list): raise FabricConfigError("models must be an array") data["models"] = tuple(PerModelUsage.from_mapping(model) for model in models) + if ( + data["models_exhaustive"] + and not data["models"] + and any( + (data[field] or 0) > 0 or (data["observed"].get(field) or 0) > 0 + for field in _MODEL_USAGE_TOKEN_FIELDS + ) + ): + raise FabricConfigError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.get("provider"), model.model) for model in data["models"]] if len(set(identities)) != len(identities): raise FabricConfigError("duplicate provider/model identity") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 4c731b546..8bd256da1 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -535,3 +535,51 @@ def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence parsed = PricingBasis.from_mapping(source) assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") assert parsed.to_mapping() == source + + +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ], +) +@pytest.mark.parametrize("observed", [False, True]) +def test_exhaustive_empty_model_list_rejects_positive_tokens(field, observed): + usage = {"models_exhaustive": True, "models": [], "gaps": [], "cost": {}} + values = {field: 1} + if field == "total_tokens": + values.update(input_tokens=1, output_tokens=0) + if observed: + usage["observed"] = values + else: + usage.update(values) + with pytest.raises(FabricConfigError, match="exhaustive empty model list"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("value", [None, 0]) +def test_exhaustive_empty_model_list_preserves_unknown_and_zero_work(value): + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] + usage = { + "models_exhaustive": True, + "models": [], + "gaps": [], + "cost": {}, + **dict.fromkeys(fields, value), + } + usage["observed"] = dict.fromkeys(fields, value) + parsed = ModelUsage.from_mapping(usage) + assert parsed.models_exhaustive is True + assert parsed.models == () + assert parsed.input_tokens == value + assert parsed.observed.input_tokens == value