From 2fb392585aad085e17b70c0e7adc2795594cc278 Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Sun, 26 Jul 2026 18:30:42 -0700 Subject: [PATCH 01/22] [cli] feat: add benchmark submit command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `osmosis benchmark submit `, which submits a managed benchmark run from an Osmosis-owned TOML contract following the train/eval mental model. The config is not a Harbor config: Harbor stays an internal execution detail, and the CLI never sees or transmits secret values — `api_key_secret` fields carry Platform secret record names that the platform resolves server-side. Secret and env collision checks are scoped per agent because the platform injects each agent's API key only into that agent's environment, so one agent's secret name may legitimately be another agent's literal env var. --- osmosis_ai/cli/commands/benchmark.py | 32 +++ osmosis_ai/cli/main.py | 2 + osmosis_ai/platform/api/client.py | 32 +++ osmosis_ai/platform/api/models.py | 25 ++ osmosis_ai/platform/cli/benchmark.py | 228 ++++++++++++++++++ osmosis_ai/platform/cli/benchmark_config.py | 196 +++++++++++++++ osmosis_ai/platform/cli/shared_config.py | 6 +- tests/unit/cli/test_benchmark_commands.py | 31 +++ .../platform/api/test_client_benchmark.py | 75 ++++++ .../platform/cli/test_benchmark_config.py | 213 ++++++++++++++++ .../platform/cli/test_benchmark_submit.py | 122 ++++++++++ 11 files changed, 961 insertions(+), 1 deletion(-) create mode 100644 osmosis_ai/cli/commands/benchmark.py create mode 100644 osmosis_ai/platform/cli/benchmark.py create mode 100644 osmosis_ai/platform/cli/benchmark_config.py create mode 100644 tests/unit/cli/test_benchmark_commands.py create mode 100644 tests/unit/platform/api/test_client_benchmark.py create mode 100644 tests/unit/platform/cli/test_benchmark_config.py create mode 100644 tests/unit/platform/cli/test_benchmark_submit.py diff --git a/osmosis_ai/cli/commands/benchmark.py b/osmosis_ai/cli/commands/benchmark.py new file mode 100644 index 00000000..d8dc5845 --- /dev/null +++ b/osmosis_ai/cli/commands/benchmark.py @@ -0,0 +1,32 @@ +"""Benchmark commands (submit).""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import typer + +app: typer.Typer = typer.Typer( + help="Manage benchmark runs (submit).", + no_args_is_help=True, +) + + +@app.command("submit") +def benchmark_submit( + config_path: Path = typer.Argument( + ..., + exists=False, + file_okay=True, + dir_okay=False, + readable=False, + resolve_path=False, + help="Path to benchmark config TOML file.", + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt."), +) -> Any: + """Submit a benchmark run.""" + from osmosis_ai.platform.cli.benchmark import submit as _submit + + return _submit(config_path, yes=yes) diff --git a/osmosis_ai/cli/main.py b/osmosis_ai/cli/main.py index 235a43a4..994383fe 100644 --- a/osmosis_ai/cli/main.py +++ b/osmosis_ai/cli/main.py @@ -187,6 +187,7 @@ def _register_commands() -> None: get_completion_inspect_parameters() # -- Command groups -- from osmosis_ai.cli.commands.auth import app as auth_app + from osmosis_ai.cli.commands.benchmark import app as benchmark_app from osmosis_ai.cli.commands.dataset import app as dataset_app from osmosis_ai.cli.commands.eval import app as eval_app from osmosis_ai.cli.commands.model import app as model_app @@ -202,6 +203,7 @@ def _register_commands() -> None: app.add_typer(train_app, name="train", rich_help_panel=_WORKFLOW) app.add_typer(model_app, name="model", rich_help_panel=_WORKFLOW) app.add_typer(eval_app, name="eval", rich_help_panel=_WORKFLOW) + app.add_typer(benchmark_app, name="benchmark", rich_help_panel=_WORKFLOW) app.add_typer(rollout_app, name="rollout", rich_help_panel=_WORKFLOW) app.add_typer(template_app, name="template", rich_help_panel=_WORKFLOW) diff --git a/osmosis_ai/platform/api/client.py b/osmosis_ai/platform/api/client.py index dd0e38f4..e0ffe197 100644 --- a/osmosis_ai/platform/api/client.py +++ b/osmosis_ai/platform/api/client.py @@ -30,6 +30,7 @@ RunDownloadFile, RunDownloadManifest, RunDownloadURLBatch, + SubmitBenchmarkRunResult, SubmitRunResult, TrainingRunCheckpoints, TrainingRunDetail, @@ -635,6 +636,37 @@ def submit_evaluation_run( ) return SubmitRunResult.from_dict(result) + def submit_benchmark_run( + self, + *, + experiment_config: dict[str, Any], + agents: list[dict[str, Any]], + tasks_config: dict[str, Any] | None = None, + execution_config: dict[str, Any] | None = None, + env_config: dict[str, str] | None = None, + credentials: Credentials | None = None, + git_identity: str, + ) -> SubmitBenchmarkRunResult: + """Submit a new benchmark run.""" + data: dict[str, Any] = { + "experiment_config": experiment_config, + "agents": agents, + } + if tasks_config: + data["tasks_config"] = tasks_config + if execution_config: + data["execution_config"] = execution_config + if env_config: + data["env_config"] = env_config + result = platform_request( + "/api/cli/benchmark-runs", + method="POST", + data=data, + credentials=credentials, + git_identity=git_identity, + ) + return SubmitBenchmarkRunResult.from_dict(result) + def list_eval_runs( self, limit: int = DEFAULT_PAGE_SIZE, diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index f2827d26..a2f0b204 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -352,6 +352,31 @@ def from_dict(cls, data: dict[str, Any]) -> SubmitRunResult: ) +@dataclass +class SubmitBenchmarkRunResult: + """Result of submitting a benchmark run.""" + + id: str + name: str + status: str + created_at: str + workflow_id: str = "" + task_count: int = 0 + platform_url: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> SubmitBenchmarkRunResult: + return cls( + id=data["id"], + name=data["name"], + status=data["status"], + created_at=data["created_at"], + workflow_id=data.get("workflow_id", ""), + task_count=data.get("task_count", 0), + platform_url=data.get("platform_url"), + ) + + # ── Training run metrics ───────────────────────────────────────── diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py new file mode 100644 index 00000000..ac672b3b --- /dev/null +++ b/osmosis_ai/platform/cli/benchmark.py @@ -0,0 +1,228 @@ +"""Handler for ``osmosis benchmark submit``.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from osmosis_ai.cli.console import console +from osmosis_ai.cli.errors import CLIError +from osmosis_ai.cli.output import OperationResult, get_output_context +from osmosis_ai.cli.prompts import require_confirmation +from osmosis_ai.platform.api.client import OsmosisClient +from osmosis_ai.platform.api.models import SubmitBenchmarkRunResult +from osmosis_ai.platform.auth.platform_client import PlatformAPIError +from osmosis_ai.platform.cli.benchmark_config import ( + BenchmarkSubmitConfig, + load_benchmark_submit_config, +) +from osmosis_ai.platform.cli.shared_config import ( + build_env_table_rows, + build_secret_table_rows, +) +from osmosis_ai.platform.cli.shared_submit import ( + _enrich_missing_secret_error, + _fetch_secret_scopes, + _missing_secret_message, +) +from osmosis_ai.platform.cli.utils import require_git_workspace_directory_context +from osmosis_ai.platform.cli.workspace_directory_context import git_result_context +from osmosis_ai.platform.cli.workspace_directory_contract import ( + ensure_workspace_directory_config_path, + validate_workspace_directory_contract, +) + + +def _agent_model_label(agent: dict[str, Any]) -> str: + model = agent["model"] + if model["type"] == "hosted": + return f"{model['base_model']}:{model['checkpoint_name']}" + return str(model["model"]) + + +def _task_selection_label(config: BenchmarkSubmitConfig) -> str: + tasks = config.tasks_config + if tasks.get("task_set"): + return str(tasks["task_set"]) + names = tasks.get("task_names") + categories = tasks.get("categories") + parts: list[str] = [] + if isinstance(names, list) and names: + parts.append(f"{len(names)} task(s)") + if isinstance(categories, list) and categories: + parts.append(f"{len(categories)} category(s)") + return ", ".join(parts) if parts else "all tasks" + + +def _submit_benchmark( + client: OsmosisClient, + config: BenchmarkSubmitConfig, + credentials: Any, + git_identity: str, +) -> SubmitBenchmarkRunResult: + return client.submit_benchmark_run( + experiment_config=config.experiment_config, + tasks_config=config.tasks_config or None, + agents=config.agents_config, + execution_config=config.execution_config or None, + env_config=config.env or None, + credentials=credentials, + git_identity=git_identity, + ) + + +def submit(config_path: Path, *, yes: bool) -> OperationResult: + """Submit a benchmark run.""" + context = require_git_workspace_directory_context() + workspace_directory = Path(context.workspace_directory) + validate_workspace_directory_contract(workspace_directory) + + path = Path(config_path) + resolved_path = path if path.is_absolute() else workspace_directory / path + ensure_workspace_directory_config_path( + resolved_path, + workspace_directory, + config_dir="configs/benchmark", + command_label="`osmosis benchmark submit`", + ) + config = load_benchmark_submit_config(resolved_path) + + execution = config.execution_config + summary_rows = [ + ("Benchmark", config.experiment.benchmark), + ("Tasks", _task_selection_label(config)), + ("Agents", str(len(config.agents))), + ("Attempts per task", str(execution.get("attempts_per_task", 1))), + ( + "Max concurrent attempts", + str(execution.get("max_concurrent_attempts", 4)), + ), + ] + console.table( + [(label, console.escape(value)) for label, value in summary_rows], + title="Benchmark Run", + ) + agent_summary_rows = [ + ( + f"{index} · {agent.harness or 'default'}", + _agent_model_label(config.agents_config[index - 1]), + ) + for index, agent in enumerate(config.agents, start=1) + ] + console.table( + [ + (console.escape(agent), console.escape(model)) + for agent, model in agent_summary_rows + ], + title=f"Agents ({len(agent_summary_rows)})", + headers=("Agent", "Model"), + ) + + full_summary: list[tuple[str, str]] = list(summary_rows) + full_summary.extend( + (f"agent.{agent}", model) for agent, model in agent_summary_rows + ) + + env_rows = build_env_table_rows(config.env) + for index, agent in enumerate(config.agents, start=1): + env_rows.extend( + ( + f"agent {index} · {name}" + + (" (overrides global)" if name in config.env else ""), + value, + ) + for name, value in build_env_table_rows(agent.env) + ) + if env_rows: + console.table( + [(name, console.escape(value)) for name, value in env_rows], + title=f"Env Vars ({len(env_rows)})", + headers=("Name", "Value"), + ) + full_summary.extend((f"env.{name}", value) for name, value in env_rows) + + if config.required_secrets: + scopes = _fetch_secret_scopes( + OsmosisClient(), + credentials=context.credentials, + git_identity=context.git_identity, + ) + if scopes is None: + secret_rows = [(name, "–") for name in sorted(config.required_secrets)] + else: + workspace_names, personal_names = scopes + missing = sorted( + name + for name in config.required_secrets + if name not in workspace_names and name not in personal_names + ) + if missing: + raise CLIError(_missing_secret_message(missing)) + secret_rows = build_secret_table_rows( + config.required_secrets, + user_secret_names=personal_names, + workspace_secret_names=workspace_names, + ) + console.table( + secret_rows, + title=f"Secrets ({len(secret_rows)})", + headers=("Name", "Scope"), + ) + full_summary.extend((f"secret.{name}", scope) for name, scope in secret_rows) + + require_confirmation( + "Submit this benchmark run?", + yes=yes, + summary=full_summary, + ) + + output = get_output_context() + with output.status("Submitting benchmark run..."): + try: + result = _submit_benchmark( + OsmosisClient(), + config, + context.credentials, + context.git_identity, + ) + except PlatformAPIError as exc: + enriched = _enrich_missing_secret_error(exc) + if enriched is not None: + raise enriched from exc + raise + + display_next_steps = [ + f"Status: {result.status}", + f"Benchmark: {config.experiment.benchmark}", + ] + structured_next_steps: list[dict[str, Any]] = [] + if result.platform_url: + display_next_steps.append(f"View: {result.platform_url}") + structured_next_steps.append({"action": "open_url", "url": result.platform_url}) + + return OperationResult( + operation="benchmark.submit", + status="success", + resource={ + "id": result.id, + "name": result.name, + "status": result.status, + "benchmark_name": config.experiment.benchmark, + "task_count": result.task_count, + "created_at": result.created_at, + **({"url": result.platform_url} if result.platform_url else {}), + **git_result_context(context), + "config": { + "experiment": config.experiment_config, + "tasks": config.tasks_config, + "agents": config.agents_config, + "execution": config.execution_config, + }, + }, + message=f"Benchmark run submitted: {result.name}", + display_next_steps=display_next_steps, + next_steps_structured=structured_next_steps, + ) + + +__all__ = ["submit"] diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py new file mode 100644 index 00000000..345cd5bb --- /dev/null +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -0,0 +1,196 @@ +"""TOML config loading and validation for benchmark runs.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Any, ClassVar, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from osmosis_ai.cli.errors import CLIError +from osmosis_ai.platform.cli.shared_config import ( + SECRET_NAME_RE, + config_issues_error, + read_toml_file, + read_toml_table, + validate_env_var_keys, + validation_issue_to_config_issue, +) + +_BENCHMARK_CONFIG_LABEL = "benchmark" + + +class _StrictSection(BaseModel): + model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid") + + +class BenchmarkExperimentSection(_StrictSection): + benchmark: str + + +class BenchmarkTasksSection(_StrictSection): + categories: Any = None + task_names: Any = None + task_set: Any = None + + +class BenchmarkProviderModel(_StrictSection): + type: Literal["provider"] + model: str + api_key_secret: str + + +class BenchmarkEndpointModel(_StrictSection): + type: Literal["endpoint"] + base_url: str + model: str + api_key_secret: str + extra_headers: dict[str, str] | None = None + + +class BenchmarkHostedModel(_StrictSection): + type: Literal["hosted"] + base_model: str + checkpoint_name: str + + +BenchmarkModel = Annotated[ + BenchmarkProviderModel | BenchmarkEndpointModel | BenchmarkHostedModel, + Field(discriminator="type"), +] + + +class BenchmarkAgentSection(_StrictSection): + harness: str | None = None + model: BenchmarkModel + env: dict[str, str] = Field(default_factory=dict) + + +class BenchmarkExecutionSection(_StrictSection): + attempts_per_task: Any = None + max_concurrent_attempts: Any = None + timeout_multiplier: Any = None + max_retries: Any = None + pass_threshold: Any = None + judge_model: Any = None + judge_api_key_secret: Any = None + + +class BenchmarkSubmitConfig(_StrictSection): + """Parsed benchmark run TOML configuration.""" + + experiment: BenchmarkExperimentSection + tasks: BenchmarkTasksSection = Field(default_factory=BenchmarkTasksSection) + agents: list[BenchmarkAgentSection] = Field(min_length=1, max_length=8) + execution: BenchmarkExecutionSection = Field( + default_factory=BenchmarkExecutionSection + ) + env: dict[str, str] = Field(default_factory=dict) + + @property + def experiment_config(self) -> dict[str, Any]: + return self.experiment.model_dump() + + @property + def tasks_config(self) -> dict[str, Any]: + return self.tasks.model_dump(exclude_none=True) + + @property + def agents_config(self) -> list[dict[str, Any]]: + return [agent.model_dump(exclude_none=True) for agent in self.agents] + + @property + def execution_config(self) -> dict[str, Any]: + return self.execution.model_dump(exclude_none=True) + + @property + def required_secrets(self) -> list[str]: + names = [ + agent.model.api_key_secret + for agent in self.agents + if isinstance(agent.model, BenchmarkProviderModel | BenchmarkEndpointModel) + ] + judge_secret = self.execution.judge_api_key_secret + if isinstance(judge_secret, str) and judge_secret: + names.append(judge_secret) + return list(dict.fromkeys(names)) + + +def _env_source_label(name: str, agent_index: int, agent_env: dict[str, str]) -> str: + if name in agent_env: + return f"[agents.env] of agent {agent_index}" + return "[env]" + + +def _validate_secret_references(config: BenchmarkSubmitConfig, path: Path) -> None: + """Validate secret record names and per-agent env collisions. + + The platform injects an agent's ``api_key_secret`` value (and any judge + secret) as an env var of the same name into that agent's runtime env, so a + literal env var with that name would be silently overwritten. Collisions + are scoped per agent: one agent's secret name may still be another agent's + literal env var. + """ + for name in config.required_secrets: + if not SECRET_NAME_RE.match(name): + raise CLIError( + f"Invalid secret name '{name}' in {path}: use uppercase " + "letters, digits, and underscores, starting with a letter " + "(e.g. MY_SECRET). Must match ^[A-Z][A-Z0-9_]*$." + ) + + judge_secret = config.execution.judge_api_key_secret + judge_name = ( + judge_secret if isinstance(judge_secret, str) and judge_secret else None + ) + for index, agent in enumerate(config.agents, start=1): + effective_env = {**config.env, **agent.env} + model = agent.model + if ( + isinstance(model, BenchmarkProviderModel | BenchmarkEndpointModel) + and model.api_key_secret in effective_env + ): + source = _env_source_label(model.api_key_secret, index, agent.env) + raise CLIError( + f"'{model.api_key_secret}' appears in {source} and as agent " + f"{index}'s api_key_secret in {path}. The platform injects the " + "secret value under that name; remove the env var or rename it." + ) + if judge_name and judge_name in effective_env: + source = _env_source_label(judge_name, index, agent.env) + raise CLIError( + f"'{judge_name}' appears in {source} and as " + f"judge_api_key_secret in {path}. The platform injects the " + "judge secret value under that name; remove the env var or " + "rename it." + ) + + +def load_benchmark_submit_config(path: Path) -> BenchmarkSubmitConfig: + """Load and validate TOML config for benchmark run submit.""" + raw = read_toml_file(path) + read_toml_table(raw, "experiment", path, required=True) + if "agents" not in raw: + raise CLIError(f"Missing [[agents]] section in {path}") + if not isinstance(raw["agents"], list): + raise CLIError(f"[[agents]] must be an array of tables in {path}") + + try: + config = BenchmarkSubmitConfig.model_validate(raw) + except ValidationError as exc: + raise config_issues_error( + issues=[ + validation_issue_to_config_issue(error=error, section_name="") + for error in exc.errors() + ], + config_label=_BENCHMARK_CONFIG_LABEL, + ) from exc + + validate_env_var_keys(env=config.env, path=path) + for agent in config.agents: + validate_env_var_keys(env=agent.env, path=path) + _validate_secret_references(config, path) + return config + + +__all__ = ["BenchmarkSubmitConfig", "load_benchmark_submit_config"] diff --git a/osmosis_ai/platform/cli/shared_config.py b/osmosis_ai/platform/cli/shared_config.py index 83a5ac78..c06aa7d1 100644 --- a/osmosis_ai/platform/cli/shared_config.py +++ b/osmosis_ai/platform/cli/shared_config.py @@ -193,7 +193,11 @@ def validation_issue_to_config_issue( loc = tuple(error.get("loc") or ()) field_path = format_field_path(loc) error_type = str(error.get("type")) - issue_key = f"{section_name}.{field_path}" if field_path else section_name + issue_key = ( + f"{section_name}.{field_path}" + if section_name and field_path + else field_path or section_name + ) if error_type == "extra_forbidden": return {"key": issue_key, "message": "Unrecognized key"} diff --git a/tests/unit/cli/test_benchmark_commands.py b/tests/unit/cli/test_benchmark_commands.py new file mode 100644 index 00000000..3f8d0bf9 --- /dev/null +++ b/tests/unit/cli/test_benchmark_commands.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +import osmosis_ai.cli.commands.benchmark as benchmark_commands +import osmosis_ai.platform.cli.benchmark as benchmark_handler + + +def test_benchmark_submit_delegates_to_handler( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + expected = object() + + def fake_submit(config_path: Path, *, yes: bool) -> object: + captured.update(config_path=config_path, yes=yes) + return expected + + monkeypatch.setattr(benchmark_handler, "submit", fake_submit) + + result = benchmark_commands.benchmark_submit( + Path("configs/benchmark/smoke.toml"), yes=True + ) + + assert result is expected + assert captured == { + "config_path": Path("configs/benchmark/smoke.toml"), + "yes": True, + } diff --git a/tests/unit/platform/api/test_client_benchmark.py b/tests/unit/platform/api/test_client_benchmark.py new file mode 100644 index 00000000..14e92975 --- /dev/null +++ b/tests/unit/platform/api/test_client_benchmark.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +from osmosis_ai.platform.api.client import OsmosisClient + + +@patch("osmosis_ai.platform.api.client.platform_request") +def test_submit_benchmark_run_posts_config_sections(mock_request: MagicMock) -> None: + mock_request.return_value = { + "id": "benchmark-run-1", + "name": "bright-otter", + "status": "pending", + "workflow_id": "benchmark-run/benchmark-run-1", + "task_count": 12, + "created_at": "2026-07-25T00:00:00Z", + "platform_url": "https://platform.osmosis.ai/acme/benchmarks/benchmark-run-1", + } + agent: dict[str, Any] = { + "harness": "codex", + "model": { + "type": "provider", + "model": "openai/gpt-5", + "api_key_secret": "OPENAI_API_KEY", + }, + } + + result = OsmosisClient().submit_benchmark_run( + experiment_config={"benchmark": "Terminal-Bench 2.1"}, + tasks_config={"task_set": "parity"}, + agents=[agent], + execution_config={"attempts_per_task": 2}, + env_config={"LOG_LEVEL": "info"}, + git_identity="acme/workspace", + ) + + assert result.id == "benchmark-run-1" + assert result.task_count == 12 + assert mock_request.call_args.args[0] == "/api/cli/benchmark-runs" + assert mock_request.call_args.kwargs == { + "method": "POST", + "data": { + "experiment_config": {"benchmark": "Terminal-Bench 2.1"}, + "tasks_config": {"task_set": "parity"}, + "agents": [agent], + "execution_config": {"attempts_per_task": 2}, + "env_config": {"LOG_LEVEL": "info"}, + }, + "credentials": None, + "git_identity": "acme/workspace", + } + + +@patch("osmosis_ai.platform.api.client.platform_request") +def test_submit_benchmark_run_omits_empty_optional_sections( + mock_request: MagicMock, +) -> None: + mock_request.return_value = { + "id": "benchmark-run-1", + "name": "bright-otter", + "status": "pending", + "created_at": "2026-07-25T00:00:00Z", + } + + OsmosisClient().submit_benchmark_run( + experiment_config={"benchmark": "Terminal-Bench 2.1"}, + agents=[{"model": {"type": "hosted"}}], + git_identity="acme/workspace", + ) + + assert mock_request.call_args.kwargs["data"] == { + "experiment_config": {"benchmark": "Terminal-Bench 2.1"}, + "agents": [{"model": {"type": "hosted"}}], + } diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py new file mode 100644 index 00000000..c69b2f9d --- /dev/null +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from osmosis_ai.cli.errors import CLIError +from osmosis_ai.platform.cli.benchmark_config import load_benchmark_submit_config + + +def _write_config(path: Path, body: str) -> Path: + path.write_text(body.strip() + "\n", encoding="utf-8") + return path + + +def test_load_benchmark_submit_config_accepts_provider_and_endpoint_agents( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[tasks] +task_names = ["git-multibranch"] + +[[agents]] +harness = "codex" + +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[agents.env] +AGENT_MODE = "strict" + +[[agents]] +harness = "claude-code" + +[agents.model] +type = "endpoint" +base_url = "https://models.example.com/v1" +model = "custom-model" +api_key_secret = "CUSTOM_API_KEY" + +[execution] +attempts_per_task = 2 +max_concurrent_attempts = 8 + +[env] +LOG_LEVEL = "info" +""", + ) + + config = load_benchmark_submit_config(path) + + assert config.experiment_config == {"benchmark": "Terminal-Bench 2.1"} + assert config.tasks_config == {"task_names": ["git-multibranch"]} + assert config.execution_config == { + "attempts_per_task": 2, + "max_concurrent_attempts": 8, + } + assert config.env == {"LOG_LEVEL": "info"} + assert config.required_secrets == ["OPENAI_API_KEY", "CUSTOM_API_KEY"] + assert config.agents_config[0] == { + "harness": "codex", + "model": { + "type": "provider", + "model": "openai/gpt-5", + "api_key_secret": "OPENAI_API_KEY", + }, + "env": {"AGENT_MODE": "strict"}, + } + + +def test_load_benchmark_submit_config_accepts_hosted_model(tmp_path: Path) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[[agents]] +harness = "codex" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "terminal-agent" +""", + ) + + config = load_benchmark_submit_config(path) + + assert config.tasks_config == {} + assert config.execution_config == {} + assert config.required_secrets == [] + + +def test_load_benchmark_submit_config_rejects_unknown_section(tmp_path: Path) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[[agents]] +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[harbor] +n_attempts = 3 +""", + ) + + with pytest.raises(CLIError) as exc_info: + load_benchmark_submit_config(path) + + assert "harbor: Unrecognized key" in str(exc_info.value) + + +def test_load_benchmark_submit_config_rejects_secret_env_collision( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[[agents]] +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[env] +OPENAI_API_KEY = "literal-secret" +""", + ) + + with pytest.raises(CLIError, match=r"agent 1's api_key_secret"): + load_benchmark_submit_config(path) + + +def test_load_benchmark_submit_config_allows_cross_agent_secret_env_names( + tmp_path: Path, +) -> None: + """Agent 2 may use agent 1's secret name as a literal env var. + + The platform injects each secret only into the env of the agent that + references it, so this configuration is valid server-side. + """ + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[[agents]] +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[[agents]] +[agents.model] +type = "endpoint" +base_url = "https://models.example.com/v1" +model = "custom-model" +api_key_secret = "CUSTOM_API_KEY" + +[agents.env] +OPENAI_API_KEY = "placeholder-for-harness" +""", + ) + + config = load_benchmark_submit_config(path) + + assert config.required_secrets == ["OPENAI_API_KEY", "CUSTOM_API_KEY"] + assert config.agents[1].env == {"OPENAI_API_KEY": "placeholder-for-harness"} + + +def test_load_benchmark_submit_config_rejects_judge_secret_env_collision( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[[agents]] +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[agents.env] +JUDGE_KEY = "literal" + +[execution] +judge_model = "openai/gpt-5" +judge_api_key_secret = "JUDGE_KEY" +""", + ) + + with pytest.raises(CLIError, match=r"judge_api_key_secret"): + load_benchmark_submit_config(path) diff --git a/tests/unit/platform/cli/test_benchmark_submit.py b/tests/unit/platform/cli/test_benchmark_submit.py new file mode 100644 index 00000000..f2532c02 --- /dev/null +++ b/tests/unit/platform/cli/test_benchmark_submit.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import osmosis_ai.platform.cli.benchmark as benchmark_module +from osmosis_ai.cli.errors import CLIError +from osmosis_ai.cli.output import OperationResult +from osmosis_ai.platform.api.models import SubmitBenchmarkRunResult +from osmosis_ai.templates.catalog import required_workspace_paths + +GIT_IDENTITY = "acme/workspace" +REPO_URL = "https://github.com/acme/workspace.git" +FAKE_CREDENTIALS = object() + + +def _make_workspace(root: Path) -> Path: + for rel_path in required_workspace_paths(): + (root / rel_path).mkdir(parents=True, exist_ok=True) + (root / "configs" / "benchmark").mkdir(parents=True) + return root + + +def _write_config(path: Path) -> Path: + path.write_text( + """ +[experiment] +benchmark = "Terminal-Bench 2.1" + +[tasks] +task_set = "parity" + +[[agents]] +harness = "codex" + +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[execution] +attempts_per_task = 2 +""".strip() + + "\n", + encoding="utf-8", + ) + return path + + +def test_submit_sends_benchmark_config_and_returns_operation_result( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workspace = _make_workspace(tmp_path / "workspace") + config_path = _write_config(workspace / "configs" / "benchmark" / "smoke.toml") + context = SimpleNamespace( + workspace_directory=workspace, + git_identity=GIT_IDENTITY, + repo_url=REPO_URL, + credentials=FAKE_CREDENTIALS, + ) + captured: dict[str, Any] = {} + + class FakeClient: + def submit_benchmark_run(self, **kwargs: Any) -> SubmitBenchmarkRunResult: + captured.update(kwargs) + return SubmitBenchmarkRunResult( + id="benchmark-run-1", + name="bright-otter", + status="pending", + workflow_id="benchmark-run/benchmark-run-1", + task_count=10, + created_at="2026-07-25T00:00:00Z", + platform_url="https://platform.osmosis.ai/acme/benchmarks/benchmark-run-1", + ) + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", lambda: context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + monkeypatch.setattr( + benchmark_module, + "_fetch_secret_scopes", + lambda *args, **kwargs: ({"OPENAI_API_KEY"}, set()), + ) + + result = benchmark_module.submit(config_path, yes=True) + + assert captured["experiment_config"] == {"benchmark": "Terminal-Bench 2.1"} + assert captured["tasks_config"] == {"task_set": "parity"} + assert captured["execution_config"] == {"attempts_per_task": 2} + assert captured["agents"][0]["model"]["model"] == "openai/gpt-5" + assert captured["credentials"] is FAKE_CREDENTIALS + assert captured["git_identity"] == GIT_IDENTITY + assert isinstance(result, OperationResult) + assert result.operation == "benchmark.submit" + assert result.resource is not None + assert result.resource["task_count"] == 10 + assert result.resource["benchmark_name"] == "Terminal-Bench 2.1" + + +def test_submit_rejects_config_outside_benchmark_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workspace = _make_workspace(tmp_path / "workspace") + config_path = _write_config(workspace / "smoke.toml") + context = SimpleNamespace( + workspace_directory=workspace, + git_identity=GIT_IDENTITY, + repo_url=REPO_URL, + credentials=FAKE_CREDENTIALS, + ) + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", lambda: context + ) + + with pytest.raises(CLIError, match="configs/benchmark"): + benchmark_module.submit(config_path, yes=True) From 87ec1e1883ed30a832a7c98347e791187d2d31d1 Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 07:24:28 -0700 Subject: [PATCH 02/22] [cli] fix: align benchmark submit contract --- docs/cli.md | 2 +- osmosis_ai/platform/api/models.py | 8 +- osmosis_ai/platform/cli/benchmark.py | 22 +- osmosis_ai/platform/cli/benchmark_config.py | 98 ++++- tests/unit/cli/test_command_groups.py | 3 + .../platform/api/test_client_benchmark.py | 77 +++- .../platform/cli/test_benchmark_config.py | 384 +++++++++++++++++- .../platform/cli/test_benchmark_submit.py | 214 +++++++++- 8 files changed, 745 insertions(+), 63 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 48421d69..b9b5d43e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -7,7 +7,7 @@ The console script is `osmosis_ai.cli.main:main` (aliases: `osmosis`, `osmosis-ai`, `osmosis_ai`). [../osmosis_ai/cli/main.py](../osmosis_ai/cli/main.py): - `main()` calls `_register_commands()` once, then runs the Typer `app` with `standalone_mode=False` so it can map exceptions to exit codes itself. -- `_register_commands()` imports each command group **lazily inside the function**. Groups attach via `app.add_typer(...)`; the standalone `doctor` / `upgrade` commands attach via `app.command(...)`. Two `rich_help_panel`s split the help: `Workflow Commands` (`dataset`, `train`, `model`, `eval`, `rollout`, `template`, `doctor`) and `Platform Commands` (`auth`, `secret`, `upgrade`). +- `_register_commands()` imports each command group **lazily inside the function**. Groups attach via `app.add_typer(...)`; the standalone `doctor` / `upgrade` commands attach via `app.command(...)`. Two `rich_help_panel`s split the help: `Workflow Commands` (`dataset`, `train`, `model`, `eval`, `benchmark`, `rollout`, `template`, `doctor`) and `Platform Commands` (`auth`, `secret`, `upgrade`). - The root `_callback` resolves `--json` / `--plain`, builds an `OutputContext`, installs it on the Typer context, registers `verify_output_emitted` on close, and loads `.env` via `python-dotenv`. `hoist_format_selectors` lets the format flags appear anywhere on the line. ## Command shells delegate; they don't do work diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index a2f0b204..ff230d7a 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -360,8 +360,8 @@ class SubmitBenchmarkRunResult: name: str status: str created_at: str - workflow_id: str = "" - task_count: int = 0 + workflow_id: str + task_count: int platform_url: str | None = None @classmethod @@ -371,8 +371,8 @@ def from_dict(cls, data: dict[str, Any]) -> SubmitBenchmarkRunResult: name=data["name"], status=data["status"], created_at=data["created_at"], - workflow_id=data.get("workflow_id", ""), - task_count=data.get("task_count", 0), + workflow_id=data["workflow_id"], + task_count=data["task_count"], platform_url=data.get("platform_url"), ) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index ac672b3b..0f89bcb4 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -32,6 +32,12 @@ validate_workspace_directory_contract, ) +_HLE_PARITY_WARNING = ( + 'For HLE, we recommend [tasks] task_set = "parity" so results are ' + "comparable with published scores. This submission uses the full or a " + "custom task selection." +) + def _agent_model_label(agent: dict[str, Any]) -> str: model = agent["model"] @@ -54,6 +60,17 @@ def _task_selection_label(config: BenchmarkSubmitConfig) -> str: return ", ".join(parts) if parts else "all tasks" +def _warn_if_hle_without_parity(config: BenchmarkSubmitConfig) -> None: + # Catch likely casing mistakes too: the route will still return its precise + # case-sensitive benchmark-name error after the user sees this guidance. + benchmark_name = config.experiment.benchmark.strip().casefold() + if benchmark_name == "hle" and config.tasks.task_set != "parity": + console.print_warning( + _HLE_PARITY_WARNING, + code="HLE_PARITY_RECOMMENDED", + ) + + def _submit_benchmark( client: OsmosisClient, config: BenchmarkSubmitConfig, @@ -141,6 +158,8 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: ) full_summary.extend((f"env.{name}", value) for name, value in env_rows) + _warn_if_hle_without_parity(config) + if config.required_secrets: scopes = _fetch_secret_scopes( OsmosisClient(), @@ -208,9 +227,10 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: "name": result.name, "status": result.status, "benchmark_name": config.experiment.benchmark, + "workflow_id": result.workflow_id, "task_count": result.task_count, "created_at": result.created_at, - **({"url": result.platform_url} if result.platform_url else {}), + **({"platform_url": result.platform_url} if result.platform_url else {}), **git_result_context(context), "config": { "experiment": config.experiment_config, diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index 345cd5bb..a37508ab 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -18,6 +18,19 @@ ) _BENCHMARK_CONFIG_LABEL = "benchmark" +_HARNESS_API_KEY_ENV = { + "cursor-cli": "CURSOR_API_KEY", + "mini-swe-agent": "MSWEA_API_KEY", +} +_RESERVED_MODEL_API_KEY_SECRET_NAMES = frozenset( + { + "HF_TOKEN", + "DAYTONA_API_KEY", + "DAYTONA_API_URL", + "SKYPILOT_SERVICE_ACCOUNT_TOKEN", + "SKYPILOT_API_SERVER_ENDPOINT", + } +) class _StrictSection(BaseModel): @@ -28,10 +41,13 @@ class BenchmarkExperimentSection(_StrictSection): benchmark: str +_NonEmptyTaskSelector = Annotated[str, Field(min_length=1)] + + class BenchmarkTasksSection(_StrictSection): - categories: Any = None - task_names: Any = None - task_set: Any = None + categories: list[_NonEmptyTaskSelector] | None = None + task_names: list[_NonEmptyTaskSelector] | None = None + task_set: Literal["parity"] | None = None class BenchmarkProviderModel(_StrictSection): @@ -62,6 +78,7 @@ class BenchmarkHostedModel(_StrictSection): class BenchmarkAgentSection(_StrictSection): harness: str | None = None + harness_api_key_secret: str | None = None model: BenchmarkModel env: dict[str, str] = Field(default_factory=dict) @@ -110,9 +127,18 @@ def required_secrets(self) -> list[str]: for agent in self.agents if isinstance(agent.model, BenchmarkProviderModel | BenchmarkEndpointModel) ] + names.extend( + agent.harness_api_key_secret + for agent in self.agents + if agent.harness_api_key_secret is not None + ) judge_secret = self.execution.judge_api_key_secret - if isinstance(judge_secret, str) and judge_secret: + if isinstance(judge_secret, str): names.append(judge_secret) + # HLE's managed adapter reads the dataset through a fixed Platform + # secret, even though the name is not repeated in the submit config. + if self.experiment.benchmark.strip() == "HLE": + names.append("HF_TOKEN") return list(dict.fromkeys(names)) @@ -125,11 +151,15 @@ def _env_source_label(name: str, agent_index: int, agent_env: dict[str, str]) -> def _validate_secret_references(config: BenchmarkSubmitConfig, path: Path) -> None: """Validate secret record names and per-agent env collisions. - The platform injects an agent's ``api_key_secret`` value (and any judge - secret) as an env var of the same name into that agent's runtime env, so a - literal env var with that name would be silently overwritten. Collisions - are scoped per agent: one agent's secret name may still be another agent's - literal env var. + The platform injects an agent model's ``api_key_secret`` value (and any + judge secret) as an env var of the same name into that agent's runtime env, + so a literal env var with that name would be silently overwritten. + Collisions are scoped per agent: one agent's secret name may still be + another agent's literal env var. Model secrets also cannot use names the + runner removes before model-key aliasing. Harness credentials travel + through a separate reserved channel; known credentialed harnesses must + reference a secret record and cannot also set their platform-managed + destination env. ``HF_TOKEN`` is always runner-reserved as a literal env. """ for name in config.required_secrets: if not SECRET_NAME_RE.match(name): @@ -146,16 +176,22 @@ def _validate_secret_references(config: BenchmarkSubmitConfig, path: Path) -> No for index, agent in enumerate(config.agents, start=1): effective_env = {**config.env, **agent.env} model = agent.model - if ( - isinstance(model, BenchmarkProviderModel | BenchmarkEndpointModel) - and model.api_key_secret in effective_env - ): - source = _env_source_label(model.api_key_secret, index, agent.env) - raise CLIError( - f"'{model.api_key_secret}' appears in {source} and as agent " - f"{index}'s api_key_secret in {path}. The platform injects the " - "secret value under that name; remove the env var or rename it." - ) + if isinstance(model, BenchmarkProviderModel | BenchmarkEndpointModel): + if model.api_key_secret in _RESERVED_MODEL_API_KEY_SECRET_NAMES: + raise CLIError( + f"Agent {index}'s api_key_secret '{model.api_key_secret}' " + f"in {path} uses a name reserved by the benchmark runner. " + "Store the model credential under a different Platform " + "secret record name." + ) + if model.api_key_secret in effective_env: + source = _env_source_label(model.api_key_secret, index, agent.env) + raise CLIError( + f"'{model.api_key_secret}' appears in {source} and as agent " + f"{index}'s api_key_secret in {path}. The platform injects " + "the secret value under that name; remove the env var or " + "rename it." + ) if judge_name and judge_name in effective_env: source = _env_source_label(judge_name, index, agent.env) raise CLIError( @@ -164,6 +200,30 @@ def _validate_secret_references(config: BenchmarkSubmitConfig, path: Path) -> No "judge secret value under that name; remove the env var or " "rename it." ) + if "HF_TOKEN" in effective_env: + source = _env_source_label("HF_TOKEN", index, agent.env) + raise CLIError( + f"'HF_TOKEN' appears in {source} but is reserved by the " + f"benchmark runner in {path}. The runner removes this literal " + "env var before starting the agent; remove it from the config. " + "For HLE, store the dataset credential in the HF_TOKEN Platform " + "secret record instead." + ) + harness_env_name = _HARNESS_API_KEY_ENV.get(agent.harness or "") + if harness_env_name and harness_env_name in effective_env: + source = _env_source_label(harness_env_name, index, agent.env) + raise CLIError( + f"'{harness_env_name}' appears in {source} but is managed by " + f"agent {index}'s {agent.harness} harness API key secret in " + f"{path}. Remove the env var and set harness_api_key_secret " + "to a Platform secret record name." + ) + if harness_env_name and agent.harness_api_key_secret is None: + raise CLIError( + f"Agent {index}'s {agent.harness} harness requires " + f"harness_api_key_secret in {path}. Set it to the name of a " + f"Platform secret record containing {harness_env_name}." + ) def load_benchmark_submit_config(path: Path) -> BenchmarkSubmitConfig: diff --git a/tests/unit/cli/test_command_groups.py b/tests/unit/cli/test_command_groups.py index 68a30abe..e8597557 100644 --- a/tests/unit/cli/test_command_groups.py +++ b/tests/unit/cli/test_command_groups.py @@ -17,6 +17,7 @@ "dataset", "train", "model", + "benchmark", "rollout", "template", "eval", @@ -44,6 +45,8 @@ ["model", "info", "--help"], ["model", "deploy", "--help"], ["model", "undeploy", "--help"], + ["benchmark", "--help"], + ["benchmark", "submit", "--help"], ["rollout", "--help"], ["template", "--help"], ["eval", "--help"], diff --git a/tests/unit/platform/api/test_client_benchmark.py b/tests/unit/platform/api/test_client_benchmark.py index 14e92975..55e17284 100644 --- a/tests/unit/platform/api/test_client_benchmark.py +++ b/tests/unit/platform/api/test_client_benchmark.py @@ -27,24 +27,34 @@ def test_submit_benchmark_run_posts_config_sections(mock_request: MagicMock) -> } result = OsmosisClient().submit_benchmark_run( - experiment_config={"benchmark": "Terminal-Bench 2.1"}, + experiment_config={"benchmark": "HLE"}, tasks_config={"task_set": "parity"}, agents=[agent], - execution_config={"attempts_per_task": 2}, + execution_config={ + "attempts_per_task": 2, + "judge_api_key_secret": "OPENAI_API_KEY", + }, env_config={"LOG_LEVEL": "info"}, git_identity="acme/workspace", ) assert result.id == "benchmark-run-1" + assert result.workflow_id == "benchmark-run/benchmark-run-1" assert result.task_count == 12 + assert result.platform_url == ( + "https://platform.osmosis.ai/acme/benchmarks/benchmark-run-1" + ) assert mock_request.call_args.args[0] == "/api/cli/benchmark-runs" assert mock_request.call_args.kwargs == { "method": "POST", "data": { - "experiment_config": {"benchmark": "Terminal-Bench 2.1"}, + "experiment_config": {"benchmark": "HLE"}, "tasks_config": {"task_set": "parity"}, "agents": [agent], - "execution_config": {"attempts_per_task": 2}, + "execution_config": { + "attempts_per_task": 2, + "judge_api_key_secret": "OPENAI_API_KEY", + }, "env_config": {"LOG_LEVEL": "info"}, }, "credentials": None, @@ -52,6 +62,41 @@ def test_submit_benchmark_run_posts_config_sections(mock_request: MagicMock) -> } +@patch("osmosis_ai.platform.api.client.platform_request") +def test_submit_benchmark_run_forwards_harness_api_key_secret( + mock_request: MagicMock, +) -> None: + mock_request.return_value = { + "id": "benchmark-run-1", + "name": "bright-otter", + "status": "pending", + "workflow_id": "benchmark-run/benchmark-run-1", + "task_count": 120, + "created_at": "2026-07-25T00:00:00Z", + "platform_url": None, + } + agent: dict[str, Any] = { + "harness": "cursor-cli", + "harness_api_key_secret": "CURSOR_API_KEY", + "model": { + "type": "provider", + "model": "openai/gpt-5", + "api_key_secret": "OPENAI_API_KEY", + }, + } + + OsmosisClient().submit_benchmark_run( + experiment_config={"benchmark": "DeepSWE"}, + agents=[agent], + git_identity="acme/workspace", + ) + + assert mock_request.call_args.kwargs["data"] == { + "experiment_config": {"benchmark": "DeepSWE"}, + "agents": [agent], + } + + @patch("osmosis_ai.platform.api.client.platform_request") def test_submit_benchmark_run_omits_empty_optional_sections( mock_request: MagicMock, @@ -60,16 +105,36 @@ def test_submit_benchmark_run_omits_empty_optional_sections( "id": "benchmark-run-1", "name": "bright-otter", "status": "pending", + "workflow_id": "benchmark-run/benchmark-run-1", + "task_count": 12, "created_at": "2026-07-25T00:00:00Z", } OsmosisClient().submit_benchmark_run( experiment_config={"benchmark": "Terminal-Bench 2.1"}, - agents=[{"model": {"type": "hosted"}}], + agents=[ + { + "harness": "codex", + "model": { + "type": "hosted", + "base_model": "Qwen/Qwen3-8B", + "checkpoint_name": "terminal-agent", + }, + } + ], git_identity="acme/workspace", ) assert mock_request.call_args.kwargs["data"] == { "experiment_config": {"benchmark": "Terminal-Bench 2.1"}, - "agents": [{"model": {"type": "hosted"}}], + "agents": [ + { + "harness": "codex", + "model": { + "type": "hosted", + "base_model": "Qwen/Qwen3-8B", + "checkpoint_name": "terminal-agent", + }, + } + ], } diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index c69b2f9d..10b05905 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -20,13 +20,14 @@ def test_load_benchmark_submit_config_accepts_provider_and_endpoint_agents( tmp_path / "benchmark.toml", """ [experiment] -benchmark = "Terminal-Bench 2.1" +benchmark = "DeepSWE" [tasks] -task_names = ["git-multibranch"] +task_names = ["abs-module-cache-flags"] [[agents]] -harness = "codex" +harness = "cursor-cli" +harness_api_key_secret = "CURSOR_API_KEY" [agents.model] type = "provider" @@ -56,16 +57,21 @@ def test_load_benchmark_submit_config_accepts_provider_and_endpoint_agents( config = load_benchmark_submit_config(path) - assert config.experiment_config == {"benchmark": "Terminal-Bench 2.1"} - assert config.tasks_config == {"task_names": ["git-multibranch"]} + assert config.experiment_config == {"benchmark": "DeepSWE"} + assert config.tasks_config == {"task_names": ["abs-module-cache-flags"]} assert config.execution_config == { "attempts_per_task": 2, "max_concurrent_attempts": 8, } assert config.env == {"LOG_LEVEL": "info"} - assert config.required_secrets == ["OPENAI_API_KEY", "CUSTOM_API_KEY"] + assert config.required_secrets == [ + "OPENAI_API_KEY", + "CUSTOM_API_KEY", + "CURSOR_API_KEY", + ] assert config.agents_config[0] == { - "harness": "codex", + "harness": "cursor-cli", + "harness_api_key_secret": "CURSOR_API_KEY", "model": { "type": "provider", "model": "openai/gpt-5", @@ -99,12 +105,114 @@ def test_load_benchmark_submit_config_accepts_hosted_model(tmp_path: Path) -> No assert config.required_secrets == [] +def test_load_benchmark_submit_config_accepts_hle_parity_with_explicit_filters( + tmp_path: Path, +) -> None: + """The platform gives task_set precedence over explicit filters.""" + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "HLE" + +[tasks] +task_set = "parity" +task_names = ["hle__sample"] +categories = ["Math"] + +[[agents]] +harness = "codex" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "hle-agent" + +[execution] +judge_model = "openai/gpt-5" +judge_api_key_secret = "OPENAI_API_KEY" +""", + ) + + config = load_benchmark_submit_config(path) + + assert config.tasks_config == { + "categories": ["Math"], + "task_names": ["hle__sample"], + "task_set": "parity", + } + assert config.required_secrets == ["OPENAI_API_KEY", "HF_TOKEN"] + + +def test_load_benchmark_submit_config_rejects_unknown_task_set( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "HLE" + +[tasks] +task_set = "full" + +[[agents]] +harness = "codex" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "hle-agent" +""", + ) + + with pytest.raises(CLIError, match=r"tasks.task_set"): + load_benchmark_submit_config(path) + + +@pytest.mark.parametrize( + "tasks_body, field_name", + [ + ('task_names = "hle__sample"', "task_names"), + ('task_names = [""]', "task_names"), + ('categories = "Math"', "categories"), + ('categories = [""]', "categories"), + ], +) +def test_load_benchmark_submit_config_rejects_invalid_explicit_task_filters( + tmp_path: Path, + tasks_body: str, + field_name: str, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "HLE" + +[tasks] +{tasks_body} + +[[agents]] +harness = "codex" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "hle-agent" +""", + ) + + with pytest.raises(CLIError, match=rf"tasks.{field_name}"): + load_benchmark_submit_config(path) + + def test_load_benchmark_submit_config_rejects_unknown_section(tmp_path: Path) -> None: path = _write_config( tmp_path / "benchmark.toml", """ [experiment] -benchmark = "Terminal-Bench 2.1" +benchmark = "DeepSWE" [[agents]] [agents.model] @@ -130,7 +238,7 @@ def test_load_benchmark_submit_config_rejects_secret_env_collision( tmp_path / "benchmark.toml", """ [experiment] -benchmark = "Terminal-Bench 2.1" +benchmark = "DeepSWE" [[agents]] [agents.model] @@ -159,7 +267,7 @@ def test_load_benchmark_submit_config_allows_cross_agent_secret_env_names( tmp_path / "benchmark.toml", """ [experiment] -benchmark = "Terminal-Bench 2.1" +benchmark = "DeepSWE" [[agents]] [agents.model] @@ -192,7 +300,7 @@ def test_load_benchmark_submit_config_rejects_judge_secret_env_collision( tmp_path / "benchmark.toml", """ [experiment] -benchmark = "Terminal-Bench 2.1" +benchmark = "DeepSWE" [[agents]] [agents.model] @@ -211,3 +319,257 @@ def test_load_benchmark_submit_config_rejects_judge_secret_env_collision( with pytest.raises(CLIError, match=r"judge_api_key_secret"): load_benchmark_submit_config(path) + + +@pytest.mark.parametrize( + "secret_name", + [ + "HF_TOKEN", + "DAYTONA_API_KEY", + "DAYTONA_API_URL", + "SKYPILOT_SERVICE_ACCOUNT_TOKEN", + "SKYPILOT_API_SERVER_ENDPOINT", + ], +) +@pytest.mark.parametrize("model_type", ["provider", "endpoint"]) +def test_load_benchmark_submit_config_rejects_reserved_model_secret_names( + tmp_path: Path, + secret_name: str, + model_type: str, +) -> None: + model_fields = ( + 'model = "openai/gpt-5"' + if model_type == "provider" + else 'base_url = "https://models.example.com/v1"\nmodel = "custom-model"' + ) + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "Terminal-Bench 2.1" + +[[agents]] +harness = "codex" + +[agents.model] +type = "{model_type}" +{model_fields} +api_key_secret = "{secret_name}" +""", + ) + + with pytest.raises(CLIError) as exc_info: + load_benchmark_submit_config(path) + + assert secret_name in str(exc_info.value) + assert "reserved by the benchmark runner" in str(exc_info.value) + + +@pytest.mark.parametrize("benchmark", ["Terminal-Bench 2.1", "HLE"]) +@pytest.mark.parametrize( + "env_section", + [ + '[env]\nHF_TOKEN = "literal-token"', + '[agents.env]\nHF_TOKEN = "literal-token"', + ], +) +def test_load_benchmark_submit_config_rejects_literal_hf_token_for_every_benchmark( + tmp_path: Path, + benchmark: str, + env_section: str, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "{benchmark}" + +[[agents]] +harness = "codex" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "benchmark-agent" + +{env_section} +""", + ) + + with pytest.raises(CLIError, match=r"reserved by the benchmark runner"): + load_benchmark_submit_config(path) + + +def test_load_benchmark_submit_config_validates_harness_secret_name( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "cursor-cli" +harness_api_key_secret = "invalid-secret" + +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" +""", + ) + + with pytest.raises(CLIError, match=r"Invalid secret name 'invalid-secret'"): + load_benchmark_submit_config(path) + + +def test_load_benchmark_submit_config_allows_harness_secret_record_name_as_env( + tmp_path: Path, +) -> None: + """The record name is separate from the harness's destination env name.""" + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "cursor-cli" +harness_api_key_secret = "MY_CURSOR_TOKEN" + +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[agents.env] +MY_CURSOR_TOKEN = "literal-for-the-agent" +""", + ) + + config = load_benchmark_submit_config(path) + + assert config.required_secrets == ["OPENAI_API_KEY", "MY_CURSOR_TOKEN"] + assert config.agents[0].env == {"MY_CURSOR_TOKEN": "literal-for-the-agent"} + + +@pytest.mark.parametrize( + "harness, harness_secret, destination_env", + [ + ("cursor-cli", "MY_CURSOR_TOKEN", "CURSOR_API_KEY"), + ("mini-swe-agent", "MY_MSWEA_TOKEN", "MSWEA_API_KEY"), + ], +) +def test_load_benchmark_submit_config_rejects_harness_destination_env_collision( + tmp_path: Path, + harness: str, + harness_secret: str, + destination_env: str, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "{harness}" +harness_api_key_secret = "{harness_secret}" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "deep-swe-agent" + +[agents.env] +{destination_env} = "literal-for-the-agent" +""", + ) + + with pytest.raises(CLIError, match=destination_env): + load_benchmark_submit_config(path) + + +def test_load_benchmark_submit_config_rejects_harness_destination_env_without_secret( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "cursor-cli" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "deep-swe-agent" + +[agents.env] +CURSOR_API_KEY = "literal-for-the-agent" +""", + ) + + with pytest.raises(CLIError, match=r"CURSOR_API_KEY"): + load_benchmark_submit_config(path) + + +def test_load_benchmark_submit_config_requires_known_harness_secret( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "mini-swe-agent" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "deep-swe-agent" +""", + ) + + with pytest.raises(CLIError, match=r"requires harness_api_key_secret"): + load_benchmark_submit_config(path) + + +@pytest.mark.parametrize( + "secret_field, section", + [ + ('harness_api_key_secret = ""', ""), + ("", '[execution]\njudge_api_key_secret = ""'), + ], +) +def test_load_benchmark_submit_config_rejects_empty_secret_references( + tmp_path: Path, + secret_field: str, + section: str, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "cursor-cli" +{secret_field} + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "deep-swe-agent" + +{section} +""", + ) + + with pytest.raises(CLIError, match=r"Invalid secret name ''"): + load_benchmark_submit_config(path) diff --git a/tests/unit/platform/cli/test_benchmark_submit.py b/tests/unit/platform/cli/test_benchmark_submit.py index f2532c02..7c92f5d5 100644 --- a/tests/unit/platform/cli/test_benchmark_submit.py +++ b/tests/unit/platform/cli/test_benchmark_submit.py @@ -28,13 +28,11 @@ def _write_config(path: Path) -> Path: path.write_text( """ [experiment] -benchmark = "Terminal-Bench 2.1" - -[tasks] -task_set = "parity" +benchmark = "DeepSWE" [[agents]] -harness = "codex" +harness = "cursor-cli" +harness_api_key_secret = "CURSOR_API_KEY" [agents.model] type = "provider" @@ -50,18 +48,66 @@ def _write_config(path: Path) -> Path: return path +def _write_hosted_config( + path: Path, + *, + benchmark: str, + tasks: str = "", +) -> Path: + path.write_text( + f""" +[experiment] +benchmark = "{benchmark}" + +{tasks} + +[[agents]] +harness = "codex" + +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "benchmark-agent" + +[execution] +judge_model = "openai/gpt-5" +judge_api_key_secret = "OPENAI_API_KEY" +""".strip() + + "\n", + encoding="utf-8", + ) + return path + + +def _context(workspace: Path) -> SimpleNamespace: + return SimpleNamespace( + workspace_directory=workspace, + git_identity=GIT_IDENTITY, + repo_url=REPO_URL, + credentials=FAKE_CREDENTIALS, + ) + + +class _FakeSubmitClient: + def submit_benchmark_run(self, **kwargs: Any) -> SubmitBenchmarkRunResult: + return SubmitBenchmarkRunResult( + id="benchmark-run-1", + name="bright-otter", + status="pending", + workflow_id="benchmark-run/benchmark-run-1", + task_count=10, + created_at="2026-07-25T00:00:00Z", + platform_url="https://platform.osmosis.ai/acme/benchmarks/benchmark-run-1", + ) + + def test_submit_sends_benchmark_config_and_returns_operation_result( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: workspace = _make_workspace(tmp_path / "workspace") config_path = _write_config(workspace / "configs" / "benchmark" / "smoke.toml") - context = SimpleNamespace( - workspace_directory=workspace, - git_identity=GIT_IDENTITY, - repo_url=REPO_URL, - credentials=FAKE_CREDENTIALS, - ) + context = _context(workspace) captured: dict[str, Any] = {} class FakeClient: @@ -84,22 +130,31 @@ def submit_benchmark_run(self, **kwargs: Any) -> SubmitBenchmarkRunResult: monkeypatch.setattr( benchmark_module, "_fetch_secret_scopes", - lambda *args, **kwargs: ({"OPENAI_API_KEY"}, set()), + lambda *args, **kwargs: ({"OPENAI_API_KEY", "CURSOR_API_KEY"}, set()), ) result = benchmark_module.submit(config_path, yes=True) - assert captured["experiment_config"] == {"benchmark": "Terminal-Bench 2.1"} - assert captured["tasks_config"] == {"task_set": "parity"} + assert captured["experiment_config"] == {"benchmark": "DeepSWE"} + assert captured["tasks_config"] is None assert captured["execution_config"] == {"attempts_per_task": 2} assert captured["agents"][0]["model"]["model"] == "openai/gpt-5" + assert captured["agents"][0]["harness_api_key_secret"] == "CURSOR_API_KEY" assert captured["credentials"] is FAKE_CREDENTIALS assert captured["git_identity"] == GIT_IDENTITY assert isinstance(result, OperationResult) assert result.operation == "benchmark.submit" assert result.resource is not None assert result.resource["task_count"] == 10 - assert result.resource["benchmark_name"] == "Terminal-Bench 2.1" + assert result.resource["benchmark_name"] == "DeepSWE" + assert result.resource["workflow_id"] == "benchmark-run/benchmark-run-1" + assert result.resource["platform_url"] == ( + "https://platform.osmosis.ai/acme/benchmarks/benchmark-run-1" + ) + assert "url" not in result.resource + assert result.resource["config"]["agents"][0]["harness_api_key_secret"] == ( + "CURSOR_API_KEY" + ) def test_submit_rejects_config_outside_benchmark_directory( @@ -108,15 +163,132 @@ def test_submit_rejects_config_outside_benchmark_directory( ) -> None: workspace = _make_workspace(tmp_path / "workspace") config_path = _write_config(workspace / "smoke.toml") - context = SimpleNamespace( - workspace_directory=workspace, - git_identity=GIT_IDENTITY, - repo_url=REPO_URL, - credentials=FAKE_CREDENTIALS, - ) + context = _context(workspace) monkeypatch.setattr( benchmark_module, "require_git_workspace_directory_context", lambda: context ) with pytest.raises(CLIError, match="configs/benchmark"): benchmark_module.submit(config_path, yes=True) + + +@pytest.mark.parametrize("benchmark_name", ["HLE", " HLE ", " hLe "]) +def test_submit_warns_before_confirmation_for_hle_without_parity( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + benchmark_name: str, +) -> None: + workspace = _make_workspace(tmp_path / "workspace") + config_path = _write_hosted_config( + workspace / "configs" / "benchmark" / "hle.toml", + benchmark=benchmark_name, + tasks='[tasks]\ntask_names = ["hle__sample"]', + ) + events: list[tuple[str, object]] = [] + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + lambda: _context(workspace), + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", _FakeSubmitClient) + monkeypatch.setattr( + benchmark_module, + "_fetch_secret_scopes", + lambda *args, **kwargs: ({"OPENAI_API_KEY", "HF_TOKEN"}, set()), + ) + monkeypatch.setattr( + benchmark_module.console, + "print_warning", + lambda message, **kwargs: events.append( + ("warning", {"message": message, **kwargs}) + ), + ) + monkeypatch.setattr( + benchmark_module, + "require_confirmation", + lambda *args, **kwargs: events.append(("confirmation", None)), + ) + + result = benchmark_module.submit(config_path, yes=True) + + assert result.status == "success" + assert [event[0] for event in events] == ["warning", "confirmation"] + warning = events[0][1] + assert isinstance(warning, dict) + assert warning["code"] == "HLE_PARITY_RECOMMENDED" + assert 'task_set = "parity"' in str(warning["message"]) + + +def test_submit_does_not_warn_when_hle_uses_parity( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workspace = _make_workspace(tmp_path / "workspace") + config_path = _write_hosted_config( + workspace / "configs" / "benchmark" / "hle.toml", + benchmark="HLE", + tasks='[tasks]\ntask_set = "parity"', + ) + warnings: list[str] = [] + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + lambda: _context(workspace), + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", _FakeSubmitClient) + monkeypatch.setattr( + benchmark_module, + "_fetch_secret_scopes", + lambda *args, **kwargs: ({"OPENAI_API_KEY", "HF_TOKEN"}, set()), + ) + monkeypatch.setattr( + benchmark_module.console, + "print_warning", + lambda message, **kwargs: warnings.append(message), + ) + + result = benchmark_module.submit(config_path, yes=True) + + assert result.status == "success" + assert warnings == [] + + +def test_submit_warns_before_hle_missing_secret_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workspace = _make_workspace(tmp_path / "workspace") + config_path = _write_hosted_config( + workspace / "configs" / "benchmark" / "hle.toml", + benchmark="HLE", + ) + warnings: list[dict[str, str]] = [] + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + lambda: _context(workspace), + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", _FakeSubmitClient) + monkeypatch.setattr( + benchmark_module, + "_fetch_secret_scopes", + lambda *args, **kwargs: (set(), set()), + ) + monkeypatch.setattr( + benchmark_module.console, + "print_warning", + lambda message, **kwargs: warnings.append({"message": message, **kwargs}), + ) + + with pytest.raises(CLIError, match=r"HF_TOKEN"): + benchmark_module.submit(config_path, yes=True) + + assert warnings == [ + { + "message": benchmark_module._HLE_PARITY_WARNING, + "code": "HLE_PARITY_RECOMMENDED", + } + ] From d798d82d130c42eceeffcd04ba0f251fac3bf052 Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 07:42:58 -0700 Subject: [PATCH 03/22] [cli] fix: validate benchmark judge secret reference --- osmosis_ai/platform/cli/benchmark_config.py | 2 +- .../platform/cli/test_benchmark_config.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index a37508ab..dca42119 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -90,7 +90,7 @@ class BenchmarkExecutionSection(_StrictSection): max_retries: Any = None pass_threshold: Any = None judge_model: Any = None - judge_api_key_secret: Any = None + judge_api_key_secret: str | None = None class BenchmarkSubmitConfig(_StrictSection): diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index 10b05905..7d1d31d4 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -321,6 +321,30 @@ def test_load_benchmark_submit_config_rejects_judge_secret_env_collision( load_benchmark_submit_config(path) +def test_load_benchmark_submit_config_rejects_non_string_judge_secret( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "DeepSWE" + +[[agents]] +[agents.model] +type = "hosted" +base_model = "Qwen/Qwen3-8B" +checkpoint_name = "deep-swe-agent" + +[execution] +judge_api_key_secret = 42 +""", + ) + + with pytest.raises(CLIError, match=r"execution.judge_api_key_secret"): + load_benchmark_submit_config(path) + + @pytest.mark.parametrize( "secret_name", [ From a7446da100f9d7734bf0d97c88d1cc0468af8c71 Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 08:17:32 -0700 Subject: [PATCH 04/22] [cli] fix: reject literal endpoint authorization headers --- osmosis_ai/platform/cli/benchmark_config.py | 16 ++++- .../platform/cli/test_benchmark_config.py | 61 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index dca42119..feb206ee 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Annotated, Any, ClassVar, Literal -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator from osmosis_ai.cli.errors import CLIError from osmosis_ai.platform.cli.shared_config import ( @@ -63,6 +63,20 @@ class BenchmarkEndpointModel(_StrictSection): api_key_secret: str extra_headers: dict[str, str] | None = None + @field_validator("extra_headers") + @classmethod + def reject_authorization_header( + cls, extra_headers: dict[str, str] | None + ) -> dict[str, str] | None: + if extra_headers is not None and any( + name.casefold() == "authorization" for name in extra_headers + ): + raise ValueError( + "extra_headers must not include an Authorization header; " + "use api_key_secret for endpoint authentication" + ) + return extra_headers + class BenchmarkHostedModel(_StrictSection): type: Literal["hosted"] diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index 7d1d31d4..5d4f20c0 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -81,6 +81,67 @@ def test_load_benchmark_submit_config_accepts_provider_and_endpoint_agents( } +@pytest.mark.parametrize( + "header_name", + ["Authorization", "authorization", "AUTHORIZATION", "aUtHoRiZaTiOn"], +) +def test_load_benchmark_submit_config_rejects_endpoint_authorization_header( + tmp_path: Path, + header_name: str, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "DeepSWE" + +[[agents]] +[agents.model] +type = "endpoint" +base_url = "https://models.example.com/v1" +model = "custom-model" +api_key_secret = "CUSTOM_API_KEY" + +[agents.model.extra_headers] +"{header_name}" = "Bearer literal-token" +""", + ) + + with pytest.raises( + CLIError, + match=r"use api_key_secret for endpoint authentication", + ): + load_benchmark_submit_config(path) + + +def test_load_benchmark_submit_config_accepts_non_authorization_endpoint_header( + tmp_path: Path, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + """ +[experiment] +benchmark = "DeepSWE" + +[[agents]] +[agents.model] +type = "endpoint" +base_url = "https://models.example.com/v1" +model = "custom-model" +api_key_secret = "CUSTOM_API_KEY" + +[agents.model.extra_headers] +"X-Request-ID" = "benchmark-run" +""", + ) + + config = load_benchmark_submit_config(path) + + assert config.agents_config[0]["model"]["extra_headers"] == { + "X-Request-ID": "benchmark-run" + } + + def test_load_benchmark_submit_config_accepts_hosted_model(tmp_path: Path) -> None: path = _write_config( tmp_path / "benchmark.toml", From 1f2d55277fa0cee526821ec3f1776f6930ab483f Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 08:30:16 -0700 Subject: [PATCH 05/22] [cli] fix: reject blank benchmark task selectors --- osmosis_ai/platform/cli/benchmark_config.py | 2 +- tests/unit/platform/cli/test_benchmark_config.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index feb206ee..78a4faec 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -41,7 +41,7 @@ class BenchmarkExperimentSection(_StrictSection): benchmark: str -_NonEmptyTaskSelector = Annotated[str, Field(min_length=1)] +_NonEmptyTaskSelector = Annotated[str, Field(min_length=1, pattern=r"\S")] class BenchmarkTasksSection(_StrictSection): diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index 5d4f20c0..51a489d1 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -236,8 +236,10 @@ def test_load_benchmark_submit_config_rejects_unknown_task_set( [ ('task_names = "hle__sample"', "task_names"), ('task_names = [""]', "task_names"), + ('task_names = [" "]', "task_names"), ('categories = "Math"', "categories"), ('categories = [""]', "categories"), + ('categories = [" "]', "categories"), ], ) def test_load_benchmark_submit_config_rejects_invalid_explicit_task_filters( From 5526187cff0fa617701951b01ca473cc7b85a587 Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 08:49:07 -0700 Subject: [PATCH 06/22] [cli] fix: label benchmark agent env errors --- osmosis_ai/platform/cli/benchmark_config.py | 8 ++++-- osmosis_ai/platform/cli/shared_config.py | 7 +++-- .../platform/cli/test_benchmark_config.py | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index 78a4faec..0db4760a 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -261,8 +261,12 @@ def load_benchmark_submit_config(path: Path) -> BenchmarkSubmitConfig: ) from exc validate_env_var_keys(env=config.env, path=path) - for agent in config.agents: - validate_env_var_keys(env=agent.env, path=path) + for index, agent in enumerate(config.agents, start=1): + validate_env_var_keys( + env=agent.env, + path=path, + source_label=f"agent {index}'s [agents.env]", + ) _validate_secret_references(config, path) return config diff --git a/osmosis_ai/platform/cli/shared_config.py b/osmosis_ai/platform/cli/shared_config.py index c06aa7d1..773ac6e5 100644 --- a/osmosis_ai/platform/cli/shared_config.py +++ b/osmosis_ai/platform/cli/shared_config.py @@ -301,17 +301,18 @@ def validate_env_var_keys( *, env: dict[str, str], path: Path, + source_label: str = "[env]", ) -> None: - """Reject invalid or reserved [env] var names.""" + """Reject invalid or reserved environment variable names.""" for key in env: if not ENV_VAR_NAME_RE.match(key): raise CLIError( - f"Invalid env var name '{key}' in [env] of {path}: " + f"Invalid env var name '{key}' in {source_label} of {path}: " "must match ^[A-Z_][A-Z0-9_]*$" ) if key.startswith(RESERVED_ENV_PREFIX): raise CLIError( - f"'{key}' in [env] of {path}: env var names starting " + f"'{key}' in {source_label} of {path}: env var names starting " f"with {RESERVED_ENV_PREFIX} are reserved by the platform; " "choose a different name." ) diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index 51a489d1..e8cd3eaf 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -294,6 +294,34 @@ def test_load_benchmark_submit_config_rejects_unknown_section(tmp_path: Path) -> assert "harbor: Unrecognized key" in str(exc_info.value) +@pytest.mark.parametrize("env_key", ["bad-name", "_OSMOSIS_INTERNAL"]) +def test_load_benchmark_submit_config_labels_invalid_agent_env( + tmp_path: Path, + env_key: str, +) -> None: + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "DeepSWE" + +[[agents]] +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" + +[agents.env] +{env_key} = "value" +""", + ) + + with pytest.raises(CLIError) as exc_info: + load_benchmark_submit_config(path) + + assert "agent 1's [agents.env]" in str(exc_info.value) + + def test_load_benchmark_submit_config_rejects_secret_env_collision( tmp_path: Path, ) -> None: From a506e1d148c0abd9645c3ff06cbddbe9f78bbcbf Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 14:29:36 -0700 Subject: [PATCH 07/22] [cli] feat: add benchmark catalog commands --- osmosis_ai/cli/commands/benchmark.py | 33 ++- osmosis_ai/platform/api/client.py | 34 +++ osmosis_ai/platform/api/models.py | 134 ++++++++++++ osmosis_ai/platform/cli/benchmark.py | 196 +++++++++++++++++- tests/unit/cli/test_benchmark_commands.py | 36 ++++ tests/unit/cli/test_command_groups.py | 2 + .../platform/api/test_client_benchmark.py | 87 ++++++++ .../platform/cli/test_benchmark_catalog.py | 167 +++++++++++++++ 8 files changed, 682 insertions(+), 7 deletions(-) create mode 100644 tests/unit/platform/cli/test_benchmark_catalog.py diff --git a/osmosis_ai/cli/commands/benchmark.py b/osmosis_ai/cli/commands/benchmark.py index d8dc5845..a2784069 100644 --- a/osmosis_ai/cli/commands/benchmark.py +++ b/osmosis_ai/cli/commands/benchmark.py @@ -1,4 +1,4 @@ -"""Benchmark commands (submit).""" +"""Benchmark catalog and run submission commands.""" from __future__ import annotations @@ -7,12 +7,41 @@ import typer +from osmosis_ai.platform.constants import DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE + app: typer.Typer = typer.Typer( - help="Manage benchmark runs (submit).", + help="Discover benchmarks and submit benchmark runs (list, info, submit).", no_args_is_help=True, ) +@app.command("list") +def benchmark_list( + limit: int = typer.Option( + DEFAULT_PAGE_SIZE, + "--limit", + min=1, + max=MAX_PAGE_SIZE, + help="Maximum number of benchmarks to show.", + ), + all_: bool = typer.Option(False, "--all", help="Show all benchmarks."), +) -> Any: + """List benchmarks available in the current workspace.""" + from osmosis_ai.platform.cli.benchmark import list_benchmarks as _list_benchmarks + + return _list_benchmarks(limit=limit, all_=all_) + + +@app.command("info") +def benchmark_info( + name_or_id: str = typer.Argument(..., help="Benchmark name or ID."), +) -> Any: + """Show benchmark metadata and task-selection options.""" + from osmosis_ai.platform.cli.benchmark import info as _info + + return _info(name_or_id) + + @app.command("submit") def benchmark_submit( config_path: Path = typer.Argument( diff --git a/osmosis_ai/platform/api/client.py b/osmosis_ai/platform/api/client.py index e0ffe197..b1ab7aad 100644 --- a/osmosis_ai/platform/api/client.py +++ b/osmosis_ai/platform/api/client.py @@ -10,6 +10,7 @@ from osmosis_ai.platform.constants import DEFAULT_PAGE_SIZE from .models import ( + BenchmarkCatalogDetail, DatasetDownloadInfo, DatasetFile, EnvironmentSecretInfo, @@ -20,6 +21,7 @@ LoraModelDetail, LoraModelSummary, PaginatedBaseModels, + PaginatedBenchmarks, PaginatedDatasets, PaginatedDevRolloutServers, PaginatedEnvironmentSecrets, @@ -667,6 +669,38 @@ def submit_benchmark_run( ) return SubmitBenchmarkRunResult.from_dict(result) + def list_benchmarks( + self, + limit: int = DEFAULT_PAGE_SIZE, + offset: int = 0, + *, + credentials: Credentials | None = None, + git_identity: str, + ) -> PaginatedBenchmarks: + """List benchmarks available in the current workspace.""" + qs = urlencode({"limit": limit, "offset": offset}) + data = platform_request( + f"/api/cli/benchmarks?{qs}", + credentials=credentials, + git_identity=git_identity, + ) + return PaginatedBenchmarks.from_dict(data) + + def get_benchmark( + self, + name_or_id: str, + *, + credentials: Credentials | None = None, + git_identity: str, + ) -> BenchmarkCatalogDetail: + """Get benchmark metadata and task-selection options.""" + data = platform_request( + f"/api/cli/benchmarks/{_safe_path(name_or_id)}", + credentials=credentials, + git_identity=git_identity, + ) + return BenchmarkCatalogDetail.from_dict(data) + def list_eval_runs( self, limit: int = DEFAULT_PAGE_SIZE, diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index ff230d7a..bb06a14a 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -352,6 +352,140 @@ def from_dict(cls, data: dict[str, Any]) -> SubmitRunResult: ) +@dataclass +class BenchmarkTaskSet: + """A named task set exposed by a benchmark.""" + + name: str + task_count: int + recommended: bool + description: str | None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchmarkTaskSet: + return cls( + name=data["name"], + task_count=data["task_count"], + recommended=data.get("recommended", False), + description=data.get("description"), + ) + + +@dataclass +class BenchmarkCategory: + """Task count for one benchmark category.""" + + name: str + task_count: int + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchmarkCategory: + return cls(name=data["name"], task_count=data["task_count"]) + + +@dataclass +class BenchmarkCatalogEntry: + """Benchmark available in the current workspace catalog.""" + + id: str + name: str + description: str | None + source_type: str + source_ref: str + task_count: int + category_count: int + task_sets: list[BenchmarkTaskSet] + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogEntry: + return cls( + id=data["id"], + name=data["name"], + description=data.get("description"), + source_type=data["source_type"], + source_ref=data["source_ref"], + task_count=data["task_count"], + category_count=data["category_count"], + task_sets=[ + BenchmarkTaskSet.from_dict(item) for item in data.get("task_sets", []) + ], + ) + + +@dataclass +class PaginatedBenchmarks: + """Paginated workspace benchmark catalog.""" + + benchmarks: list[BenchmarkCatalogEntry] + total_count: int + has_more: bool + next_offset: int | None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> PaginatedBenchmarks: + return cls( + benchmarks=[ + BenchmarkCatalogEntry.from_dict(item) + for item in data.get("benchmarks", []) + ], + total_count=data.get("total_count", 0), + has_more=data.get("has_more", False), + next_offset=data.get("next_offset"), + ) + + +@dataclass +class BenchmarkCatalogDetail: + """Detailed benchmark metadata and task-selection options.""" + + id: str + name: str + description: str | None + source_type: str + source_ref: str + task_count: int + category_count: int + task_sets: list[BenchmarkTaskSet] + runner_family: str + supports_harness: bool + requires_harness: bool + requires_judge_model: bool + judge_model_default: str | None + pass_threshold: float + categories: list[BenchmarkCategory] + tasks: list[dict[str, Any]] + unavailable_tasks: dict[str, Any] | None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: + benchmark = data["benchmark"] + return cls( + id=benchmark["id"], + name=benchmark["name"], + description=benchmark.get("description"), + source_type=benchmark["source_type"], + source_ref=benchmark["source_ref"], + task_count=benchmark["task_count"], + category_count=benchmark["category_count"], + task_sets=[ + BenchmarkTaskSet.from_dict(item) + for item in benchmark.get("task_sets", []) + ], + runner_family=benchmark["runner_family"], + supports_harness=benchmark["supports_harness"], + requires_harness=benchmark["requires_harness"], + requires_judge_model=benchmark["requires_judge_model"], + judge_model_default=benchmark.get("judge_model_default"), + pass_threshold=float(benchmark["pass_threshold"]), + categories=[ + BenchmarkCategory.from_dict(item) + for item in benchmark.get("categories", []) + ], + tasks=list(benchmark.get("tasks", [])), + unavailable_tasks=benchmark.get("unavailable_tasks"), + ) + + @dataclass class SubmitBenchmarkRunResult: """Result of submitting a benchmark run.""" diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 0f89bcb4..19454b44 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -1,4 +1,4 @@ -"""Handler for ``osmosis benchmark submit``.""" +"""Handlers for benchmark catalog discovery and run submission.""" from __future__ import annotations @@ -7,10 +7,22 @@ from osmosis_ai.cli.console import console from osmosis_ai.cli.errors import CLIError -from osmosis_ai.cli.output import OperationResult, get_output_context +from osmosis_ai.cli.output import ( + DetailResult, + ListColumn, + ListResult, + OperationResult, + detail_fields, + get_output_context, +) from osmosis_ai.cli.prompts import require_confirmation from osmosis_ai.platform.api.client import OsmosisClient -from osmosis_ai.platform.api.models import SubmitBenchmarkRunResult +from osmosis_ai.platform.api.models import ( + BenchmarkCatalogDetail, + BenchmarkCatalogEntry, + BenchmarkTaskSet, + SubmitBenchmarkRunResult, +) from osmosis_ai.platform.auth.platform_client import PlatformAPIError from osmosis_ai.platform.cli.benchmark_config import ( BenchmarkSubmitConfig, @@ -25,7 +37,11 @@ _fetch_secret_scopes, _missing_secret_message, ) -from osmosis_ai.platform.cli.utils import require_git_workspace_directory_context +from osmosis_ai.platform.cli.utils import ( + paginated_fetch, + require_git_workspace_directory_context, + validate_list_options, +) from osmosis_ai.platform.cli.workspace_directory_context import git_result_context from osmosis_ai.platform.cli.workspace_directory_contract import ( ensure_workspace_directory_config_path, @@ -38,6 +54,176 @@ "custom task selection." ) +_BENCHMARK_COLUMNS = [ + ListColumn(key="name", label="Name", ratio=4, overflow="fold"), + ListColumn(key="task_count", label="Tasks", no_wrap=True, ratio=1), + ListColumn(key="category_count", label="Categories", no_wrap=True, ratio=1), + ListColumn(key="task_sets", label="Named Task Sets", ratio=2, overflow="fold"), + ListColumn(key="source", label="Source", no_wrap=True, ratio=1), +] + + +def _task_set_resource(task_set: BenchmarkTaskSet) -> dict[str, Any]: + return { + "name": task_set.name, + "task_count": task_set.task_count, + "recommended": task_set.recommended, + "description": task_set.description, + } + + +def _benchmark_resource( + benchmark: BenchmarkCatalogEntry | BenchmarkCatalogDetail, +) -> dict[str, Any]: + return { + "id": benchmark.id, + "name": benchmark.name, + "description": benchmark.description, + "source_type": benchmark.source_type, + "source_ref": benchmark.source_ref, + "task_count": benchmark.task_count, + "category_count": benchmark.category_count, + "task_sets": [_task_set_resource(task_set) for task_set in benchmark.task_sets], + } + + +def _task_set_display(task_sets: list[BenchmarkTaskSet]) -> str: + if not task_sets: + return "–" + labels = [] + for task_set in task_sets: + suffix = ", recommended" if task_set.recommended else "" + labels.append(f"{task_set.name} ({task_set.task_count:,}{suffix})") + return ", ".join(labels) + + +def list_benchmarks(*, limit: int, all_: bool) -> ListResult: + """List benchmarks available in the current workspace.""" + effective_limit, fetch_all = validate_list_options(limit=limit, all_=all_) + context = require_git_workspace_directory_context() + client = OsmosisClient() + output = get_output_context() + + with output.status("Fetching benchmarks..."): + benchmarks, total_count, has_more, next_offset = paginated_fetch( + lambda lim, off: client.list_benchmarks( + limit=lim, + offset=off, + credentials=context.credentials, + git_identity=context.git_identity, + ), + items_attr="benchmarks", + limit=effective_limit, + fetch_all=fetch_all, + ) + + return ListResult( + title="Benchmarks", + items=[_benchmark_resource(benchmark) for benchmark in benchmarks], + total_count=total_count, + has_more=has_more, + next_offset=next_offset, + extra=git_result_context(context), + columns=_BENCHMARK_COLUMNS, + display_items=[ + { + **_benchmark_resource(benchmark), + "task_count": f"{benchmark.task_count:,}", + "category_count": f"{benchmark.category_count:,}", + "task_sets": _task_set_display(benchmark.task_sets), + "source": ( + "Managed" + if benchmark.source_type == "osmosis_managed" + else "Harbor" + ), + } + for benchmark in benchmarks + ], + display_hints=[ + "Use osmosis benchmark info for task sets, categories, and tasks." + ], + ) + + +def info(name_or_id: str) -> DetailResult: + """Show benchmark metadata and task-selection options.""" + context = require_git_workspace_directory_context() + client = OsmosisClient() + output = get_output_context() + + with output.status(f'Fetching benchmark "{console.escape(name_or_id)}"...'): + benchmark = client.get_benchmark( + name_or_id, + credentials=context.credentials, + git_identity=context.git_identity, + ) + + harness = ( + "Required" + if benchmark.requires_harness + else "Optional" + if benchmark.supports_harness + else "Not supported" + ) + judge = "Not required" + if benchmark.requires_judge_model: + judge = "Required" + if benchmark.judge_model_default: + judge += f" (default: {benchmark.judge_model_default})" + + category_display = ", ".join( + f"{category.name} ({category.task_count:,})" + for category in benchmark.categories + ) + rows = [ + ("Name", console.escape(benchmark.name)), + ("Description", console.escape(benchmark.description or "–")), + ("Source", f"{benchmark.source_type}: {benchmark.source_ref}"), + ("Runner", benchmark.runner_family), + ("Tasks", f"{benchmark.task_count:,}"), + ("Categories", category_display or "–"), + ("Named Task Sets", _task_set_display(benchmark.task_sets)), + ("Harness", harness), + ("LLM Judge", judge), + ("Pass Threshold", f"{benchmark.pass_threshold:g}"), + ] + + benchmark_data = { + **_benchmark_resource(benchmark), + "runner_family": benchmark.runner_family, + "supports_harness": benchmark.supports_harness, + "requires_harness": benchmark.requires_harness, + "requires_judge_model": benchmark.requires_judge_model, + "judge_model_default": benchmark.judge_model_default, + "pass_threshold": benchmark.pass_threshold, + "categories": [ + {"name": category.name, "task_count": category.task_count} + for category in benchmark.categories + ], + "tasks": benchmark.tasks, + "unavailable_tasks": benchmark.unavailable_tasks, + } + + display_hints = [ + f"Omit [tasks] to select all {benchmark.task_count:,} tasks.", + "Use task_names or categories under [tasks] for a custom subset.", + "Use osmosis --json benchmark info to inspect the full task list.", + ] + for task_set in benchmark.task_sets: + if task_set.recommended: + display_hints.insert( + 0, + f"For {benchmark.name}, we recommend [tasks] task_set = " + f'"{task_set.name}" ({task_set.task_count:,} tasks).', + ) + + return DetailResult( + title="Benchmark Info", + data={"benchmark": benchmark_data, **git_result_context(context)}, + fields=detail_fields(rows), + display_hints=display_hints, + ) + def _agent_model_label(agent: dict[str, Any]) -> str: model = agent["model"] @@ -245,4 +431,4 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: ) -__all__ = ["submit"] +__all__ = ["info", "list_benchmarks", "submit"] diff --git a/tests/unit/cli/test_benchmark_commands.py b/tests/unit/cli/test_benchmark_commands.py index 3f8d0bf9..00d695aa 100644 --- a/tests/unit/cli/test_benchmark_commands.py +++ b/tests/unit/cli/test_benchmark_commands.py @@ -8,6 +8,42 @@ import osmosis_ai.platform.cli.benchmark as benchmark_handler +def test_benchmark_list_delegates_to_handler( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + expected = object() + + def fake_list_benchmarks(*, limit: int, all_: bool) -> object: + captured.update(limit=limit, all_=all_) + return expected + + monkeypatch.setattr(benchmark_handler, "list_benchmarks", fake_list_benchmarks) + + result = benchmark_commands.benchmark_list(limit=25, all_=True) + + assert result is expected + assert captured == {"limit": 25, "all_": True} + + +def test_benchmark_info_delegates_to_handler( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + expected = object() + + def fake_info(name_or_id: str) -> object: + captured["name_or_id"] = name_or_id + return expected + + monkeypatch.setattr(benchmark_handler, "info", fake_info) + + result = benchmark_commands.benchmark_info("Terminal-Bench 2.1") + + assert result is expected + assert captured == {"name_or_id": "Terminal-Bench 2.1"} + + def test_benchmark_submit_delegates_to_handler( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/cli/test_command_groups.py b/tests/unit/cli/test_command_groups.py index e8597557..7cdb5e5d 100644 --- a/tests/unit/cli/test_command_groups.py +++ b/tests/unit/cli/test_command_groups.py @@ -46,6 +46,8 @@ ["model", "deploy", "--help"], ["model", "undeploy", "--help"], ["benchmark", "--help"], + ["benchmark", "list", "--help"], + ["benchmark", "info", "--help"], ["benchmark", "submit", "--help"], ["rollout", "--help"], ["template", "--help"], diff --git a/tests/unit/platform/api/test_client_benchmark.py b/tests/unit/platform/api/test_client_benchmark.py index 55e17284..d8ae3432 100644 --- a/tests/unit/platform/api/test_client_benchmark.py +++ b/tests/unit/platform/api/test_client_benchmark.py @@ -6,6 +6,93 @@ from osmosis_ai.platform.api.client import OsmosisClient +@patch("osmosis_ai.platform.api.client.platform_request") +def test_list_benchmarks_gets_paginated_catalog(mock_request: MagicMock) -> None: + mock_request.return_value = { + "benchmarks": [ + { + "id": "benchmark-1", + "name": "HLE", + "description": "Humanity's Last Exam", + "source_type": "osmosis_managed", + "source_ref": "hle", + "task_count": 2_500, + "category_count": 30, + "task_sets": [ + { + "name": "parity", + "task_count": 249, + "recommended": True, + "description": "Published comparison sample.", + } + ], + } + ], + "total_count": 7, + "has_more": True, + "next_offset": 1, + } + + result = OsmosisClient().list_benchmarks( + limit=1, + offset=0, + git_identity="acme/workspace", + ) + + assert result.total_count == 7 + assert result.has_more is True + assert result.next_offset == 1 + assert result.benchmarks[0].name == "HLE" + assert result.benchmarks[0].task_sets[0].name == "parity" + assert result.benchmarks[0].task_sets[0].recommended is True + assert mock_request.call_args.args[0] == "/api/cli/benchmarks?limit=1&offset=0" + assert mock_request.call_args.kwargs == { + "credentials": None, + "git_identity": "acme/workspace", + } + + +@patch("osmosis_ai.platform.api.client.platform_request") +def test_get_benchmark_encodes_name_and_parses_detail(mock_request: MagicMock) -> None: + mock_request.return_value = { + "benchmark": { + "id": "benchmark-2", + "name": "Terminal-Bench 2.1", + "description": "Terminal benchmark", + "source_type": "osmosis_managed", + "source_ref": "terminal-bench-2-1", + "task_count": 89, + "category_count": 1, + "task_sets": [], + "runner_family": "harbor", + "supports_harness": True, + "requires_harness": True, + "requires_judge_model": False, + "judge_model_default": None, + "pass_threshold": 1, + "categories": [{"name": "terminal", "task_count": 89}], + "tasks": [{"name": "task-1", "category": "terminal"}], + "unavailable_tasks": None, + } + } + + result = OsmosisClient().get_benchmark( + "Terminal-Bench 2.1", + git_identity="acme/workspace", + ) + + assert result.name == "Terminal-Bench 2.1" + assert result.categories[0].name == "terminal" + assert result.tasks == [{"name": "task-1", "category": "terminal"}] + assert mock_request.call_args.args[0] == ( + "/api/cli/benchmarks/Terminal-Bench%202.1" + ) + assert mock_request.call_args.kwargs == { + "credentials": None, + "git_identity": "acme/workspace", + } + + @patch("osmosis_ai.platform.api.client.platform_request") def test_submit_benchmark_run_posts_config_sections(mock_request: MagicMock) -> None: mock_request.return_value = { diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py new file mode 100644 index 00000000..dccf7edc --- /dev/null +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import osmosis_ai.platform.cli.benchmark as benchmark_module +from osmosis_ai.cli.output import DetailResult, ListResult +from osmosis_ai.platform.api.models import ( + BenchmarkCatalogDetail, + BenchmarkCatalogEntry, + BenchmarkCategory, + BenchmarkTaskSet, + PaginatedBenchmarks, +) + +GIT_IDENTITY = "acme/workspace" +REPO_URL = "https://github.com/acme/workspace.git" +FAKE_CREDENTIALS = object() + + +def _context() -> SimpleNamespace: + return SimpleNamespace( + workspace_directory=Path("/repo"), + git_identity=GIT_IDENTITY, + repo_url=REPO_URL, + credentials=FAKE_CREDENTIALS, + ) + + +def test_list_benchmarks_returns_catalog_and_git_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + parity = BenchmarkTaskSet( + name="parity", + task_count=249, + recommended=True, + description="Published comparison sample.", + ) + + class FakeClient: + def list_benchmarks(self, **kwargs: Any) -> PaginatedBenchmarks: + calls.append(kwargs) + return PaginatedBenchmarks( + benchmarks=[ + BenchmarkCatalogEntry( + id="benchmark-1", + name="HLE", + description="Humanity's Last Exam", + source_type="osmosis_managed", + source_ref="hle", + task_count=2_500, + category_count=30, + task_sets=[parity], + ) + ], + total_count=1, + has_more=False, + next_offset=None, + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.list_benchmarks(limit=50, all_=False) + + assert isinstance(result, ListResult) + assert result.total_count == 1 + assert result.items[0]["name"] == "HLE" + assert result.items[0]["task_sets"] == [ + { + "name": "parity", + "task_count": 249, + "recommended": True, + "description": "Published comparison sample.", + } + ] + assert result.display_items[0]["task_sets"] == "parity (249, recommended)" + assert result.extra["git"]["identity"] == GIT_IDENTITY + assert calls == [ + { + "limit": 50, + "offset": 0, + "credentials": FAKE_CREDENTIALS, + "git_identity": GIT_IDENTITY, + } + ] + + +def test_info_exposes_selection_metadata_and_full_task_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + parity = BenchmarkTaskSet( + name="parity", + task_count=249, + recommended=True, + description="Published comparison sample.", + ) + + class FakeClient: + def get_benchmark( + self, + name_or_id: str, + **kwargs: Any, + ) -> BenchmarkCatalogDetail: + calls.append({"name_or_id": name_or_id, **kwargs}) + return BenchmarkCatalogDetail( + id="benchmark-1", + name="HLE", + description="Humanity's Last Exam", + source_type="osmosis_managed", + source_ref="hle", + task_count=2, + category_count=2, + task_sets=[parity], + runner_family="harbor", + supports_harness=True, + requires_harness=False, + requires_judge_model=True, + judge_model_default="openai/gpt-5", + pass_threshold=1, + categories=[ + BenchmarkCategory(name="math", task_count=1), + BenchmarkCategory(name="science", task_count=1), + ], + tasks=[ + {"name": "hle__math", "category": "math"}, + {"name": "hle__science", "category": "science"}, + ], + unavailable_tasks=None, + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.info("HLE") + + assert isinstance(result, DetailResult) + assert result.data["benchmark"]["tasks"] == [ + {"name": "hle__math", "category": "math"}, + {"name": "hle__science", "category": "science"}, + ] + assert result.data["benchmark"]["categories"] == [ + {"name": "math", "task_count": 1}, + {"name": "science", "task_count": 1}, + ] + assert 'task_set = "parity"' in result.display_hints[0] + assert "Omit [tasks]" in result.display_hints[1] + assert calls == [ + { + "name_or_id": "HLE", + "credentials": FAKE_CREDENTIALS, + "git_identity": GIT_IDENTITY, + } + ] From 78344290f3739735c081da049996b514ca379abe Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Thu, 30 Jul 2026 16:25:25 -0700 Subject: [PATCH 08/22] [cli] feat: add benchmark run lifecycle commands --- CONTRIBUTING.md | 8 + docs/README.md | 2 +- docs/cli.md | 5 + docs/run-downloads.md | 26 +- osmosis_ai/cli/commands/benchmark.py | 126 +++++- osmosis_ai/cli/metrics_export.py | 46 +- osmosis_ai/cli/output/__init__.py | 2 + osmosis_ai/cli/output/error.py | 3 + osmosis_ai/cli/output/serializers.py | 24 ++ osmosis_ai/platform/api/client.py | 110 ++++- osmosis_ai/platform/api/models.py | 228 +++++++++- osmosis_ai/platform/cli/benchmark.py | 401 +++++++++++++++++- osmosis_ai/platform/cli/run_download.py | 44 +- osmosis_ai/platform/cli/utils.py | 17 + tests/unit/cli/output/test_error.py | 22 + tests/unit/cli/test_benchmark_commands.py | 74 +++- tests/unit/cli/test_benchmark_download.py | 282 ++++++++++++ tests/unit/cli/test_command_groups.py | 6 + .../platform/api/test_client_benchmark.py | 219 +++++++++- .../platform/cli/test_benchmark_catalog.py | 46 +- .../unit/platform/cli/test_benchmark_runs.py | 336 +++++++++++++++ 21 files changed, 1976 insertions(+), 51 deletions(-) create mode 100644 tests/unit/cli/test_benchmark_download.py create mode 100644 tests/unit/platform/cli/test_benchmark_runs.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74803b26..dddddb5e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,6 +44,14 @@ The table below lists all development commands. If you installed with **pip**, d Coverage configuration is in `pyproject.toml` under `[tool.coverage.*]`. CI enforces a minimum coverage threshold of 70%. +When changing remote run commands, preserve the lifecycle convention: +`train`, `eval`, and `benchmark` use top-level `submit`, `list`, `info`, +`logs`, and `stop` for run management. Benchmark-definition discovery lives +under `osmosis benchmark catalog list|info`. Eval and benchmark downloads +share the manifest transfer engine in `osmosis_ai/platform/cli/run_download.py`; +add domain-specific routes and fixed path classifiers instead of copying the +transfer loop. + ## Linting & Formatting This project uses [Ruff](https://docs.astral.sh/ruff/) for both linting and code formatting. Configuration lives in `pyproject.toml` under `[tool.ruff]`. diff --git a/docs/README.md b/docs/README.md index 8fd28e3b..d3f4fc24 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,7 +31,7 @@ The package (`osmosis_ai/`) is organized into top-level domains. See [architectu - [datasets.md](./datasets.md) — the dataset row contract enforced by the SDK validator. - [troubleshooting.md](./troubleshooting.md) — engineering issues (rollout timeouts, event-loop blocking, concurrency tuning). - [cli.md](./cli.md) — CLI internals for contributors (command shells, lazy imports, JSON envelopes). -- [run-downloads.md](./run-downloads.md) — eval download command, platform route contract, fixed local layout, resume, confirmation, and retry behavior. +- [run-downloads.md](./run-downloads.md) — eval and benchmark download commands, platform route contracts, fixed local layouts, resume, confirmation, and retry behavior. ## See also diff --git a/docs/cli.md b/docs/cli.md index b9b5d43e..c0ee2703 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -19,6 +19,11 @@ Files in [../osmosis_ai/cli/commands/](../osmosis_ai/cli/commands/) are thin Typ Module-level imports in `commands/` are kept light: `typer`, `cli.console`, `cli.errors`, the lightweight `osmosis_ai.platform.constants` (pagination limits), and stdlib. Everything heavy (`rollout.*`, `platform.api.*`, `platform.cli.*`, `eval.*`) must be imported **inside the function** to keep CLI startup fast — see the lazy-loading section of [architecture.md](./architecture.md). +`osmosis benchmark` follows the same run-lifecycle surface as train/eval: +`submit`, `list`, `info`, `logs`, `stop`, and `download`. Benchmark-definition +discovery is a separate nested namespace, `benchmark catalog list|info`, so +top-level `list` and `info` always refer to submitted runs. + ## Commands return results; they don't print The Typer app is created with `result_callback=render_command_result` ([../osmosis_ai/cli/main.py](../osmosis_ai/cli/main.py)). A command function **returns** a `CommandResult`; the callback renders it in the active format. Do not `print()` from a command — return a typed result instead. diff --git a/docs/run-downloads.md b/docs/run-downloads.md index 8b6d1358..ad3c14e1 100644 --- a/docs/run-downloads.md +++ b/docs/run-downloads.md @@ -1,6 +1,6 @@ -# Evaluation run output download contract +# Run output download contracts -The eval download implementation lives in [`platform/cli/run_download.py`](../osmosis_ai/platform/cli/run_download.py). Its Typer shell remains thin and lives in [`cli/commands/eval.py`](../osmosis_ai/cli/commands/eval.py). +The shared manifest-to-disk engine lives in [`platform/cli/run_download.py`](../osmosis_ai/platform/cli/run_download.py). Eval and benchmark handlers provide their route loader, fixed path classifier, output-root resolver, operation name, and resource key; they do not duplicate transfer behavior. ## Commands @@ -11,9 +11,15 @@ osmosis eval download NAME_OR_ID -o, --output ROOT --overwrite -y, --yes + +osmosis benchmark download NAME_OR_ID + --type summary,results|artifacts|logs|all + -o, --output ROOT + --overwrite + -y, --yes ``` -`--type` replaces the default selection and defaults to `metrics,trajectories`. A row selection includes every run for each selected row. +`--type` replaces the default selection. Eval defaults to `metrics,trajectories`; benchmark defaults to `summary,results`. Eval row selection includes every run for each selected row. ## Platform routes @@ -24,6 +30,13 @@ GET /api/cli/eval-runs/[id]/samples/manifest?types=&rows= POST /api/cli/eval-runs/[id]/samples/download-urls ``` +Benchmark downloads use the parallel output routes: + +```text +GET /api/cli/benchmark-runs/[id]/outputs/manifest?types= +POST /api/cli/benchmark-runs/[id]/outputs/download-urls +``` + The manifest returns `{files: [{token?, path, size}], totals}`. `path` is the final path relative to the local run root, and `token` is an opaque server handle (a rollout id or an export snapshot token). URL requests contain at most 500 `{token, path}` items. The platform derives full S3 keys server-side and returns 15-minute presigned GET URLs; the SDK never accepts raw object keys. ## Fixed local layout @@ -36,10 +49,15 @@ The manifest returns `{files: [{token?, path, size}], totals}`. `path` is the fi │ ├── trajectories/row_3_run_0.json │ ├── artifacts/row_3_run_0/logs/agent.log │ └── logs.txt +├── benchmarks// +│ ├── summary.csv +│ ├── results.csv +│ ├── artifacts// +│ └── logs.txt └── metrics/ # legacy eval exports; never deleted ``` -`--output` relocates the run root; filenames and subdirectories below it do not change. Rich-mode `eval info` uses the same resolver and writes the same run-scoped `metrics.json` path. Names that require filesystem sanitization gain a stable run-ID suffix so distinct runs never share a local directory. Training download and training metrics-path migration are intentionally out of scope until the platform routes are ready; `train info` keeps its existing export behavior for now. +`--output` relocates the run root; filenames and subdirectories below it do not change. Rich-mode `eval info` uses the same resolver and writes the same run-scoped `metrics.json` path. Names that require filesystem sanitization gain a stable run-ID suffix so distinct runs never share a local directory. Each domain supplies a strict path classifier: an eval manifest cannot write benchmark filenames and a benchmark manifest cannot write eval filenames. Training download and training metrics-path migration are intentionally out of scope until the platform routes are ready; `train info` keeps its existing export behavior for now. ## Transfer behavior diff --git a/osmosis_ai/cli/commands/benchmark.py b/osmosis_ai/cli/commands/benchmark.py index a2784069..ca21163a 100644 --- a/osmosis_ai/cli/commands/benchmark.py +++ b/osmosis_ai/cli/commands/benchmark.py @@ -1,4 +1,4 @@ -"""Benchmark catalog and run submission commands.""" +"""Benchmark catalog and run management commands.""" from __future__ import annotations @@ -7,16 +7,25 @@ import typer -from osmosis_ai.platform.constants import DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE +from osmosis_ai.platform.constants import ( + DEFAULT_PAGE_SIZE, + MAX_LOG_PAGE_SIZE, + MAX_PAGE_SIZE, +) app: typer.Typer = typer.Typer( - help="Discover benchmarks and submit benchmark runs (list, info, submit).", + help="Manage benchmark runs.", + no_args_is_help=True, +) +catalog_app: typer.Typer = typer.Typer( + help="Discover available benchmarks and task-selection options.", no_args_is_help=True, ) +app.add_typer(catalog_app, name="catalog") -@app.command("list") -def benchmark_list( +@catalog_app.command("list") +def benchmark_catalog_list( limit: int = typer.Option( DEFAULT_PAGE_SIZE, "--limit", @@ -32,12 +41,12 @@ def benchmark_list( return _list_benchmarks(limit=limit, all_=all_) -@app.command("info") -def benchmark_info( +@catalog_app.command("info") +def benchmark_catalog_info( name_or_id: str = typer.Argument(..., help="Benchmark name or ID."), ) -> Any: """Show benchmark metadata and task-selection options.""" - from osmosis_ai.platform.cli.benchmark import info as _info + from osmosis_ai.platform.cli.benchmark import catalog_info as _info return _info(name_or_id) @@ -59,3 +68,104 @@ def benchmark_submit( from osmosis_ai.platform.cli.benchmark import submit as _submit return _submit(config_path, yes=yes) + + +@app.command("list") +def benchmark_list( + limit: int = typer.Option( + DEFAULT_PAGE_SIZE, + "--limit", + min=1, + max=MAX_PAGE_SIZE, + help="Maximum number of benchmark runs to show.", + ), + all_: bool = typer.Option(False, "--all", help="Show all benchmark runs."), +) -> Any: + """List benchmark runs for the current workspace directory.""" + from osmosis_ai.platform.cli.benchmark import list_benchmark_runs as _list + + return _list(limit=limit, all_=all_) + + +@app.command("info") +def benchmark_info( + name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), +) -> Any: + """Show benchmark run details, progress, and results.""" + from osmosis_ai.platform.cli.benchmark import run_info as _info + + return _info(name_or_id) + + +@app.command("logs") +def benchmark_logs( + name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), + limit: int = typer.Option( + DEFAULT_PAGE_SIZE, + "--limit", + min=1, + max=MAX_LOG_PAGE_SIZE, + help="Maximum number of recent log entries to show.", + ), + cursor: str | None = typer.Option( + None, + "--cursor", + help="Page further back using the next_cursor value from a previous page.", + ), +) -> Any: + """Show recent logs for a benchmark run, oldest first.""" + from osmosis_ai.platform.cli.benchmark import logs as _logs + + return _logs(name_or_id, limit=limit, cursor=cursor) + + +@app.command("stop") +def benchmark_stop( + name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt."), +) -> Any: + """Stop a benchmark run.""" + from osmosis_ai.platform.cli.benchmark import stop as _stop + + return _stop(name_or_id, yes=yes) + + +@app.command("download") +def benchmark_download( + name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), + output: str | None = typer.Option( + None, + "--output", + "-o", + help="Run output root (default: .osmosis/benchmarks//).", + ), + types: str = typer.Option( + "summary,results", + "--type", + help=( + "Comma-separated selector: summary, results, artifacts, logs, all. " + "Replaces the default selection." + ), + ), + overwrite: bool = typer.Option( + False, + "--overwrite", + help="Re-download files that already exist locally.", + ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip size confirmation.", + ), +) -> Any: + """Download benchmark summary, results, artifacts, or logs.""" + from osmosis_ai.platform.cli.benchmark import download as _download + + return _download( + name_or_id, + output=output, + types=types, + overwrite=overwrite, + yes=yes, + ) diff --git a/osmosis_ai/cli/metrics_export.py b/osmosis_ai/cli/metrics_export.py index 23cdbf9c..739cc923 100644 --- a/osmosis_ai/cli/metrics_export.py +++ b/osmosis_ai/cli/metrics_export.py @@ -83,18 +83,18 @@ def resolve_default_metrics_output( return metrics_dir / default_metrics_filename(run_name, run_id) -def resolve_eval_output_dir( +def _resolve_run_output_dir( run_name: str | None, run_id: str, *, + scope: str, workspace_directory: Path, output: str | None = None, create: bool = True, ) -> Path: - """Resolve the fixed eval-scoped output root used by ``info`` and download. + """Resolve a fixed run-scoped output root under ``.osmosis/``. ``output`` relocates the run root itself; the layout below it is fixed. - Without an explicit root, evals live under ``.osmosis/evals/``. Unnamed runs fall back to their full ID so separate runs never share the unnamed directory. """ @@ -115,7 +115,7 @@ def resolve_eval_output_dir( safe_run_id = safe_name(run_id).strip("_") or "run" suffix = f"--{safe_run_id}" safe_run_name = f"{safe_run_name[: 255 - len(suffix)]}{suffix}" - root = workspace_directory / ".osmosis" / "evals" / (safe_run_name or run_id) + root = workspace_directory / ".osmosis" / scope / (safe_run_name or run_id) try: if root.exists() and not root.is_dir(): @@ -127,6 +127,44 @@ def resolve_eval_output_dir( return root +def resolve_eval_output_dir( + run_name: str | None, + run_id: str, + *, + workspace_directory: Path, + output: str | None = None, + create: bool = True, +) -> Path: + """Resolve the fixed eval-scoped output root used by ``info`` and download.""" + return _resolve_run_output_dir( + run_name, + run_id, + scope="evals", + workspace_directory=workspace_directory, + output=output, + create=create, + ) + + +def resolve_benchmark_output_dir( + run_name: str | None, + run_id: str, + *, + workspace_directory: Path, + output: str | None = None, + create: bool = True, +) -> Path: + """Resolve the fixed benchmark-scoped output root used by download.""" + return _resolve_run_output_dir( + run_name, + run_id, + scope="benchmarks", + workspace_directory=workspace_directory, + output=output, + create=create, + ) + + def resolve_eval_metrics_output( run_name: str | None, run_id: str, diff --git a/osmosis_ai/cli/output/__init__.py b/osmosis_ai/cli/output/__init__.py index 718f2d42..1a7310cd 100644 --- a/osmosis_ai/cli/output/__init__.py +++ b/osmosis_ai/cli/output/__init__.py @@ -29,6 +29,7 @@ detail_fields, ) from .serializers import ( + serialize_benchmark_run, serialize_checkpoint, serialize_dataset, serialize_dev_rollout_server, @@ -64,6 +65,7 @@ "render", "render_command_result", "resolve_format_selectors", + "serialize_benchmark_run", "serialize_checkpoint", "serialize_dataset", "serialize_dev_rollout_server", diff --git a/osmosis_ai/cli/output/error.py b/osmosis_ai/cli/output/error.py index 90ec1c7e..b5f817ca 100644 --- a/osmosis_ai/cli/output/error.py +++ b/osmosis_ai/cli/output/error.py @@ -17,6 +17,7 @@ _SUPPORTED_COMMAND_GROUPS = { "auth", + "benchmark", "dataset", "eval", "model", @@ -130,6 +131,8 @@ def _argv_command_path(argv: list[str]) -> str: return command if (command, tokens[1]) in _REMOVED_TWO_TOKEN_COMMANDS: return " ".join(tokens[:2]) + if command == "benchmark" and tokens[1] == "catalog" and len(tokens) >= 3: + return " ".join(tokens[:3]) if command == "eval" and tokens[1] == "cache" and len(tokens) >= 3: return " ".join(tokens[:3]) if command in _SUPPORTED_COMMAND_GROUPS: diff --git a/osmosis_ai/cli/output/serializers.py b/osmosis_ai/cli/output/serializers.py index a72e3245..829088b2 100644 --- a/osmosis_ai/cli/output/serializers.py +++ b/osmosis_ai/cli/output/serializers.py @@ -6,6 +6,7 @@ from osmosis_ai.platform.api.models import ( BaseModelInfo, + BenchmarkRun, DatasetFile, DevRolloutServerInfo, EnvironmentSecretInfo, @@ -18,6 +19,29 @@ ) +def serialize_benchmark_run(run: BenchmarkRun) -> dict[str, Any]: + """Serialize a benchmark run for the public JSON contract.""" + data: dict[str, Any] = { + "id": run.id, + "name": run.name, + "status": run.status, + "benchmark_id": run.benchmark_id, + "benchmark_name": run.benchmark_name, + "agent_count": run.agent_count, + "best_pass_at_1": run.best_pass_at_1, + "ingested_results": run.ingested_results, + "expected_results": run.expected_results, + "creator_name": run.creator_name, + "creator_email": run.creator_email, + "created_at": run.created_at, + "started_at": run.started_at, + "completed_at": run.completed_at, + } + if run.platform_url: + data["platform_url"] = run.platform_url + return data + + def serialize_dataset(df: DatasetFile) -> dict[str, Any]: """Serialize a dataset for the public JSON contract.""" data = { diff --git a/osmosis_ai/platform/api/client.py b/osmosis_ai/platform/api/client.py index b1ab7aad..11e85a86 100644 --- a/osmosis_ai/platform/api/client.py +++ b/osmosis_ai/platform/api/client.py @@ -11,6 +11,7 @@ from .models import ( BenchmarkCatalogDetail, + BenchmarkRunDetail, DatasetDownloadInfo, DatasetFile, EnvironmentSecretInfo, @@ -21,6 +22,7 @@ LoraModelDetail, LoraModelSummary, PaginatedBaseModels, + PaginatedBenchmarkRuns, PaginatedBenchmarks, PaginatedDatasets, PaginatedDevRolloutServers, @@ -86,6 +88,7 @@ def _get_run_download_manifest( *, types: Sequence[str], rows: str | None = None, + route: str = "samples", credentials: Credentials | None = None, git_identity: str, ) -> RunDownloadManifest: @@ -93,7 +96,7 @@ def _get_run_download_manifest( if rows is not None: params["rows"] = rows data = platform_request( - f"{resource_path}/samples/manifest?{urlencode(params)}", + f"{resource_path}/{route}/manifest?{urlencode(params)}", credentials=credentials, git_identity=git_identity, ) @@ -104,6 +107,7 @@ def _get_run_download_urls( resource_path: str, *, items: Sequence[RunDownloadFile], + route: str = "samples", credentials: Credentials | None = None, git_identity: str, ) -> RunDownloadURLBatch: @@ -112,7 +116,7 @@ def _get_run_download_urls( "Download URL batches must contain between 1 and 500 items" ) data = platform_request( - f"{resource_path}/samples/download-urls", + f"{resource_path}/{route}/download-urls", method="POST", data={"items": [item.to_request_item() for item in items]}, credentials=credentials, @@ -701,6 +705,108 @@ def get_benchmark( ) return BenchmarkCatalogDetail.from_dict(data) + def list_benchmark_runs( + self, + limit: int = DEFAULT_PAGE_SIZE, + offset: int = 0, + *, + credentials: Credentials | None = None, + git_identity: str, + ) -> PaginatedBenchmarkRuns: + """List benchmark runs in the current workspace.""" + qs = urlencode({"limit": limit, "offset": offset}) + data = platform_request( + f"/api/cli/benchmark-runs?{qs}", + credentials=credentials, + git_identity=git_identity, + ) + return PaginatedBenchmarkRuns.from_dict(data) + + def get_benchmark_run( + self, + name_or_id: str, + *, + credentials: Credentials | None = None, + git_identity: str, + ) -> BenchmarkRunDetail: + """Get benchmark run details by name or ID.""" + data = platform_request( + f"/api/cli/benchmark-runs/{_safe_path(name_or_id)}", + credentials=credentials, + git_identity=git_identity, + ) + return BenchmarkRunDetail.from_dict(data) + + def get_benchmark_run_logs( + self, + name_or_id: str, + *, + limit: int = DEFAULT_PAGE_SIZE, + cursor: str | None = None, + direction: str = "older", + credentials: Credentials | None = None, + git_identity: str, + ) -> LogsPage: + """Fetch one cursor page of benchmark run logs.""" + return self._get_logs( + f"/api/cli/benchmark-runs/{_safe_path(name_or_id)}", + limit=limit, + cursor=cursor, + direction=direction, + credentials=credentials, + git_identity=git_identity, + ) + + def get_benchmark_run_download_manifest( + self, + benchmark_run_id: str, + *, + types: Sequence[str], + credentials: Credentials | None = None, + git_identity: str, + ) -> RunDownloadManifest: + """Get the fixed-layout download manifest for a benchmark run.""" + return self._get_run_download_manifest( + f"/api/cli/benchmark-runs/{_safe_path(benchmark_run_id)}", + types=types, + route="outputs", + credentials=credentials, + git_identity=git_identity, + ) + + def get_benchmark_run_download_urls( + self, + benchmark_run_id: str, + *, + items: Sequence[RunDownloadFile], + credentials: Credentials | None = None, + git_identity: str, + ) -> RunDownloadURLBatch: + """Exchange benchmark manifest items for bounded presigned URLs.""" + return self._get_run_download_urls( + f"/api/cli/benchmark-runs/{_safe_path(benchmark_run_id)}", + items=items, + route="outputs", + credentials=credentials, + git_identity=git_identity, + ) + + def stop_benchmark_run( + self, + name_or_id: str, + *, + credentials: Credentials | None = None, + git_identity: str, + ) -> dict[str, Any]: + """Stop a non-terminal benchmark run.""" + return platform_request( + f"/api/cli/benchmark-runs/{_safe_path(name_or_id)}/stop", + method="POST", + data={}, + credentials=credentials, + git_identity=git_identity, + ) + def list_eval_runs( self, limit: int = DEFAULT_PAGE_SIZE, diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index bb06a14a..920f8f07 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -3,8 +3,8 @@ from __future__ import annotations import math -from dataclasses import dataclass -from typing import Any, Literal +from dataclasses import dataclass, field +from typing import Any, Literal, TypedDict # ── Dataset status constants ───────────────────────────────────── # Single source of truth for status classification. @@ -178,6 +178,19 @@ def from_dict(cls, data: dict[str, Any]) -> PaginatedDatasets: EVAL_RUN_STATUSES_SUCCESS | EVAL_RUN_STATUSES_ERROR | EVAL_RUN_STATUSES_STOPPED ) +# ── Benchmark run status constants ─────────────────────────────── + +BENCHMARK_RUN_STATUSES_SUCCESS: frozenset[str] = frozenset({"finished"}) +BENCHMARK_RUN_STATUSES_PENDING: frozenset[str] = frozenset({"pending", "queued"}) +BENCHMARK_RUN_STATUSES_IN_PROGRESS: frozenset[str] = frozenset({"running"}) +BENCHMARK_RUN_STATUSES_ERROR: frozenset[str] = frozenset({"failed"}) +BENCHMARK_RUN_STATUSES_STOPPED: frozenset[str] = frozenset({"stopped"}) +BENCHMARK_RUN_STATUSES_TERMINAL: frozenset[str] = ( + BENCHMARK_RUN_STATUSES_SUCCESS + | BENCHMARK_RUN_STATUSES_ERROR + | BENCHMARK_RUN_STATUSES_STOPPED +) + def _number_or_none(value: Any) -> int | float | None: if isinstance(value, bool): @@ -383,6 +396,44 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCategory: return cls(name=data["name"], task_count=data["task_count"]) +BenchmarkTaskDifficulty = Literal["easy", "medium", "hard"] + + +class BenchmarkCatalogTask(TypedDict): + """One task exposed by the benchmark catalog.""" + + name: str + category: str | None + difficulty: BenchmarkTaskDifficulty | None + + +def _parse_benchmark_catalog_task(data: dict[str, Any]) -> BenchmarkCatalogTask: + difficulty = data.get("difficulty") + if difficulty not in ("easy", "medium", "hard"): + difficulty = None + return { + "name": data["name"], + "category": data.get("category"), + "difficulty": difficulty, + } + + +def _parse_unavailable_benchmark_tasks(data: Any) -> dict[str, Any] | None: + if not isinstance(data, dict): + return None + raw_tasks = data.get("tasks", []) + tasks = ( + [ + _parse_benchmark_catalog_task(task) + for task in raw_tasks + if isinstance(task, dict) + ] + if isinstance(raw_tasks, list) + else [] + ) + return {**data, "tasks": tasks} + + @dataclass class BenchmarkCatalogEntry: """Benchmark available in the current workspace catalog.""" @@ -453,8 +504,9 @@ class BenchmarkCatalogDetail: judge_model_default: str | None pass_threshold: float categories: list[BenchmarkCategory] - tasks: list[dict[str, Any]] + tasks: list[BenchmarkCatalogTask] unavailable_tasks: dict[str, Any] | None + required_secret_names: list[str] = field(default_factory=list) @classmethod def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: @@ -481,8 +533,14 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: BenchmarkCategory.from_dict(item) for item in benchmark.get("categories", []) ], - tasks=list(benchmark.get("tasks", [])), - unavailable_tasks=benchmark.get("unavailable_tasks"), + tasks=[ + _parse_benchmark_catalog_task(item) + for item in benchmark.get("tasks", []) + ], + unavailable_tasks=_parse_unavailable_benchmark_tasks( + benchmark.get("unavailable_tasks") + ), + required_secret_names=list(benchmark.get("required_secret_names", [])), ) @@ -511,6 +569,166 @@ def from_dict(cls, data: dict[str, Any]) -> SubmitBenchmarkRunResult: ) +@dataclass +class BenchmarkRun: + """A benchmark run in the current workspace.""" + + id: str + name: str + status: str + benchmark_name: str + created_at: str + benchmark_id: str | None = None + started_at: str | None = None + completed_at: str | None = None + creator_name: str | None = None + creator_email: str | None = None + platform_url: str | None = None + agent_count: int = 0 + best_pass_at_1: float | None = None + ingested_results: int = 0 + expected_results: int = 0 + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchmarkRun: + benchmark = data.get("benchmark") + if not isinstance(benchmark, dict): + benchmark = {} + best_pass_at_1 = _number_or_none(data.get("best_pass_at_1")) + return cls( + id=data["id"], + name=data.get("name") or data.get("benchmark_run_name", ""), + status=data.get("status", ""), + benchmark_name=data.get("benchmark_name") or benchmark.get("name", ""), + benchmark_id=data.get("benchmark_id") or benchmark.get("id"), + created_at=data.get("created_at", ""), + started_at=data.get("started_at"), + completed_at=data.get("completed_at"), + creator_name=data.get("creator_name"), + creator_email=data.get("creator_email"), + platform_url=data.get("platform_url"), + agent_count=int(data.get("agent_count") or 0), + best_pass_at_1=( + float(best_pass_at_1) if best_pass_at_1 is not None else None + ), + ingested_results=int(data.get("ingested_results") or 0), + expected_results=int(data.get("expected_results") or 0), + ) + + +@dataclass +class BenchmarkRunDetail(BenchmarkRun): + """Detailed benchmark run with configuration, agents, and result totals.""" + + configuration: dict[str, Any] | None = None + agents: list[dict[str, Any]] | None = None + progress: dict[str, Any] | None = None + totals: dict[str, Any] | None = None + agent_metrics: list[dict[str, Any]] | None = None + is_internal_user: bool = False + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchmarkRunDetail: + run = data["benchmark_run"] + configuration = data.get("configuration") + if not isinstance(configuration, dict): + configuration = None + progress = data.get("progress") + if not isinstance(progress, dict): + progress = None + totals = data.get("totals") + if not isinstance(totals, dict): + totals = None + raw_agents = data.get("agents") + agents = ( + [item for item in raw_agents if isinstance(item, dict)] + if isinstance(raw_agents, list) + else [] + ) + raw_agent_metrics = data.get("agent_metrics") + agent_metrics = ( + [item for item in raw_agent_metrics if isinstance(item, dict)] + if isinstance(raw_agent_metrics, list) + else [] + ) + benchmark_name = run.get("benchmark_name") + if not isinstance(benchmark_name, str): + benchmark_name = ( + configuration.get("benchmark_name", "") if configuration else "" + ) + benchmark_id = run.get("benchmark_id") + if not isinstance(benchmark_id, str): + configured_id = configuration.get("benchmark_id") if configuration else None + benchmark_id = configured_id if isinstance(configured_id, str) else None + merged_run = { + **run, + "benchmark_id": benchmark_id, + "benchmark_name": benchmark_name, + } + base = BenchmarkRun.from_dict(merged_run) + ingested_results = base.ingested_results + expected_results = base.expected_results + if progress is not None: + ingested_results = int(progress.get("ingested") or 0) + expected_results = int(progress.get("expected") or 0) + pass_at_1_values = [] + for metrics in agent_metrics: + interval = metrics.get("pass_at_1") + if not isinstance(interval, dict): + continue + value = _number_or_none(interval.get("value")) + if value is not None: + pass_at_1_values.append(float(value)) + return cls( + id=base.id, + name=base.name, + status=base.status, + benchmark_name=base.benchmark_name, + benchmark_id=base.benchmark_id, + created_at=base.created_at, + started_at=base.started_at, + completed_at=base.completed_at, + creator_name=base.creator_name, + creator_email=base.creator_email, + platform_url=base.platform_url, + agent_count=base.agent_count or len(agents), + best_pass_at_1=( + base.best_pass_at_1 + if base.best_pass_at_1 is not None + else max(pass_at_1_values, default=None) + ), + ingested_results=ingested_results, + expected_results=expected_results, + configuration=configuration, + agents=agents, + progress=progress, + totals=totals, + agent_metrics=agent_metrics, + is_internal_user=data.get("is_internal_user", False), + ) + + +@dataclass +class PaginatedBenchmarkRuns: + """Paginated benchmark runs for a workspace.""" + + benchmark_runs: list[BenchmarkRun] + total_count: int + has_more: bool + next_offset: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> PaginatedBenchmarkRuns: + return cls( + benchmark_runs=[ + BenchmarkRun.from_dict(item) for item in data.get("benchmark_runs", []) + ], + total_count=data.get("total_count", 0), + has_more=data.get("has_more", False), + next_offset=data.get("next_offset"), + ) + + # ── Training run metrics ───────────────────────────────────────── diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 19454b44..fefe3f75 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -1,4 +1,4 @@ -"""Handlers for benchmark catalog discovery and run submission.""" +"""Handlers for benchmark catalog discovery and run management.""" from __future__ import annotations @@ -9,17 +9,25 @@ from osmosis_ai.cli.errors import CLIError from osmosis_ai.cli.output import ( DetailResult, + DetailSection, ListColumn, ListResult, OperationResult, detail_fields, get_output_context, + serialize_benchmark_run, ) +from osmosis_ai.cli.output.display import format_local_date, format_local_datetime from osmosis_ai.cli.prompts import require_confirmation from osmosis_ai.platform.api.client import OsmosisClient from osmosis_ai.platform.api.models import ( + BENCHMARK_RUN_STATUSES_ERROR, + BENCHMARK_RUN_STATUSES_PENDING, + BENCHMARK_RUN_STATUSES_TERMINAL, BenchmarkCatalogDetail, BenchmarkCatalogEntry, + BenchmarkRun, + BenchmarkRunDetail, BenchmarkTaskSet, SubmitBenchmarkRunResult, ) @@ -38,6 +46,14 @@ _missing_secret_message, ) from osmosis_ai.platform.cli.utils import ( + build_logs_result, + format_benchmark_status, + format_env_config, + format_progress, + format_secret_scopes, + jsonish, + kv_section, + make_progress, paginated_fetch, require_git_workspace_directory_context, validate_list_options, @@ -140,12 +156,13 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: for benchmark in benchmarks ], display_hints=[ - "Use osmosis benchmark info for task sets, categories, and tasks." + "Use osmosis benchmark catalog info for task sets, " + "categories, and tasks." ], ) -def info(name_or_id: str) -> DetailResult: +def catalog_info(name_or_id: str) -> DetailResult: """Show benchmark metadata and task-selection options.""" context = require_git_workspace_directory_context() client = OsmosisClient() @@ -185,6 +202,10 @@ def info(name_or_id: str) -> DetailResult: ("Named Task Sets", _task_set_display(benchmark.task_sets)), ("Harness", harness), ("LLM Judge", judge), + ( + "Required Secret Records", + ", ".join(benchmark.required_secret_names) or "–", + ), ("Pass Threshold", f"{benchmark.pass_threshold:g}"), ] @@ -195,6 +216,7 @@ def info(name_or_id: str) -> DetailResult: "requires_harness": benchmark.requires_harness, "requires_judge_model": benchmark.requires_judge_model, "judge_model_default": benchmark.judge_model_default, + "required_secret_names": benchmark.required_secret_names, "pass_threshold": benchmark.pass_threshold, "categories": [ {"name": category.name, "task_count": category.task_count} @@ -207,7 +229,8 @@ def info(name_or_id: str) -> DetailResult: display_hints = [ f"Omit [tasks] to select all {benchmark.task_count:,} tasks.", "Use task_names or categories under [tasks] for a custom subset.", - "Use osmosis --json benchmark info to inspect the full task list.", + "Use osmosis --json benchmark catalog info to inspect the full " + "task list.", ] for task_set in benchmark.task_sets: if task_set.recommended: @@ -225,6 +248,359 @@ def info(name_or_id: str) -> DetailResult: ) +def _benchmark_progress(run: BenchmarkRun) -> dict[str, Any] | None: + return make_progress(run.ingested_results, run.expected_results, "results") + + +def _format_pass_at_1(value: float | None) -> str: + return "–" if value is None else f"{value:.1%}" + + +def list_benchmark_runs(*, limit: int, all_: bool) -> ListResult: + """List benchmark runs for the current workspace directory.""" + effective_limit, fetch_all = validate_list_options(limit=limit, all_=all_) + context = require_git_workspace_directory_context() + client = OsmosisClient() + output = get_output_context() + with output.status("Fetching benchmark runs..."): + runs, total_count, has_more, next_offset = paginated_fetch( + lambda lim, off: client.list_benchmark_runs( + limit=lim, + offset=off, + credentials=context.credentials, + git_identity=context.git_identity, + ), + items_attr="benchmark_runs", + limit=effective_limit, + fetch_all=fetch_all, + ) + + return ListResult( + title="Benchmark Runs", + items=[ + { + **serialize_benchmark_run(run), + "progress": _benchmark_progress(run), + } + for run in runs + ], + total_count=total_count, + has_more=has_more, + next_offset=next_offset, + extra=git_result_context(context), + columns=[ + ListColumn(key="name", label="Name", ratio=3, overflow="fold"), + ListColumn(key="status", label="Status", no_wrap=True, ratio=1), + ListColumn(key="benchmark", label="Benchmark", ratio=2, overflow="fold"), + ListColumn(key="progress", label="Progress", no_wrap=True, ratio=2), + ListColumn(key="best_pass_at_1", label="Best Pass@1", no_wrap=True), + ListColumn(key="created_at", label="Submitted", no_wrap=True, ratio=1), + ListColumn(key="creator_name", label="Submitted By", no_wrap=True), + ], + display_items=[ + { + **serialize_benchmark_run(run), + "status": format_benchmark_status(run), + "benchmark": run.benchmark_name or "–", + "progress": format_progress(_benchmark_progress(run)) or "–", + "best_pass_at_1": _format_pass_at_1(run.best_pass_at_1), + "created_at": format_local_date(run.created_at), + "creator_name": run.creator_name or "–", + } + for run in runs + ], + display_hints=["Use osmosis benchmark info for details."], + ) + + +def _configuration_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: + configuration = detail.configuration or {} + rows: list[tuple[str, str]] = [] + source_type = configuration.get("source_type") + source_ref = configuration.get("source_ref") + if source_type or source_ref: + rows.append( + ( + "Source", + ": ".join(str(value) for value in (source_type, source_ref) if value), + ) + ) + version = configuration.get("resolved_version") + if version: + rows.append(("Version", str(version))) + digest = configuration.get("resolved_digest") + if digest: + rows.append(("Digest", str(digest))) + task_filters = configuration.get("task_filters") + if task_filters: + rows.append(("Tasks", jsonish(task_filters))) + config = configuration.get("config") + if config: + rows.append(("Settings", jsonish(config))) + scopes = format_secret_scopes(configuration.get("resolved_secret_scopes")) + if scopes: + rows.append(("Secrets", scopes)) + env = format_env_config(configuration.get("env_config")) + if env: + rows.append(("Environment Variables", env)) + return rows + + +def _agent_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: + rows: list[tuple[str, str]] = [] + for position, agent in enumerate(detail.agents or [], start=1): + index = agent.get("agent_index") + label = f"Agent {index + 1 if isinstance(index, int) else position}" + model = ( + agent.get("model_display_name") + or agent.get("model") + or agent.get("model_ref") + or "–" + ) + if not isinstance(model, str): + model = jsonish(model) + harness = agent.get("harness") or "default" + status = agent.get("status") + value = f"{harness} · {model}" + if status: + value += f" · {status}" + rows.append((label, value)) + return rows + + +def _result_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: + totals = detail.totals or {} + rows: list[tuple[str, str]] = [] + outcome_parts = [ + f"{int(totals[key]):,} {key}" + for key in ("passed", "failed", "errored", "cancelled") + if isinstance(totals.get(key), int | float) + ] + if outcome_parts: + rows.append(("Outcomes", ", ".join(outcome_parts))) + for key, label in ( + ("total_input_tokens", "Input Tokens"), + ("total_output_tokens", "Output Tokens"), + ("total_cost_usd", "Reported Cost"), + ): + value = totals.get(key) + if not isinstance(value, int | float) or isinstance(value, bool): + continue + rows.append( + ( + label, + f"${value:,.4f}" if key == "total_cost_usd" else f"{int(value):,}", + ) + ) + return rows + + +def run_info(name_or_id: str) -> DetailResult: + """Show benchmark run details, progress, configuration, and results.""" + context = require_git_workspace_directory_context() + client = OsmosisClient() + output = get_output_context() + with output.status("Fetching benchmark run..."): + detail = client.get_benchmark_run( + name_or_id, + credentials=context.credentials, + git_identity=context.git_identity, + ) + + rows: list[tuple[str, str]] = [ + ("Name", console.escape(detail.name)), + ("ID", detail.id), + ] + rows.extend( + [ + ("Status", detail.status.replace("_", " ").title()), + ("Benchmark", console.escape(detail.benchmark_name or "–")), + ] + ) + progress = detail.progress + if isinstance(progress, dict): + progress = make_progress( + progress.get("ingested"), + progress.get("expected"), + "results", + ) + else: + progress = _benchmark_progress(detail) + progress_label = format_progress(progress) + if progress_label: + rows.append(("Progress", progress_label)) + if detail.best_pass_at_1 is not None: + rows.append(("Best Pass@1", _format_pass_at_1(detail.best_pass_at_1))) + rows.append(("Agents", f"{detail.agent_count:,}")) + if detail.created_at: + rows.append(("Submitted", format_local_datetime(detail.created_at))) + if detail.creator_name: + rows.append(("Submitted By", console.escape(detail.creator_name))) + if detail.started_at: + rows.append(("Started", format_local_datetime(detail.started_at))) + if detail.completed_at: + rows.append(("Completed", format_local_datetime(detail.completed_at))) + + sections: list[DetailSection] = [] + for section in ( + kv_section("Configuration", _configuration_rows(detail)), + kv_section("Agents", _agent_rows(detail)), + kv_section("Results", _result_rows(detail)), + ): + if section is not None: + sections.append(section) + + display_hints: list[str] = [] + if detail.platform_url: + display_hints.append(f"View: {detail.platform_url}") + if detail.status in BENCHMARK_RUN_STATUSES_ERROR: + display_hints.append(f"See logs with: osmosis benchmark logs {detail.name}") + if detail.status not in BENCHMARK_RUN_STATUSES_TERMINAL: + display_hints.append(f"Stop with: osmosis benchmark stop {detail.name}") + display_hints.append( + f"Download outputs with: osmosis benchmark download {detail.name}" + ) + + return DetailResult( + title="Benchmark Run", + data={ + "benchmark_run": serialize_benchmark_run(detail), + "configuration": detail.configuration, + "agents": detail.agents, + "progress": progress, + "totals": detail.totals, + "agent_metrics": detail.agent_metrics, + **git_result_context(context), + }, + fields=detail_fields(rows), + sections=sections, + display_hints=display_hints, + ) + + +def logs(name_or_id: str, *, limit: int, cursor: str | None = None) -> ListResult: + """Show the most recent logs for a benchmark run, oldest-first.""" + context = require_git_workspace_directory_context() + client = OsmosisClient() + output = get_output_context() + with output.status("Fetching logs..."): + page = client.get_benchmark_run_logs( + name_or_id, + limit=limit, + cursor=cursor, + credentials=context.credentials, + git_identity=context.git_identity, + ) + return build_logs_result( + title=f"Benchmark Run Logs: {name_or_id}", + page=page, + context=context, + next_step_hint=f"Use osmosis benchmark info {name_or_id} for run details.", + ) + + +def download( + name_or_id: str, + *, + output: str | None, + types: str = "summary,results", + overwrite: bool = False, + yes: bool = False, +) -> OperationResult: + """Download selected benchmark run outputs through the manifest contract.""" + from osmosis_ai.cli.metrics_export import resolve_benchmark_output_dir + from osmosis_ai.platform.cli.run_download import ( + BENCHMARK_DOWNLOAD_TYPES, + benchmark_path_category, + parse_download_types, + run_download, + ) + + selected_types = parse_download_types(types, allowed=BENCHMARK_DOWNLOAD_TYPES) + context = require_git_workspace_directory_context() + client = OsmosisClient() + output_ctx = get_output_context() + with output_ctx.status("Fetching benchmark run..."): + detail = client.get_benchmark_run( + name_or_id, + credentials=context.credentials, + git_identity=context.git_identity, + ) + if detail.status in BENCHMARK_RUN_STATUSES_PENDING: + raise CLIError( + "Outputs are not yet available for pending or queued benchmark runs.", + code="CONFLICT", + ) + try: + return run_download( + run_id=detail.id, + run_name=detail.name, + run_status=detail.status, + selected_types=selected_types, + output=output, + overwrite=overwrite, + yes=yes, + workspace_directory=context.workspace_directory, + result_context=git_result_context(context), + manifest_loader=lambda requested_types: ( + client.get_benchmark_run_download_manifest( + detail.id, + types=requested_types, + credentials=context.credentials, + git_identity=context.git_identity, + ) + ), + url_loader=lambda items: client.get_benchmark_run_download_urls( + detail.id, + items=items, + credentials=context.credentials, + git_identity=context.git_identity, + ), + output_resolver=resolve_benchmark_output_dir, + path_category=benchmark_path_category, + operation="benchmark.download", + resource_key="benchmark_run", + ) + except PlatformAPIError as exc: + if exc.status_code == 404: + raise CLIError( + "Benchmark run output route was not found. The run may have been " + "deleted or the platform may not support benchmark downloads yet." + ) from exc + raise + + +def stop(name_or_id: str, *, yes: bool) -> OperationResult: + """Stop a benchmark run.""" + context = require_git_workspace_directory_context() + client = OsmosisClient() + output = get_output_context() + with output.status("Fetching benchmark run..."): + detail = client.get_benchmark_run( + name_or_id, + credentials=context.credentials, + git_identity=context.git_identity, + ) + require_confirmation( + f'Stop benchmark run "{detail.name}"?', + yes=yes, + default=False, + summary=[("Name", detail.name), ("ID", detail.id)], + ) + with output.status("Stopping benchmark run..."): + client.stop_benchmark_run( + detail.id, + credentials=context.credentials, + git_identity=context.git_identity, + ) + return OperationResult( + operation="benchmark.stop", + status="success", + resource={"id": detail.id, "name": detail.name, "status": "stopped"}, + message=f'Benchmark run "{detail.name}" stopped.', + ) + + def _agent_model_label(agent: dict[str, Any]) -> str: model = agent["model"] if model["type"] == "hosted": @@ -399,8 +775,12 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: display_next_steps = [ f"Status: {result.status}", f"Benchmark: {config.experiment.benchmark}", + f"Check status with: osmosis benchmark info {result.name}", + ] + structured_next_steps: list[dict[str, Any]] = [ + {"action": "benchmark_info", "name": result.name}, + {"action": "benchmark_list"}, ] - structured_next_steps: list[dict[str, Any]] = [] if result.platform_url: display_next_steps.append(f"View: {result.platform_url}") structured_next_steps.append({"action": "open_url", "url": result.platform_url}) @@ -431,4 +811,13 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: ) -__all__ = ["info", "list_benchmarks", "submit"] +__all__ = [ + "catalog_info", + "download", + "list_benchmark_runs", + "list_benchmarks", + "logs", + "run_info", + "stop", + "submit", +] diff --git a/osmosis_ai/platform/cli/run_download.py b/osmosis_ai/platform/cli/run_download.py index 01d85717..1cd03316 100644 --- a/osmosis_ai/platform/cli/run_download.py +++ b/osmosis_ai/platform/cli/run_download.py @@ -1,4 +1,4 @@ -"""Manifest-to-disk download engine for evaluation runs.""" +"""Shared manifest-to-disk download engine for remote runs.""" from __future__ import annotations @@ -35,9 +35,12 @@ DOWNLOAD_RETRY_BASE_SECONDS = 0.5 EVAL_DOWNLOAD_TYPES = ("metrics", "trajectories", "artifacts", "logs") +BENCHMARK_DOWNLOAD_TYPES = ("summary", "results", "artifacts", "logs") ManifestLoader = Callable[[Sequence[str]], RunDownloadManifest] URLLoader = Callable[[Sequence[RunDownloadFile]], RunDownloadURLBatch] +OutputResolver = Callable[..., Path] +PathCategory = Callable[[str], str | None] _ROWS_RE = re.compile(r"\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*") _URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) @@ -110,10 +113,25 @@ def _path_category(path: str) -> str | None: return None +def benchmark_path_category(path: str) -> str | None: + """Map a fixed benchmark download path to its public selector.""" + if path == "summary.csv": + return "summary" + if path == "results.csv": + return "results" + if path == "logs.txt": + return "logs" + parts = path.split("/") + if len(parts) >= 3 and parts[0] == "artifacts": + return "artifacts" + return None + + def _safe_relative_path( path: str, *, selected_types: set[str], + path_category: PathCategory = _path_category, ) -> tuple[Path, str] | None: if not path or path.startswith("/") or "\\" in path: return None @@ -122,7 +140,7 @@ def _safe_relative_path( return None if _RESERVED_ARTIFACT_MANIFEST_RE.fullmatch(path) is not None: return None - category = _path_category(path) + category = path_category(path) if category is None or category not in selected_types: return None return Path(*parts), category @@ -133,6 +151,7 @@ def _prepare_files( *, output_dir: Path, selected_types: Sequence[str], + path_category: PathCategory = _path_category, ) -> tuple[list[_PreparedFile], list[dict[str, str]]]: prepared: list[_PreparedFile] = [] rejected: list[dict[str, str]] = [] @@ -140,7 +159,11 @@ def _prepare_files( selected = set(selected_types) resolved_root = output_dir.resolve(strict=False) for item in manifest.files: - safe = _safe_relative_path(item.path, selected_types=selected) + safe = _safe_relative_path( + item.path, + selected_types=selected, + path_category=path_category, + ) if safe is None: rejected.append( { @@ -312,9 +335,13 @@ def run_download( manifest_loader: ManifestLoader, url_loader: URLLoader, selection: dict[str, Any] | None = None, + output_resolver: OutputResolver = resolve_eval_output_dir, + path_category: PathCategory = _path_category, + operation: str = "eval.download", + resource_key: str = "eval_run", ) -> OperationResult: - """Plan, confirm, and execute one eval download using the route contract.""" - output_dir = resolve_eval_output_dir( + """Plan, confirm, and execute a run download using the manifest contract.""" + output_dir = output_resolver( run_name, run_id, workspace_directory=workspace_directory, @@ -329,6 +356,7 @@ def run_download( manifest, output_dir=output_dir, selected_types=selected_types, + path_category=path_category, ) if not prepared: if rejected: @@ -376,7 +404,7 @@ def run_download( notes=[f"Selected types: {', '.join(selected_types)}"], ) - resolve_eval_output_dir( + output_resolver( run_name, run_id, workspace_directory=workspace_directory, @@ -468,7 +496,7 @@ def run_download( message += f" ({len(failures):,} failed; re-run to retry)" resource: dict[str, Any] = { - "eval_run": {"id": run_id, "name": run_name}, + resource_key: {"id": run_id, "name": run_name}, "status": run_status, "selected_types": list(selected_types), "files_downloaded": downloaded, @@ -486,7 +514,7 @@ def run_download( resource.update(selection) return OperationResult( - operation="eval.download", + operation=operation, status="partial" if partial else "success", resource=resource, message=message, diff --git a/osmosis_ai/platform/cli/utils.py b/osmosis_ai/platform/cli/utils.py index eddaf13b..6ab9e66f 100644 --- a/osmosis_ai/platform/cli/utils.py +++ b/osmosis_ai/platform/cli/utils.py @@ -12,6 +12,11 @@ from osmosis_ai.cli.output import DetailSection, ListColumn, ListResult from osmosis_ai.cli.output.display import format_local_datetime from osmosis_ai.platform.api.models import ( + BENCHMARK_RUN_STATUSES_ERROR, + BENCHMARK_RUN_STATUSES_IN_PROGRESS, + BENCHMARK_RUN_STATUSES_PENDING, + BENCHMARK_RUN_STATUSES_STOPPED, + BENCHMARK_RUN_STATUSES_SUCCESS, DEPLOYMENT_STATUSES_SUCCESS, EVAL_RUN_STATUSES_ERROR, EVAL_RUN_STATUSES_IN_PROGRESS, @@ -144,6 +149,13 @@ def require_git_workspace_directory_context() -> GitWorkspaceDirectoryContext: (EVAL_RUN_STATUSES_ERROR, "red"), (EVAL_RUN_STATUSES_STOPPED, "dim"), ) +_BENCHMARK_STATUS_STYLES: _StatusStyleMap = ( + (BENCHMARK_RUN_STATUSES_SUCCESS, "green"), + (BENCHMARK_RUN_STATUSES_PENDING, "orange3"), + (BENCHMARK_RUN_STATUSES_IN_PROGRESS, "blue"), + (BENCHMARK_RUN_STATUSES_ERROR, "red"), + (BENCHMARK_RUN_STATUSES_STOPPED, "dim"), +) def format_status_token( @@ -200,6 +212,11 @@ def format_eval_status(run: Any) -> str: return format_status_token(run.status, _EVAL_STATUS_STYLES) +def format_benchmark_status(run: Any) -> str: + """Format a benchmark run status token with Rich styling.""" + return format_status_token(run.status, _BENCHMARK_STATUS_STYLES) + + def format_reward(reward: float | None) -> str: """Format a training reward to two decimals, en dash when unset.""" if reward is None: diff --git a/tests/unit/cli/output/test_error.py b/tests/unit/cli/output/test_error.py index 99160bed..cc06d95a 100644 --- a/tests/unit/cli/output/test_error.py +++ b/tests/unit/cli/output/test_error.py @@ -113,6 +113,28 @@ def test_command_path_falls_back_to_argv_when_no_context(monkeypatch) -> None: assert command_path_for_error(None) == "dataset list" +@pytest.mark.parametrize( + ("argv", "expected"), + [ + ( + ["osmosis", "--json", "benchmark", "download", "hle-smoke"], + "benchmark download", + ), + ( + ["osmosis", "--json", "benchmark", "catalog", "info", "HLE"], + "benchmark catalog info", + ), + ], +) +def test_benchmark_command_path_falls_back_to_full_command( + monkeypatch: pytest.MonkeyPatch, + argv: list[str], + expected: str, +) -> None: + monkeypatch.setattr("sys.argv", argv) + assert command_path_for_error(None) == expected + + def test_command_path_fallback_excludes_top_level_argument(monkeypatch) -> None: monkeypatch.setattr("sys.argv", ["osmosis", "--json", "deploy", "ckpt-name"]) assert command_path_for_error(None) == "deploy" diff --git a/tests/unit/cli/test_benchmark_commands.py b/tests/unit/cli/test_benchmark_commands.py index 00d695aa..b1d049e8 100644 --- a/tests/unit/cli/test_benchmark_commands.py +++ b/tests/unit/cli/test_benchmark_commands.py @@ -8,7 +8,7 @@ import osmosis_ai.platform.cli.benchmark as benchmark_handler -def test_benchmark_list_delegates_to_handler( +def test_benchmark_catalog_list_delegates_to_handler( monkeypatch: pytest.MonkeyPatch, ) -> None: captured: dict[str, object] = {} @@ -20,13 +20,13 @@ def fake_list_benchmarks(*, limit: int, all_: bool) -> object: monkeypatch.setattr(benchmark_handler, "list_benchmarks", fake_list_benchmarks) - result = benchmark_commands.benchmark_list(limit=25, all_=True) + result = benchmark_commands.benchmark_catalog_list(limit=25, all_=True) assert result is expected assert captured == {"limit": 25, "all_": True} -def test_benchmark_info_delegates_to_handler( +def test_benchmark_catalog_info_delegates_to_handler( monkeypatch: pytest.MonkeyPatch, ) -> None: captured: dict[str, object] = {} @@ -36,9 +36,9 @@ def fake_info(name_or_id: str) -> object: captured["name_or_id"] = name_or_id return expected - monkeypatch.setattr(benchmark_handler, "info", fake_info) + monkeypatch.setattr(benchmark_handler, "catalog_info", fake_info) - result = benchmark_commands.benchmark_info("Terminal-Bench 2.1") + result = benchmark_commands.benchmark_catalog_info("Terminal-Bench 2.1") assert result is expected assert captured == {"name_or_id": "Terminal-Bench 2.1"} @@ -65,3 +65,67 @@ def fake_submit(config_path: Path, *, yes: bool) -> object: "config_path": Path("configs/benchmark/smoke.toml"), "yes": True, } + + +def test_benchmark_run_commands_delegate_to_handlers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, object]] = [] + expected = object() + + monkeypatch.setattr( + benchmark_handler, + "list_benchmark_runs", + lambda *, limit, all_: calls.append(("list", (limit, all_))) or expected, + ) + monkeypatch.setattr( + benchmark_handler, + "run_info", + lambda name_or_id: calls.append(("info", name_or_id)) or expected, + ) + monkeypatch.setattr( + benchmark_handler, + "logs", + lambda name_or_id, *, limit, cursor: ( + calls.append(("logs", (name_or_id, limit, cursor))) or expected + ), + ) + monkeypatch.setattr( + benchmark_handler, + "stop", + lambda name_or_id, *, yes: ( + calls.append(("stop", (name_or_id, yes))) or expected + ), + ) + monkeypatch.setattr( + benchmark_handler, + "download", + lambda name_or_id, *, output, types, overwrite, yes: ( + calls.append(("download", (name_or_id, output, types, overwrite, yes))) + or expected + ), + ) + + assert benchmark_commands.benchmark_list(limit=10, all_=False) is expected + assert benchmark_commands.benchmark_info("run-1") is expected + assert ( + benchmark_commands.benchmark_logs("run-1", limit=25, cursor="older") is expected + ) + assert benchmark_commands.benchmark_stop("run-1", yes=True) is expected + assert ( + benchmark_commands.benchmark_download( + "run-1", + output="out", + types="all", + overwrite=True, + yes=True, + ) + is expected + ) + assert calls == [ + ("list", (10, False)), + ("info", "run-1"), + ("logs", ("run-1", 25, "older")), + ("stop", ("run-1", True)), + ("download", ("run-1", "out", "all", True, True)), + ] diff --git a/tests/unit/cli/test_benchmark_download.py b/tests/unit/cli/test_benchmark_download.py new file mode 100644 index 00000000..698f72a0 --- /dev/null +++ b/tests/unit/cli/test_benchmark_download.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import osmosis_ai.cli.main as cli +import osmosis_ai.platform.cli.benchmark as benchmark_module +import osmosis_ai.platform.cli.run_download as run_download_module +from osmosis_ai.cli.errors import CLIError +from osmosis_ai.platform.api.models import ( + BenchmarkRunDetail, + RunDownloadFile, + RunDownloadManifest, + RunDownloadURL, + RunDownloadURLBatch, +) +from osmosis_ai.platform.auth import PlatformAPIError + +GIT_IDENTITY = "acme/workspace" + +FILES = { + "summary": [RunDownloadFile("summary.csv", 10, token="export-1")], + "results": [RunDownloadFile("results.csv", 20, token="export-1")], + "artifacts": [ + RunDownloadFile( + "artifacts/result_01/logs/agent.log", + 30, + token="result-01", + ) + ], + "logs": [RunDownloadFile("logs.txt", 40)], +} + + +def _detail(status: str = "finished") -> BenchmarkRunDetail: + return BenchmarkRunDetail( + id="benchmark-run-1", + name="hle-smoke", + status=status, + benchmark_name="HLE", + created_at="2026-07-30T00:00:00Z", + ) + + +def _stub_context(monkeypatch: pytest.MonkeyPatch, workspace: Path) -> object: + credentials = object() + context = SimpleNamespace( + workspace_directory=workspace, + git_identity=GIT_IDENTITY, + repo_url="https://github.com/acme/workspace.git", + credentials=credentials, + ) + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + lambda: context, + ) + return credentials + + +def _fake_client( + *, + status: str = "finished", + manifest_error: Exception | None = None, + manifest_files: list[RunDownloadFile] | None = None, +): + manifest_calls: list[tuple[str, ...]] = [] + url_calls: list[list[RunDownloadFile]] = [] + + class FakeClient: + def get_benchmark_run(self, name_or_id, *, git_identity, credentials=None): + assert name_or_id == "hle-smoke" + return _detail(status) + + def get_benchmark_run_download_manifest( + self, run_id, *, types, git_identity, credentials=None + ): + if manifest_error is not None: + raise manifest_error + assert run_id == "benchmark-run-1" + manifest_calls.append(tuple(types)) + files = ( + manifest_files + if manifest_files is not None + else [item for kind in types for item in FILES[kind]] + ) + return RunDownloadManifest( + files=files, + totals={"files": len(files), "bytes": sum(item.size for item in files)}, + ) + + def get_benchmark_run_download_urls( + self, run_id, *, items, git_identity, credentials=None + ): + assert run_id == "benchmark-run-1" + url_calls.append(list(items)) + return RunDownloadURLBatch( + items=[ + RunDownloadURL( + path=item.path, + token=item.token, + url=f"https://example.test/{item.path}", + ) + for item in items + ] + ) + + FakeClient.manifest_calls = manifest_calls + FakeClient.url_calls = url_calls + return FakeClient + + +def _stub_download(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_download(url, destination, *, expected_size=None): + size = expected_size or 0 + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b"x" * size) + return size + + monkeypatch.setattr(run_download_module, "download_file_to", fake_download) + + +def test_default_benchmark_download_uses_run_scoped_layout( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _stub_context(monkeypatch, tmp_path) + fake_client = _fake_client() + monkeypatch.setattr(benchmark_module, "OsmosisClient", fake_client) + _stub_download(monkeypatch) + + exit_code = cli.main(["--json", "benchmark", "download", "hle-smoke"]) + captured = capsys.readouterr() + + assert exit_code == 0 + envelope = json.loads(captured.out) + resource = envelope["resource"] + root = tmp_path / ".osmosis" / "benchmarks" / "hle-smoke" + assert envelope["operation"] == "benchmark.download" + assert resource["benchmark_run"] == { + "id": "benchmark-run-1", + "name": "hle-smoke", + } + assert resource["selected_types"] == ["summary", "results"] + assert fake_client.manifest_calls == [("summary", "results")] + assert (root / "summary.csv").is_file() + assert (root / "results.csv").is_file() + assert not (root / "logs.txt").exists() + + +def test_all_benchmark_download_accepts_stable_artifact_paths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _stub_context(monkeypatch, tmp_path) + fake_client = _fake_client() + monkeypatch.setattr(benchmark_module, "OsmosisClient", fake_client) + _stub_download(monkeypatch) + + result = benchmark_module.download( + "hle-smoke", + output=None, + types="all", + overwrite=False, + yes=True, + ) + + root = tmp_path / ".osmosis" / "benchmarks" / "hle-smoke" + assert result.status == "success" + assert (root / "artifacts" / "result_01" / "logs" / "agent.log").is_file() + assert (root / "logs.txt").is_file() + + +@pytest.mark.parametrize("status", ["pending", "queued"]) +def test_benchmark_download_rejects_pending_statuses( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + status: str, +) -> None: + _stub_context(monkeypatch, tmp_path) + fake_client = _fake_client(status=status) + monkeypatch.setattr(benchmark_module, "OsmosisClient", fake_client) + _stub_download(monkeypatch) + + exit_code = cli.main(["--json", "benchmark", "download", "hle-smoke"]) + captured = capsys.readouterr() + + assert exit_code == 1 + envelope = json.loads(captured.err) + assert envelope["command"] == "benchmark download" + assert envelope["error"]["code"] == "CONFLICT" + assert envelope["error"]["message"] == ( + "Outputs are not yet available for pending or queued benchmark runs." + ) + assert fake_client.manifest_calls == [] + + +def test_benchmark_download_allows_running( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _stub_context(monkeypatch, tmp_path) + fake_client = _fake_client(status="running") + monkeypatch.setattr(benchmark_module, "OsmosisClient", fake_client) + _stub_download(monkeypatch) + + result = benchmark_module.download( + "hle-smoke", + output=None, + types="summary", + overwrite=False, + yes=True, + ) + + assert result.status == "success" + assert fake_client.manifest_calls == [("summary",)] + + +def test_benchmark_download_rejects_unknown_types( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _stub_context(monkeypatch, tmp_path) + monkeypatch.setattr(benchmark_module, "OsmosisClient", _fake_client()) + + exit_code = cli.main( + ["--json", "benchmark", "download", "hle-smoke", "--type", "metrics"] + ) + + assert exit_code == 1 + + +def test_benchmark_download_translates_missing_route( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _stub_context(monkeypatch, tmp_path) + monkeypatch.setattr( + benchmark_module, + "OsmosisClient", + _fake_client(manifest_error=PlatformAPIError("not found", status_code=404)), + ) + + with pytest.raises(CLIError, match="platform may not support benchmark downloads"): + benchmark_module.download( + "hle-smoke", + output=None, + types="summary", + overwrite=False, + yes=True, + ) + + +def test_benchmark_download_rejects_paths_outside_fixed_layout( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _stub_context(monkeypatch, tmp_path) + monkeypatch.setattr( + benchmark_module, + "OsmosisClient", + _fake_client( + manifest_files=[ + RunDownloadFile("../summary.csv", 10), + RunDownloadFile("summary.json", 10), + ] + ), + ) + + with pytest.raises(CLIError, match="no usable run-scoped paths"): + benchmark_module.download( + "hle-smoke", + output=None, + types="summary", + overwrite=False, + yes=True, + ) diff --git a/tests/unit/cli/test_command_groups.py b/tests/unit/cli/test_command_groups.py index 7cdb5e5d..ba1ee3b2 100644 --- a/tests/unit/cli/test_command_groups.py +++ b/tests/unit/cli/test_command_groups.py @@ -46,8 +46,14 @@ ["model", "deploy", "--help"], ["model", "undeploy", "--help"], ["benchmark", "--help"], + ["benchmark", "catalog", "--help"], + ["benchmark", "catalog", "list", "--help"], + ["benchmark", "catalog", "info", "--help"], ["benchmark", "list", "--help"], ["benchmark", "info", "--help"], + ["benchmark", "logs", "--help"], + ["benchmark", "stop", "--help"], + ["benchmark", "download", "--help"], ["benchmark", "submit", "--help"], ["rollout", "--help"], ["template", "--help"], diff --git a/tests/unit/platform/api/test_client_benchmark.py b/tests/unit/platform/api/test_client_benchmark.py index d8ae3432..243ecdac 100644 --- a/tests/unit/platform/api/test_client_benchmark.py +++ b/tests/unit/platform/api/test_client_benchmark.py @@ -69,10 +69,26 @@ def test_get_benchmark_encodes_name_and_parses_detail(mock_request: MagicMock) - "requires_harness": True, "requires_judge_model": False, "judge_model_default": None, + "required_secret_names": ["HF_TOKEN"], "pass_threshold": 1, "categories": [{"name": "terminal", "task_count": 89}], - "tasks": [{"name": "task-1", "category": "terminal"}], - "unavailable_tasks": None, + "tasks": [ + { + "name": "task-1", + "category": "terminal", + "difficulty": "hard", + } + ], + "unavailable_tasks": { + "reason": "Missing fixture", + "tasks": [ + { + "name": "task-2", + "category": "terminal", + "difficulty": None, + } + ], + }, } } @@ -83,7 +99,14 @@ def test_get_benchmark_encodes_name_and_parses_detail(mock_request: MagicMock) - assert result.name == "Terminal-Bench 2.1" assert result.categories[0].name == "terminal" - assert result.tasks == [{"name": "task-1", "category": "terminal"}] + assert result.tasks == [ + {"name": "task-1", "category": "terminal", "difficulty": "hard"} + ] + assert result.unavailable_tasks == { + "reason": "Missing fixture", + "tasks": [{"name": "task-2", "category": "terminal", "difficulty": None}], + } + assert result.required_secret_names == ["HF_TOKEN"] assert mock_request.call_args.args[0] == ( "/api/cli/benchmarks/Terminal-Bench%202.1" ) @@ -93,6 +116,51 @@ def test_get_benchmark_encodes_name_and_parses_detail(mock_request: MagicMock) - } +@patch("osmosis_ai.platform.api.client.platform_request") +def test_get_benchmark_defaults_missing_required_secret_names( + mock_request: MagicMock, +) -> None: + mock_request.return_value = { + "benchmark": { + "id": "benchmark-2", + "name": "Terminal-Bench 2.1", + "description": "Terminal benchmark", + "source_type": "osmosis_managed", + "source_ref": "terminal-bench-2-1", + "task_count": 89, + "category_count": 1, + "task_sets": [], + "runner_family": "harbor", + "supports_harness": True, + "requires_harness": True, + "requires_judge_model": False, + "judge_model_default": None, + "pass_threshold": 1, + "categories": [{"name": "terminal", "task_count": 89}], + "tasks": [ + {"name": "task-1", "category": "terminal"}, + { + "name": "task-2", + "category": "terminal", + "difficulty": "extreme", + }, + ], + "unavailable_tasks": None, + } + } + + result = OsmosisClient().get_benchmark( + "Terminal-Bench 2.1", + git_identity="acme/workspace", + ) + + assert result.required_secret_names == [] + assert result.tasks == [ + {"name": "task-1", "category": "terminal", "difficulty": None}, + {"name": "task-2", "category": "terminal", "difficulty": None}, + ] + + @patch("osmosis_ai.platform.api.client.platform_request") def test_submit_benchmark_run_posts_config_sections(mock_request: MagicMock) -> None: mock_request.return_value = { @@ -225,3 +293,148 @@ def test_submit_benchmark_run_omits_empty_optional_sections( } ], } + + +@patch("osmosis_ai.platform.api.client.platform_request") +def test_list_benchmark_runs_gets_paginated_runs(mock_request: MagicMock) -> None: + mock_request.return_value = { + "benchmark_runs": [ + { + "id": "run-1", + "name": "hle-smoke", + "status": "running", + "benchmark": {"id": "benchmark-1", "name": "HLE"}, + "agent_count": 2, + "best_pass_at_1": 0.42, + "ingested_results": 50, + "expected_results": 100, + "created_at": "2026-07-30T00:00:00Z", + } + ], + "total_count": 1, + "has_more": False, + "next_offset": None, + } + + result = OsmosisClient().list_benchmark_runs( + limit=25, + offset=50, + git_identity="acme/workspace", + ) + + assert result.benchmark_runs[0].name == "hle-smoke" + assert result.benchmark_runs[0].best_pass_at_1 == 0.42 + mock_request.assert_called_once_with( + "/api/cli/benchmark-runs?limit=25&offset=50", + credentials=None, + git_identity="acme/workspace", + ) + + +@patch("osmosis_ai.platform.api.client.platform_request") +def test_get_benchmark_run_parses_detail(mock_request: MagicMock) -> None: + mock_request.return_value = { + "benchmark_run": { + "id": "run-1", + "name": "hle-smoke", + "status": "finished", + "created_at": "2026-07-30T00:00:00Z", + "platform_url": "https://platform.example/benchmarks/run-1", + }, + "configuration": { + "benchmark_id": "benchmark-1", + "benchmark_name": "HLE", + "task_filters": {"task_set": "parity"}, + }, + "agents": [{"agent_index": 0, "model_display_name": "GPT-5"}], + "progress": {"ingested": 249, "expected": 249}, + "totals": {"passed": 100, "failed": 149}, + "agent_metrics": [ + { + "benchmark_run_agent_id": "agent-1", + "pass_at_1": {"value": 0.75}, + } + ], + } + + result = OsmosisClient().get_benchmark_run( + "hle smoke", + git_identity="acme/workspace", + ) + + assert result.benchmark_name == "HLE" + assert result.benchmark_id == "benchmark-1" + assert result.agent_count == 1 + assert result.progress == {"ingested": 249, "expected": 249} + assert result.ingested_results == 249 + assert result.expected_results == 249 + assert result.best_pass_at_1 == 0.75 + mock_request.assert_called_once_with( + "/api/cli/benchmark-runs/hle%20smoke", + credentials=None, + git_identity="acme/workspace", + ) + + +@patch("osmosis_ai.platform.api.client.platform_request") +def test_benchmark_logs_stop_and_output_routes(mock_request: MagicMock) -> None: + client = OsmosisClient() + + mock_request.return_value = {"logs": [], "next_cursor": None} + client.get_benchmark_run_logs( + "run/name", + limit=20, + cursor="cursor-1", + git_identity="acme/workspace", + ) + assert mock_request.call_args.args[0] == ( + "/api/cli/benchmark-runs/run%2Fname/logs?" + "limit=20&direction=older&cursor=cursor-1" + ) + + mock_request.return_value = {} + client.stop_benchmark_run("run/name", git_identity="acme/workspace") + mock_request.assert_called_with( + "/api/cli/benchmark-runs/run%2Fname/stop", + method="POST", + data={}, + credentials=None, + git_identity="acme/workspace", + ) + + mock_request.return_value = { + "files": [{"path": "summary.csv", "size": 10, "token": "export-1"}], + "totals": {"files": 1, "bytes": 10}, + } + manifest = client.get_benchmark_run_download_manifest( + "run/name", + types=["summary", "results"], + git_identity="acme/workspace", + ) + assert manifest.files[0].path == "summary.csv" + assert mock_request.call_args.args[0] == ( + "/api/cli/benchmark-runs/run%2Fname/outputs/manifest?types=summary%2Cresults" + ) + + mock_request.return_value = { + "items": [ + { + "path": "summary.csv", + "token": "export-1", + "url": "https://example.test/summary.csv", + } + ] + } + urls = client.get_benchmark_run_download_urls( + "run/name", + items=manifest.files, + git_identity="acme/workspace", + ) + assert urls.items[0].token == "export-1" + mock_request.assert_called_with( + "/api/cli/benchmark-runs/run%2Fname/outputs/download-urls", + method="POST", + data={"items": [{"path": "summary.csv", "token": "export-1"}]}, + credentials=None, + git_identity="acme/workspace", + ) diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index dccf7edc..a8bc8d1e 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -132,10 +132,19 @@ def get_benchmark( BenchmarkCategory(name="science", task_count=1), ], tasks=[ - {"name": "hle__math", "category": "math"}, - {"name": "hle__science", "category": "science"}, + { + "name": "hle__math", + "category": "math", + "difficulty": None, + }, + { + "name": "hle__science", + "category": "science", + "difficulty": None, + }, ], unavailable_tasks=None, + required_secret_names=["HF_TOKEN"], ) monkeypatch.setattr( @@ -145,17 +154,20 @@ def get_benchmark( ) monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) - result = benchmark_module.info("HLE") + result = benchmark_module.catalog_info("HLE") assert isinstance(result, DetailResult) assert result.data["benchmark"]["tasks"] == [ - {"name": "hle__math", "category": "math"}, - {"name": "hle__science", "category": "science"}, + {"name": "hle__math", "category": "math", "difficulty": None}, + {"name": "hle__science", "category": "science", "difficulty": None}, ] assert result.data["benchmark"]["categories"] == [ {"name": "math", "task_count": 1}, {"name": "science", "task_count": 1}, ] + assert result.data["benchmark"]["required_secret_names"] == ["HF_TOKEN"] + fields = {field.label: field.value for field in result.fields} + assert fields["Required Secret Records"] == "HF_TOKEN" assert 'task_set = "parity"' in result.display_hints[0] assert "Omit [tasks]" in result.display_hints[1] assert calls == [ @@ -165,3 +177,27 @@ def get_benchmark( "git_identity": GIT_IDENTITY, } ] + + +def test_catalog_detail_defaults_required_secret_names() -> None: + detail = BenchmarkCatalogDetail( + id="benchmark-1", + name="Example", + description=None, + source_type="harbor_registry", + source_ref="example@1", + task_count=1, + category_count=0, + task_sets=[], + runner_family="harbor", + supports_harness=True, + requires_harness=True, + requires_judge_model=False, + judge_model_default=None, + pass_threshold=1, + categories=[], + tasks=[{"name": "task-1", "category": None, "difficulty": None}], + unavailable_tasks=None, + ) + + assert detail.required_secret_names == [] diff --git a/tests/unit/platform/cli/test_benchmark_runs.py b/tests/unit/platform/cli/test_benchmark_runs.py new file mode 100644 index 00000000..35f5396f --- /dev/null +++ b/tests/unit/platform/cli/test_benchmark_runs.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import osmosis_ai.cli.main as cli +import osmosis_ai.platform.cli.benchmark as benchmark_module +from osmosis_ai.cli.output import DetailResult, ListResult, OperationResult +from osmosis_ai.platform.api.models import ( + BenchmarkRun, + BenchmarkRunDetail, + LogEntry, + LogsPage, + PaginatedBenchmarkRuns, +) + +GIT_IDENTITY = "acme/workspace" +FAKE_CREDENTIALS = object() + + +def _context(tmp_path: Path | None = None) -> SimpleNamespace: + return SimpleNamespace( + workspace_directory=tmp_path or Path("/repo"), + git_identity=GIT_IDENTITY, + repo_url="https://github.com/acme/workspace.git", + credentials=FAKE_CREDENTIALS, + ) + + +def _run(*, status: str = "running") -> BenchmarkRun: + return BenchmarkRun( + id="run-1", + name="hle-smoke", + status=status, + benchmark_id="benchmark-1", + benchmark_name="HLE", + agent_count=2, + best_pass_at_1=0.42, + ingested_results=50, + expected_results=100, + creator_name="Ada", + created_at="2026-07-30T00:00:00Z", + ) + + +def _detail( + *, + status: str = "running", + is_internal_user: bool = True, +) -> BenchmarkRunDetail: + run = _run(status=status) + return BenchmarkRunDetail( + **run.__dict__, + configuration={ + "source_type": "osmosis_managed", + "source_ref": "hle", + "resolved_version": "1", + "task_filters": {"task_set": "parity"}, + "config": {"attempts_per_task": 2}, + "resolved_secret_scopes": {"OPENAI_API_KEY": "workspace"}, + }, + agents=[ + { + "agent_index": 0, + "harness": "codex", + "model_display_name": "GPT-5", + "status": "running", + }, + { + "agent_index": 1, + "harness": None, + "model_display_name": "Qwen", + "status": "pending", + }, + ], + progress={"ingested": 50, "expected": 100}, + totals={ + "passed": 20, + "failed": 25, + "errored": 5, + "cancelled": 0, + "total_input_tokens": 1000, + "total_output_tokens": 500, + "total_cost_usd": 1.25, + }, + agent_metrics=[{"benchmark_run_agent_id": "agent-1"}], + is_internal_user=is_internal_user, + ) + + +def test_list_benchmark_runs_returns_public_and_display_shapes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + + class FakeClient: + def list_benchmark_runs(self, **kwargs: Any) -> PaginatedBenchmarkRuns: + calls.append(kwargs) + return PaginatedBenchmarkRuns( + benchmark_runs=[_run()], + total_count=1, + has_more=False, + ) + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.list_benchmark_runs(limit=50, all_=False) + + assert isinstance(result, ListResult) + assert result.items[0]["benchmark_name"] == "HLE" + assert result.items[0]["progress"] == { + "completed": 50, + "total": 100, + "unit": "results", + } + assert result.display_items[0]["progress"] == "50 / 100 results" + assert result.display_items[0]["best_pass_at_1"] == "42.0%" + assert calls == [ + { + "limit": 50, + "offset": 0, + "credentials": FAKE_CREDENTIALS, + "git_identity": GIT_IDENTITY, + } + ] + + +def test_benchmark_list_json_envelope( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + class FakeClient: + def list_benchmark_runs(self, **kwargs: Any) -> PaginatedBenchmarkRuns: + return PaginatedBenchmarkRuns( + benchmark_runs=[_run()], + total_count=1, + has_more=False, + ) + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + exit_code = cli.main(["--json", "benchmark", "list"]) + envelope = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert envelope["schema_version"] == 1 + assert envelope["items"][0]["benchmark_name"] == "HLE" + assert envelope["items"][0]["progress"]["completed"] == 50 + + +def test_run_info_returns_config_agents_results_and_next_steps( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeClient: + def get_benchmark_run( + self, name_or_id: str, **kwargs: Any + ) -> BenchmarkRunDetail: + assert name_or_id == "hle-smoke" + assert kwargs["credentials"] is FAKE_CREDENTIALS + return _detail() + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.run_info("hle-smoke") + + assert isinstance(result, DetailResult) + assert result.data["benchmark_run"]["benchmark_name"] == "HLE" + assert result.data["benchmark_run"]["benchmark_id"] == "benchmark-1" + assert result.data["benchmark_run"]["best_pass_at_1"] == 0.42 + assert result.data["progress"] == { + "completed": 50, + "total": 100, + "unit": "results", + } + assert result.data["totals"]["passed"] == 20 + assert [section.plain_lines[0] for section in result.sections] == [ + "Configuration:", + "Agents:", + "Results:", + ] + assert "osmosis benchmark stop hle-smoke" in result.display_hints[-2] + assert "osmosis benchmark download hle-smoke" in result.display_hints[-1] + + +def test_run_info_always_displays_canonical_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeClient: + def get_benchmark_run( + self, name_or_id: str, **kwargs: Any + ) -> BenchmarkRunDetail: + return _detail(is_internal_user=False) + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.run_info("hle-smoke") + + fields = {field.label: field.value for field in result.fields} + assert fields["ID"] == "run-1" + + +def test_logs_uses_shared_cursor_result(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeClient: + def get_benchmark_run_logs(self, name_or_id: str, **kwargs: Any) -> LogsPage: + assert name_or_id == "run-1" + assert kwargs["cursor"] == "cursor-1" + return LogsPage( + logs=[ + LogEntry( + timestamp="2026-07-30T00:00:00Z", + level="info", + step="runner", + message="Started", + ) + ], + next_cursor="cursor-2", + ) + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.logs("run-1", limit=20, cursor="cursor-1") + + assert isinstance(result, ListResult) + assert result.items[0]["message"] == "Started" + assert result.extra["next_cursor"] == "cursor-2" + + +def test_stop_confirms_and_calls_platform(monkeypatch: pytest.MonkeyPatch) -> None: + confirmations: list[dict[str, Any]] = [] + calls: list[dict[str, Any]] = [] + + class FakeClient: + def get_benchmark_run( + self, name_or_id: str, **kwargs: Any + ) -> BenchmarkRunDetail: + calls.append({"operation": "get", "name_or_id": name_or_id, **kwargs}) + return _detail() + + def stop_benchmark_run(self, name_or_id: str, **kwargs: Any) -> dict[str, Any]: + calls.append({"operation": "stop", "name_or_id": name_or_id, **kwargs}) + return {"status": "stopped"} + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + monkeypatch.setattr( + benchmark_module, + "require_confirmation", + lambda prompt, **kwargs: confirmations.append({"prompt": prompt, **kwargs}), + ) + + result = benchmark_module.stop("run-1", yes=True) + + assert isinstance(result, OperationResult) + assert result.operation == "benchmark.stop" + assert result.resource == { + "id": "run-1", + "name": "hle-smoke", + "status": "stopped", + } + assert confirmations[0]["yes"] is True + assert confirmations[0]["prompt"] == 'Stop benchmark run "hle-smoke"?' + assert confirmations[0]["summary"] == [ + ("Name", "hle-smoke"), + ("ID", "run-1"), + ] + assert calls == [ + { + "operation": "get", + "name_or_id": "run-1", + "credentials": FAKE_CREDENTIALS, + "git_identity": GIT_IDENTITY, + }, + { + "operation": "stop", + "name_or_id": "run-1", + "credentials": FAKE_CREDENTIALS, + "git_identity": GIT_IDENTITY, + }, + ] + + +def test_stop_resolves_name_before_stopping_canonical_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + class FakeClient: + def get_benchmark_run( + self, name_or_id: str, **kwargs: Any + ) -> BenchmarkRunDetail: + calls.append(("get", name_or_id)) + return _detail() + + def stop_benchmark_run(self, name_or_id: str, **kwargs: Any) -> dict[str, Any]: + calls.append(("stop", name_or_id)) + return {"status": "stopped"} + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + monkeypatch.setattr( + benchmark_module, + "require_confirmation", + lambda *args, **kwargs: None, + ) + + result = benchmark_module.stop("hle-smoke", yes=True) + + assert calls == [("get", "hle-smoke"), ("stop", "run-1")] + assert result.resource == { + "id": "run-1", + "name": "hle-smoke", + "status": "stopped", + } From 4b2052f9c77cb471dfa2c5a9c3c17db70536de1c Mon Sep 17 00:00:00 2001 From: JoyboyBrian Date: Fri, 31 Jul 2026 12:42:02 -0700 Subject: [PATCH 09/22] [cli] feat: add shell-safe benchmark catalog keys --- docs/cli.md | 4 ++++ osmosis_ai/cli/commands/benchmark.py | 2 +- osmosis_ai/platform/cli/benchmark.py | 10 ++++++---- tests/unit/platform/cli/test_benchmark_catalog.py | 13 +++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index c0ee2703..d7bb150a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -24,6 +24,10 @@ Module-level imports in `commands/` are kept light: `typer`, `cli.console`, `cli discovery is a separate nested namespace, `benchmark catalog list|info`, so top-level `list` and `info` always refer to submitted runs. +Catalog output includes a shell-safe benchmark `Key`, such as +`terminal-bench-2-1`. Pass that key to `osmosis benchmark catalog info `; +exact display names and UUIDs remain supported for compatibility. + ## Commands return results; they don't print The Typer app is created with `result_callback=render_command_result` ([../osmosis_ai/cli/main.py](../osmosis_ai/cli/main.py)). A command function **returns** a `CommandResult`; the callback renders it in the active format. Do not `print()` from a command — return a typed result instead. diff --git a/osmosis_ai/cli/commands/benchmark.py b/osmosis_ai/cli/commands/benchmark.py index ca21163a..8c12880e 100644 --- a/osmosis_ai/cli/commands/benchmark.py +++ b/osmosis_ai/cli/commands/benchmark.py @@ -43,7 +43,7 @@ def benchmark_catalog_list( @catalog_app.command("info") def benchmark_catalog_info( - name_or_id: str = typer.Argument(..., help="Benchmark name or ID."), + name_or_id: str = typer.Argument(..., help="Benchmark key, name, or ID."), ) -> Any: """Show benchmark metadata and task-selection options.""" from osmosis_ai.platform.cli.benchmark import catalog_info as _info diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index fefe3f75..003faac9 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -71,11 +71,11 @@ ) _BENCHMARK_COLUMNS = [ - ListColumn(key="name", label="Name", ratio=4, overflow="fold"), + ListColumn(key="name", label="Name", ratio=3, overflow="fold"), + ListColumn(key="key", label="Key", no_wrap=True, min_width=20), ListColumn(key="task_count", label="Tasks", no_wrap=True, ratio=1), ListColumn(key="category_count", label="Categories", no_wrap=True, ratio=1), ListColumn(key="task_sets", label="Named Task Sets", ratio=2, overflow="fold"), - ListColumn(key="source", label="Source", no_wrap=True, ratio=1), ] @@ -94,6 +94,7 @@ def _benchmark_resource( return { "id": benchmark.id, "name": benchmark.name, + "key": benchmark.source_ref, "description": benchmark.description, "source_type": benchmark.source_type, "source_ref": benchmark.source_ref, @@ -156,7 +157,7 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: for benchmark in benchmarks ], display_hints=[ - "Use osmosis benchmark catalog info for task sets, " + "Use osmosis benchmark catalog info for task sets, " "categories, and tasks." ], ) @@ -194,6 +195,7 @@ def catalog_info(name_or_id: str) -> DetailResult: ) rows = [ ("Name", console.escape(benchmark.name)), + ("Key", console.escape(benchmark.source_ref)), ("Description", console.escape(benchmark.description or "–")), ("Source", f"{benchmark.source_type}: {benchmark.source_ref}"), ("Runner", benchmark.runner_family), @@ -229,7 +231,7 @@ def catalog_info(name_or_id: str) -> DetailResult: display_hints = [ f"Omit [tasks] to select all {benchmark.task_count:,} tasks.", "Use task_names or categories under [tasks] for a custom subset.", - "Use osmosis --json benchmark catalog info to inspect the full " + "Use osmosis --json benchmark catalog info to inspect the full " "task list.", ] for task_set in benchmark.task_sets: diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index a8bc8d1e..de367904 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -74,6 +74,7 @@ def list_benchmarks(self, **kwargs: Any) -> PaginatedBenchmarks: assert isinstance(result, ListResult) assert result.total_count == 1 assert result.items[0]["name"] == "HLE" + assert result.items[0]["key"] == "hle" assert result.items[0]["task_sets"] == [ { "name": "parity", @@ -82,7 +83,17 @@ def list_benchmarks(self, **kwargs: Any) -> PaginatedBenchmarks: "description": "Published comparison sample.", } ] + assert result.display_items[0]["key"] == "hle" assert result.display_items[0]["task_sets"] == "parity (249, recommended)" + assert [column.key for column in result.columns] == [ + "name", + "key", + "task_count", + "category_count", + "task_sets", + ] + assert result.columns[1].no_wrap is True + assert result.columns[1].min_width == 20 assert result.extra["git"]["identity"] == GIT_IDENTITY assert calls == [ { @@ -157,6 +168,7 @@ def get_benchmark( result = benchmark_module.catalog_info("HLE") assert isinstance(result, DetailResult) + assert result.data["benchmark"]["key"] == "hle" assert result.data["benchmark"]["tasks"] == [ {"name": "hle__math", "category": "math", "difficulty": None}, {"name": "hle__science", "category": "science", "difficulty": None}, @@ -167,6 +179,7 @@ def get_benchmark( ] assert result.data["benchmark"]["required_secret_names"] == ["HF_TOKEN"] fields = {field.label: field.value for field in result.fields} + assert fields["Key"] == "hle" assert fields["Required Secret Records"] == "HF_TOKEN" assert 'task_set = "parity"' in result.display_hints[0] assert "Omit [tasks]" in result.display_hints[1] From 7815408b9b071eb80a82696db58be9c423a79661 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Sat, 1 Aug 2026 23:11:44 +0000 Subject: [PATCH 10/22] refactor changes --- osmosis_ai/platform/api/models.py | 28 +++++ osmosis_ai/platform/cli/benchmark.py | 53 ++++++++- osmosis_ai/platform/cli/benchmark_config.py | 15 ++- .../platform/cli/test_benchmark_catalog.py | 110 ++++++++++++++++++ .../platform/cli/test_benchmark_config.py | 67 ++++++++--- 5 files changed, 252 insertions(+), 21 deletions(-) diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index 920f8f07..ee688cb8 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -446,6 +446,15 @@ class BenchmarkCatalogEntry: task_count: int category_count: int task_sets: list[BenchmarkTaskSet] + sync_status: str = "ready" + synced_task_count: int = 0 + sync_error: str | None = None + platform_url: str | None = None + + @property + def is_ready(self) -> bool: + """Whether the task list has finished paging in from the registry.""" + return self.sync_status == "ready" @classmethod def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogEntry: @@ -460,6 +469,10 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogEntry: task_sets=[ BenchmarkTaskSet.from_dict(item) for item in data.get("task_sets", []) ], + sync_status=data.get("sync_status", "ready"), + synced_task_count=int(data.get("synced_task_count") or 0), + sync_error=data.get("sync_error"), + platform_url=data.get("platform_url"), ) @@ -507,6 +520,16 @@ class BenchmarkCatalogDetail: tasks: list[BenchmarkCatalogTask] unavailable_tasks: dict[str, Any] | None required_secret_names: list[str] = field(default_factory=list) + default_harness: str | None = None + sync_status: str = "ready" + synced_task_count: int = 0 + sync_error: str | None = None + platform_url: str | None = None + + @property + def is_ready(self) -> bool: + """Whether the task list has finished paging in from the registry.""" + return self.sync_status == "ready" @classmethod def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: @@ -541,6 +564,11 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: benchmark.get("unavailable_tasks") ), required_secret_names=list(benchmark.get("required_secret_names", [])), + default_harness=benchmark.get("default_harness"), + sync_status=benchmark.get("sync_status", "ready"), + synced_task_count=int(benchmark.get("synced_task_count") or 0), + sync_error=benchmark.get("sync_error"), + platform_url=benchmark.get("platform_url"), ) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 003faac9..0cf6eb5d 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -101,9 +101,44 @@ def _benchmark_resource( "task_count": benchmark.task_count, "category_count": benchmark.category_count, "task_sets": [_task_set_resource(task_set) for task_set in benchmark.task_sets], + "sync_status": benchmark.sync_status, + "synced_task_count": benchmark.synced_task_count, + "sync_error": benchmark.sync_error, + "platform_url": benchmark.platform_url, } +def _task_count_display( + benchmark: BenchmarkCatalogEntry | BenchmarkCatalogDetail, +) -> str: + """Task total, or sync progress while the registry manifest pages in.""" + if benchmark.is_ready: + return f"{benchmark.task_count:,}" + if benchmark.sync_status == "failed": + return "unavailable" + return f"{benchmark.synced_task_count:,} / {benchmark.task_count:,} syncing" + + +def _sync_hints( + benchmark: BenchmarkCatalogEntry | BenchmarkCatalogDetail, +) -> list[str]: + if benchmark.is_ready: + return [] + location = ( + f" Retry its sync at {benchmark.platform_url}." + if benchmark.platform_url + else "" + ) + if benchmark.sync_status == "failed": + reason = benchmark.sync_error or "Its task list failed to sync." + return [f"{benchmark.name} is not runnable: {reason}{location}"] + return [ + f"{benchmark.name} is still syncing its task list " + f"({benchmark.synced_task_count:,} of {benchmark.task_count:,} tasks); " + "submit once it is ready." + ] + + def _task_set_display(task_sets: list[BenchmarkTaskSet]) -> str: if not task_sets: return "–" @@ -145,7 +180,7 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: display_items=[ { **_benchmark_resource(benchmark), - "task_count": f"{benchmark.task_count:,}", + "task_count": _task_count_display(benchmark), "category_count": f"{benchmark.category_count:,}", "task_sets": _task_set_display(benchmark.task_sets), "source": ( @@ -158,7 +193,8 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: ], display_hints=[ "Use osmosis benchmark catalog info for task sets, " - "categories, and tasks." + "categories, and tasks.", + *[hint for benchmark in benchmarks for hint in _sync_hints(benchmark)], ], ) @@ -183,6 +219,8 @@ def catalog_info(name_or_id: str) -> DetailResult: if benchmark.supports_harness else "Not supported" ) + if benchmark.default_harness: + harness += f" (default: {benchmark.default_harness})" judge = "Not required" if benchmark.requires_judge_model: judge = "Required" @@ -199,7 +237,7 @@ def catalog_info(name_or_id: str) -> DetailResult: ("Description", console.escape(benchmark.description or "–")), ("Source", f"{benchmark.source_type}: {benchmark.source_ref}"), ("Runner", benchmark.runner_family), - ("Tasks", f"{benchmark.task_count:,}"), + ("Tasks", _task_count_display(benchmark)), ("Categories", category_display or "–"), ("Named Task Sets", _task_set_display(benchmark.task_sets)), ("Harness", harness), @@ -216,6 +254,7 @@ def catalog_info(name_or_id: str) -> DetailResult: "runner_family": benchmark.runner_family, "supports_harness": benchmark.supports_harness, "requires_harness": benchmark.requires_harness, + "default_harness": benchmark.default_harness, "requires_judge_model": benchmark.requires_judge_model, "judge_model_default": benchmark.judge_model_default, "required_secret_names": benchmark.required_secret_names, @@ -233,7 +272,15 @@ def catalog_info(name_or_id: str) -> DetailResult: "Use task_names or categories under [tasks] for a custom subset.", "Use osmosis --json benchmark catalog info to inspect the full " "task list.", + *_sync_hints(benchmark), ] + if benchmark.default_harness: + display_hints.insert( + 0, + f"{benchmark.name}'s published scores were measured on " + f'harness = "{benchmark.default_harness}"; another harness is ' + "not comparable with them.", + ) for task_set in benchmark.task_sets: if task_set.recommended: display_hints.insert( diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index 0db4760a..91783760 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -171,9 +171,10 @@ def _validate_secret_references(config: BenchmarkSubmitConfig, path: Path) -> No Collisions are scoped per agent: one agent's secret name may still be another agent's literal env var. Model secrets also cannot use names the runner removes before model-key aliasing. Harness credentials travel - through a separate reserved channel; known credentialed harnesses must - reference a secret record and cannot also set their platform-managed - destination env. ``HF_TOKEN`` is always runner-reserved as a literal env. + through a separate reserved channel; a credentialed harness must reference + a secret record named exactly for the variable it reads, and cannot also + set that name as a literal env var. ``HF_TOKEN`` is always runner-reserved + as a literal env. """ for name in config.required_secrets: if not SECRET_NAME_RE.match(name): @@ -238,6 +239,14 @@ def _validate_secret_references(config: BenchmarkSubmitConfig, path: Path) -> No f"harness_api_key_secret in {path}. Set it to the name of a " f"Platform secret record containing {harness_env_name}." ) + if harness_env_name and agent.harness_api_key_secret != harness_env_name: + raise CLIError( + f"Agent {index}'s harness_api_key_secret " + f"'{agent.harness_api_key_secret}' in {path} does not match " + f"the variable the {agent.harness} harness reads. Store the " + f"credential in a Platform secret record named exactly " + f"{harness_env_name} and reference that name." + ) def load_benchmark_submit_config(path: Path) -> BenchmarkSubmitConfig: diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index de367904..8e796af6 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -214,3 +214,113 @@ def test_catalog_detail_defaults_required_secret_names() -> None: ) assert detail.required_secret_names == [] + + +def _syncing_entry(**overrides: Any) -> BenchmarkCatalogEntry: + return BenchmarkCatalogEntry( + id="benchmark-2", + name="acme/suite", + description=None, + source_type="harbor_registry", + source_ref="acme/suite@3", + task_count=4_000, + category_count=0, + task_sets=[], + platform_url="https://platform.example/Acme/benchmarks", + **overrides, + ) + + +def _list_with( + monkeypatch: pytest.MonkeyPatch, entry: BenchmarkCatalogEntry +) -> ListResult: + class FakeClient: + def list_benchmarks(self, **_: Any) -> PaginatedBenchmarks: + return PaginatedBenchmarks( + benchmarks=[entry], + total_count=1, + has_more=False, + next_offset=None, + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + return benchmark_module.list_benchmarks(limit=50, all_=False) + + +def test_list_benchmarks_shows_registry_sync_progress( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result = _list_with( + monkeypatch, + _syncing_entry(sync_status="syncing", synced_task_count=1_000), + ) + + assert result.display_items[0]["task_count"] == "1,000 / 4,000 syncing" + assert result.items[0]["sync_status"] == "syncing" + assert any("still syncing" in hint for hint in result.display_hints) + + +def test_list_benchmarks_surfaces_failed_sync_and_platform_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result = _list_with( + monkeypatch, + _syncing_entry( + sync_status="failed", + sync_error="Task list unavailable.", + ), + ) + + assert result.display_items[0]["task_count"] == "unavailable" + assert result.items[0]["sync_error"] == "Task list unavailable." + assert any( + "Task list unavailable." in hint + and "https://platform.example/Acme/benchmarks" in hint + for hint in result.display_hints + ) + + +def test_catalog_info_surfaces_the_default_harness( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeClient: + def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: + return BenchmarkCatalogDetail( + id="benchmark-1", + name="Terminal-Bench 2.1", + description=None, + source_type="osmosis_managed", + source_ref="terminal-bench-2-1", + task_count=89, + category_count=1, + task_sets=[], + runner_family="harbor", + supports_harness=True, + requires_harness=True, + requires_judge_model=False, + judge_model_default=None, + pass_threshold=1, + categories=[], + tasks=[], + unavailable_tasks=None, + default_harness="terminus-2", + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.catalog_info("terminal-bench-2-1") + + assert result.data["benchmark"]["default_harness"] == "terminus-2" + fields = {field.label: field.value for field in result.fields} + assert fields["Harness"] == "Required (default: terminus-2)" + assert 'harness = "terminus-2"' in result.display_hints[0] diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index e8cd3eaf..d061b2af 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -539,47 +539,84 @@ def test_load_benchmark_submit_config_validates_harness_secret_name( load_benchmark_submit_config(path) -def test_load_benchmark_submit_config_allows_harness_secret_record_name_as_env( +@pytest.mark.parametrize( + "harness, destination_env", + [ + ("cursor-cli", "CURSOR_API_KEY"), + ("mini-swe-agent", "MSWEA_API_KEY"), + ], +) +def test_load_benchmark_submit_config_accepts_pinned_harness_secret_name( tmp_path: Path, + harness: str, + destination_env: str, ) -> None: - """The record name is separate from the harness's destination env name.""" path = _write_config( tmp_path / "benchmark.toml", - """ + f""" [experiment] benchmark = "DeepSWE" [[agents]] -harness = "cursor-cli" -harness_api_key_secret = "MY_CURSOR_TOKEN" +harness = "{harness}" +harness_api_key_secret = "{destination_env}" [agents.model] type = "provider" model = "openai/gpt-5" api_key_secret = "OPENAI_API_KEY" - -[agents.env] -MY_CURSOR_TOKEN = "literal-for-the-agent" """, ) config = load_benchmark_submit_config(path) - assert config.required_secrets == ["OPENAI_API_KEY", "MY_CURSOR_TOKEN"] - assert config.agents[0].env == {"MY_CURSOR_TOKEN": "literal-for-the-agent"} + assert config.required_secrets == ["OPENAI_API_KEY", destination_env] + + +@pytest.mark.parametrize( + "harness, destination_env", + [ + ("cursor-cli", "CURSOR_API_KEY"), + ("mini-swe-agent", "MSWEA_API_KEY"), + ], +) +def test_load_benchmark_submit_config_rejects_unpinned_harness_secret_name( + tmp_path: Path, + harness: str, + destination_env: str, +) -> None: + """The record must be named for the variable the harness actually reads.""" + path = _write_config( + tmp_path / "benchmark.toml", + f""" +[experiment] +benchmark = "DeepSWE" + +[[agents]] +harness = "{harness}" +harness_api_key_secret = "MY_HARNESS_TOKEN" + +[agents.model] +type = "provider" +model = "openai/gpt-5" +api_key_secret = "OPENAI_API_KEY" +""", + ) + + with pytest.raises(CLIError, match=rf"named exactly {destination_env}"): + load_benchmark_submit_config(path) @pytest.mark.parametrize( - "harness, harness_secret, destination_env", + "harness, destination_env", [ - ("cursor-cli", "MY_CURSOR_TOKEN", "CURSOR_API_KEY"), - ("mini-swe-agent", "MY_MSWEA_TOKEN", "MSWEA_API_KEY"), + ("cursor-cli", "CURSOR_API_KEY"), + ("mini-swe-agent", "MSWEA_API_KEY"), ], ) def test_load_benchmark_submit_config_rejects_harness_destination_env_collision( tmp_path: Path, harness: str, - harness_secret: str, destination_env: str, ) -> None: path = _write_config( @@ -590,7 +627,7 @@ def test_load_benchmark_submit_config_rejects_harness_destination_env_collision( [[agents]] harness = "{harness}" -harness_api_key_secret = "{harness_secret}" +harness_api_key_secret = "{destination_env}" [agents.model] type = "hosted" From 68980854f22239f1295cd258ead8dd644b754cc4 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Mon, 3 Aug 2026 18:28:27 +0000 Subject: [PATCH 11/22] improve cli --- osmosis_ai/cli/commands/benchmark.py | 4 +- osmosis_ai/cli/commands/eval.py | 4 +- osmosis_ai/platform/cli/benchmark.py | 13 ++- osmosis_ai/platform/cli/model.py | 2 +- osmosis_ai/platform/cli/train.py | 4 +- tests/conftest.py | 12 +++ tests/unit/cli/test_model_commands.py | 8 +- tests/unit/cli/test_train_commands.py | 2 +- tests/unit/cli/test_train_info_checkpoints.py | 4 +- .../platform/cli/test_benchmark_catalog.py | 83 ++++++++++++++++++- 10 files changed, 119 insertions(+), 17 deletions(-) diff --git a/osmosis_ai/cli/commands/benchmark.py b/osmosis_ai/cli/commands/benchmark.py index 8c12880e..f05ae4b5 100644 --- a/osmosis_ai/cli/commands/benchmark.py +++ b/osmosis_ai/cli/commands/benchmark.py @@ -137,7 +137,7 @@ def benchmark_download( None, "--output", "-o", - help="Run output root (default: .osmosis/benchmarks//).", + help="Run output root (default: .osmosis/benchmarks//).", ), types: str = typer.Option( "summary,results", @@ -159,7 +159,7 @@ def benchmark_download( help="Skip size confirmation.", ), ) -> Any: - """Download benchmark summary, results, artifacts, or logs.""" + """Download benchmark run summary, results, artifacts, or logs.""" from osmosis_ai.platform.cli.benchmark import download as _download return _download( diff --git a/osmosis_ai/cli/commands/eval.py b/osmosis_ai/cli/commands/eval.py index d73cf0ab..e77db7f7 100644 --- a/osmosis_ai/cli/commands/eval.py +++ b/osmosis_ai/cli/commands/eval.py @@ -138,7 +138,7 @@ def eval_info( None, "--output", "-o", - help="Run output root (default in rich mode: .osmosis/evals//).", + help="Run output root (default in rich mode: .osmosis/evals//).", ), ) -> Any: """Show evaluation run details, results, and metrics.""" @@ -154,7 +154,7 @@ def eval_download( None, "--output", "-o", - help="Run output root (default: .osmosis/evals//).", + help="Run output root (default: .osmosis/evals//).", ), types: str = typer.Option( "metrics,trajectories", diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 0cf6eb5d..752a8915 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -217,10 +217,12 @@ def catalog_info(name_or_id: str) -> DetailResult: if benchmark.requires_harness else "Optional" if benchmark.supports_harness - else "Not supported" + else "Official scaffold only" ) if benchmark.default_harness: harness += f" (default: {benchmark.default_harness})" + elif benchmark.supports_harness and not benchmark.requires_harness: + harness += " (default: official scaffold)" judge = "Not required" if benchmark.requires_judge_model: judge = "Required" @@ -281,6 +283,13 @@ def catalog_info(name_or_id: str) -> DetailResult: f'harness = "{benchmark.default_harness}"; another harness is ' "not comparable with them.", ) + elif benchmark.supports_harness and not benchmark.requires_harness: + display_hints.insert( + 0, + "Every agent needs a [[agents]] entry. Omit its harness to run " + f"{benchmark.name}'s official scaffold, or name one to compare " + "scaffolds in the same run.", + ) for task_set in benchmark.task_sets: if task_set.recommended: display_hints.insert( @@ -358,7 +367,7 @@ def list_benchmark_runs(*, limit: int, all_: bool) -> ListResult: } for run in runs ], - display_hints=["Use osmosis benchmark info for details."], + display_hints=["Use osmosis benchmark info for details."], ) diff --git a/osmosis_ai/platform/cli/model.py b/osmosis_ai/platform/cli/model.py index cba7dc42..405635cd 100644 --- a/osmosis_ai/platform/cli/model.py +++ b/osmosis_ai/platform/cli/model.py @@ -209,7 +209,7 @@ def _fetch_lora(lim: int, off: int) -> Any: "inference deployments used" ) if lora_models: - display_hints.append("Use osmosis model info for details.") + display_hints.append("Use osmosis model info for details.") lora_extra = {**git_result_context(context)} if deployment_info["present"]: lora_extra["active_deployments"] = active_deployments diff --git a/osmosis_ai/platform/cli/train.py b/osmosis_ai/platform/cli/train.py index 9902272a..5456863d 100644 --- a/osmosis_ai/platform/cli/train.py +++ b/osmosis_ai/platform/cli/train.py @@ -385,7 +385,7 @@ def info(name: str, *, output: str | None) -> DetailResult: from rich.text import Text table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Checkpoint", overflow="fold") + table.add_column("LoRA Model", overflow="fold") table.add_column("Step", no_wrap=True) table.add_column("Status", no_wrap=True) if run.is_internal_user: @@ -395,7 +395,7 @@ def info(name: str, *, output: str | None) -> DetailResult: cp_name = cp.checkpoint_name or "(unnamed)" cells = [Text(cp_name), str(cp.checkpoint_step), cp.status] plain_line = ( - f"Checkpoint: {cp_name} step {cp.checkpoint_step} [{cp.status}]" + f"LoRA Model: {cp_name} step {cp.checkpoint_step} [{cp.status}]" ) if run.is_internal_user: cells.append(cp.id[:8]) diff --git a/tests/conftest.py b/tests/conftest.py index 25b082fa..8d1ae551 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,18 @@ import pytest +@pytest.fixture(autouse=True) +def _neutralize_color_forcing_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep Rich's terminal detection independent of the developer's shell. + + Rich forces terminal mode when FORCE_COLOR is merely present, whatever its + value, so a shell exporting FORCE_COLOR=0 makes Console emit ANSI styles + into assertions that expect plain text. + """ + for key in ("FORCE_COLOR", "CLICOLOR_FORCE"): + monkeypatch.delenv(key, raising=False) + + @pytest.fixture(autouse=True) def _reset_output_context_var() -> None: """Keep CLI output context state isolated across tests.""" diff --git a/tests/unit/cli/test_model_commands.py b/tests/unit/cli/test_model_commands.py index fb72bff4..2b440639 100644 --- a/tests/unit/cli/test_model_commands.py +++ b/tests/unit/cli/test_model_commands.py @@ -285,7 +285,7 @@ def test_list_sections_base_models_before_lora_models( assert lora_section.display_items[0]["reward"] == "0.85" assert base_section.total_count == 1 assert lora_section.total_count == 1 - assert result.display_hints == ["Use osmosis model info for details."] + assert result.display_hints == ["Use osmosis model info for details."] def test_list_items_carry_no_type_discriminator( self, monkeypatch: pytest.MonkeyPatch, console_capture: StringIO @@ -518,7 +518,7 @@ def test_list_type_lora_only_calls_lora_endpoint( assert all("type" not in item for item in result.items) assert [c.key for c in result.columns] == _LORA_COLUMN_KEYS assert result.next_offset == 1 - assert result.display_hints == ["Use osmosis model info for details."] + assert result.display_hints == ["Use osmosis model info for details."] assert_git_context(result.extra) def test_list_rejects_invalid_type(self) -> None: @@ -570,7 +570,7 @@ def test_list_shows_deployment_quota_hint_under_lora_table( assert isinstance(result, SectionedListResult) assert result.display_hints == [ "2 of 5 inference deployments used", - "Use osmosis model info for details.", + "Use osmosis model info for details.", ] def test_list_type_lora_shows_deployment_quota_hint( @@ -602,7 +602,7 @@ def test_list_omits_quota_hint_when_server_reports_no_quota( result = platform_model_module.list_models(limit=30, all_=False) assert isinstance(result, SectionedListResult) - assert result.display_hints == ["Use osmosis model info for details."] + assert result.display_hints == ["Use osmosis model info for details."] def test_list_all_captures_quota_from_first_page( self, monkeypatch: pytest.MonkeyPatch, console_capture: StringIO diff --git a/tests/unit/cli/test_train_commands.py b/tests/unit/cli/test_train_commands.py index 5a77a249..7b86f4b2 100644 --- a/tests/unit/cli/test_train_commands.py +++ b/tests/unit/cli/test_train_commands.py @@ -737,7 +737,7 @@ def list_training_run_checkpoints( assert result.sections section = result.sections[0] assert [column.header for column in section.rich.columns] == [ - "Checkpoint", + "LoRA Model", "Step", "Status", ] diff --git a/tests/unit/cli/test_train_info_checkpoints.py b/tests/unit/cli/test_train_info_checkpoints.py index e47df740..af9c1db5 100644 --- a/tests/unit/cli/test_train_info_checkpoints.py +++ b/tests/unit/cli/test_train_info_checkpoints.py @@ -299,7 +299,7 @@ def list_training_run_checkpoints( assert result.sections section = result.sections[0] assert [column.header for column in section.rich.columns] == [ - "Checkpoint", + "LoRA Model", "Step", "Status", "ID", @@ -335,7 +335,7 @@ def list_training_run_checkpoints( assert isinstance(result, DetailResult) assert result.data["checkpoints"] == [] assert all( - field.label not in {"Checkpoint", "Deploy"} for field in result.fields + field.label not in {"LoRA Model", "Deploy"} for field in result.fields ) assert result.sections == [] expected_url = "https://platform.osmosis.ai/ws/training/run_1" diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index 8e796af6..f3aeb309 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -134,7 +134,7 @@ def get_benchmark( task_sets=[parity], runner_family="harbor", supports_harness=True, - requires_harness=False, + requires_harness=True, requires_judge_model=True, judge_model_default="openai/gpt-5", pass_threshold=1, @@ -324,3 +324,84 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: fields = {field.label: field.value for field in result.fields} assert fields["Harness"] == "Required (default: terminus-2)" assert 'harness = "terminus-2"' in result.display_hints[0] + + +def test_catalog_info_names_the_official_scaffold_as_the_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A benchmark that allows but does not require a harness defaults to its own scaffold.""" + + class FakeClient: + def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: + return BenchmarkCatalogDetail( + id="benchmark-3", + name="BrowseComp", + description=None, + source_type="osmosis_managed", + source_ref="browsecomp", + task_count=10, + category_count=1, + task_sets=[], + runner_family="harbor", + supports_harness=True, + requires_harness=False, + requires_judge_model=False, + judge_model_default=None, + pass_threshold=1, + categories=[], + tasks=[], + unavailable_tasks=None, + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.catalog_info("browsecomp") + + fields = {field.label: field.value for field in result.fields} + assert fields["Harness"] == "Optional (default: official scaffold)" + assert "[[agents]] entry" in result.display_hints[0] + + +def test_catalog_info_reports_a_scaffold_only_benchmark( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A benchmark with no harness support runs its own scaffold, not "nothing".""" + + class FakeClient: + def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: + return BenchmarkCatalogDetail( + id="benchmark-4", + name="Toolathlon-Verified", + description=None, + source_type="osmosis_managed", + source_ref="toolathlon-verified", + task_count=5, + category_count=1, + task_sets=[], + runner_family="toolathlon", + supports_harness=False, + requires_harness=False, + requires_judge_model=False, + judge_model_default=None, + pass_threshold=1, + categories=[], + tasks=[], + unavailable_tasks=None, + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.catalog_info("toolathlon-verified") + + fields = {field.label: field.value for field in result.fields} + assert fields["Harness"] == "Official scaffold only" From 2511ce51da8abdf6b2cd2349b3402539aebd0bc4 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Mon, 3 Aug 2026 22:14:47 +0000 Subject: [PATCH 12/22] [cli] fix: rename hosted model checkpoint_name to lora_model_name --- osmosis_ai/platform/cli/benchmark.py | 2 +- osmosis_ai/platform/cli/benchmark_config.py | 2 +- .../platform/cli/test_benchmark_config.py | 20 +++++++++---------- .../platform/cli/test_benchmark_submit.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 752a8915..865c160d 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -662,7 +662,7 @@ def stop(name_or_id: str, *, yes: bool) -> OperationResult: def _agent_model_label(agent: dict[str, Any]) -> str: model = agent["model"] if model["type"] == "hosted": - return f"{model['base_model']}:{model['checkpoint_name']}" + return f"{model['base_model']}:{model['lora_model_name']}" return str(model["model"]) diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index 91783760..cd3f9943 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -81,7 +81,7 @@ def reject_authorization_header( class BenchmarkHostedModel(_StrictSection): type: Literal["hosted"] base_model: str - checkpoint_name: str + lora_model_name: str BenchmarkModel = Annotated[ diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index d061b2af..095b2d07 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -155,7 +155,7 @@ def test_load_benchmark_submit_config_accepts_hosted_model(tmp_path: Path) -> No [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "terminal-agent" +lora_model_name = "terminal-agent" """, ) @@ -187,7 +187,7 @@ def test_load_benchmark_submit_config_accepts_hle_parity_with_explicit_filters( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "hle-agent" +lora_model_name = "hle-agent" [execution] judge_model = "openai/gpt-5" @@ -223,7 +223,7 @@ def test_load_benchmark_submit_config_rejects_unknown_task_set( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "hle-agent" +lora_model_name = "hle-agent" """, ) @@ -262,7 +262,7 @@ def test_load_benchmark_submit_config_rejects_invalid_explicit_task_filters( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "hle-agent" +lora_model_name = "hle-agent" """, ) @@ -425,7 +425,7 @@ def test_load_benchmark_submit_config_rejects_non_string_judge_secret( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "deep-swe-agent" +lora_model_name = "deep-swe-agent" [execution] judge_api_key_secret = 42 @@ -505,7 +505,7 @@ def test_load_benchmark_submit_config_rejects_literal_hf_token_for_every_benchma [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "benchmark-agent" +lora_model_name = "benchmark-agent" {env_section} """, @@ -632,7 +632,7 @@ def test_load_benchmark_submit_config_rejects_harness_destination_env_collision( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "deep-swe-agent" +lora_model_name = "deep-swe-agent" [agents.env] {destination_env} = "literal-for-the-agent" @@ -658,7 +658,7 @@ def test_load_benchmark_submit_config_rejects_harness_destination_env_without_se [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "deep-swe-agent" +lora_model_name = "deep-swe-agent" [agents.env] CURSOR_API_KEY = "literal-for-the-agent" @@ -684,7 +684,7 @@ def test_load_benchmark_submit_config_requires_known_harness_secret( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "deep-swe-agent" +lora_model_name = "deep-swe-agent" """, ) @@ -717,7 +717,7 @@ def test_load_benchmark_submit_config_rejects_empty_secret_references( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "deep-swe-agent" +lora_model_name = "deep-swe-agent" {section} """, diff --git a/tests/unit/platform/cli/test_benchmark_submit.py b/tests/unit/platform/cli/test_benchmark_submit.py index 7c92f5d5..0575b2e1 100644 --- a/tests/unit/platform/cli/test_benchmark_submit.py +++ b/tests/unit/platform/cli/test_benchmark_submit.py @@ -67,7 +67,7 @@ def _write_hosted_config( [agents.model] type = "hosted" base_model = "Qwen/Qwen3-8B" -checkpoint_name = "benchmark-agent" +lora_model_name = "benchmark-agent" [execution] judge_model = "openai/gpt-5" From 60d22cdc57e6521959e4a43d9bf64e00d41f8607 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Mon, 3 Aug 2026 22:14:53 +0000 Subject: [PATCH 13/22] [cli] fix: label train info checkpoints as Checkpoint --- osmosis_ai/platform/cli/train.py | 4 ++-- tests/unit/cli/test_train_commands.py | 2 +- tests/unit/cli/test_train_info_checkpoints.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/osmosis_ai/platform/cli/train.py b/osmosis_ai/platform/cli/train.py index 5456863d..9902272a 100644 --- a/osmosis_ai/platform/cli/train.py +++ b/osmosis_ai/platform/cli/train.py @@ -385,7 +385,7 @@ def info(name: str, *, output: str | None) -> DetailResult: from rich.text import Text table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("LoRA Model", overflow="fold") + table.add_column("Checkpoint", overflow="fold") table.add_column("Step", no_wrap=True) table.add_column("Status", no_wrap=True) if run.is_internal_user: @@ -395,7 +395,7 @@ def info(name: str, *, output: str | None) -> DetailResult: cp_name = cp.checkpoint_name or "(unnamed)" cells = [Text(cp_name), str(cp.checkpoint_step), cp.status] plain_line = ( - f"LoRA Model: {cp_name} step {cp.checkpoint_step} [{cp.status}]" + f"Checkpoint: {cp_name} step {cp.checkpoint_step} [{cp.status}]" ) if run.is_internal_user: cells.append(cp.id[:8]) diff --git a/tests/unit/cli/test_train_commands.py b/tests/unit/cli/test_train_commands.py index 7b86f4b2..5a77a249 100644 --- a/tests/unit/cli/test_train_commands.py +++ b/tests/unit/cli/test_train_commands.py @@ -737,7 +737,7 @@ def list_training_run_checkpoints( assert result.sections section = result.sections[0] assert [column.header for column in section.rich.columns] == [ - "LoRA Model", + "Checkpoint", "Step", "Status", ] diff --git a/tests/unit/cli/test_train_info_checkpoints.py b/tests/unit/cli/test_train_info_checkpoints.py index af9c1db5..e47df740 100644 --- a/tests/unit/cli/test_train_info_checkpoints.py +++ b/tests/unit/cli/test_train_info_checkpoints.py @@ -299,7 +299,7 @@ def list_training_run_checkpoints( assert result.sections section = result.sections[0] assert [column.header for column in section.rich.columns] == [ - "LoRA Model", + "Checkpoint", "Step", "Status", "ID", @@ -335,7 +335,7 @@ def list_training_run_checkpoints( assert isinstance(result, DetailResult) assert result.data["checkpoints"] == [] assert all( - field.label not in {"LoRA Model", "Deploy"} for field in result.fields + field.label not in {"Checkpoint", "Deploy"} for field in result.fields ) assert result.sections == [] expected_url = "https://platform.osmosis.ai/ws/training/run_1" From 3822bf28d2cbf273e4fdde88b3dc0a6bd4176c6d Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Mon, 3 Aug 2026 22:14:53 +0000 Subject: [PATCH 14/22] style: reformat model command test assertions --- tests/unit/cli/test_model_commands.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/unit/cli/test_model_commands.py b/tests/unit/cli/test_model_commands.py index 2b440639..a1b25395 100644 --- a/tests/unit/cli/test_model_commands.py +++ b/tests/unit/cli/test_model_commands.py @@ -285,7 +285,9 @@ def test_list_sections_base_models_before_lora_models( assert lora_section.display_items[0]["reward"] == "0.85" assert base_section.total_count == 1 assert lora_section.total_count == 1 - assert result.display_hints == ["Use osmosis model info for details."] + assert result.display_hints == [ + "Use osmosis model info for details." + ] def test_list_items_carry_no_type_discriminator( self, monkeypatch: pytest.MonkeyPatch, console_capture: StringIO @@ -518,7 +520,9 @@ def test_list_type_lora_only_calls_lora_endpoint( assert all("type" not in item for item in result.items) assert [c.key for c in result.columns] == _LORA_COLUMN_KEYS assert result.next_offset == 1 - assert result.display_hints == ["Use osmosis model info for details."] + assert result.display_hints == [ + "Use osmosis model info for details." + ] assert_git_context(result.extra) def test_list_rejects_invalid_type(self) -> None: @@ -602,7 +606,9 @@ def test_list_omits_quota_hint_when_server_reports_no_quota( result = platform_model_module.list_models(limit=30, all_=False) assert isinstance(result, SectionedListResult) - assert result.display_hints == ["Use osmosis model info for details."] + assert result.display_hints == [ + "Use osmosis model info for details." + ] def test_list_all_captures_quota_from_first_page( self, monkeypatch: pytest.MonkeyPatch, console_capture: StringIO From f2dd3987ef697439a778b87551c83b8218d8c56a Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 04:37:24 +0000 Subject: [PATCH 15/22] update cli commands --- CONTRIBUTING.md | 10 +- docs/cli.md | 11 +- docs/run-downloads.md | 2 +- osmosis_ai/cli/commands/benchmark.py | 50 ++--- osmosis_ai/cli/output/error.py | 2 +- osmosis_ai/platform/api/client.py | 11 +- osmosis_ai/platform/api/models.py | 12 ++ osmosis_ai/platform/cli/benchmark.py | 177 ++++++++++++++++-- .../cli_output/benchmark_run_serializer.json | 20 ++ tests/unit/cli/output/test_error.py | 8 +- tests/unit/cli/output/test_serializers.py | 27 +++ tests/unit/cli/test_benchmark_commands.py | 33 ++-- tests/unit/cli/test_benchmark_download.py | 8 +- tests/unit/cli/test_command_groups.py | 12 +- .../platform/cli/test_benchmark_catalog.py | 46 ++++- .../unit/platform/cli/test_benchmark_runs.py | 6 +- 16 files changed, 343 insertions(+), 92 deletions(-) create mode 100644 tests/golden/cli_output/benchmark_run_serializer.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dddddb5e..1beaabe4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,10 +44,12 @@ The table below lists all development commands. If you installed with **pip**, d Coverage configuration is in `pyproject.toml` under `[tool.coverage.*]`. CI enforces a minimum coverage threshold of 70%. -When changing remote run commands, preserve the lifecycle convention: -`train`, `eval`, and `benchmark` use top-level `submit`, `list`, `info`, -`logs`, and `stop` for run management. Benchmark-definition discovery lives -under `osmosis benchmark catalog list|info`. Eval and benchmark downloads +When changing remote run commands, preserve the naming convention: bare +verbs act on a group's primary noun. `train` and `eval` manage runs with +top-level `submit`, `list`, `info`, `logs`, and `stop` because the run is +their noun; `benchmark list|info` act on benchmarks themselves (catalog and +benchmark page), with run lifecycle nested under `osmosis benchmark runs +list|info|logs|stop|download`. Eval and benchmark downloads share the manifest transfer engine in `osmosis_ai/platform/cli/run_download.py`; add domain-specific routes and fixed path classifiers instead of copying the transfer loop. diff --git a/docs/cli.md b/docs/cli.md index d7bb150a..902d7cd1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -19,13 +19,14 @@ Files in [../osmosis_ai/cli/commands/](../osmosis_ai/cli/commands/) are thin Typ Module-level imports in `commands/` are kept light: `typer`, `cli.console`, `cli.errors`, the lightweight `osmosis_ai.platform.constants` (pagination limits), and stdlib. Everything heavy (`rollout.*`, `platform.api.*`, `platform.cli.*`, `eval.*`) must be imported **inside the function** to keep CLI startup fast — see the lazy-loading section of [architecture.md](./architecture.md). -`osmosis benchmark` follows the same run-lifecycle surface as train/eval: -`submit`, `list`, `info`, `logs`, `stop`, and `download`. Benchmark-definition -discovery is a separate nested namespace, `benchmark catalog list|info`, so -top-level `list` and `info` always refer to submitted runs. +`osmosis benchmark` puts the benchmark first, mirroring the platform's +Benchmarks pages: top-level `list` and `info` act on benchmarks (the catalog +and one benchmark's page - `info` shows its metadata, leaderboard, and runs), +`submit` starts a run, and run lifecycle lives under the nested `benchmark +runs list|info|logs|stop|download` namespace. Catalog output includes a shell-safe benchmark `Key`, such as -`terminal-bench-2-1`. Pass that key to `osmosis benchmark catalog info `; +`terminal-bench-2-1`. Pass that key to `osmosis benchmark info `; exact display names and UUIDs remain supported for compatibility. ## Commands return results; they don't print diff --git a/docs/run-downloads.md b/docs/run-downloads.md index ad3c14e1..46bdea53 100644 --- a/docs/run-downloads.md +++ b/docs/run-downloads.md @@ -12,7 +12,7 @@ osmosis eval download NAME_OR_ID --overwrite -y, --yes -osmosis benchmark download NAME_OR_ID +osmosis benchmark runs download NAME_OR_ID --type summary,results|artifacts|logs|all -o, --output ROOT --overwrite diff --git a/osmosis_ai/cli/commands/benchmark.py b/osmosis_ai/cli/commands/benchmark.py index f05ae4b5..9f38a8a4 100644 --- a/osmosis_ai/cli/commands/benchmark.py +++ b/osmosis_ai/cli/commands/benchmark.py @@ -14,18 +14,18 @@ ) app: typer.Typer = typer.Typer( - help="Manage benchmark runs.", + help="Manage benchmarks and their runs.", no_args_is_help=True, ) -catalog_app: typer.Typer = typer.Typer( - help="Discover available benchmarks and task-selection options.", +runs_app: typer.Typer = typer.Typer( + help="Manage benchmark runs.", no_args_is_help=True, ) -app.add_typer(catalog_app, name="catalog") +app.add_typer(runs_app, name="runs") -@catalog_app.command("list") -def benchmark_catalog_list( +@app.command("list") +def benchmark_list( limit: int = typer.Option( DEFAULT_PAGE_SIZE, "--limit", @@ -41,14 +41,22 @@ def benchmark_catalog_list( return _list_benchmarks(limit=limit, all_=all_) -@catalog_app.command("info") -def benchmark_catalog_info( +@app.command("info") +def benchmark_info( name_or_id: str = typer.Argument(..., help="Benchmark key, name, or ID."), + limit: int = typer.Option( + DEFAULT_PAGE_SIZE, + "--limit", + min=1, + max=MAX_PAGE_SIZE, + help="Maximum number of runs to show in the runs section.", + ), + all_: bool = typer.Option(False, "--all", help="Show all of the benchmark's runs."), ) -> Any: - """Show benchmark metadata and task-selection options.""" - from osmosis_ai.platform.cli.benchmark import catalog_info as _info + """Show a benchmark: metadata, task options, leaderboard, and runs.""" + from osmosis_ai.platform.cli.benchmark import benchmark_info as _info - return _info(name_or_id) + return _info(name_or_id, limit=limit, all_=all_) @app.command("submit") @@ -70,8 +78,8 @@ def benchmark_submit( return _submit(config_path, yes=yes) -@app.command("list") -def benchmark_list( +@runs_app.command("list") +def benchmark_runs_list( limit: int = typer.Option( DEFAULT_PAGE_SIZE, "--limit", @@ -87,8 +95,8 @@ def benchmark_list( return _list(limit=limit, all_=all_) -@app.command("info") -def benchmark_info( +@runs_app.command("info") +def benchmark_runs_info( name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), ) -> Any: """Show benchmark run details, progress, and results.""" @@ -97,8 +105,8 @@ def benchmark_info( return _info(name_or_id) -@app.command("logs") -def benchmark_logs( +@runs_app.command("logs") +def benchmark_runs_logs( name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), limit: int = typer.Option( DEFAULT_PAGE_SIZE, @@ -119,8 +127,8 @@ def benchmark_logs( return _logs(name_or_id, limit=limit, cursor=cursor) -@app.command("stop") -def benchmark_stop( +@runs_app.command("stop") +def benchmark_runs_stop( name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt."), ) -> Any: @@ -130,8 +138,8 @@ def benchmark_stop( return _stop(name_or_id, yes=yes) -@app.command("download") -def benchmark_download( +@runs_app.command("download") +def benchmark_runs_download( name_or_id: str = typer.Argument(..., help="Benchmark run name or ID."), output: str | None = typer.Option( None, diff --git a/osmosis_ai/cli/output/error.py b/osmosis_ai/cli/output/error.py index b5f817ca..3a16e81f 100644 --- a/osmosis_ai/cli/output/error.py +++ b/osmosis_ai/cli/output/error.py @@ -131,7 +131,7 @@ def _argv_command_path(argv: list[str]) -> str: return command if (command, tokens[1]) in _REMOVED_TWO_TOKEN_COMMANDS: return " ".join(tokens[:2]) - if command == "benchmark" and tokens[1] == "catalog" and len(tokens) >= 3: + if command == "benchmark" and tokens[1] == "runs" and len(tokens) >= 3: return " ".join(tokens[:3]) if command == "eval" and tokens[1] == "cache" and len(tokens) >= 3: return " ".join(tokens[:3]) diff --git a/osmosis_ai/platform/api/client.py b/osmosis_ai/platform/api/client.py index 11e85a86..2e2a77cc 100644 --- a/osmosis_ai/platform/api/client.py +++ b/osmosis_ai/platform/api/client.py @@ -710,11 +710,18 @@ def list_benchmark_runs( limit: int = DEFAULT_PAGE_SIZE, offset: int = 0, *, + benchmark: str | None = None, credentials: Credentials | None = None, git_identity: str, ) -> PaginatedBenchmarkRuns: - """List benchmark runs in the current workspace.""" - qs = urlencode({"limit": limit, "offset": offset}) + """List benchmark runs in the current workspace. + + `benchmark` (a benchmark ID) scopes the list to one benchmark's runs. + """ + params: dict[str, Any] = {"limit": limit, "offset": offset} + if benchmark is not None: + params["benchmark"] = benchmark + qs = urlencode(params) data = platform_request( f"/api/cli/benchmark-runs?{qs}", credentials=credentials, diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index ee688cb8..84bfffd7 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -450,6 +450,9 @@ class BenchmarkCatalogEntry: synced_task_count: int = 0 sync_error: str | None = None platform_url: str | None = None + run_count: int = 0 + running_count: int = 0 + last_run_at: str | None = None @property def is_ready(self) -> bool: @@ -473,6 +476,9 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogEntry: synced_task_count=int(data.get("synced_task_count") or 0), sync_error=data.get("sync_error"), platform_url=data.get("platform_url"), + run_count=int(data.get("run_count") or 0), + running_count=int(data.get("running_count") or 0), + last_run_at=data.get("last_run_at"), ) @@ -525,6 +531,9 @@ class BenchmarkCatalogDetail: synced_task_count: int = 0 sync_error: str | None = None platform_url: str | None = None + # Server-computed standings; metric shapes travel verbatim like + # BenchmarkRunDetail.agent_metrics, so the estimator stays server-side. + leaderboard: list[dict[str, Any]] = field(default_factory=list) @property def is_ready(self) -> bool: @@ -569,6 +578,9 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: synced_task_count=int(benchmark.get("synced_task_count") or 0), sync_error=benchmark.get("sync_error"), platform_url=benchmark.get("platform_url"), + leaderboard=[ + item for item in data.get("leaderboard", []) if isinstance(item, dict) + ], ) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 865c160d..594ba398 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -73,9 +73,10 @@ _BENCHMARK_COLUMNS = [ ListColumn(key="name", label="Name", ratio=3, overflow="fold"), ListColumn(key="key", label="Key", no_wrap=True, min_width=20), + ListColumn(key="status", label="Status", no_wrap=True, ratio=1), + ListColumn(key="run_count", label="Runs", no_wrap=True, ratio=1), + ListColumn(key="last_run_at", label="Last Run", no_wrap=True, ratio=1), ListColumn(key="task_count", label="Tasks", no_wrap=True, ratio=1), - ListColumn(key="category_count", label="Categories", no_wrap=True, ratio=1), - ListColumn(key="task_sets", label="Named Task Sets", ratio=2, overflow="fold"), ] @@ -108,6 +109,26 @@ def _benchmark_resource( } +def _catalog_status(benchmark: BenchmarkCatalogEntry) -> str: + """Activity beats sync state; a quiet benchmark is Ready, never blank.""" + if benchmark.running_count > 0: + return f"Running ({benchmark.running_count})" + if benchmark.sync_status in ("pending", "syncing"): + return "Syncing" + if benchmark.sync_status == "failed": + return "Sync failed" + return "Ready" + + +def _benchmark_list_resource(benchmark: BenchmarkCatalogEntry) -> dict[str, Any]: + return { + **_benchmark_resource(benchmark), + "run_count": benchmark.run_count, + "running_count": benchmark.running_count, + "last_run_at": benchmark.last_run_at, + } + + def _task_count_display( benchmark: BenchmarkCatalogEntry | BenchmarkCatalogDetail, ) -> str: @@ -171,7 +192,7 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: return ListResult( title="Benchmarks", - items=[_benchmark_resource(benchmark) for benchmark in benchmarks], + items=[_benchmark_list_resource(benchmark) for benchmark in benchmarks], total_count=total_count, has_more=has_more, next_offset=next_offset, @@ -179,10 +200,15 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: columns=_BENCHMARK_COLUMNS, display_items=[ { - **_benchmark_resource(benchmark), + **_benchmark_list_resource(benchmark), + "status": _catalog_status(benchmark), + "run_count": f"{benchmark.run_count:,}", + "last_run_at": ( + format_local_date(benchmark.last_run_at) + if benchmark.last_run_at + else "–" + ), "task_count": _task_count_display(benchmark), - "category_count": f"{benchmark.category_count:,}", - "task_sets": _task_set_display(benchmark.task_sets), "source": ( "Managed" if benchmark.source_type == "osmosis_managed" @@ -192,15 +218,81 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: for benchmark in benchmarks ], display_hints=[ - "Use osmosis benchmark catalog info for task sets, " - "categories, and tasks.", + "Use osmosis benchmark info for its leaderboard, runs, " + "task sets, and tasks.", *[hint for benchmark in benchmarks for hint in _sync_hints(benchmark)], ], ) -def catalog_info(name_or_id: str) -> DetailResult: - """Show benchmark metadata and task-selection options.""" +def _format_rate_interval(rate: Any) -> str: + if not isinstance(rate, dict) or rate.get("value") is None: + return "–" + value = float(rate["value"]) + low, high = rate.get("ci_low"), rate.get("ci_high") + if isinstance(low, int | float) and isinstance(high, int | float): + margin = max(value - low, high - value) + return f"{value:.1%} ± {margin:.1%}" + return f"{value:.1%}" + + +def _leaderboard_rows(entries: list[dict[str, Any]]) -> list[tuple[str, str]]: + rows: list[tuple[str, str]] = [] + for entry in entries: + rank = entry.get("rank") + label = f"#{rank}" if isinstance(rank, int) else "–" + model = str(entry.get("model") or "–") + harness = entry.get("harness") + name = f"{model} ({harness})" if harness else model + tags = [ + tag + for tag, present in ( + ("tied", bool(entry.get("tied"))), + ("parity", entry.get("task_set") == "parity"), + ) + if present + ] + parts = [ + name + (f" [{', '.join(tags)}]" if tags else ""), + f"pass@1 {_format_rate_interval(entry.get('pass_at_1'))}", + ] + points = entry.get("pass_at_k") or [] + deepest = points[-1] if points else None + if isinstance(deepest, dict) and deepest.get("value") is not None: + parts.append(f"pass@{deepest.get('k')} {float(deepest['value']):.1%}") + cost = entry.get("reported_cost_usd") + if isinstance(cost, int | float): + parts.append(f"${cost:,.2f}") + seconds = entry.get("mean_duration_seconds") + if isinstance(seconds, int | float): + parts.append(f"{seconds:,.0f}s/task") + run = entry.get("run") + if isinstance(run, dict) and run.get("name"): + parts.append(f"run {run['name']}") + rows.append((label, " · ".join(parts))) + return rows + + +def _benchmark_run_rows(runs: list[BenchmarkRun]) -> list[tuple[str, str]]: + return [ + ( + run.name, + " · ".join( + [ + format_benchmark_status(run), + format_progress(_benchmark_progress(run)) or "–", + f"best pass@1 {_format_pass_at_1(run.best_pass_at_1)}", + format_local_date(run.created_at), + ] + ), + ) + for run in runs + ] + + +def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: + """Show a benchmark: metadata, task options, leaderboard, and runs.""" + effective_limit, fetch_all = validate_list_options(limit=limit, all_=all_) context = require_git_workspace_directory_context() client = OsmosisClient() output = get_output_context() @@ -211,6 +303,18 @@ def catalog_info(name_or_id: str) -> DetailResult: credentials=context.credentials, git_identity=context.git_identity, ) + runs, runs_total_count, _runs_have_more, _runs_next_offset = paginated_fetch( + lambda lim, off: client.list_benchmark_runs( + limit=lim, + offset=off, + benchmark=benchmark.id, + credentials=context.credentials, + git_identity=context.git_identity, + ), + items_attr="benchmark_runs", + limit=effective_limit, + fetch_all=fetch_all, + ) harness = ( "Required" @@ -249,6 +353,7 @@ def catalog_info(name_or_id: str) -> DetailResult: ", ".join(benchmark.required_secret_names) or "–", ), ("Pass Threshold", f"{benchmark.pass_threshold:g}"), + ("Runs", f"{runs_total_count:,}"), ] benchmark_data = { @@ -272,10 +377,16 @@ def catalog_info(name_or_id: str) -> DetailResult: display_hints = [ f"Omit [tasks] to select all {benchmark.task_count:,} tasks.", "Use task_names or categories under [tasks] for a custom subset.", - "Use osmosis --json benchmark catalog info to inspect the full " - "task list.", + "Use osmosis --json benchmark info to inspect the full task list.", + "Use osmosis benchmark runs info for a run's details.", *_sync_hints(benchmark), ] + if not benchmark.leaderboard: + display_hints.insert( + 0, + "No leaderboard entries yet: finished runs on the full task set " + "with scores rank here.", + ) if benchmark.default_harness: display_hints.insert( 0, @@ -298,10 +409,34 @@ def catalog_info(name_or_id: str) -> DetailResult: f'"{task_set.name}" ({task_set.task_count:,} tasks).', ) + sections: list[DetailSection] = [] + for section in ( + kv_section("Leaderboard", _leaderboard_rows(benchmark.leaderboard)), + kv_section( + f"Runs ({len(runs):,} of {runs_total_count:,})", + _benchmark_run_rows(runs), + ), + ): + if section is not None: + sections.append(section) + return DetailResult( title="Benchmark Info", - data={"benchmark": benchmark_data, **git_result_context(context)}, + data={ + "benchmark": benchmark_data, + "leaderboard": benchmark.leaderboard, + "runs": [ + { + **serialize_benchmark_run(run), + "progress": _benchmark_progress(run), + } + for run in runs + ], + "runs_total_count": runs_total_count, + **git_result_context(context), + }, fields=detail_fields(rows), + sections=sections, display_hints=display_hints, ) @@ -367,7 +502,7 @@ def list_benchmark_runs(*, limit: int, all_: bool) -> ListResult: } for run in runs ], - display_hints=["Use osmosis benchmark info for details."], + display_hints=["Use osmosis benchmark runs info for details."], ) @@ -512,11 +647,13 @@ def run_info(name_or_id: str) -> DetailResult: if detail.platform_url: display_hints.append(f"View: {detail.platform_url}") if detail.status in BENCHMARK_RUN_STATUSES_ERROR: - display_hints.append(f"See logs with: osmosis benchmark logs {detail.name}") + display_hints.append( + f"See logs with: osmosis benchmark runs logs {detail.name}" + ) if detail.status not in BENCHMARK_RUN_STATUSES_TERMINAL: - display_hints.append(f"Stop with: osmosis benchmark stop {detail.name}") + display_hints.append(f"Stop with: osmosis benchmark runs stop {detail.name}") display_hints.append( - f"Download outputs with: osmosis benchmark download {detail.name}" + f"Download outputs with: osmosis benchmark runs download {detail.name}" ) return DetailResult( @@ -553,7 +690,7 @@ def logs(name_or_id: str, *, limit: int, cursor: str | None = None) -> ListResul title=f"Benchmark Run Logs: {name_or_id}", page=page, context=context, - next_step_hint=f"Use osmosis benchmark info {name_or_id} for run details.", + next_step_hint=f"Use osmosis benchmark runs info {name_or_id} for run details.", ) @@ -833,7 +970,7 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: display_next_steps = [ f"Status: {result.status}", f"Benchmark: {config.experiment.benchmark}", - f"Check status with: osmosis benchmark info {result.name}", + f"Check status with: osmosis benchmark runs info {result.name}", ] structured_next_steps: list[dict[str, Any]] = [ {"action": "benchmark_info", "name": result.name}, @@ -870,7 +1007,7 @@ def submit(config_path: Path, *, yes: bool) -> OperationResult: __all__ = [ - "catalog_info", + "benchmark_info", "download", "list_benchmark_runs", "list_benchmarks", diff --git a/tests/golden/cli_output/benchmark_run_serializer.json b/tests/golden/cli_output/benchmark_run_serializer.json new file mode 100644 index 00000000..ee46d3fe --- /dev/null +++ b/tests/golden/cli_output/benchmark_run_serializer.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "keys": [ + "id", + "name", + "status", + "benchmark_id", + "benchmark_name", + "agent_count", + "best_pass_at_1", + "ingested_results", + "expected_results", + "creator_name", + "creator_email", + "created_at", + "started_at", + "completed_at", + "platform_url" + ] +} diff --git a/tests/unit/cli/output/test_error.py b/tests/unit/cli/output/test_error.py index cc06d95a..ab514bb6 100644 --- a/tests/unit/cli/output/test_error.py +++ b/tests/unit/cli/output/test_error.py @@ -117,12 +117,12 @@ def test_command_path_falls_back_to_argv_when_no_context(monkeypatch) -> None: ("argv", "expected"), [ ( - ["osmosis", "--json", "benchmark", "download", "hle-smoke"], - "benchmark download", + ["osmosis", "--json", "benchmark", "info", "HLE"], + "benchmark info", ), ( - ["osmosis", "--json", "benchmark", "catalog", "info", "HLE"], - "benchmark catalog info", + ["osmosis", "--json", "benchmark", "runs", "download", "hle-smoke"], + "benchmark runs download", ), ], ) diff --git a/tests/unit/cli/output/test_serializers.py b/tests/unit/cli/output/test_serializers.py index 57220301..2b915508 100644 --- a/tests/unit/cli/output/test_serializers.py +++ b/tests/unit/cli/output/test_serializers.py @@ -6,6 +6,7 @@ from pathlib import Path from osmosis_ai.cli.output.serializers import ( + serialize_benchmark_run, serialize_checkpoint, serialize_dataset, serialize_lora_model, @@ -15,6 +16,7 @@ ) from osmosis_ai.platform.api.models import ( BaseModelInfo, + BenchmarkRun, DatasetFile, LoraCheckpointInfo, LoraModelInfo, @@ -174,3 +176,28 @@ def test_serialize_rollout_keys() -> None: ) payload = serialize_rollout(rollout) _assert_keys_match_golden(payload, "rollout_serializer.json") + + +def test_serialize_benchmark_run_keys() -> None: + run = BenchmarkRun.from_dict( + { + "id": "run-1", + "name": "warm-gull", + "status": "finished", + "benchmark": {"id": "benchmark-1", "name": "HLE"}, + "agent_count": 2, + "best_pass_at_1": 0.75, + "ingested_results": 216, + "expected_results": 216, + "creator_name": "Ada Lovelace", + "creator_email": "ada@example.com", + "created_at": "2026-08-01T00:00:00Z", + "started_at": "2026-08-01T00:01:00Z", + "completed_at": "2026-08-01T02:00:00Z", + "platform_url": "https://platform.example/Acme/benchmarks/runs/run-1", + } + ) + payload = serialize_benchmark_run(run) + _assert_keys_match_golden(payload, "benchmark_run_serializer.json") + assert payload["benchmark_id"] == "benchmark-1" + assert payload["best_pass_at_1"] == 0.75 diff --git a/tests/unit/cli/test_benchmark_commands.py b/tests/unit/cli/test_benchmark_commands.py index b1d049e8..5cc0a2bd 100644 --- a/tests/unit/cli/test_benchmark_commands.py +++ b/tests/unit/cli/test_benchmark_commands.py @@ -8,7 +8,7 @@ import osmosis_ai.platform.cli.benchmark as benchmark_handler -def test_benchmark_catalog_list_delegates_to_handler( +def test_benchmark_list_delegates_to_handler( monkeypatch: pytest.MonkeyPatch, ) -> None: captured: dict[str, object] = {} @@ -20,28 +20,34 @@ def fake_list_benchmarks(*, limit: int, all_: bool) -> object: monkeypatch.setattr(benchmark_handler, "list_benchmarks", fake_list_benchmarks) - result = benchmark_commands.benchmark_catalog_list(limit=25, all_=True) + result = benchmark_commands.benchmark_list(limit=25, all_=True) assert result is expected assert captured == {"limit": 25, "all_": True} -def test_benchmark_catalog_info_delegates_to_handler( +def test_benchmark_info_delegates_to_handler( monkeypatch: pytest.MonkeyPatch, ) -> None: captured: dict[str, object] = {} expected = object() - def fake_info(name_or_id: str) -> object: - captured["name_or_id"] = name_or_id + def fake_info(name_or_id: str, *, limit: int, all_: bool) -> object: + captured.update(name_or_id=name_or_id, limit=limit, all_=all_) return expected - monkeypatch.setattr(benchmark_handler, "catalog_info", fake_info) + monkeypatch.setattr(benchmark_handler, "benchmark_info", fake_info) - result = benchmark_commands.benchmark_catalog_info("Terminal-Bench 2.1") + result = benchmark_commands.benchmark_info( + "Terminal-Bench 2.1", limit=15, all_=False + ) assert result is expected - assert captured == {"name_or_id": "Terminal-Bench 2.1"} + assert captured == { + "name_or_id": "Terminal-Bench 2.1", + "limit": 15, + "all_": False, + } def test_benchmark_submit_delegates_to_handler( @@ -106,14 +112,15 @@ def test_benchmark_run_commands_delegate_to_handlers( ), ) - assert benchmark_commands.benchmark_list(limit=10, all_=False) is expected - assert benchmark_commands.benchmark_info("run-1") is expected + assert benchmark_commands.benchmark_runs_list(limit=10, all_=False) is expected + assert benchmark_commands.benchmark_runs_info("run-1") is expected assert ( - benchmark_commands.benchmark_logs("run-1", limit=25, cursor="older") is expected + benchmark_commands.benchmark_runs_logs("run-1", limit=25, cursor="older") + is expected ) - assert benchmark_commands.benchmark_stop("run-1", yes=True) is expected + assert benchmark_commands.benchmark_runs_stop("run-1", yes=True) is expected assert ( - benchmark_commands.benchmark_download( + benchmark_commands.benchmark_runs_download( "run-1", output="out", types="all", diff --git a/tests/unit/cli/test_benchmark_download.py b/tests/unit/cli/test_benchmark_download.py index 698f72a0..01883760 100644 --- a/tests/unit/cli/test_benchmark_download.py +++ b/tests/unit/cli/test_benchmark_download.py @@ -133,7 +133,7 @@ def test_default_benchmark_download_uses_run_scoped_layout( monkeypatch.setattr(benchmark_module, "OsmosisClient", fake_client) _stub_download(monkeypatch) - exit_code = cli.main(["--json", "benchmark", "download", "hle-smoke"]) + exit_code = cli.main(["--json", "benchmark", "runs", "download", "hle-smoke"]) captured = capsys.readouterr() assert exit_code == 0 @@ -187,12 +187,12 @@ def test_benchmark_download_rejects_pending_statuses( monkeypatch.setattr(benchmark_module, "OsmosisClient", fake_client) _stub_download(monkeypatch) - exit_code = cli.main(["--json", "benchmark", "download", "hle-smoke"]) + exit_code = cli.main(["--json", "benchmark", "runs", "download", "hle-smoke"]) captured = capsys.readouterr() assert exit_code == 1 envelope = json.loads(captured.err) - assert envelope["command"] == "benchmark download" + assert envelope["command"] == "benchmark runs download" assert envelope["error"]["code"] == "CONFLICT" assert envelope["error"]["message"] == ( "Outputs are not yet available for pending or queued benchmark runs." @@ -229,7 +229,7 @@ def test_benchmark_download_rejects_unknown_types( monkeypatch.setattr(benchmark_module, "OsmosisClient", _fake_client()) exit_code = cli.main( - ["--json", "benchmark", "download", "hle-smoke", "--type", "metrics"] + ["--json", "benchmark", "runs", "download", "hle-smoke", "--type", "metrics"] ) assert exit_code == 1 diff --git a/tests/unit/cli/test_command_groups.py b/tests/unit/cli/test_command_groups.py index ba1ee3b2..dfaa7a7a 100644 --- a/tests/unit/cli/test_command_groups.py +++ b/tests/unit/cli/test_command_groups.py @@ -46,15 +46,15 @@ ["model", "deploy", "--help"], ["model", "undeploy", "--help"], ["benchmark", "--help"], - ["benchmark", "catalog", "--help"], - ["benchmark", "catalog", "list", "--help"], - ["benchmark", "catalog", "info", "--help"], ["benchmark", "list", "--help"], ["benchmark", "info", "--help"], - ["benchmark", "logs", "--help"], - ["benchmark", "stop", "--help"], - ["benchmark", "download", "--help"], ["benchmark", "submit", "--help"], + ["benchmark", "runs", "--help"], + ["benchmark", "runs", "list", "--help"], + ["benchmark", "runs", "info", "--help"], + ["benchmark", "runs", "logs", "--help"], + ["benchmark", "runs", "stop", "--help"], + ["benchmark", "runs", "download", "--help"], ["rollout", "--help"], ["template", "--help"], ["eval", "--help"], diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index f3aeb309..33d2b233 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -13,8 +13,20 @@ BenchmarkCatalogEntry, BenchmarkCategory, BenchmarkTaskSet, + PaginatedBenchmarkRuns, PaginatedBenchmarks, ) +from osmosis_ai.platform.constants import DEFAULT_PAGE_SIZE + + +def _empty_runs_page(**kwargs: Any) -> PaginatedBenchmarkRuns: + return PaginatedBenchmarkRuns( + benchmark_runs=[], + total_count=0, + has_more=False, + next_offset=None, + ) + GIT_IDENTITY = "acme/workspace" REPO_URL = "https://github.com/acme/workspace.git" @@ -84,13 +96,16 @@ def list_benchmarks(self, **kwargs: Any) -> PaginatedBenchmarks: } ] assert result.display_items[0]["key"] == "hle" - assert result.display_items[0]["task_sets"] == "parity (249, recommended)" + assert result.display_items[0]["status"] == "Ready" + assert result.display_items[0]["run_count"] == "0" + assert result.display_items[0]["last_run_at"] == "–" assert [column.key for column in result.columns] == [ "name", "key", + "status", + "run_count", + "last_run_at", "task_count", - "category_count", - "task_sets", ] assert result.columns[1].no_wrap is True assert result.columns[1].min_width == 20 @@ -158,6 +173,8 @@ def get_benchmark( required_secret_names=["HF_TOKEN"], ) + list_benchmark_runs = staticmethod(_empty_runs_page) + monkeypatch.setattr( benchmark_module, "require_git_workspace_directory_context", @@ -165,7 +182,7 @@ def get_benchmark( ) monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) - result = benchmark_module.catalog_info("HLE") + result = benchmark_module.benchmark_info("HLE", limit=DEFAULT_PAGE_SIZE, all_=False) assert isinstance(result, DetailResult) assert result.data["benchmark"]["key"] == "hle" @@ -182,7 +199,8 @@ def get_benchmark( assert fields["Key"] == "hle" assert fields["Required Secret Records"] == "HF_TOKEN" assert 'task_set = "parity"' in result.display_hints[0] - assert "Omit [tasks]" in result.display_hints[1] + assert "No leaderboard entries yet" in result.display_hints[1] + assert any("Omit [tasks]" in hint for hint in result.display_hints) assert calls == [ { "name_or_id": "HLE", @@ -311,6 +329,8 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: default_harness="terminus-2", ) + list_benchmark_runs = staticmethod(_empty_runs_page) + monkeypatch.setattr( benchmark_module, "require_git_workspace_directory_context", @@ -318,7 +338,9 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: ) monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) - result = benchmark_module.catalog_info("terminal-bench-2-1") + result = benchmark_module.benchmark_info( + "terminal-bench-2-1", limit=DEFAULT_PAGE_SIZE, all_=False + ) assert result.data["benchmark"]["default_harness"] == "terminus-2" fields = {field.label: field.value for field in result.fields} @@ -353,6 +375,8 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: unavailable_tasks=None, ) + list_benchmark_runs = staticmethod(_empty_runs_page) + monkeypatch.setattr( benchmark_module, "require_git_workspace_directory_context", @@ -360,7 +384,9 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: ) monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) - result = benchmark_module.catalog_info("browsecomp") + result = benchmark_module.benchmark_info( + "browsecomp", limit=DEFAULT_PAGE_SIZE, all_=False + ) fields = {field.label: field.value for field in result.fields} assert fields["Harness"] == "Optional (default: official scaffold)" @@ -394,6 +420,8 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: unavailable_tasks=None, ) + list_benchmark_runs = staticmethod(_empty_runs_page) + monkeypatch.setattr( benchmark_module, "require_git_workspace_directory_context", @@ -401,7 +429,9 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: ) monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) - result = benchmark_module.catalog_info("toolathlon-verified") + result = benchmark_module.benchmark_info( + "toolathlon-verified", limit=DEFAULT_PAGE_SIZE, all_=False + ) fields = {field.label: field.value for field in result.fields} assert fields["Harness"] == "Official scaffold only" diff --git a/tests/unit/platform/cli/test_benchmark_runs.py b/tests/unit/platform/cli/test_benchmark_runs.py index 35f5396f..0e6c9a85 100644 --- a/tests/unit/platform/cli/test_benchmark_runs.py +++ b/tests/unit/platform/cli/test_benchmark_runs.py @@ -149,7 +149,7 @@ def list_benchmark_runs(self, **kwargs: Any) -> PaginatedBenchmarkRuns: ) monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) - exit_code = cli.main(["--json", "benchmark", "list"]) + exit_code = cli.main(["--json", "benchmark", "runs", "list"]) envelope = json.loads(capsys.readouterr().out) assert exit_code == 0 @@ -191,8 +191,8 @@ def get_benchmark_run( "Agents:", "Results:", ] - assert "osmosis benchmark stop hle-smoke" in result.display_hints[-2] - assert "osmosis benchmark download hle-smoke" in result.display_hints[-1] + assert "osmosis benchmark runs stop hle-smoke" in result.display_hints[-2] + assert "osmosis benchmark runs download hle-smoke" in result.display_hints[-1] def test_run_info_always_displays_canonical_id( From cc5b9dc66b968bd488509b1a472d8bb86561ba5e Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 05:04:33 +0000 Subject: [PATCH 16/22] fix cubic comments --- .../platform/cli/test_benchmark_catalog.py | 199 +++++++++++++++++- 1 file changed, 193 insertions(+), 6 deletions(-) diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index 33d2b233..b12de6bc 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -12,6 +12,7 @@ BenchmarkCatalogDetail, BenchmarkCatalogEntry, BenchmarkCategory, + BenchmarkRun, BenchmarkTaskSet, PaginatedBenchmarkRuns, PaginatedBenchmarks, @@ -237,14 +238,14 @@ def test_catalog_detail_defaults_required_secret_names() -> None: def _syncing_entry(**overrides: Any) -> BenchmarkCatalogEntry: return BenchmarkCatalogEntry( id="benchmark-2", - name="acme/suite", + name="acme/custom", description=None, source_type="harbor_registry", - source_ref="acme/suite@3", + source_ref="acme/custom@3", task_count=4_000, category_count=0, task_sets=[], - platform_url="https://platform.example/Acme/benchmarks", + platform_url="https://platform.example/Acme/benchmarks/benchmark-2", **overrides, ) @@ -303,7 +304,7 @@ def test_list_benchmarks_surfaces_failed_sync_and_platform_url( ) -def test_catalog_info_surfaces_the_default_harness( +def test_benchmark_info_surfaces_the_default_harness( monkeypatch: pytest.MonkeyPatch, ) -> None: class FakeClient: @@ -348,7 +349,7 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: assert 'harness = "terminus-2"' in result.display_hints[0] -def test_catalog_info_names_the_official_scaffold_as_the_default( +def test_benchmark_info_names_the_official_scaffold_as_the_default( monkeypatch: pytest.MonkeyPatch, ) -> None: """A benchmark that allows but does not require a harness defaults to its own scaffold.""" @@ -393,7 +394,7 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: assert "[[agents]] entry" in result.display_hints[0] -def test_catalog_info_reports_a_scaffold_only_benchmark( +def test_benchmark_info_reports_a_scaffold_only_benchmark( monkeypatch: pytest.MonkeyPatch, ) -> None: """A benchmark with no harness support runs its own scaffold, not "nothing".""" @@ -435,3 +436,189 @@ def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: fields = {field.label: field.value for field in result.fields} assert fields["Harness"] == "Official scaffold only" + + +def test_benchmark_info_renders_leaderboard_and_runs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + leaderboard = [ + { + "rank": 1, + "tied": False, + "task_set": "parity", + "harness": "codex", + "model": "GPT-5.5", + "pass_at_1": { + "value": 0.75, + "ci_low": 0.719, + "ci_high": 0.781, + "n": 249, + "method": "wilson", + }, + "pass_at_k": [ + {"k": 1, "value": 0.75, "ci_low": 0.719, "ci_high": 0.781, "n": 249}, + {"k": 2, "value": 0.812, "ci_low": 0.77, "ci_high": 0.85, "n": 249}, + ], + "tokens_per_task": 1_100_000, + "mean_duration_seconds": 54, + "reported_cost_usd": 4.2, + "run": { + "id": "run-1", + "name": "warm-gull", + "platform_url": "https://platform.example/Acme/benchmarks/runs/run-1", + }, + }, + # A sparse entrant: every optional metric missing or the wrong type. + {"rank": None, "tied": True, "task_set": "full", "model": "122-test-lora"}, + ] + + class FakeClient: + def get_benchmark(self, *_: Any, **__: Any) -> BenchmarkCatalogDetail: + return BenchmarkCatalogDetail( + id="benchmark-1", + name="HLE", + description=None, + source_type="osmosis_managed", + source_ref="hle", + task_count=2_500, + category_count=30, + task_sets=[], + runner_family="harbor", + supports_harness=True, + requires_harness=True, + requires_judge_model=True, + judge_model_default="openai/gpt-5", + pass_threshold=1, + categories=[], + tasks=[], + unavailable_tasks=None, + leaderboard=leaderboard, + ) + + def list_benchmark_runs(self, **_: Any) -> PaginatedBenchmarkRuns: + return PaginatedBenchmarkRuns( + benchmark_runs=[ + BenchmarkRun.from_dict( + { + "id": "run-1", + "name": "warm-gull", + "status": "finished", + "benchmark": {"id": "benchmark-1", "name": "HLE"}, + "agent_count": 1, + "best_pass_at_1": 0.75, + "ingested_results": 498, + "expected_results": 498, + "created_at": "2026-08-01T00:00:00Z", + } + ) + ], + total_count=12, + has_more=True, + next_offset=10, + ) + + monkeypatch.setattr( + benchmark_module, + "require_git_workspace_directory_context", + _context, + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.benchmark_info("hle", limit=DEFAULT_PAGE_SIZE, all_=False) + + assert result.data["leaderboard"] == leaderboard + assert result.data["runs_total_count"] == 12 + assert result.data["runs"][0]["name"] == "warm-gull" + fields = {field.label: field.value for field in result.fields} + assert fields["Runs"] == "12" + assert len(result.sections) == 2 + assert result.sections[0].plain_lines[0] == "Leaderboard:" + assert any("GPT-5.5 (codex)" in line for line in result.sections[0].plain_lines) + assert result.sections[1].plain_lines[0] == "Runs (1 of 12):" + assert not any( + "No leaderboard entries yet" in hint for hint in result.display_hints + ) + + +def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: + full, sparse = benchmark_module._leaderboard_rows( + [ + { + "rank": 1, + "tied": False, + "task_set": "parity", + "harness": "codex", + "model": "GPT-5.5", + "pass_at_1": {"value": 0.75, "ci_low": 0.719, "ci_high": 0.781}, + "pass_at_k": [ + {"k": 2, "value": 0.812, "ci_low": 0.77, "ci_high": 0.85} + ], + "reported_cost_usd": 4.2, + "mean_duration_seconds": 54, + "run": {"name": "warm-gull"}, + }, + {"rank": "not-a-rank", "tied": True, "task_set": "full"}, + ] + ) + + label, value = full + assert label == "#1" + assert "GPT-5.5 (codex) [parity]" in value + assert "pass@1 75.0% ± 3.1%" in value + assert "pass@2 81.2%" in value + assert "$4.20" in value + assert "54s/task" in value + assert "run warm-gull" in value + + label, value = sparse + assert label == "–" + assert value.startswith("– [tied]") + assert "pass@1 –" in value + + +def test_catalog_status_prefers_activity_over_sync_state() -> None: + def entry(**overrides: Any) -> BenchmarkCatalogEntry: + return BenchmarkCatalogEntry( + id="benchmark-1", + name="HLE", + description=None, + source_type="osmosis_managed", + source_ref="hle", + task_count=1, + category_count=1, + task_sets=[], + **overrides, + ) + + assert benchmark_module._catalog_status(entry(running_count=2)) == "Running (2)" + assert ( + benchmark_module._catalog_status(entry(running_count=1, sync_status="failed")) + == "Running (1)" + ) + assert benchmark_module._catalog_status(entry(sync_status="pending")) == "Syncing" + assert benchmark_module._catalog_status(entry(sync_status="syncing")) == "Syncing" + assert ( + benchmark_module._catalog_status(entry(sync_status="failed")) == "Sync failed" + ) + assert benchmark_module._catalog_status(entry()) == "Ready" + + +def test_benchmark_run_rows_render_status_progress_and_date() -> None: + run = BenchmarkRun.from_dict( + { + "id": "run-1", + "name": "warm-gull", + "status": "finished", + "benchmark": {"id": "benchmark-1", "name": "HLE"}, + "best_pass_at_1": 0.75, + "ingested_results": 498, + "expected_results": 498, + "created_at": "2026-08-01T00:00:00Z", + } + ) + + [(label, value)] = benchmark_module._benchmark_run_rows([run]) + + assert label == "warm-gull" + assert "best pass@1 75.0%" in value + assert "498" in value From 6d985b45ad6dcb305337c4176e029d1088d796d0 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 18:13:36 +0000 Subject: [PATCH 17/22] trim catalog wording --- CONTRIBUTING.md | 2 +- docs/cli.md | 10 ++++----- osmosis_ai/platform/cli/benchmark.py | 15 ++++++++----- osmosis_ai/platform/cli/benchmark_config.py | 4 ---- .../platform/cli/test_benchmark_catalog.py | 21 ++++++++++++++----- .../platform/cli/test_benchmark_config.py | 2 +- .../platform/cli/test_benchmark_submit.py | 2 +- 7 files changed, 34 insertions(+), 22 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1beaabe4..c3b297e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ Coverage configuration is in `pyproject.toml` under `[tool.coverage.*]`. CI enfo When changing remote run commands, preserve the naming convention: bare verbs act on a group's primary noun. `train` and `eval` manage runs with top-level `submit`, `list`, `info`, `logs`, and `stop` because the run is -their noun; `benchmark list|info` act on benchmarks themselves (catalog and +their noun; `benchmark list|info` act on benchmarks themselves (workspace list and benchmark page), with run lifecycle nested under `osmosis benchmark runs list|info|logs|stop|download`. Eval and benchmark downloads share the manifest transfer engine in `osmosis_ai/platform/cli/run_download.py`; diff --git a/docs/cli.md b/docs/cli.md index 902d7cd1..bb5bb022 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,12 +20,12 @@ Files in [../osmosis_ai/cli/commands/](../osmosis_ai/cli/commands/) are thin Typ Module-level imports in `commands/` are kept light: `typer`, `cli.console`, `cli.errors`, the lightweight `osmosis_ai.platform.constants` (pagination limits), and stdlib. Everything heavy (`rollout.*`, `platform.api.*`, `platform.cli.*`, `eval.*`) must be imported **inside the function** to keep CLI startup fast — see the lazy-loading section of [architecture.md](./architecture.md). `osmosis benchmark` puts the benchmark first, mirroring the platform's -Benchmarks pages: top-level `list` and `info` act on benchmarks (the catalog -and one benchmark's page - `info` shows its metadata, leaderboard, and runs), -`submit` starts a run, and run lifecycle lives under the nested `benchmark -runs list|info|logs|stop|download` namespace. +Benchmarks pages: top-level `list` and `info` act on benchmarks (the workspace +list and one benchmark's page - `info` shows its metadata, leaderboard, and +runs), `submit` starts a run, and run lifecycle lives under the nested +`benchmark runs list|info|logs|stop|download` namespace. -Catalog output includes a shell-safe benchmark `Key`, such as +List output includes a shell-safe benchmark `Key`, such as `terminal-bench-2-1`. Pass that key to `osmosis benchmark info `; exact display names and UUIDs remain supported for compatibility. diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 594ba398..97ee942e 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -230,10 +230,15 @@ def _format_rate_interval(rate: Any) -> str: return "–" value = float(rate["value"]) low, high = rate.get("ci_low"), rate.get("ci_high") + sample = rate.get("n") + decimals = 1 if isinstance(sample, int | float) and sample >= 100 else 0 + + def pct(fraction: float) -> str: + return f"{fraction * 100:.{decimals}f}" + if isinstance(low, int | float) and isinstance(high, int | float): - margin = max(value - low, high - value) - return f"{value:.1%} ± {margin:.1%}" - return f"{value:.1%}" + return f"{pct(value)}% ({pct(float(low))}–{pct(float(high))})" + return f"{pct(value)}%" def _leaderboard_rows(entries: list[dict[str, Any]]) -> list[tuple[str, str]]: @@ -384,8 +389,8 @@ def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: if not benchmark.leaderboard: display_hints.insert( 0, - "No leaderboard entries yet: finished runs on the full task set " - "with scores rank here.", + "No eligible benchmark runs. Finished benchmark runs on the full " + "task set with scores will appear here.", ) if benchmark.default_harness: display_hints.insert( diff --git a/osmosis_ai/platform/cli/benchmark_config.py b/osmosis_ai/platform/cli/benchmark_config.py index cd3f9943..a1ef14ee 100644 --- a/osmosis_ai/platform/cli/benchmark_config.py +++ b/osmosis_ai/platform/cli/benchmark_config.py @@ -149,10 +149,6 @@ def required_secrets(self) -> list[str]: judge_secret = self.execution.judge_api_key_secret if isinstance(judge_secret, str): names.append(judge_secret) - # HLE's managed adapter reads the dataset through a fixed Platform - # secret, even though the name is not repeated in the submit config. - if self.experiment.benchmark.strip() == "HLE": - names.append("HF_TOKEN") return list(dict.fromkeys(names)) diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index b12de6bc..142be16a 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -200,7 +200,7 @@ def get_benchmark( assert fields["Key"] == "hle" assert fields["Required Secret Records"] == "HF_TOKEN" assert 'task_set = "parity"' in result.display_hints[0] - assert "No leaderboard entries yet" in result.display_hints[1] + assert "No eligible benchmark runs" in result.display_hints[1] assert any("Omit [tasks]" in hint for hint in result.display_hints) assert calls == [ { @@ -536,7 +536,7 @@ def list_benchmark_runs(self, **_: Any) -> PaginatedBenchmarkRuns: assert any("GPT-5.5 (codex)" in line for line in result.sections[0].plain_lines) assert result.sections[1].plain_lines[0] == "Runs (1 of 12):" assert not any( - "No leaderboard entries yet" in hint for hint in result.display_hints + "No eligible benchmark runs" in hint for hint in result.display_hints ) @@ -549,9 +549,20 @@ def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: "task_set": "parity", "harness": "codex", "model": "GPT-5.5", - "pass_at_1": {"value": 0.75, "ci_low": 0.719, "ci_high": 0.781}, + "pass_at_1": { + "value": 0.75, + "ci_low": 0.719, + "ci_high": 0.781, + "n": 249, + }, "pass_at_k": [ - {"k": 2, "value": 0.812, "ci_low": 0.77, "ci_high": 0.85} + { + "k": 2, + "value": 0.812, + "ci_low": 0.77, + "ci_high": 0.85, + "n": 249, + } ], "reported_cost_usd": 4.2, "mean_duration_seconds": 54, @@ -564,7 +575,7 @@ def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: label, value = full assert label == "#1" assert "GPT-5.5 (codex) [parity]" in value - assert "pass@1 75.0% ± 3.1%" in value + assert "pass@1 75.0% (71.9–78.1)" in value assert "pass@2 81.2%" in value assert "$4.20" in value assert "54s/task" in value diff --git a/tests/unit/platform/cli/test_benchmark_config.py b/tests/unit/platform/cli/test_benchmark_config.py index 095b2d07..75bfaef7 100644 --- a/tests/unit/platform/cli/test_benchmark_config.py +++ b/tests/unit/platform/cli/test_benchmark_config.py @@ -202,7 +202,7 @@ def test_load_benchmark_submit_config_accepts_hle_parity_with_explicit_filters( "task_names": ["hle__sample"], "task_set": "parity", } - assert config.required_secrets == ["OPENAI_API_KEY", "HF_TOKEN"] + assert config.required_secrets == ["OPENAI_API_KEY"] def test_load_benchmark_submit_config_rejects_unknown_task_set( diff --git a/tests/unit/platform/cli/test_benchmark_submit.py b/tests/unit/platform/cli/test_benchmark_submit.py index 0575b2e1..39c71539 100644 --- a/tests/unit/platform/cli/test_benchmark_submit.py +++ b/tests/unit/platform/cli/test_benchmark_submit.py @@ -283,7 +283,7 @@ def test_submit_warns_before_hle_missing_secret_failure( lambda message, **kwargs: warnings.append({"message": message, **kwargs}), ) - with pytest.raises(CLIError, match=r"HF_TOKEN"): + with pytest.raises(CLIError, match=r"OPENAI_API_KEY"): benchmark_module.submit(config_path, yes=True) assert warnings == [ From b2591bafa68fe339d2437fb9f37c7f185c764a0d Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 19:05:11 +0000 Subject: [PATCH 18/22] [cli] feat: surface run history and per-agent metrics in benchmark output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the benchmarks table Status column with a Last Run cell that shows sync progress until tasks land, then the newest run’s status, age, and name. Add per-agent rank, cost/task, and tokens/task to run info alongside a Duration field, and report leaderboard cost per task instead of run totals. Co-authored-by: Cursor --- osmosis_ai/cli/output/display.py | 42 +++++ osmosis_ai/platform/api/models.py | 6 + osmosis_ai/platform/cli/benchmark.py | 172 +++++++++++++----- osmosis_ai/platform/cli/utils.py | 10 + tests/unit/cli/output/test_display.py | 36 +++- .../platform/cli/test_benchmark_catalog.py | 95 ++++++---- .../unit/platform/cli/test_benchmark_runs.py | 79 ++++++++ 7 files changed, 366 insertions(+), 74 deletions(-) diff --git a/osmosis_ai/cli/output/display.py b/osmosis_ai/cli/output/display.py index a3ac3722..0ffff39f 100644 --- a/osmosis_ai/cli/output/display.py +++ b/osmosis_ai/cli/output/display.py @@ -57,6 +57,48 @@ def format_local_datetime( ) +def format_elapsed( + started_at: str | None, + completed_at: str | None = None, + *, + now: datetime | None = None, +) -> str | None: + """Wall-clock span from *started_at*, to *completed_at* or to now.""" + started = _parse_iso_datetime(started_at) + if started is None: + return None + finished = _parse_iso_datetime(completed_at) + if finished is None: + finished = now.astimezone() if now is not None else datetime.now(started.tzinfo) + return format_duration_ms(max(0.0, (finished - started).total_seconds()) * 1000) + + +def format_relative_time(value: str | None, *, now: datetime | None = None) -> str: + """Compact age for table cells (e.g. ``2d ago``).""" + parsed = _parse_iso_datetime(value) + if parsed is None: + return "" if value is None else str(value) + reference = now.astimezone() if now is not None else datetime.now(parsed.tzinfo) + minutes = max(0.0, (reference - parsed).total_seconds()) / 60 + if minutes < 1: + return "just now" + if minutes < 60: + return f"{int(minutes)}m ago" + hours = minutes / 60 + if hours < 24: + return f"{int(hours)}h ago" + days = hours / 24 + if days < 7: + return f"{int(days)}d ago" + weeks = days / 7 + if weeks < 5: + return f"{int(weeks)}w ago" + months = days / 30.44 + if months < 12: + return f"{max(1, int(months))}mo ago" + return f"{max(1, round(months / 12))}y ago" + + def format_duration_ms(duration_ms: float) -> str: """Human-readable duration from milliseconds (e.g. ``2h 47m``).""" duration_ms = max(0.0, duration_ms) diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index 84bfffd7..b64357c2 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -453,6 +453,9 @@ class BenchmarkCatalogEntry: run_count: int = 0 running_count: int = 0 last_run_at: str | None = None + last_run_status: str | None = None + last_run_name: str | None = None + creator_name: str | None = None @property def is_ready(self) -> bool: @@ -479,6 +482,9 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogEntry: run_count=int(data.get("run_count") or 0), running_count=int(data.get("running_count") or 0), last_run_at=data.get("last_run_at"), + last_run_status=data.get("last_run_status"), + last_run_name=data.get("last_run_name"), + creator_name=data.get("creator_name"), ) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 97ee942e..dd96406f 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -17,7 +17,12 @@ get_output_context, serialize_benchmark_run, ) -from osmosis_ai.cli.output.display import format_local_date, format_local_datetime +from osmosis_ai.cli.output.display import ( + format_elapsed, + format_local_date, + format_local_datetime, + format_relative_time, +) from osmosis_ai.cli.prompts import require_confirmation from osmosis_ai.platform.api.client import OsmosisClient from osmosis_ai.platform.api.models import ( @@ -48,6 +53,7 @@ from osmosis_ai.platform.cli.utils import ( build_logs_result, format_benchmark_status, + format_benchmark_status_label, format_env_config, format_progress, format_secret_scopes, @@ -73,12 +79,18 @@ _BENCHMARK_COLUMNS = [ ListColumn(key="name", label="Name", ratio=3, overflow="fold"), ListColumn(key="key", label="Key", no_wrap=True, min_width=20), - ListColumn(key="status", label="Status", no_wrap=True, ratio=1), - ListColumn(key="run_count", label="Runs", no_wrap=True, ratio=1), - ListColumn(key="last_run_at", label="Last Run", no_wrap=True, ratio=1), + ListColumn(key="last_run", label="Last Run", ratio=3, overflow="fold"), ListColumn(key="task_count", label="Tasks", no_wrap=True, ratio=1), + ListColumn(key="run_count", label="Runs", no_wrap=True, ratio=1), + ListColumn(key="creator_name", label="Added By", no_wrap=True, ratio=1), ] +_SYNC_STATUS_DISPLAY: dict[str, tuple[str, str]] = { + "pending": ("Queued", "orange3"), + "syncing": ("Syncing", "blue"), + "failed": ("Failed", "red"), +} + def _task_set_resource(task_set: BenchmarkTaskSet) -> dict[str, Any]: return { @@ -109,15 +121,34 @@ def _benchmark_resource( } -def _catalog_status(benchmark: BenchmarkCatalogEntry) -> str: - """Activity beats sync state; a quiet benchmark is Ready, never blank.""" - if benchmark.running_count > 0: - return f"Running ({benchmark.running_count})" - if benchmark.sync_status in ("pending", "syncing"): - return "Syncing" +def _sync_detail(benchmark: BenchmarkCatalogEntry) -> str: if benchmark.sync_status == "failed": - return "Sync failed" - return "Ready" + return benchmark.sync_error or "Failed to sync tasks" + if benchmark.sync_status == "pending": + return "Waiting to start" + if benchmark.task_count == 0: + return "Starting" + synced = min(benchmark.synced_task_count, benchmark.task_count) + return f"{synced:,} / {benchmark.task_count:,} tasks" + + +def _last_run_cell(benchmark: BenchmarkCatalogEntry) -> str: + """Sync state until the task list lands, then the newest run's state.""" + if not benchmark.is_ready: + label, style = _SYNC_STATUS_DISPLAY.get( + benchmark.sync_status, (benchmark.sync_status.title(), "") + ) + styled = console.format_styled(label, style) if style else label + return f"{styled} · {console.escape(_sync_detail(benchmark))}" + if not benchmark.last_run_at or not benchmark.last_run_status: + return "No benchmark runs yet" + parts = [ + format_benchmark_status_label(benchmark.last_run_status), + format_relative_time(benchmark.last_run_at), + ] + if benchmark.last_run_name: + parts.append(console.escape(benchmark.last_run_name)) + return " · ".join(parts) def _benchmark_list_resource(benchmark: BenchmarkCatalogEntry) -> dict[str, Any]: @@ -126,6 +157,9 @@ def _benchmark_list_resource(benchmark: BenchmarkCatalogEntry) -> dict[str, Any] "run_count": benchmark.run_count, "running_count": benchmark.running_count, "last_run_at": benchmark.last_run_at, + "last_run_status": benchmark.last_run_status, + "last_run_name": benchmark.last_run_name, + "creator_name": benchmark.creator_name, } @@ -140,6 +174,13 @@ def _task_count_display( return f"{benchmark.synced_task_count:,} / {benchmark.task_count:,} syncing" +def _list_task_count_display(benchmark: BenchmarkCatalogEntry) -> str: + """Mid-sync rows leave Tasks blank; the Last Run cell carries the progress.""" + if benchmark.is_ready or benchmark.sync_status == "failed": + return _task_count_display(benchmark) + return "–" + + def _sync_hints( benchmark: BenchmarkCatalogEntry | BenchmarkCatalogDetail, ) -> list[str]: @@ -201,19 +242,10 @@ def list_benchmarks(*, limit: int, all_: bool) -> ListResult: display_items=[ { **_benchmark_list_resource(benchmark), - "status": _catalog_status(benchmark), + "last_run": _last_run_cell(benchmark), + "task_count": _list_task_count_display(benchmark), "run_count": f"{benchmark.run_count:,}", - "last_run_at": ( - format_local_date(benchmark.last_run_at) - if benchmark.last_run_at - else "–" - ), - "task_count": _task_count_display(benchmark), - "source": ( - "Managed" - if benchmark.source_type == "osmosis_managed" - else "Harbor" - ), + "creator_name": benchmark.creator_name or "–", } for benchmark in benchmarks ], @@ -241,6 +273,35 @@ def pct(fraction: float) -> str: return f"{pct(value)}%" +def _format_tokens(value: Any) -> str | None: + if not isinstance(value, int | float) or isinstance(value, bool): + return None + if value >= 1_000_000: + return f"{value / 1_000_000:.1f}M tokens" + if value >= 1_000: + return f"{value / 1_000:.1f}k tokens" + return f"{round(value):,} tokens" + + +def _metric_parts(entry: dict[str, Any]) -> list[str]: + """The platform's metric set, in the order its leaderboard ranks by.""" + parts = [f"pass@1 {_format_rate_interval(entry.get('pass_at_1'))}"] + points = entry.get("pass_at_k") or [] + deepest = points[-1] if points else None + if isinstance(deepest, dict) and deepest.get("value") is not None: + parts.append(f"pass@{deepest.get('k')} {float(deepest['value']):.1%}") + cost = entry.get("cost_per_task") + if isinstance(cost, int | float): + parts.append(f"${cost:,.2f}/task") + seconds = entry.get("mean_duration_seconds") + if isinstance(seconds, int | float): + parts.append(f"{seconds:,.0f}s/task") + tokens = _format_tokens(entry.get("tokens_per_task")) + if tokens is not None: + parts.append(f"{tokens}/task") + return parts + + def _leaderboard_rows(entries: list[dict[str, Any]]) -> list[tuple[str, str]]: rows: list[tuple[str, str]] = [] for entry in entries: @@ -252,25 +313,15 @@ def _leaderboard_rows(entries: list[dict[str, Any]]) -> list[tuple[str, str]]: tags = [ tag for tag, present in ( - ("tied", bool(entry.get("tied"))), + ("tied for first", bool(entry.get("tied"))), ("parity", entry.get("task_set") == "parity"), ) if present ] parts = [ name + (f" [{', '.join(tags)}]" if tags else ""), - f"pass@1 {_format_rate_interval(entry.get('pass_at_1'))}", + *_metric_parts(entry), ] - points = entry.get("pass_at_k") or [] - deepest = points[-1] if points else None - if isinstance(deepest, dict) and deepest.get("value") is not None: - parts.append(f"pass@{deepest.get('k')} {float(deepest['value']):.1%}") - cost = entry.get("reported_cost_usd") - if isinstance(cost, int | float): - parts.append(f"${cost:,.2f}") - seconds = entry.get("mean_duration_seconds") - if isinstance(seconds, int | float): - parts.append(f"{seconds:,.0f}s/task") run = entry.get("run") if isinstance(run, dict) and run.get("name"): parts.append(f"run {run['name']}") @@ -489,8 +540,9 @@ def list_benchmark_runs(*, limit: int, all_: bool) -> ListResult: columns=[ ListColumn(key="name", label="Name", ratio=3, overflow="fold"), ListColumn(key="status", label="Status", no_wrap=True, ratio=1), - ListColumn(key="benchmark", label="Benchmark", ratio=2, overflow="fold"), ListColumn(key="progress", label="Progress", no_wrap=True, ratio=2), + ListColumn(key="benchmark", label="Benchmark", ratio=2, overflow="fold"), + ListColumn(key="agent_count", label="Agents", no_wrap=True), ListColumn(key="best_pass_at_1", label="Best Pass@1", no_wrap=True), ListColumn(key="created_at", label="Submitted", no_wrap=True, ratio=1), ListColumn(key="creator_name", label="Submitted By", no_wrap=True), @@ -499,8 +551,9 @@ def list_benchmark_runs(*, limit: int, all_: bool) -> ListResult: { **serialize_benchmark_run(run), "status": format_benchmark_status(run), - "benchmark": run.benchmark_name or "–", "progress": format_progress(_benchmark_progress(run)) or "–", + "benchmark": run.benchmark_name or "–", + "agent_count": f"{run.agent_count:,}", "best_pass_at_1": _format_pass_at_1(run.best_pass_at_1), "created_at": format_local_date(run.created_at), "creator_name": run.creator_name or "–", @@ -544,7 +597,33 @@ def _configuration_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: return rows +def _agent_metric_entry( + agent: dict[str, Any], metrics: dict[str, Any] +) -> dict[str, Any]: + """cloud-eval stores the agent's summed spend; the per-task rate is derived + against the same task count that backs tokens_per_task.""" + aggregates = agent.get("aggregates") or {} + total_cost = aggregates.get("reported_cost_usd") + tasks = metrics.get("n_tasks") + cost_per_task = ( + total_cost / tasks + if isinstance(total_cost, int | float) and isinstance(tasks, int) and tasks > 0 + else None + ) + return { + "pass_at_1": metrics.get("pass_at_1"), + "pass_at_k": metrics.get("pass_at_k") or [], + "cost_per_task": cost_per_task, + "mean_duration_seconds": aggregates.get("mean_duration_seconds"), + "tokens_per_task": aggregates.get("tokens_per_task"), + } + + def _agent_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: + metrics_by_agent = { + metrics.get("benchmark_run_agent_id"): metrics + for metrics in (detail.agent_metrics or []) + } rows: list[tuple[str, str]] = [] for position, agent in enumerate(detail.agents or [], start=1): index = agent.get("agent_index") @@ -558,11 +637,17 @@ def _agent_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: if not isinstance(model, str): model = jsonish(model) harness = agent.get("harness") or "default" + parts = [f"{harness} · {model}"] status = agent.get("status") - value = f"{harness} · {model}" if status: - value += f" · {status}" - rows.append((label, value)) + parts.append(str(status)) + metrics = metrics_by_agent.get(agent.get("id")) + if metrics is not None: + rank = metrics.get("rank") + if isinstance(rank, int): + parts.append(f"#{rank}") + parts.extend(_metric_parts(_agent_metric_entry(agent, metrics))) + rows.append((label, " · ".join(parts))) return rows @@ -579,7 +664,7 @@ def _result_rows(detail: BenchmarkRunDetail) -> list[tuple[str, str]]: for key, label in ( ("total_input_tokens", "Input Tokens"), ("total_output_tokens", "Output Tokens"), - ("total_cost_usd", "Reported Cost"), + ("total_cost_usd", "LLM Cost"), ): value = totals.get(key) if not isinstance(value, int | float) or isinstance(value, bool): @@ -627,6 +712,9 @@ def run_info(name_or_id: str) -> DetailResult: progress_label = format_progress(progress) if progress_label: rows.append(("Progress", progress_label)) + duration = format_elapsed(detail.started_at, detail.completed_at) + if duration: + rows.append(("Duration", duration)) if detail.best_pass_at_1 is not None: rows.append(("Best Pass@1", _format_pass_at_1(detail.best_pass_at_1))) rows.append(("Agents", f"{detail.agent_count:,}")) diff --git a/osmosis_ai/platform/cli/utils.py b/osmosis_ai/platform/cli/utils.py index 6ab9e66f..4643695a 100644 --- a/osmosis_ai/platform/cli/utils.py +++ b/osmosis_ai/platform/cli/utils.py @@ -217,6 +217,16 @@ def format_benchmark_status(run: Any) -> str: return format_status_token(run.status, _BENCHMARK_STATUS_STYLES) +def format_benchmark_status_label(status: str) -> str: + """Status as a plain styled word: the benchmarks table reads state as text + alongside its detail, not as a ``[token]``.""" + text = status.replace("_", " ").title() + for statuses, style in _BENCHMARK_STATUS_STYLES: + if status in statuses: + return console.format_styled(text, style) + return console.escape(text) + + def format_reward(reward: float | None) -> str: """Format a training reward to two decimals, en dash when unset.""" if reward is None: diff --git a/tests/unit/cli/output/test_display.py b/tests/unit/cli/output/test_display.py index 59f2c216..4a35fec3 100644 --- a/tests/unit/cli/output/test_display.py +++ b/tests/unit/cli/output/test_display.py @@ -1,16 +1,20 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta, timezone from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import pytest from osmosis_ai.cli.output.display import ( format_duration_ms, + format_elapsed, format_local_date, format_local_datetime, + format_relative_time, ) +NOW = datetime(2026, 8, 4, 12, 0, tzinfo=UTC) + def test_format_local_date_uses_explicit_timezone() -> None: formatted = format_local_date("2026-05-13T12:34:56Z", tz=ZoneInfo("UTC")) @@ -120,3 +124,33 @@ def test_format_local_date_uses_compact_fallback_for_offsetless_input() -> None: ) def test_format_duration_ms(duration_ms: float, expected: str) -> None: assert format_duration_ms(duration_ms) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("2026-08-04T11:59:30Z", "just now"), + ("2026-08-04T11:35:00Z", "25m ago"), + ("2026-08-04T07:00:00Z", "5h ago"), + ("2026-08-02T12:00:00Z", "2d ago"), + ("2026-07-14T12:00:00Z", "3w ago"), + ("2026-05-04T12:00:00Z", "3mo ago"), + ("2024-08-04T12:00:00Z", "2y ago"), + ], +) +def test_format_relative_time(value: str, expected: str) -> None: + assert format_relative_time(value, now=NOW) == expected + + +def test_format_relative_time_falls_back_for_invalid_input() -> None: + assert format_relative_time("not-a-date", now=NOW) == "not-a-date" + assert format_relative_time(None, now=NOW) == "" + + +def test_format_elapsed_measures_to_completion_or_now() -> None: + assert ( + format_elapsed("2026-08-04T09:15:00Z", "2026-08-04T11:00:00Z", now=NOW) + == "1h 45m" + ) + assert format_elapsed("2026-08-04T11:00:00Z", None, now=NOW) == "1h" + assert format_elapsed(None, "2026-08-04T11:00:00Z", now=NOW) is None diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index 142be16a..73724eb7 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -68,6 +68,11 @@ def list_benchmarks(self, **kwargs: Any) -> PaginatedBenchmarks: task_count=2_500, category_count=30, task_sets=[parity], + run_count=14, + last_run_at="2026-08-01T00:00:00Z", + last_run_status="finished", + last_run_name="brave-otter", + creator_name="Brian", ) ], total_count=1, @@ -97,16 +102,23 @@ def list_benchmarks(self, **kwargs: Any) -> PaginatedBenchmarks: } ] assert result.display_items[0]["key"] == "hle" - assert result.display_items[0]["status"] == "Ready" - assert result.display_items[0]["run_count"] == "0" - assert result.display_items[0]["last_run_at"] == "–" + assert result.display_items[0]["run_count"] == "14" + assert result.display_items[0]["task_count"] == "2,500" + assert result.display_items[0]["creator_name"] == "Brian" + last_run = result.display_items[0]["last_run"] + assert "Finished" in last_run + assert "ago" in last_run + assert last_run.endswith("brave-otter") + assert result.items[0]["last_run_status"] == "finished" + assert result.items[0]["last_run_name"] == "brave-otter" + assert result.items[0]["creator_name"] == "Brian" assert [column.key for column in result.columns] == [ "name", "key", - "status", - "run_count", - "last_run_at", + "last_run", "task_count", + "run_count", + "creator_name", ] assert result.columns[1].no_wrap is True assert result.columns[1].min_width == 20 @@ -279,7 +291,9 @@ def test_list_benchmarks_shows_registry_sync_progress( _syncing_entry(sync_status="syncing", synced_task_count=1_000), ) - assert result.display_items[0]["task_count"] == "1,000 / 4,000 syncing" + assert result.display_items[0]["task_count"] == "–" + assert "1,000 / 4,000 tasks" in result.display_items[0]["last_run"] + assert "Syncing" in result.display_items[0]["last_run"] assert result.items[0]["sync_status"] == "syncing" assert any("still syncing" in hint for hint in result.display_hints) @@ -296,6 +310,7 @@ def test_list_benchmarks_surfaces_failed_sync_and_platform_url( ) assert result.display_items[0]["task_count"] == "unavailable" + assert "Task list unavailable." in result.display_items[0]["last_run"] assert result.items[0]["sync_error"] == "Task list unavailable." assert any( "Task list unavailable." in hint @@ -564,8 +579,9 @@ def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: "n": 249, } ], - "reported_cost_usd": 4.2, + "cost_per_task": 4.2, "mean_duration_seconds": 54, + "tokens_per_task": 1_100_000, "run": {"name": "warm-gull"}, }, {"rank": "not-a-rank", "tied": True, "task_set": "full"}, @@ -577,41 +593,58 @@ def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: assert "GPT-5.5 (codex) [parity]" in value assert "pass@1 75.0% (71.9–78.1)" in value assert "pass@2 81.2%" in value - assert "$4.20" in value + assert "$4.20/task" in value assert "54s/task" in value + assert "1.1M tokens/task" in value assert "run warm-gull" in value label, value = sparse assert label == "–" - assert value.startswith("– [tied]") + assert value.startswith("– [tied for first]") assert "pass@1 –" in value -def test_catalog_status_prefers_activity_over_sync_state() -> None: +def test_last_run_cell_shows_sync_state_before_run_history() -> None: def entry(**overrides: Any) -> BenchmarkCatalogEntry: - return BenchmarkCatalogEntry( - id="benchmark-1", - name="HLE", - description=None, - source_type="osmosis_managed", - source_ref="hle", - task_count=1, - category_count=1, - task_sets=[], - **overrides, - ) + fields: dict[str, Any] = { + "id": "benchmark-1", + "name": "HLE", + "description": None, + "source_type": "osmosis_managed", + "source_ref": "hle", + "task_count": 1, + "category_count": 1, + "task_sets": [], + } + return BenchmarkCatalogEntry(**{**fields, **overrides}) + + syncing = benchmark_module._last_run_cell( + entry(sync_status="syncing", task_count=4_000, synced_task_count=1_000) + ) + assert "Syncing" in syncing + assert "1,000 / 4,000 tasks" in syncing + + queued = benchmark_module._last_run_cell(entry(sync_status="pending")) + assert "Queued" in queued + assert "Waiting to start" in queued - assert benchmark_module._catalog_status(entry(running_count=2)) == "Running (2)" - assert ( - benchmark_module._catalog_status(entry(running_count=1, sync_status="failed")) - == "Running (1)" + failed = benchmark_module._last_run_cell( + entry(sync_status="failed", sync_error="Registry unreachable.") ) - assert benchmark_module._catalog_status(entry(sync_status="pending")) == "Syncing" - assert benchmark_module._catalog_status(entry(sync_status="syncing")) == "Syncing" - assert ( - benchmark_module._catalog_status(entry(sync_status="failed")) == "Sync failed" + assert "Failed" in failed + assert "Registry unreachable." in failed + + assert benchmark_module._last_run_cell(entry()) == "No benchmark runs yet" + + ran = benchmark_module._last_run_cell( + entry( + last_run_at="2026-08-01T00:00:00Z", + last_run_status="running", + last_run_name="calm-yak", + ) ) - assert benchmark_module._catalog_status(entry()) == "Ready" + assert "Running" in ran + assert ran.endswith("calm-yak") def test_benchmark_run_rows_render_status_progress_and_date() -> None: diff --git a/tests/unit/platform/cli/test_benchmark_runs.py b/tests/unit/platform/cli/test_benchmark_runs.py index 0e6c9a85..fdd205ea 100644 --- a/tests/unit/platform/cli/test_benchmark_runs.py +++ b/tests/unit/platform/cli/test_benchmark_runs.py @@ -122,6 +122,17 @@ def list_benchmark_runs(self, **kwargs: Any) -> PaginatedBenchmarkRuns: } assert result.display_items[0]["progress"] == "50 / 100 results" assert result.display_items[0]["best_pass_at_1"] == "42.0%" + assert result.display_items[0]["agent_count"] == "2" + assert [column.key for column in result.columns] == [ + "name", + "status", + "progress", + "benchmark", + "agent_count", + "best_pass_at_1", + "created_at", + "creator_name", + ] assert calls == [ { "limit": 50, @@ -195,6 +206,74 @@ def get_benchmark_run( assert "osmosis benchmark runs download hle-smoke" in result.display_hints[-1] +def test_run_info_reports_duration_and_per_agent_metrics( + monkeypatch: pytest.MonkeyPatch, +) -> None: + detail = _detail(status="finished") + detail.started_at = "2026-07-30T00:05:00Z" + detail.completed_at = "2026-07-30T01:35:00Z" + detail.agents = [ + { + "id": "agent-1", + "agent_index": 0, + "harness": "codex", + "model_display_name": "GPT-5", + "status": "finished", + "aggregates": { + "reported_cost_usd": 498.0, + "mean_duration_seconds": 54, + "tokens_per_task": 1_100_000, + }, + } + ] + detail.agent_metrics = [ + { + "benchmark_run_agent_id": "agent-1", + "rank": 1, + "n_tasks": 249, + "pass_at_1": { + "value": 0.75, + "ci_low": 0.719, + "ci_high": 0.781, + "n": 249, + }, + "pass_at_k": [ + {"k": 2, "value": 0.812, "ci_low": 0.77, "ci_high": 0.85, "n": 249} + ], + } + ] + + class FakeClient: + def get_benchmark_run(self, *_: Any, **__: Any) -> BenchmarkRunDetail: + return detail + + monkeypatch.setattr( + benchmark_module, "require_git_workspace_directory_context", _context + ) + monkeypatch.setattr(benchmark_module, "OsmosisClient", FakeClient) + + result = benchmark_module.run_info("hle-smoke") + + fields = {field.label: field.value for field in result.fields} + assert fields["Duration"] == "1h 30m" + + agents = next( + section for section in result.sections if section.plain_lines[0] == "Agents:" + ) + agent_line = " ".join(agents.plain_lines) + assert "#1" in agent_line + assert "pass@1 75.0% (71.9–78.1)" in agent_line + assert "pass@2 81.2%" in agent_line + assert "$2.00/task" in agent_line + assert "54s/task" in agent_line + assert "1.1M tokens/task" in agent_line + + results = next( + section for section in result.sections if section.plain_lines[0] == "Results:" + ) + assert any("LLM Cost" in line for line in results.plain_lines) + + def test_run_info_always_displays_canonical_id( monkeypatch: pytest.MonkeyPatch, ) -> None: From 6efe0c59e84c3a2314307590c9b4ce476a01306e Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 22:17:30 +0000 Subject: [PATCH 19/22] fix source url and misc changes --- osmosis_ai/platform/api/models.py | 4 ++++ osmosis_ai/platform/cli/benchmark.py | 7 ++++--- tests/unit/platform/cli/test_benchmark_catalog.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/osmosis_ai/platform/api/models.py b/osmosis_ai/platform/api/models.py index b64357c2..0f8b81f2 100644 --- a/osmosis_ai/platform/api/models.py +++ b/osmosis_ai/platform/api/models.py @@ -446,6 +446,7 @@ class BenchmarkCatalogEntry: task_count: int category_count: int task_sets: list[BenchmarkTaskSet] + source_url: str | None = None sync_status: str = "ready" synced_task_count: int = 0 sync_error: str | None = None @@ -470,6 +471,7 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogEntry: description=data.get("description"), source_type=data["source_type"], source_ref=data["source_ref"], + source_url=data.get("source_url"), task_count=data["task_count"], category_count=data["category_count"], task_sets=[ @@ -533,6 +535,7 @@ class BenchmarkCatalogDetail: unavailable_tasks: dict[str, Any] | None required_secret_names: list[str] = field(default_factory=list) default_harness: str | None = None + source_url: str | None = None sync_status: str = "ready" synced_task_count: int = 0 sync_error: str | None = None @@ -555,6 +558,7 @@ def from_dict(cls, data: dict[str, Any]) -> BenchmarkCatalogDetail: description=benchmark.get("description"), source_type=benchmark["source_type"], source_ref=benchmark["source_ref"], + source_url=benchmark.get("source_url"), task_count=benchmark["task_count"], category_count=benchmark["category_count"], task_sets=[ diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index dd96406f..552caa4f 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -111,6 +111,7 @@ def _benchmark_resource( "description": benchmark.description, "source_type": benchmark.source_type, "source_ref": benchmark.source_ref, + "source_url": benchmark.source_url, "task_count": benchmark.task_count, "category_count": benchmark.category_count, "task_sets": [_task_set_resource(task_set) for task_set in benchmark.task_sets], @@ -383,7 +384,7 @@ def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: harness += f" (default: {benchmark.default_harness})" elif benchmark.supports_harness and not benchmark.requires_harness: harness += " (default: official scaffold)" - judge = "Not required" + judge = "–" if benchmark.requires_judge_model: judge = "Required" if benchmark.judge_model_default: @@ -397,7 +398,7 @@ def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: ("Name", console.escape(benchmark.name)), ("Key", console.escape(benchmark.source_ref)), ("Description", console.escape(benchmark.description or "–")), - ("Source", f"{benchmark.source_type}: {benchmark.source_ref}"), + ("Source", benchmark.source_url or benchmark.source_ref), ("Runner", benchmark.runner_family), ("Tasks", _task_count_display(benchmark)), ("Categories", category_display or "–"), @@ -405,7 +406,7 @@ def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: ("Harness", harness), ("LLM Judge", judge), ( - "Required Secret Records", + "Required Secrets", ", ".join(benchmark.required_secret_names) or "–", ), ("Pass Threshold", f"{benchmark.pass_threshold:g}"), diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index 73724eb7..b81464bd 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -210,7 +210,7 @@ def get_benchmark( assert result.data["benchmark"]["required_secret_names"] == ["HF_TOKEN"] fields = {field.label: field.value for field in result.fields} assert fields["Key"] == "hle" - assert fields["Required Secret Records"] == "HF_TOKEN" + assert fields["Required Secrets"] == "HF_TOKEN" assert 'task_set = "parity"' in result.display_hints[0] assert "No eligible benchmark runs" in result.display_hints[1] assert any("Omit [tasks]" in hint for hint in result.display_hints) From de69156796d3fc500010887d3b25957e9b8f3ae1 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 23:07:00 +0000 Subject: [PATCH 20/22] fix benchmark info tables --- osmosis_ai/platform/cli/benchmark.py | 255 ++++++++++++++---- .../platform/cli/test_benchmark_catalog.py | 57 ++-- 2 files changed, 236 insertions(+), 76 deletions(-) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 552caa4f..61b62dba 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -278,73 +278,215 @@ def _format_tokens(value: Any) -> str | None: if not isinstance(value, int | float) or isinstance(value, bool): return None if value >= 1_000_000: - return f"{value / 1_000_000:.1f}M tokens" + return f"{value / 1_000_000:.1f}M" if value >= 1_000: - return f"{value / 1_000:.1f}k tokens" - return f"{round(value):,} tokens" + return f"{value / 1_000:.1f}k" + return f"{round(value):,}" + + +def _agent_label(entry: dict[str, Any]) -> str: + model = str(entry.get("model") or "–") + harness = entry.get("harness") + name = f"{model} ({harness})" if harness else model + tags = [ + tag + for tag, present in ( + ("tied for first", bool(entry.get("tied"))), + ("parity", entry.get("task_set") == "parity"), + ) + if present + ] + return name + (f" [{', '.join(tags)}]" if tags else "") -def _metric_parts(entry: dict[str, Any]) -> list[str]: - """The platform's metric set, in the order its leaderboard ranks by.""" - parts = [f"pass@1 {_format_rate_interval(entry.get('pass_at_1'))}"] +def _deepest_pass_at_k(entry: dict[str, Any]) -> dict[str, Any] | None: points = entry.get("pass_at_k") or [] deepest = points[-1] if points else None if isinstance(deepest, dict) and deepest.get("value") is not None: - parts.append(f"pass@{deepest.get('k')} {float(deepest['value']):.1%}") - cost = entry.get("cost_per_task") - if isinstance(cost, int | float): - parts.append(f"${cost:,.2f}/task") - seconds = entry.get("mean_duration_seconds") - if isinstance(seconds, int | float): - parts.append(f"{seconds:,.0f}s/task") - tokens = _format_tokens(entry.get("tokens_per_task")) - if tokens is not None: - parts.append(f"{tokens}/task") - return parts - - -def _leaderboard_rows(entries: list[dict[str, Any]]) -> list[tuple[str, str]]: - rows: list[tuple[str, str]] = [] + return deepest + return None + + +def _format_pass_at_k(point: dict[str, Any] | None) -> str: + if point is None or point.get("value") is None: + return "–" + return f"{float(point['value']):.1%}" + + +def _format_cost_per_task(cost: Any) -> str: + if not isinstance(cost, int | float) or isinstance(cost, bool): + return "–" + return f"${cost:,.2f}" + + +def _format_seconds_per_task(seconds: Any) -> str: + if not isinstance(seconds, int | float) or isinstance(seconds, bool): + return "–" + return f"{seconds:,.0f}s" + + +def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: + """Multi-column standings; drops metric columns that are empty for everyone.""" + if not entries: + return None + + from rich import box + from rich.table import Table + from rich.text import Text + + show_pass_k = any(_deepest_pass_at_k(entry) for entry in entries) + show_cost = any( + isinstance(entry.get("cost_per_task"), int | float) + and not isinstance(entry.get("cost_per_task"), bool) + for entry in entries + ) + show_time = any( + isinstance(entry.get("mean_duration_seconds"), int | float) + and not isinstance(entry.get("mean_duration_seconds"), bool) + for entry in entries + ) + show_tokens = any(_format_tokens(entry.get("tokens_per_task")) for entry in entries) + show_run = any( + isinstance(entry.get("run"), dict) and entry["run"].get("name") + for entry in entries + ) + + max_k: int | None = None + if show_pass_k: + for entry in entries: + point = _deepest_pass_at_k(entry) + k = point.get("k") if point else None + if isinstance(k, int): + max_k = k if max_k is None else max(max_k, k) + pass_k_header = f"Pass@{max_k}" if max_k is not None else "Pass@k" + + table = Table( + title="Leaderboard", + box=box.ROUNDED, + show_header=True, + header_style="bold", + title_justify="left", + expand=False, + ) + table.add_column("Rank", style="cyan", no_wrap=True) + table.add_column("Agent", overflow="fold") + table.add_column("Pass@1", no_wrap=True) + if show_pass_k: + table.add_column(pass_k_header, no_wrap=True) + if show_cost: + table.add_column("Cost/task", no_wrap=True) + if show_time: + table.add_column("Time/task", no_wrap=True) + if show_tokens: + table.add_column("Tokens/task", no_wrap=True) + if show_run: + table.add_column("Run", no_wrap=True, overflow="ellipsis") + + plain_lines = ["Leaderboard:"] for entry in entries: rank = entry.get("rank") - label = f"#{rank}" if isinstance(rank, int) else "–" - model = str(entry.get("model") or "–") - harness = entry.get("harness") - name = f"{model} ({harness})" if harness else model - tags = [ - tag - for tag, present in ( - ("tied for first", bool(entry.get("tied"))), - ("parity", entry.get("task_set") == "parity"), + rank_label = f"#{rank}" if isinstance(rank, int) else "–" + agent = _agent_label(entry) + pass_at_1 = _format_rate_interval(entry.get("pass_at_1")) + cells: list[Any] = [rank_label, Text(agent), pass_at_1] + plain_parts = [rank_label, agent, f"pass@1 {pass_at_1}"] + + if show_pass_k: + point = _deepest_pass_at_k(entry) + pass_at_k = _format_pass_at_k(point) + cells.append(pass_at_k) + if pass_at_k != "–" and point is not None: + plain_parts.append(f"pass@{point.get('k')} {pass_at_k}") + + if show_cost: + cost = _format_cost_per_task(entry.get("cost_per_task")) + cells.append(cost) + if cost != "–": + plain_parts.append(f"{cost}/task") + + if show_time: + duration = _format_seconds_per_task(entry.get("mean_duration_seconds")) + cells.append(duration) + if duration != "–": + plain_parts.append(f"{duration}/task") + + if show_tokens: + tokens = _format_tokens(entry.get("tokens_per_task")) or "–" + cells.append(tokens) + if tokens != "–": + plain_parts.append(f"{tokens} tokens/task") + + if show_run: + run = entry.get("run") + run_name = ( + str(run["name"]) + if isinstance(run, dict) and run.get("name") + else "–" ) - if present - ] - parts = [ - name + (f" [{', '.join(tags)}]" if tags else ""), - *_metric_parts(entry), - ] - run = entry.get("run") - if isinstance(run, dict) and run.get("name"): - parts.append(f"run {run['name']}") - rows.append((label, " · ".join(parts))) - return rows + cells.append(Text(run_name)) + if run_name != "–": + plain_parts.append(f"run {run_name}") + table.add_row(*cells) + plain_lines.append(" · ".join(plain_parts)) -def _benchmark_run_rows(runs: list[BenchmarkRun]) -> list[tuple[str, str]]: - return [ - ( - run.name, + return DetailSection(rich=table, plain_lines=plain_lines) + + +def _benchmark_runs_section( + runs: list[BenchmarkRun], + *, + shown: int, + total: int, +) -> DetailSection | None: + if not runs: + return None + + from rich import box + from rich.table import Table + from rich.text import Text + + title = f"Runs ({shown:,} of {total:,})" + table = Table( + title=title, + box=box.ROUNDED, + show_header=True, + header_style="bold", + title_justify="left", + expand=False, + ) + table.add_column("Name", overflow="fold") + table.add_column("Status", no_wrap=True) + table.add_column("Progress", no_wrap=True) + table.add_column("Best Pass@1", no_wrap=True) + table.add_column("Submitted", no_wrap=True) + + plain_lines = [f"{title}:"] + for run in runs: + progress = format_progress(_benchmark_progress(run)) or "–" + best = _format_pass_at_1(run.best_pass_at_1) + submitted = format_local_date(run.created_at) + # Markup string so Rich paints the status; Text() would show raw tags. + table.add_row( + Text(run.name), + format_benchmark_status(run), + progress, + best, + submitted, + ) + plain_lines.append( " · ".join( [ - format_benchmark_status(run), - format_progress(_benchmark_progress(run)) or "–", - f"best pass@1 {_format_pass_at_1(run.best_pass_at_1)}", - format_local_date(run.created_at), + run.name, + f"[{run.status}]", + progress, + f"best pass@1 {best}", + submitted, ] - ), + ) ) - for run in runs - ] + + return DetailSection(rich=table, plain_lines=plain_lines) def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: @@ -468,10 +610,11 @@ def benchmark_info(name_or_id: str, *, limit: int, all_: bool) -> DetailResult: sections: list[DetailSection] = [] for section in ( - kv_section("Leaderboard", _leaderboard_rows(benchmark.leaderboard)), - kv_section( - f"Runs ({len(runs):,} of {runs_total_count:,})", - _benchmark_run_rows(runs), + _leaderboard_section(benchmark.leaderboard), + _benchmark_runs_section( + runs, + shown=len(runs), + total=runs_total_count, ), ): if section is not None: diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index b81464bd..705d05d7 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -555,8 +555,8 @@ def list_benchmark_runs(self, **_: Any) -> PaginatedBenchmarkRuns: ) -def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: - full, sparse = benchmark_module._leaderboard_rows( +def test_leaderboard_section_format_and_tolerate_sparse_entries() -> None: + section = benchmark_module._leaderboard_section( [ { "rank": 1, @@ -588,20 +588,22 @@ def test_leaderboard_rows_format_and_tolerate_sparse_entries() -> None: ] ) - label, value = full - assert label == "#1" - assert "GPT-5.5 (codex) [parity]" in value - assert "pass@1 75.0% (71.9–78.1)" in value - assert "pass@2 81.2%" in value - assert "$4.20/task" in value - assert "54s/task" in value - assert "1.1M tokens/task" in value - assert "run warm-gull" in value + assert section is not None + assert section.plain_lines[0] == "Leaderboard:" + full = section.plain_lines[1] + assert full.startswith("#1 · GPT-5.5 (codex) [parity]") + assert "pass@1 75.0% (71.9–78.1)" in full + assert "pass@2 81.2%" in full + assert "$4.20/task" in full + assert "54s/task" in full + assert "1.1M tokens/task" in full + assert "run warm-gull" in full - label, value = sparse - assert label == "–" - assert value.startswith("– [tied for first]") - assert "pass@1 –" in value + sparse = section.plain_lines[2] + assert sparse.startswith("– · – [tied for first]") + assert "pass@1 –" in sparse + # Rich table should expose the dynamic Pass@k header. + assert any(getattr(col, "header", None) == "Pass@2" for col in section.rich.columns) def test_last_run_cell_shows_sync_state_before_run_history() -> None: @@ -647,7 +649,7 @@ def entry(**overrides: Any) -> BenchmarkCatalogEntry: assert ran.endswith("calm-yak") -def test_benchmark_run_rows_render_status_progress_and_date() -> None: +def test_benchmark_runs_section_render_status_progress_and_date() -> None: run = BenchmarkRun.from_dict( { "id": "run-1", @@ -661,8 +663,23 @@ def test_benchmark_run_rows_render_status_progress_and_date() -> None: } ) - [(label, value)] = benchmark_module._benchmark_run_rows([run]) + section = benchmark_module._benchmark_runs_section( + [run], + shown=1, + total=12, + ) - assert label == "warm-gull" - assert "best pass@1 75.0%" in value - assert "498" in value + assert section is not None + assert section.plain_lines[0] == "Runs (1 of 12):" + line = section.plain_lines[1] + assert line.startswith("warm-gull · ") + assert "[finished]" in line + assert "best pass@1 75.0%" in line + assert "498" in line + assert [col.header for col in section.rich.columns] == [ + "Name", + "Status", + "Progress", + "Best Pass@1", + "Submitted", + ] From 8a3dddf290333f2f6e795509154334a4aa1a70b9 Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 23:28:50 +0000 Subject: [PATCH 21/22] fix leaderboard rank labels and drop Run column --- osmosis_ai/platform/cli/benchmark.py | 43 ++++++------------- .../platform/cli/test_benchmark_catalog.py | 5 ++- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 61b62dba..7fc8808e 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -284,19 +284,21 @@ def _format_tokens(value: Any) -> str | None: return f"{round(value):,}" +def _rank_label(entry: dict[str, Any]) -> str: + rank = entry.get("rank") + label = f"#{rank}" if isinstance(rank, int) else "–" + if entry.get("tied"): + label += " [tied for first]" + return label + + def _agent_label(entry: dict[str, Any]) -> str: model = str(entry.get("model") or "–") harness = entry.get("harness") name = f"{model} ({harness})" if harness else model - tags = [ - tag - for tag, present in ( - ("tied for first", bool(entry.get("tied"))), - ("parity", entry.get("task_set") == "parity"), - ) - if present - ] - return name + (f" [{', '.join(tags)}]" if tags else "") + if entry.get("task_set") == "parity": + name += " [parity]" + return name def _deepest_pass_at_k(entry: dict[str, Any]) -> dict[str, Any] | None: @@ -346,10 +348,6 @@ def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: for entry in entries ) show_tokens = any(_format_tokens(entry.get("tokens_per_task")) for entry in entries) - show_run = any( - isinstance(entry.get("run"), dict) and entry["run"].get("name") - for entry in entries - ) max_k: int | None = None if show_pass_k: @@ -379,16 +377,14 @@ def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: table.add_column("Time/task", no_wrap=True) if show_tokens: table.add_column("Tokens/task", no_wrap=True) - if show_run: - table.add_column("Run", no_wrap=True, overflow="ellipsis") plain_lines = ["Leaderboard:"] for entry in entries: - rank = entry.get("rank") - rank_label = f"#{rank}" if isinstance(rank, int) else "–" + rank_label = _rank_label(entry) agent = _agent_label(entry) pass_at_1 = _format_rate_interval(entry.get("pass_at_1")) - cells: list[Any] = [rank_label, Text(agent), pass_at_1] + # Text() so "[tied for first]" is literal, not Rich markup. + cells: list[Any] = [Text(rank_label), Text(agent), pass_at_1] plain_parts = [rank_label, agent, f"pass@1 {pass_at_1}"] if show_pass_k: @@ -416,17 +412,6 @@ def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: if tokens != "–": plain_parts.append(f"{tokens} tokens/task") - if show_run: - run = entry.get("run") - run_name = ( - str(run["name"]) - if isinstance(run, dict) and run.get("name") - else "–" - ) - cells.append(Text(run_name)) - if run_name != "–": - plain_parts.append(f"run {run_name}") - table.add_row(*cells) plain_lines.append(" · ".join(plain_parts)) diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index 705d05d7..8e4dc76f 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -597,10 +597,11 @@ def test_leaderboard_section_format_and_tolerate_sparse_entries() -> None: assert "$4.20/task" in full assert "54s/task" in full assert "1.1M tokens/task" in full - assert "run warm-gull" in full + assert "run warm-gull" not in full + assert "[tied for first]" not in full # full row is not tied sparse = section.plain_lines[2] - assert sparse.startswith("– · – [tied for first]") + assert sparse.startswith("– [tied for first] · –") assert "pass@1 –" in sparse # Rich table should expose the dynamic Pass@k header. assert any(getattr(col, "header", None) == "Pass@2" for col in section.rich.columns) From bda6cee2b249cfb90017d5d3c5763ecdf1d2449b Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Tue, 4 Aug 2026 23:56:57 +0000 Subject: [PATCH 22/22] [cli] fix: compact leaderboard ties with * and expand table layout --- osmosis_ai/platform/cli/benchmark.py | 45 ++++++++++++++++--- .../platform/cli/test_benchmark_catalog.py | 13 ++++-- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/osmosis_ai/platform/cli/benchmark.py b/osmosis_ai/platform/cli/benchmark.py index 7fc8808e..bc53c195 100644 --- a/osmosis_ai/platform/cli/benchmark.py +++ b/osmosis_ai/platform/cli/benchmark.py @@ -285,13 +285,28 @@ def _format_tokens(value: Any) -> str | None: def _rank_label(entry: dict[str, Any]) -> str: + """Plain-text rank; ties live on rank (not the agent name).""" rank = entry.get("rank") label = f"#{rank}" if isinstance(rank, int) else "–" if entry.get("tied"): - label += " [tied for first]" + # Asterisk keeps the column narrow; full wording is in the section note. + label += "*" return label +def _rank_cell(entry: dict[str, Any]) -> Any: + """Rich rank cell. `*` = tied for first (see section note under the table).""" + from rich.text import Text + + rank = entry.get("rank") + label = f"#{rank}" if isinstance(rank, int) else "–" + if not entry.get("tied"): + return Text(label) + cell = Text(label) + cell.append("*", style="dim") + return cell + + def _agent_label(entry: dict[str, Any]) -> str: model = str(entry.get("model") or "–") harness = entry.get("harness") @@ -358,16 +373,30 @@ def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: max_k = k if max_k is None else max(max_k, k) pass_k_header = f"Pass@{max_k}" if max_k is not None else "Pass@k" + any_tied = any(bool(entry.get("tied")) for entry in entries) table = Table( title="Leaderboard", box=box.ROUNDED, show_header=True, header_style="bold", title_justify="left", - expand=False, + expand=True, + caption=( + "* tied for first (not distinguishable from the leader)" + if any_tied + else None + ), + caption_justify="left", + ) + table.add_column("Rank", style="cyan", no_wrap=True, min_width=4) + # no_wrap + ellipsis: fold collapses Agent when many metric cols compete. + table.add_column( + "Agent", + no_wrap=True, + overflow="ellipsis", + min_width=16, + ratio=2, ) - table.add_column("Rank", style="cyan", no_wrap=True) - table.add_column("Agent", overflow="fold") table.add_column("Pass@1", no_wrap=True) if show_pass_k: table.add_column(pass_k_header, no_wrap=True) @@ -383,8 +412,7 @@ def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: rank_label = _rank_label(entry) agent = _agent_label(entry) pass_at_1 = _format_rate_interval(entry.get("pass_at_1")) - # Text() so "[tied for first]" is literal, not Rich markup. - cells: list[Any] = [Text(rank_label), Text(agent), pass_at_1] + cells: list[Any] = [_rank_cell(entry), Text(agent), pass_at_1] plain_parts = [rank_label, agent, f"pass@1 {pass_at_1}"] if show_pass_k: @@ -415,6 +443,11 @@ def _leaderboard_section(entries: list[dict[str, Any]]) -> DetailSection | None: table.add_row(*cells) plain_lines.append(" · ".join(plain_parts)) + if any_tied: + plain_lines.append( + "* tied for first (not distinguishable from the leader)" + ) + return DetailSection(rich=table, plain_lines=plain_lines) diff --git a/tests/unit/platform/cli/test_benchmark_catalog.py b/tests/unit/platform/cli/test_benchmark_catalog.py index 8e4dc76f..330475b7 100644 --- a/tests/unit/platform/cli/test_benchmark_catalog.py +++ b/tests/unit/platform/cli/test_benchmark_catalog.py @@ -598,13 +598,20 @@ def test_leaderboard_section_format_and_tolerate_sparse_entries() -> None: assert "54s/task" in full assert "1.1M tokens/task" in full assert "run warm-gull" not in full - assert "[tied for first]" not in full # full row is not tied + assert not full.startswith("#1*") # leader is not marked tied sparse = section.plain_lines[2] - assert sparse.startswith("– [tied for first] · –") + assert sparse.startswith("–* · –") assert "pass@1 –" in sparse - # Rich table should expose the dynamic Pass@k header. + assert ( + plain_lines[-1] + if (plain_lines := section.plain_lines) + else "" + ).startswith("* tied for first") + # Rich table should expose the dynamic Pass@k header + caption. assert any(getattr(col, "header", None) == "Pass@2" for col in section.rich.columns) + assert section.rich.caption is not None + assert "tied for first" in str(section.rich.caption) def test_last_run_cell_shows_sync_state_before_run_history() -> None: