From a7ef80fa4c0d462d9a90caf3e66a90df3af103aa Mon Sep 17 00:00:00 2001 From: Valmik Nahata Date: Wed, 23 Sep 2026 06:50:54 -0400 Subject: [PATCH] fix(schema): declare judge_context.gold_answer so the answer judge can read it _build_answer_msg reads judge_context.gold_answer and docs/answer-mode-tasks.md documents it, but judge_context in test-cases/task.schema.json declares only rubric, reference_solution and source_task_yaml under additionalProperties: false. No schema-valid task can set gold_answer, so the branch is unreachable and validate-task would reject any task that tried. The two drifted in 68da27a, which added the answer judge and the fourth key without extending the schema that 2e5ccbf had introduced. Declares the property and adds tests that hold the schema and the judges together in both directions: every key a judge reads must be declared, and every declared key must be read by a judge. Both new assertions fail against the schema as it stands on main. The field earns its place next to reference_solution because the judge labels each context value by its key. Across the 19 bundled claw-eval tasks reference_solution holds a worked procedure in some and the answer itself in others, so a judge reading it cannot tell which it has; gold_answer says so. --- CHANGELOG.md | 1 + src/clawbench/runner/judge.py | 15 ++++++++- test-cases/task.schema.json | 4 +++ tests/test_answer_judge.py | 63 +++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e321b51d..02de9945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - Changed the default Harbor version to `0.22.0`. ### Fixed +- Declared `judge_context.gold_answer` in `test-cases/task.schema.json`. The answer judge reads it and [`docs/answer-mode-tasks.md`](docs/answer-mode-tasks.md) documents it, but `judge_context` sets `additionalProperties: false`, so no schema-valid task could ever set it and the branch was unreachable. `tests/test_answer_judge.py` now holds the schema's declared keys and the keys the judges read together, in both directions. - Isolate `clawbench-reproduce` downloads in a per-invocation cache directory so cleanup preserves existing work-directory files and removes only owned downloads, including on failure. - Align public discovery metadata with the canonical repository and shipping corpus, label historical V1 scores in both READMEs, and correct the v0.10.0 citation release date. - Host-timeout container termination now uses the lazy container-engine resolver. diff --git a/src/clawbench/runner/judge.py b/src/clawbench/runner/judge.py index 6b15453a..af04b6b9 100644 --- a/src/clawbench/runner/judge.py +++ b/src/clawbench/runner/judge.py @@ -313,13 +313,26 @@ def judge_request( """ +# judge_context keys the answer judge reads, labelled to the judge by key. +# Every one of these must be a declared property of judge_context in +# test-cases/task.schema.json, which sets additionalProperties: false: a key +# only listed here is unreachable, because no schema-valid task can set it. +# tests/test_answer_judge.py holds the two lists together. +ANSWER_CONTEXT_KEYS = ( + "rubric", + "reference_solution", + "gold_answer", + "source_task_yaml", +) + + def _build_answer_msg( instruction: str, answer: str, judge_context: dict[str, Any] | None ) -> str: rubric = "" if isinstance(judge_context, dict): pieces = [] - for key in ("rubric", "reference_solution", "gold_answer", "source_task_yaml"): + for key in ANSWER_CONTEXT_KEYS: value = judge_context.get(key) if isinstance(value, str) and value.strip(): pieces.append(f"{key}:\n{value.strip()[:6000]}") diff --git a/test-cases/task.schema.json b/test-cases/task.schema.json index aa66ba19..1841d695 100644 --- a/test-cases/task.schema.json +++ b/test-cases/task.schema.json @@ -134,6 +134,10 @@ "type": "string", "description": "Reference solution or expected answer details" }, + "gold_answer": { + "type": "string", + "description": "The expected final answer, for answer-submitting tasks. Read by the answer judge, which labels each context field by its key, so use this rather than reference_solution when the value is the answer itself and not the working that reaches it." + }, "source_task_yaml": { "type": "string", "description": "Raw source task.yaml content, when converted from another corpus" diff --git a/tests/test_answer_judge.py b/tests/test_answer_judge.py index d92ba459..fafe65f3 100644 --- a/tests/test_answer_judge.py +++ b/tests/test_answer_judge.py @@ -2,7 +2,13 @@ from __future__ import annotations +import json + +import pytest +from jsonschema import Draft202012Validator + from clawbench.runner import judge +from clawbench.utils.paths import asset_path CFG = { "base_url": "https://j.example/v1", @@ -69,3 +75,60 @@ def test_run_judge_is_shared() -> None: # both judges route through it assert "_run_judge" in judge.judge_request.__code__.co_names assert "_run_judge" in judge.judge_answer.__code__.co_names + + +def _judge_context_schema() -> dict: + path = asset_path("test-cases", "task.schema.json") + if not path.is_file(): + pytest.skip("bundled task.schema.json not available") + schema = json.loads(path.read_text(encoding="utf-8")) + return schema["properties"]["judge_context"] + + +def test_schema_declares_every_key_the_answer_judge_reads() -> None: + """A key the judge reads but the schema omits is unreachable. + + judge_context sets additionalProperties: false, so a task carrying an + undeclared key fails both validate-task in CI and any local validation. + gold_answer was in exactly that state: read here, documented in + docs/answer-mode-tasks.md, rejected by the schema. + """ + declared = set(_judge_context_schema()["properties"]) + missing = set(judge.ANSWER_CONTEXT_KEYS) - declared + assert missing == set(), ( + f"judge_context keys read by the answer judge but not declared in " + f"task.schema.json: {sorted(missing)}" + ) + + +def test_schema_declares_no_judge_context_key_no_judge_reads() -> None: + """The reverse drift: a schema field nothing consumes is a promise we break.""" + declared = set(_judge_context_schema()["properties"]) + request_keys = {"rubric", "reference_solution", "source_task_yaml"} + consumed = set(judge.ANSWER_CONTEXT_KEYS) | request_keys + assert declared - consumed == set() + # The request judge's keys are a literal list in _context_text; if that + # changes, this constant has to change with it. + for key in request_keys: + assert key in judge._context_text.__code__.co_consts + + +def test_a_task_carrying_a_gold_answer_validates() -> None: + validator = Draft202012Validator( + json.loads( + asset_path("test-cases", "task.schema.json").read_text(encoding="utf-8") + ) + ) + task = { + "instruction": "What is the capital of France?", + "eval_schema": {"url_pattern": "/api/task-submit", "method": "POST"}, + "time_limit": 10, + "judge_context": {"gold_answer": "Paris", "rubric": "exact city name"}, + } + assert list(validator.iter_errors(task)) == [] + + +def test_gold_answer_reaches_the_judge_labelled_by_its_key() -> None: + """The judge sees the key name, so gold_answer and reference_solution differ.""" + msg = judge._build_answer_msg("q", "Lyon", {"gold_answer": "Paris"}) + assert "gold_answer:\nParis" in msg