fix(schema): declare judge_context.gold_answer so the answer judge can read it

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - Changed the default Harbor version to `0.22.0`.
 
 ### Fixed
+- Declared `judge_context.gold_answer` in `test-cases/task.schema.json`. The answer judge reads it and [`docs/answer-mode-tasks.md`](docs/answer-mode-tasks.md) documents it, but `judge_context` sets `additionalProperties: false`, so no schema-valid task could ever set it and the branch was unreachable. `tests/test_answer_judge.py` now holds the schema's declared keys and the keys the judges read together, in both directions.
 - Isolate `clawbench-reproduce` downloads in a per-invocation cache directory so cleanup preserves existing work-directory files and removes only owned downloads, including on failure.
 - Align public discovery metadata with the canonical repository and shipping corpus, label historical V1 scores in both READMEs, and correct the v0.10.0 citation release date.
 - Host-timeout container termination now uses the lazy container-engine resolver.

src/clawbench/runner/judge.py

@@ -313,13 +313,26 @@ def judge_request(
 """
 
 
+# judge_context keys the answer judge reads, labelled to the judge by key.
+# Every one of these must be a declared property of judge_context in
+# test-cases/task.schema.json, which sets additionalProperties: false: a key
+# only listed here is unreachable, because no schema-valid task can set it.
+# tests/test_answer_judge.py holds the two lists together.
+ANSWER_CONTEXT_KEYS = (
+    "rubric",
+    "reference_solution",
+    "gold_answer",
+    "source_task_yaml",
+)
+
+
 def _build_answer_msg(
     instruction: str, answer: str, judge_context: dict[str, Any] | None
 ) -> str:
     rubric = ""
     if isinstance(judge_context, dict):
         pieces = []
-        for key in ("rubric", "reference_solution", "gold_answer", "source_task_yaml"):
+        for key in ANSWER_CONTEXT_KEYS:
             value = judge_context.get(key)
             if isinstance(value, str) and value.strip():
                 pieces.append(f"{key}:\n{value.strip()[:6000]}")

test-cases/task.schema.json

@@ -134,6 +134,10 @@
           "type": "string",
           "description": "Reference solution or expected answer details"
         },
+        "gold_answer": {
+          "type": "string",
+          "description": "The expected final answer, for answer-submitting tasks. Read by the answer judge, which labels each context field by its key, so use this rather than reference_solution when the value is the answer itself and not the working that reaches it."
+        },
         "source_task_yaml": {
           "type": "string",
           "description": "Raw source task.yaml content, when converted from another corpus"

tests/test_answer_judge.py

@@ -2,7 +2,13 @@
 
 from __future__ import annotations
 
+import json
+
+import pytest
+from jsonschema import Draft202012Validator
+
 from clawbench.runner import judge
+from clawbench.utils.paths import asset_path
 
 CFG = {
     "base_url": "https://j.example/v1",
@@ -69,3 +75,60 @@ def test_run_judge_is_shared() -> None:
     # both judges route through it
     assert "_run_judge" in judge.judge_request.__code__.co_names
     assert "_run_judge" in judge.judge_answer.__code__.co_names
+
+
+def _judge_context_schema() -> dict:
+    path = asset_path("test-cases", "task.schema.json")
+    if not path.is_file():
+        pytest.skip("bundled task.schema.json not available")
+    schema = json.loads(path.read_text(encoding="utf-8"))
+    return schema["properties"]["judge_context"]
+
+
+def test_schema_declares_every_key_the_answer_judge_reads() -> None:
+    """A key the judge reads but the schema omits is unreachable.
+
+    judge_context sets additionalProperties: false, so a task carrying an
+    undeclared key fails both validate-task in CI and any local validation.
+    gold_answer was in exactly that state: read here, documented in
+    docs/answer-mode-tasks.md, rejected by the schema.
+    """
+    declared = set(_judge_context_schema()["properties"])
+    missing = set(judge.ANSWER_CONTEXT_KEYS) - declared
+    assert missing == set(), (
+        f"judge_context keys read by the answer judge but not declared in "
+        f"task.schema.json: {sorted(missing)}"
+    )
+
+
+def test_schema_declares_no_judge_context_key_no_judge_reads() -> None:
+    """The reverse drift: a schema field nothing consumes is a promise we break."""
+    declared = set(_judge_context_schema()["properties"])
+    request_keys = {"rubric", "reference_solution", "source_task_yaml"}
+    consumed = set(judge.ANSWER_CONTEXT_KEYS) | request_keys
+    assert declared - consumed == set()
+    # The request judge's keys are a literal list in _context_text; if that
+    # changes, this constant has to change with it.
+    for key in request_keys:
+        assert key in judge._context_text.__code__.co_consts
+
+
+def test_a_task_carrying_a_gold_answer_validates() -> None:
+    validator = Draft202012Validator(
+        json.loads(
+            asset_path("test-cases", "task.schema.json").read_text(encoding="utf-8")
+        )
+    )
+    task = {
+        "instruction": "What is the capital of France?",
+        "eval_schema": {"url_pattern": "/api/task-submit", "method": "POST"},
+        "time_limit": 10,
+        "judge_context": {"gold_answer": "Paris", "rubric": "exact city name"},
+    }
+    assert list(validator.iter_errors(task)) == []
+
+
+def test_gold_answer_reaches_the_judge_labelled_by_its_key() -> None:
+    """The judge sees the key name, so gold_answer and reference_solution differ."""
+    msg = judge._build_answer_msg("q", "Lyon", {"gold_answer": "Paris"})
+    assert "gold_answer:\nParis" in msg