Add cluster-agnostic eval system

.gitignore

@@ -1,3 +1,6 @@
+# Claude Code local instructions (personal, not shared)
+CLAUDE.local.md
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[codz]
@@ -9,6 +12,16 @@ scripts/beam/helm-chart/charts/localstack-0.6.5.tgz
 # database bu
 scripts/database/backups
 
+# Eval logs (generated at runtime, not synced)
+eval/logs/
+eval/local/
+
+# Local eval memory/notes (per-cluster, not synced)
+eval/.local_notes/
+
+# Personal Claude Code instructions (not shared)
+CLAUDE.local.md
+
 # C extensions
 *.so
 
@@ -410,3 +423,17 @@ train/hpc/dotenv/secret.env
 # Claude
 CLAUDE.md
 development_progress.md
+
+# Eval runtime data (logs, tracking, job output)
+eval/logs/
+eval/local/
+eval/MBZ/logs/
+eval/MBZ/tracking/
+eval/MBZ/__pycache__/
+eval/jupiter/logs/
+jobs/
+
+# Secrets (never commit)
+secret.env
+secrets.env
+*.env.local

database/unified_db/utils.py

@@ -2015,7 +2015,8 @@ def register_sandbox_job(
     stats: Optional[Dict[str, Any]] = None,
     forced_update: bool = True,
     hf_traces_link: Optional[str] = None,
-    job_status: Optional[str] = None
+    job_status: Optional[str] = None,
+    is_overlong: bool = False,
 ) -> Dict[str, Any]:
     """
     Register a sandbox job with minimal auto-filling.
@@ -2055,7 +2056,8 @@ def register_sandbox_job(
             "benchmark_id": benchmark_id,
             "n_rep_eval": n_rep_eval,
             "hf_traces_link": hf_traces_link,
-            "job_status": job_status
+            "job_status": job_status,
+            "is_overlong": is_overlong,
         }
 
         # Include job_id if provided (preserves local ID from result.json)
@@ -3447,6 +3449,7 @@ def upload_job_and_trial_records(
     register_benchmark: bool = False,
     hf_dataset_url: Optional[str] = None,
     forced_update: bool = False,
+    is_overlong: bool = False,
 ) -> Dict[str, Any]:
     """
     Upload job and trial records to database (with optional HF dataset URL for trials).
@@ -3858,7 +3861,7 @@ def upload_job_and_trial_records(
         job_metadata["hf_traces_link"] = hf_dataset_url
         job_metadata["job_status"] = "Finished"
 
-        job_record = register_sandbox_job(**job_metadata, forced_update=forced_update)
+        job_record = register_sandbox_job(**job_metadata, forced_update=forced_update, is_overlong=is_overlong)
 
         if not job_record.get("success"):
             raise Exception(f"Job registration failed: {job_record.get('error')}")
@@ -4141,6 +4144,7 @@ def upload_traces_to_hf(
             verbose=verbose,
             success_filter=success_filter,
             include_verifier_output=include_verifier_output,
+            export_subagents=export_subagents,
         )
         logger.info(f"Extracted {len(dataset)} conversation rows from trials")
     except Exception as e:
@@ -4301,6 +4305,7 @@ def upload_eval_results(
     hf_verbose: bool = False,
     hf_export_subagents: bool = False,
     forced_update: bool = False,
+    is_overlong: bool = False,
 ) -> Dict[str, Any]:
     """
     Upload evaluation results from a job directory to HuggingFace and database.
@@ -4434,6 +4439,7 @@ def upload_eval_results(
             register_benchmark=register_benchmark,
             hf_dataset_url=hf_dataset_url,  # Will be None if HF upload failed
             forced_update=forced_update,
+            is_overlong=is_overlong,
         )
 
         # Add HF-related information to result

eval/AGENT_SETUP_PROMPT.md

@@ -0,0 +1,48 @@
+# Agent Setup Prompt — Eval System on New Cluster
+
+Copy the prompt below and paste it into a Claude Code session on the target cluster. It will read the onboarding doc, detect the cluster, and complete the setup.
+
+---
+
+## Prompt
+
+```
+I need you to set up the eval system on this cluster. Follow the guide at eval/docs/CLUSTER_ONBOARDING.md exactly.
+
+Context:
+- This repo is OpenThoughts-Agent, an eval system that runs LLM evals on HPC clusters via SLURM
+- The eval listener (eval/unified_eval_listener.py) submits sbatch jobs that start vLLM + harbor evals
+- Each cluster needs: a cluster config YAML (eval/clusters/<name>.yaml), a dotenv (hpc/dotenv/<name>.env), conda env(s) with vLLM + harbor, and ~/secrets.env
+- Existing cluster configs to reference: eval/clusters/m2.yaml (MBZ H200), eval/clusters/jupiter.yaml (JSC GH200)
+
+Steps:
+1. Read eval/docs/CLUSTER_ONBOARDING.md for the full setup guide
+2. Detect this cluster:
+   - Run: hostname, sinfo -N --format="%.30N %.6t %.5c %.10G %.10m" | head -5, sinfo --format="%P %D %G" --noheader, uname -m, nvidia-smi | head -3, ls /usr/local/cuda*
+   - Run: conda env list
+   - Check if ~/secrets.env exists and has DAYTONA_API_KEY, SUPABASE_URL, HF_TOKEN
+3. Check if eval/clusters/<detected_cluster>.yaml already exists. If yes, verify it matches hardware. If no, create it using m2.yaml as template.
+4. Check if hpc/dotenv/<detected_cluster>.env already exists. If no, create it using m2.env as template. Use $USER for paths.
+5. Verify harbor is installed: python -c "import harbor; print(harbor.__version__)"
+   - If not installed: check if harbor repo exists locally, install with pip install -e /path/to/harbor
+   - Verify harbor is pinned to commit 6fdb92e7 or later (but NOT e371289f)
+6. Verify hf_transfer is installed: python -c "import hf_transfer"
+   - If not: pip install hf_transfer
+7. Pre-download datasets:
+   - python eval/snapshot_download.py DCAgent/dev_set_v2
+   - python eval/snapshot_download.py DCAgent2/terminal_bench_2
+   - python eval/snapshot_download.py DCAgent2/swebench-verified-random-100-folders
+8. Create log directories: mkdir -p eval/local/<cluster>/logs experiments/listener_logs
+9. Dry-run the listener:
+   source ~/secrets.env && PYTHONPATH=$PWD python eval/unified_eval_listener.py \
+     --cluster-config eval/clusters/<cluster>.yaml \
+     --preset v2 \
+     --priority-file eval/lists/a1_nl2bash.txt \
+     --require-priority-list \
+     --baseline-model-config eval/baseline_model_configs.yaml \
+     --timeout-multiplier 2.0 --tp-size 2 --enable-thinking \
+     --dry-run --once --verbose
+10. Report the results: what cluster was detected, what was created/verified, and whether the dry-run passed.
+
+If anything is missing (secrets, harbor, conda env), tell me what's needed and stop — don't guess at secrets or install packages without confirming.
+```

eval/baseline_model_configs.yaml

@@ -9,56 +9,89 @@
 # The eval script will default to the node's GPU count (e.g. TP=4 on Polaris A100-40GB).
 # Only set it here if a model requires a specific TP size.
 #
-# Pattern configs (under "patterns") are matched when no exact model name is found.
-# Each pattern entry has a "match" regex and the same config fields as a model entry.
+# Format:
+#   models:      Per-model overrides (exact HF name -> config).
+#   groups:      Apply one config to many models. Each entry has "models" (list) + config fields.
+#                Per-model entries in "models:" are merged on top of group config (override wins).
+#   patterns:    Regex fallback when no exact/group match. First match wins.
 
-models:
-  "NovaSky-AI/SA-SWE-32B":
+# --- Groups: many models sharing the same config ---
+groups:
+  - models:
+      - "nvidia/Nemotron-Terminal-32B"
+      - "DCAgent/staqc-sandboxes-traces-terminus-2_Qwen3-32B"
+      - "NovaSky-AI/SA-SWE-32B"
+      - "Qwen/Qwen2.5-Coder-32B-Instruct"
+      - "Qwen/Qwen3-32B"
+      - "R2E-Gym/R2EGym-32B-Agent"
+      - "SWE-Swiss/SWE-Swiss-32B"
+      - "SWE-bench/SWE-agent-LM-32B"
+      - "Skywork/Skywork-SWE-32B"
+      - "allenai/SERA-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_adam-beta1_0-91_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_adam-beta1_0-93_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_adam-beta1_0-95_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_global-batch-size_32_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_global-batch-size_64_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_learning-rate_1e-06_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_lr_1e-5_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_4.0_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_6.0_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_7.0_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_8-0_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_warmup-ratio_0-01_Qwen3-32B"
+      - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_warmup-ratio_0-05_Qwen3-32B"
+      - "laion/Qwen3-32B-NL2Bash-31step"
+      - "laion/Qwen3-32B-R2EGYM-256-3epochs"
+      - "laion/Qwen3-32B-SweSmith-20step"
+      - "laion/open-thoughts-4-code-qwen3-32b-annotated"
+      - "laion/rl__40GPU_base_32b__exp_rpt_codeelo-v2__sft_GLM-4-7-swesmith"
+      - "laion/rl__40GPU_base_32b__exp_rpt_nemotron-bash__sft_GLM-4-7-swesmith"
+      - "laion/sft_GLM-4-7-swesmith-sandboxes-with_tests-oracle_verified_120s-maxeps-131k_Qwen3-32B"
+      - "laion/sft_r2egym-nl2bash-stackoverflow-inferredbugs-32B_Qwen3-32B"
+      - "laion/syh-r2eg-askl-glm_4-7_trac_jupi_-gfi-swes-rand-filt-10K_glm_4-7_trac_jupi_32B"
+      - "laion/rl__48GPU_shaped_32b__swe_rebench_patched_oracle__Qwen3-32B-45"
+    tensor_parallel_size: 4
     max_model_len: 32768
     swap_space: 32
     trust_remote_code: true
     tool_call_parser: hermes
-    reasoning_parser: qwen3
     extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching"
 
-  "Qwen/Qwen2.5-Coder-32B-Instruct":
-    max_model_len: 32768
-    swap_space: 32
-    trust_remote_code: true
-    tool_call_parser: hermes
-    extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching"
+# --- Per-model overrides (merged on top of group config) ---
+models:
+  # Adds reasoning_parser on top of the group defaults
+  "NovaSky-AI/SA-SWE-32B":
+    reasoning_parser: qwen3
 
-  "R2E-Gym/R2EGym-32B-Agent":
-    max_model_len: 32768
-    swap_space: 32
-    trust_remote_code: true
-    tool_call_parser: hermes
-    extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching"
+  # Override: lower swap_space (group default is 32)
+  "allenai/SERA-32B":
+    swap_space: 12
 
-  "SWE-Swiss/SWE-Swiss-32B":
-    max_model_len: 32768
-    swap_space: 32
+# Pattern-based configs: matched by regex when no exact/group match is found.
+# Checked in order; first match wins.
+patterns:
+  - match: "(?i)qwen3\\.5"
+    conda_env: otagent2
     trust_remote_code: true
-    tool_call_parser: hermes
-    extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching"
-
-  "Skywork/Skywork-SWE-32B":
+    tensor_parallel_size: 4
     max_model_len: 32768
     swap_space: 32
-    trust_remote_code: true
-    tool_call_parser: hermes
-    extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching"
+    extra_args: "--enforce-eager"
 
-  "allenai/SERA-32B":
-    max_model_len: 32768
+  - match: "(?i)(?:32b.*(131k|-lc)|(131k|-lc).*32b)"
+    tensor_parallel_size: 4
+    max_model_len: 131072
     swap_space: 12
-    tool_call_parser: hermes
-    extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching"
+    hf_overrides: '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}}'
 
-# Pattern-based configs: matched by regex when no exact model name is found.
-# Checked in order; first match wins.
-patterns:
-  - match: "131k|-lc$"
+  - match: "(?i)32[Bb]"
+    tensor_parallel_size: 4
+    max_model_len: 32768
+    swap_space: 32
+
+  - match: "(131k|-lc)$"
     max_model_len: 131072
     swap_space: 12
-    extra_args: "--hf-overrides '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}}'"
+    hf_overrides: '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}}'

eval/build_vllm_cmd.sh

@@ -21,6 +21,7 @@
 #   EVAL_VLLM_REASONING_PARSER      (default: unset)
 #   EVAL_VLLM_DATA_PARALLEL_SIZE    (default: unset; vLLM v0.8+ only)
 #   EVAL_VLLM_EXTRA_ARGS            (default: unset; space-separated string)
+#   EVAL_VLLM_HF_OVERRIDES          (default: unset; JSON string for --hf-overrides)
 # ==============================================================================
 
 build_vllm_cmd() {
@@ -31,18 +32,19 @@ build_vllm_cmd() {
     # Read overrides from env (set by listener via sbatch --export)
     local tp="${EVAL_VLLM_TENSOR_PARALLEL_SIZE:-4}"
     local dp="${EVAL_VLLM_DATA_PARALLEL_SIZE:-}"
-    local max_model_len="${EVAL_VLLM_MAX_MODEL_LEN:-}"
+    local max_model_len="${EVAL_VLLM_MAX_MODEL_LEN:-32768}"
     local swap_space="${EVAL_VLLM_SWAP_SPACE:-4}"
     local trust_remote_code="${EVAL_VLLM_TRUST_REMOTE_CODE:-}"
     local tool_call_parser="${EVAL_VLLM_TOOL_CALL_PARSER:-}"
     local reasoning_parser="${EVAL_VLLM_REASONING_PARSER:-}"
     local extra_args="${EVAL_VLLM_EXTRA_ARGS:-}"
+    local hf_overrides="${EVAL_VLLM_HF_OVERRIDES:-}"
 
     # Build command array
     VLLM_CMD=(
         "$python_bin" -m vllm.entrypoints.openai.api_server
         --model "$model"
-        --host 0.0.0.0 --port 8000
+        --host 0.0.0.0 --port "${VLLM_PORT:-8000}"
         --served-model-name "$model"
         --tensor-parallel-size "$tp"
         --gpu-memory-utilization "$gpu_mem_util"
@@ -70,6 +72,11 @@ build_vllm_cmd() {
         VLLM_CMD+=(--reasoning-parser "$reasoning_parser")
     fi
 
+    # HF model config overrides (JSON string, properly quoted)
+    if [ -n "$hf_overrides" ]; then
+        VLLM_CMD+=(--hf-overrides "$hf_overrides")
+    fi
+
     # Append extra args (space-separated string)
     if [ -n "$extra_args" ]; then
         # shellcheck disable=SC2206