diff --git a/.gitignore b/.gitignore index 228383a3b..b50be3a91 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Claude Code local instructions (personal, not shared) +CLAUDE.local.md + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] @@ -9,6 +12,16 @@ scripts/beam/helm-chart/charts/localstack-0.6.5.tgz # database bu scripts/database/backups +# Eval logs (generated at runtime, not synced) +eval/logs/ +eval/local/ + +# Local eval memory/notes (per-cluster, not synced) +eval/.local_notes/ + +# Personal Claude Code instructions (not shared) +CLAUDE.local.md + # C extensions *.so @@ -410,3 +423,17 @@ train/hpc/dotenv/secret.env # Claude CLAUDE.md development_progress.md + +# Eval runtime data (logs, tracking, job output) +eval/logs/ +eval/local/ +eval/MBZ/logs/ +eval/MBZ/tracking/ +eval/MBZ/__pycache__/ +eval/jupiter/logs/ +jobs/ + +# Secrets (never commit) +secret.env +secrets.env +*.env.local \ No newline at end of file diff --git a/database/unified_db/utils.py b/database/unified_db/utils.py index 32397611c..3d81a4ce9 100644 --- a/database/unified_db/utils.py +++ b/database/unified_db/utils.py @@ -2015,7 +2015,8 @@ def register_sandbox_job( stats: Optional[Dict[str, Any]] = None, forced_update: bool = True, hf_traces_link: Optional[str] = None, - job_status: Optional[str] = None + job_status: Optional[str] = None, + is_overlong: bool = False, ) -> Dict[str, Any]: """ Register a sandbox job with minimal auto-filling. @@ -2055,7 +2056,8 @@ def register_sandbox_job( "benchmark_id": benchmark_id, "n_rep_eval": n_rep_eval, "hf_traces_link": hf_traces_link, - "job_status": job_status + "job_status": job_status, + "is_overlong": is_overlong, } # Include job_id if provided (preserves local ID from result.json) @@ -3447,6 +3449,7 @@ def upload_job_and_trial_records( register_benchmark: bool = False, hf_dataset_url: Optional[str] = None, forced_update: bool = False, + is_overlong: bool = False, ) -> Dict[str, Any]: """ Upload job and trial records to database (with optional HF dataset URL for trials). @@ -3858,7 +3861,7 @@ def upload_job_and_trial_records( job_metadata["hf_traces_link"] = hf_dataset_url job_metadata["job_status"] = "Finished" - job_record = register_sandbox_job(**job_metadata, forced_update=forced_update) + job_record = register_sandbox_job(**job_metadata, forced_update=forced_update, is_overlong=is_overlong) if not job_record.get("success"): raise Exception(f"Job registration failed: {job_record.get('error')}") @@ -4141,6 +4144,7 @@ def upload_traces_to_hf( verbose=verbose, success_filter=success_filter, include_verifier_output=include_verifier_output, + export_subagents=export_subagents, ) logger.info(f"Extracted {len(dataset)} conversation rows from trials") except Exception as e: @@ -4301,6 +4305,7 @@ def upload_eval_results( hf_verbose: bool = False, hf_export_subagents: bool = False, forced_update: bool = False, + is_overlong: bool = False, ) -> Dict[str, Any]: """ Upload evaluation results from a job directory to HuggingFace and database. @@ -4434,6 +4439,7 @@ def upload_eval_results( register_benchmark=register_benchmark, hf_dataset_url=hf_dataset_url, # Will be None if HF upload failed forced_update=forced_update, + is_overlong=is_overlong, ) # Add HF-related information to result diff --git a/eval/AGENT_SETUP_PROMPT.md b/eval/AGENT_SETUP_PROMPT.md new file mode 100644 index 000000000..b0f8f18e8 --- /dev/null +++ b/eval/AGENT_SETUP_PROMPT.md @@ -0,0 +1,48 @@ +# Agent Setup Prompt — Eval System on New Cluster + +Copy the prompt below and paste it into a Claude Code session on the target cluster. It will read the onboarding doc, detect the cluster, and complete the setup. + +--- + +## Prompt + +``` +I need you to set up the eval system on this cluster. Follow the guide at eval/docs/CLUSTER_ONBOARDING.md exactly. + +Context: +- This repo is OpenThoughts-Agent, an eval system that runs LLM evals on HPC clusters via SLURM +- The eval listener (eval/unified_eval_listener.py) submits sbatch jobs that start vLLM + harbor evals +- Each cluster needs: a cluster config YAML (eval/clusters/.yaml), a dotenv (hpc/dotenv/.env), conda env(s) with vLLM + harbor, and ~/secrets.env +- Existing cluster configs to reference: eval/clusters/m2.yaml (MBZ H200), eval/clusters/jupiter.yaml (JSC GH200) + +Steps: +1. Read eval/docs/CLUSTER_ONBOARDING.md for the full setup guide +2. Detect this cluster: + - Run: hostname, sinfo -N --format="%.30N %.6t %.5c %.10G %.10m" | head -5, sinfo --format="%P %D %G" --noheader, uname -m, nvidia-smi | head -3, ls /usr/local/cuda* + - Run: conda env list + - Check if ~/secrets.env exists and has DAYTONA_API_KEY, SUPABASE_URL, HF_TOKEN +3. Check if eval/clusters/.yaml already exists. If yes, verify it matches hardware. If no, create it using m2.yaml as template. +4. Check if hpc/dotenv/.env already exists. If no, create it using m2.env as template. Use $USER for paths. +5. Verify harbor is installed: python -c "import harbor; print(harbor.__version__)" + - If not installed: check if harbor repo exists locally, install with pip install -e /path/to/harbor + - Verify harbor is pinned to commit 6fdb92e7 or later (but NOT e371289f) +6. Verify hf_transfer is installed: python -c "import hf_transfer" + - If not: pip install hf_transfer +7. Pre-download datasets: + - python eval/snapshot_download.py DCAgent/dev_set_v2 + - python eval/snapshot_download.py DCAgent2/terminal_bench_2 + - python eval/snapshot_download.py DCAgent2/swebench-verified-random-100-folders +8. Create log directories: mkdir -p eval/local//logs experiments/listener_logs +9. Dry-run the listener: + source ~/secrets.env && PYTHONPATH=$PWD python eval/unified_eval_listener.py \ + --cluster-config eval/clusters/.yaml \ + --preset v2 \ + --priority-file eval/lists/a1_nl2bash.txt \ + --require-priority-list \ + --baseline-model-config eval/baseline_model_configs.yaml \ + --timeout-multiplier 2.0 --tp-size 2 --enable-thinking \ + --dry-run --once --verbose +10. Report the results: what cluster was detected, what was created/verified, and whether the dry-run passed. + +If anything is missing (secrets, harbor, conda env), tell me what's needed and stop — don't guess at secrets or install packages without confirming. +``` diff --git a/eval/baseline_model_configs.yaml b/eval/baseline_model_configs.yaml index 1302c3709..10961b196 100644 --- a/eval/baseline_model_configs.yaml +++ b/eval/baseline_model_configs.yaml @@ -9,56 +9,89 @@ # The eval script will default to the node's GPU count (e.g. TP=4 on Polaris A100-40GB). # Only set it here if a model requires a specific TP size. # -# Pattern configs (under "patterns") are matched when no exact model name is found. -# Each pattern entry has a "match" regex and the same config fields as a model entry. +# Format: +# models: Per-model overrides (exact HF name -> config). +# groups: Apply one config to many models. Each entry has "models" (list) + config fields. +# Per-model entries in "models:" are merged on top of group config (override wins). +# patterns: Regex fallback when no exact/group match. First match wins. -models: - "NovaSky-AI/SA-SWE-32B": +# --- Groups: many models sharing the same config --- +groups: + - models: + - "nvidia/Nemotron-Terminal-32B" + - "DCAgent/staqc-sandboxes-traces-terminus-2_Qwen3-32B" + - "NovaSky-AI/SA-SWE-32B" + - "Qwen/Qwen2.5-Coder-32B-Instruct" + - "Qwen/Qwen3-32B" + - "R2E-Gym/R2EGym-32B-Agent" + - "SWE-Swiss/SWE-Swiss-32B" + - "SWE-bench/SWE-agent-LM-32B" + - "Skywork/Skywork-SWE-32B" + - "allenai/SERA-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_adam-beta1_0-91_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_adam-beta1_0-93_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_adam-beta1_0-95_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_global-batch-size_32_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_global-batch-size_64_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_learning-rate_1e-06_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_lr_1e-5_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_4.0_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_6.0_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_7.0_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_num-train-epochs_8-0_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_warmup-ratio_0-01_Qwen3-32B" + - "laion/GLM-4.6-stackexchange-overflow-sandboxes-32eps-65k-reasoning_warmup-ratio_0-05_Qwen3-32B" + - "laion/Qwen3-32B-NL2Bash-31step" + - "laion/Qwen3-32B-R2EGYM-256-3epochs" + - "laion/Qwen3-32B-SweSmith-20step" + - "laion/open-thoughts-4-code-qwen3-32b-annotated" + - "laion/rl__40GPU_base_32b__exp_rpt_codeelo-v2__sft_GLM-4-7-swesmith" + - "laion/rl__40GPU_base_32b__exp_rpt_nemotron-bash__sft_GLM-4-7-swesmith" + - "laion/sft_GLM-4-7-swesmith-sandboxes-with_tests-oracle_verified_120s-maxeps-131k_Qwen3-32B" + - "laion/sft_r2egym-nl2bash-stackoverflow-inferredbugs-32B_Qwen3-32B" + - "laion/syh-r2eg-askl-glm_4-7_trac_jupi_-gfi-swes-rand-filt-10K_glm_4-7_trac_jupi_32B" + - "laion/rl__48GPU_shaped_32b__swe_rebench_patched_oracle__Qwen3-32B-45" + tensor_parallel_size: 4 max_model_len: 32768 swap_space: 32 trust_remote_code: true tool_call_parser: hermes - reasoning_parser: qwen3 extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching" - "Qwen/Qwen2.5-Coder-32B-Instruct": - max_model_len: 32768 - swap_space: 32 - trust_remote_code: true - tool_call_parser: hermes - extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching" +# --- Per-model overrides (merged on top of group config) --- +models: + # Adds reasoning_parser on top of the group defaults + "NovaSky-AI/SA-SWE-32B": + reasoning_parser: qwen3 - "R2E-Gym/R2EGym-32B-Agent": - max_model_len: 32768 - swap_space: 32 - trust_remote_code: true - tool_call_parser: hermes - extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching" + # Override: lower swap_space (group default is 32) + "allenai/SERA-32B": + swap_space: 12 - "SWE-Swiss/SWE-Swiss-32B": - max_model_len: 32768 - swap_space: 32 +# Pattern-based configs: matched by regex when no exact/group match is found. +# Checked in order; first match wins. +patterns: + - match: "(?i)qwen3\\.5" + conda_env: otagent2 trust_remote_code: true - tool_call_parser: hermes - extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching" - - "Skywork/Skywork-SWE-32B": + tensor_parallel_size: 4 max_model_len: 32768 swap_space: 32 - trust_remote_code: true - tool_call_parser: hermes - extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching" + extra_args: "--enforce-eager" - "allenai/SERA-32B": - max_model_len: 32768 + - match: "(?i)(?:32b.*(131k|-lc)|(131k|-lc).*32b)" + tensor_parallel_size: 4 + max_model_len: 131072 swap_space: 12 - tool_call_parser: hermes - extra_args: "--dtype bfloat16 --block-size 16 --enable-chunked-prefill --max-num-partial-prefills 1 --enable-prefix-caching" + hf_overrides: '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}}' -# Pattern-based configs: matched by regex when no exact model name is found. -# Checked in order; first match wins. -patterns: - - match: "131k|-lc$" + - match: "(?i)32[Bb]" + tensor_parallel_size: 4 + max_model_len: 32768 + swap_space: 32 + + - match: "(131k|-lc)$" max_model_len: 131072 swap_space: 12 - extra_args: "--hf-overrides '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}}'" + hf_overrides: '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}}' diff --git a/eval/build_vllm_cmd.sh b/eval/build_vllm_cmd.sh index d027ff2d8..a731eaeee 100644 --- a/eval/build_vllm_cmd.sh +++ b/eval/build_vllm_cmd.sh @@ -21,6 +21,7 @@ # EVAL_VLLM_REASONING_PARSER (default: unset) # EVAL_VLLM_DATA_PARALLEL_SIZE (default: unset; vLLM v0.8+ only) # EVAL_VLLM_EXTRA_ARGS (default: unset; space-separated string) +# EVAL_VLLM_HF_OVERRIDES (default: unset; JSON string for --hf-overrides) # ============================================================================== build_vllm_cmd() { @@ -31,18 +32,19 @@ build_vllm_cmd() { # Read overrides from env (set by listener via sbatch --export) local tp="${EVAL_VLLM_TENSOR_PARALLEL_SIZE:-4}" local dp="${EVAL_VLLM_DATA_PARALLEL_SIZE:-}" - local max_model_len="${EVAL_VLLM_MAX_MODEL_LEN:-}" + local max_model_len="${EVAL_VLLM_MAX_MODEL_LEN:-32768}" local swap_space="${EVAL_VLLM_SWAP_SPACE:-4}" local trust_remote_code="${EVAL_VLLM_TRUST_REMOTE_CODE:-}" local tool_call_parser="${EVAL_VLLM_TOOL_CALL_PARSER:-}" local reasoning_parser="${EVAL_VLLM_REASONING_PARSER:-}" local extra_args="${EVAL_VLLM_EXTRA_ARGS:-}" + local hf_overrides="${EVAL_VLLM_HF_OVERRIDES:-}" # Build command array VLLM_CMD=( "$python_bin" -m vllm.entrypoints.openai.api_server --model "$model" - --host 0.0.0.0 --port 8000 + --host 0.0.0.0 --port "${VLLM_PORT:-8000}" --served-model-name "$model" --tensor-parallel-size "$tp" --gpu-memory-utilization "$gpu_mem_util" @@ -70,6 +72,11 @@ build_vllm_cmd() { VLLM_CMD+=(--reasoning-parser "$reasoning_parser") fi + # HF model config overrides (JSON string, properly quoted) + if [ -n "$hf_overrides" ]; then + VLLM_CMD+=(--hf-overrides "$hf_overrides") + fi + # Append extra args (space-separated string) if [ -n "$extra_args" ]; then # shellcheck disable=SC2206 diff --git a/eval/check_progress.py b/eval/check_progress.py new file mode 100644 index 000000000..dcc6437ff --- /dev/null +++ b/eval/check_progress.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +"""Check progress of all running eval jobs on Jupiter. + +Usage: + python check_progress.py # grouped text output (default) + python check_progress.py --live # rich live dashboard + python check_progress.py --live -i 3 # live with 3s refresh + python check_progress.py --sort elapsed + python check_progress.py --jobs-dir /path/to/other/jobs # override jobs dir +""" + +import argparse +import subprocess +import json +import os +import sys +import time +from collections import defaultdict +from datetime import datetime +from pathlib import Path + +REPO_DIR = Path(__file__).resolve().parent.parent +LOGS_DIR = REPO_DIR / "eval" / "logs" +EXTRA_LOG_DIRS = [ + REPO_DIR / "eval" / "local" / "logs", # legacy v6 default + REPO_DIR / "eval" / "MBZ" / "logs", # legacy MBZ v4 + REPO_DIR / "eval" / "local" / "mbz" / "logs", # legacy MBZ v6 + REPO_DIR / "eval" / "local" / "m2" / "logs", # legacy M2 +] +DEFAULT_JOBS_DIR = REPO_DIR / "jobs" + + +# --------------------------------------------------------------------------- +# Data collection helpers (unchanged logic, refactored for reuse) +# --------------------------------------------------------------------------- + +def get_running_jobs(): + result = subprocess.run( + ["squeue", "-u", os.environ["USER"], "--format=%.10i %.15j %.8T %.20S", "--noheader"], + capture_output=True, text=True, + ) + jobs = [] + for line in result.stdout.strip().split("\n"): + parts = line.split() + if len(parts) >= 4: + jobs.append((parts[0].strip(), parts[1].strip(), parts[2].strip(), parts[3].strip())) + return jobs + + +def parse_eval_log(jid, job_name): + """Try multiple log naming patterns. SLURM %x is captured at submit time, + so renamed jobs (eval_dp -> eval_dp_v2) need fallback to original name.""" + # Search primary log dir + all extra log dirs + all_log_dirs = [LOGS_DIR] + [d for d in EXTRA_LOG_DIRS if d.exists()] + candidates = [] + for log_dir in all_log_dirs: + candidates.append(log_dir / f"{job_name}_{jid}.out") + if job_name.startswith("eval_dp_"): + for log_dir in all_log_dirs: + candidates.append(log_dir / f"eval_dp_{jid}.out") + elif job_name.startswith("eval_"): + for log_dir in all_log_dirs: + candidates.append(log_dir / f"eval_{jid}.out") + if job_name.startswith("res_dp_"): + for log_dir in all_log_dirs: + candidates.append(log_dir / f"res_dp_{jid}.out") + elif job_name.startswith("res_"): + for log_dir in all_log_dirs: + candidates.append(log_dir / f"res_{jid}.out") + model = bench = run_tag = None + num_shards = 0 + for log in candidates: + if log.exists(): + with open(log) as f: + for line in f: + if line.startswith("Model: "): + model = line.strip()[7:] + elif line.startswith("Dataset: "): + bench = line.strip()[9:] + elif line.startswith("Run tag: "): + run_tag = line.strip()[9:] + elif "total shards)" in line: + try: + num_shards = int(line.split("total shards")[0].split(",")[-1].strip()) + except (ValueError, IndexError): + pass + break + return model, bench, run_tag, num_shards + + +VALID_ERROR_TYPES = { + "AgentTimeoutError", "ContextLengthExceededError", + "SummarizationTimeout", "SummarizationTimeoutError", "BadRequestError", +} + + +def get_progress_single(run_tag, jobs_dir): + """Get progress from a single (non-DP) job dir.""" + if not run_tag: + return None, None, None, None, None, None + job_dir = jobs_dir / run_tag + rf = job_dir / "result.json" + if not rf.exists(): + return None, None, None, None, None, None + try: + with open(rf) as f: + d = json.load(f) + completed = d.get("stats", {}).get("n_trials", None) + total = d.get("n_total_trials", None) + finished = d.get("finished_at") is not None + invalid_trials = set() + evals = d.get("stats", {}).get("evals", {}) + # Extract accuracy from metrics + accuracy = None + for eval_data in evals.values(): + for error_type, trial_names in eval_data.get("exception_stats", {}).items(): + if error_type not in VALID_ERROR_TYPES and isinstance(trial_names, list): + invalid_trials.update(trial_names) + metrics = eval_data.get("metrics", []) + if metrics and isinstance(metrics, list): + mr = metrics[0].get("mean_reward") + if mr is not None: + accuracy = mr + # Use reward_stats to count completed trials (avoids expensive dir scan) + n_on_disk = None + for eval_data in evals.values(): + rs = eval_data.get("reward_stats", {}).get("reward", {}) + if rs: + n_on_disk = sum(len(v) for v in rs.values() if isinstance(v, list)) + break + if n_on_disk is None: + # Fallback: count from stats + n_on_disk = completed + return completed, total, len(invalid_trials), finished, n_on_disk, accuracy + except Exception: + return None, None, None, None, None, None + + +def get_progress(run_tag, num_shards, jobs_dir): + """Get progress, aggregating across shards for DP jobs.""" + if not run_tag: + return None, None, None, None, None, None + if num_shards > 1: + total_completed = 0 + total_total = 0 + total_errors = 0 + total_on_disk = 0 + all_finished = True + found_any = False + all_accuracies = [] + for shard_idx in range(num_shards): + shard_tag = f"{run_tag}_shard{shard_idx}" + c, t, e, fin, od, acc = get_progress_single(shard_tag, jobs_dir) + if c is not None: + found_any = True + total_completed += c + total_total += (t or 0) + total_errors += (e or 0) + total_on_disk += (od or 0) + if acc is not None: + all_accuracies.append(acc) + if not fin: + all_finished = False + if found_any: + avg_acc = (sum(all_accuracies) / len(all_accuracies)) if all_accuracies else None + return total_completed, total_total, total_errors, all_finished, total_on_disk, avg_acc + return None, None, None, None, None, None + else: + return get_progress_single(run_tag, jobs_dir) + + +def format_elapsed(start_str): + try: + if start_str == "N/A": + return "-", 0 + st = datetime.fromisoformat(start_str.replace("T", " ")) + delta = datetime.now() - st + secs = int(delta.total_seconds()) + hours, rem = divmod(secs, 3600) + mins, _ = divmod(rem, 60) + return f"{hours}h{mins:02d}m", secs + except Exception: + return "?", 0 + + +# --------------------------------------------------------------------------- +# Unified data collection +# --------------------------------------------------------------------------- + +def collect_job_data(jobs_dir): + """Collect all job data into structured dicts. + + Returns (running_data: list[dict], pending_jobs: list[tuple]). + """ + all_jobs = get_running_jobs() + if not all_jobs: + return [], [] + + running_raw = [(jid, name, state, start) for jid, name, state, start in all_jobs if state == "RUNNING"] + pending = [(jid, name, state, start) for jid, name, state, start in all_jobs if state == "PENDING"] + + running_data = [] + for jid, name, state, start_time in running_raw: + model, bench, run_tag, num_shards = parse_eval_log(jid, name) + completed, total, invalid_errors, finished, n_on_disk, accuracy = get_progress(run_tag, num_shards, jobs_dir) + + if finished: + status = "done" + elif completed is not None and total and completed >= total: + status = "retry" + else: + status = "active" + + elapsed, elapsed_secs = format_elapsed(start_time) + m_short = model.split("/")[-1] if model and "/" in model else (model or "?") + b_short = bench.split("/")[-1] if bench and "/" in bench else (bench or "?") + + is_resume = name.startswith("res_") + tags = [] + if is_resume: + tags.append("RES") + if num_shards > 1: + tags.append(f"{num_shards}x DP") + + # Progress percentage (based on disk for ground truth) + if n_on_disk is not None and total and total > 0: + progress_pct = n_on_disk / total + elif completed is not None and total and total > 0: + progress_pct = completed / total + else: + progress_pct = 0.0 + + running_data.append({ + "jid": jid, + "job_name": name, + "model": m_short, + "model_full": model or "?", + "bench": b_short, + "bench_full": bench or "?", + "run_tag": run_tag, + "num_shards": num_shards, + "completed": completed, + "total": total, + "n_on_disk": n_on_disk, + "n_invalid_errors": invalid_errors, + "accuracy": accuracy, + "finished": finished, + "elapsed": elapsed, + "elapsed_secs": elapsed_secs, + "status": status, + "tags": tags, + "progress_pct": progress_pct, + "is_resume": is_resume, + }) + + return running_data, pending + + +def group_by_benchmark(jobs, sort_key="progress"): + """Group jobs by benchmark, sort groups alphabetically, sort jobs within groups. + + sort_key: 'progress' (default), 'elapsed', 'model', 'errors' + """ + groups = defaultdict(list) + for job in jobs: + groups[job["bench"]].append(job) + + # Sort within each group + for bench in groups: + if sort_key == "elapsed": + groups[bench].sort(key=lambda j: j["elapsed_secs"], reverse=True) + elif sort_key == "model": + groups[bench].sort(key=lambda j: j["model"].lower()) + elif sort_key == "errors": + groups[bench].sort(key=lambda j: (j["n_invalid_errors"] or 0), reverse=True) + else: # progress (default) + groups[bench].sort(key=lambda j: j["progress_pct"], reverse=True) + + # Return as OrderedDict-like sorted by benchmark name + return dict(sorted(groups.items())) + + +def get_pending_reasons(): + """Get pending job reasons from squeue.""" + result = subprocess.run( + ["squeue", "-u", os.environ["USER"], "--format=%.10i %.8T %R", "--noheader"], + capture_output=True, text=True, + ) + reasons = {} + for line in result.stdout.strip().split("\n"): + parts = line.split(None, 2) + if len(parts) >= 3 and parts[1].strip() == "PENDING": + reasons[parts[0].strip()] = parts[2].strip() + return reasons + + +# --------------------------------------------------------------------------- +# Default text mode (enhanced with grouping) +# --------------------------------------------------------------------------- + +def print_default(running_data, pending, sort_key="progress"): + if not running_data and not pending: + print("No jobs in queue.") + return + + if running_data: + groups = group_by_benchmark(running_data, sort_key) + + # Count statuses + status_counts = defaultdict(int) + for j in running_data: + status_counts[j["status"]] += 1 + + print(f"\nRUNNING ({len(running_data)}):") + + header = f" {'JID':>8s} {'Progress':>10s} {'On Disk':>8s} {'Acc':>6s} {'Errors':>6s} {'Status':>6s} {'Elapsed':>8s} Model" + sep = " " + "-" * (len(header) - 2) + + for bench_name, jobs in groups.items(): + # Per-benchmark summary + n_active = sum(1 for j in jobs if j["status"] == "active") + n_retry = sum(1 for j in jobs if j["status"] == "retry") + n_done = sum(1 for j in jobs if j["status"] == "done") + parts = [] + if n_active: + parts.append(f"{n_active} active") + if n_retry: + parts.append(f"{n_retry} retry") + if n_done: + parts.append(f"{n_done} done") + status_str = ", ".join(parts) if parts else "0 jobs" + + print(f"\n === {bench_name} ({len(jobs)} jobs: {status_str}) ===") + print(header) + print(sep) + + for j in jobs: + progress = f"{j['completed']}/{j['total']}" if j["completed"] is not None else "-" + on_disk = f"{j['n_on_disk']}/{j['total']}" if j["n_on_disk"] is not None and j["total"] else "-" + acc = f"{j['accuracy']:.1%}" if j["accuracy"] is not None else "-" + errors = str(j["n_invalid_errors"]) if j["n_invalid_errors"] is not None else "-" + tag_str = f" [{', '.join(j['tags'])}]" if j["tags"] else "" + print(f" {j['jid']:>8s} {progress:>10s} {on_disk:>8s} {acc:>6s} {errors:>6s} {j['status']:>6s} {j['elapsed']:>8s} {j['model']}{tag_str}") + + if pending: + reasons = get_pending_reasons() + print(f"\n PENDING ({len(pending)}):") + print(f" {'JID':>8s} Reason") + print(" " + "-" * 38) + for jid, name, state, start in pending: + reason = reasons.get(jid, "?") + print(f" {jid:>8s} {reason}") + + # Summary line + status_counts = defaultdict(int) + for j in running_data: + status_counts[j["status"]] += 1 + parts = [] + for s in ["active", "retry", "done"]: + if status_counts[s]: + parts.append(f"{status_counts[s]} {s}") + status_detail = f" ({', '.join(parts)})" if parts else "" + print(f"\n Total: {len(running_data)} running{status_detail}, {len(pending)} pending\n") + + +# --------------------------------------------------------------------------- +# Rich Live dashboard +# --------------------------------------------------------------------------- + +def run_live_dashboard(jobs_dir, interval, sort_key="progress", compact=False, + benchmark_filter=None, page_mode=False): + from rich.console import Console, Group + from rich.table import Table + from rich.live import Live + from rich.text import Text + from rich.panel import Panel + from rich.progress_bar import ProgressBar + + console = Console() + page_idx = [0] # mutable for closure + + STATUS_STYLES = { + "active": "green", + "active_res": "purple", + "retry": "yellow", + "done": "dim", + } + + def _job_style(j): + """Get style key for a job: distinguish resume (cyan) from fresh (green) for active jobs.""" + if j["status"] == "active" and j["is_resume"]: + return "active_res" + return j["status"] + + def make_progress_bar(pct, style_key): + """Create a colored progress bar.""" + color = STATUS_STYLES.get(style_key, "white") + bar = ProgressBar(total=100, completed=int(pct * 100), width=10, + complete_style=color, finished_style=color) + return bar + + def render(): + running_data, pending = collect_job_data(jobs_dir) + now = datetime.now().strftime("%H:%M:%S") + + renderables = [] + + # Header + n_fresh = sum(1 for j in running_data if j["status"] == "active" and not j["is_resume"]) + n_resume = sum(1 for j in running_data if j["status"] == "active" and j["is_resume"]) + n_retry = sum(1 for j in running_data if j["status"] == "retry") + n_done = sum(1 for j in running_data if j["status"] == "done") + header_parts = [ + f"[bold]EVAL DASHBOARD[/bold]", + f"[green]{n_fresh} fresh[/green]" if n_fresh else None, + f"[purple]{n_resume} resume[/purple]" if n_resume else None, + f"[yellow]{n_retry} retry[/yellow]" if n_retry else None, + f"[dim]{n_done} done[/dim]" if n_done else None, + f"{len(pending)} pending" if pending else None, + f"[dim]Updated {now}[/dim]", + ] + header_text = " | ".join(p for p in header_parts if p) + renderables.append(Text.from_markup(header_text)) + renderables.append(Text("")) + + if not running_data and not pending: + renderables.append(Text("No jobs in queue.", style="dim")) + return Group(*renderables) + + # Group by benchmark + groups = group_by_benchmark(running_data, sort_key) + + # Apply benchmark filter + if benchmark_filter: + groups = {k: v for k, v in groups.items() + if benchmark_filter.lower() in k.lower()} + + # Page mode: show one benchmark at a time, rotating each refresh + if page_mode and groups: + bench_names = list(groups.keys()) + idx = page_idx[0] % len(bench_names) + selected = bench_names[idx] + groups = {selected: groups[selected]} + page_idx[0] += 1 + renderables.append(Text.from_markup( + f"[dim]Page {idx + 1}/{len(bench_names)}[/dim]")) + renderables.append(Text("")) + + for bench_name, jobs in groups.items(): + # Per-benchmark stats + b_fresh = sum(1 for j in jobs if j["status"] == "active" and not j["is_resume"]) + b_resume = sum(1 for j in jobs if j["status"] == "active" and j["is_resume"]) + b_retry = sum(1 for j in jobs if j["status"] == "retry") + b_done = sum(1 for j in jobs if j["status"] == "done") + b_errors = sum(j["n_invalid_errors"] or 0 for j in jobs) + avg_pct = sum(j["progress_pct"] for j in jobs) / len(jobs) if jobs else 0 + + status_parts = [] + if b_fresh: + status_parts.append(f"[green]{b_fresh} fresh[/green]") + if b_resume: + status_parts.append(f"[purple]{b_resume} resume[/purple]") + if b_retry: + status_parts.append(f"[yellow]{b_retry} retry[/yellow]") + if b_done: + status_parts.append(f"[dim]{b_done} done[/dim]") + error_str = f" [red]{b_errors} err[/red]" if b_errors > 10 else (f" {b_errors} err" if b_errors else "") + + title = (f"[bold cyan]{bench_name}[/bold cyan] " + f"{len(jobs)} jobs | {' | '.join(status_parts)}{error_str} | " + f"avg {avg_pct:.0%}") + + # In compact mode, only show active jobs in the table + display_jobs = [j for j in jobs if j["status"] == "active"] if compact else jobs + n_hidden = len(jobs) - len(display_jobs) + + table = Table( + show_header=True, header_style="bold", box=None, + padding=(0, 1), expand=True, + ) + table.add_column("JID", style="dim", width=7, justify="right") + table.add_column("Progress", width=8, justify="right") + if not compact: + table.add_column("Bar", width=12) + table.add_column("Disk", width=8, justify="right") + table.add_column("Acc", width=6, justify="right") + table.add_column("Err", width=4, justify="right") + table.add_column("Status", width=6, justify="right") + table.add_column("Elapsed", width=6, justify="right") + table.add_column("Model", no_wrap=False, ratio=1) + + for j in display_jobs: + skey = _job_style(j) + style = STATUS_STYLES.get(skey, "") + progress = f"{j['completed']}/{j['total']}" if j["completed"] is not None else "-" + on_disk = f"{j['n_on_disk']}/{j['total']}" if j["n_on_disk"] is not None and j["total"] else "-" + acc = f"{j['accuracy']:.1%}" if j["accuracy"] is not None else "-" + errors = str(j["n_invalid_errors"]) if j["n_invalid_errors"] is not None else "-" + error_style = "red bold" if (j["n_invalid_errors"] or 0) > 10 else "" + tag_str = f" [{', '.join(j['tags'])}]" if j["tags"] else "" + model_display = j["model_full"] + + row = [ + j["jid"], + Text(progress, style=style), + ] + if not compact: + row.append(make_progress_bar(j["progress_pct"], skey)) + row.extend([ + Text(on_disk, style=style), + Text(acc, style="cyan" if j["accuracy"] is not None else style), + Text(errors, style=error_style or style), + Text("resume" if skey == "active_res" else j["status"], style=style), + Text(j["elapsed"], style=style), + Text(f"{model_display}{tag_str}", style=style), + ]) + table.add_row(*row) + + renderables.append(Text.from_markup(title)) + if display_jobs: + renderables.append(table) + if n_hidden: + renderables.append(Text.from_markup( + f" [dim]... {n_hidden} done/retry job(s) hidden (use without --compact to show)[/dim]")) + renderables.append(Text("")) + + # Pending section + if pending: + reasons = get_pending_reasons() + pending_lines = [] + for jid, name, state, start in pending: + reason = reasons.get(jid, "?") + pending_lines.append(f" {jid} {reason}") + pending_text = "\n".join(pending_lines[:10]) + if len(pending) > 10: + pending_text += f"\n ... and {len(pending) - 10} more" + renderables.append(Panel( + pending_text, + title=f"[dim]PENDING ({len(pending)})[/dim]", + border_style="dim", + expand=False, + )) + + # Footer + tags = [] + if compact: + tags.append("compact") + if page_mode: + tags.append("paging") + if benchmark_filter: + tags.append(f"filter: {benchmark_filter}") + extra = f" | {', '.join(tags)}" if tags else "" + renderables.append(Text.from_markup( + f"[dim]Ctrl+C to exit | Refreshing every {interval}s | " + f"Sort: {sort_key}{extra}[/dim]" + )) + + return Group(*renderables) + + try: + with Live(render(), console=console, refresh_per_second=1, + vertical_overflow="ellipsis") as live: + while True: + time.sleep(interval) + live.update(render()) + except KeyboardInterrupt: + console.print("\n[dim]Dashboard stopped.[/dim]") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def parse_args(): + parser = argparse.ArgumentParser( + description="Check progress of running eval jobs on Jupiter.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +examples: + %(prog)s # grouped text output + %(prog)s --live # rich live dashboard + %(prog)s --live -i 3 # 3s refresh interval + %(prog)s --sort elapsed # sort by elapsed time + %(prog)s --sort errors # sort by error count + %(prog)s --live --compact # live, hide done/retry + no bar (fits more) + %(prog)s --live -b tb2 # live, only terminal_bench_2 + %(prog)s --live --page # live, rotate one benchmark per refresh +""", + ) + parser.add_argument( + "--live", "-w", action="store_true", + help="Launch Rich live dashboard (auto-refreshing)", + ) + parser.add_argument( + "--interval", "-i", type=int, default=5, + help="Refresh interval in seconds for --live mode (default: 5, min: 3)", + ) + parser.add_argument( + "--sort", "-s", choices=["progress", "elapsed", "model", "errors"], + default="progress", + help="Sort order within benchmark groups (default: progress)", + ) + parser.add_argument( + "--compact", "-c", action="store_true", + help="In --live mode, hide done/retry jobs and progress bar (fits more rows)", + ) + parser.add_argument( + "--benchmark", "-b", type=str, default=None, + help="Filter to benchmarks matching this substring (e.g. 'tb2', 'dev_set')", + ) + parser.add_argument( + "--page", "-p", action="store_true", + help="In --live mode, show one benchmark per page, rotating each refresh", + ) + parser.add_argument( + "--jobs-dir", type=Path, default=DEFAULT_JOBS_DIR, + help=f"Path to eval jobs directory (default: {DEFAULT_JOBS_DIR})", + ) + parser.add_argument( + "--logs-dir", type=Path, default=LOGS_DIR, + help=f"Path to eval logs directory (default: {LOGS_DIR})", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + global LOGS_DIR + LOGS_DIR = args.logs_dir + + if args.live: + interval = max(3, args.interval) + run_live_dashboard(args.jobs_dir, interval, sort_key=args.sort, + compact=args.compact, benchmark_filter=args.benchmark, + page_mode=args.page) + else: + running_data, pending = collect_job_data(args.jobs_dir) + if args.benchmark: + running_data = [j for j in running_data + if args.benchmark.lower() in j["bench"].lower()] + print_default(running_data, pending, sort_key=args.sort) + + +if __name__ == "__main__": + main() diff --git a/eval/clusters/jupiter.yaml b/eval/clusters/jupiter.yaml new file mode 100644 index 000000000..3a6a33d52 --- /dev/null +++ b/eval/clusters/jupiter.yaml @@ -0,0 +1,41 @@ +# Cluster configuration for Jupiter (JSC GH200 ARM nodes) +# Used by unified_eval_listener_v6.py --cluster-config eval/clusters/jupiter.yaml + +cluster_name: jupiter + +# SLURM +slurm_partition: booster +slurm_account: "reformo" +slurm_time: "12:00:00" + +# Conda environments: name → prefix directory (passed as OTAGENT_DIR to sbatch) +conda_envs: + otagent: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent + otagent2: /e/scratch/jureap59/zhuang1/conda/envs/otagent2 + +# Paths +paths: + project_root: /e/scratch/jureap59/zhuang1/OpenThoughts-Agent + hf_cache: /e/data1/datasets/playground/ot/hf_hub + eval_jobs_dir: /e/data1/datasets/playground/mmlaion/shared/zhuang1_eval_jobs + eval_logs_dir: eval/logs + listener_logs_dir: experiments/listener_logs + sbatch_script: eval/unified_eval_harbor.sbatch + dp_sbatch_script: eval/unified_eval_harbor_dp.sbatch + harbor_src: /e/scratch/jureap59/feuer1/harbor/src + datasets_dirs: + - /e/data1/datasets/playground/ot/datasets + - /e/scratch/jureap59/${USER}/datasets + secrets_file: ~/secrets.env + +# Proxy (Jupiter compute nodes have no internet) +proxy: + enabled: true + login_node: jpbl-s01-02 + proxychains_bin: /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4 + +# Hardware +hardware: + gpus_per_node: 4 + arch: aarch64 + cuda_home: /e/software/default/stages/2026/software/CUDA/13 diff --git a/eval/configs/dcagent_eval_config.yaml b/eval/configs/dcagent_eval_config.yaml new file mode 100644 index 000000000..414188fe4 --- /dev/null +++ b/eval/configs/dcagent_eval_config.yaml @@ -0,0 +1,47 @@ +# Harbor eval config (default) +# NOTE: jobs_dir is NOT set here — it's passed via `--jobs-dir` CLI flag +# from the sbatch, which reads EVAL_JOBS_DIR from the cluster config. +n_attempts: 3 +timeout_multiplier: 1.0 +orchestrator: + type: local + n_concurrent_trials: 16 + quiet: false + plain_output: true + retry: + max_retries: 3 + exclude_exceptions: + - AgentTimeoutError + - AgentEnvironmentTimeoutError + - BadRequestError + - VerifierTimeoutError + - SummarizationTimeout + - SummarizationTimeoutError + - ContextLengthExceededError + wait_multiplier: 1.0 + min_wait_sec: 1.0 + max_wait_sec: 60.0 +environment: + type: daytona + force_build: false +agents: + - name: terminus-2 + max_timeout_sec: 7200 + trajectory_config: + raw_content: true + linear_history: true + kwargs: + record_terminal_session: false + enable_episode_logging: false + enable_pane_logging: false + collect_rollout_details: false + collect_engine_metrics: false + metrics_endpoint: https://replace-with-vllm-host/metrics + metrics_timeout_sec: 10 + model_info: + max_input_tokens: 32768 + max_output_tokens: 8192 + input_cost_per_token: 0 + output_cost_per_token: 0 +datasets: + - path: examples/tasks diff --git a/eval/configs/dcagent_eval_config_no_override.yaml b/eval/configs/dcagent_eval_config_no_override.yaml new file mode 100644 index 000000000..b3df7d862 --- /dev/null +++ b/eval/configs/dcagent_eval_config_no_override.yaml @@ -0,0 +1,47 @@ +# Harbor eval config (no-override variant for swebench/tb2) +# NOTE: jobs_dir is NOT set here — it's passed via `--jobs-dir` CLI flag. +n_attempts: 3 +timeout_multiplier: 1.0 +orchestrator: + type: local + n_concurrent_trials: 4 + quiet: false + plain_output: true + retry: + max_retries: 3 + exclude_exceptions: + - AgentTimeoutError + - AgentEnvironmentTimeoutError + - BadRequestError + - VerifierTimeoutError + - SummarizationTimeout + - SummarizationTimeoutError + - ContextLengthExceededError + wait_multiplier: 1.0 + min_wait_sec: 1.0 + max_wait_sec: 60.0 +environment: + type: daytona + force_build: true + delete: false +agents: + - name: terminus-2 + max_timeout_sec: 7200 + trajectory_config: + raw_content: true + linear_history: true + kwargs: + record_terminal_session: false + enable_episode_logging: false + enable_pane_logging: false + collect_rollout_details: false + collect_engine_metrics: false + metrics_endpoint: https://replace-with-vllm-host/metrics + metrics_timeout_sec: 10 + model_info: + max_input_tokens: 32768 + max_output_tokens: 8192 + input_cost_per_token: 0 + output_cost_per_token: 0 +datasets: + - path: examples/tasks diff --git a/eval/docs/CLUSTER_ONBOARDING.md b/eval/docs/CLUSTER_ONBOARDING.md new file mode 100644 index 000000000..6cf71f833 --- /dev/null +++ b/eval/docs/CLUSTER_ONBOARDING.md @@ -0,0 +1,260 @@ +# Eval System — Cluster Onboarding Guide + +Set up the cluster-agnostic eval system on a new HPC cluster. After these steps you can run evals with `git pull` + a listener command. + +## Architecture + +``` +Listener (login node) + └── submits sbatch jobs ──> SLURM scheduler + │ + Compute node + ├── vLLM server (TP across GPUs) + ├── Harbor orchestrator (N concurrent trials) + │ └── Daytona cloud sandboxes (Docker) + └── Upload: HF traces + Supabase DB +``` + +**Key files:** + +| File | Purpose | +|------|---------| +| `eval/unified_eval_listener.py` | Polls DB, submits SLURM eval jobs | +| `eval/unified_eval_harbor.sbatch` | Single-node sbatch (starts vLLM + harbor) | +| `eval/unified_eval_harbor_dp.sbatch` | Multi-node data-parallel sbatch | +| `eval/clusters/.yaml` | Cluster config (paths, hardware, SLURM settings) | +| `hpc/dotenv/.env` | Env vars sourced by sbatch on compute nodes | +| `eval/configs/dcagent_eval_config.yaml` | Harbor agent/orchestrator config | +| `eval/baseline_model_configs.yaml` | Per-model vLLM overrides (TP, rope, etc.) | + +--- + +## Step 0: Gather Cluster Info + +Run on the cluster and record: + +```bash +sinfo --format="%P %D %G" --noheader # partitions, node count, GPUs +sinfo -N --format="%.30N %.5c %.10G %.10m" | head -5 # CPUs, GPUs, memory per node +uname -m # x86_64 or aarch64 +nvidia-smi | head -3 # GPU type, CUDA version +``` + +You need: **GPU count/type per node, CPUs per node, memory per node, architecture, CUDA path, SLURM partition/account, internet on compute nodes (yes/no).** + +--- + +## Step 1: Create Cluster Config YAML + +Create `eval/clusters/.yaml`. This is the single source of truth for all cluster-specific settings. + +```yaml +cluster_name: mycluster + +# SLURM +slurm_partition: gpu +slurm_account: "" # empty = no --account flag +slurm_time: "24:00:00" + +# Conda environments: name → prefix path +conda_envs: + otagent: /path/to/conda/envs/otagent + +# Paths (use ${USER} for multi-user support) +paths: + project_root: /path/to/OpenThoughts-Agent + hf_cache: /path/to/.cache/huggingface/hub + eval_jobs_dir: /path/to/OpenThoughts-Agent/jobs + eval_logs_dir: eval/logs + listener_logs_dir: experiments/listener_logs + sbatch_script: eval/unified_eval_harbor.sbatch + dp_sbatch_script: eval/unified_eval_harbor_dp.sbatch + harbor_src: /path/to/harbor/src + datasets_dirs: + - /path/to/.cache/huggingface/hub + secrets_file: ~/secrets.env + +# Proxy (for clusters where compute nodes have no internet) +proxy: + enabled: false + # If enabled: + # login_node: login01 + # proxychains_bin: /path/to/proxychains4 + +# Hardware (used for GPU/CPU scaling and job packing) +hardware: + gpus_per_node: 8 + cpus_per_node: 128 + mem_per_node_mb: 1600000 # from sinfo; omit to skip --mem in sbatch + arch: x86_64 # x86_64 or aarch64 + cuda_home: /usr/local/cuda +``` + +Reference: `eval/clusters/jupiter.yaml` for a working example (aarch64, proxy, 4 GPUs/node). + +--- + +## Step 2: Create Cluster Dotenv + +Create `hpc/dotenv/.env`. Sourced by the sbatch on compute nodes. + +```bash +export SCRATCH="/path/to/$USER" +export DCFT="$SCRATCH/OpenThoughts-Agent" +export DC_AGENT_SECRET_ENV=~/secrets.env +export HF_CACHE_DIR="$SCRATCH/.cache/huggingface" +export HF_HUB_CACHE="$HF_CACHE_DIR/hub" +export HF_HOME="$HF_CACHE_DIR" +export DATASETS_DIR="$HF_HUB_CACHE" +export MODELS_DIR="$HF_HUB_CACHE" +export VLLM_CACHE_ROOT="$SCRATCH/.cache/vllm" +export TRITON_CACHE_DIR="$SCRATCH/.cache/triton" +export FLASHINFER_CACHE_DIR="$SCRATCH/.cache/flashinfer" +export HF_HUB_ENABLE_HF_TRANSFER=1 +export PYTHONPATH="${DCFT}${PYTHONPATH:+:$PYTHONPATH}" +``` + +No secrets here — those go in `~/secrets.env`. + +--- + +## Step 3: Secrets (never commit) + +Create `~/secrets.env` on the cluster: + +```bash +export DAYTONA_API_KEY="dtn_..." +export DAYTONA_TARGET="" # empty = default region +export HF_TOKEN="hf_..." +export SUPABASE_URL="https://..." +export SUPABASE_ANON_KEY="..." +export SUPABASE_SERVICE_ROLE_KEY="..." +``` + +See `eval/secret.env.template` for the full template. + +--- + +## Step 4: Conda Environment + +### Required packages + +| Package | Version | Notes | +|---------|---------|-------| +| Python | 3.12 | | +| vLLM | ≥0.16 | `pip install vllm` (x86_64) or build from source (aarch64) | +| harbor | 0.1.45+ | `pip install -e /path/to/harbor` at commit `6fdb92e7` or later | +| hf_transfer | latest | `pip install hf_transfer` (fast HF downloads) | +| supabase | latest | For DB registration | + +```bash +conda create -n otagent python=3.12 -y +conda activate otagent +pip install vllm hf_transfer +pip install -e /path/to/harbor # clone from github.com/laude-institute/harbor +pip install -e /path/to/OpenThoughts-Agent +``` + +### Architecture notes + +- **x86_64** (H100/H200/A100): `pip install vllm` works directly +- **aarch64** (GH200): Must build vLLM from source or use a pre-built wheel. PyPI vLLM wheels are x86_64-only. See the Jupiter setup for reference. + +### Optional: second env for newer models + +Models like Qwen3.5 require vLLM ≥0.17 with newer transformers. Create a second env and add it to the cluster YAML: + +```yaml +conda_envs: + otagent: /path/to/envs/otagent + otagent2: /path/to/envs/otagent2 +``` + +Use `--conda-env otagent2` when evaluating these models. + +--- + +## Step 5: Pre-download Datasets + +Run on the login node (which has internet): + +```bash +source ~/secrets.env +python eval/snapshot_download.py DCAgent/dev_set_v2 +python eval/snapshot_download.py DCAgent2/terminal_bench_2 +python eval/snapshot_download.py DCAgent2/swebench-verified-random-100-folders +``` + +--- + +## Step 6: Create Directories & Verify + +```bash +mkdir -p eval/logs experiments/listener_logs +``` + +Dry-run the listener: + +```bash +source ~/secrets.env +PYTHONPATH=$PWD python eval/unified_eval_listener.py \ + --cluster-config eval/clusters/.yaml \ + --preset v2 \ + --priority-file eval/lists/example_models.txt \ + --baseline-model-configs eval/baseline_model_configs.yaml \ + --timeout-multiplier 2.0 --tp-size 2 --enable-thinking \ + --dry-run --once --verbose +``` + +Check for: `[v6] Cluster config: `, correct sbatch params, `[DRY RUN] Would submit`, no ERROR lines. + +--- + +## Running Evals + +### Basic launch + +```bash +source ~/secrets.env +PYTHONPATH=$PWD python eval/unified_eval_listener.py \ + --cluster-config eval/clusters/.yaml \ + --preset v2 \ + --priority-file eval/lists/.txt \ + --baseline-model-configs eval/baseline_model_configs.yaml \ + --timeout-multiplier 2.0 --tp-size 2 --enable-thinking \ + --auto-snapshot --once +``` + +### Presets + +| Preset | Dataset | Notes | +|--------|---------|-------| +| `v2` | DCAgent/dev_set_v2 | Dev eval (100 tasks) | +| `tb2` | DCAgent2/terminal_bench_2 | Terminal benchmark (89 tasks) | +| `swebench` | DCAgent2/swebench-verified-random-100-folders | SWE-bench (use `--no-auto-snapshot`) | +| `aider` | DCAgent2/aider_polyglot | Aider benchmark | +| `bfcl` | DCAgent2/bfcl-parity | BFCL benchmark | + +### Key flags + +| Flag | Purpose | +|------|---------| +| `--tp-size {1,2,4}` | Tensor parallelism (GPUs per vLLM replica) | +| `--dp-size {1,2,4,8}` | Data parallelism (vLLM replicas per job) | +| `--n-concurrent N` | Concurrent trials per job | +| `--timeout-multiplier F` | Scale agent timeout | +| `--force-reeval` | Bypass DB status checks | +| `--resume-only` | Only submit resume jobs | +| `--max-jobs-submitted N` | Cap active SLURM jobs | +| `--conda-env NAME` | Select conda env from cluster YAML | +| `--auto-snapshot` / `--no-auto-snapshot` | Daytona snapshot control | +| `--once` | Single iteration then exit | +| `--dry-run` | Show what would be submitted | + +### Monitoring + +```bash +python eval/check_progress.py # text output +python eval/check_progress.py --live # rich dashboard +python eval/check_progress.py --live -b tb2 # filter by benchmark +``` diff --git a/eval/lists/example_models.txt b/eval/lists/example_models.txt new file mode 100644 index 000000000..9c4d50a1d --- /dev/null +++ b/eval/lists/example_models.txt @@ -0,0 +1,7 @@ +# Example model priority list for the eval listener. +# One HuggingFace model name per line. Lines starting with # are comments. +# Usage: --priority-file eval/lists/example_models.txt +# +# DCAgent/a1-ghactions +# DCAgent/a1-magicoder +# myorg/my-finetuned-model diff --git a/eval/precreate_snapshots.py b/eval/precreate_snapshots.py new file mode 100644 index 000000000..0fee49667 --- /dev/null +++ b/eval/precreate_snapshots.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Pre-create Daytona snapshots for eval datasets. + +Computes unique environment hashes per dataset, then creates snapshots +on the Daytona org specified by the secrets file. Handles RL region routing +(builds via "us" but registers for "RL"). + +Usage: + python eval/MBZ/precreate_snapshots.py \ + --dataset DCAgent/dev_set_v2 \ + --dataset DCAgent2/terminal_bench_2 \ + --secrets-file ~/secrets.env + + python eval/MBZ/precreate_snapshots.py \ + --dataset DCAgent/dev_set_v2 \ + --dataset DCAgent2/terminal_bench_2 \ + --secrets-file ~/secrets_rl_org.env +""" + +import argparse +import asyncio +import os +import sys +from pathlib import Path + +# Ensure harbor is importable +REPO_ROOT = Path(__file__).resolve().parent.parent +HARBOR_SRC = REPO_ROOT / "harbor" / "src" +if str(HARBOR_SRC) not in sys.path: + sys.path.insert(0, str(HARBOR_SRC)) + + +def load_secrets(secrets_file: str) -> None: + """Parse 'export KEY=value' lines from a secrets file into os.environ.""" + with open(os.path.expanduser(secrets_file)) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[7:] + k, v = line.split("=", 1) + os.environ[k.strip()] = v.strip().strip('"').strip("'") + + +def resolve_dataset_path(repo_id: str) -> Path: + """Resolve a HF repo ID to the local cached snapshot path.""" + hf_cache = os.getenv("HF_HUB_CACHE", os.path.expanduser("~/.cache/huggingface/hub")) + dataset_dir = Path(hf_cache) / f"datasets--{repo_id.replace('/', '--')}" + snapshots_dir = dataset_dir / "snapshots" + + if not snapshots_dir.exists(): + raise FileNotFoundError( + f"Dataset {repo_id} not found in HF cache at {snapshots_dir}. " + "Download it first with snapshot_download.py." + ) + + snapshots = sorted( + [d for d in snapshots_dir.iterdir() if d.is_dir()], + key=lambda p: p.name, + ) + if not snapshots: + raise FileNotFoundError(f"No snapshots found in {snapshots_dir}") + + return snapshots[-1] + + +def get_task_dirs(dataset_path: Path) -> list[Path]: + """Find task directories (those containing instruction.md).""" + return sorted([ + d for d in dataset_path.iterdir() + if d.is_dir() + and not d.name.startswith(".") + and (d / "instruction.md").exists() + ]) + + +def compute_unique_snapshots( + datasets: list[tuple[str, Path]], + target: str | None, +) -> dict[str, str]: + """Compute unique snapshot names and a representative Dockerfile for each. + + Returns: + Dict of snapshot_name -> representative Dockerfile path (as string). + """ + from harbor.utils.container_cache import get_task_environment_hash + + snapshots: dict[str, str] = {} # snapshot_name -> dockerfile_path + + for repo_id, dataset_path in datasets: + task_dirs = get_task_dirs(dataset_path) + print(f"\n=== {repo_id} ({len(task_dirs)} tasks) ===") + + hash_counts: dict[str, int] = {} + hash_example: dict[str, Path] = {} + + for task_dir in task_dirs: + env_hash = get_task_environment_hash(task_dir) + if env_hash is None: + continue + hash_counts[env_hash] = hash_counts.get(env_hash, 0) + 1 + if env_hash not in hash_example: + hash_example[env_hash] = task_dir + + print(f"Unique environment hashes: {len(hash_counts)}") + + for env_hash, count in sorted(hash_counts.items(), key=lambda x: -x[1]): + if target: + snap_name = f"harbor__{env_hash}__{target}__snapshot" + else: + snap_name = f"harbor__{env_hash}__snapshot" + + example_task = hash_example[env_hash] + dockerfile = example_task / "environment" / "Dockerfile" + + print(f" {snap_name} ({count} tasks) e.g. {example_task.name}") + snapshots[snap_name] = str(dockerfile) + + return snapshots + + +async def create_snapshot( + client, + name: str, + dockerfile_path: str, + target: str | None, + label: str, +) -> bool: + """Create a single snapshot, handling already-exists gracefully.""" + from daytona import AsyncDaytona, DaytonaConfig, CreateSnapshotParams, Image, Resources + from daytona._async.snapshot import SnapshotState + + # Check existing state + try: + snap = await client.snapshot.get(name) + if snap.state == SnapshotState.ACTIVE: + print(f" [{label}] {name}: already ACTIVE, skipping") + return True + elif snap.state == SnapshotState.ERROR: + print(f" [{label}] {name}: ERROR state, deleting and recreating...") + try: + await client.snapshot.delete(snap) + except Exception as e: + print(f" [{label}] {name}: failed to delete ERROR snapshot: {e}") + except Exception: + pass # Doesn't exist yet + + # RL region has no build runners — build via "us", register for RL + build_client = client + if target and target.upper() == "RL": + api_key = os.environ.get("DAYTONA_API_KEY", "") + build_client = AsyncDaytona(DaytonaConfig(api_key=api_key, target="us")) + + print(f" [{label}] Creating {name} from {dockerfile_path}...") + try: + await build_client.snapshot.create( + CreateSnapshotParams( + name=name, + image=Image.from_dockerfile(dockerfile_path), + resources=Resources(cpu=1, memory=1, disk=3), + region_id=target if target else None, + ) + ) + except Exception as e: + error_msg = str(e).lower() + if "already exists" in error_msg or "conflict" in error_msg: + print(f" [{label}] {name}: already exists (global), OK") + return True + print(f" [{label}] {name}: create FAILED: {e}") + return False + finally: + if build_client is not client: + await build_client.close() + + # Poll for ACTIVE state (up to 10 minutes, 5s intervals) + for i in range(120): + await asyncio.sleep(5) + try: + snap = await client.snapshot.get(name) + if snap.state == SnapshotState.ACTIVE: + print(f" [{label}] {name}: ACTIVE (took ~{(i+1)*5}s)") + return True + elif snap.state == SnapshotState.ERROR: + reason = getattr(snap, "error_reason", "unknown") + print(f" [{label}] {name}: entered ERROR state: {reason}") + return False + except Exception: + pass + + print(f" [{label}] {name}: TIMEOUT waiting for ACTIVE (600s)") + return False + + +async def main(): + parser = argparse.ArgumentParser( + description="Pre-create Daytona snapshots for eval datasets" + ) + parser.add_argument( + "--dataset", + action="append", + required=True, + help="HF repo ID (e.g. DCAgent/dev_set_v2). Can be repeated.", + ) + parser.add_argument( + "--secrets-file", + required=True, + help="Path to secrets env file (must contain DAYTONA_API_KEY and DAYTONA_TARGET)", + ) + args = parser.parse_args() + + # Load secrets into environment + load_secrets(args.secrets_file) + + api_key = os.environ.get("DAYTONA_API_KEY") + target = os.environ.get("DAYTONA_TARGET") + + if not api_key: + print("ERROR: DAYTONA_API_KEY not found in secrets file", file=sys.stderr) + sys.exit(1) + + label = target or "default" + print(f"Daytona target: {label}") + print(f"API key: {api_key[:12]}...") + + # Resolve dataset paths + datasets: list[tuple[str, Path]] = [] + for repo_id in args.dataset: + path = resolve_dataset_path(repo_id) + print(f"Dataset {repo_id} -> {path}") + datasets.append((repo_id, path)) + + # Compute unique snapshots + snapshots = compute_unique_snapshots(datasets, target) + + if not snapshots: + print("\nNo snapshots to create (no tasks with Dockerfiles found).") + return + + print(f"\n--- Creating {len(snapshots)} unique snapshots on '{label}' ---\n") + + # Create snapshots + from daytona import AsyncDaytona, DaytonaConfig + + client = AsyncDaytona(DaytonaConfig(api_key=api_key, target=target or "us")) + try: + results = {} + for snap_name, dockerfile in snapshots.items(): + ok = await create_snapshot(client, snap_name, dockerfile, target, label) + results[snap_name] = ok + finally: + await client.close() + + # Summary + succeeded = sum(1 for v in results.values() if v) + failed = sum(1 for v in results.values() if not v) + print(f"\n=== Done: {succeeded} succeeded, {failed} failed ===") + + if failed: + print("\nFailed snapshots:") + for name, ok in results.items(): + if not ok: + print(f" - {name}") + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/eval/secret.env.template b/eval/secret.env.template new file mode 100644 index 000000000..720a1ca7c --- /dev/null +++ b/eval/secret.env.template @@ -0,0 +1,7 @@ +export DAYTONA_API_KEY='YOUR_DAYTONA_API_KEY' +export DAYTONA_TARGET='us' +export HF_TOKEN='YOUR_HF_TOKEN' +export SUPABASE_URL='YOUR_SUPABASE_URL' +export SUPABASE_ANON_KEY='YOUR_SUPABASE_ANON_KEY' +export SUPABASE_SERVICE_ROLE_KEY='YOUR_SUPABASE_SERVICE_ROLE_KEY' +export SSH_KEY="$HOME/.ssh/id_ed25519_jsc" diff --git a/eval/snapshot_download.py b/eval/snapshot_download.py new file mode 100644 index 000000000..882c1f85e --- /dev/null +++ b/eval/snapshot_download.py @@ -0,0 +1,165 @@ +import os +import sys +import argparse +import fcntl +import time +from huggingface_hub import snapshot_download + +def is_valid_task_dir(path): + """ + Check if a directory is a valid task directory by verifying it has instruction.md + and excludes Git-related files. + """ + # Skip Git-related files and directories + basename = os.path.basename(path) + if basename.startswith('.git') or basename in {'.gitignore', '.gitattributes', '.github'}: + return False + + # Check if it's a directory and has instruction.md + return (os.path.isdir(path) and + os.path.isfile(os.path.join(path, 'instruction.md'))) + +def get_dataset_path(repo_id): + """ + Get the path to the dataset without downloading + Returns the path if it exists, None otherwise + + Args: + repo_id (str): The repository ID to look for + """ + + # Construct the path using HF_HUB_CACHE environment variable + hf_cache = os.getenv('HF_HUB_CACHE', os.path.expanduser('~/.cache/huggingface/hub')) + dataset_path = os.path.join(hf_cache, f"datasets--{repo_id.replace('/', '--')}") + + # Find the latest snapshot + snapshots_dir = os.path.join(dataset_path, 'snapshots') + if os.path.exists(snapshots_dir): + try: + snapshots = [d for d in os.listdir(snapshots_dir) if os.path.isdir(os.path.join(snapshots_dir, d))] + except OSError as e: + print(f"Could not read snapshots directory {snapshots_dir}: {e}", file=sys.stderr) + snapshots = [] + if snapshots: + latest_snapshot = sorted(snapshots)[-1] # Get the latest snapshot + snapshot_path = os.path.join(snapshots_dir, latest_snapshot) + + # Verify we have valid task directories + if os.path.exists(snapshot_path): + task_dirs = [d for d in os.listdir(snapshot_path) + if is_valid_task_dir(os.path.join(snapshot_path, d))] + + if task_dirs: + print(f"Found dataset at {snapshot_path}") + print(f"Found {len(task_dirs)} valid task directories") + return snapshot_path + else: + print("No valid task directories found in snapshot") + return None + + print("Dataset not found, downloading...") + return None + +def download_sandboxes_dataset(repo_id, local_dir=None, cache_dir=None): + """ + Download the dataset using snapshot_download + + Args: + repo_id (str): The repository ID (e.g., 'mlfoundations-dev/sandboxes-tasks') + local_dir (str, optional): Local directory to save the dataset + cache_dir (str, optional): Cache directory for huggingface hub + """ + + try: + print(f"Starting download of {repo_id}...") + + # Download the entire dataset repository + local_path = snapshot_download( + repo_id=repo_id, + repo_type="dataset", + local_dir=local_dir, + cache_dir=cache_dir, + ) + + # Remove .gitattributes file if it exists + gitattributes_path = os.path.join(local_path, '.gitattributes') + if os.path.exists(gitattributes_path): + os.remove(gitattributes_path) + print("Removed .gitattributes file") + + # Verify we have valid task directories + task_dirs = [d for d in os.listdir(local_path) + if is_valid_task_dir(os.path.join(local_path, d))] + + if task_dirs: + print(f"DATASET_PATH={local_path}") + print(f"Found {len(task_dirs)} valid task directories") + return local_path + else: + print("No valid task directories found in downloaded dataset") + return None + + except Exception as e: + print(f"Error downloading dataset: {e}") + return None + + +def main(): + parser = argparse.ArgumentParser(description='Download or locate a Hugging Face dataset') + parser.add_argument('repo_id', help='Repository ID (e.g., mlfoundations-dev/clean-sandboxes-tasks)') + parser.add_argument('--local-dir', help='Local directory to save the dataset') + parser.add_argument('--cache-dir', help='Cache directory for huggingface hub') + + args = parser.parse_args() + + path = None + if args.local_dir: + # When --local-dir is specified, download real files (no symlinks). + # Use a file lock to prevent race conditions when multiple SLURM jobs + # download the same dataset concurrently. + lock_path = args.local_dir.rstrip("/") + ".lock" + os.makedirs(os.path.dirname(lock_path) or ".", exist_ok=True) + lock_fd = open(lock_path, "w") + try: + print(f"Acquiring dataset lock: {lock_path}", file=sys.stderr) + fcntl.flock(lock_fd, fcntl.LOCK_EX) + print("Lock acquired.", file=sys.stderr) + + # Check if local_dir already has valid task dirs + if os.path.isdir(args.local_dir): + task_dirs = [d for d in os.listdir(args.local_dir) + if is_valid_task_dir(os.path.join(args.local_dir, d))] + if task_dirs: + print(f"Found existing dataset at {args.local_dir} with {len(task_dirs)} tasks") + path = args.local_dir + if not path: + print("Downloading dataset to local dir (real files, no symlinks)...", file=sys.stderr) + path = download_sandboxes_dataset( + repo_id=args.repo_id, + local_dir=args.local_dir, + cache_dir=args.cache_dir + ) + finally: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + lock_fd.close() + else: + # First try to get existing cached path + path = get_dataset_path(args.repo_id) + if not path: + # If not found, download it + print("Dataset not found, downloading...", file=sys.stderr) + path = download_sandboxes_dataset( + repo_id=args.repo_id, + local_dir=args.local_dir, + cache_dir=args.cache_dir + ) + + if path: + print(f"DATASET_PATH={path}") + return 0 + else: + print("Failed to download dataset", file=sys.stderr) + return 1 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/test_dp_eval.sh b/eval/test_dp_eval.sh new file mode 100644 index 000000000..e9de47c76 --- /dev/null +++ b/eval/test_dp_eval.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# ============================================================================== +# Test: vLLM native Data Parallel eval (DP=4, TP=1) on single node +# +# Uses vLLM's --data-parallel-size flag to run 4 model replicas (one per GPU), +# each with TP=1. vLLM load-balances requests across replicas internally. +# Harbor sees a single endpoint — no sharding needed. +# +# Usage: +# bash eval/MBZ/test_dp_eval.sh +# +# To clean up DB entries from failed tests: +# python -c " +# from database.unified_db.utils import delete_sandbox_job_by_id, load_supabase_keys +# load_supabase_keys() +# delete_sandbox_job_by_id('') +# " +# ============================================================================== + +set -euo pipefail + +MODEL="DCAgent/a1-nl2bash" +DATASET="DCAgent/dev_set_v2" +BENCHMARK_ID="b94dfab2-c438-4c32-ba29-23e46d566763" + +# DP config: 4 replicas, TP=1 each, 16 concurrent harbor trials per replica = 64 total +DP_SIZE=4 +TP_SIZE=1 +N_CONCURRENT=64 # 16 per DP replica × 4 replicas +GPU_MEMORY_UTIL=0.9 +TIMEOUT_MULTIPLIER=2.0 + +echo "=== vLLM Native DP Test ===" +echo "Model: $MODEL" +echo "Dataset: $DATASET" +echo "DP=$DP_SIZE, TP=$TP_SIZE, N_CONCURRENT=$N_CONCURRENT" +echo "" + +# Submit using the v6 sbatch with DP env vars +EVAL_VLLM_TENSOR_PARALLEL_SIZE=$TP_SIZE \ +EVAL_VLLM_DATA_PARALLEL_SIZE=$DP_SIZE \ +EVAL_N_CONCURRENT=$N_CONCURRENT \ +EVAL_GPU_MEMORY_UTIL=$GPU_MEMORY_UTIL \ +EVAL_ENABLE_THINKING=true \ +EVAL_TIMEOUT_MULTIPLIER=$TIMEOUT_MULTIPLIER \ +EVAL_VLLM_MAX_RETRIES=10 \ +EVAL_DAYTONA_THRESHOLD=10 \ +EVAL_CONDA_ENV=otagent2 \ +EVAL_AUTO_SNAPSHOT=true \ +EVAL_CONFIG_YAML=dcagent_eval_config_no_override.yaml \ +sbatch \ + --time 24:00:00 \ + --partition main \ + --gres gpu:4 \ + --cpus-per-task=32 \ + --job-name data_dp_test \ + --output eval/local/logs/data_dp_test_%j.out \ + eval/unified_eval_harbor.sbatch \ + "$MODEL" "$DATASET" "$BENCHMARK_ID" "" + +echo "Submitted! Monitor with:" +echo " squeue -u \$USER" +echo " tail -f eval/local/logs/data_dp_test_*.out" +echo " tail -f eval/local/logs/vllm_*.log" diff --git a/eval/unified_eval_harbor.sbatch b/eval/unified_eval_harbor.sbatch new file mode 100644 index 000000000..5d74e866e --- /dev/null +++ b/eval/unified_eval_harbor.sbatch @@ -0,0 +1,1021 @@ +#!/bin/bash +#SBATCH -p booster +#SBATCH --time=12:00:00 +#SBATCH --signal=B:TERM@120 +#SBATCH --nodes 1 +#SBATCH --ntasks-per-node 1 +#SBATCH --cpus-per-task=24 +#SBATCH --gres=gpu:2 +#SBATCH --output=eval/logs/%x_%j.out +#SBATCH --job-name=eval + +# ============================================================================== +# Unified Eval Harbor v4 — Jupiter Cluster (JSC GH200) +# +# Starts vLLM, runs Harbor eval, checks errors, uploads results. +# Merged from Jupiter v1 env setup + TACC v4 features (configurable params, +# Pending→Started DB flow, unified error checking, thinking, export-traces). +# +# Positional args: +# $1 = MODEL (HF model name, e.g. mlfoundations-dev/some_model) +# $2 = REPO_ID (HF dataset repo, e.g. DCAgent/dev_set_v2) +# $3 = BENCHMARK_ID (optional, DB benchmark UUID) +# $4 = RUN_TAG_ARG (optional, override run tag) +# +# Env vars from listener (v4 SbatchParams.to_env()): +# EVAL_N_CONCURRENT (default: 128) +# EVAL_N_ATTEMPTS (default: 3) +# EVAL_GPU_MEMORY_UTIL (default: 0.95) +# EVAL_DAYTONA_THRESHOLD (default: 999999) +# EVAL_VLLM_MAX_RETRIES (default: 20) +# EVAL_AGENT_PARSER (default: "") +# EVAL_ENABLE_THINKING (default: "false") +# EVAL_AGENT_NAME (default: "terminus-2") +# EVAL_STARTS_LOG (optional, shared eval starts log) +# EVAL_TIMEOUT_MULTIPLIER (default: 1.0) +# EVAL_CONFIG_YAML (default: dcagent_eval_config.yaml) +# EVAL_DB_JOB_ID (optional, pre-created Pending job ID) +# EVAL_UPLOAD_USERNAME (optional, override upload username) +# EVAL_OVERRIDE_MEMORY_MB (optional, memory override for harbor) +# EVAL_AUTO_SNAPSHOT (optional, "true"/"false" override for auto_snapshot) +# EVAL_SNAPSHOT_NAME (optional, Daytona snapshot template) +# ============================================================================== + +set -eo pipefail +ulimit -c 0 # Disable core dumps +ulimit -n 65536 2>/dev/null || true + +TIMESTAMP=$(date +'%Y%m%d_%H%M%S') + +# --- Parse positional args --- +MODEL="${1:-mlfoundations-dev/claude_3_7_20250219_tbench_traces_sharegptv1}" +REPO_ID="${2:-DCAgent/dev_set_71_tasks}" +BENCHMARK_ID="${3:-}" +RUN_TAG_ARG="${4:-}" + +# --- Read env vars from listener (with defaults) --- +N_CONCURRENT="${EVAL_N_CONCURRENT:-128}" +N_ATTEMPTS="${EVAL_N_ATTEMPTS:-3}" +GPU_MEMORY_UTIL="${EVAL_GPU_MEMORY_UTIL:-0.95}" +ERROR_THRESHOLD="${EVAL_DAYTONA_THRESHOLD:-999999}" +VLLM_MAX_RETRIES="${EVAL_VLLM_MAX_RETRIES:-20}" +AGENT_PARSER="${EVAL_AGENT_PARSER:-}" +ENABLE_THINKING="${EVAL_ENABLE_THINKING:-false}" +AGENT_NAME="${EVAL_AGENT_NAME:-terminus-2}" +EVAL_STARTS_LOG="${EVAL_STARTS_LOG:-}" +TIMEOUT_MULTIPLIER="${EVAL_TIMEOUT_MULTIPLIER:-1.0}" +CONFIG_YAML="${EVAL_CONFIG_YAML:-dcagent_eval_config.yaml}" +DB_JOB_ID="${EVAL_DB_JOB_ID:-}" +UPLOAD_USERNAME="${EVAL_UPLOAD_USERNAME:-$USER}" + +# Strip slashes and special chars for file-safe names +SAFE_MODEL=$(echo "$MODEL" | tr '/:' '_') +if [[ "$REPO_ID" == /* ]]; then + # Local path: use basename as the safe repo name + SAFE_REPO=$(basename "$REPO_ID") +else + SAFE_REPO=$(echo "$REPO_ID" | tr '/:' '_') +fi + +# Derive benchmark shorthand for SLURM job name (for squeue readability) +# Listener passes --job-name to override, but for manual runs derive from REPO_ID. +declare -A BENCH_SHORT=( + ["DCAgent_dev_set_v2"]="v2" + ["DCAgent2_swebench-verified-random-100-folders"]="swe" + ["DCAgent2_terminal_bench_2"]="tb2" + ["DCAgent2_aider_polyglot"]="aider" + ["DCAgent2_bfcl-parity"]="bfcl" + ["DCAgent_dev_set_71_tasks"]="v1" +) +BENCH_TAG="${BENCH_SHORT[$SAFE_REPO]:-${SAFE_REPO:0:12}}" +# Only rename if still the default "eval" (i.e. listener didn't override) +if [ "$SLURM_JOB_NAME" = "eval" ]; then + scontrol update JobId="$SLURM_JOB_ID" JobName="eval_${BENCH_TAG}" +fi + +echo "==============================================" +echo "Jupiter Eval Harbor v4" +echo "==============================================" +echo "Model: $MODEL" +echo "Dataset: $REPO_ID" +echo "Benchmark ID: ${BENCHMARK_ID:-}" +echo "N concurrent: $N_CONCURRENT" +echo "N attempts: $N_ATTEMPTS" +echo "GPU memory util: $GPU_MEMORY_UTIL" +echo "Error threshold: $ERROR_THRESHOLD" +echo "vLLM max retries: $VLLM_MAX_RETRIES" +echo "Agent: $AGENT_NAME" +echo "Agent parser: ${AGENT_PARSER:-}" +echo "Thinking: $ENABLE_THINKING" +echo "Timeout multiplier: $TIMEOUT_MULTIPLIER" +echo "Config YAML: $CONFIG_YAML" +echo "DB Job ID (pending): ${DB_JOB_ID:-}" +echo "Upload username: $UPLOAD_USERNAME" +echo "==============================================" + +# ============================================================================== +# Cluster-Agnostic Environment Setup +# ============================================================================== + +DCFT="${EVAL_PROJECT_ROOT:-${DCFT:?ERROR: EVAL_PROJECT_ROOT not set. Launch via the listener with --cluster-config.}}" +CLUSTER_NAME="${EVAL_CLUSTER_NAME:?ERROR: EVAL_CLUSTER_NAME not set. Launch via the listener with --cluster-config.}" + +# Reset PYTHONPATH to avoid inheriting submitter's environment (e.g. numpy 2.4 +# from guha1/site-packages which breaks numba/vLLM). +unset PYTHONPATH + +# Preserve listener-provided overrides before dotenv clobbers them +_SAVED_SECRET_ENV="${DC_AGENT_SECRET_ENV:-}" + +# Source cluster-specific dotenv (e.g. hpc/dotenv/jupiter.env or hpc/dotenv/mbz.env) +DOTENV_FILE="$DCFT/hpc/dotenv/${CLUSTER_NAME}.env" +if [ -f "$DOTENV_FILE" ]; then + source "$DOTENV_FILE" + echo "Sourced dotenv: $DOTENV_FILE" +else + echo "WARNING: dotenv not found: $DOTENV_FILE" +fi + +# Source secrets (listener override takes precedence over dotenv) +DC_AGENT_SECRET_ENV="${_SAVED_SECRET_ENV:-${DC_AGENT_SECRET_ENV:-$HOME/secrets.env}}" +if [ -f "$DC_AGENT_SECRET_ENV" ]; then + source "$DC_AGENT_SECRET_ENV" +fi + +# --- Daytona API key (sourced from secrets.env above) --- +if [ -z "${DAYTONA_API_KEY:-}" ]; then + echo "WARNING: DAYTONA_API_KEY not set. Check $DC_AGENT_SECRET_ENV" +fi +echo "Daytona API key: ${DAYTONA_API_KEY:0:12}..." + +# --- vLLM / Ray / Triton env vars --- +export VLLM_USE_V1=1 +export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook +export VLLM_CONFIG_ROOT="${VLLM_CACHE_ROOT:-/tmp/vllm_config_${USER}}" +export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-/tmp/triton_cache_${USER}}" +export FLASHINFER_WORKSPACE_BASE="${FLASHINFER_CACHE_DIR:-/tmp/flashinfer_cache_${USER}}" +export UV_CACHE_DIR="${UV_CACHE_DIR:-/tmp/uv_cache_${USER}}" +export HYDRA_FULL_ERROR=1 +# Use naive all2all backend for DP (pplx_kernels requires separate install) +export VLLM_ALL2ALL_BACKEND="${VLLM_ALL2ALL_BACKEND:-naive}" +# Model/dataset cache — overridden by EVAL_HF_CACHE from listener +export HF_HUB_CACHE="${EVAL_HF_CACHE:-${HF_HUB_CACHE:?ERROR: HF_HUB_CACHE not set. Source your cluster dotenv or use --cluster-config.}}" +export HF_HOME="${HF_HUB_CACHE}" +export HF_CACHE_DIR="$HF_HUB_CACHE" +# XET staging cache — must be user-writable +export HF_XET_CACHE="${HF_XET_CACHE:-/tmp/hf_xet_cache_${USER}}" +mkdir -p "$HF_XET_CACHE" + +# Harbor and DB operations +HARBOR_SRC="${EVAL_HARBOR_SRC:-}" +export PYTHONPATH="${HARBOR_SRC}:${DCFT}:${PYTHONPATH:-}" + +# ============================================================================== +# LD_LOADER Wrappers (needed on aarch64 Jupiter where shared conda lacks exec perms) +# ============================================================================== + +OTAGENT_DIR="${OTAGENT_DIR:?ERROR: OTAGENT_DIR not set. Use --conda-env with the listener.}" +export PATH="$OTAGENT_DIR/bin:$PATH" +export CONDA_PREFIX="$OTAGENT_DIR" + +PYTHON_REAL="$OTAGENT_DIR/bin/python3.12" + +# LD_LOADER Wrappers: required on aarch64 (Jupiter GH200) where shared conda +# lacks execute permissions. Skipped on x86_64 where the conda env is directly usable. +if [ "$(uname -m)" = "aarch64" ] && [ -f /lib/ld-linux-aarch64.so.1 ]; then + LD_LOADER="/lib/ld-linux-aarch64.so.1" + WRAPPER_DIR="/tmp/eval_wrappers_${SLURM_JOB_ID}" + mkdir -p "$WRAPPER_DIR" + + cat > "$WRAPPER_DIR/python3" < "$WRAPPER_DIR/python" < "$WRAPPER_DIR/harbor" </dev/null +export RAY_DEDUP_LOGS=0 + +# Set CUDA_HOME for flashinfer JIT compilation (needs nvcc) +# The CUDA module provides nvcc at this path on Jupiter GH200 nodes. +_CUDA_HOME="${EVAL_CUDA_HOME:-/e/software/default/stages/2026/software/CUDA/13}" +if [ -n "${_CUDA_HOME}" ] && [ -d "${_CUDA_HOME}" ]; then + export CUDA_HOME="${_CUDA_HOME}" + export PATH="$CUDA_HOME/bin:$PATH" +fi + + +# ============================================================================== +# SSH Tunnel + Proxychains Setup (no internet on compute nodes) +# ============================================================================== + +NODE_HOST=$(hostname -s) +TUNNEL_PORT=7003 +LOGIN_NODE="${EVAL_LOGIN_NODE:-jpbl-s01-02}" +PROXYCHAINS_BIN="${EVAL_PROXYCHAINS_BIN:-}" +TUNNEL_PID="" + +setup_proxy() { + echo "[proxy] Setting up SSH tunnel to $LOGIN_NODE..." + + if [ -z "${SSH_KEY:-}" ]; then + echo "[proxy] WARNING: SSH_KEY not set, skipping tunnel setup" + echo "[proxy] External connectivity (Daytona, HF) will fail" + return 1 + fi + + # Get node IP for multi-node access + NODE_IP=$(nslookup "$NODE_HOST" 2>/dev/null | grep 'Address' | tail -n1 | awk '{print $2}') + NODE_IP="${NODE_IP:-127.0.0.1}" + + # Create SSH tunnel with SOCKS5 proxy + ssh -g -f -N -D ${TUNNEL_PORT} \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=1000 \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=30 \ + -o TCPKeepAlive=yes \ + -o ExitOnForwardFailure=yes \ + -o BatchMode=yes \ + -i "${SSH_KEY}" \ + "${USER}@${LOGIN_NODE}" + + sleep 5 + + # Verify tunnel + if pgrep -f "ssh.*-D.*${TUNNEL_PORT}" > /dev/null; then + echo "[proxy] SSH tunnel started successfully" + TUNNEL_PID=$(pgrep -f "ssh.*-D.*${TUNNEL_PORT}" | head -1) + else + echo "[proxy] ERROR: SSH tunnel failed to start" + return 1 + fi + + # Generate proxychains config + CFG_PATH="$HOME/.proxychains/proxychains_${SLURM_JOB_ID}.conf" + mkdir -p "$HOME/.proxychains" + + cat > "$CFG_PATH" </dev/null; then + echo "[proxy] HuggingFace connectivity test passed" + else + echo "[proxy] WARNING: HuggingFace connectivity test failed" + fi + fi + + # Test Daytona API connectivity + if [ -n "${DAYTONA_API_KEY:-}" ]; then + echo "[daytona] Testing API connectivity..." + DAYTONA_HTTP_CODE=$("$PROXYCHAINS_BIN" -f "$CFG_PATH" curl -s --connect-timeout 15 \ + -H "Authorization: Bearer $DAYTONA_API_KEY" \ + "${DAYTONA_API_URL:-https://app.daytona.io/api}/health" \ + -o /tmp/daytona_health_$$.json -w "%{http_code}" 2>/dev/null) || DAYTONA_HTTP_CODE="FAIL" + echo "[daytona] Health check HTTP code: $DAYTONA_HTTP_CODE" + if [ -f /tmp/daytona_health_$$.json ]; then + echo "[daytona] Response: $(cat /tmp/daytona_health_$$.json | head -c 200)" + rm -f /tmp/daytona_health_$$.json + fi + if [ "$DAYTONA_HTTP_CODE" = "200" ]; then + echo "[daytona] API connectivity OK" + else + echo "[daytona] WARNING: API returned $DAYTONA_HTTP_CODE — sandbox creation may fail" + fi + fi + + echo "[proxy] Setup complete" + return 0 +} + +# Helper: run command through proxychains +proxied() { + if [ -x "$PROXYCHAINS_BIN" ] && [ -n "${PROXYCHAINS_CONF_FILE:-}" ]; then + "$PROXYCHAINS_BIN" -f "$PROXYCHAINS_CONF_FILE" "$@" + else + "$@" + fi +} + +# Setup proxy (skip if EVAL_PROXY_ENABLED=false, e.g. clusters with direct internet) +if [ "${EVAL_PROXY_ENABLED:-true}" = "true" ]; then + setup_proxy || echo "[proxy] Continuing without proxy (some operations may fail)" +else + echo "[proxy] Proxy disabled (EVAL_PROXY_ENABLED=false), assuming direct internet access" +fi + +# ============================================================================== +# Cleanup trap +# ============================================================================== +cleanup() { + echo "Cleaning up..." + # Kill vLLM + if [ -n "${VLLM_PID:-}" ]; then + kill "$VLLM_PID" 2>/dev/null || true + fi + # Kill SSH tunnel + if [ -n "${TUNNEL_PID:-}" ]; then + kill "$TUNNEL_PID" 2>/dev/null || true + fi + # Remove wrappers and per-job cache + [ -n "${WRAPPER_DIR:-}" ] && rm -rf "$WRAPPER_DIR" 2>/dev/null || true + [ -n "${_JOB_CACHE:-}" ] && rm -rf "$_JOB_CACHE" 2>/dev/null || true + # Remove proxychains config + rm -f "${PROXYCHAINS_CONF_FILE:-}" 2>/dev/null || true + echo "Cleanup done." +} +trap cleanup EXIT + +# ============================================================================== +# Start vLLM Server +# ============================================================================== +EVAL_LOGS_DIR="${EVAL_LOGS_DIR:-eval/logs}" +mkdir -p "$EVAL_LOGS_DIR" + +# Derive unique vLLM port. When the listener packs jobs (EVAL_VLLM_PORT is set), +# use the listener-assigned port directly — it centrally plans non-overlapping +# port ranges across all jobs on a node. Otherwise fall back to job-ID-based port. +# IMPORTANT: Do NOT export VLLM_PORT as an env var — vLLM's _get_open_port() +# reads the VLLM_PORT env var and all DP subprocess children would try to bind +# the same port, causing an infinite port-collision loop. +_PHYSICAL_GPUS="${SLURM_JOB_GPUS:-0}" +FIRST_GPU=$(echo "$_PHYSICAL_GPUS" | cut -d',' -f1) +if [ -n "${EVAL_VLLM_PORT:-}" ]; then + VLLM_PORT="$EVAL_VLLM_PORT" +else + VLLM_PORT=$(( 10000 + (SLURM_JOB_ID % 50000) )) +fi +unset -v VLLM_PORT_ENV # ensure no VLLM_PORT env var leaks to subprocesses +echo "Physical GPUs: $_PHYSICAL_GPUS (CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES), vLLM port: $VLLM_PORT" + +# Pre-download model via proxychains (compute nodes have no direct internet) +echo "Pre-downloading model: $MODEL" +proxied $PYTHON_BIN -c " +from huggingface_hub import snapshot_download +import os +cache = os.environ.get('HF_HUB_CACHE') +path = snapshot_download('$MODEL', cache_dir=cache) +print(f'Model cached at: {path}') +" +if [ $? -ne 0 ]; then + echo "ERROR: Model pre-download failed for $MODEL. Exiting." + exit 1 +fi + +echo "Starting vLLM server for model: $MODEL" +source "$DCFT/eval/build_vllm_cmd.sh" +build_vllm_cmd "$PYTHON_BIN" "$MODEL" "$GPU_MEMORY_UTIL" + +# Per-job cache dirs to avoid JIT compilation conflicts when packing +_JOB_CACHE="/tmp/${USER}_${SLURM_JOB_ID}" +mkdir -p "$_JOB_CACHE" + +# With DP > 1, vLLM needs torch.compile (TORCHDYNAMO_DISABLE must be unset) +_DP_SIZE="${EVAL_VLLM_DATA_PARALLEL_SIZE:-1}" +_DYNAMO_DISABLE=1 +if [ "$_DP_SIZE" -gt 1 ] 2>/dev/null; then + _DYNAMO_DISABLE=0 +fi + +env TORCHDYNAMO_DISABLE=$_DYNAMO_DISABLE \ + TMPDIR="$_JOB_CACHE" \ + TRITON_CACHE_DIR="$_JOB_CACHE/triton" \ + TORCH_COMPILE_CACHE_DIR="$_JOB_CACHE/torch" \ + TORCHINDUCTOR_CACHE_DIR="$_JOB_CACHE/torchinductor" \ + RAY_TMPDIR="$_JOB_CACHE/ray" \ + HF_HOME="$HF_HOME" \ + HF_HUB_CACHE="$HF_HUB_CACHE" \ + HF_HUB_OFFLINE=1 \ +"${VLLM_CMD[@]}" \ + > "$EVAL_LOGS_DIR/vllm_${SLURM_JOB_ID}.log" 2>&1 & +VLLM_PID=$! + +# Health check loop (use configurable retries) +MAX_RETRIES=$VLLM_MAX_RETRIES +RETRY_INTERVAL=15 +for i in $(seq 1 $MAX_RETRIES); do + if curl -s http://localhost:${VLLM_PORT}/v1/models > /dev/null 2>&1; then + echo "vLLM server is ready" + break + fi + echo "Waiting for vLLM server (attempt $i/$MAX_RETRIES)..." + sleep $RETRY_INTERVAL + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: vLLM server failed to start" + # Dump last 50 lines of vLLM log + tail -50 "$EVAL_LOGS_DIR/vllm_${SLURM_JOB_ID}.log" 2>/dev/null || true + exit 1 + fi +done + +# ============================================================================== +# Download dataset (via proxychains for HF access) +# Use --local-dir to get real files instead of symlinks (Daytona needs real files) +# Supports local paths: if REPO_ID starts with /, use it directly. +# ============================================================================== +if [[ "$REPO_ID" == /* ]]; then + # Local dataset path + echo "Using local dataset path: $REPO_ID" + DATASET_PATH="$REPO_ID" + if [ ! -d "$DATASET_PATH" ]; then + echo "ERROR: Local dataset path does not exist: $DATASET_PATH" + exit 1 + fi + TASK_COUNT=$(ls -d "$DATASET_PATH"/*/instruction.md 2>/dev/null | wc -l) + echo "[DEBUG] Found $TASK_COUNT tasks with instruction.md" +else + echo "Downloading/locating dataset: $REPO_ID" + # Check dataset directories (from EVAL_DATASETS_DIRS or fallback) + _DS_DIRS="${EVAL_DATASETS_DIRS:?ERROR: EVAL_DATASETS_DIRS not set. Set datasets_dirs in cluster config YAML.}" + DATASET_LOCAL_DIR="" + IFS=':' read -ra _DS_DIRS_ARR <<< "$_DS_DIRS" + for _ds_dir in "${_DS_DIRS_ARR[@]}"; do + if [ -d "${_ds_dir}/${SAFE_REPO}" ]; then + DATASET_LOCAL_DIR="${_ds_dir}/${SAFE_REPO}" + break + fi + done + # Default to first dir if nothing found (snapshot_download will create it) + DATASET_LOCAL_DIR="${DATASET_LOCAL_DIR:-${_DS_DIRS_ARR[0]}/${SAFE_REPO}}" + DATASET_LOCAL_DIR_ALT="" # no longer needed; absorbed into EVAL_DATASETS_DIRS + echo "[DEBUG] DATASET_LOCAL_DIR=$DATASET_LOCAL_DIR" + echo "[DEBUG] Local dir exists: $([ -d "$DATASET_LOCAL_DIR" ] && echo yes || echo no)" + if [ -d "$DATASET_LOCAL_DIR" ]; then + TASK_COUNT=$(ls -d "$DATASET_LOCAL_DIR"/*/instruction.md 2>/dev/null | wc -l) + echo "[DEBUG] Found $TASK_COUNT tasks with instruction.md in local dir" + echo "[DEBUG] Sample files:" + file "$DATASET_LOCAL_DIR"/$(ls "$DATASET_LOCAL_DIR" | head -1)/environment/Dockerfile 2>/dev/null || echo "[DEBUG] No Dockerfile found" + fi + DOWNLOAD_LOG=$(mktemp /tmp/download_XXXXXX.log) + proxied $PYTHON_BIN "$DCFT/eval/snapshot_download.py" "$REPO_ID" --local-dir "$DATASET_LOCAL_DIR" > "$DOWNLOAD_LOG" 2>&1 + DOWNLOAD_EXIT=$? + cat "$DOWNLOAD_LOG" + DATASET_PATH=$(grep DATASET_PATH "$DOWNLOAD_LOG" | tail -n 1 | cut -d'=' -f2) + rm -f "$DOWNLOAD_LOG" + if [ $DOWNLOAD_EXIT -ne 0 ] || [ -z "${DATASET_PATH:-}" ]; then + echo "ERROR: Failed to get dataset path (exit code: $DOWNLOAD_EXIT)" + exit 1 + fi +fi +echo "Using dataset path: $DATASET_PATH" +echo "[DEBUG] Dataset source: $([ -L "$DATASET_PATH/$(ls "$DATASET_PATH" | head -1)/environment/Dockerfile" ] && echo 'SYMLINK' || echo 'REAL FILE')" +echo "[DEBUG] Task count in dataset: $(ls -d "$DATASET_PATH"/*/instruction.md 2>/dev/null | wc -l)" + +# ============================================================================== +# Construct run tag and directory +# ============================================================================== +if [ -n "$RUN_TAG_ARG" ]; then + RUN_TAG="$RUN_TAG_ARG" +else + RUN_TAG="${SAFE_REPO}_${SAFE_MODEL}" +fi +EVAL_JOBS_DIR="${EVAL_JOBS_DIR:?ERROR: EVAL_JOBS_DIR not set. Set eval_jobs_dir in cluster config YAML.}" +RUN_DIR="${EVAL_JOBS_DIR}/${RUN_TAG}" +mkdir -p "$RUN_DIR" + +echo "Run tag: $RUN_TAG" +echo "Run dir: $RUN_DIR" + +# --- Compute canonical benchmark name for DB uploads --- +declare -A BENCHMARK_NAME_MAP=( + ["DCAgent2_aider_polyglot"]="aider_polyglot" + ["DCAgent_dev_set_v2"]="dev_set_v2" + ["DCAgent_dev_set_71_tasks"]="dev_set_71_tasks" + ["DCAgent2_terminal_bench_2"]="terminal_bench_2" + ["DCAgent_swebench_verified_eval_set"]="swebench-verified-random-100-folders" + ["DCAgent2_bfcl-parity"]="bfcl-parity" +) +BASE_NAME="${BENCHMARK_NAME_MAP[$SAFE_REPO]:-$SAFE_REPO}" + +# Append suffix for non-default timeout/memory configs. +BENCHMARK_SUFFIX="" +if [ -n "${EVAL_OVERRIDE_MEMORY_MB:-}" ] && [ "$EVAL_OVERRIDE_MEMORY_MB" != "1024" ]; then + mem_gb=$(( EVAL_OVERRIDE_MEMORY_MB / 1024 )) + BENCHMARK_SUFFIX="${BENCHMARK_SUFFIX}_${mem_gb}gb" +fi +if [ "$TIMEOUT_MULTIPLIER" != "1" ] && [ "$TIMEOUT_MULTIPLIER" != "1.0" ]; then + BENCHMARK_SUFFIX="${BENCHMARK_SUFFIX}_${TIMEOUT_MULTIPLIER}x" +fi +if [ -n "$BENCHMARK_SUFFIX" ]; then + BENCHMARK_NAME="${BASE_NAME}${BENCHMARK_SUFFIX}" +else + BENCHMARK_NAME="$BASE_NAME" +fi +export BENCHMARK_NAME +echo "Benchmark name: $BENCHMARK_NAME" + +# ============================================================================== +# Update DB: Pending → Started (v4 flow) +# ============================================================================== +echo "Creating/updating DB job entry..." +export MODEL REPO_ID RUN_TAG SLURM_JOB_ID BENCHMARK_NAME + +HARBOR_VERSION=$($PYTHON_BIN -c "import harbor; print(harbor.__version__)" 2>/dev/null || echo "unknown") +export HARBOR_VERSION + +if [ -n "$DB_JOB_ID" ]; then + # v4 flow: listener already created a Pending entry — transition to Started + echo "Updating Pending job $DB_JOB_ID → Started (run_tag=$RUN_TAG)" + proxied $PYTHON_BIN - <<'PY' +import os, sys, json +from database.unified_db.utils import update_job_status_to_started + +run_tag = os.environ["RUN_TAG"] +agent_name = os.environ.get("EVAL_AGENT_NAME", "terminus-2") +n_concurrent = int(os.environ.get("EVAL_N_CONCURRENT", "128")) +n_attempts = int(os.environ.get("EVAL_N_ATTEMPTS", "3")) +timeout_multiplier = float(os.environ.get("EVAL_TIMEOUT_MULTIPLIER", "1.0")) +harbor_version = os.environ.get("HARBOR_VERSION", "unknown") + +config = { + "agent": agent_name, + "env": "daytona", + "timeout_multiplier": timeout_multiplier, +} + +result = update_job_status_to_started( + job_name=run_tag, + n_trials=n_concurrent, + n_rep_eval=n_attempts, + config=config, + harbor_package_version=harbor_version, +) +if not result.get("success"): + print(f"WARNING: Pending→Started update failed: {result.get('error')}", file=sys.stderr) + print("Will create a new Started entry as fallback.", file=sys.stderr) + sys.exit(1) +else: + print(f"DB job {run_tag} → Started") +PY + _db_update_rc=$? +else + _db_update_rc=1 # No DB_JOB_ID — force fallback +fi + +if [ "$_db_update_rc" -ne 0 ]; then + # Fallback: create Started entry directly (Pending entry missing/deleted, or no DB_JOB_ID) + echo "Creating new Started entry (fallback)..." + proxied $PYTHON_BIN - <<'PY' || true +import os, sys + +from database.unified_db.utils import create_job_entry_started + +model_hf = os.environ["MODEL"] +dataset_hf = os.environ["REPO_ID"] +run_tag = os.environ["RUN_TAG"] +slurm_job_id = os.environ["SLURM_JOB_ID"] +harbor_version = os.environ.get("HARBOR_VERSION", "unknown") +agent_name = os.environ.get("EVAL_AGENT_NAME", "terminus-2") +n_concurrent = int(os.environ.get("EVAL_N_CONCURRENT", "128")) +n_attempts = int(os.environ.get("EVAL_N_ATTEMPTS", "3")) + +result = create_job_entry_started( + model_hf_name=model_hf, + benchmark_hf_name=dataset_hf, + job_name=run_tag, + username=os.environ.get("USER", "jupiter"), + slurm_job_id=slurm_job_id, + harbor_package_version=harbor_version, + agent_name=agent_name, + config={"agent": agent_name, "env": "daytona"}, + n_trials=n_concurrent, + n_rep_eval=n_attempts +) + +if not result.get("success"): + print(f"WARNING: DB create failed: {result.get('error')}", file=sys.stderr) + sys.exit(0) + +db_job_id = result["job"]["id"] +print(f"DB job created with ID: {db_job_id}") +PY + + # Get the DB job ID for later use + DB_JOB_ID=$(proxied $PYTHON_BIN - <<'PY' || true +import os, sys + +from database.unified_db.utils import get_latest_job_for_model_benchmark + +model_hf = os.environ["MODEL"] +dataset_hf = os.environ["REPO_ID"] + +try: + result = get_latest_job_for_model_benchmark(model_hf, dataset_hf) + if result and result.get("id"): + print(result["id"]) +except Exception as e: + print(f"WARNING: DB lookup failed: {e}", file=sys.stderr) +PY +) +fi + +if [ -z "${DB_JOB_ID:-}" ]; then + echo "WARNING: Failed to get DB job ID" +else + echo "DB job entry: $DB_JOB_ID" +fi + +# ============================================================================== +# Eval Starts Log (v4 — for model retry tracking) +# ============================================================================== +if [ -n "$EVAL_STARTS_LOG" ]; then + echo "${TIMESTAMP} ${MODEL} ${REPO_ID} ${SLURM_JOB_ID} ${RUN_TAG}" >> "$EVAL_STARTS_LOG" + echo "Logged eval start to: $EVAL_STARTS_LOG" +fi + +# ============================================================================== +# Run Harbor Eval (via proxychains for Daytona access) +# ============================================================================== +set +e + +# Resolve config path: check eval/configs/ (canonical), then legacy per-cluster dirs +if [[ "$CONFIG_YAML" != /* ]]; then + if [ -f "$DCFT/eval/configs/$CONFIG_YAML" ]; then + HARBOR_CONFIG="$DCFT/eval/configs/$CONFIG_YAML" + elif [ -f "$DCFT/eval/${CLUSTER_NAME}/$CONFIG_YAML" ]; then + HARBOR_CONFIG="$DCFT/eval/${CLUSTER_NAME}/$CONFIG_YAML" + elif [ -f "$DCFT/eval/MBZ/$CONFIG_YAML" ]; then + HARBOR_CONFIG="$DCFT/eval/MBZ/$CONFIG_YAML" + else + echo "ERROR: Harbor config not found: $CONFIG_YAML" + exit 1 + fi +else + HARBOR_CONFIG="$CONFIG_YAML" +fi +echo "Resolved harbor config: $HARBOR_CONFIG" + +# Build extra harbor args +EXTRA_HARBOR_ARGS="" +if [ -n "${EVAL_SNAPSHOT_NAME:-}" ]; then + echo "Using Daytona snapshot: $EVAL_SNAPSHOT_NAME" + EXTRA_HARBOR_ARGS="--environment-kwarg snapshot_template_name=$EVAL_SNAPSHOT_NAME --no-force-build" +fi +if [ "$TIMEOUT_MULTIPLIER" != "1" ] && [ "$TIMEOUT_MULTIPLIER" != "1.0" ]; then + echo "Using timeout multiplier: $TIMEOUT_MULTIPLIER" + EXTRA_HARBOR_ARGS="$EXTRA_HARBOR_ARGS --timeout-multiplier $TIMEOUT_MULTIPLIER" +fi +if [ -n "${EVAL_OVERRIDE_MEMORY_MB:-}" ]; then + echo "Using memory override: ${EVAL_OVERRIDE_MEMORY_MB}MB" + EXTRA_HARBOR_ARGS="$EXTRA_HARBOR_ARGS --override-memory-mb $EVAL_OVERRIDE_MEMORY_MB" +fi +# auto_snapshot override from listener (overrides YAML config value) +if [ -n "${EVAL_AUTO_SNAPSHOT:-}" ]; then + echo "Auto snapshot override: $EVAL_AUTO_SNAPSHOT" + EXTRA_HARBOR_ARGS="$EXTRA_HARBOR_ARGS --environment-kwarg auto_snapshot=$EVAL_AUTO_SNAPSHOT" +fi + +# Build thinking args +THINKING_ARGS="" +if [ "$ENABLE_THINKING" = "true" ]; then + echo "Thinking enabled" + THINKING_ARGS='--agent-kwarg "enable_thinking=true"' +fi + +# Build agent-parser args +PARSER_ARGS="" +if [ -n "$AGENT_PARSER" ]; then + echo "Agent parser: $AGENT_PARSER" + PARSER_ARGS="--agent-kwarg \"parser=$AGENT_PARSER\"" +fi + +echo "[DEBUG] EXTRA_HARBOR_ARGS=$EXTRA_HARBOR_ARGS" +echo "[DEBUG] DAYTONA_API_KEY=${DAYTONA_API_KEY:0:20}..." + +# Check if a previous job dir exists for this run tag (for resume) +# Skip resume if EVAL_FORCE_FRESH=true +EXISTING_JOB_DIR="${EVAL_JOBS_DIR}/${RUN_TAG}" +if [ "${EVAL_FORCE_FRESH:-false}" != "true" ] && [ -d "$EXISTING_JOB_DIR" ] && [ -f "$EXISTING_JOB_DIR/config.json" ]; then + echo "Found existing job dir, resuming: $EXISTING_JOB_DIR" + proxied $HARBOR_BIN jobs resume \ + -p "$EXISTING_JOB_DIR" \ + --filter-error-type EnvironmentStartTimeoutError \ + --filter-error-type DaytonaError \ + --filter-error-type DaytonaRateLimitError \ + --filter-error-type CancelledError \ + --filter-error-type AgentEnvironmentTimeoutError \ + --filter-error-type SandboxBuildFailedError +else + echo "Starting new job" + # Build command as array to handle quoting properly + HARBOR_CMD=( + proxied $HARBOR_BIN jobs start + -p "$DATASET_PATH" + --n-concurrent "$N_CONCURRENT" + --agent "$AGENT_NAME" + --model "hosted_vllm/$MODEL" + --env "daytona" + --agent-kwarg "api_base=http://localhost:${VLLM_PORT}/v1" + --agent-kwarg "key=fake_key" + --agent-kwarg "max_tokens=16384" + --agent-kwarg "model_info={\"max_output_tokens\":16384,\"max_input_tokens\":32768,\"input_cost_per_token\":0.0,\"output_cost_per_token\":0.0}" + --n-attempts "$N_ATTEMPTS" + --job-name "$RUN_TAG" + --export-traces + --config "$HARBOR_CONFIG" + --jobs-dir "$EVAL_JOBS_DIR" + --debug + ) + # Add optional args + if [ "$ENABLE_THINKING" = "true" ]; then + HARBOR_CMD+=(--agent-kwarg "enable_thinking=true") + fi + if [ -n "$AGENT_PARSER" ]; then + HARBOR_CMD+=(--agent-kwarg "parser=$AGENT_PARSER") + fi + + echo "[DEBUG] Full harbor command:" + echo " ${HARBOR_CMD[*]} $EXTRA_HARBOR_ARGS" + + "${HARBOR_CMD[@]}" $EXTRA_HARBOR_ARGS +fi +SB_EXIT=$? +set -e + +# ============================================================================== +# Save meta.env (v4 — extended fields) +# ============================================================================== +# Track resume count +RESUME_COUNT=0 +if [ -f "$RUN_DIR/meta.env" ]; then + OLD_RESUME_COUNT=$(grep -oP 'RESUME_COUNT=\K[0-9]+' "$RUN_DIR/meta.env" 2>/dev/null || echo "0") + RESUME_COUNT=$((OLD_RESUME_COUNT + 1)) + echo "Resume count: $RESUME_COUNT (was $OLD_RESUME_COUNT)" +fi + +mkdir -p "$RUN_DIR" +{ + echo "MODEL=$MODEL" + echo "REPO_ID=$REPO_ID" + echo "TIMESTAMP=$TIMESTAMP" + echo "SLURM_JOB_ID=$SLURM_JOB_ID" + echo "DB_JOB_ID=${DB_JOB_ID:-}" + echo "BENCHMARK_ID=${BENCHMARK_ID:-}" + echo "N_CONCURRENT=$N_CONCURRENT" + echo "N_ATTEMPTS=$N_ATTEMPTS" + echo "GPU_MEMORY_UTIL=$GPU_MEMORY_UTIL" + echo "AGENT_NAME=$AGENT_NAME" + echo "AGENT_PARSER=${AGENT_PARSER:-}" + echo "ENABLE_THINKING=$ENABLE_THINKING" + echo "TIMEOUT_MULTIPLIER=$TIMEOUT_MULTIPLIER" + echo "BENCHMARK_NAME=$BENCHMARK_NAME" + echo "RESUME_COUNT=$RESUME_COUNT" +} > "$RUN_DIR/meta.env" + +# If eval failed, don't attempt upload +if [ ${SB_EXIT:-0} -ne 0 ]; then + echo "Harbor eval exited with non-zero status: ${SB_EXIT}. Skipping upload." + exit ${SB_EXIT} +fi + +# Ensure run dir exists +if [ ! -d "$RUN_DIR" ]; then + echo "Expected run directory not found: $RUN_DIR" + exit 2 +fi + +# ============================================================================== +# Check for errors before upload (v4 unified error checking) +# Counts ALL errors except known-benign types: +# AgentTimeoutError, BadRequestError, ContextLengthExceededError, +# SummarizationTimeout, SummarizationTimeoutError +# NOTE: AgentEnvironmentTimeoutError is NOT benign (infra issue, not model's fault) +# but is excluded from retry (retrying won't help) +# ============================================================================== +RESULT_FILE="$RUN_DIR/result.json" +ERROR_LOG="${EVAL_JOBS_DIR}/invalid_errors_${SLURM_JOB_ID}.log" + +if [ -f "$RESULT_FILE" ]; then + echo "Checking for invalid errors in $RESULT_FILE..." + + INVALID_COUNT=$($PYTHON_BIN -c " +import json, sys +try: + with open('$RESULT_FILE', 'r') as f: + data = json.load(f) + + BENIGN_ERRORS = { + 'AgentTimeoutError', + 'BadRequestError', + 'ContextLengthExceededError', + 'SummarizationTimeout', + 'SummarizationTimeoutError', + } + + total_invalid = 0 + if 'stats' in data and 'evals' in data['stats']: + for eval_key, eval_data in data['stats']['evals'].items(): + if 'exception_stats' in eval_data: + for err_type, err_ids in eval_data['exception_stats'].items(): + if err_type not in BENIGN_ERRORS: + if isinstance(err_ids, list): + total_invalid += len(err_ids) + print(total_invalid) +except Exception as e: + print(f'Error parsing result.json: {e}', file=sys.stderr) + print('0') +" 2>&1 | tail -n 1) + + echo "Invalid error count: ${INVALID_COUNT} (threshold: ${ERROR_THRESHOLD})" + + if [ "${INVALID_COUNT:-0}" -gt "$ERROR_THRESHOLD" ]; then + echo "Job has ${INVALID_COUNT} invalid errors (> ${ERROR_THRESHOLD}), skipping upload" + + { + echo "===============================================" + echo "Timestamp: $(date)" + echo "Job: ${RUN_TAG}" + echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" + echo "Model: ${MODEL}" + echo "Repo: ${REPO_ID}" + echo "Invalid errors: ${INVALID_COUNT}" + echo "Result file: ${RESULT_FILE}" + $PYTHON_BIN -c " +import json +BENIGN = {'AgentTimeoutError', 'ContextLengthExceededError', 'SummarizationTimeout', 'SummarizationTimeoutError'} +with open('$RESULT_FILE', 'r') as f: + data = json.load(f) +if 'stats' in data and 'evals' in data['stats']: + for eval_key, eval_data in data['stats']['evals'].items(): + if 'exception_stats' in eval_data: + for err_type, err_ids in eval_data['exception_stats'].items(): + if err_type not in BENIGN and err_ids: + print(f'Eval: {eval_key} / {err_type}') + for i, error_id in enumerate(err_ids[:10], 1): + print(f' {i}. {error_id}') + if len(err_ids) > 10: + print(f' ... and {len(err_ids) - 10} more') +" 2>/dev/null || true + echo "===============================================" + } >> "$ERROR_LOG" + + echo "Error details logged to: $ERROR_LOG" + echo "Job completed but not uploaded due to excessive invalid errors" + exit 0 + fi +else + echo "Warning: result.json not found, continuing with upload" +fi + +# ============================================================================== +# Upload results to DB (via proxychains for HF upload) +# ============================================================================== +export RUN_DIR +export UPLOAD_USERNAME +export UPLOAD_MODE="${UPLOAD_MODE:-skip_on_error}" +export RUN_TAG + +UPLOAD_LOG="${EVAL_LOGS_DIR:-eval/logs}/upload_${SLURM_JOB_ID}.log" +mkdir -p "$(dirname "$UPLOAD_LOG")" + +echo "Uploading results from: $RUN_DIR" | tee -a "$UPLOAD_LOG" +echo "Using username=${UPLOAD_USERNAME}, mode=${UPLOAD_MODE}" | tee -a "$UPLOAD_LOG" + +proxied $PYTHON_BIN - <<'PY' 2>&1 | tee -a "$UPLOAD_LOG" +import os, sys, re, hashlib + +from database.unified_db.utils import upload_eval_results + + +def sanitize_hf_repo_id(repo_id: str, max_length: int = 96) -> str: + def collapse(s: str) -> str: + prev = None + while s != prev: + prev = s + s = s.replace("--", "-").replace("..", ".") + return s + + org, name = repo_id.split("/", 1) if "/" in repo_id else (None, repo_id) + name = re.sub(r"[^A-Za-z0-9._-]", "-", name) + name = collapse(name).strip("-.") + if not name: + name = "repo" + + limit = max_length - (len(org) + 1 if org else 0) + if len(name) > limit: + digest = hashlib.sha1(name.encode()).hexdigest()[:8] + keep = max(1, limit - len(digest)) + base = name[:keep].rstrip("-.") + if not base: + base = "r" + name = f"{base}{digest}" + name = collapse(name).strip("-.") + + name = collapse(name).strip("-.") + if name[0] in "-.": + name = "r" + name[1:] + if name[-1] in "-.": + name = name[:-1] + "0" + + return f"{org}/{name}" if org else name + + +run_dir = os.environ["RUN_DIR"] +run_tag = os.environ["RUN_TAG"] +username = os.environ.get("UPLOAD_USERNAME", os.environ.get("USER", "jupiter")) +error_mode = os.environ.get("UPLOAD_MODE", "skip_on_error") +hf_repo_id = sanitize_hf_repo_id(f"DCAgent2/{run_tag}") +hf_token = os.environ["HF_TOKEN"] + +print(f"[uploader] upload_eval_results(path={run_dir!r}, username={username!r}, " + f"error_mode={error_mode!r}, hf_repo_id={hf_repo_id!r})") +dataset_hf = os.environ.get("REPO_ID", "") +# Use the canonical benchmark name computed in the shell section, +# which handles local paths and timeout/memory suffixes correctly. +benchmark_name = os.environ.get("BENCHMARK_NAME", "") +if not benchmark_name: + # Fallback: derive from REPO_ID (legacy behavior) + benchmark_name = dataset_hf.split("/")[-1] if "/" in dataset_hf else dataset_hf + +# Compute a stable benchmark_version_hash from the benchmark name +import hashlib +benchmark_version_hash = hashlib.sha256(benchmark_name.encode()).hexdigest() +print(f"[uploader] benchmark_name={benchmark_name!r}, version_hash={benchmark_version_hash[:16]}...") + +upload_eval_results( + run_dir, + username=username, + error_mode=error_mode, + hf_token=hf_token, + hf_repo_id=hf_repo_id, + register_benchmark=True, + benchmark_name=benchmark_name, + benchmark_version_hash=benchmark_version_hash, +) +print("[uploader] done.") +PY +UPLOAD_EXIT=${PIPESTATUS[0]} + +if [ $UPLOAD_EXIT -ne 0 ]; then + echo "Upload failed with exit code: $UPLOAD_EXIT" + exit $UPLOAD_EXIT +fi + +echo "==============================================" +echo "Eval and upload finished successfully." +echo "==============================================" diff --git a/eval/unified_eval_harbor_dp.sbatch b/eval/unified_eval_harbor_dp.sbatch new file mode 100644 index 000000000..69bff03a7 --- /dev/null +++ b/eval/unified_eval_harbor_dp.sbatch @@ -0,0 +1,992 @@ +#!/bin/bash +#SBATCH -p booster +#SBATCH --time=12:00:00 +#SBATCH --signal=B:TERM@120 +#SBATCH --ntasks-per-node 1 +#SBATCH --cpus-per-task=72 +#SBATCH --gres=gpu:4 +#SBATCH --output=eval/logs/%x_%j.out +#SBATCH --job-name=eval_dp + +# ============================================================================== +# Unified Eval Harbor DP — Jupiter Cluster (JSC GH200) +# +# Multi-node data-parallel version of unified_eval_harbor.sbatch. +# Runs N vLLM replicas (one per node/shard) with Ray, shards the dataset, +# and runs N parallel harbor jobs — one per shard. +# +# IMPORTANT: Must pass --nodes N to sbatch (or set via EVAL_NUM_SHARDS). +# +# Positional args (same as single-node): +# $1 = MODEL (HF model name) +# $2 = REPO_ID (HF dataset repo or local path) +# $3 = BENCHMARK_ID (optional, DB benchmark UUID) +# $4 = RUN_TAG_ARG (optional, override run tag) +# +# Extra env vars (DP-specific): +# EVAL_NUM_SHARDS (default: $SLURM_JOB_NUM_NODES) +# EVAL_NODES_PER_SHARD (default: 1) +# EVAL_VLLM_PIPELINE_PARALLEL_SIZE (default: 1) +# +# All other env vars from unified_eval_harbor.sbatch are supported: +# EVAL_N_CONCURRENT, EVAL_N_ATTEMPTS, EVAL_GPU_MEMORY_UTIL, etc. +# ============================================================================== + +set -eo pipefail +ulimit -c 0 +ulimit -n 65536 2>/dev/null || true + +TIMESTAMP=$(date +'%Y%m%d_%H%M%S') + +# --- Parse positional args --- +MODEL="${1:-mlfoundations-dev/claude_3_7_20250219_tbench_traces_sharegptv1}" +REPO_ID="${2:-DCAgent/dev_set_71_tasks}" +BENCHMARK_ID="${3:-}" +RUN_TAG_ARG="${4:-}" + +# --- Read env vars from listener (with defaults) --- +N_CONCURRENT="${EVAL_N_CONCURRENT:-128}" +N_ATTEMPTS="${EVAL_N_ATTEMPTS:-3}" +GPU_MEMORY_UTIL="${EVAL_GPU_MEMORY_UTIL:-0.95}" +ERROR_THRESHOLD="${EVAL_DAYTONA_THRESHOLD:-999999}" +VLLM_MAX_RETRIES="${EVAL_VLLM_MAX_RETRIES:-20}" +AGENT_PARSER="${EVAL_AGENT_PARSER:-}" +ENABLE_THINKING="${EVAL_ENABLE_THINKING:-false}" +AGENT_NAME="${EVAL_AGENT_NAME:-terminus-2}" +EVAL_STARTS_LOG="${EVAL_STARTS_LOG:-}" +TIMEOUT_MULTIPLIER="${EVAL_TIMEOUT_MULTIPLIER:-1.0}" +CONFIG_YAML="${EVAL_CONFIG_YAML:-dcagent_eval_config.yaml}" +DB_JOB_ID="${EVAL_DB_JOB_ID:-}" +UPLOAD_USERNAME="${EVAL_UPLOAD_USERNAME:-$USER}" + +# --- DP-specific env vars --- +GPUS_PER_NODE="${EVAL_GPUS_PER_NODE:-4}" # From cluster config (Jupiter=4, MBZ=8) +TP_SIZE="${EVAL_VLLM_TENSOR_PARALLEL_SIZE:-2}" +PP_SIZE="${EVAL_VLLM_PIPELINE_PARALLEL_SIZE:-1}" +NODES_PER_SHARD="${EVAL_NODES_PER_SHARD:-1}" +# Shards per node: how many vLLM replicas fit on one node (default: 4/TP) +SHARDS_PER_NODE="${EVAL_SHARDS_PER_NODE:-$((GPUS_PER_NODE / TP_SIZE))}" +GPUS_PER_SHARD=$((TP_SIZE * PP_SIZE)) +# Total shards (default: nodes * shards_per_node) +NUM_SHARDS="${EVAL_NUM_SHARDS:-$((SLURM_JOB_NUM_NODES * SHARDS_PER_NODE))}" + +# Strip slashes and special chars for file-safe names +SAFE_MODEL=$(echo "$MODEL" | tr '/:' '_') +if [[ "$REPO_ID" == /* ]]; then + SAFE_REPO=$(basename "$REPO_ID") +else + SAFE_REPO=$(echo "$REPO_ID" | tr '/:' '_') +fi + +# Benchmark shorthand for squeue +declare -A BENCH_SHORT=( + ["DCAgent_dev_set_v2"]="v2" + ["DCAgent2_swebench-verified-random-100-folders"]="swe" + ["DCAgent2_terminal_bench_2"]="tb2" + ["DCAgent2_aider_polyglot"]="aider" + ["DCAgent2_bfcl-parity"]="bfcl" + ["DCAgent_dev_set_71_tasks"]="v1" +) +BENCH_TAG="${BENCH_SHORT[$SAFE_REPO]:-${SAFE_REPO:0:12}}" +if [ "$SLURM_JOB_NAME" = "eval_dp" ]; then + scontrol update JobId="$SLURM_JOB_ID" JobName="eval_dp_${BENCH_TAG}" +fi + +echo "==============================================" +echo "Jupiter Eval Harbor DP (${NUM_SHARDS}-way)" +echo "==============================================" +echo "Model: $MODEL" +echo "Dataset: $REPO_ID" +echo "Benchmark ID: ${BENCHMARK_ID:-}" +echo "Nodes: $SLURM_JOB_NUM_NODES ($SHARDS_PER_NODE shards/node, $NUM_SHARDS total shards)" +echo "GPUs: TP=$TP_SIZE, PP=$PP_SIZE, $GPUS_PER_SHARD GPUs/shard" +echo "N concurrent (per shard): $N_CONCURRENT" +echo "N attempts: $N_ATTEMPTS" +echo "GPU memory util: $GPU_MEMORY_UTIL" +echo "Error threshold: $ERROR_THRESHOLD" +echo "vLLM max retries: $VLLM_MAX_RETRIES" +echo "Agent: $AGENT_NAME" +echo "Thinking: $ENABLE_THINKING" +echo "Timeout multiplier: $TIMEOUT_MULTIPLIER" +echo "Pipeline parallel: $PP_SIZE" +echo "Config YAML: $CONFIG_YAML" +echo "DB Job ID (pending): ${DB_JOB_ID:-}" +echo "==============================================" + +# ============================================================================== +# Cluster-Agnostic Environment Setup (same as single-node sbatch) +# ============================================================================== + +DCFT="${EVAL_PROJECT_ROOT:-${DCFT:-/e/scratch/jureap59/$USER/OpenThoughts-Agent}}" +CLUSTER_NAME="${EVAL_CLUSTER_NAME:-jupiter}" +unset PYTHONPATH + +# Source cluster-specific dotenv +DOTENV_FILE="$DCFT/hpc/dotenv/${CLUSTER_NAME}.env" +if [ -f "$DOTENV_FILE" ]; then + source "$DOTENV_FILE" + echo "Sourced dotenv: $DOTENV_FILE" +else + echo "WARNING: dotenv not found: $DOTENV_FILE" +fi + +DC_AGENT_SECRET_ENV="${DC_AGENT_SECRET_ENV:-$HOME/secrets.env}" +if [ -f "$DC_AGENT_SECRET_ENV" ]; then + source "$DC_AGENT_SECRET_ENV" +fi + +if [ -z "${DAYTONA_API_KEY:-}" ]; then + echo "WARNING: DAYTONA_API_KEY not set. Check $DC_AGENT_SECRET_ENV" +fi +echo "Daytona API key: ${DAYTONA_API_KEY:0:12}..." + +# vLLM / Ray / Triton env vars +export VLLM_USE_V1=1 +export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook +export RAY_CGRAPH_get_timeout=900 +export VLLM_CONFIG_ROOT="${VLLM_CACHE_ROOT:-/tmp/vllm_config_${USER}}" +export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-/tmp/triton_cache_${USER}}" +export FLASHINFER_WORKSPACE_BASE="${FLASHINFER_CACHE_DIR:-/tmp/flashinfer_cache_${USER}}" +export UV_CACHE_DIR="${UV_CACHE_DIR:-/tmp/uv_cache_${USER}}" +export HYDRA_FULL_ERROR=1 +export HF_HUB_CACHE="${EVAL_HF_CACHE:-${HF_HUB_CACHE:-/e/data1/datasets/playground/ot/hf_hub}}" +export HF_HOME="${HF_HUB_CACHE}" +export HF_CACHE_DIR="$HF_HUB_CACHE" +export HF_XET_CACHE="${HF_XET_CACHE:-/tmp/hf_xet_cache_${USER}}" +mkdir -p "$HF_XET_CACHE" + +HARBOR_SRC="${EVAL_HARBOR_SRC:-/e/scratch/jureap59/feuer1/harbor/src}" +export PYTHONPATH="${HARBOR_SRC}:${DCFT}:${PYTHONPATH:-}" + +# ============================================================================== +# LD_LOADER Wrappers (needed on aarch64 Jupiter where shared conda lacks exec perms) +# ============================================================================== + +OTAGENT_DIR="${OTAGENT_DIR:-/e/scratch/jureap59/feuer1/miniforge3/envs/otagent}" +export PATH="$OTAGENT_DIR/bin:$PATH" +export CONDA_PREFIX="$OTAGENT_DIR" + +PYTHON_REAL="$OTAGENT_DIR/bin/python3.12" + +# LD_LOADER Wrappers: required on aarch64 (Jupiter GH200) where shared conda +# lacks execute permissions. Skipped on x86_64 where the conda env is directly usable. +if [ "$(uname -m)" = "aarch64" ] && [ -f /lib/ld-linux-aarch64.so.1 ]; then + LD_LOADER="/lib/ld-linux-aarch64.so.1" + # Use shared filesystem (not /tmp) so all nodes can see the wrappers + WRAPPER_DIR="${DCFT}/eval/${CLUSTER_NAME}/.wrappers/${SLURM_JOB_ID}" + mkdir -p "$WRAPPER_DIR" + + cat > "$WRAPPER_DIR/python3" < "$WRAPPER_DIR/python" < "$WRAPPER_DIR/ray" < "$WRAPPER_DIR/harbor" </dev/null +export RAY_DEDUP_LOGS=0 + +# Set CUDA_HOME for flashinfer JIT compilation (needs nvcc) +_CUDA_HOME="${EVAL_CUDA_HOME:-/e/software/default/stages/2026/software/CUDA/13}" +if [ -n "${_CUDA_HOME}" ] && [ -d "${_CUDA_HOME}" ]; then + export CUDA_HOME="${_CUDA_HOME}" + export PATH="$CUDA_HOME/bin:$PATH" +fi + + +# Get all nodes +ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") +ALL_NODES_ARRAY=($ALL_NODES) +echo "All nodes: ${ALL_NODES_ARRAY[@]}" + +# Env vars to export to srun workers +export SRUN_EXPORT_ENV="ALL,LD_LIBRARY_PATH=$LD_LIBRARY_PATH,PATH=$PATH,HF_TOKEN=$HF_TOKEN" +RAY_ENV_VARS="LD_LIBRARY_PATH=$LD_LIBRARY_PATH PATH=$PATH HF_TOKEN=$HF_TOKEN" + +# ============================================================================== +# SSH Tunnel + Proxychains Setup (from head node = first node) +# ============================================================================== + +NODE_HOST=$(hostname -s) +TUNNEL_PORT=7003 +LOGIN_NODE="${EVAL_LOGIN_NODE:-jpbl-s01-02}" +PROXYCHAINS_BIN="${EVAL_PROXYCHAINS_BIN:-/e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4}" +TUNNEL_PID="" + +setup_proxy() { + echo "[proxy] Setting up SSH tunnel to $LOGIN_NODE..." + if [ -z "${SSH_KEY:-}" ]; then + echo "[proxy] WARNING: SSH_KEY not set, skipping tunnel setup" + return 1 + fi + NODE_IP=$(nslookup "$NODE_HOST" 2>/dev/null | grep 'Address' | tail -n1 | awk '{print $2}') + NODE_IP="${NODE_IP:-127.0.0.1}" + ssh -g -f -N -D ${TUNNEL_PORT} \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=1000 \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=30 \ + -o TCPKeepAlive=yes \ + -o ExitOnForwardFailure=yes \ + -o BatchMode=yes \ + -i "${SSH_KEY}" \ + "${USER}@${LOGIN_NODE}" + sleep 5 + if pgrep -f "ssh.*-D.*${TUNNEL_PORT}" > /dev/null; then + echo "[proxy] SSH tunnel started successfully" + TUNNEL_PID=$(pgrep -f "ssh.*-D.*${TUNNEL_PORT}" | head -1) + else + echo "[proxy] ERROR: SSH tunnel failed to start" + return 1 + fi + CFG_PATH="$HOME/.proxychains/proxychains_${SLURM_JOB_ID}.conf" + mkdir -p "$HOME/.proxychains" + cat > "$CFG_PATH" </dev/null \ + && echo "[proxy] HuggingFace connectivity OK" \ + || echo "[proxy] WARNING: HuggingFace connectivity test failed" + fi + + if [ -n "${DAYTONA_API_KEY:-}" ]; then + echo "[daytona] Testing API connectivity..." + DAYTONA_HTTP_CODE=$("$PROXYCHAINS_BIN" -f "$CFG_PATH" curl -s --connect-timeout 15 \ + -H "Authorization: Bearer $DAYTONA_API_KEY" \ + "${DAYTONA_API_URL:-https://app.daytona.io/api}/health" \ + -o /dev/null -w "%{http_code}" 2>/dev/null) || DAYTONA_HTTP_CODE="FAIL" + echo "[daytona] Health check: $DAYTONA_HTTP_CODE" + fi + echo "[proxy] Setup complete" + return 0 +} + +proxied() { + if [ -x "$PROXYCHAINS_BIN" ] && [ -n "${PROXYCHAINS_CONF_FILE:-}" ]; then + "$PROXYCHAINS_BIN" -f "$PROXYCHAINS_CONF_FILE" "$@" + else + "$@" + fi +} + +if [ "${EVAL_PROXY_ENABLED:-true}" = "true" ]; then + setup_proxy || echo "[proxy] Continuing without proxy" +else + echo "[proxy] Proxy disabled (EVAL_PROXY_ENABLED=false), assuming direct internet access" +fi + +# ============================================================================== +# Cleanup trap (DP: kill all vLLM + Ray + tunnel) +# ============================================================================== +cleanup() { + echo "Cleaning up DP resources..." + # Kill vLLM processes + for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + if [ -n "${VLLM_PIDS[$shard_idx]:-}" ]; then + kill "${VLLM_PIDS[$shard_idx]}" 2>/dev/null || true + fi + done + # Stop Ray on all nodes + for node in "${ALL_NODES_ARRAY[@]}"; do + srun --nodes=1 --ntasks=1 --overlap -w "$node" "$RAY_BIN" stop --force 2>/dev/null & + done + wait + # Kill SSH tunnel + if [ -n "${TUNNEL_PID:-}" ]; then + kill "$TUNNEL_PID" 2>/dev/null || true + fi + pkill -f "ssh.*-D.*${TUNNEL_PORT}" 2>/dev/null || true + # Remove wrappers + proxychains config + [ -n "${WRAPPER_DIR:-}" ] && rm -rf "$WRAPPER_DIR" 2>/dev/null || true + rm -f "${PROXYCHAINS_CONF_FILE:-}" 2>/dev/null || true + echo "Cleanup done." +} +trap cleanup EXIT + +# ============================================================================== +# Pre-download model (compute nodes have no internet) +# ============================================================================== +EVAL_LOGS_DIR="${EVAL_LOGS_DIR:-eval/logs}" +mkdir -p "$EVAL_LOGS_DIR" + +echo "Pre-downloading model: $MODEL" +proxied $PYTHON_BIN -c " +from huggingface_hub import snapshot_download +import os +cache = os.environ.get('HF_HUB_CACHE') +path = snapshot_download('$MODEL', cache_dir=cache) +print(f'Model cached at: {path}') +" +if [ $? -ne 0 ]; then + echo "ERROR: Model pre-download failed. Exiting." + exit 1 +fi + +# ============================================================================== +# Build vLLM command args (reuse build_vllm_cmd.sh) +# ============================================================================== +source "$DCFT/eval/build_vllm_cmd.sh" +build_vllm_cmd "$PYTHON_BIN" "$MODEL" "$GPU_MEMORY_UTIL" +# VLLM_CMD includes --host 0.0.0.0 --port 8000 by default. +# For DP, we strip --port and --host from VLLM_CMD and re-add per shard. +VLLM_CMD_BASE=() +skip_next=false +for arg in "${VLLM_CMD[@]}"; do + if $skip_next; then + skip_next=false + continue + fi + if [ "$arg" = "--port" ] || [ "$arg" = "--host" ]; then + skip_next=true + continue + fi + VLLM_CMD_BASE+=("$arg") +done + +echo "vLLM DP config: TP=$TP_SIZE, PP=$PP_SIZE, shards=$NUM_SHARDS, shards_per_node=$SHARDS_PER_NODE" + +# ============================================================================== +# Start Ray Clusters + vLLM (supports multiple shards per node) +# +# Topology: +# - Each node has GPUS_PER_NODE GPUs (4 on Jupiter GH200) +# - Each shard uses TP_SIZE GPUs (via CUDA_VISIBLE_DEVICES) +# - SHARDS_PER_NODE = GPUS_PER_NODE / TP_SIZE (e.g. 4/2=2) +# - Each shard gets its own Ray cluster (separate port), vLLM (separate port) +# +# Example: 1 node, TP=2, 2 shards/node +# Shard 0: GPUs 0,1, Ray port 6379, API port 8000 +# Shard 1: GPUs 2,3, Ray port 6479, API port 8001 +# +# Example: 2 nodes, TP=2, 2 shards/node = 4 total shards +# Node 0: Shard 0 (GPUs 0,1, port 8000), Shard 1 (GPUs 2,3, port 8001) +# Node 1: Shard 2 (GPUs 0,1, port 8000), Shard 3 (GPUs 2,3, port 8001) +# ============================================================================== +echo "" +echo "=== Starting $NUM_SHARDS shards ($SHARDS_PER_NODE per node, TP=$TP_SIZE) ===" + +declare -a VLLM_PIDS +declare -a SHARD_NODES # which node each shard runs on +declare -a SHARD_GPU_IDS # CUDA_VISIBLE_DEVICES for each shard +declare -a API_PORTS +declare -a RAY_PORTS +declare -a HEAD_IPS + +BASE_RAY_PORT=6379 +BASE_API_PORT=8000 + +# Compute shard → node mapping and GPU assignment +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + node_idx=$((shard_idx / SHARDS_PER_NODE)) + local_shard=$((shard_idx % SHARDS_PER_NODE)) + gpu_start=$((local_shard * TP_SIZE)) + + # Build CUDA_VISIBLE_DEVICES string (e.g. "0,1" or "2,3") + gpu_ids="" + for ((g = gpu_start; g < gpu_start + TP_SIZE; g++)); do + [ -n "$gpu_ids" ] && gpu_ids="${gpu_ids}," + gpu_ids="${gpu_ids}${g}" + done + + SHARD_NODES[$shard_idx]="${ALL_NODES_ARRAY[$node_idx]}" + SHARD_GPU_IDS[$shard_idx]="$gpu_ids" + # Use different Ray ports per shard (even on same node) + RAY_PORTS[$shard_idx]=$((BASE_RAY_PORT + shard_idx * 100)) + # API ports: offset by local_shard so shards on same node don't collide + API_PORTS[$shard_idx]=$((BASE_API_PORT + local_shard)) + + echo " Shard $shard_idx: node=${SHARD_NODES[$shard_idx]}, GPUs=${gpu_ids}, Ray port=${RAY_PORTS[$shard_idx]}, API port=${API_PORTS[$shard_idx]}" +done + +# --- Step 3a: Clean up existing Ray processes on all nodes --- +echo "" +echo "Cleaning up existing Ray processes..." +for node in "${ALL_NODES_ARRAY[@]}"; do + srun --nodes=1 --ntasks=1 --overlap -w "$node" bash -c \ + "pkill -9 -u $USER -f 'ray::' 2>/dev/null; $RAY_BIN stop --force 2>/dev/null; rm -rf /tmp/ray_${USER}_shard* 2>/dev/null" || true & +done +wait +sleep 2 + +# --- Step 3b: Start Ray clusters (one per shard) --- +echo "" +echo "Starting Ray clusters..." + +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + node="${SHARD_NODES[$shard_idx]}" + gpu_ids="${SHARD_GPU_IDS[$shard_idx]}" + ray_port="${RAY_PORTS[$shard_idx]}" + ray_temp_dir="/tmp/ray_${USER}_shard${shard_idx}" + + # Get node IP via InfiniBand + head_ip="" + if srun --nodes=1 --ntasks=1 --overlap -w "$node" ip -o -4 addr show ib0 >/dev/null 2>&1; then + head_ip=$(srun --nodes=1 --ntasks=1 --overlap -w "$node" ip -o -4 addr show ib0 | awk '{print $4}' | cut -d/ -f1) + else + head_ip=$(srun --nodes=1 --ntasks=1 --overlap -w "$node" hostname --ip-address) + head_ip=${head_ip%% *} + fi + HEAD_IPS[$shard_idx]="$head_ip" + + echo " Shard $shard_idx: Starting Ray head on $node ($head_ip:$ray_port), GPUs=$gpu_ids" + + # Start Ray head with only this shard's GPUs visible + srun --export="$SRUN_EXPORT_ENV" --nodes=1 --ntasks=1 --overlap -w "$node" bash -c \ + "env $RAY_ENV_VARS CUDA_VISIBLE_DEVICES=$gpu_ids \ + $RAY_BIN start --head --node-ip-address=${head_ip} --port=${ray_port} \ + --num-gpus=${TP_SIZE} --num-cpus=$((72 / SHARDS_PER_NODE)) \ + --temp-dir=${ray_temp_dir}" & + sleep 3 +done + +echo "Waiting for Ray clusters to stabilize..." +sleep 15 + +# --- Step 3c: Start vLLM on each shard --- +echo "" +echo "=== Starting $NUM_SHARDS vLLM Servers ===" + +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + node="${SHARD_NODES[$shard_idx]}" + gpu_ids="${SHARD_GPU_IDS[$shard_idx]}" + head_ip="${HEAD_IPS[$shard_idx]}" + ray_port="${RAY_PORTS[$shard_idx]}" + api_port="${API_PORTS[$shard_idx]}" + ray_address="${head_ip}:${ray_port}" + vllm_log="$EVAL_LOGS_DIR/vllm_dp_shard${shard_idx}_${SLURM_JOB_ID}.log" + + echo " Shard $shard_idx: vLLM on $node, GPUs=$gpu_ids, port=$api_port" + + srun --export="$SRUN_EXPORT_ENV" --nodes=1 --ntasks=1 --overlap -w "$node" \ + env LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \ + PATH="$PATH" \ + PYTHONPATH="$PYTHONPATH" \ + HF_HOME="$HF_HOME" \ + HF_HUB_CACHE="$HF_HUB_CACHE" \ + HF_HUB_OFFLINE="1" \ + HF_TOKEN="$HF_TOKEN" \ + CUDA_VISIBLE_DEVICES="$gpu_ids" \ + RAY_ADDRESS="$ray_address" \ + VLLM_USE_V1="$VLLM_USE_V1" \ + VLLM_ALL2ALL_BACKEND="pplx" \ + TORCHDYNAMO_DISABLE="1" \ + TMPDIR="/tmp" \ + TRITON_CACHE_DIR="/tmp/triton_cache_${USER}" \ + TORCH_COMPILE_CACHE_DIR="/tmp/torch_cache_${USER}" \ + TORCHINDUCTOR_CACHE_DIR="/tmp/torchinductor_${USER}" \ + "${VLLM_CMD_BASE[@]}" \ + --host "0.0.0.0" \ + --port "$api_port" \ + --pipeline-parallel-size "$PP_SIZE" \ + >> "$vllm_log" 2>&1 & + + VLLM_PIDS[$shard_idx]=$! + echo " PID: ${VLLM_PIDS[$shard_idx]}, Log: $vllm_log" +done + +# --- Step 3d: Health check all shards --- +echo "" +echo "Waiting for all vLLM servers to become healthy..." + +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + node="${SHARD_NODES[$shard_idx]}" + api_port="${API_PORTS[$shard_idx]}" + health_url="http://127.0.0.1:${api_port}/v1/models" + + echo " Checking shard $shard_idx (port $api_port on $node)..." + for i in $(seq 1 $VLLM_MAX_RETRIES); do + if srun --export="$SRUN_EXPORT_ENV" --nodes=1 --ntasks=1 --overlap -w "$node" \ + curl -s "$health_url" > /dev/null 2>&1; then + echo " Shard $shard_idx is healthy!" + break + fi + if [ "$i" -eq "$VLLM_MAX_RETRIES" ]; then + echo "ERROR: Shard $shard_idx failed health check after $VLLM_MAX_RETRIES attempts" + tail -50 "$EVAL_LOGS_DIR/vllm_dp_shard${shard_idx}_${SLURM_JOB_ID}.log" 2>/dev/null || true + exit 1 + fi + sleep 15 + done +done + +echo "All vLLM servers are ready!" + +# ============================================================================== +# Download and shard dataset +# ============================================================================== +echo "" +echo "=== Downloading and sharding dataset ===" + +# Download/locate dataset (same logic as single-node sbatch) +if [[ "$REPO_ID" == /* ]]; then + echo "Using local dataset path: $REPO_ID" + DATASET_PATH="$REPO_ID" + if [ ! -d "$DATASET_PATH" ]; then + echo "ERROR: Local dataset path does not exist: $DATASET_PATH" + exit 1 + fi +else + echo "Downloading/locating dataset: $REPO_ID" + # Check dataset directories (from EVAL_DATASETS_DIRS or fallback) + _DS_DIRS="${EVAL_DATASETS_DIRS:-/e/data1/datasets/playground/ot/datasets:/e/scratch/jureap59/${USER}/datasets}" + DATASET_LOCAL_DIR="" + IFS=':' read -ra _DS_DIRS_ARR <<< "$_DS_DIRS" + for _ds_dir in "${_DS_DIRS_ARR[@]}"; do + if [ -d "${_ds_dir}/${SAFE_REPO}" ]; then + DATASET_LOCAL_DIR="${_ds_dir}/${SAFE_REPO}" + break + fi + done + DATASET_LOCAL_DIR="${DATASET_LOCAL_DIR:-${_DS_DIRS_ARR[0]}/${SAFE_REPO}}" + DOWNLOAD_LOG=$(mktemp /tmp/download_XXXXXX.log) + proxied $PYTHON_BIN "$DCFT/eval/snapshot_download.py" "$REPO_ID" --local-dir "$DATASET_LOCAL_DIR" > "$DOWNLOAD_LOG" 2>&1 + DOWNLOAD_EXIT=$? + cat "$DOWNLOAD_LOG" + DATASET_PATH=$(grep DATASET_PATH "$DOWNLOAD_LOG" | tail -n 1 | cut -d'=' -f2) + rm -f "$DOWNLOAD_LOG" + if [ $DOWNLOAD_EXIT -ne 0 ] || [ -z "${DATASET_PATH:-}" ]; then + echo "ERROR: Failed to get dataset path (exit code: $DOWNLOAD_EXIT)" + exit 1 + fi +fi +echo "Dataset path: $DATASET_PATH" + +# Shard dataset into N parts (simple round-robin symlink) +SHARD_DIR="/tmp/eval_shards_${SLURM_JOB_ID}" +mkdir -p "$SHARD_DIR" + +echo "Sharding dataset into $NUM_SHARDS parts..." +$PYTHON_BIN -c " +import os, sys +from pathlib import Path + +dataset_path = Path('$DATASET_PATH') +num_shards = $NUM_SHARDS +shard_dir = Path('$SHARD_DIR') + +task_dirs = sorted([d for d in dataset_path.iterdir() if d.is_dir()]) +print(f'Found {len(task_dirs)} tasks to shard into {num_shards} parts') + +for shard_idx in range(num_shards): + shard_path = shard_dir / f'shard_{shard_idx}' + shard_path.mkdir(exist_ok=True) + shard_tasks = task_dirs[shard_idx::num_shards] + print(f'Shard {shard_idx}: {len(shard_tasks)} tasks') + for task_dir in shard_tasks: + dest = shard_path / task_dir.name + if not dest.exists(): + os.symlink(task_dir, dest) +" +if [ $? -ne 0 ]; then + echo "ERROR: Dataset sharding failed" + exit 1 +fi + +TASK_COUNT=$(ls -d "$DATASET_PATH"/*/instruction.md 2>/dev/null | wc -l) +echo "Total tasks: $TASK_COUNT, shards: $NUM_SHARDS (~$((TASK_COUNT / NUM_SHARDS)) tasks/shard)" + +# ============================================================================== +# Construct run tag and directory +# ============================================================================== +if [ -n "$RUN_TAG_ARG" ]; then + RUN_TAG="$RUN_TAG_ARG" +else + RUN_TAG="${SAFE_REPO}_${SAFE_MODEL}" +fi +EVAL_JOBS_DIR="${EVAL_JOBS_DIR:-/e/data1/datasets/playground/mmlaion/shared/zhuang1_eval_jobs}" # fallback; listener sets EVAL_JOBS_DIR from cluster config + +echo "Run tag: $RUN_TAG" + +# Benchmark name (same logic as single-node) +declare -A BENCHMARK_NAME_MAP=( + ["DCAgent2_aider_polyglot"]="aider_polyglot" + ["DCAgent_dev_set_v2"]="dev_set_v2" + ["DCAgent_dev_set_71_tasks"]="dev_set_71_tasks" + ["DCAgent2_terminal_bench_2"]="terminal_bench_2" + ["DCAgent_swebench_verified_eval_set"]="swebench-verified-random-100-folders" + ["DCAgent2_bfcl-parity"]="bfcl-parity" +) +BASE_NAME="${BENCHMARK_NAME_MAP[$SAFE_REPO]:-$SAFE_REPO}" +BENCHMARK_SUFFIX="" +if [ -n "${EVAL_OVERRIDE_MEMORY_MB:-}" ] && [ "$EVAL_OVERRIDE_MEMORY_MB" != "1024" ]; then + mem_gb=$(( EVAL_OVERRIDE_MEMORY_MB / 1024 )) + BENCHMARK_SUFFIX="${BENCHMARK_SUFFIX}_${mem_gb}gb" +fi +if [ "$TIMEOUT_MULTIPLIER" != "1" ] && [ "$TIMEOUT_MULTIPLIER" != "1.0" ]; then + BENCHMARK_SUFFIX="${BENCHMARK_SUFFIX}_${TIMEOUT_MULTIPLIER}x" +fi +BENCHMARK_NAME="${BASE_NAME}${BENCHMARK_SUFFIX}" +export BENCHMARK_NAME MODEL REPO_ID SLURM_JOB_ID +echo "Benchmark name: $BENCHMARK_NAME" + +# ============================================================================== +# Update DB: Pending → Started +# ============================================================================== +echo "Creating/updating DB job entry..." +HARBOR_VERSION=$($PYTHON_BIN -c "import harbor; print(harbor.__version__)" 2>/dev/null || echo "unknown") +export HARBOR_VERSION RUN_TAG + +if [ -n "$DB_JOB_ID" ]; then + echo "Updating Pending job $DB_JOB_ID → Started" + proxied $PYTHON_BIN - <<'PY' || true +import os, sys +from database.unified_db.utils import update_job_status_to_started +run_tag = os.environ["RUN_TAG"] +agent_name = os.environ.get("EVAL_AGENT_NAME", "terminus-2") +timeout_multiplier = float(os.environ.get("EVAL_TIMEOUT_MULTIPLIER", "1.0")) +harbor_version = os.environ.get("HARBOR_VERSION", "unknown") +config = {"agent": agent_name, "env": "daytona", "timeout_multiplier": timeout_multiplier} +result = update_job_status_to_started( + job_name=run_tag, + n_trials=int(os.environ.get("EVAL_N_CONCURRENT", "128")), + n_rep_eval=int(os.environ.get("EVAL_N_ATTEMPTS", "3")), + config=config, + harbor_package_version=harbor_version, +) +if not result.get("success"): + print(f"WARNING: Pending→Started update failed: {result.get('error')}", file=sys.stderr) +else: + print(f"DB job {run_tag} → Started") +PY +fi + +# ============================================================================== +# Resolve harbor config +# ============================================================================== +if [[ "$CONFIG_YAML" != /* ]]; then + if [ -f "$DCFT/eval/${CLUSTER_NAME}/$CONFIG_YAML" ]; then + HARBOR_CONFIG="$DCFT/eval/${CLUSTER_NAME}/$CONFIG_YAML" + elif [ -f "$DCFT/eval/MBZ/$CONFIG_YAML" ]; then + HARBOR_CONFIG="$DCFT/eval/MBZ/$CONFIG_YAML" + else + HARBOR_CONFIG="$DCFT/eval/jupiter/$CONFIG_YAML" + fi +else + HARBOR_CONFIG="$CONFIG_YAML" +fi + +# Build extra harbor args +EXTRA_HARBOR_ARGS="" +if [ -n "${EVAL_SNAPSHOT_NAME:-}" ]; then + EXTRA_HARBOR_ARGS="--environment-kwarg snapshot_template_name=$EVAL_SNAPSHOT_NAME --no-force-build" +fi +if [ "$TIMEOUT_MULTIPLIER" != "1" ] && [ "$TIMEOUT_MULTIPLIER" != "1.0" ]; then + EXTRA_HARBOR_ARGS="$EXTRA_HARBOR_ARGS --timeout-multiplier $TIMEOUT_MULTIPLIER" +fi +if [ -n "${EVAL_OVERRIDE_MEMORY_MB:-}" ]; then + EXTRA_HARBOR_ARGS="$EXTRA_HARBOR_ARGS --override-memory-mb $EVAL_OVERRIDE_MEMORY_MB" +fi +if [ -n "${EVAL_AUTO_SNAPSHOT:-}" ]; then + EXTRA_HARBOR_ARGS="$EXTRA_HARBOR_ARGS --environment-kwarg auto_snapshot=$EVAL_AUTO_SNAPSHOT" +fi + +# ============================================================================== +# Run Harbor Jobs in Parallel (one per shard) +# ============================================================================== +echo "" +echo "=== Running $NUM_SHARDS Harbor Jobs in Parallel ===" +set +e + +declare -a HARBOR_PIDS +declare -a HARBOR_LOGS +declare -a SHARD_RUN_DIRS + +run_harbor_for_shard() { + local shard_idx=$1 + local head_node="${SHARD_NODES[$shard_idx]}" + local api_port=${API_PORTS[$shard_idx]} + local shard_dataset="$SHARD_DIR/shard_${shard_idx}" + local shard_run_tag="${RUN_TAG}_shard${shard_idx}" + local harbor_log="$EVAL_LOGS_DIR/harbor_dp_shard${shard_idx}_${SLURM_JOB_ID}.log" + local shard_run_dir="${EVAL_JOBS_DIR}/${shard_run_tag}" + + HARBOR_LOGS[$shard_idx]="$harbor_log" + SHARD_RUN_DIRS[$shard_idx]="$shard_run_dir" + + echo " Shard $shard_idx: dataset=$shard_dataset, port=$api_port, run_tag=$shard_run_tag" + + # Check for resume (existing job dir for this shard) + local existing_job_dir="$shard_run_dir" + local harbor_cmd + + if [ -d "$existing_job_dir" ] && [ -f "$existing_job_dir/config.json" ]; then + echo " Resuming existing job at $existing_job_dir" + harbor_cmd="harbor jobs resume \ + --job-path \"$existing_job_dir\" \ + --filter-error-type EnvironmentStartTimeoutError \ + --filter-error-type DaytonaError \ + --filter-error-type DaytonaRateLimitError \ + --filter-error-type CancelledError \ + --filter-error-type AgentEnvironmentTimeoutError \ + --filter-error-type SandboxBuildFailedError" + else + echo " Starting new job" + harbor_cmd="harbor jobs start \ + -p \"$shard_dataset\" \ + --jobs-dir \"$EVAL_JOBS_DIR\" \ + --n-concurrent \"$N_CONCURRENT\" \ + --agent \"$AGENT_NAME\" \ + --model \"hosted_vllm/$MODEL\" \ + --env \"daytona\" \ + --agent-kwarg \"api_base=http://localhost:${api_port}/v1\" \ + --agent-kwarg \"key=fake_key\" \ + --agent-kwarg \"max_tokens=16384\" \ + --agent-kwarg 'model_info={\"max_output_tokens\":16384,\"max_input_tokens\":32768,\"input_cost_per_token\":0.0,\"output_cost_per_token\":0.0}' \ + --n-attempts \"$N_ATTEMPTS\" \ + --job-name \"$shard_run_tag\" \ + --export-traces \ + --config \"$HARBOR_CONFIG\" \ + --debug" + + if [ "$ENABLE_THINKING" = "true" ]; then + harbor_cmd="$harbor_cmd --agent-kwarg \"enable_thinking=true\"" + fi + if [ -n "$AGENT_PARSER" ]; then + harbor_cmd="$harbor_cmd --agent-kwarg \"parser=$AGENT_PARSER\"" + fi + if [ -n "$EXTRA_HARBOR_ARGS" ]; then + harbor_cmd="$harbor_cmd $EXTRA_HARBOR_ARGS" + fi + fi + + # Run harbor on the shard's head node (via proxychains for Daytona) + if [ -x "$PROXYCHAINS_BIN" ] && [ -n "${PROXYCHAINS_CONF_FILE:-}" ]; then + srun --export="$SRUN_EXPORT_ENV" --nodes=1 --ntasks=1 --overlap -w "$head_node" \ + "$PROXYCHAINS_BIN" -f "$PROXYCHAINS_CONF_FILE" \ + bash -c "$harbor_cmd" \ + >> "$harbor_log" 2>&1 & + else + srun --export="$SRUN_EXPORT_ENV" --nodes=1 --ntasks=1 --overlap -w "$head_node" \ + bash -c "$harbor_cmd" \ + >> "$harbor_log" 2>&1 & + fi + + HARBOR_PIDS[$shard_idx]=$! + echo " PID: ${HARBOR_PIDS[$shard_idx]}, Log: $harbor_log" +} + +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + run_harbor_for_shard $shard_idx +done + +echo "" +echo "All Harbor jobs started. Waiting for completion..." + +FAILED=0 +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + wait ${HARBOR_PIDS[$shard_idx]} || FAILED=$((FAILED + 1)) + echo " Shard $shard_idx completed (exit: $?)" +done + +set -e + +# ============================================================================== +# Save meta.env (per shard) +# ============================================================================== +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + shard_run_dir="${SHARD_RUN_DIRS[$shard_idx]}" + mkdir -p "$shard_run_dir" + { + echo "MODEL=$MODEL" + echo "REPO_ID=$REPO_ID" + echo "TIMESTAMP=$TIMESTAMP" + echo "SLURM_JOB_ID=$SLURM_JOB_ID" + echo "DB_JOB_ID=${DB_JOB_ID:-}" + echo "BENCHMARK_ID=${BENCHMARK_ID:-}" + echo "N_CONCURRENT=$N_CONCURRENT" + echo "N_ATTEMPTS=$N_ATTEMPTS" + echo "AGENT_NAME=$AGENT_NAME" + echo "ENABLE_THINKING=$ENABLE_THINKING" + echo "TIMEOUT_MULTIPLIER=$TIMEOUT_MULTIPLIER" + echo "BENCHMARK_NAME=$BENCHMARK_NAME" + echo "NUM_SHARDS=$NUM_SHARDS" + echo "SHARD_IDX=$shard_idx" + echo "DP_MODE=true" + } > "$shard_run_dir/meta.env" +done + +# ============================================================================== +# Error check per shard + aggregate +# ============================================================================== +echo "" +echo "=== Checking errors across all shards ===" + +TOTAL_INVALID=0 +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + shard_run_dir="${SHARD_RUN_DIRS[$shard_idx]}" + result_file="$shard_run_dir/result.json" + if [ -f "$result_file" ]; then + SHARD_INVALID=$($PYTHON_BIN -c " +import json +try: + with open('$result_file') as f: + data = json.load(f) + BENIGN = {'AgentTimeoutError','ContextLengthExceededError','SummarizationTimeout','SummarizationTimeoutError'} + total = 0 + if 'stats' in data and 'evals' in data['stats']: + for ek, ev in data['stats']['evals'].items(): + if 'exception_stats' in ev: + for et, ids in ev['exception_stats'].items(): + if et not in BENIGN and isinstance(ids, list): + total += len(ids) + print(total) +except: print('0') +" 2>&1 | tail -1) + echo " Shard $shard_idx: $SHARD_INVALID invalid errors" + TOTAL_INVALID=$((TOTAL_INVALID + SHARD_INVALID)) + else + echo " Shard $shard_idx: no result.json" + fi +done + +echo "Total invalid errors across all shards: $TOTAL_INVALID (threshold: $ERROR_THRESHOLD)" + +if [ "$TOTAL_INVALID" -gt "$ERROR_THRESHOLD" ]; then + echo "Too many invalid errors ($TOTAL_INVALID > $ERROR_THRESHOLD), skipping upload" + exit 0 +fi + +# ============================================================================== +# Upload results (merge all shard dirs) +# ============================================================================== +echo "" +echo "=== Uploading results ===" + +# Collect all shard job dirs +JOB_DIRS="" +for shard_idx in $(seq 0 $((NUM_SHARDS - 1))); do + shard_run_dir="${SHARD_RUN_DIRS[$shard_idx]}" + if [ -d "$shard_run_dir" ]; then + JOB_DIRS="$JOB_DIRS $shard_run_dir" + fi +done + +echo "Uploading from shard dirs: $JOB_DIRS" + +# Use upload_traces.py to merge and upload all shard results +export PYTHONPATH="${DCFT}:${DCFT}/data/sbatches/teacher_scripts:${PYTHONPATH:-}" +proxied $PYTHON_BIN -c " +import os, sys, re, hashlib + +from database.unified_db.utils import upload_eval_results + + +def sanitize_hf_repo_id(repo_id, max_length=96): + def collapse(s): + prev = None + while s != prev: + prev = s + s = s.replace('--', '-').replace('..', '.') + return s + org, name = repo_id.split('/', 1) if '/' in repo_id else (None, repo_id) + name = re.sub(r'[^A-Za-z0-9._-]', '-', name) + name = collapse(name).strip('-.') + if not name: name = 'repo' + limit = max_length - (len(org) + 1 if org else 0) + if len(name) > limit: + digest = hashlib.sha1(name.encode()).hexdigest()[:8] + keep = max(1, limit - len(digest)) + base = name[:keep].rstrip('-.') + if not base: base = 'r' + name = f'{base}{digest}' + name = collapse(name).strip('-.') + if name[0] in '-.': name = 'r' + name[1:] + if name[-1] in '-.': name = name[:-1] + '0' + return f'{org}/{name}' if org else name + + +# Upload each shard separately +shard_dirs = '$JOB_DIRS'.split() +run_tag = os.environ['RUN_TAG'] +username = os.environ.get('UPLOAD_USERNAME', os.environ.get('USER', 'jupiter')) +hf_token = os.environ['HF_TOKEN'] +benchmark_name = os.environ.get('BENCHMARK_NAME', '') + +for shard_dir in shard_dirs: + shard_name = os.path.basename(shard_dir) + hf_repo_id = sanitize_hf_repo_id(f'DCAgent2/{shard_name}') + print(f'[uploader] Uploading shard {shard_name} to {hf_repo_id}') + + benchmark_version_hash = hashlib.sha256(benchmark_name.encode()).hexdigest() + + try: + upload_eval_results( + shard_dir, + username=username, + error_mode='skip_on_error', + hf_token=hf_token, + hf_repo_id=hf_repo_id, + register_benchmark=True, + benchmark_name=benchmark_name, + benchmark_version_hash=benchmark_version_hash, + ) + print(f'[uploader] {shard_name} done.') + except Exception as e: + print(f'[uploader] WARNING: {shard_name} upload failed: {e}') +" 2>&1 +UPLOAD_EXIT=$? + +# Clean up shard directory +rm -rf "$SHARD_DIR" 2>/dev/null || true + +echo "==============================================" +if [ $FAILED -eq 0 ] && [ $UPLOAD_EXIT -eq 0 ]; then + echo "DP Eval COMPLETED ($NUM_SHARDS shards)" +else + echo "DP Eval completed with issues (failed shards: $FAILED, upload exit: $UPLOAD_EXIT)" +fi +echo "==============================================" diff --git a/eval/unified_eval_listener.py b/eval/unified_eval_listener.py new file mode 100644 index 000000000..c69006528 --- /dev/null +++ b/eval/unified_eval_listener.py @@ -0,0 +1,3759 @@ +#!/usr/bin/env python3 +""" +Unified Eval Listener v6 - Polls Supabase for models and submits SLURM eval jobs. + +Based on v5, with disk-based resume replacing the v5 DB-based DaytonaError resume. +Scans the eval jobs directory for incomplete/error-heavy jobs and resubmits them +with the same run_tag so harbor auto-resumes (skips completed trials, retries failed ones). + +Key v6 features over v5: + - Disk-based resume: scans --jobs-dir for incomplete/errored job dirs + - Persistent sliding-window batch_size across iterations + - hf_overrides support in baseline model configs + - Supabase queries wrapped in try/except for resilience +Uses unified_eval_harbor.sbatch as the SLURM job template. + + +=============================================================================== +FLAG REFERENCE +=============================================================================== + +--- Preset & Dataset Selection --- + +--preset, -p {aider,bfcl,swebench,v2,tb2,v1} + Load a named preset that bundles dataset, concurrency, error threshold, and + other defaults tuned for a specific benchmark. CLI flags override any preset + value. Almost all runs should start with a preset. + + Preset details: + aider Dataset: DCAgent2/aider_polyglot. n_concurrent=32, error_threshold=10, thinking=on. + bfcl Dataset: DCAgent2/bfcl-parity. n_concurrent=32, error_threshold=10, thinking=on, vllm_retries=20. + swebench Dataset: DCAgent2/swebench-verified-*. n_concurrent=32, error_threshold=15, thinking=on, vllm_retries=20, + agent_parser=xml, gpu_mem=0.95, config_yaml=no_override. HF existence check on. + v2 Dataset: DCAgent/dev_set_v2. n_concurrent=32, error_threshold=10, thinking=on, vllm_retries=20. + tb2 Dataset: DCAgent2/terminal_bench_2. n_concurrent=32, error_threshold=10, thinking=on, + gpu_mem=0.95, slurm_time=48h, config_yaml=no_override. + v1 Dataset: DCAgent/dev_set_71_tasks. n_concurrent=32, error_threshold=10, thinking=on, vllm_retries=20. + + Tuning: Pick the preset matching your benchmark. Override individual params + with CLI flags (e.g. --n-concurrent 64 to double concurrency). + +--datasets, -d + Comma- or space-separated list of HuggingFace dataset repos. Overrides the + preset's dataset list. Use this for one-off evals against custom datasets. + Example: --datasets "DCAgent/dev_set_v2,DCAgent2/terminal_bench_2" + +--sbatch-script, -s + Path to the sbatch template. Default: unified_eval_harbor_v4.sbatch (or + whatever the preset specifies). Only change this if you have a custom sbatch. + + +--- Model Filtering --- + +--priority-file + Text file listing HuggingFace model names (org/model), one per line. + Lines starting with # are comments; blank lines are ignored. + File order = submission priority: earlier lines are submitted first. + Hot-reloaded every iteration — edit the file without restarting the listener. + + Env: EVAL_LISTENER_PRIORITY_FILE + +--priority-mode {filter_only,priority_first} [default: filter_only] + filter_only — Only evaluate models IN the priority file. All others skipped. + priority_first — Evaluate ALL models, but submit priority models first. + + Tuning: Use filter_only (default) when you have a curated list of models to + evaluate. Use priority_first when you want to evaluate everything but ensure + specific models get SLURM slots first. + + Env: EVAL_LISTENER_PRIORITY_MODE + +--require-priority-list + Safety flag. If set and no priority file is loaded (missing file or empty), + the listener skips ALL models instead of evaluating everything. Prevents + accidental mass submissions when a priority file path is misconfigured. + + Env: EVAL_LISTENER_REQUIRE_PRIORITY_LIST="1" + +--blacklist-file [v4 NEW] + Text file listing models that should NEVER be submitted, same format as + --priority-file (one model per line, # comments, blank lines ignored). + Blacklist overrides priority: if a model appears in both files, it is blocked. + Hot-reloaded every iteration, same as --priority-file. + + Tuning: Use this to permanently exclude known-bad models (e.g. broken + checkpoints, models that consistently OOM, duplicates you don't want to + re-evaluate). Faster than removing them from the priority file because + the blacklist is checked first — no DB queries wasted on blocked models. + + Env: EVAL_LISTENER_BLACKLIST_FILE + +--check-hf-exists + Before submitting, validate that the model actually exists on HuggingFace Hub. + Adds a network round-trip per model but prevents wasted SLURM jobs on typos + or deleted models. The swebench preset enables this by default. + + Env: EVAL_LISTENER_CHECK_HF_EXISTS="1" + + +--- Timing & Lifecycle --- + +--lookback-days [default: 1000] + How far back to query the Supabase `models` table (by creation_time). + Priority models bypass this window — they are always fetched by name + regardless of when they were added. + + Tuning: Keep this large (default 1000) to catch old models. Reduce only if + DB queries are slow and you know all target models are recent. + + Env: EVAL_LISTENER_LOOKBACK_DAYS + +--check-hours [default: 4.0] + Hours to sleep between iterations. Each iteration re-queries the DB, hot- + reloads priority/blacklist files, and submits any new jobs. + + Tuning: For active development with frequent model uploads, use 1-2h. + For stable production runs, 4-12h is fine. Ignored when --once is set. + + Env: EVAL_LISTENER_CHECK_HOURS + +--stale-hours [default: 24] + A job in "Started" status older than this is considered stale and will be + resubmitted. Covers cases where the sbatch job crashed without updating + the DB to Finished. + + Tuning: Set to at least 1.5x your SLURM time limit. If --slurm-time is + 24:00:00, keep this at 24 (default). If you use --slurm-time 48:00:00 + (like tb2), bump to 48-72. + +--stale-pending-hours [default: 48] + A job in "Pending" status older than this is considered stale. The listener + will scancel the old SLURM job (if tracked) and resubmit. + + Tuning: Should be >= --stale-hours. Default of 48h gives Pending jobs extra + time to get through the SLURM queue before being killed. + + +--- Sbatch / vLLM Parameters (passed to sbatch via env vars) --- + +--n-concurrent [default: 64, preset overrides] + Number of concurrent Harbor evaluation jobs inside the sbatch. Controls how + many sandbox tasks run in parallel against the vLLM server. + + Tuning: Depends on model size and GPU memory. + - 7-8B models on GH200 (96GB): 32-64 is safe. + - 32B models: 8-16 (higher causes vLLM queue buildup → AgentTimeoutError). + - 131K context models: 4-8 (KV cache fills fast at high concurrency). + If you see many AgentTimeoutErrors, reduce this. If eval is slow and vLLM + GPU utilization is low, increase it. + +--n-attempts [default: 3] + Number of retry attempts per Harbor task. If a task fails (e.g. sandbox + timeout), Harbor retries it up to this many times. + + Tuning: 3 is good for most benchmarks. Raise to 5 for flaky benchmarks. + Lowering to 1 speeds up runs but increases noise from transient failures. + +--gpu-memory-util [default: 0.9] + Fraction of GPU memory allocated to vLLM via --gpu-memory-utilization. + + Tuning: + - 0.90 (default): safe for 7-8B models on GH200 (96GB). Leaves headroom + for GPU memory variance across nodes. + - 0.95: used by swebench/tb2 presets for larger models or when you need + maximum KV cache capacity. Risk: some GH200 nodes have slightly less + available memory and will OOM at 0.95 (use --exclude in sbatch). + - Never go above 0.95. Below 0.85 wastes memory. + +--error-threshold [default: 3, preset overrides] + Maximum number of "invalid" errors allowed before the sbatch script aborts + result upload. Invalid = any error type EXCEPT AgentTimeoutError, + ContextLengthExceededError, SummarizationTimeout, SummarizationTimeoutError. + + Tuning: Controls quality gating. Low values (3) are strict — a few + DaytonaErrors or unexpected crashes abort the upload. Higher values (10-15) + are more tolerant, appropriate for benchmarks where some sandbox flakiness + is expected. + - aider: 3 (strict, small dataset) + - v2/tb2: 10 (moderate, larger datasets with occasional flakes) + - swebench: 15 (lenient, swebench sandboxes are flakier) + + --daytona-threshold is a backward-compatible alias for this flag. + +--vllm-max-retries [default: 5, preset overrides] + Number of times the sbatch script retries starting the vLLM server. + vLLM occasionally fails to start on first attempt (port conflicts, + CUDA initialization issues). + + Tuning: 5 is fine for quick detection of real failures. Presets like v2 + and swebench use 20 for more resilience on busy clusters. + +--agent-parser [default: "" (none)] + Parser type for Harbor agent output. Set to "xml" for swebench (which + uses XML-structured agent responses). Leave empty for all other benchmarks. + + Tuning: Only change this if you're adding a new benchmark with a custom + agent output format. The swebench preset sets this automatically. + +--slurm-time [default: "24:00:00"] + SLURM wall-clock time limit for the sbatch job. Format: HH:MM:SS. + + Tuning: 24h is enough for most benchmarks. tb2 preset uses 48h because + terminal_bench_2 tasks are longer-running. If jobs are hitting the time + limit and getting killed, increase this and also bump --stale-hours. + +--slurm-partition [default: "gh"] + SLURM partition to submit jobs to. On TACC, "gh" is the GH200 GPU partition. + +--agent-name [default: "terminus-2"] + Agent name written to DB entries and used by Harbor for evaluation config. + This determines which agent implementation Harbor uses to run the eval tasks. + +--enable-thinking + Enable thinking/reasoning blocks in vLLM model inference. Most presets + enable this by default. Only disable if the model doesn't support thinking + or you want to test non-thinking mode. + +--upload-username [default: current OS user] + Username recorded in DB entries and result uploads. Auto-detected from + the OS user if not specified. + + Env: EVAL_UPLOAD_USERNAME + + +--- v3 Enhancement: Per-Listener SLURM Throttle --- + +--max-jobs-submitted [default: 20] + Maximum number of active SLURM jobs this listener instance is allowed to + have running simultaneously. The listener tracks which SLURM job IDs it + submitted and checks squeue to count only those still active. + + Tuning: This is PER-LISTENER, not global. Multiple listeners can run in + parallel with independent budgets. Set based on your fair-share allocation: + - Single listener: 10-20 is typical. + - Multiple listeners: split your budget (e.g. v2=10, swebench=5). + When the limit is reached, the listener queues submissions by priority + order and drops the lowest-priority ones. + + Env: EVAL_LISTENER_MAX_JOBS + + +--- v3 Enhancement: Daytona Resource Pre-flight --- + +--check-daytona-resources + Enable Daytona API sandbox count check at startup and each iteration. + If active sandboxes are at or above the limit, the listener skips that + iteration entirely. Requires DAYTONA_API_KEY in environment. + + Tuning: Enable this in production to prevent overwhelming the Daytona + sandbox pool. Not needed for small-scale or development runs. + +--daytona-sandbox-limit [default: 2000] + Maximum expected active sandboxes. The listener skips submissions when + the active count reaches this number. + +--daytona-warning-buffer [default: 0.9] + Fraction of the sandbox limit at which a warning is logged. At 0.9 with + limit=2000, warns when active sandboxes reach 1800. + + +--- v3 Enhancement: Model Retry Tracking --- + +--track-model-retries + Enable tracking of how many times each model has been started. Models + exceeding the retry threshold are deprioritized (moved to end of the + submission queue, not blocked entirely). + + Tuning: Enable this for long-running listeners to prevent repeatedly + resubmitting models that keep failing. The sbatch script appends to the + shared log when transitioning a job from Pending → Started. + +--model-retry-threshold [default: 5] + Number of eval starts before a model is deprioritized. Deprioritized + models are still submitted, just last in the queue (and may be dropped + if --max-jobs-submitted truncates the list). + + Tuning: 3-5 for strict environments. Higher (10+) if transient failures + are common and you want to give models more chances. + +--eval-starts-log [default: auto-generated] + Path to the shared append-only log file where eval starts are recorded. + Auto-generated with a benchmark+timestamp suffix if not specified. + Multiple listeners using the same log file will share retry counts. + + Tuning: If you run multiple listeners for the same benchmark and want + shared retry tracking, point them at the same log file. + + +--- v3 Enhancement: Timeout-Config-Sensitive Dedup --- + +--timeout-aware + Change job dedup logic to check model + benchmark + agent + timeout_multiplier + instead of just model + benchmark. This allows running the same model with + different timeout configurations without one blocking the other. + + Tuning: Enable when running A/B experiments with different timeout settings. + When disabled (default), two listeners submitting the same model with + different --timeout-multiplier values will conflict (one sees the other's + job and skips). + +--timeout-multiplier [default: 1.0] + Harbor timeout multiplier, passed to the sbatch job and stored in the DB + job config. Values >1.0 give tasks more time; <1.0 makes them stricter. + + Tuning: Use with --timeout-aware for controlled experiments: + --timeout-multiplier 0.25 (aggressive timeout, fast failures) + --timeout-multiplier 1.0 (default) + --timeout-multiplier 2.0 (lenient, for slow models) + --timeout-multiplier 4.0 (very lenient, for debugging) + + +--- Execution Mode --- + +--dry-run + Preview mode: runs one full iteration (DB queries, filtering, status checks) + but does NOT submit any sbatch jobs. Logs what WOULD be submitted. Implies + --once. Use this to verify your flags before a real run. + + Env: EVAL_LISTENER_DRY_RUN="1" + +--once + Run a single iteration and exit. Useful for cron-triggered runs or one-shot + submissions. Without this, the listener loops forever (sleeping --check-hours + between iterations). + +--verbose, -v + Enable detailed logging: shows every model skipped (with reason), priority + list contents, blacklist contents, and per-model DB status checks. + +--log-file + Explicit log file path. Default: auto-generated in experiments/listener_logs/ + with a preset+timestamp name. + + Env: EVAL_LISTENER_LOG_DIR (for the directory) + + +=============================================================================== +ENVIRONMENT VARIABLES (all optional, CLI args take precedence) +=============================================================================== + + EVAL_LISTENER_LOOKBACK_DAYS Days to look back for models (default: 1000) + EVAL_LISTENER_CHECK_HOURS Hours between iterations (default: 4.0) + EVAL_LISTENER_SBATCH SBATCH script to use + EVAL_LISTENER_LOG_DIR Log directory (default: experiments/listener_logs) + EVAL_LISTENER_DATASETS Comma/space/newline list of HF dataset repos + EVAL_LISTENER_PRIORITY_FILE Path to priority models file (hot-reloaded) + EVAL_LISTENER_BLACKLIST_FILE Path to blacklist models file (hot-reloaded) [v4] + EVAL_LISTENER_DRY_RUN "1" or "true" to enable dry run mode + EVAL_LISTENER_REQUIRE_PRIORITY_LIST "1" or "true" to require priority list + EVAL_LISTENER_PRIORITY_MODE "filter_only" or "priority_first" + EVAL_LISTENER_CHECK_HF_EXISTS "1" or "true" to validate HF model existence + EVAL_LISTENER_MAX_JOBS Per-listener SLURM job limit (default: 20) + EVAL_UPLOAD_USERNAME Username for DB entries (default: OS user) + DAYTONA_API_KEY Required for --check-daytona-resources + + +=============================================================================== +QUICK START EXAMPLES +=============================================================================== + + # Most common: evaluate priority models on dev_set_v2 + python unified_eval_listener_v4.py --preset v2 \\ + --priority-file v2_priority_models_richard.txt + + # Preview what would be submitted (no actual jobs) + python unified_eval_listener_v4.py --preset v2 --dry-run --once \\ + --priority-file v2_priority_models_richard.txt --verbose + + # Block known-bad models + python unified_eval_listener_v4.py --preset v2 \\ + --priority-file v2_priority_models_richard.txt \\ + --blacklist-file bad_models.txt + + # Full v3/v4 features enabled + python unified_eval_listener_v4.py --preset v2 \\ + --priority-file v2_priority_models_richard.txt \\ + --blacklist-file bad_models.txt \\ + --error-threshold 10 --max-jobs-submitted 15 \\ + --check-daytona-resources \\ + --track-model-retries --model-retry-threshold 3 \\ + --timeout-aware --timeout-multiplier 2.0 + + # Two listeners with independent SLURM budgets + python unified_eval_listener_v4.py --preset v2 --max-jobs-submitted 10 & + python unified_eval_listener_v4.py --preset swebench --max-jobs-submitted 5 & + + # A/B timeout experiment (requires --timeout-aware on both) + python unified_eval_listener_v4.py --preset v2 --timeout-aware \\ + --timeout-multiplier 1.0 --max-jobs-submitted 10 & + python unified_eval_listener_v4.py --preset v2 --timeout-aware \\ + --timeout-multiplier 2.0 --max-jobs-submitted 5 & +""" + +import argparse +import getpass +import json +import os +import re +import subprocess +import sys +import time +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +import yaml + +# Add leaderboard utilities to path +# Add project root to path for database.unified_db imports +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_ROOT = os.path.abspath(os.path.join(_SCRIPT_DIR, "..", "..")) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +from database.unified_db.utils import get_supabase_client, load_supabase_keys + + +# --------------------------------------------------------------------------- +# Secrets loading (Jupiter-specific: load ~/secrets.env at import time) +# --------------------------------------------------------------------------- +def _load_secrets(path: Optional[str] = None) -> None: + """Load secrets from env file, then call unified_db's load_supabase_keys.""" + path = ( + path + or os.environ.get("DC_AGENT_SECRET_ENV") + or os.environ.get("KEYS") + or os.path.expanduser("~/secrets.env") + ) + if path and os.path.isfile(os.path.expanduser(path)): + with open(os.path.expanduser(path)) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[7:].strip() + k, v = line.split("=", 1) + os.environ[k.strip()] = v.strip().strip("'\"") + # Alias SUPABASE_KEY -> SUPABASE_ANON_KEY if the latter is missing + # (some secrets.env files use the shorter name) + if os.environ.get("SUPABASE_KEY") and not os.environ.get("SUPABASE_ANON_KEY"): + os.environ["SUPABASE_ANON_KEY"] = os.environ["SUPABASE_KEY"] + try: + load_supabase_keys() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Harbor config parsing -- extract eval config fields for dedup +# --------------------------------------------------------------------------- +def parse_harbor_eval_config(path: Optional[str]) -> Dict: + """Parse eval-relevant config fields from a Harbor YAML config. + + Returns dict with keys: timeout_multiplier, override_cpus, + override_memory_mb, override_storage_mb (only if set). + """ + if not path or not os.path.isfile(path): + return {} + try: + import yaml + with open(path) as f: + cfg = yaml.safe_load(f) or {} + except Exception as e: + log(f"WARNING: failed to parse harbor config {path}: {e}") + return {} + result: Dict = {} + if cfg.get("timeout_multiplier") is not None: + result["timeout_multiplier"] = float(cfg["timeout_multiplier"]) + env_cfg = cfg.get("environment") or {} + for key in ("override_cpus", "override_memory_mb", "override_storage_mb"): + if env_cfg.get(key) is not None: + result[key] = int(env_cfg[key]) + return result + + +# --------------------------------------------------------------------------- +# Baseline model config mapping -- per-model vLLM overrides +# --------------------------------------------------------------------------- +_BASELINE_MODEL_CONFIGS: Optional[Dict[str, Dict]] = None +_BASELINE_MODEL_PATTERNS: Optional[List[Dict]] = None + + +def load_baseline_model_configs(path: Optional[str]) -> Dict[str, Dict]: + """Load baseline model -> vLLM config mapping from YAML file. + + Returns dict mapping HF model name to vLLM serving params. + Also loads pattern-based fallback configs (stored in _BASELINE_MODEL_PATTERNS). + """ + global _BASELINE_MODEL_CONFIGS, _BASELINE_MODEL_PATTERNS + if _BASELINE_MODEL_CONFIGS is not None: + return _BASELINE_MODEL_CONFIGS + + if not path or not os.path.isfile(path): + _BASELINE_MODEL_CONFIGS = {} + _BASELINE_MODEL_PATTERNS = [] + return _BASELINE_MODEL_CONFIGS + + try: + import yaml + with open(path) as f: + data = yaml.safe_load(f) or {} + + # Start with per-model entries + per_model = data.get("models", {}) + + # Expand groups: each group has a "models" list + shared config fields. + # Group config is the base; per-model entries are merged on top (override wins). + expanded: Dict[str, Dict] = {} + for group in data.get("groups", []): + model_names = group.get("models", []) + shared_cfg = {k: v for k, v in group.items() if k != "models"} + for name in model_names: + expanded[name] = dict(shared_cfg) # copy so mutations are isolated + + # Merge per-model overrides on top of group defaults + for name, overrides in per_model.items(): + if name in expanded: + expanded[name].update(overrides) + else: + expanded[name] = dict(overrides) + + _BASELINE_MODEL_CONFIGS = expanded + _BASELINE_MODEL_PATTERNS = data.get("patterns", []) + n_groups = len(data.get("groups", [])) + log(f"Loaded {len(_BASELINE_MODEL_CONFIGS)} baseline model config(s) " + f"({n_groups} group(s), {len(per_model)} override(s)) and " + f"{len(_BASELINE_MODEL_PATTERNS)} pattern(s) from {path}") + except Exception as e: + log(f"WARNING: failed to load baseline model configs from {path}: {e}") + _BASELINE_MODEL_CONFIGS = {} + _BASELINE_MODEL_PATTERNS = [] + + return _BASELINE_MODEL_CONFIGS + + +def _match_pattern_config(hf_model: str) -> Optional[Dict]: + """Try to match a model name against pattern-based configs. + + Patterns are checked in order; first match wins. + Each pattern has a 'match' field (regex or substring) and config fields. + """ + if not _BASELINE_MODEL_PATTERNS: + return None + for pattern_entry in _BASELINE_MODEL_PATTERNS: + pattern = pattern_entry.get("match", "") + if not pattern: + continue + if re.search(pattern, hf_model): + return {k: v for k, v in pattern_entry.items() if k != "match"} + return None + + +def get_vllm_env_overrides(hf_model: str, configs: Dict[str, Dict]) -> Dict[str, str]: + """Get vLLM env var overrides for a model from the baseline config mapping. + + Tries exact model name match first, then falls back to pattern matching. + Returns dict of EVAL_VLLM_* env vars to pass to the eval script. + """ + match_source = None + cfg = configs.get(hf_model) + if cfg: + match_source = "exact/group" + else: + cfg = _match_pattern_config(hf_model) + if cfg: + match_source = "pattern" + if not cfg: + return {} + + log(f" Baseline config [{match_source}] for {hf_model}: {cfg}") + + env: Dict[str, str] = {} + if cfg.get("tensor_parallel_size") is not None: + env["EVAL_VLLM_TENSOR_PARALLEL_SIZE"] = str(cfg["tensor_parallel_size"]) + if cfg.get("max_model_len") is not None: + env["EVAL_VLLM_MAX_MODEL_LEN"] = str(cfg["max_model_len"]) + if cfg.get("swap_space") is not None: + env["EVAL_VLLM_SWAP_SPACE"] = str(cfg["swap_space"]) + if cfg.get("trust_remote_code"): + env["EVAL_VLLM_TRUST_REMOTE_CODE"] = "1" + if cfg.get("tool_call_parser"): + env["EVAL_VLLM_TOOL_CALL_PARSER"] = cfg["tool_call_parser"] + if cfg.get("reasoning_parser"): + env["EVAL_VLLM_REASONING_PARSER"] = cfg["reasoning_parser"] + if cfg.get("extra_args"): + env["EVAL_VLLM_EXTRA_ARGS"] = cfg["extra_args"] + if cfg.get("hf_overrides"): + env["EVAL_VLLM_HF_OVERRIDES"] = cfg["hf_overrides"] + + return env + + +def get_conda_env_override(hf_model: str, configs: Dict[str, Dict]) -> Optional[str]: + """Get conda_env override for a model from the baseline config mapping. + + Tries exact/group match first, then pattern match. Returns the conda_env + string (e.g. "otagent2") or None if no override is configured. + """ + cfg = configs.get(hf_model) + if not cfg: + cfg = _match_pattern_config(hf_model) + if cfg and cfg.get("conda_env"): + return cfg["conda_env"] + return None + + +# ---------- v6: Disk-Based Resume Scanner ---------- + +# Infrastructure errors that harbor's resume filters will retry +INFRA_ERROR_TYPES = { + "DaytonaError", + "EnvironmentStartTimeoutError", + "DaytonaRateLimitError", + "CancelledError", + "SandboxBuildFailedError", + "AgentEnvironmentTimeoutError", +} + + +def _parse_job_dir(job_dir: Path) -> Optional[Dict]: + """Parse a harbor job directory, extracting model/dataset/progress info. + + Returns dict with keys: run_tag, hf_model, dataset, n_completed, n_total, + finished_at, infra_errors, total_errors, db_job_id, slurm_job_id, resume_count. + Returns None if dir is not a valid harbor job dir. + """ + config_path = job_dir / "config.json" + if not config_path.exists(): + return None + + run_tag = job_dir.name + info: Dict = { + "run_tag": run_tag, + "hf_model": None, + "dataset": None, + "n_completed": 0, + "n_total": 0, + "finished_at": None, + "infra_errors": 0, + "total_errors": 0, + "db_job_id": None, + "slurm_job_id": None, + "resume_count": 0, + } + + # Parse config.json for model and dataset + try: + import json as _json + config = _json.loads(config_path.read_text()) + agents = config.get("agents", []) + if agents and isinstance(agents, list): + model_name = agents[0].get("model_name", "") + # Strip "hosted_vllm/" prefix + if model_name.startswith("hosted_vllm/"): + model_name = model_name[len("hosted_vllm/"):] + info["hf_model"] = model_name or None + datasets = config.get("datasets", []) + if datasets and isinstance(datasets, list): + ds_path = datasets[0].get("path", "") + if ds_path: + # Extract dataset name from path: /e/.../DCAgent_dev_set_v2 → DCAgent/dev_set_v2 + ds_name = Path(ds_path).name # e.g., "DCAgent_dev_set_v2" + # Convert first underscore to slash (org/name convention) + parts = ds_name.split("_", 1) + if len(parts) == 2: + info["dataset"] = f"{parts[0]}/{parts[1]}" + else: + info["dataset"] = ds_name + except Exception: + pass + + # Parse meta.env for DB_JOB_ID, SLURM_JOB_ID, RESUME_COUNT + meta_path = job_dir / "meta.env" + if meta_path.exists(): + try: + for line in meta_path.read_text().splitlines(): + if line.startswith("DB_JOB_ID="): + info["db_job_id"] = line.split("=", 1)[1].strip() or None + elif line.startswith("SLURM_JOB_ID="): + info["slurm_job_id"] = line.split("=", 1)[1].strip() or None + elif line.startswith("RESUME_COUNT="): + try: + info["resume_count"] = int(line.split("=", 1)[1].strip()) + except ValueError: + pass + elif line.startswith("MODEL=") and not info["hf_model"]: + info["hf_model"] = line.split("=", 1)[1].strip() or None + except Exception: + pass + + # Parse result.json for progress + result_path = job_dir / "result.json" + if result_path.exists(): + try: + import json as _json + result = _json.loads(result_path.read_text()) + info["n_total"] = result.get("n_total_trials", 0) + stats = result.get("stats", {}) + info["n_completed"] = stats.get("n_trials", 0) + info["finished_at"] = result.get("finished_at") + + # Count infrastructure errors + infra_count = 0 + total_err_count = 0 + for eval_data in stats.get("evals", {}).values(): + for exc_type, ids in eval_data.get("exception_stats", {}).items(): + n = len(ids) if isinstance(ids, list) else 1 + total_err_count += n + if exc_type in INFRA_ERROR_TYPES: + infra_count += n + info["infra_errors"] = infra_count + info["total_errors"] = total_err_count + except Exception: + pass + + return info + + +def scan_jobs_dir_for_resume( + jobs_dir: str, + dataset_prefixes: List[str], + active_slurm_ids: Set[str], + infra_error_threshold: int = 3, + max_resume_count: int = 5, +) -> List[Dict]: + """Scan eval jobs directory for jobs that need to be resumed. + + Args: + jobs_dir: Path to the eval jobs directory + dataset_prefixes: List of dataset name prefixes to filter (e.g., ["dev_set_v2"]) + active_slurm_ids: Set of SLURM job IDs currently in squeue + infra_error_threshold: Min infra errors to trigger resume for PARTIAL jobs + max_resume_count: Skip dirs with RESUME_COUNT >= this (prevent infinite loops) + + Returns: + List of dicts with keys: hf_model, dataset, run_tag, reason, db_job_id + """ + jobs_path = Path(jobs_dir) + if not jobs_path.is_dir(): + log(f"[v6-resume] Jobs dir not found: {jobs_dir}") + return [] + + # Build prefix patterns from dataset names + # "DCAgent/dev_set_v2" → "dev_set_v2_" + # Must normalize hyphens/dots to underscores to match generate_run_tag() output + dir_prefixes = [] + for ds in dataset_prefixes: + # Dataset format: "DCAgent/dev_set_v2" or "DCAgent2/terminal_bench_2" + ds_short = ds.split("/")[-1] if "/" in ds else ds + ds_safe = ds_short.replace("-", "_").replace(".", "_") + dir_prefixes.append(f"{ds_safe}_") + + candidates = [] + scanned = 0 + skipped_active = 0 + skipped_done = 0 + skipped_resume_limit = 0 + + for entry in sorted(jobs_path.iterdir()): + if not entry.is_dir(): + continue + + # Filter by dataset prefix + if not any(entry.name.startswith(p) for p in dir_prefixes): + continue + + info = _parse_job_dir(entry) + if info is None: + continue + scanned += 1 + + # Skip if SLURM job still running + if info["slurm_job_id"] and info["slurm_job_id"] in active_slurm_ids: + skipped_active += 1 + continue + + # Skip if resume count too high + if info["resume_count"] >= max_resume_count: + skipped_resume_limit += 1 + continue + + # Classify job state + n_completed = info["n_completed"] + n_total = info["n_total"] + finished_at = info["finished_at"] + infra_errors = info["infra_errors"] + + reason = None + + if n_total == 0 and not (jobs_path / entry.name / "result.json").exists(): + # EARLY_KILL: killed before any trial completed + reason = f"early_kill (no result.json, resume #{info['resume_count']+1})" + elif n_completed < n_total and finished_at is None: + # INCOMPLETE: SLURM killed mid-run + reason = f"incomplete ({n_completed}/{n_total} trials, resume #{info['resume_count']+1})" + elif n_completed < n_total and finished_at is not None: + # PARTIAL: harbor finished but some trials failed + if infra_errors > infra_error_threshold: + reason = f"partial ({n_completed}/{n_total}, {infra_errors} infra errors, resume #{info['resume_count']+1})" + elif n_completed == n_total: + # DONE: all trials completed + if infra_errors > infra_error_threshold: + reason = f"done_with_errors ({n_completed}/{n_total}, {infra_errors} infra errors, resume #{info['resume_count']+1})" + else: + skipped_done += 1 + continue + else: + continue + + if reason and info["hf_model"]: + candidates.append({ + "hf_model": info["hf_model"], + "dataset": info["dataset"], + "run_tag": info["run_tag"], + "reason": f"v6_resume: {reason}", + "db_job_id": info["db_job_id"], + }) + + log(f"[v6-resume] Scanned {scanned} job dirs: " + f"{len(candidates)} resume candidates, " + f"{skipped_active} still running, " + f"{skipped_done} completed, " + f"{skipped_resume_limit} at resume limit") + + return candidates + + +# ---------- Preset Definitions ---------- +# Each preset can configure: +# - datasets: list of HF dataset repos +# - sbatch_script: sbatch script to use (default: unified_eval_harbor_v4.sbatch) +# - log_suffix: suffix for log file +# - check_hf_exists: validate model exists on HuggingFace +# - n_concurrent: Harbor --n-concurrent (default: 64) +# - n_attempts: Harbor --n-attempts (default: 3) +# - gpu_memory_util: VLLM --gpu-memory-utilization (default: 0.9) +# - error_threshold: Max invalid errors before abort (default: 3) +# - vllm_max_retries: VLLM startup retries (default: 5) +# - agent_parser: Agent parser type (default: "", use "xml" for swebench) +# - slurm_time: SLURM time limit (default: "24:00:00") +PRESETS: Dict[str, Dict] = { + "aider": { + "datasets": ["DCAgent2/aider_polyglot"], + "log_suffix": "aider", + "n_concurrent": 32, + "error_threshold": 10, + "enable_thinking": True, + }, + "bfcl": { + "datasets": ["DCAgent2/bfcl-parity"], + "log_suffix": "bfcl", + "n_concurrent": 32, + "error_threshold": 10, + "vllm_max_retries": 20, + "enable_thinking": True, + }, + # NOTE: swebench and tb2 use dcagent_eval_config_no_override.yaml (no model overrides) + "swebench": { + "datasets": ["DCAgent2/swebench-verified-random-100-folders"], + "log_suffix": "swebench", + "n_concurrent": 32, + "error_threshold": 20, + "agent_parser": "xml", + "vllm_max_retries": 10, + "enable_thinking": True, + "config_yaml": "dcagent_eval_config_no_override.yaml", + }, + "v2": { + "datasets": ["DCAgent/dev_set_v2"], + "log_suffix": "v2", + "n_concurrent": 32, + "error_threshold": 10, + "vllm_max_retries": 10, + "enable_thinking": True, + "config_yaml": "dcagent_eval_config_no_override.yaml", + "auto_snapshot": True, + }, + "tb2": { + "datasets": ["DCAgent2/terminal_bench_2"], + "log_suffix": "tb2", + "n_concurrent": 32, + "error_threshold": 10, + "enable_thinking": True, + "vllm_max_retries": 10, + "config_yaml": "dcagent_eval_config_no_override.yaml", + "auto_snapshot": True, + }, + "v1": { + "datasets": ["DCAgent/dev_set_71_tasks"], + "log_suffix": "v1", + "n_concurrent": 32, + "error_threshold": 10, + "vllm_max_retries": 10, + "enable_thinking": True, + }, +} + +# ---------- Cluster Config ---------- +_CLUSTER_CONFIG_REQUIRED_KEYS = ["cluster_name", "slurm_partition", "paths"] +_CLUSTER_CONFIG_REQUIRED_PATHS = ["eval_jobs_dir", "sbatch_script"] + +# Global cluster config (set by --cluster-config, None = use hardcoded defaults) +_CLUSTER_CONFIG: Optional[Dict[str, Any]] = None + + +def load_cluster_config(path: str) -> Dict[str, Any]: + """Load and validate a cluster config YAML. + + Returns the parsed config dict. Raises SystemExit on validation failure. + """ + path = os.path.expanduser(path) + if not os.path.isfile(path): + print(f"ERROR: Cluster config not found: {path}") + sys.exit(2) + + with open(path) as f: + cfg = yaml.safe_load(f) + + if not isinstance(cfg, dict): + print(f"ERROR: Cluster config must be a YAML mapping, got {type(cfg).__name__}") + sys.exit(2) + + # Expand $USER / ${USER} and ~ in all string values (paths, conda env dirs, etc.) + def _expand(obj): + if isinstance(obj, str): + return os.path.expandvars(os.path.expanduser(obj)) + elif isinstance(obj, dict): + return {k: _expand(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [_expand(v) for v in obj] + return obj + cfg = _expand(cfg) + + for key in _CLUSTER_CONFIG_REQUIRED_KEYS: + if key not in cfg: + print(f"ERROR: Cluster config missing required key: {key}") + sys.exit(2) + + paths = cfg.get("paths", {}) + for key in _CLUSTER_CONFIG_REQUIRED_PATHS: + if key not in paths: + print(f"ERROR: Cluster config paths.{key} is required") + sys.exit(2) + + return cfg + + +def _cc_get(key: str, default: Any = None) -> Any: + """Get a top-level key from the cluster config, or *default* if not loaded.""" + if _CLUSTER_CONFIG is None: + return default + return _CLUSTER_CONFIG.get(key, default) + + +def _cc_path(key: str, default: Any = None) -> Any: + """Get a paths.* key from the cluster config, or *default*.""" + if _CLUSTER_CONFIG is None: + return default + return _CLUSTER_CONFIG.get("paths", {}).get(key, default) + + +# ---------- Constants ---------- +HF_URL_RE = re.compile(r'https?://(?:www\.)?huggingface\.co/([^/\s]+)/([^/\s#?]+)') +JOB_STATUS_PENDING = "Pending" +JOB_STATUS_STARTED = "Started" +JOB_STATUS_FINISHED = "Finished" +JOB_STATUS_FAILED = "Failed" +DEFAULT_STALE_JOB_HOURS = 24 +DEFAULT_STALE_PENDING_HOURS = 48 +DEFAULT_LOOKBACK_DAYS = 1000 +DEFAULT_CHECK_HOURS = 4.0 +DEFAULT_LOG_DIR = "experiments/listener_logs" + +# Sbatch parameter defaults +DEFAULT_N_CONCURRENT = 64 +DEFAULT_N_ATTEMPTS = 3 +DEFAULT_GPU_MEMORY_UTIL = 0.9 +DEFAULT_ERROR_THRESHOLD = 10 +DEFAULT_VLLM_MAX_RETRIES = 5 +DEFAULT_AGENT_PARSER = "" +DEFAULT_SLURM_TIME = "12:00:00" +DEFAULT_AGENT_NAME = "terminus-2" +DEFAULT_SLURM_PARTITION = "booster" +DEFAULT_SLURM_ACCOUNT = "" # empty = use sbatch header default +DEFAULT_ENABLE_THINKING = False +DEFAULT_TP_SIZE = 1 +DEFAULT_SBATCH_SCRIPT = "eval/unified_eval_harbor.sbatch" + +# Fallback defaults (used when no --cluster-config is provided). +# Empty strings force explicit cluster config — no implicit Jupiter defaults. +_FALLBACK_EVAL_JOBS_DIR = "" +_FALLBACK_HF_CACHE = "" +_FALLBACK_EVAL_LOGS_DIR = "eval/logs" + +# Conda env paths: env name → prefix directory (passed as OTAGENT_DIR to sbatch) +# Overridden by cluster_config["conda_envs"] when --cluster-config is used. +CONDA_ENV_PATHS: Dict[str, str] = {} + +# Dataset repo name → short benchmark tag for SLURM job names (squeue readability) +_BENCH_SHORT: Dict[str, str] = { + "dev_set_v2": "v2", + "swebench-verified-random-100-folders": "swe", + "terminal_bench_2": "tb2", + "aider_polyglot": "aider", + "bfcl-parity": "bfcl", + "dev_set_71_tasks": "v1", +} + +# Enhancement 2: SLURM job submission throttle +DEFAULT_MAX_JOBS_SUBMITTED = 20 + +# Enhancement 3: Daytona resource pre-flight check +DEFAULT_DAYTONA_SANDBOX_LIMIT = 2000 +DEFAULT_DAYTONA_WARNING_BUFFER = 0.9 + +# Enhancement 5: Timeout-config-sensitive dedup +DEFAULT_TIMEOUT_MULTIPLIER = 1.0 + + +# ---------- Configuration ---------- +@dataclass +class ListenerConfig: + """Configuration for the eval listener. + + Core fields: + datasets HF dataset repos to evaluate against. + sbatch_script Path to the sbatch script to submit. + priority_models Ordered list of HF model names from the priority file. + File order = submission priority (first = highest). + priority_file Path to the priority file (hot-reloaded each iteration). + + Sbatch parameters (forwarded to sbatch via env vars): + n_concurrent Harbor --n-concurrent. + n_attempts Harbor --n-attempts. + gpu_memory_util VLLM --gpu-memory-utilization. + error_threshold Max invalid errors before aborting upload (v3 Enhancement 1). + Replaces v2's daytona_threshold. Env var kept as + EVAL_DAYTONA_THRESHOLD for sbatch backward compat. + agent_name Agent name for harbor and DB entries. + timeout_multiplier Harbor timeout multiplier (v3 Enhancement 5). + + v3 enhancement fields: + max_jobs_submitted Per-listener SLURM job limit (Enhancement 2). + Each listener tracks its own submitted job IDs and + only counts those still active in squeue. + check_daytona_resources Enable Daytona API pre-flight check (Enhancement 3). + daytona_sandbox_limit Max expected active sandboxes for pre-flight check. + daytona_warning_buffer Fraction of limit to trigger warning (e.g. 0.95). + timeout_aware Enable config-sensitive job dedup (Enhancement 5). + """ + datasets: List[str] + sbatch_script: str + log_file: Optional[Path] + lookback_days: int + check_interval_hours: float + stale_job_hours: int + stale_pending_hours: int + priority_file: Optional[str] + require_priority_list: bool + priority_models: List[str] + check_hf_exists: bool + dry_run: bool + run_once: bool + verbose: bool + # Priority mode: "filter_only" (skip non-priority) or "priority_first" (all models, priority first) + priority_mode: str = "filter_only" + # Sbatch parameters (passed to sbatch via env vars) + n_concurrent: int = DEFAULT_N_CONCURRENT + n_attempts: int = DEFAULT_N_ATTEMPTS + gpu_memory_util: float = DEFAULT_GPU_MEMORY_UTIL + error_threshold: int = DEFAULT_ERROR_THRESHOLD + vllm_max_retries: int = DEFAULT_VLLM_MAX_RETRIES + agent_parser: str = DEFAULT_AGENT_PARSER + slurm_time: str = DEFAULT_SLURM_TIME + enable_thinking: bool = DEFAULT_ENABLE_THINKING + agent_name: str = DEFAULT_AGENT_NAME + slurm_partition: str = DEFAULT_SLURM_PARTITION + slurm_account: str = DEFAULT_SLURM_ACCOUNT + tp_size: int = DEFAULT_TP_SIZE + dp_size: int = 1 # vLLM native data-parallel replicas (total GPUs = tp_size * dp_size) + upload_username: str = "" + log_prefix: str = "[unified-eval-listener-v6]" + # v3 Enhancement 2: Per-listener SLURM throttle + max_jobs_submitted: int = DEFAULT_MAX_JOBS_SUBMITTED + # v3 Enhancement 3: Daytona pre-flight + check_daytona_resources: bool = False + daytona_sandbox_limit: int = DEFAULT_DAYTONA_SANDBOX_LIMIT + daytona_warning_buffer: float = DEFAULT_DAYTONA_WARNING_BUFFER + # v3 Enhancement 5: Timeout-config-sensitive dedup + timeout_multiplier: float = DEFAULT_TIMEOUT_MULTIPLIER + timeout_aware: bool = False + # Config YAML for harbor (overrides vs no-overrides) + config_yaml: str = "dcagent_eval_config.yaml" + # Model blacklist + blacklist_file: Optional[str] = None + blacklisted_models: Set[str] = field(default_factory=set) + # Daytona auto_snapshot: None = use YAML config default, True/False = override + auto_snapshot: Optional[bool] = None + # Per-model vLLM overrides (baseline model configs) + baseline_model_configs: Optional[str] = None + # Harbor config path + harbor_config: Optional[str] = None + # Parsed eval config from harbor YAML (for config-aware dedup) + eval_config: Dict = field(default_factory=dict) + # Pre-download model weights before submitting jobs + pre_download: bool = False + # Sliding-window batch dependencies + batch_size: Optional[int] = None + # Conda env selector (otagent / otagent2) + conda_env: str = "otagent" + # v6: Disk-based resume + jobs_dirs: List[str] = field(default_factory=list) # Set from CLI or EVAL_JOBS_DIR env var + enable_disk_resume: bool = True + resume_infra_error_threshold: int = 10 + max_resume_count: int = 5 + force_reeval: bool = False # Bypass DB status check (submit even if Finished/Started) + resume_only: bool = False # Only submit resume jobs, skip fresh submissions + submission_delay: float = 1.0 # Seconds to sleep between sbatch submissions + stagger_delay: int = 0 # Minutes between job starts via SLURM after: dependency chain (0 = disabled) + chain_batch_size: int = 1 # Jobs per stagger batch (1 = every job waits, 10 = fire 10 then wait) + pack_jobs: bool = False # Pack multiple jobs onto same node via --nodelist + # DP: data-parallel multi-node eval + dp_nodes: int = 0 # 0 = single-node (default), >0 = use DP sbatch with N nodes + dp_sbatch_script: str = "eval/unified_eval_harbor_dp.sbatch" + # Inherit: seed _submitted_jobs from previous listener logs + inherit_log: Optional[List[str]] = None + # Cluster config (loaded from --cluster-config YAML) + cluster_config: Optional[Dict[str, Any]] = None + + @property + def check_interval_seconds(self) -> int: + return int(self.check_interval_hours * 60 * 60) + + +# ---------- Logging ---------- +_LOG_FILE: Optional[Path] = None +_VERBOSE: bool = False + + +def set_log_file(path: Optional[Path]) -> None: + global _LOG_FILE + _LOG_FILE = path + + +def log(msg: str, prefix: str = "[unified-eval-listener-v6]", verbose_only: bool = False) -> None: + """Log a message to stdout and optionally to file. + + If verbose_only=True, the message is only emitted when _VERBOSE is set. + """ + if verbose_only and not _VERBOSE: + return + ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + line = f"{prefix} {ts} {msg}" + print(line, flush=True) + if _LOG_FILE: + try: + with _LOG_FILE.open("a") as f: + f.write(line + "\n") + except Exception: + pass + + +# ---------- Priority Models Loading ---------- +def load_priority_models(filepath: Optional[str]) -> List[str]: + """ + Load priority models from a text file, preserving file order as rank. + + File order determines submission priority: models listed earlier are + submitted first. When the per-listener SLURM job limit truncates the + submission list, higher-priority (earlier) models are kept. + + File format: + - One model per line (HuggingFace format: org/model) + - Lines starting with # are comments + - Blank lines are ignored + + Returns: + Ordered list of model names (duplicates removed, order preserved). + Empty list if file is missing or empty. + """ + if not filepath: + return [] + + path = Path(filepath) + if not path.exists(): + log(f"Priority file not found: {filepath}") + return [] + + models: List[str] = [] + seen: Set[str] = set() + try: + with path.open("r") as f: + for line in f: + line = line.strip() + # Skip empty lines and comments + if not line or line.startswith("#"): + continue + if line not in seen: + seen.add(line) + models.append(line) + log(f"Loaded {len(models)} model(s) from priority file: {filepath}") + return models + except Exception as e: + log(f"ERROR reading priority file {filepath}: {e}") + return [] + + +# ---------- Model Blacklist Loading ---------- +def load_blacklist(filepath: Optional[str]) -> Set[str]: + """Load blacklisted models from a text file. Same format as priority file.""" + return set(load_priority_models(filepath)) + + +# ---------- HuggingFace Utilities ---------- +def check_hf_model_exists(model_name: str) -> bool: + """ + Check if a model exists on HuggingFace Hub. + + Args: + model_name: HF model name (e.g., "org/model-name") + + Returns: + True if model exists and is accessible, False otherwise + """ + if not model_name or not isinstance(model_name, str): + return False + + try: + from huggingface_hub import model_info + model_info(model_name) + return True + except Exception as e: + log(f"HF check failed for {model_name}: {e}") + return False + + +def _parse_hf_from_str(val: Optional[str]) -> Optional[str]: + """Parse HuggingFace model name from a string (URL or org/repo).""" + if not isinstance(val, str): + return None + m = HF_URL_RE.search(val) + if m: + return f"{m.group(1)}/{m.group(2)}" + return None + + +def resolve_hf_model_name(model_row: Dict) -> Optional[str]: + """ + Resolve HF model name from a database model row. + + Checks multiple fields in order of priority. + """ + # Check name field first + v = model_row.get("name") + if isinstance(v, str) and "/" in v and not v.startswith("hosted_vllm/"): + return v + + # Check other URL fields + for field in ("weights_location", "training_parameters", "url", "hf_url"): + vv = model_row.get(field) + if isinstance(vv, str): + name = _parse_hf_from_str(vv) + if name: + return name + + # Check training_parameters as JSON + vv = model_row.get("training_parameters") + if isinstance(vv, str): + try: + obj = json.loads(vv) + except Exception: + obj = None + else: + obj = vv + + if isinstance(obj, dict): + for sval in obj.values(): + if isinstance(sval, str): + name = _parse_hf_from_str(sval) + if name: + return name + + return None + + +# ---------- Dataset Parsing ---------- +def parse_datasets(s: str) -> List[str]: + """ + Parse dataset list from string. + + Supports comma, space, or newline separated values. + Normalizes HF URLs to org/repo format. + """ + parts = [p.strip() for p in re.split(r"[,\s]+", s) if p.strip()] + out = [] + for p in parts: + m = HF_URL_RE.search(p) + out.append(f"{m.group(1)}/{m.group(2)}" if m else p) + + # Dedup while preserving order + seen: Set[str] = set() + uniq: List[str] = [] + for d in out: + if d not in seen: + seen.add(d) + uniq.append(d) + return uniq + + +def dataset_repo_name(dataset_hf: str) -> str: + """Convert 'org/repo' or HF URL to 'repo' (just the repo name).""" + if not dataset_hf: + return dataset_hf + m = HF_URL_RE.search(dataset_hf) + if m: + return m.group(2) + if "/" in dataset_hf: + return dataset_hf.rsplit("/", 1)[-1] + return dataset_hf + + +# ---------- Database Operations ---------- +_BENCH_CACHE: Dict[str, Optional[str]] = {} + + +def _iso(dt: datetime) -> str: + """Convert datetime to ISO format string.""" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat() + + +def _time_filters(q, since_iso: str): + """Apply time filter to Supabase query (handles both column names).""" + try: + return q.gte('creation_time', since_iso) + except Exception: + return q.gte('created_at', since_iso) + + +def fetch_recent_models(days: int) -> List[Dict]: + """Fetch recent models from Supabase within the lookback window. + + Filters out: + - Models with created_by == "precomputed_hf" + - Models with a non-empty "duplicate_of" field (v3: prevents duplicate + eval submissions when the same HF model appears under multiple DB rows) + """ + client = get_supabase_client() + since = _iso(datetime.now(timezone.utc) - timedelta(days=days)) + try: + resp = _time_filters(client.table('models').select('*'), since).execute() + rows = list(resp.data or []) + except Exception as e: + log(f"ERROR: failed querying models by time: {e}") + return [] + + # Filter out precomputed models and duplicates + out: List[Dict] = [] + skipped_dupes = 0 + for r in rows: + if r.get("created_by") == "precomputed_hf": + continue + if r.get("duplicate_of"): + skipped_dupes += 1 + continue + out.append(r) + if skipped_dupes: + log(f"Filtered out {skipped_dupes} duplicate model(s) (duplicate_of set)") + return out + + +def fetch_priority_models(priority_names: List[str]) -> List[Dict]: + """Fetch models by name from Supabase, bypassing the lookback window. + + This ensures priority models are always evaluated even if they were + registered long ago (outside the lookback window). + + Filters out: + - Models with created_by == "precomputed_hf" + - Models with a non-empty "duplicate_of" field + """ + if not priority_names: + return [] + + client = get_supabase_client() + try: + resp = ( + client.table('models') + .select('*') + .in_('name', priority_names) + .execute() + ) + rows = list(resp.data or []) + except Exception as e: + log(f"ERROR: failed querying priority models by name: {e}") + return [] + + out: List[Dict] = [] + for r in rows: + if r.get("created_by") == "precomputed_hf": + continue + if r.get("duplicate_of"): + continue + out.append(r) + return out + + +def resolve_benchmark_id(dataset_hf: str) -> Optional[str]: + """ + Look up benchmark ID from database for a given dataset. + + Caches results for performance. + """ + repo_name = dataset_repo_name(dataset_hf) + if repo_name in _BENCH_CACHE: + return _BENCH_CACHE[repo_name] + + try: + client = get_supabase_client() + resp = ( + client.table('benchmarks') + .select('id,name') + .eq('name', repo_name) + .limit(1) + .execute() + ) + rows = resp.data or [] + bench_id = rows[0]['id'] if rows else None + _BENCH_CACHE[repo_name] = bench_id + if not bench_id: + log(f"No benchmark row found for dataset '{dataset_hf}' (wanted name='{repo_name}').") + return bench_id + except Exception as e: + log(f"ERROR resolving benchmark id for dataset '{dataset_hf}': {e}") + return None + + +def check_job_status( + model_id: str, benchmark_id: Optional[str] +) -> Tuple[bool, Optional[str], Optional[datetime], Optional[datetime], Optional[str]]: + """Check if a job exists for (model_id, benchmark_id) and its status. + + Delegates to check_job_status_v3 (single-ID, non-timeout-aware path). + Kept as a thin wrapper for backward compatibility with callers that don't + need timeout-aware or duplicate-group queries. + """ + return check_job_status_v3(model_id, benchmark_id) + + +# ---------- Cross-Duplicate Aggregation ---------- +_DUP_GROUP_CACHE: Dict[str, List[str]] = {} + + +def get_duplicate_group_ids(table: str, entity_id: str) -> List[str]: + """Get all IDs in the duplicate group for a model or benchmark. + + Given an entity_id, finds the canonical ID and all its duplicates. + - If entity has duplicate_of set, canonical = duplicate_of + - Otherwise canonical = entity_id + - Then finds all rows WHERE duplicate_of = canonical_id + - Returns [canonical_id] + [all duplicate IDs] + + Results are cached per (table, entity_id). + """ + cache_key = f"{table}:{entity_id}" + if cache_key in _DUP_GROUP_CACHE: + return _DUP_GROUP_CACHE[cache_key] + + try: + client = get_supabase_client() + + # Step 1: Find the canonical ID + resp = client.table(table).select('id,duplicate_of').eq('id', entity_id).limit(1).execute() + rows = resp.data or [] + if not rows: + _DUP_GROUP_CACHE[cache_key] = [entity_id] + return [entity_id] + + canonical_id = rows[0].get('duplicate_of') or entity_id + + # Step 2: Find all duplicates of the canonical + resp2 = client.table(table).select('id').eq('duplicate_of', canonical_id).execute() + dup_ids = [r['id'] for r in (resp2.data or [])] + + group = list(set([canonical_id] + dup_ids)) + # Cache for all members of the group + for gid in group: + _DUP_GROUP_CACHE[f"{table}:{gid}"] = group + return group + + except Exception as e: + log(f"WARNING: Failed to get duplicate group for {table}/{entity_id}: {e}") + _DUP_GROUP_CACHE[cache_key] = [entity_id] + return [entity_id] + + +# ---------- v3 Enhancement 5: Timeout-Config-Sensitive Job Dedup ---------- +def check_job_status_v3( + model_id: str, + benchmark_id: Optional[str], + timeout_aware: bool = False, + agent_name: str = DEFAULT_AGENT_NAME, + timeout_multiplier: float = DEFAULT_TIMEOUT_MULTIPLIER, + duplicate_model_ids: Optional[List[str]] = None, + duplicate_benchmark_ids: Optional[List[str]] = None, +) -> Tuple[bool, Optional[str], Optional[datetime], Optional[datetime], Optional[str]]: + """ + Check if a job exists for (model_id, benchmark_id) and its status. + + When timeout_aware=True, filters to only match jobs with the same + agent_name and timeout_multiplier in their config. + + When duplicate_model_ids/duplicate_benchmark_ids are provided, queries + across the entire duplicate group using .in_() instead of .eq(). + + Returns: + (job_exists, job_status, started_at, submitted_at, slurm_job_id) + """ + if not benchmark_id: + return (False, None, None, None, None) + + # Determine which IDs to query + model_ids = duplicate_model_ids if duplicate_model_ids else [model_id] + bench_ids = duplicate_benchmark_ids if duplicate_benchmark_ids else [benchmark_id] + + try: + client = get_supabase_client() + q = client.table('sandbox_jobs').select( + 'id,job_status,started_at,submitted_at,slurm_job_id,config' + ) + + # Use .in_() for duplicate groups, .eq() for singles + if len(model_ids) == 1: + q = q.eq('model_id', model_ids[0]) + else: + q = q.in_('model_id', model_ids) + + if len(bench_ids) == 1: + q = q.eq('benchmark_id', bench_ids[0]) + else: + q = q.in_('benchmark_id', bench_ids) + + q = q.order('created_at', desc=True).limit(50) + data = (q.execute().data) or [] + + if not data: + return (False, None, None, None, None) + + # Filter to matching config if timeout_aware + for job in data: + if timeout_aware: + config = job.get('config') + if isinstance(config, str): + try: + config = json.loads(config) + except Exception: + config = {} + if not isinstance(config, dict): + config = {} + + job_agent = config.get('agent', DEFAULT_AGENT_NAME) + job_tm = config.get('timeout_multiplier', DEFAULT_TIMEOUT_MULTIPLIER) + + # Skip if agent_name or timeout_multiplier don't match + if job_agent != agent_name or float(job_tm) != float(timeout_multiplier): + continue + + job_status = job.get('job_status') + started_at_str = job.get('started_at') + submitted_at_str = job.get('submitted_at') + slurm_job_id = job.get('slurm_job_id') + + started_at = None + if started_at_str: + try: + started_at = datetime.fromisoformat(started_at_str.replace('Z', '+00:00')) + except Exception: + pass + + submitted_at = None + if submitted_at_str: + try: + submitted_at = datetime.fromisoformat(submitted_at_str.replace('Z', '+00:00')) + except Exception: + pass + + return (True, job_status, started_at, submitted_at, slurm_job_id) + + # No matching job found + return (False, None, None, None, None) + + except Exception as e: + log(f"WARNING: sandbox_jobs v3 check failed for model_id={model_id}, benchmark_id={benchmark_id}: {e}") + return (False, None, None, None, None) # fail-open + + +def is_job_stale(started_at: Optional[datetime], hours: int = DEFAULT_STALE_JOB_HOURS) -> bool: + """Check if a job started more than the specified hours ago.""" + if not started_at: + # If started_at is null but job exists with status='Started', treat as stale + return True + now = datetime.now(timezone.utc) + if started_at.tzinfo is None: + started_at = started_at.replace(tzinfo=timezone.utc) + age = now - started_at + return age > timedelta(hours=hours) + + +def _config_matches_eval(job_config: Optional[Dict], eval_config: Dict) -> bool: + """Check if a DB job's config JSONB matches the current eval config fields. + + Compares: timeout_multiplier, override_cpus, override_memory_mb, override_storage_mb. + A job with no config is treated as defaults (timeout=1.0, no overrides). + If eval_config is empty (no harbor config), any job config matches (backwards compat). + """ + if not eval_config: + return True # no config constraints -- any existing job counts + + job_cfg = job_config or {} + job_env = job_cfg.get("environment") or {} + + # timeout_multiplier: top-level in config JSONB + if "timeout_multiplier" in eval_config: + job_tm = job_cfg.get("timeout_multiplier") + # Treat None/missing as 1.0 + job_tm = float(job_tm) if job_tm is not None else 1.0 + if float(eval_config["timeout_multiplier"]) != job_tm: + return False + + # Environment overrides: nested under config.environment + for key in ("override_cpus", "override_memory_mb", "override_storage_mb"): + if key in eval_config: + job_val = job_env.get(key) + # Treat None/missing as the default (None means no override) + job_val = int(job_val) if job_val is not None else None + eval_val = int(eval_config[key]) + if job_val != eval_val: + return False + + return True + + +def should_start_job( + model_id: str, + benchmark_id: Optional[str], + stale_hours: int = DEFAULT_STALE_JOB_HOURS, + stale_pending_hours: int = DEFAULT_STALE_PENDING_HOURS, + timeout_aware: bool = False, + agent_name: str = DEFAULT_AGENT_NAME, + timeout_multiplier: float = DEFAULT_TIMEOUT_MULTIPLIER, + duplicate_model_ids: Optional[List[str]] = None, + duplicate_benchmark_ids: Optional[List[str]] = None, + eval_config: Optional[Dict] = None, +) -> Tuple[bool, str, Optional[str]]: + """ + Determine if a job should be started based on DB status. + + When timeout_aware=True (v3 Enhancement 5), uses check_job_status_v3() + which filters jobs by agent_name and timeout_multiplier in config. This + allows running the same model with different configs without one blocking + the other. + + When duplicate_model_ids/duplicate_benchmark_ids are provided, checks + across the entire duplicate group for existing jobs. + + When eval_config is provided (from harbor YAML), performs config-aware + dedup: checks that existing jobs match the current resource overrides + (timeout_multiplier, override_cpus, override_memory_mb, override_storage_mb). + + Returns: + (should_start, reason, slurm_job_id) + slurm_job_id is provided so the caller can scancel stale jobs. + """ + job_exists, job_status, started_at, submitted_at, slurm_job_id = check_job_status_v3( + model_id, benchmark_id, + timeout_aware=timeout_aware, + agent_name=agent_name, + timeout_multiplier=timeout_multiplier, + duplicate_model_ids=duplicate_model_ids, + duplicate_benchmark_ids=duplicate_benchmark_ids, + ) + + if not job_exists: + return (True, "no existing job", None) + + ec = eval_config or {} + + if job_status == JOB_STATUS_FINISHED: + if ec: + # Check if any Finished job matches our eval config + try: + client = get_supabase_client() + q = ( + client.table("sandbox_jobs") + .select("config, metrics") + .eq("model_id", model_id) + .eq("benchmark_id", benchmark_id) + .eq("job_status", "Finished") + .order("created_at", desc=True) + .limit(10) + ) + rows = (q.execute().data) or [] + matching = [r for r in rows if _config_matches_eval(r.get("config"), ec)] + if not matching: + return (True, "no finished job with matching config", slurm_job_id) + if matching[0] and not matching[0].get("metrics"): + return (True, "finished with matching config but metrics cleared", slurm_job_id) + except Exception as e: + log(f"WARNING: config-aware check failed: {e}") + # v6: DaytonaError resume is now handled by disk-based scanning in + # scan_jobs_dir_for_resume(), not here. This avoids the circular dependency + # where DB stats only exist after upload, but upload is skipped on error. + + return (False, "job finished", slurm_job_id) + + if job_status == JOB_STATUS_PENDING: + if ec: + try: + client = get_supabase_client() + q = ( + client.table("sandbox_jobs") + .select("config, slurm_job_id, created_at") + .eq("model_id", model_id) + .eq("benchmark_id", benchmark_id) + .eq("job_status", "Pending") + .order("created_at", desc=True) + .limit(5) + ) + rows = (q.execute().data) or [] + matching = [r for r in rows if _config_matches_eval(r.get("config"), ec)] + if not matching: + return (True, "no pending job with matching config", slurm_job_id) + except Exception as e: + log(f"WARNING: config-aware check failed: {e}") + # Job submitted but not yet running - check if stale using separate pending threshold + if is_job_stale(submitted_at, stale_pending_hours): + submitted_str = submitted_at.isoformat() if submitted_at else "null" + return (True, f"stale pending job (submitted_at={submitted_str})", slurm_job_id) + else: + submitted_str = submitted_at.isoformat() if submitted_at else "null" + return (False, f"job pending in SLURM queue (submitted_at={submitted_str})", slurm_job_id) + + if job_status == JOB_STATUS_STARTED: + if ec: + try: + client = get_supabase_client() + q = ( + client.table("sandbox_jobs") + .select("config, started_at") + .eq("model_id", model_id) + .eq("benchmark_id", benchmark_id) + .eq("job_status", "Started") + .order("created_at", desc=True) + .limit(5) + ) + rows = (q.execute().data) or [] + matching = [r for r in rows if _config_matches_eval(r.get("config"), ec)] + if not matching: + return (True, "no in-progress job with matching config", slurm_job_id) + except Exception as e: + log(f"WARNING: config-aware check failed: {e}") + if is_job_stale(started_at, stale_hours): + started_str = started_at.isoformat() if started_at else "null" + return (True, f"stale job (started_at={started_str})", slurm_job_id) + else: + started_str = started_at.isoformat() if started_at else "null" + return (False, f"job in progress (started_at={started_str})", slurm_job_id) + + # Unknown status - start job to be safe + return (True, f"unknown job status: {job_status}", slurm_job_id) + + +# ---------- v3 Enhancement 2: Per-Listener SLURM Job Throttle ---------- +def get_active_slurm_job_ids() -> Set[str]: + """Return set of SLURM job IDs currently queued/running for this user. + + Used by EvalListener to determine which of its submitted jobs are still + active. The listener intersects this with its internal _submitted_jobs + set to get a per-listener active count. + """ + try: + user = getpass.getuser() + code, out = _run(["squeue", "-u", user, "--noheader", "-h", "-o", "%i"]) + if code != 0: + log(f"WARNING: squeue failed (exit {code}), returning empty set") + return set() + return {line.strip() for line in out.strip().split('\n') if line.strip()} + except Exception as e: + log(f"WARNING: Failed to query squeue: {e}") + return set() + + +def get_active_model_dataset_pairs( + log_dir: str = "eval/logs", +) -> Tuple[Set[str], Set[Tuple[str, str]], Dict[str, str]]: + """Return (active_models, active_model_dataset_pairs, active_run_tags) for all active SLURM eval jobs. + + Queries squeue for all active jobs (RUNNING/PENDING/COMPLETING), then parses + each job's eval log file to extract the model name, dataset, and run_tag. + + Args: + log_dir: Directory containing eval log files ({job_name}_{slurm_id}.out). + + Returns: + active_models: Set of HF model names currently running/queued. + active_pairs: Set of (hf_model, dataset_hf) tuples currently running/queued. + active_run_tags: Dict mapping run_tag → slurm_job_id for active jobs. + """ + active_models: Set[str] = set() + active_pairs: Set[Tuple[str, str]] = set() + active_run_tags: Dict[str, str] = {} + + try: + user = getpass.getuser() + code, out = _run(["squeue", "-u", user, "--noheader", "-o", "%i %j"]) + if code != 0: + log(f"WARNING: squeue failed (exit {code}), returning empty active sets") + return active_models, active_pairs, active_run_tags + except Exception as e: + log(f"WARNING: Failed to query squeue: {e}") + return active_models, active_pairs, active_run_tags + + log_path = Path(log_dir) + for line in out.strip().split("\n"): + line = line.strip() + if not line: + continue + parts = line.split(None, 1) + if len(parts) < 2: + continue + job_id, job_name = parts[0], parts[1] + + # Try to parse the eval log file for this job + log_file = log_path / f"{job_name}_{job_id}.out" + if not log_file.exists(): + continue + + model = dataset = run_tag = None + try: + with open(log_file, "r") as f: + for i, fline in enumerate(f): + if i > 200: # Only scan first 200 lines + break + if fline.startswith("Model: "): + model = fline.strip()[7:] + elif fline.startswith("Dataset: "): + dataset = fline.strip()[9:] + elif fline.startswith("Run tag: "): + run_tag = fline.strip()[9:] + if model and dataset and run_tag: + break + except (OSError, IOError): + continue + + if model: + active_models.add(model) + if model and dataset: + # Normalize dataset: local path → HF name (e.g. /e/.../DCAgent_dev_set_v2 → DCAgent/dev_set_v2) + ds_normalized = dataset + if "/" in dataset and not dataset.startswith("/"): + # Already HF format like DCAgent/dev_set_v2 or DCAgent2/terminal_bench_2 + ds_normalized = dataset + active_pairs.add((model, ds_normalized)) + if run_tag: + active_run_tags[run_tag] = job_id + + return active_models, active_pairs, active_run_tags + + +def _parse_job_ids_from_single_log(log_path: str) -> Set[str]: + """Parse SLURM job IDs from a single listener log file. + + Matches two patterns: + 1. "-> Submitted as SLURM job 293324 (job_name=...)" — direct submissions + 2. "[inherit-log] Inherited jobs: 293324,293325,..." — inherited from previous log + """ + job_ids: Set[str] = set() + try: + with open(log_path, "r") as f: + for line in f: + if "Submitted as SLURM job" in line: + parts = line.split("Submitted as SLURM job ") + if len(parts) >= 2: + jid = parts[1].split()[0].strip() + if jid.isdigit(): + job_ids.add(jid) + elif "[inherit-log] Inherited jobs:" in line: + # Parse comma-separated job IDs + parts = line.split("Inherited jobs:") + if len(parts) >= 2: + for jid in parts[1].strip().split(","): + jid = jid.strip() + if jid.isdigit(): + job_ids.add(jid) + except (OSError, IOError) as e: + log(f"WARNING: Failed to read log {log_path}: {e}") + return job_ids + + +def parse_submitted_jobs_from_logs(log_paths: List[str]) -> Set[str]: + """Parse SLURM job IDs from one or more listener logs. + + Aggregates across all logs, then filters to jobs still active in squeue. + """ + all_job_ids: Set[str] = set() + for lp in log_paths: + ids = _parse_job_ids_from_single_log(lp) + log(f"[inherit-log] Parsed {len(ids)} job(s) from {lp}") + all_job_ids |= ids + + active_ids = get_active_slurm_job_ids() + still_active = all_job_ids & active_ids + log(f"[inherit-log] Total: {len(all_job_ids)} job(s) across {len(log_paths)} log(s), " + f"{len(still_active)} still active in squeue") + return still_active + + +# ---------- v3 Enhancement 3: Daytona Resource Pre-flight Check ---------- +def check_daytona_resources(sandbox_limit: int, warning_buffer: float) -> bool: + """ + Check Daytona resource usage via API. + + Called at listener startup and optionally each iteration when + --check-daytona-resources is enabled. Requires DAYTONA_API_KEY in env. + + Returns True if OK to proceed, False if active sandboxes >= sandbox_limit. + Logs a warning when active sandboxes >= sandbox_limit * warning_buffer. + """ + try: + from daytona_api_client import ApiClient, Configuration, SandboxApi + except ImportError: + log("WARNING: daytona_api_client not installed, skipping resource check") + return True + + api_key = os.environ.get("DAYTONA_API_KEY") + api_url = os.environ.get("DAYTONA_API_URL", "https://app.daytona.io/api") + if not api_key: + log("WARNING: DAYTONA_API_KEY not set, skipping resource check") + return True + + try: + config = Configuration(host=api_url) + client = ApiClient(config) + client.default_headers["Authorization"] = f"Bearer {api_key}" + api = SandboxApi(client) + + result = api.list_sandboxes_paginated(states=["started"], limit=1, page=1) + active_count = result.total + + threshold = int(sandbox_limit * warning_buffer) + if active_count >= sandbox_limit: + log(f"ERROR: Daytona resources at limit: {active_count}/{sandbox_limit} active sandboxes " + f"({active_count/sandbox_limit:.1%})") + return False + elif active_count >= threshold: + log(f"WARNING: Daytona resources at {active_count}/{sandbox_limit} active sandboxes " + f"({active_count/sandbox_limit:.1%}) - approaching limit!") + return True + else: + log(f"Daytona resources OK: {active_count}/{sandbox_limit} active sandboxes " + f"({active_count/sandbox_limit:.1%})") + return True + except Exception as e: + log(f"WARNING: Daytona resource check failed: {e}") + return True # fail-open + + +# ---------- Job Submission ---------- +@dataclass +class SbatchParams: + """Parameters passed to the sbatch script via environment variables. + + The listener converts these to EVAL_* env vars via to_env(), which the + sbatch script reads at startup. + + v3 additions: + error_threshold Mapped to EVAL_DAYTONA_THRESHOLD (name kept for compat). + Controls the unified invalid error threshold. + timeout_multiplier Mapped to EVAL_TIMEOUT_MULTIPLIER. Passed to harbor + --timeout-multiplier and stored in DB job config. + + Cluster config additions (v6): + When a cluster config YAML is loaded (--cluster-config), to_env() also + exports EVAL_PROJECT_ROOT, EVAL_HF_CACHE, EVAL_HARBOR_SRC, + EVAL_DATASETS_DIRS, EVAL_PROXY_ENABLED, EVAL_LOGIN_NODE, + EVAL_PROXYCHAINS_BIN, EVAL_CUDA_HOME, EVAL_ARCH, EVAL_GPUS_PER_NODE, + and EVAL_LOGS_DIR so sbatch scripts can be cluster-agnostic. + """ + n_concurrent: int = DEFAULT_N_CONCURRENT + n_attempts: int = DEFAULT_N_ATTEMPTS + gpu_memory_util: float = DEFAULT_GPU_MEMORY_UTIL + error_threshold: int = DEFAULT_ERROR_THRESHOLD + vllm_max_retries: int = DEFAULT_VLLM_MAX_RETRIES + agent_parser: str = DEFAULT_AGENT_PARSER + slurm_time: str = DEFAULT_SLURM_TIME + enable_thinking: bool = DEFAULT_ENABLE_THINKING + agent_name: str = DEFAULT_AGENT_NAME + slurm_partition: str = DEFAULT_SLURM_PARTITION + slurm_account: str = DEFAULT_SLURM_ACCOUNT + tp_size: int = DEFAULT_TP_SIZE + dp_size: int = 1 # vLLM native data-parallel replicas (total GPUs = tp_size * dp_size) + upload_username: str = "" + timeout_multiplier: float = DEFAULT_TIMEOUT_MULTIPLIER # v3 Enhancement 5 + config_yaml: str = "dcagent_eval_config.yaml" + auto_snapshot: Optional[bool] = None # None = use YAML default + + def to_env(self) -> Dict[str, str]: + """Convert to environment variables for sbatch.""" + env = { + "EVAL_N_CONCURRENT": str(self.n_concurrent), + "EVAL_N_ATTEMPTS": str(self.n_attempts), + "EVAL_GPU_MEMORY_UTIL": str(self.gpu_memory_util), + "EVAL_DAYTONA_THRESHOLD": str(self.error_threshold), + "EVAL_VLLM_MAX_RETRIES": str(self.vllm_max_retries), + "EVAL_AGENT_PARSER": self.agent_parser, + "EVAL_SLURM_TIME": self.slurm_time, + "EVAL_ENABLE_THINKING": "true" if self.enable_thinking else "false", + "EVAL_AGENT_NAME": self.agent_name, + } + # Always send tp_size so build_vllm_cmd.sh doesn't fall back to its own default + env["EVAL_VLLM_TENSOR_PARALLEL_SIZE"] = str(self.tp_size) + if self.dp_size > 1: + env["EVAL_VLLM_DATA_PARALLEL_SIZE"] = str(self.dp_size) + if self.upload_username: + env["EVAL_UPLOAD_USERNAME"] = self.upload_username + # Enhancement 5: Pass timeout multiplier + if self.timeout_multiplier != DEFAULT_TIMEOUT_MULTIPLIER: + env["EVAL_TIMEOUT_MULTIPLIER"] = str(self.timeout_multiplier) + # Pass config YAML (no-override for tb2/swebench) + if self.config_yaml != "dcagent_eval_config.yaml": + env["EVAL_CONFIG_YAML"] = self.config_yaml + # Daytona auto_snapshot override (None = use YAML default) + if self.auto_snapshot is not None: + env["EVAL_AUTO_SNAPSHOT"] = "true" if self.auto_snapshot else "false" + # Forward EVAL_JOBS_DIR to sbatch (default to user-writable location) + fallback_jobs_dir = _cc_path("eval_jobs_dir", _FALLBACK_EVAL_JOBS_DIR) + env["EVAL_JOBS_DIR"] = os.environ.get("EVAL_JOBS_DIR", fallback_jobs_dir) + + # --- Cluster config env vars (for sbatch parameterization) --- + cc = _CLUSTER_CONFIG + if cc: + paths = cc.get("paths", {}) + proxy = cc.get("proxy", {}) + hw = cc.get("hardware", {}) + + if cc.get("cluster_name"): + env["EVAL_CLUSTER_NAME"] = cc["cluster_name"] + if paths.get("project_root"): + env["EVAL_PROJECT_ROOT"] = paths["project_root"] + if paths.get("hf_cache"): + env["EVAL_HF_CACHE"] = paths["hf_cache"] + if paths.get("harbor_src"): + env["EVAL_HARBOR_SRC"] = paths["harbor_src"] + if paths.get("datasets_dirs"): + env["EVAL_DATASETS_DIRS"] = ":".join(paths["datasets_dirs"]) + if paths.get("eval_logs_dir"): + env["EVAL_LOGS_DIR"] = paths["eval_logs_dir"] + + env["EVAL_PROXY_ENABLED"] = "true" if proxy.get("enabled") else "false" + if proxy.get("login_node"): + env["EVAL_LOGIN_NODE"] = proxy["login_node"] + if proxy.get("proxychains_bin"): + env["EVAL_PROXYCHAINS_BIN"] = proxy["proxychains_bin"] + + if hw.get("cuda_home"): + env["EVAL_CUDA_HOME"] = hw["cuda_home"] + if hw.get("arch"): + env["EVAL_ARCH"] = hw["arch"] + if hw.get("gpus_per_node"): + env["EVAL_GPUS_PER_NODE"] = str(hw["gpus_per_node"]) + if hw.get("cpus_per_node"): + env["EVAL_CPUS_PER_NODE"] = str(hw["cpus_per_node"]) + + return env + + def __str__(self) -> str: + """String representation for logging.""" + parts = [ + f"n_concurrent={self.n_concurrent}", + f"n_attempts={self.n_attempts}", + f"gpu_memory_util={self.gpu_memory_util}", + f"error_threshold={self.error_threshold}", + f"vllm_max_retries={self.vllm_max_retries}", + ] + if self.agent_parser: + parts.append(f"agent_parser={self.agent_parser}") + if self.slurm_time != DEFAULT_SLURM_TIME: + parts.append(f"slurm_time={self.slurm_time}") + if self.tp_size != DEFAULT_TP_SIZE: + parts.append(f"tp_size={self.tp_size}") + if self.dp_size > 1: + parts.append(f"dp_size={self.dp_size}") + if self.enable_thinking: + parts.append("enable_thinking=True") + if self.agent_name != DEFAULT_AGENT_NAME: + parts.append(f"agent_name={self.agent_name}") + if self.slurm_partition != DEFAULT_SLURM_PARTITION: + parts.append(f"slurm_partition={self.slurm_partition}") + if self.upload_username: + parts.append(f"upload_username={self.upload_username}") + if self.timeout_multiplier != DEFAULT_TIMEOUT_MULTIPLIER: + parts.append(f"timeout_multiplier={self.timeout_multiplier}") + return ", ".join(parts) + + +def _run(cmd: List[str], env: Optional[Dict[str, str]] = None) -> Tuple[int, str]: + """Run a command and return exit code and output.""" + # Merge with current environment if extra env vars provided + run_env = None + if env: + run_env = os.environ.copy() + run_env.update(env) + + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=run_env + ) + out_lines = [] + assert proc.stdout is not None + for line in proc.stdout: + out_lines.append(line.rstrip()) + code = proc.wait() + return code, "\n".join(out_lines) + + +def get_idle_nodes(partition: str) -> List[str]: + """Get list of idle nodes on a SLURM partition.""" + code, out = _run(["sinfo", "-p", partition, "-N", "--format=%N %t", "--noheader"]) + if code != 0: + return [] + nodes = [] + for line in out.strip().split("\n"): + parts = line.split() + if len(parts) >= 2 and parts[1].strip() == "idle": + nodes.append(parts[0].strip()) + return nodes + + +def generate_run_tag(dataset_hf: str, model_hf: str) -> str: + """ + Generate a unique RUN_TAG for the job. + + Format: {safe_repo}_{safe_model}_{timestamp} + """ + safe_repo = dataset_repo_name(dataset_hf).replace("-", "_").replace(".", "_") + safe_model = model_hf.split("/")[-1].replace("-", "_").replace(".", "_") + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + return f"{safe_repo}_{safe_model}_{timestamp}" + + +def cancel_slurm_job(slurm_job_id: str, dry_run: bool = False) -> bool: + """Cancel a SLURM job via scancel. Returns True if successful.""" + if dry_run: + log(f"[DRY RUN] Would cancel SLURM job {slurm_job_id}") + return True + code, out = _run(["scancel", slurm_job_id]) + if code == 0: + log(f"Cancelled SLURM job {slurm_job_id}") + return True + else: + log(f"WARNING: scancel failed for job {slurm_job_id}: {out}") + return False + + +def update_pending_job_slurm_id(db_job_id: str, slurm_job_id: str) -> None: + """Update the Pending job entry with the SLURM job ID after successful sbatch.""" + try: + client = get_supabase_client() + client.table("sandbox_jobs").update( + {"slurm_job_id": slurm_job_id} + ).eq("id", db_job_id).execute() + log(f"Updated job {db_job_id} with slurm_job_id={slurm_job_id}", verbose_only=True) + except Exception as e: + log(f"WARNING: failed to update job {db_job_id} with slurm_job_id: {e}") + + +def submit_eval( + hf_model_name: str, + dataset_hf: str, + benchmark_id: Optional[str], + sbatch_script: str, + sbatch_params: Optional[SbatchParams] = None, + dry_run: bool = False, + upload_username: str = "", + timeout_multiplier: float = DEFAULT_TIMEOUT_MULTIPLIER, + vllm_overrides: Optional[Dict[str, str]] = None, + dependency: Optional[str] = None, + eval_config: Optional[Dict] = None, + conda_env: str = "otagent", + run_tag_override: Optional[str] = None, + dp_nodes: int = 0, + nodelist: Optional[str] = None, + extra_env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[str]]: + """ + Create a Pending DB entry, then submit sbatch job and update with SLURM ID. + + sbatch positional args: + $1 = model HF name + $2 = dataset HF repo (org/repo) + $3 = benchmark_id (uuid) [optional] + $4 = job_name (RUN_TAG) + + Environment variables (from SbatchParams.to_env()): + EVAL_N_CONCURRENT, EVAL_N_ATTEMPTS, EVAL_GPU_MEMORY_UTIL, + EVAL_DAYTONA_THRESHOLD, EVAL_VLLM_MAX_RETRIES, EVAL_AGENT_PARSER, + EVAL_SLURM_TIME, EVAL_ENABLE_THINKING, EVAL_AGENT_NAME, + EVAL_STARTS_LOG (v3), EVAL_TIMEOUT_MULTIPLIER (v3) + + The Pending DB entry includes timeout_multiplier in its config dict + so that timeout-aware dedup (Enhancement 5) can match on it. + + Args: + vllm_overrides: Optional dict of EVAL_VLLM_* env vars from baseline + model config. Merged into sbatch env vars. + dependency: Optional SLURM dependency string (e.g. 'afterany:12345'). + eval_config: Optional harbor eval config dict for DB job config. + run_tag_override: v5 - reuse original run_tag for resume (triggers sbatch auto-resume). + + Returns: + (slurm_job_id, job_name) if successful, ("DRY_RUN", job_name) if dry run, (None, None) on failure + """ + # Generate unique job name, or reuse for resume + if run_tag_override: + job_name = run_tag_override + log(f" [v5] Reusing run_tag for resume: {job_name}") + else: + job_name = generate_run_tag(dataset_hf, hf_model_name) + + # Early return for dry-run — no DB writes, no sbatch + if dry_run: + log(f"[DRY RUN] Would submit: model={hf_model_name} dataset={dataset_hf} job={job_name}") + if sbatch_params: + log(f"[DRY RUN] With params: {sbatch_params}") + if vllm_overrides: + log(f"[DRY RUN] vLLM overrides: {list(vllm_overrides.keys())}", verbose_only=True) + return ("DRY_RUN", job_name) + + # Step 1: Create or reuse DB entry BEFORE sbatch submission + agent = sbatch_params.agent_name if sbatch_params else DEFAULT_AGENT_NAME + tm = sbatch_params.timeout_multiplier if sbatch_params else timeout_multiplier + config: Dict = {"agent": agent, "env": "daytona", "timeout_multiplier": tm, "run_tag": job_name} + # Include harbor eval config fields in DB entry for config-aware dedup + if eval_config: + if "timeout_multiplier" in eval_config: + config["timeout_multiplier"] = eval_config["timeout_multiplier"] + env_overrides = {} + for key in ("override_cpus", "override_memory_mb", "override_storage_mb"): + if key in eval_config: + env_overrides[key] = eval_config[key] + if env_overrides: + config["environment"] = env_overrides + + db_job_id: Optional[str] = None + try: + from database.unified_db.utils import ( + create_job_entry_pending, get_supabase_client, + get_model_by_name, get_benchmark_by_name, + get_job_by_model_benchmark, update_sandbox_job, + ) + + if run_tag_override: + # v6 resume: find existing DB entry and update its job_name to match + # the resume run_tag, so sbatch's update_job_status_to_started() can + # find it by name. Also reset status to Pending for clean state. + model_row = get_model_by_name(hf_model_name) + bench_name = dataset_hf.split("/")[-1] if "/" in dataset_hf else dataset_hf + bench_row = get_benchmark_by_name(bench_name) + if model_row and bench_row: + existing = get_job_by_model_benchmark(model_row['id'], bench_row['id']) + if existing: + old_name = existing.get('job_name', '?') + old_status = existing.get('job_status', '?') + db_job_id = str(existing['id']) + # Update the existing entry: reset to Pending with new job_name + update_sandbox_job(db_job_id, { + "job_name": job_name, + "job_status": "Pending", + "slurm_job_id": "pending", + "config": config, + "submitted_at": datetime.now().isoformat(), + "started_at": None, + }) + log(f" [v6] Reused DB entry {db_job_id}: {old_status} '{old_name}' → Pending '{job_name}'", + verbose_only=True) + + if not db_job_id: + # Normal path: create new Pending entry + result = create_job_entry_pending( + job_name=job_name, + model_hf=hf_model_name, + benchmark_hf=dataset_hf, + agent_name=agent, + slurm_job_id="pending", + username=upload_username or "listener", + config=config, + ) + if result.get("success") and result.get("job"): + db_job_id = str(result["job"].get("id")) + log(f"Created Pending DB entry: {db_job_id}", verbose_only=True) + else: + log(f"WARNING: Failed to create Pending DB entry: {result.get('error')}") + except Exception as e: + log(f"WARNING: Exception creating Pending DB entry: {e}") + + # Step 2: Build sbatch command + cmd = ["sbatch"] + if sbatch_params: + cmd.extend(["--time", sbatch_params.slurm_time]) + cmd.extend(["--partition", sbatch_params.slurm_partition]) + if sbatch_params.slurm_account: + cmd.extend(["--account", sbatch_params.slurm_account]) + # Request GPUs, CPUs, and memory proportional to total GPU count (TP × DP). + # Per-model baseline config can override TP (e.g. 32B models need TP=4). + effective_tp = sbatch_params.tp_size + if vllm_overrides and "EVAL_VLLM_TENSOR_PARALLEL_SIZE" in vllm_overrides: + effective_tp = int(vllm_overrides["EVAL_VLLM_TENSOR_PARALLEL_SIZE"]) + effective_dp = sbatch_params.dp_size + total_gpus = effective_tp * effective_dp + cmd.extend(["--gres", f"gpu:{total_gpus}"]) + cc = _CLUSTER_CONFIG or {} + hw = cc.get("hardware", {}) + gpus_per_node = int(hw.get("gpus_per_node", 8)) + cpus_per_node = int(hw.get("cpus_per_node", 96)) + mem_per_node_mb = int(hw.get("mem_per_node_mb", 1860000)) + cpus_needed = (cpus_per_node * total_gpus) // gpus_per_node + mem_needed = (mem_per_node_mb * total_gpus) // gpus_per_node + cmd.extend(["--cpus-per-task", str(cpus_needed)]) + cmd.extend(["--mem", f"{mem_needed}M"]) + # Override sbatch --output to use cluster-configured log dir + cc_logs = (_CLUSTER_CONFIG or {}).get("paths", {}).get("eval_logs_dir", _FALLBACK_EVAL_LOGS_DIR) + cmd.extend(["--output", f"{cc_logs}/%x_%j.out"]) + if nodelist: + cmd.extend(["--nodelist", nodelist]) + if dependency: + cmd.append(f"--dependency={dependency}") + # DP: request multiple nodes + if dp_nodes > 0: + cmd.extend(["--nodes", str(dp_nodes)]) + # Benchmark-aware SLURM job name for squeue readability + repo_name = dataset_repo_name(dataset_hf) + bench_tag = _BENCH_SHORT.get(repo_name, repo_name[:12]) + if run_tag_override: + job_prefix = "res_dp_" if dp_nodes > 0 else "res_" + else: + job_prefix = "eval_dp_" if dp_nodes > 0 else "eval_" + slurm_job_name = os.environ.get("EVAL_SLURM_JOB_NAME", "data") + cmd.extend(["--job-name", slurm_job_name]) + cmd.append(sbatch_script) + cmd.extend([hf_model_name, dataset_hf]) + # $3 = benchmark_id (or empty placeholder), $4 = run_tag override + cmd.append(str(benchmark_id) if benchmark_id else "") + cmd.append(job_name) # 4th arg: job_name (RUN_TAG) + + # Get env vars from params and merge vllm overrides + harbor config env vars + env_vars = sbatch_params.to_env() if sbatch_params else {} + if db_job_id: + env_vars["EVAL_DB_JOB_ID"] = db_job_id + if vllm_overrides: + env_vars.update(vllm_overrides) + # Pass harbor eval config fields as sbatch env vars + if eval_config: + if eval_config.get("timeout_multiplier") is not None: + env_vars["EVAL_TIMEOUT_MULTIPLIER"] = str(eval_config["timeout_multiplier"]) + if eval_config.get("override_memory_mb") is not None: + env_vars["EVAL_OVERRIDE_MEMORY_MB"] = str(eval_config["override_memory_mb"]) + + # Pass conda env path so sbatch uses the right Python/vLLM installation + otagent_dir = CONDA_ENV_PATHS.get(conda_env) + if otagent_dir: + env_vars["OTAGENT_DIR"] = otagent_dir + # Merge extra env vars (e.g. EVAL_VLLM_PORT from pack-jobs port planner) + if extra_env: + env_vars.update(extra_env) + # DP: let sbatch compute NUM_SHARDS from GPUS_PER_NODE / TP_SIZE * nodes + # Do NOT pass EVAL_NUM_SHARDS — it would override the per-node shard calculation + + # Step 3: Run sbatch + code, out = _run(cmd, env=env_vars) + log(f"sbatch: {' '.join(cmd)}\n{out}") + + if code != 0: + # sbatch failed; pending entry remains (will be detected as stale later) + return (None, None) + + m = re.search(r"Submitted batch job (\d+)", out) + slurm_job_id = m.group(1) if m else None + + if not slurm_job_id: + log("ERROR: Could not parse SLURM job ID from sbatch output") + return (None, None) + + # Step 4: Update pending entry with actual SLURM job ID + if db_job_id: + update_pending_job_slurm_id(db_job_id, slurm_job_id) + + return (slurm_job_id, job_name) + + +# ---------- Main Listener Class ---------- +class EvalListener: + """Unified eval listener v3 that handles all benchmark configurations. + + Lifecycle: + 1. run() logs config, runs Daytona pre-flight (if enabled), enters main loop + 2. Each iteration: hot-reload priority file, fetch models, filter, build + submissions list, sort by priority rank, apply retry deprioritization, + throttle to per-listener SLURM limit, submit + 3. Sleep for check_interval_hours, then repeat + + Per-listener SLURM tracking: + _submitted_jobs tracks SLURM job IDs submitted by THIS listener instance. + Each iteration, completed jobs are pruned via squeue intersection. This + allows multiple listeners to run in parallel with independent job budgets. + """ + + def __init__(self, config: ListenerConfig): + self.config = config + self._submitted_jobs: Set[str] = set() # SLURM job IDs submitted by THIS listener + self._dep_chain: List[str] = [] # Persistent sliding-window dependency chain across iterations + self._resume_run_tags: Dict[str, str] = {} # v6: hf_model → run_tag for disk resume + set_log_file(config.log_file) + # Seed _submitted_jobs from previous listener logs (--inherit-log) + if config.inherit_log: + inherited = parse_submitted_jobs_from_logs(config.inherit_log) + self._submitted_jobs = inherited + if inherited: + # Log the inherited IDs so future --inherit-log on THIS log picks them up + log(f"[inherit-log] Inherited {len(inherited)} active job(s)") + log(f"[inherit-log] Inherited jobs: {','.join(sorted(inherited))}") + + def run_iteration(self) -> int: + """ + Run one check iteration. + + Returns: + Number of jobs submitted (or would submit in dry-run mode) + """ + # Hot-reload priority models from file (enables editing during long runs) + if self.config.priority_file: + new_priority = load_priority_models(self.config.priority_file) + if new_priority != self.config.priority_models: + log(f"Priority list reloaded: {len(new_priority)} model(s)") + self.config.priority_models = new_priority + + # Hot-reload blacklist from file + if self.config.blacklist_file: + new_blacklist = load_blacklist(self.config.blacklist_file) + if new_blacklist != self.config.blacklisted_models: + log(f"Blacklist reloaded: {len(new_blacklist)} model(s)") + self.config.blacklisted_models = new_blacklist + + # v6: Clear per-iteration resume state + self._resume_run_tags = {} + + log("Checking for new models...") + + # Optimization: in filter_only mode with a priority file, skip the + # expensive fetch_recent_models() (which returns ALL models in the + # lookback window) and only fetch priority models by name. + if (self.config.priority_mode == "filter_only" + and self.config.priority_models): + models = fetch_priority_models(self.config.priority_models) + log(f"Fetched {len(models)} priority model(s) directly (filter_only mode, skipped full scan).") + else: + models = fetch_recent_models(self.config.lookback_days) + log(f"Found {len(models)} model(s) in lookback window.") + + # Priority models bypass lookback window. + # Fetch priority models by name regardless of creation_time, then merge. + if self.config.priority_models: + priority_models_from_db = fetch_priority_models(self.config.priority_models) + seen_ids = {str(m.get("id")) for m in models} + added = 0 + for pm in priority_models_from_db: + if str(pm.get("id")) not in seen_ids: + models.append(pm) + seen_ids.add(str(pm.get("id"))) + added += 1 + if added: + log(f"Added {added} priority model(s) outside lookback window.") + + log(f"Total {len(models)} model(s) to check. Filtering...") + + # Check if we should skip all models due to require_priority_list + if not self.config.priority_models and self.config.require_priority_list: + log("No priority list configured and --require-priority-list is set. Skipping all models.") + return 0 + + submissions: List[Tuple[str, str, str, Optional[str], str, Optional[str]]] = [] + # (model_id, hf_model_name, dataset_hf, benchmark_id, reason, slurm_job_id) + finished_in_db: Set[str] = set() # v6: models DB considers done (skip for resume) + + # v6: Build set of (model, dataset) pairs currently running in squeue (by parsing eval logs). + # Used to prevent both resume and fresh submissions for already-running models. + active_models, active_pairs, active_run_tags = get_active_model_dataset_pairs( + log_dir=_cc_path("eval_logs_dir", _FALLBACK_EVAL_LOGS_DIR), + ) + if active_pairs: + log(f"[v6-active] Found {len(active_pairs)} (model, dataset) pair(s) currently active in squeue") + if self.config.verbose: + for m, d in sorted(active_pairs): + log(f" [v6-active] {m} on {d}") + + # Track stats + skipped_not_in_priority = 0 + skipped_hf_not_exists = 0 + + # Resolve all benchmarks up front (once per loop) + dataset_to_bench: Dict[str, Optional[str]] = { + ds: resolve_benchmark_id(ds) for ds in self.config.datasets + } + + # Precompute benchmark duplicate groups for cross-duplicate aggregation + bench_dup_groups: Dict[str, List[str]] = {} + for ds, bench_id in dataset_to_bench.items(): + if bench_id: + bench_dup_groups[bench_id] = get_duplicate_group_ids('benchmarks', bench_id) + + for m in models: + model_id = str(m.get("id")) + if not model_id: + continue + + hf_model = resolve_hf_model_name(m) + if not hf_model: + if self.config.verbose: + log(f"Skip: cannot resolve HF model for id={model_id}, name={m.get('name')}") + continue + + # Blacklist check (overrides priority) + if hf_model in self.config.blacklisted_models: + if self.config.verbose: + log(f"Skip: model={hf_model} is blacklisted") + continue + + # Priority handling depends on mode + is_priority = bool(self.config.priority_models and hf_model in self.config.priority_models) + + if self.config.priority_mode == "filter_only": + # Only evaluate models in the priority list + if self.config.priority_models and not is_priority: + skipped_not_in_priority += 1 + continue + # priority_first: don't skip, just track is_priority for sorting + + # HuggingFace existence check + if self.config.check_hf_exists: + if not check_hf_model_exists(hf_model): + log(f"Skip: model not found on HuggingFace: {hf_model} (model_id={model_id})") + skipped_hf_not_exists += 1 + continue + + # Compute model duplicate group for cross-duplicate aggregation + model_dup_ids = get_duplicate_group_ids('models', model_id) + + for dataset_hf in self.config.datasets: + bench_id = dataset_to_bench.get(dataset_hf) + + # Get benchmark duplicate group (precomputed above) + bench_dup_ids = bench_dup_groups.get(bench_id) if bench_id else None + + # Check DB status to decide if we should start + # (Enhancement 5: timeout-aware, cross-duplicate aggregation, config-aware dedup) + if self.config.force_reeval: + should_start, reason, old_slurm_job_id = True, "force-reeval", None + else: + should_start, reason, old_slurm_job_id = should_start_job( + model_id, bench_id, self.config.stale_job_hours, + stale_pending_hours=self.config.stale_pending_hours, + timeout_aware=self.config.timeout_aware, + agent_name=self.config.agent_name, + timeout_multiplier=self.config.timeout_multiplier, + duplicate_model_ids=model_dup_ids, + duplicate_benchmark_ids=bench_dup_ids, + eval_config=self.config.eval_config if self.config.eval_config else None, + ) + + if should_start: + # v6: Skip if (model, dataset) already running in squeue (even if DB says "no existing job", + # e.g. when DB entry was deleted but SLURM job is still active) + # Bypass this check in force-reeval mode. + if not self.config.force_reeval and (hf_model, dataset_hf) in active_pairs: + if self.config.verbose: + log(f"Skip: model={hf_model}, dataset={dataset_hf}, reason=currently running in squeue") + continue + submissions.append((model_id, hf_model, dataset_hf, bench_id, reason, old_slurm_job_id)) + else: + # Track models the DB considers done (for v6 resume filtering) + if "finished" in reason: + finished_in_db.add(hf_model) + if self.config.verbose: + log(f"Skip: model={hf_model}, dataset={dataset_hf}, reason={reason}") + + # Log filtering stats + if self.config.priority_mode == "filter_only" and self.config.priority_models and skipped_not_in_priority > 0: + log(f"Skipped {skipped_not_in_priority} model(s) not in priority list") + if self.config.check_hf_exists and skipped_hf_not_exists > 0: + log(f"Skipped {skipped_hf_not_exists} model(s) not found on HuggingFace") + + # v6: Disk-based resume — scan jobs dir for incomplete/errored jobs + resume_submissions = [] + if self.config.enable_disk_resume and self.config.jobs_dirs: + # Always query squeue (even in dry-run) for accurate filtering + active_slurm = get_active_slurm_job_ids() + # Build dataset prefixes from config + ds_prefixes = [] + for ds in self.config.datasets: + ds_short = ds.split("/")[-1] if "/" in ds else ds + ds_prefixes.append(ds_short) + # Scan all configured jobs directories + resume_candidates = [] + for jdir in self.config.jobs_dirs: + resume_candidates.extend(scan_jobs_dir_for_resume( + jobs_dir=jdir, + dataset_prefixes=ds_prefixes, + active_slurm_ids=active_slurm, + infra_error_threshold=self.config.resume_infra_error_threshold, + max_resume_count=self.config.max_resume_count, + )) + # Filter resume candidates through blacklist and priority (same as normal models) + if self.config.blacklisted_models: + before = len(resume_candidates) + resume_candidates = [rc for rc in resume_candidates + if rc["hf_model"] not in self.config.blacklisted_models] + skipped_bl = before - len(resume_candidates) + if skipped_bl: + log(f"[v6-resume] Filtered out {skipped_bl} blacklisted resume candidate(s)") + if self.config.priority_mode == "filter_only" and self.config.priority_models: + before = len(resume_candidates) + resume_candidates = [rc for rc in resume_candidates + if rc["hf_model"] in self.config.priority_models] + skipped_prio = before - len(resume_candidates) + if skipped_prio: + log(f"[v6-resume] Filtered out {skipped_prio} non-priority resume candidate(s)") + + # v6: Filter out (model, dataset) pairs currently running in squeue. + # This prevents resuming old dirs when a job for the same model+dataset is active. + if active_pairs: + before = len(resume_candidates) + resume_candidates = [rc for rc in resume_candidates + if (rc["hf_model"], rc.get("dataset", "")) not in active_pairs] + skipped_active = before - len(resume_candidates) + if skipped_active: + log(f"[v6-resume] Filtered out {skipped_active} currently-running resume candidate(s)") + + # Filter out models that DB already considers finished (stale disk dirs + # from older runs that have been superseded by a successful resubmission) + if finished_in_db: + before = len(resume_candidates) + resume_candidates = [rc for rc in resume_candidates + if rc["hf_model"] not in finished_in_db] + skipped_fin = before - len(resume_candidates) + if skipped_fin: + log(f"[v6-resume] Filtered out {skipped_fin} already-finished-in-DB resume candidate(s)") + + # Dedup: pick the most recent dir per model (reverse so latest timestamp wins). + seen_resume_models: Set[str] = set() + for rc in reversed(resume_candidates): + if rc["hf_model"] not in seen_resume_models: + seen_resume_models.add(rc["hf_model"]) + # Use a sentinel model_id since we don't have it from DB + resume_submissions.append( + ("__resume__", rc["hf_model"], rc["dataset"] or "", + None, rc["reason"], None) + ) + # Store run_tag mapping for submit_eval + self._resume_run_tags[rc["hf_model"]] = rc["run_tag"] + if resume_submissions: + log(f"[v6-resume] Adding {len(resume_submissions)} resume job(s) (priority over new models)") + + # v6: Resume takes priority — remove normal submissions for models + # that already have a resume candidate (avoid duplicate fresh + resume). + resume_model_set = {s[1] for s in resume_submissions} # s[1] = hf_model + if resume_model_set: + before = len(submissions) + submissions = [s for s in submissions if s[1] not in resume_model_set] + skipped_dup = before - len(submissions) + if skipped_dup: + log(f"[v6-resume] Suppressed {skipped_dup} fresh submission(s) in favor of resume") + + # --resume-only: drop all fresh submissions, keep only resume jobs + if self.config.resume_only: + if submissions: + log(f"[v6-resume] --resume-only: dropping {len(submissions)} fresh submission(s)") + submissions = [] + + if not submissions and not resume_submissions: + log("No eligible (model, dataset) pairs to submit.") + return 0 + + # Prepend resume submissions (higher priority than new models) + submissions = resume_submissions + submissions + + # Sort submissions by priority file order (earlier in file = higher priority). + # Models not in the priority list get lowest rank (submitted last). + if self.config.priority_models: + priority_rank = {m: i for i, m in enumerate(self.config.priority_models)} + fallback_rank = len(self.config.priority_models) + submissions.sort(key=lambda s: priority_rank.get(s[1], fallback_rank)) + if self.config.priority_mode == "priority_first": + n_priority = sum(1 for s in submissions if s[1] in priority_rank) + n_non_priority = len(submissions) - n_priority + log(f"Priority-first ordering: {n_priority} priority + {n_non_priority} non-priority submissions") + + prefix = "[DRY RUN] Would submit" if self.config.dry_run else "Submitting" + log(f"{prefix} {len(submissions)} eval(s)...") + + # Enhancement 2: Per-listener SLURM job submission throttle. + # Track which SLURM job IDs this listener submitted. Prune finished ones + # via squeue, then cap new submissions at remaining slots. + if not self.config.dry_run: + active_ids = get_active_slurm_job_ids() + # Prune jobs that are no longer in squeue (finished/failed/cancelled) + still_active = self._submitted_jobs & active_ids + finished = len(self._submitted_jobs) - len(still_active) + self._submitted_jobs = still_active + active_count = len(self._submitted_jobs) + remaining_slots = self.config.max_jobs_submitted - active_count + log(f"Listener SLURM jobs: {active_count} active " + f"({finished} finished since last check), " + f"{remaining_slots} slots available (max {self.config.max_jobs_submitted})") + if remaining_slots <= 0: + log(f"WARNING: At per-listener job limit " + f"({active_count}/{self.config.max_jobs_submitted}), " + f"skipping all submissions this iteration") + return 0 + if len(submissions) > remaining_slots: + log(f"Capping submissions from {len(submissions)} to {remaining_slots} " + f"(per-listener limit: {self.config.max_jobs_submitted})") + submissions = submissions[:remaining_slots] + + # Create sbatch params from config + sbatch_params = SbatchParams( + n_concurrent=self.config.n_concurrent, + n_attempts=self.config.n_attempts, + gpu_memory_util=self.config.gpu_memory_util, + error_threshold=self.config.error_threshold, + vllm_max_retries=self.config.vllm_max_retries, + agent_parser=self.config.agent_parser, + slurm_time=self.config.slurm_time, + enable_thinking=self.config.enable_thinking, + agent_name=self.config.agent_name, + slurm_partition=self.config.slurm_partition, + slurm_account=self.config.slurm_account, + tp_size=self.config.tp_size, + dp_size=self.config.dp_size, + upload_username=self.config.upload_username, + timeout_multiplier=self.config.timeout_multiplier, + config_yaml=self.config.config_yaml, + auto_snapshot=self.config.auto_snapshot, + ) + + # Load baseline model configs for per-model vLLM overrides + baseline_configs = load_baseline_model_configs(self.config.baseline_model_configs) + + # Add harbor config env vars to sbatch params + if self.config.harbor_config: + # Will be merged in submit_eval via eval_config, but also pass path + pass # harbor config fields are passed via eval_config to submit_eval + + # Pre-download setup (for no-internet compute nodes) + if self.config.pre_download: + from huggingface_hub import snapshot_download + downloaded_models: set = set() + + # Sliding-window dependency tracking (persistent across iterations) + # self._dep_chain carries job IDs from previous iterations so new jobs + # respect the concurrency limit even across sleep cycles. + batch_size = self.config.batch_size + if batch_size and batch_size > 0: + if not self.config.dry_run: + active_ids = get_active_slurm_job_ids() + else: + active_ids = set() + active_in_chain = sum(1 for jid in self._dep_chain if jid in active_ids) + log(f"Sliding-window batch-size={batch_size}: " + f"{active_in_chain} active jobs in dependency chain from previous iterations") + + # Node packing: query idle nodes and track GPU + port slots per node + pack_node_list: List[str] = [] + pack_gpus_per_node = 8 + pack_node_gpu_used: Dict[int, int] = {} # node_idx -> GPUs used so far + pack_node_port_next: Dict[int, int] = {} # node_idx -> next available port offset + pack_node_idx = 0 + if self.config.pack_jobs: + cc = self.config.cluster_config or {} + hw = cc.get("hardware", {}) + pack_gpus_per_node = int(hw.get("gpus_per_node", 8)) + pack_node_list = get_idle_nodes(self.config.slurm_partition) + if pack_node_list: + log(f"Pack mode: {len(pack_node_list)} idle nodes, {pack_gpus_per_node} GPUs/node") + else: + log("Pack mode: no idle nodes found, falling back to default scheduling") + + submitted = 0 + for idx, (mid, hf_model, dataset_hf, bench_id, reason, old_slurm_job_id) in enumerate(submissions): + + # Pre-download this model before submitting (download-then-submit per model) + # Uses the shared HF cache so compute nodes (no internet) find it via HF_HUB_OFFLINE=1 + if self.config.pre_download and hf_model not in downloaded_models: + hf_cache = os.environ.get("HF_HUB_CACHE", _cc_path("hf_cache", _FALLBACK_HF_CACHE)) + log(f" Pre-downloading model {hf_model} to {hf_cache}...") + try: + # Run snapshot_download in a subprocess thread with timeout + # to avoid indefinite hangs on network issues + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + snapshot_download, repo_id=hf_model, repo_type="model", cache_dir=hf_cache + ) + path = future.result(timeout=300) # 5 minute timeout + log(f" Cached at {path}") + except concurrent.futures.TimeoutError: + log(f" WARNING: Pre-download of {hf_model} timed out after 300s, skipping (will retry next iteration)") + except Exception as e: + log(f" WARNING: Failed to download {hf_model}: {e}") + downloaded_models.add(hf_model) + + dry_prefix = "[DRY RUN] " if self.config.dry_run else "" + prio_tag = " [PRIORITY]" if (self.config.priority_mode == "priority_first" + and self.config.priority_models + and hf_model in self.config.priority_models) else "" + # v6: Pretty-print resume reason + display_reason = reason + log(f"{dry_prefix}Submitting [{idx+1}/{len(submissions)}]: model={hf_model}, dataset={dataset_hf}, reason={display_reason}{prio_tag}") + + # Cancel stale Pending SLURM job before resubmission + if reason.startswith("stale pending") and old_slurm_job_id: + cancel_slurm_job(old_slurm_job_id, dry_run=self.config.dry_run) + + # Per-model vLLM overrides from baseline config mapping + vllm_overrides = get_vllm_env_overrides(hf_model, baseline_configs) + if vllm_overrides: + log(f" Applying baseline model vLLM overrides: {list(vllm_overrides.keys())}", verbose_only=True) + + # Per-model conda env override (e.g. otagent2 for Qwen3.5) + model_conda_env = get_conda_env_override(hf_model, baseline_configs) or self.config.conda_env + if model_conda_env != self.config.conda_env: + log(f" Using conda env '{model_conda_env}' for {hf_model}") + + # Build sliding-window dependency using persistent chain. + # Look back batch_size positions in self._dep_chain. If that job is + # still active (running/pending in squeue), depend on it. If it already + # finished, the concurrency slot is free — no dependency needed. + job_dependency: Optional[str] = None + if batch_size and batch_size > 0: + chain_pos = len(self._dep_chain) # where this job will be appended + if chain_pos >= batch_size: + dep_candidate = self._dep_chain[chain_pos - batch_size] + if not self.config.dry_run and dep_candidate in active_ids: + job_dependency = f"afterany:{dep_candidate}" + log(f" Depends on job {dep_candidate} (chain pos {chain_pos - batch_size})", verbose_only=True) + + # Stagger chain: jobs wait N minutes after the previous batch STARTS. + # Uses SLURM "after:jobid+minutes" so jobs start sequentially even when + # many nodes are idle (prevents Daytona sandbox creation burst). + # With chain_batch_size=K, K jobs fire together, then the next K wait + # for the first job of the previous batch to have started + delay. + if self.config.stagger_delay > 0 and self._dep_chain: + cbs = max(self.config.chain_batch_size, 1) + chain_len = len(self._dep_chain) + # Determine which batch this job belongs to + current_batch = chain_len // cbs + if current_batch > 0: + # Depend on the first job of the previous batch + prev_batch_first = (current_batch - 1) * cbs + prev_job = self._dep_chain[prev_batch_first] + if not prev_job.startswith(("DRY_", "FAILED_")): + stagger_dep = f"after:{prev_job}+{self.config.stagger_delay}" + if job_dependency: + job_dependency = f"{job_dependency},{stagger_dep}" + else: + job_dependency = stagger_dep + if chain_len % cbs == 0: + log(f" Stagger: batch {current_batch} boundary, wait {self.config.stagger_delay}m after job {prev_job} starts") + else: + log(f" Stagger: batch {current_batch} (pos {chain_len % cbs}/{cbs}), wait {self.config.stagger_delay}m after job {prev_job} starts", verbose_only=True) + + # v6: Extract run_tag_override for disk-based resume + resume_run_tag = self._resume_run_tags.get(hf_model) + + # DP: use DP sbatch when dp_nodes > 0 + actual_sbatch = self.config.dp_sbatch_script if self.config.dp_nodes > 0 else self.config.sbatch_script + + # Node packing: assign a target node and port based on GPU slots + target_node = None + pack_port = None + if pack_node_list: + # Determine effective GPU count for this model (TP × DP) + effective_tp = self.config.tp_size + if vllm_overrides and "EVAL_VLLM_TENSOR_PARALLEL_SIZE" in vllm_overrides: + effective_tp = int(vllm_overrides["EVAL_VLLM_TENSOR_PARALLEL_SIZE"]) + effective_dp = self.config.dp_size + total_gpus = effective_tp * effective_dp + # Find a node with enough free GPU slots + while pack_node_idx < len(pack_node_list): + used = pack_node_gpu_used.get(pack_node_idx, 0) + if used + total_gpus <= pack_gpus_per_node: + target_node = pack_node_list[pack_node_idx] + pack_node_gpu_used[pack_node_idx] = used + total_gpus + # Assign a non-overlapping port for this job + port_offset = pack_node_port_next.get(pack_node_idx, 0) + pack_port = 10000 + port_offset + pack_node_port_next[pack_node_idx] = port_offset + max(effective_dp, 1) + break + pack_node_idx += 1 + if target_node: + log(f" Pack: {target_node} (GPUs {pack_node_gpu_used[pack_node_idx]}/{pack_gpus_per_node}, port {pack_port})", verbose_only=True) + + # Pass listener-assigned port to sbatch when packing + extra_env = {} + if pack_port is not None: + extra_env["EVAL_VLLM_PORT"] = str(pack_port) + + slurm_job_id, job_name = submit_eval( + hf_model, + dataset_hf, + bench_id, + actual_sbatch, + sbatch_params=sbatch_params, + dry_run=self.config.dry_run, + upload_username=self.config.upload_username, + timeout_multiplier=self.config.timeout_multiplier, + vllm_overrides=vllm_overrides if vllm_overrides else None, + dependency=job_dependency, + eval_config=self.config.eval_config if self.config.eval_config else None, + conda_env=model_conda_env, + run_tag_override=resume_run_tag, + dp_nodes=self.config.dp_nodes, + nodelist=target_node, + extra_env=extra_env, + ) + + if slurm_job_id: + if self.config.dry_run: + node_str = f" on {target_node}" if target_node else "" + log(f" -> Would submit as SLURM job (job_name={job_name}){node_str}") + self._dep_chain.append(f"DRY_{idx}") + else: + log(f" -> Submitted as SLURM job {slurm_job_id} (job_name={job_name})") + self._submitted_jobs.add(slurm_job_id) + self._dep_chain.append(slurm_job_id) + submitted += 1 + else: + log(f" -> Submission failed") + self._dep_chain.append(f"FAILED_{idx}") + + if not self.config.dry_run and self.config.submission_delay > 0: + time.sleep(self.config.submission_delay) + + return submitted + + def run(self) -> None: + """Main event loop.""" + # Log configuration + hdr = ( + f"lookback={self.config.lookback_days}d, " + f"every {self.config.check_interval_hours}h, " + f"sbatch={self.config.sbatch_script}" + ) + log(f"Starting listener v3 for datasets={self.config.datasets}: {hdr}") + log( + f"Job logic: restart if 'Started' and started_at > {self.config.stale_job_hours}h ago, " + f"restart+scancel if 'Pending' and submitted_at > {self.config.stale_pending_hours}h ago, " + f"skip if 'Finished'" + ) + log(f"Dry run mode: {self.config.dry_run}") + log(f"Run once mode: {self.config.run_once}") + if self.config.force_reeval: + log("WARNING: --force-reeval is ON — bypassing DB status checks, will re-submit even if Finished") + log(f"Check HF exists: {self.config.check_hf_exists}") + log(f"Require priority list: {self.config.require_priority_list}") + + if self.config.priority_models: + mode_desc = "filter_only (skip non-priority)" if self.config.priority_mode == "filter_only" else "priority_first (all models, priority first)" + log(f"Priority mode: {mode_desc}, {len(self.config.priority_models)} model(s) in list") + if self.config.priority_file: + log(f"Priority file: {self.config.priority_file} (hot-reloaded each iteration)") + if self.config.verbose: + for m in sorted(self.config.priority_models): + log(f" - {m}") + else: + log("Priority: disabled (no priority file or empty)") + + if self.config.blacklisted_models: + log(f"Blacklist: {len(self.config.blacklisted_models)} model(s) from {self.config.blacklist_file}") + if self.config.verbose: + for m in sorted(self.config.blacklisted_models): + log(f" - {m}") + else: + log("Blacklist: disabled (no blacklist file or empty)") + + # Log sbatch parameters + sbatch_params = SbatchParams( + n_concurrent=self.config.n_concurrent, + n_attempts=self.config.n_attempts, + gpu_memory_util=self.config.gpu_memory_util, + error_threshold=self.config.error_threshold, + vllm_max_retries=self.config.vllm_max_retries, + agent_parser=self.config.agent_parser, + slurm_time=self.config.slurm_time, + enable_thinking=self.config.enable_thinking, + agent_name=self.config.agent_name, + slurm_partition=self.config.slurm_partition, + slurm_account=self.config.slurm_account, + tp_size=self.config.tp_size, + dp_size=self.config.dp_size, + timeout_multiplier=self.config.timeout_multiplier, + config_yaml=self.config.config_yaml, + auto_snapshot=self.config.auto_snapshot, + ) + log(f"Sbatch params: {sbatch_params}") + + # Log v3 enhancement status + log(f"[v3] Max SLURM jobs per listener: {self.config.max_jobs_submitted}") + log(f"[v3] Daytona resource check: {'enabled' if self.config.check_daytona_resources else 'disabled'}") + log(f"[v3] Timeout-aware dedup: {'enabled' if self.config.timeout_aware else 'disabled'}") + if self.config.timeout_multiplier != DEFAULT_TIMEOUT_MULTIPLIER: + log(f"[v3] Timeout multiplier: {self.config.timeout_multiplier}") + if self.config.stagger_delay > 0: + log(f"[v6] Stagger delay: {self.config.stagger_delay}m between batches of {self.config.chain_batch_size} jobs (SLURM after: chain)") + + # Enhancement 3: Daytona resource pre-flight check at startup + if self.config.check_daytona_resources: + ok = check_daytona_resources( + self.config.daytona_sandbox_limit, + self.config.daytona_warning_buffer, + ) + if not ok: + log("ERROR: Daytona resources at limit. Exiting.") + sys.exit(1) + + while True: + try: + # Enhancement 3: Optional per-iteration Daytona resource check + if self.config.check_daytona_resources: + ok = check_daytona_resources( + self.config.daytona_sandbox_limit, + self.config.daytona_warning_buffer, + ) + if not ok: + log("WARNING: Daytona resources at limit, skipping this iteration") + if self.config.run_once or self.config.dry_run: + break + hours = self.config.check_interval_hours + log(f"Sleeping for {hours} hours...\n") + time.sleep(self.config.check_interval_seconds) + continue + + self.run_iteration() + + # Exit after one iteration if requested + if self.config.run_once or self.config.dry_run: + mode = "DRY RUN" if self.config.dry_run else "ONCE" + log(f"[{mode}] Complete. Exiting after one iteration.") + break + + hours = self.config.check_interval_hours + log(f"Sleeping for {hours} hours...\n") + time.sleep(self.config.check_interval_seconds) + + except KeyboardInterrupt: + log("Interrupted by user. Exiting.") + sys.exit(0) + except Exception as e: + log(f"ERROR in main loop: {e}. Backing off 30s.") + time.sleep(30) + + +# ---------- CLI Argument Parsing ---------- +def parse_args() -> argparse.Namespace: + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Unified Eval Listener v4 - Run models on benchmark datasets", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +See the module docstring (top of file) for detailed flag reference with tuning +guidance. Quick summary below. + +Presets: aider, bfcl, swebench, v2, tb2, v1 + +v4 new: --blacklist-file PATH Block models from eval (overrides priority list) +v3 opt-in enhancements (all backward compatible): + --error-threshold N Unified invalid error threshold + --max-jobs-submitted N Per-listener SLURM job limit + --check-daytona-resources Daytona sandbox pre-flight check + --track-model-retries Deprioritize repeatedly-started models + --timeout-aware Dedup by model+benchmark+timeout_multiplier + +Examples: + python unified_eval_listener_v4.py --preset v2 \\ + --priority-file priority_models.txt + + python unified_eval_listener_v4.py --preset v2 \\ + --priority-file priority_models.txt \\ + --blacklist-file bad_models.txt + + python unified_eval_listener_v4.py --preset v2 --dry-run --once --verbose + """, + ) + + # Preset configuration + parser.add_argument( + "--preset", "-p", + choices=list(PRESETS.keys()), + help="Use a preset configuration (aider, bfcl, swebench, v2, tb2, v1)", + ) + + # Dataset configuration + parser.add_argument( + "--datasets", "-d", + help="Comma/space separated HF dataset repos (overrides preset)", + ) + parser.add_argument( + "--sbatch-script", "-s", + help="SBATCH script to use (overrides preset)", + ) + parser.add_argument( + "--log-file", + help="Log file path (default: auto-generated based on preset)", + ) + parser.add_argument( + "--log-dir", + help=f"Directory for listener logs (default: {DEFAULT_LOG_DIR}, env: EVAL_LISTENER_LOG_DIR)", + ) + + # Cluster configuration + parser.add_argument( + "--cluster-config", + help="Path to cluster config YAML (e.g. eval/clusters/jupiter.yaml). " + "Provides cluster-specific defaults for SLURM, paths, proxy, and hardware. " + "CLI flags still override cluster config values.", + ) + + # Timing configuration + parser.add_argument( + "--lookback-days", + type=int, + help=f"Days to look back for models (default: {DEFAULT_LOOKBACK_DAYS})", + ) + parser.add_argument( + "--check-hours", + type=float, + help=f"Hours between iterations (default: {DEFAULT_CHECK_HOURS})", + ) + parser.add_argument( + "--stale-hours", + type=int, + help=f"Hours before 'Started' job is stale (default: {DEFAULT_STALE_JOB_HOURS})", + ) + parser.add_argument( + "--stale-pending-hours", + type=int, + help=f"Hours before 'Pending' job is stale (default: {DEFAULT_STALE_PENDING_HOURS})", + ) + + # Priority filtering + parser.add_argument( + "--priority-file", + help="Path to priority models file (one model per line)", + ) + parser.add_argument( + "--require-priority-list", + action="store_true", + help="Skip all models when priority list is empty/missing", + ) + parser.add_argument( + "--blacklist-file", + help="Path to blacklisted models file (one model per line). " + "Models in this file are never submitted. Overrides priority list.", + ) + parser.add_argument( + "--priority-mode", + choices=["filter_only", "priority_first"], + help='Priority mode: "filter_only" (default) only evaluates priority models; ' + '"priority_first" evaluates all models but submits priority ones first', + ) + + # Validation options + parser.add_argument( + "--check-hf-exists", + action="store_true", + help="Validate model exists on HuggingFace before submit", + ) + + # Eval parameters (passed to sbatch via env vars) + parser.add_argument( + "--n-concurrent", + type=int, + help=f"Harbor concurrent jobs (default: {DEFAULT_N_CONCURRENT}, preset overrides)", + ) + parser.add_argument( + "--n-attempts", + type=int, + help=f"Retry attempts per task (default: {DEFAULT_N_ATTEMPTS})", + ) + parser.add_argument( + "--gpu-memory-util", + type=float, + help=f"VLLM GPU memory fraction (default: {DEFAULT_GPU_MEMORY_UTIL})", + ) + # Enhancement 1: Unified error threshold (with backward-compat alias) + parser.add_argument( + "--error-threshold", + type=int, + dest="error_threshold", + help=f"Max invalid errors before abort upload (default: {DEFAULT_ERROR_THRESHOLD})", + ) + parser.add_argument( + "--daytona-threshold", + type=int, + dest="error_threshold_compat", + help=f"Alias for --error-threshold (backward compat, default: {DEFAULT_ERROR_THRESHOLD})", + ) + parser.add_argument( + "--vllm-max-retries", + type=int, + help=f"VLLM startup retries (default: {DEFAULT_VLLM_MAX_RETRIES})", + ) + parser.add_argument( + "--agent-parser", + help=f"Agent parser type (default: \"{DEFAULT_AGENT_PARSER}\", use \"xml\" for swebench)", + ) + parser.add_argument( + "--slurm-time", + help=f"SLURM time limit (default: \"{DEFAULT_SLURM_TIME}\")", + ) + parser.add_argument( + "--agent-name", + help=f"Agent name for harbor and DB entries (default: \"{DEFAULT_AGENT_NAME}\")", + ) + parser.add_argument( + "--slurm-partition", + help=f"SLURM partition (default: \"{DEFAULT_SLURM_PARTITION}\")", + ) + parser.add_argument( + "--slurm-account", + help="SLURM account for job submission (e.g. 'reformo'). " + "Overrides the #SBATCH --account in the sbatch script.", + ) + parser.add_argument( + "--tp-size", + type=int, + choices=[1, 2, 4], + help=f"vLLM tensor parallel size — number of GPUs per model " + f"(default: {DEFAULT_TP_SIZE})", + ) + parser.add_argument( + "--dp-size", + type=int, + default=1, + choices=[1, 2, 4, 8], + help="vLLM native data-parallel size — number of model replicas. " + "Total GPUs = tp_size × dp_size. vLLM load-balances requests " + "across replicas internally. (default: 1)", + ) + parser.add_argument( + "--enable-thinking", + action="store_true", + help="Enable thinking blocks for model inference (default: False)", + ) + parser.add_argument( + "--upload-username", + help="Username for DB entries and result uploads (default: current OS user)", + ) + + # v3 Enhancement 2: Per-listener SLURM job throttle + parser.add_argument( + "--max-jobs-submitted", + type=int, + help=f"Per-listener SLURM job limit. Each listener tracks its own " + f"submitted jobs independently (default: {DEFAULT_MAX_JOBS_SUBMITTED})", + ) + + # v3 Enhancement 3: Daytona resource pre-flight check + parser.add_argument( + "--check-daytona-resources", + action="store_true", + help="Query Daytona API for active sandbox count; skip if at limit. " + "Requires DAYTONA_API_KEY in env", + ) + parser.add_argument( + "--daytona-sandbox-limit", + type=int, + help=f"Max expected active sandboxes (default: {DEFAULT_DAYTONA_SANDBOX_LIMIT})", + ) + parser.add_argument( + "--daytona-warning-buffer", + type=float, + help=f"Warn when active sandboxes reach this fraction of limit " + f"(default: {DEFAULT_DAYTONA_WARNING_BUFFER})", + ) + + # v3 Enhancement 5: Timeout-config-sensitive dedup + parser.add_argument( + "--timeout-multiplier", + type=float, + help=f"Harbor timeout multiplier, stored in DB job config " + f"(default: {DEFAULT_TIMEOUT_MULTIPLIER})", + ) + parser.add_argument( + "--timeout-aware", + action="store_true", + help="Dedup jobs by model+benchmark+agent+timeout_multiplier instead " + "of just model+benchmark. Allows same model with different configs", + ) + + # Baseline model configs (per-model vLLM overrides) + parser.add_argument( + "--baseline-model-configs", + help="Path to YAML mapping baseline models to vLLM serving params " + "(e.g., eval/baseline_model_configs.yaml)", + ) + + # Harbor config + parser.add_argument( + "--harbor-config", + help="Path to Harbor YAML config (parsed for timeout_multiplier, " + "resource overrides; passed as EVAL_HARBOR_CONFIG to sbatch)", + ) + + # Pre-download model weights + parser.add_argument( + "--pre-download", + action="store_true", + help="Pre-download all model weights on login node before submitting jobs. " + "Essential for no-internet compute nodes (Leonardo, Jupiter).", + ) + + # Sliding-window batch dependencies + parser.add_argument( + "--batch-size", + type=int, + help="Max concurrent jobs via sliding-window SLURM dependencies. " + "Job N depends on job N-batch_size finishing (afterany), " + "so at most batch-size jobs run at once.", + ) + + # Conda environment selector + parser.add_argument( + "--conda-env", + default="otagent", + help="Conda environment to use for eval jobs. 'otagent2' has vLLM 0.17+ " + "for Qwen3.5 and newer architectures. Available envs are defined in " + "the cluster config YAML. (default: otagent)", + ) + + # v6: Disk-based resume + parser.add_argument( + "--jobs-dir", + nargs="+", + default=None, # resolved in build_config from cluster config / env / fallback + help="Path(s) to eval jobs directories for disk-based resume scanning. " + "Can specify multiple dirs. (default: $EVAL_JOBS_DIR or cluster config paths.eval_jobs_dir)", + ) + parser.add_argument( + "--no-disk-resume", + action="store_true", + help="Disable v6 disk-based resume scanning.", + ) + parser.add_argument( + "--resume-only", + action="store_true", + help="Only submit resume jobs from disk scan, skip all fresh submissions.", + ) + parser.add_argument( + "--force-reeval", + action="store_true", + help="Force re-evaluation: bypass DB status check (submit even if Finished/Started). " + "Use with --priority-file to re-run specific models.", + ) + parser.add_argument( + "--dp-nodes", + type=int, + default=0, + help="Use DP (data-parallel) eval with N SLURM nodes. " + "0 = single-node (default). Each node runs shards_per_node vLLM replicas (4/TP).", + ) + parser.add_argument( + "--inherit-log", + nargs="+", + default=None, + help="Path(s) to previous listener log file(s). Seeds _submitted_jobs with SLURM IDs " + "still active in squeue. Supports multiple logs for chained takeovers. " + "Future --inherit-log on THIS listener's log will also pick up inherited IDs.", + ) + parser.add_argument( + "--submission-delay", + type=float, + default=1.0, + help="Seconds to sleep between sbatch submissions (default: 1.0). " + "Increase to avoid Daytona rate limits (e.g. 30 for 600 sandboxes/min).", + ) + parser.add_argument( + "--stagger-delay", + type=int, + default=0, + help="Minutes between job starts via SLURM 'after:' dependency chain (default: 0 = disabled). " + "Each batch of --chain-batch-size jobs waits N minutes after the previous batch STARTS. " + "Prevents Daytona sandbox burst when many pending jobs start simultaneously. " + "Minimum 1 (SLURM after: granularity is minutes).", + ) + parser.add_argument( + "--chain-batch-size", + type=int, + default=1, + help="Jobs per stagger batch (default: 1). With --stagger-delay=1 --chain-batch-size=10, " + "10 jobs fire immediately, then the next 10 wait 1 minute after the first batch starts. " + "Only meaningful when --stagger-delay > 0.", + ) + parser.add_argument( + "--pack-jobs", + action="store_true", + help="Pack multiple jobs onto the same node. Queries idle nodes and assigns " + "jobs round-robin so that GPUs_PER_NODE / TP_SIZE jobs share one node.", + ) + parser.add_argument( + "--resume-error-threshold", + type=int, + default=10, + help="Min infrastructure errors to trigger resume for completed jobs. " + "(default: 3)", + ) + parser.add_argument( + "--max-resume-count", + type=int, + default=5, + help="Max times to resume a job dir before giving up. " + "(default: 5)", + ) + + # Execution mode + snapshot_group = parser.add_mutually_exclusive_group() + snapshot_group.add_argument( + "--auto-snapshot", + action="store_true", + default=None, + dest="auto_snapshot", + help="Enable Daytona auto_snapshot (overrides YAML config)", + ) + snapshot_group.add_argument( + "--no-auto-snapshot", + action="store_false", + dest="auto_snapshot", + help="Disable Daytona auto_snapshot (overrides YAML config)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview mode, no actual submission (implies --once)", + ) + parser.add_argument( + "--once", + action="store_true", + help="Run single iteration and exit", + ) + parser.add_argument( + "--verbose", "-v", + action="store_true", + help="Enable verbose logging", + ) + + return parser.parse_args() + + +def _env_bool(name: str) -> bool: + """Get boolean from environment variable.""" + return os.getenv(name, "").lower() in ("1", "true", "yes") + + +def build_config(args: argparse.Namespace) -> ListenerConfig: + """Build configuration from args, env vars, cluster config, and preset defaults. + + Resolution order for most fields: + CLI flag > Preset > Cluster config > Hardcoded default + """ + global _CLUSTER_CONFIG, CONDA_ENV_PATHS + + # --- Load cluster config (if provided) --- + cluster_config: Optional[Dict[str, Any]] = None + if args.cluster_config: + cluster_config = load_cluster_config(args.cluster_config) + _CLUSTER_CONFIG = cluster_config + # Override CONDA_ENV_PATHS from cluster config + if cluster_config.get("conda_envs"): + CONDA_ENV_PATHS = cluster_config["conda_envs"] + + # Helper: get cluster config value + def _cc(key: str, default: Any = None) -> Any: + if cluster_config is None: + return default + return cluster_config.get(key, default) + + def _cc_p(key: str, default: Any = None) -> Any: + if cluster_config is None: + return default + return cluster_config.get("paths", {}).get(key, default) + + # Start with preset if specified + preset_config: Dict = {} + if args.preset: + preset_config = PRESETS.get(args.preset, {}) + + # Resolve datasets: CLI > ENV > Preset + datasets_str = args.datasets or os.getenv("EVAL_LISTENER_DATASETS") or "" + if datasets_str: + datasets = parse_datasets(datasets_str) + else: + datasets = preset_config.get("datasets", []) + + if not datasets: + print("ERROR: No datasets specified. Use --datasets, EVAL_LISTENER_DATASETS, or --preset") + sys.exit(2) + + # Resolve sbatch script: CLI > ENV > Preset > Cluster config > Default + sbatch_script = ( + args.sbatch_script + or os.getenv("EVAL_LISTENER_SBATCH") + or preset_config.get("sbatch_script") + or _cc_p("sbatch_script", DEFAULT_SBATCH_SCRIPT) + ) + + # Resolve timing: CLI > ENV > Default + lookback_days = ( + args.lookback_days + if args.lookback_days is not None + else int(os.getenv("EVAL_LISTENER_LOOKBACK_DAYS", str(DEFAULT_LOOKBACK_DAYS))) + ) + check_hours = ( + args.check_hours + if args.check_hours is not None + else float(os.getenv("EVAL_LISTENER_CHECK_HOURS", str(DEFAULT_CHECK_HOURS))) + ) + stale_hours = args.stale_hours if args.stale_hours is not None else DEFAULT_STALE_JOB_HOURS + stale_pending_hours = args.stale_pending_hours if args.stale_pending_hours is not None else DEFAULT_STALE_PENDING_HOURS + + # Resolve log file (CLI --log-dir > ENV > Cluster config > default) + log_dir = Path( + args.log_dir + or os.getenv("EVAL_LISTENER_LOG_DIR") + or _cc_p("listener_logs_dir", DEFAULT_LOG_DIR) + ) + log_dir.mkdir(parents=True, exist_ok=True) + + suffix = preset_config.get("log_suffix", "unified") + current_time = datetime.now().strftime("%Y%m%d_%H%M%S") + + resume_tag = "_resume" if args.resume_only else "" + dryrun_tag = "_dryrun" if args.dry_run else "" + + if args.log_file: + log_file = Path(args.log_file) + else: + log_file = log_dir / f"{suffix}_eval_listener_v6{resume_tag}{dryrun_tag}_{current_time}.log" + + # Resolve priority file: CLI > ENV + priority_file = args.priority_file or os.getenv("EVAL_LISTENER_PRIORITY_FILE") + priority_models = load_priority_models(priority_file) + + # Resolve blacklist file: CLI > ENV + blacklist_file = args.blacklist_file or os.getenv("EVAL_LISTENER_BLACKLIST_FILE") + blacklisted_models = load_blacklist(blacklist_file) + + # Resolve priority mode: CLI > ENV > default + priority_mode = ( + args.priority_mode + or os.getenv("EVAL_LISTENER_PRIORITY_MODE") + or "filter_only" + ) + + # Resolve boolean flags: CLI > ENV > Preset + require_priority = args.require_priority_list or _env_bool("EVAL_LISTENER_REQUIRE_PRIORITY_LIST") + dry_run = args.dry_run or _env_bool("EVAL_LISTENER_DRY_RUN") + check_hf_exists = ( + args.check_hf_exists + or _env_bool("EVAL_LISTENER_CHECK_HF_EXISTS") + or preset_config.get("check_hf_exists", False) + ) + + # Resolve sbatch parameters: CLI > Preset > Cluster config > Default + def _resolve(cli_val, preset_key: str, default): + if cli_val is not None: + return cli_val + return preset_config.get(preset_key, default) + + n_concurrent = _resolve(args.n_concurrent, "n_concurrent", DEFAULT_N_CONCURRENT) + n_attempts = _resolve(args.n_attempts, "n_attempts", DEFAULT_N_ATTEMPTS) + gpu_memory_util = _resolve(args.gpu_memory_util, "gpu_memory_util", DEFAULT_GPU_MEMORY_UTIL) + + # Enhancement 1: Resolve error_threshold with backward compat + error_threshold_cli = args.error_threshold + if error_threshold_cli is None: + error_threshold_cli = getattr(args, 'error_threshold_compat', None) + error_threshold = _resolve(error_threshold_cli, "error_threshold", DEFAULT_ERROR_THRESHOLD) + + vllm_max_retries = _resolve(args.vllm_max_retries, "vllm_max_retries", DEFAULT_VLLM_MAX_RETRIES) + agent_parser = _resolve(args.agent_parser, "agent_parser", DEFAULT_AGENT_PARSER) + slurm_time = _resolve(args.slurm_time, "slurm_time", _cc("slurm_time", DEFAULT_SLURM_TIME)) + agent_name = _resolve(args.agent_name, "agent_name", DEFAULT_AGENT_NAME) + slurm_partition = _resolve(args.slurm_partition, "slurm_partition", _cc("slurm_partition", DEFAULT_SLURM_PARTITION)) + slurm_account = _resolve(args.slurm_account, "slurm_account", _cc("slurm_account", DEFAULT_SLURM_ACCOUNT)) + tp_size = _resolve(args.tp_size, "tp_size", DEFAULT_TP_SIZE) + dp_size = args.dp_size if args.dp_size else 1 + enable_thinking = args.enable_thinking or preset_config.get("enable_thinking", DEFAULT_ENABLE_THINKING) + + # Resolve upload_username: CLI > ENV > current OS user + upload_username = ( + args.upload_username + or os.getenv("EVAL_UPLOAD_USERNAME") + or getpass.getuser() + ) + + # Enhancement 2: SLURM throttle + max_jobs_submitted = ( + args.max_jobs_submitted + if args.max_jobs_submitted is not None + else int(os.getenv("EVAL_LISTENER_MAX_JOBS", str(DEFAULT_MAX_JOBS_SUBMITTED))) + ) + + # Enhancement 3: Daytona resource check + check_daytona = args.check_daytona_resources + daytona_sandbox_limit = ( + args.daytona_sandbox_limit + if args.daytona_sandbox_limit is not None + else DEFAULT_DAYTONA_SANDBOX_LIMIT + ) + daytona_warning_buffer = ( + args.daytona_warning_buffer + if args.daytona_warning_buffer is not None + else DEFAULT_DAYTONA_WARNING_BUFFER + ) + + # Enhancement 5: Timeout-config-sensitive dedup + timeout_multiplier = ( + args.timeout_multiplier + if args.timeout_multiplier is not None + else DEFAULT_TIMEOUT_MULTIPLIER + ) + timeout_aware = args.timeout_aware + + # auto_snapshot: CLI > Preset > None (use YAML default) + auto_snapshot = args.auto_snapshot + if auto_snapshot is None: + auto_snapshot = preset_config.get("auto_snapshot") + + # Config YAML: Preset > Default + config_yaml = preset_config.get("config_yaml", "dcagent_eval_config.yaml") + + # Harbor config (parse eval-relevant fields for config-aware dedup) + harbor_config = args.harbor_config or preset_config.get("harbor_config") + eval_config = parse_harbor_eval_config(harbor_config) + + # Baseline model configs for per-model vLLM overrides + baseline_model_configs_path = args.baseline_model_configs + + # Pre-download model weights + pre_download = args.pre_download + + # Sliding-window batch dependencies + batch_size = args.batch_size + + # Conda env selector + conda_env = args.conda_env + + # Resolve jobs_dirs: CLI > ENV > Cluster config > Fallback + fallback_jobs_dir = _cc_p("eval_jobs_dir", _FALLBACK_EVAL_JOBS_DIR) + jobs_dirs = args.jobs_dir or [os.environ.get("EVAL_JOBS_DIR", fallback_jobs_dir)] + + # Resolve DP sbatch script: Cluster config > Default + dp_sbatch_script = _cc_p("dp_sbatch_script", "eval/unified_eval_harbor_dp.sbatch") + + return ListenerConfig( + datasets=datasets, + sbatch_script=sbatch_script, + log_file=log_file, + lookback_days=lookback_days, + check_interval_hours=check_hours, + stale_job_hours=stale_hours, + stale_pending_hours=stale_pending_hours, + priority_file=priority_file, + require_priority_list=require_priority, + priority_models=priority_models, + priority_mode=priority_mode, + check_hf_exists=check_hf_exists, + dry_run=dry_run, + run_once=args.once, + verbose=args.verbose, + # Sbatch parameters + n_concurrent=n_concurrent, + n_attempts=n_attempts, + gpu_memory_util=gpu_memory_util, + error_threshold=error_threshold, + vllm_max_retries=vllm_max_retries, + agent_parser=agent_parser, + slurm_time=slurm_time, + enable_thinking=enable_thinking, + agent_name=agent_name, + slurm_partition=slurm_partition, + slurm_account=slurm_account, + tp_size=tp_size, + dp_size=dp_size, + upload_username=upload_username, + # Enhancement 2 + max_jobs_submitted=max_jobs_submitted, + # Enhancement 3 + check_daytona_resources=check_daytona, + daytona_sandbox_limit=daytona_sandbox_limit, + daytona_warning_buffer=daytona_warning_buffer, + # Enhancement 5 + timeout_multiplier=timeout_multiplier, + timeout_aware=timeout_aware, + config_yaml=config_yaml, + auto_snapshot=auto_snapshot, + blacklist_file=blacklist_file, + blacklisted_models=blacklisted_models, + # New features + baseline_model_configs=baseline_model_configs_path, + harbor_config=harbor_config, + eval_config=eval_config, + pre_download=pre_download, + batch_size=batch_size, + conda_env=conda_env, + # v6: Disk-based resume + jobs_dirs=jobs_dirs, + enable_disk_resume=not args.no_disk_resume, + resume_infra_error_threshold=args.resume_error_threshold, + max_resume_count=args.max_resume_count, + force_reeval=args.force_reeval, + resume_only=args.resume_only, + submission_delay=args.submission_delay, + stagger_delay=max(args.stagger_delay, 0), + chain_batch_size=max(args.chain_batch_size, 1), + pack_jobs=args.pack_jobs, + dp_nodes=args.dp_nodes, + dp_sbatch_script=dp_sbatch_script, + inherit_log=args.inherit_log, + # Cluster config + cluster_config=cluster_config, + ) + + +# ---------- Main ---------- +def main() -> None: + global _VERBOSE + _load_secrets() + args = parse_args() + config = build_config(args) + _VERBOSE = config.verbose + if config.cluster_config: + log(f"[v6] Cluster config: {config.cluster_config.get('cluster_name', '?')}") + listener = EvalListener(config) + listener.run() + + +if __name__ == "__main__": + main() diff --git a/scripts/database/batch_upload_eval.py b/scripts/database/batch_upload_eval.py new file mode 100644 index 000000000..9a344a52f --- /dev/null +++ b/scripts/database/batch_upload_eval.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Batch upload eval job results to Supabase + HuggingFace. + +Supports two modes: +1. Explicit job dirs: upload specific completed jobs +2. Auto-detect overlong: scan a jobs directory for timed-out jobs + +Usage: + # Upload specific job dirs (normal eval) + python scripts/database/batch_upload_eval.py \ + jobs/terminal_bench_2_model_A_20260407_* \ + jobs/dev_set_v2_model_B_20260407_* + + # Upload as overlong (timed-out jobs) + python scripts/database/batch_upload_eval.py --overlong \ + jobs/terminal_bench_2_slow_model_* + + # Auto-detect overlong jobs in a directory + python scripts/database/batch_upload_eval.py --auto-detect-overlong \ + --jobs-dir /path/to/jobs + + # Parallel upload with HF traces + python scripts/database/batch_upload_eval.py -p 8 jobs/swebench_* + + # Skip HF upload (DB only) + python scripts/database/batch_upload_eval.py --skip-hf jobs/swebench_* + + # Dry run + python scripts/database/batch_upload_eval.py --dry-run jobs/swebench_* +""" + +import argparse +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +_repo_root = Path(__file__).resolve().parents[2] +if str(_repo_root) not in sys.path: + sys.path.insert(0, str(_repo_root)) + +_db_path = _repo_root / "database" / "unified_db" +if str(_db_path) not in sys.path: + sys.path.insert(0, str(_db_path)) + +DEFAULT_JOBS_DIR = _repo_root / "jobs" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def parse_iso(ts_str): + if not ts_str: + return None + try: + dt = datetime.fromisoformat(ts_str) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except (ValueError, TypeError): + return None + + +def derive_hf_repo_id(job_dir): + """Derive HF repo ID from job directory name.""" + name = Path(job_dir).name + sanitized = name.replace("@", "-").replace(" ", "-") + return f"DCAgent2/{sanitized}-traces" + + +def get_job_info(job_dir): + """Extract basic info from a job directory.""" + job_dir = Path(job_dir) + result_file = job_dir / "result.json" + if not result_file.exists(): + return None + + try: + d = json.load(open(result_file)) + except (json.JSONDecodeError, OSError): + return None + + n_total = d.get("n_total_trials", 0) + n_trials = d.get("stats", {}).get("n_trials", 0) + finished = d.get("finished_at") is not None + + # Accuracy + accuracy = None + for eval_data in d.get("stats", {}).get("evals", {}).values(): + metrics = eval_data.get("metrics", []) + if metrics and isinstance(metrics, list): + mr = metrics[0].get("mean_reward") + if mr is not None: + accuracy = mr + + # Model from meta.env + model = None + meta_env = job_dir / "meta.env" + if meta_env.exists(): + with open(meta_env) as f: + for line in f: + if line.startswith("MODEL="): + model = line.strip().split("=", 1)[1] + break + + # Benchmark from trial source + benchmark = None + for entry in sorted(job_dir.iterdir()): + if entry.is_dir() and "__" in entry.name: + tr = entry / "result.json" + if tr.exists(): + try: + td = json.load(open(tr)) + source = td.get("source", "") + if source: + for pfx in ["DCAgent2_", "DCAgent_"]: + if source.startswith(pfx): + source = source[len(pfx):] + break + benchmark = source + except (json.JSONDecodeError, OSError): + pass + break + + model_short = (model.split("/")[-1] if model and "/" in model else model) or job_dir.name + + return { + "dir": job_dir, + "model": model or job_dir.name, + "model_short": model_short, + "benchmark": benchmark or "unknown", + "n_trials": n_trials, + "n_total": n_total, + "finished": finished, + "accuracy": accuracy, + } + + +def detect_overlong_jobs(jobs_dir, min_elapsed_h=20, benchmark_filter=None): + """Find jobs that timed out (no finished_at, elapsed > min_elapsed_h).""" + results = [] + jobs_dir = Path(jobs_dir) + + for entry in sorted(jobs_dir.iterdir()): + if not entry.is_dir(): + continue + info = get_job_info(entry) + if not info or info["finished"] or info["n_trials"] == 0: + continue + + # Check elapsed + try: + d = json.load(open(entry / "result.json")) + except: + continue + started = parse_iso(d.get("started_at")) + if not started: + continue + + latest = None + for te in entry.iterdir(): + if not te.is_dir() or "__" not in te.name: + continue + tr = te / "result.json" + if tr.exists(): + try: + td = json.load(open(tr)) + fa = td.get("finished_at") + if fa: + ft = parse_iso(fa) + if ft and (latest is None or ft > latest): + latest = ft + except: + pass + + if latest is None: + continue + elapsed_h = (latest - started).total_seconds() / 3600 + if elapsed_h < min_elapsed_h: + continue + + if benchmark_filter and benchmark_filter.lower() not in info["benchmark"].lower(): + continue + + info["elapsed_h"] = elapsed_h + results.append(info) + + # Deduplicate: same model+benchmark, keep best progress + seen = {} + for r in results: + key = (r["model"], r["benchmark"]) + if key not in seen or r["n_trials"] > seen[key]["n_trials"]: + seen[key] = r + return sorted(seen.values(), key=lambda r: (r["benchmark"], r["model"])) + + +def upload_one(job_dir, benchmark_name=None, skip_hf=False, is_overlong=False): + """Upload a single job. Returns (success, label, message).""" + from hpc.launch_utils import sync_eval_to_database + + job_dir = Path(job_dir) + label = job_dir.name + hf_repo_id = None if skip_hf else derive_hf_repo_id(job_dir) + + try: + result = sync_eval_to_database( + job_dir=job_dir, + error_mode="skip_on_error", + benchmark_name=benchmark_name, + register_benchmark=True, + forced_update=True, + is_overlong=is_overlong, + hf_repo_id=hf_repo_id, + hf_token=os.environ.get("HF_TOKEN"), + ) + if result.get("success"): + job_id = result.get("job_id") + n_trials = result.get("n_trials_uploaded", 0) + hf_url = result.get("hf_dataset_url", "n/a") + return True, label, f"job_id={job_id}, trials={n_trials}, hf={hf_url}" + else: + return False, label, result.get("error", "unknown") + except Exception as e: + return False, label, str(e) + + +def run_uploads(jobs_to_upload, n_workers, skip_hf, is_overlong): + """Run uploads sequentially or in parallel.""" + n_workers = min(n_workers, len(jobs_to_upload)) + + if n_workers <= 1: + success = failed = 0 + for job in jobs_to_upload: + ok, label, msg = upload_one( + job["dir"], job.get("benchmark"), skip_hf, is_overlong) + print(f" [{'OK' if ok else 'FAIL'}] {label}: {msg}") + if ok: + success += 1 + else: + failed += 1 + else: + from concurrent.futures import ThreadPoolExecutor, as_completed + success = failed = 0 + + def _worker(job): + return upload_one(job["dir"], job.get("benchmark"), skip_hf, is_overlong) + + with ThreadPoolExecutor(max_workers=n_workers) as pool: + futures = {pool.submit(_worker, j): j for j in jobs_to_upload} + for future in as_completed(futures): + ok, label, msg = future.result() + print(f" [{'OK' if ok else 'FAIL'}] {label}: {msg}") + if ok: + success += 1 + else: + failed += 1 + + print(f"\nDone: {success} uploaded, {failed} failed") + + +def main(): + parser = argparse.ArgumentParser( + description="Batch upload eval results to Supabase + HuggingFace.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Mode + parser.add_argument("job_dirs", nargs="*", type=Path, + help="Job directories to upload (glob-friendly)") + parser.add_argument("--auto-detect-overlong", action="store_true", + help="Auto-detect overlong jobs (>20h elapsed, incomplete)") + parser.add_argument("--jobs-dir", type=Path, default=DEFAULT_JOBS_DIR, + help="Jobs directory for --auto-detect-overlong mode") + + # Upload options + parser.add_argument("--overlong", action="store_true", + help="Mark uploaded jobs as overlong (is_overlong=true)") + parser.add_argument("--skip-hf", action="store_true", + help="Skip HuggingFace upload (DB only)") + parser.add_argument("--force", action="store_true", + help="Override quality gates (low accuracy, incomplete trials)") + parser.add_argument("--parallel", "-p", type=int, default=1, + help="Number of parallel upload workers (default: 1)") + + # Filters (for auto-detect mode) + parser.add_argument("--benchmark", "-b", type=str, default=None, + help="Filter by benchmark substring") + parser.add_argument("--min-elapsed", type=float, default=20, + help="Min elapsed hours for overlong detection (default: 20)") + + # Safety + parser.add_argument("--dry-run", action="store_true", + help="Show what would be uploaded without uploading") + + args = parser.parse_args() + + if args.force: + os.environ["EVAL_UPLOAD_FORCE"] = "1" + + if args.auto_detect_overlong: + # Auto-detect mode + print(f"Scanning {args.jobs_dir} for overlong jobs (>{args.min_elapsed}h elapsed, incomplete)...") + jobs_to_upload = detect_overlong_jobs(args.jobs_dir, args.min_elapsed, args.benchmark) + is_overlong = True + print(f"Found {len(jobs_to_upload)} overlong model/benchmark pairs\n") + elif args.job_dirs: + # Explicit job dirs mode + jobs_to_upload = [] + for jd in args.job_dirs: + jd = jd.resolve() + if not jd.exists(): + print(f" [SKIP] {jd}: directory not found") + continue + info = get_job_info(jd) + if info is None: + print(f" [SKIP] {jd}: no result.json") + continue + jobs_to_upload.append(info) + is_overlong = args.overlong + print(f"Found {len(jobs_to_upload)} job(s) to upload\n") + else: + parser.print_help() + return + + if not jobs_to_upload: + print("Nothing to upload.") + return + + # Display table + hdr_elapsed = "Elapsed" if args.auto_detect_overlong else "" + print(f" {'Model':<55} {'Benchmark':<25} {'Progress':>10} {'Acc':>7} {hdr_elapsed:>8}") + print(" " + "-" * 110) + for r in jobs_to_upload: + acc = f"{r['accuracy']:.1%}" if r['accuracy'] is not None else "N/A" + elapsed = f"{r.get('elapsed_h', 0):.1f}h" if args.auto_detect_overlong else "" + print(f" {r['model_short']:<55} {r['benchmark']:<25} {r['n_trials']:>4}/{r['n_total']:<4} {acc:>7} {elapsed:>8}") + + mode = "overlong" if is_overlong else "normal" + hf = "skip" if args.skip_hf else "enabled" + print(f"\n Mode: {mode} | HF: {hf} | Workers: {args.parallel}") + + if args.dry_run: + print("\n Dry run — pass without --dry-run to upload.") + return + + print() + run_uploads(jobs_to_upload, args.parallel, args.skip_hf, is_overlong) + + +if __name__ == "__main__": + main()