Master

.env.example

@@ -1,40 +1,156 @@
+# ============================================================================
+# WildClawBench (LiteLLM/Bedrock fork) — environment template
+# ----------------------------------------------------------------------------
+# Copy to .env and fill in your values:   cp .env.example .env
+# .env is gitignored; NEVER put real secrets in this .env.example (it is tracked).
+# Variable names below are exactly what the code reads (config.py / run_batch.py
+# / docker_utils.py). KENSEI_* names are the primary keys; generic aliases noted.
+# ============================================================================
+
+# ===== Container / runtime =====
 DOCKER_IMAGE=wildclawbench-ubuntu:v1.3
 GATEWAY_PORT=18789
 TMP_WORKSPACE=/tmp_workspace
-
 TASKS_SUBDIR=tasks
 OUTPUT_SUBDIR=output
-
-DEFAULT_MODEL=openrouter/xxx
-
 DEFAULT_PARALLEL=1
-
+# Default model. In LiteLLM mode use a sidecar id (claude-opus-4.7 / gpt-5.5);
+# otherwise an OpenRouter string (e.g. openrouter/anthropic/claude-sonnet-4.6).
+DEFAULT_MODEL=openrouter/anthropic/claude-sonnet-4.6
+
+# ===== Inner proxy (optional; leave blank for none -> NO_PROXY defaults to *) =====
 HTTP_PROXY_INNER=
 HTTPS_PROXY_INNER=
 NO_PROXY_INNER=
 
-OPENROUTER_BASE_URL='https://openrouter.ai/api/v1'
-JUDGE_MODEL=openai/gpt-5.4
+# ===== OpenRouter (fallback routing when LiteLLM is not enabled) =====
 OPENROUTER_API_KEY=
-BRAVE_API_KEY=
-
-# Using a Custom Model Endpoint
+OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
+# LLM judge used by some tasks' automated checks
+JUDGE_MODEL=openai/gpt-5.4
+# Brave key is passed to the container; 'placeholder' prevents a gateway crash
+BRAVE_API_KEY=placeholder
+# For --models-config files that contain ${MY_PROXY_API_KEY}
 # MY_PROXY_API_KEY=
 
-# Lobster profile env keys (add values here for skills that need them)
-# GEMINI_API_KEY=
-# FIRECRAWL_API_KEY=
-# EXA_API_KEY=
+# ============================================================================
+# LiteLLM / Bedrock routing (openclaw backend)
+# If Bedrock (ARN + bearer token) OR OpenAI is set, openclaw auto-routes through
+# a shared LiteLLM sidecar instead of OpenRouter. Force with --litellm / --no-litellm.
+# ============================================================================
+# --- AWS Bedrock (Claude via Bedrock API key / bearer token) ---
+# alias: BEDROCK_MODEL_ARN
+KENSEI_BEDROCK_MODEL_ARN=          # arn:aws:bedrock:<region>:<acct>:application-inference-profile/<id>  (Opus 4.6/4.7 profile; also used by test-gen)
+# Sonnet 4.6 inference profile (used when claude-sonnet-4-6 routes through the
+# sidecar). Same application-inference-profile ARN format as above.
+# alias: BEDROCK_SONNET_ARN
+KENSEI_BEDROCK_SONNET_ARN=         # arn:aws:bedrock:ap-south-1:<acct>:application-inference-profile/<id>
+# alias: AWS_REGION
+KENSEI_AWS_REGION=ap-south-1
+
+# --- Claude Max OAuth subscription (per-user, per-laptop) ---
+# These power `wcb login` so trajectories + the Sonnet judge run on YOUR own
+# Claude Max subscription instead of a metered API key. Each teammate fills in
+# THEIR OWN values here on THEIR OWN machine — never commit real secrets (.env
+# is gitignored; only this .env.example template is tracked). No KENSEI_ prefix
+# fallback exists for these four; they are read verbatim.
+# WCB_USE_CLAUDE_OAUTH=1
+# WCB_CC_ACCOUNT_POOL=/Users/<you>/.wcb/oauth_pool/account_a.json
+# WCB_CC_BRIDGE_SECRET=<openssl rand -hex 32>
+# WCB_CC_STUB_KEY=sk-wcb-oauth-stub
 
-# Task env
-# 01_Productivity_Flow
+# --- LLM-council judge ARNs (rubric grading) ---
+# The 3-model council (Sonnet 4.6 + GLM 5 + Kimi K2.5) runs in parallel for
+# rubric grading. Each member is a full bedrock/<application-inference-profile>
+# ARN, read ONLY from these vars (no defaults in source) — set them to YOUR
+# account's profiles. The VARIABLE NAME (not the ARN) decides the model family,
+# so pricing/evidence-budget/cache rules live in code keyed by family and the
+# opaque profile-id is NOT load-bearing — rotate the ARNs freely (e.g. monthly)
+# and costs still resolve correctly. Per-model regions: GLM is us-east-1,
+# Sonnet + Kimi are ap-south-1.
+# JUDGE_COUNCIL_SONNET_ARN=bedrock/arn:aws:bedrock:ap-south-1:<acct>:application-inference-profile/<id>
+# JUDGE_COUNCIL_GLM_ARN=bedrock/arn:aws:bedrock:us-east-1:<acct>:application-inference-profile/<id>
+# JUDGE_COUNCIL_KIMI_ARN=bedrock/arn:aws:bedrock:ap-south-1:<acct>:application-inference-profile/<id>
+# For a custom roster, set JUDGE_COUNCIL_MEMBERS to a comma-separated list of
+# family=arn entries (overrides the three vars above), e.g.:
+# JUDGE_COUNCIL_MEMBERS=sonnet=bedrock/arn:...:profile/abc,glm=bedrock/arn:...:profile/def
+# alias: AWS_BEARER_TOKEN_BEDROCK
+KENSEI_AWS_BEARER_TOKEN=          # your Bedrock API key (bearer token)
 
-# 02_Code_Intelligence
+# --- OpenAI (gpt-5.5 via LiteLLM) ---
+# alias: OPENAI_API_KEY
+KENSEI_OPENAI_API_KEY=
 
-# 03_Social_Interaction
+# --- Whisper / audio transcription (via LiteLLM /v1/audio/transcriptions) ---
+# Dedicated key for the whisper-1 (and gpt-4o-transcribe) routes the
+# audio-extract skill uses. Optional: if left empty, transcription falls back
+# to KENSEI_OPENAI_API_KEY above. Must be an OpenAI key with audio access.
+# alias: OPENAI_WHISPER_API_KEY
+KENSEI_OPENAI_WHISPER_API_KEY=
 
-# 04_Search_Retrieval
+# --- Meta vendor model (internal Llama API, OpenAI-compatible, via LiteLLM) ---
+# When KENSEI_META_API_KEY *and* KENSEI_META_MODEL are both set, the sidecar
+# registers an OpenAI-compatible upstream pointed at KENSEI_META_BASE_URL and
+# LiteLLM auto-enables. The harness-facing model id IS KENSEI_META_MODEL, so
+# select it with `--model <that id>`. Per the vendor onboarding guide, ALL
+# inference params are kept at their defaults (no reasoning_effort/temperature/
+# top_p/top_k overrides); litellm_settings.drop_params strips anything callers
+# emit that the relay doesn't support (no structured output / parallel tools).
+# alias: META_API_KEY
+KENSEI_META_API_KEY=
+# alias: META_MODEL — must match exactly what the relay's /models returns
+KENSEI_META_MODEL=
+# alias: META_API_BASE_URL (default below; note the required /v1 suffix)
+KENSEI_META_BASE_URL=https://api.ai.meta.com/v1
 
-# 05_Creative_Synthesis
+# --- LiteLLM proxy (sidecar) ---
+# aliases: KENSEI3_LITELLM_MASTER_KEY, LITELLM_MASTER_KEY
+KENSEI_LITELLM_MASTER_KEY=sk-talos-litellm
+# alias: LITELLM_PORT
+KENSEI3_LITELLM_PORT=4000
+# Model id used when --litellm is active and --model is not a sidecar id
+LITELLM_DEFAULT_MODEL=claude-opus-4.7
 
-# 06_Safety_Alignment
+# ===== Cost-estimate fallback rates (Codex / Claude Code harnesses) =====
+# The Codex and Claude Code harnesses run outside the LiteLLM sidecar, so
+# litellm.completion_cost() never sees their tokens. When the harness transcript
+# does NOT report a provider cost, cost_usd is ESTIMATED from these per-million-
+# token (MTok) rates. If left unset they default to 0 -> cost_usd is recorded as
+# $0 for those runs. Set them to the published rates of the model you are running.
+# Codex estimator subtracts cache_read+cache_write from input before pricing;
+# Claude Code estimator prices the full input_tokens (no cache subtraction).
+# Example (Claude Opus 4.6/4.7): input 5, output 25, cache_read 0.5, cache_write 6.25
+# CODEX_INPUT_PRICE_PER_MTOK=
+# CODEX_OUTPUT_PRICE_PER_MTOK=
+# CODEX_CACHE_READ_PRICE_PER_MTOK=
+# CODEX_CACHE_WRITE_PRICE_PER_MTOK=
+# CLAUDECODE_INPUT_PRICE_PER_MTOK=
+# CLAUDECODE_OUTPUT_PRICE_PER_MTOK=
+# CLAUDECODE_CACHE_READ_PRICE_PER_MTOK=
+# CLAUDECODE_CACHE_WRITE_PRICE_PER_MTOK=
+
+# ===== Mock-API stack (101 services under environment/) =====
+# Enable per run with --mock-stack ; agents reach each API at
+# http://<mock-container>:<port> via injected *_API_URL env vars (builds kensei3-mocks:v1).
+
+# ===== Skills (sourced from environment/skills/: *-connector + media skills) =====
+# Required-API connectors auto-inject per task. These inject into EVERY task:
+WILDCLAW_DEFAULT_SKILLS=video-frames,pdf-extract,audio-extract
+# WILDCLAW_SKILLS_DIR=             # override skill catalog dir (default: environment/skills)
+
+# ===== Harbor bundle (optional) =====
+# MIN_HARBOR_SCORE=                # skip bundle export below this score
+
+# ===== S3 trajectory-media upload (optional; default keeps media as local files) =====
+UPLOAD_MEDIA_TO_S3=false
+S3_BUCKET=
+S3_PREFIX=WildClaw
+S3_REGION=us-east-1
+# aliases: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
+KENSEI_S3_ACCESS_KEY_ID=
+KENSEI_S3_SECRET_ACCESS_KEY=
+
+# ===== Lobster profile env keys (for skills that need API keys) =====
+# GEMINI_API_KEY=
+# FIRECRAWL_API_KEY=
+# EXA_API_KEY=

.gitignore

@@ -8,6 +8,10 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+.venv/
+venv/
+work/
+
 
 *.tar
 *.zip
@@ -25,12 +29,7 @@ __pycache__/
 Images/
 _workspace/
 workspace/
-script/
 tools/
-output/
-output_sum/
-output_qwen/
-output*/
 .cache/
 .hfd/
 hfd.sh
@@ -39,4 +38,59 @@ grocery/
 cookies.txt
 output.rar
 output_qwen.rar
-*.rar
+*.rar
+
+# Benchmark run output (trajectories, scores, artifacts) — large, machine-specific.
+output/
+# Repackaged Harbor bundle (derived from output/; large, machine-specific).
+output_bundle/
+
+# Local review / audit documents.
+review_docs/
+
+# Per-run telemetry written by run.sh tee — local, timestamped, machine-specific.
+logs/
+
+# macOS Finder duplicate-on-copy directories ("foo copy/", "foo copy 2/").
+*\ copy/
+*\ copy\ [0-9]*/
+
+# Audit deliverables (findings.json, REPORT.md, BUGS.md, REMEDIATION.md, logs/) — local-only, machine-specific.
+audit/
+
+# macOS Finder duplicate-on-copy of audit/ (created when the folder is duplicated in Finder).
+/audit 2/
+
+# Operator/maintainer knowledge bases — NOT loaded by the harness at runtime.
+# Persona-side AGENTS.md files under input/<task>/persona/ ARE loaded into the
+# agent container via docker cp and must remain tracked; only the source-tree
+# variants below are ignored.
+/AGENTS.md
+/environment/AGENTS.md
+/src/agents/AGENTS.md
+/src/utils/AGENTS.md
+/tests/AGENTS.md
+
+# Local-only triage / verification notes that aren't required to run the harness.
+CRUCIBLE.md
+SIX_CHECK_VERIFICATION_REPORT.md
+
+# Never commit .env or any timestamped backup of it — secrets live here.
+.env
+.env.*
+.env.backup*
+
+# Curated/published trajectories (derived from output/ + input/; large, machine-specific).
+trajectories/
+
+# Node tooling pulled in locally — not part of the benchmark.
+node_modules/
+package.json
+package-lock.json
+
+# SQLite WAL/SHM side-files — transient, never meant to be tracked.
+*.db-shm
+*.db-wal
+
+# Local helper task list generated by tooling.
+tasks_all.txt

.opencode/command/audit.md

@@ -0,0 +1,19 @@
+---
+description: Run the CRUCIBLE audit gate (run instruments → review → verify)
+agent: build
+---
+
+The contract is `@CRUCIBLE.md`. The run playbook is `@audit/README.md`. Do not restate the contract.
+
+Phase 1 — generate evidence:
+!`uv run --project audit python audit/audit.py run`
+
+Phase 2 — review: read `@REVIEW.md` as your instructions and `@audit/evidence.yaml` as the
+ONLY source of instrumented evidence. Write `findings.yaml` and `REPORT.md` at the project
+root (strip any `_template`/`_example` keys; `REPORT.md` is the single human report and its
+**Bug Tickets** section holds the JIRA-style tickets).
+
+Phase 3 — gate:
+!`uv run --project audit python audit/audit.py verify --findings findings.yaml --context audit/evidence.yaml`
+
+Findings are UNGATED until `verify` exits 0.

.opencode/skill/audit-gate/SKILL.md

@@ -0,0 +1,12 @@
+---
+name: audit-gate
+description: Use when auditing this project's deliverables against the CRUCIBLE gate — run instruments, review evidence, and verify findings fail-closed.
+---
+
+This skill defers entirely to the contract in `CRUCIBLE.md` and the run playbook in
+`audit/README.md`. It never restates the axes, severity scale, or disposition vocabulary.
+
+The two reviewer artifacts are `findings.yaml` and `REPORT.md` (project root). The only
+admissible evidence source is `audit/evidence.yaml`. The gate is
+`uv run --project audit python audit/audit.py verify --findings findings.yaml --context audit/evidence.yaml`,
+which exits 0 only when the report is internally honest and every required instrument ran clean.