diff --git a/llm_configs/claude-4.7-opus-no-thinking.json b/llm_configs/claude-4.7-opus-no-thinking.json new file mode 100644 index 00000000..c11bacfe --- /dev/null +++ b/llm_configs/claude-4.7-opus-no-thinking.json @@ -0,0 +1,10 @@ +{ + "model_name": "claude-opus-4-7", + "provider": "anthropic", + "settings": { + "max_tokens": 128000, + "anthropic_thinking": { + "type": "disabled" + } + } +} diff --git a/llm_configs/claude-4.8-opus-no-thinking.json b/llm_configs/claude-4.8-opus-no-thinking.json new file mode 100644 index 00000000..30120439 --- /dev/null +++ b/llm_configs/claude-4.8-opus-no-thinking.json @@ -0,0 +1,10 @@ +{ + "model_name": "claude-opus-4-8", + "provider": "anthropic", + "settings": { + "max_tokens": 128000, + "anthropic_thinking": { + "type": "disabled" + } + } +} diff --git a/llm_configs/deepseek-v4-flash-high.json b/llm_configs/deepseek-v4-flash-high.json new file mode 100644 index 00000000..bd7a471a --- /dev/null +++ b/llm_configs/deepseek-v4-flash-high.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "high", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-flash-max.json b/llm_configs/deepseek-v4-flash-max.json new file mode 100644 index 00000000..cd08abf3 --- /dev/null +++ b/llm_configs/deepseek-v4-flash-max.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "xhigh", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-flash-no-thinking.json b/llm_configs/deepseek-v4-flash-no-thinking.json new file mode 100644 index 00000000..b83e152b --- /dev/null +++ b/llm_configs/deepseek-v4-flash-no-thinking.json @@ -0,0 +1,22 @@ +{ + "model_name": "deepseek/deepseek-v4-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-pro-high.json b/llm_configs/deepseek-v4-pro-high.json new file mode 100644 index 00000000..9fde30c4 --- /dev/null +++ b/llm_configs/deepseek-v4-pro-high.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "high", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-pro-max.json b/llm_configs/deepseek-v4-pro-max.json new file mode 100644 index 00000000..3add1a7a --- /dev/null +++ b/llm_configs/deepseek-v4-pro-max.json @@ -0,0 +1,23 @@ +{ + "model_name": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "effort": "xhigh", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/deepseek-v4-pro-no-thinking.json b/llm_configs/deepseek-v4-pro-no-thinking.json new file mode 100644 index 00000000..0d0c462e --- /dev/null +++ b/llm_configs/deepseek-v4-pro-no-thinking.json @@ -0,0 +1,22 @@ +{ + "model_name": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "allow", + "order": ["deepseek"], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/gemini-3.5-flash-high.json b/llm_configs/gemini-3.5-flash-high.json new file mode 100644 index 00000000..7ba223e7 --- /dev/null +++ b/llm_configs/gemini-3.5-flash-high.json @@ -0,0 +1,10 @@ +{ + "model_name": "gemini-3.5-flash", + "provider": "google", + "settings": { + "google_thinking_config": { + "include_thoughts": true, + "thinking_level": "high" + } + } +} diff --git a/llm_configs/gemma-4-31b-it.json b/llm_configs/gemma-4-31b-it.json new file mode 100644 index 00000000..3d392cfa --- /dev/null +++ b/llm_configs/gemma-4-31b-it.json @@ -0,0 +1,22 @@ +{ + "model_name": "google/gemma-4-31B-it", + "provider": "vllm-chat", + "base_url": "http://localhost:18765/v1", + "api_key": "ale-bench-eval", + "settings": { + "temperature": 1.0, + "top_p": 0.95, + "extra_body": { + "top_k": 64, + "chat_template_kwargs": { + "enable_thinking": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/gpt-5.5-high.json b/llm_configs/gpt-5.5-high.json new file mode 100644 index 00000000..cfdfeef2 --- /dev/null +++ b/llm_configs/gpt-5.5-high.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "high", + "openai_reasoning_summary": "detailed" + } +} diff --git a/llm_configs/gpt-5.5-medium.json b/llm_configs/gpt-5.5-medium.json new file mode 100644 index 00000000..db4c02c9 --- /dev/null +++ b/llm_configs/gpt-5.5-medium.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "medium", + "openai_reasoning_summary": "detailed" + } +} diff --git a/llm_configs/gpt-5.5-none.json b/llm_configs/gpt-5.5-none.json new file mode 100644 index 00000000..8209c0ce --- /dev/null +++ b/llm_configs/gpt-5.5-none.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "none", + "openai_reasoning_summary": "detailed" + } +} diff --git a/llm_configs/gpt-5.5-xhigh.json b/llm_configs/gpt-5.5-xhigh.json new file mode 100644 index 00000000..1d6d93d1 --- /dev/null +++ b/llm_configs/gpt-5.5-xhigh.json @@ -0,0 +1,8 @@ +{ + "model_name": "gpt-5.5-2026-04-23", + "provider": "openai", + "settings": { + "openai_reasoning_effort": "xhigh", + "openai_reasoning_summary": "detailed" + } +} diff --git a/llm_configs/grok-4.3.json b/llm_configs/grok-4.3.json new file mode 100644 index 00000000..f037a6ed --- /dev/null +++ b/llm_configs/grok-4.3.json @@ -0,0 +1,15 @@ +{ + "model_name": "x-ai/grok-4.3", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "extra_body": { + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/kimi-k2.6.json b/llm_configs/kimi-k2.6.json new file mode 100644 index 00000000..2c3b3d0f --- /dev/null +++ b/llm_configs/kimi-k2.6.json @@ -0,0 +1,20 @@ +{ + "model_name": "moonshotai/kimi-k2.6", + "provider": "openrouter", + "settings": { + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["moonshotai/int4"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/mercury-2.json b/llm_configs/mercury-2.json new file mode 100644 index 00000000..8aa34cf9 --- /dev/null +++ b/llm_configs/mercury-2.json @@ -0,0 +1,22 @@ +{ + "model_name": "inception/mercury-2", + "provider": "openrouter", + "settings": { + "temperature": 0.75, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["inception"], + "require_parameters": true + }, + "reasoning": { + "enabled": true, + "effort": "high" + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/mimo-v2.5-pro.json b/llm_configs/mimo-v2.5-pro.json new file mode 100644 index 00000000..344162b0 --- /dev/null +++ b/llm_configs/mimo-v2.5-pro.json @@ -0,0 +1,22 @@ +{ + "model_name": "xiaomi/mimo-v2.5-pro", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 0.95, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["xiaomi/fp8"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/mimo-v2.5.json b/llm_configs/mimo-v2.5.json new file mode 100644 index 00000000..90e55c3a --- /dev/null +++ b/llm_configs/mimo-v2.5.json @@ -0,0 +1,22 @@ +{ + "model_name": "xiaomi/mimo-v2.5", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "top_p": 0.95, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["xiaomi/fp8"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/mistral-medium-3.5.json b/llm_configs/mistral-medium-3.5.json new file mode 100644 index 00000000..2dfe5bb4 --- /dev/null +++ b/llm_configs/mistral-medium-3.5.json @@ -0,0 +1,23 @@ +{ + "model_name": "mistralai/mistral-medium-3-5", + "provider": "openrouter", + "settings": { + "temperature": 0.7, + "top_p": 0.95, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["mistral"], + "require_parameters": true + }, + "reasoning": { + "effort": "high", + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/qwen3.5-27b.json b/llm_configs/qwen3.5-27b.json new file mode 100644 index 00000000..fa91b86d --- /dev/null +++ b/llm_configs/qwen3.5-27b.json @@ -0,0 +1,23 @@ +{ + "model_name": "qwen/qwen3.5-27b", + "provider": "openrouter", + "settings": { + "temperature": 0.6, + "top_p": 0.95, + "extra_body": { + "presence_penalty": 0.0, + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["alibaba"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/qwen3.6-35b-a3b.json b/llm_configs/qwen3.6-35b-a3b.json new file mode 100644 index 00000000..444742bb --- /dev/null +++ b/llm_configs/qwen3.6-35b-a3b.json @@ -0,0 +1,25 @@ +{ + "model_name": "Qwen/Qwen3.6-35B-A3B", + "provider": "vllm-chat", + "base_url": "http://localhost:18000/v1", + "api_key": "ale-bench-eval", + "settings": { + "temperature": 0.6, + "top_p": 0.95, + "extra_body": { + "top_k": 20, + "min_p": 0.0, + "presence_penalty": 0.0, + "repetition_penalty": 1.0, + "chat_template_kwargs": { + "enable_thinking": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/qwen3.6-flash.json b/llm_configs/qwen3.6-flash.json new file mode 100644 index 00000000..2a7b27a0 --- /dev/null +++ b/llm_configs/qwen3.6-flash.json @@ -0,0 +1,21 @@ +{ + "model_name": "qwen/qwen3.6-flash", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["alibaba"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/llm_configs/qwen3.7-max.json b/llm_configs/qwen3.7-max.json new file mode 100644 index 00000000..058e329e --- /dev/null +++ b/llm_configs/qwen3.7-max.json @@ -0,0 +1,21 @@ +{ + "model_name": "qwen/qwen3.7-max", + "provider": "openrouter", + "settings": { + "temperature": 1.0, + "extra_body": { + "provider": { + "allow_fallbacks": false, + "data_collection": "deny", + "order": ["alibaba"], + "require_parameters": true + }, + "reasoning": { + "enabled": true + }, + "usage": { + "include": true + } + } + } +} diff --git a/src/ale_bench_eval/calc_cost.py b/src/ale_bench_eval/calc_cost.py index b0094a43..3b3e1ac8 100644 --- a/src/ale_bench_eval/calc_cost.py +++ b/src/ale_bench_eval/calc_cost.py @@ -5,6 +5,11 @@ from pydantic_ai.usage import RunUsage FALLBACK_DICT = { + "gpt-5.5-2026-04-23": ModelPrice( + input_mtok=Decimal(5), + cache_read_mtok=Decimal(5) / Decimal(10), + output_mtok=Decimal(30), + ), "gpt-5.4-nano-2026-03-17": ModelPrice( input_mtok=Decimal(2) / Decimal(10), cache_read_mtok=Decimal(2) / Decimal(100), @@ -40,6 +45,11 @@ cache_read_mtok=Decimal(125) / Decimal(1000), output_mtok=Decimal(10), ), + "gemini-3.5-flash": ModelPrice( + input_mtok=Decimal(15) / Decimal(10), + cache_read_mtok=Decimal(15) / Decimal(100), + output_mtok=Decimal(9), + ), "gemini-3-flash-preview": ModelPrice( input_mtok=Decimal(5) / Decimal(10), cache_read_mtok=Decimal(5) / Decimal(100), @@ -58,6 +68,16 @@ cache_read_mtok=Decimal(25) / Decimal(1000), output_mtok=Decimal(15) / Decimal(10), ), + "gemma-4-31b-it": ModelPrice( + input_mtok=Decimal(14) / Decimal(100), + cache_read_mtok=Decimal(7) / Decimal(100), + output_mtok=Decimal(4) / Decimal(10), + ), + "gemma-4-31B-it": ModelPrice( + input_mtok=Decimal(14) / Decimal(100), + cache_read_mtok=Decimal(7) / Decimal(100), + output_mtok=Decimal(4) / Decimal(10), + ), "gemma-4-26b-a4b-it": ModelPrice(input_mtok=Decimal(13) / Decimal(100), output_mtok=Decimal(4) / Decimal(10)), "claude-sonnet-4": ModelPrice( input_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=200000, price=Decimal(6))]), @@ -109,6 +129,18 @@ cache_read_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(15), ), + "claude-opus-4-7": ModelPrice( + input_mtok=Decimal(5), + cache_write_mtok=Decimal(625) / Decimal(100), + cache_read_mtok=Decimal(5) / Decimal(10), + output_mtok=Decimal(25), + ), + "claude-opus-4-8": ModelPrice( + input_mtok=Decimal(5), + cache_write_mtok=Decimal(625) / Decimal(100), + cache_read_mtok=Decimal(5) / Decimal(10), + output_mtok=Decimal(25), + ), "grok-4.1-fast": ModelPrice( input_mtok=Decimal(2) / Decimal(10), cache_read_mtok=Decimal(5) / Decimal(100), @@ -119,6 +151,15 @@ cache_read_mtok=Decimal(2) / Decimal(10), output_mtok=TieredPrices(base=Decimal(6), tiers=[Tier(start=200000, price=Decimal(12))]), ), + "grok-4.3": ModelPrice( + input_mtok=TieredPrices( + base=Decimal(125) / Decimal(100), tiers=[Tier(start=200000, price=Decimal(25) / Decimal(10))] + ), + cache_read_mtok=TieredPrices( + base=Decimal(2) / Decimal(10), tiers=[Tier(start=200000, price=Decimal(4) / Decimal(10))] + ), + output_mtok=TieredPrices(base=Decimal(25) / Decimal(10), tiers=[Tier(start=200000, price=Decimal(5))]), + ), "nova-premier-v1": ModelPrice( input_mtok=Decimal(25) / Decimal(10), cache_read_mtok=Decimal(625) / Decimal(1000), @@ -129,6 +170,16 @@ "deepseek-v3.1-terminus": ModelPrice(input_mtok=Decimal(27) / Decimal(100), output_mtok=Decimal(1)), "deepseek-v3.2": ModelPrice(input_mtok=Decimal(27) / Decimal(100), output_mtok=Decimal(42) / Decimal(10)), "deepseek-r1-0528": ModelPrice(input_mtok=Decimal(79) / Decimal(100), output_mtok=Decimal(4)), + "deepseek-v4-pro": ModelPrice( + input_mtok=Decimal(174) / Decimal(100), + output_mtok=Decimal(348) / Decimal(100), + cache_read_mtok=Decimal(145) / Decimal(1000), + ), + "deepseek-v4-flash": ModelPrice( + input_mtok=Decimal(14) / Decimal(100), + output_mtok=Decimal(28) / Decimal(100), + cache_read_mtok=Decimal(28) / Decimal(1000), + ), "mimo-v2-flash:free": ModelPrice(input_mtok=Decimal(1) / Decimal(10), output_mtok=Decimal(3) / Decimal(10)), "mimo-v2-pro": ModelPrice( input_mtok=TieredPrices( @@ -144,6 +195,34 @@ tiers=[Tier(start=256000, price=Decimal(4) / Decimal(10))], ), ), + "mimo-v2.5": ModelPrice( + input_mtok=TieredPrices( + base=Decimal(4) / Decimal(10), + tiers=[Tier(start=256000, price=Decimal(8) / Decimal(10))], + ), + output_mtok=TieredPrices( + base=Decimal(2), + tiers=[Tier(start=256000, price=Decimal(4))], + ), + cache_read_mtok=TieredPrices( + base=Decimal(8) / Decimal(100), + tiers=[Tier(start=256000, price=Decimal(16) / Decimal(100))], + ), + ), + "mimo-v2.5-pro": ModelPrice( + input_mtok=TieredPrices( + base=Decimal(1), + tiers=[Tier(start=256000, price=Decimal(2))], + ), + output_mtok=TieredPrices( + base=Decimal(3), + tiers=[Tier(start=256000, price=Decimal(6))], + ), + cache_read_mtok=TieredPrices( + base=Decimal(2) / Decimal(10), + tiers=[Tier(start=256000, price=Decimal(4) / Decimal(10))], + ), + ), "glm-4.5": ModelPrice(input_mtok=Decimal(59) / Decimal(100), output_mtok=Decimal(21) / Decimal(10)), "glm-4.6": ModelPrice( input_mtok=Decimal(6) / Decimal(10), @@ -178,6 +257,7 @@ "mistral-medium-3.1": ModelPrice(input_mtok=Decimal(4) / Decimal(10), output_mtok=Decimal(2)), "mistral-large-2512": ModelPrice(input_mtok=Decimal(5) / Decimal(10), output_mtok=Decimal(15) / Decimal(10)), "mistral-small-2603": ModelPrice(input_mtok=Decimal(15) / Decimal(100), output_mtok=Decimal(6) / Decimal(10)), + "mistral-medium-3-5": ModelPrice(input_mtok=Decimal(15) / Decimal(10), output_mtok=Decimal(75) / Decimal(10)), "kimi-k2": ModelPrice( input_mtok=Decimal(6) / Decimal(10), output_mtok=Decimal(25) / Decimal(10), @@ -198,6 +278,16 @@ output_mtok=Decimal(3), cache_read_mtok=Decimal(1) / Decimal(10), ), + "kimi-k2.6": ModelPrice( + input_mtok=Decimal(95) / Decimal(100), + output_mtok=Decimal(4), + cache_read_mtok=Decimal(16) / Decimal(100), + ), + "mercury-2": ModelPrice( + input_mtok=Decimal(25) / Decimal(100), + cache_read_mtok=Decimal(25) / Decimal(1000), + output_mtok=Decimal(75) / Decimal(100), + ), "minimax-m2.1": ModelPrice( input_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(12) / Decimal(10), @@ -247,6 +337,7 @@ "qwen3-next-80b-a3b-thinking": ModelPrice( input_mtok=Decimal(14) / Decimal(100), output_mtok=Decimal(14) / Decimal(10) ), + "qwen3.5-27b": ModelPrice(input_mtok=Decimal(3) / Decimal(10), output_mtok=Decimal(24) / Decimal(10)), "qwen3.5-35b-a3b": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)), "qwen3.5-397b-a17b": ModelPrice(input_mtok=Decimal(6) / Decimal(10), output_mtok=Decimal(36) / Decimal(10)), "qwen3.5-plus-02-15": ModelPrice( @@ -262,6 +353,23 @@ input_mtok=TieredPrices(base=Decimal(5) / Decimal(10), tiers=[Tier(start=256000, price=Decimal(2))]), output_mtok=TieredPrices(base=Decimal(3), tiers=[Tier(start=256000, price=Decimal(6))]), ), + "qwen3.6-flash": ModelPrice( + input_mtok=TieredPrices(base=Decimal(25) / Decimal(100), tiers=[Tier(start=256000, price=Decimal(1))]), + output_mtok=TieredPrices(base=Decimal(15) / Decimal(10), tiers=[Tier(start=256000, price=Decimal(4))]), + cache_read_mtok=TieredPrices( + base=Decimal(3125) / Decimal(10000), tiers=[Tier(start=256000, price=Decimal(125) / Decimal(100))] + ), + cache_write_mtok=TieredPrices( + base=Decimal(25) / Decimal(1000), tiers=[Tier(start=256000, price=Decimal(1) / Decimal(10))] + ), + ), + "Qwen3.6-35B-A3B": ModelPrice(input_mtok=Decimal(25) / Decimal(100), output_mtok=Decimal(2)), + "qwen3.7-max": ModelPrice( + input_mtok=Decimal(25) / Decimal(10), + output_mtok=Decimal(75) / Decimal(10), + cache_read_mtok=Decimal(25) / Decimal(100), + cache_write_mtok=Decimal(3125) / Decimal(1000), + ), }