Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ OPENROUTER_API_KEY=
OPENROUTER_MODEL=anthropic/claude-4.5-haiku
# See all models at https://openrouter.ai/models (text-only models are auto-filtered in the UI)

# Requesty Settings (OpenAI-compatible router, provider/model naming)
# Your Requesty API key (from https://app.requesty.ai/api-keys)
REQUESTY_API_KEY=
REQUESTY_MODEL=openai/gpt-4o-mini

# Mistral AI Settings
# Your Mistral API key (from https://console.mistral.ai)
MISTRAL_API_KEY=
Expand Down
2 changes: 1 addition & 1 deletion benchmark/aggregator.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)


CLOUD_PROVIDERS = {"openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"}
CLOUD_PROVIDERS = {"openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"}


@dataclass
Expand Down
54 changes: 51 additions & 3 deletions benchmark/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from benchmark.translator import (
get_available_ollama_models,
get_available_openrouter_models,
get_available_requesty_models,
get_available_openai_models,
)

Expand Down Expand Up @@ -98,6 +99,7 @@ def cmd_run(args: argparse.Namespace) -> int:
evaluator_provider = getattr(args, 'evaluator_provider', DEFAULT_EVALUATOR_PROVIDER)
config = BenchmarkConfig.from_cli_args(
openrouter_key=args.openrouter_key,
requesty_key=getattr(args, 'requesty_key', None),
openai_key=args.openai_key,
openai_endpoint=args.openai_endpoint,
poe_key=args.poe_key,
Expand Down Expand Up @@ -129,6 +131,14 @@ def cmd_run(args: argparse.Namespace) -> int:
# Extract model IDs
models = [m["id"] if isinstance(m, dict) else m for m in models_data[:10]]
print(colored(f"Found {len(models_data)} models. Using top 10: {', '.join(models[:3])}...", Colors.GREEN))
elif provider == "requesty":
print(colored("Fetching available Requesty models...", Colors.CYAN))
models_data = asyncio.run(get_available_requesty_models(config))
if not models_data:
log_callback("error", "No Requesty models available.")
return 1
models = [m["id"] if isinstance(m, dict) else m for m in models_data[:10]]
print(colored(f"Found {len(models_data)} models. Using top 10: {', '.join(models[:3])}...", Colors.GREEN))
elif provider == "openai":
print(colored("Fetching available OpenAI-compatible models...", Colors.CYAN))
models_data = asyncio.run(get_available_openai_models(config))
Expand Down Expand Up @@ -370,6 +380,9 @@ def _fetch_model_ids(provider: str, config: BenchmarkConfig) -> list[str]:
if provider == "openrouter":
models = asyncio.run(get_available_openrouter_models(config)) or []
return [m.get("id") if isinstance(m, dict) else m for m in models]
if provider == "requesty":
models = asyncio.run(get_available_requesty_models(config)) or []
return [m.get("id") if isinstance(m, dict) else m for m in models]
if provider == "openai":
models = asyncio.run(get_available_openai_models(config)) or []
return [m.get("id") if isinstance(m, dict) else m for m in models]
Expand Down Expand Up @@ -415,6 +428,7 @@ def cmd_models(args: argparse.Namespace) -> int:
"""List available models for benchmarking, or validate a specific id."""
config = BenchmarkConfig.from_cli_args(
openrouter_key=args.openrouter_key,
requesty_key=getattr(args, "requesty_key", None),
openai_key=args.openai_key,
openai_endpoint=args.openai_endpoint,
poe_key=getattr(args, "poe_key", None),
Expand Down Expand Up @@ -474,6 +488,31 @@ def cmd_models(args: argparse.Namespace) -> int:
print(colored("Tip: Use -m to specify models, e.g.:", Colors.YELLOW))
print(" python -m benchmark.cli run -p openrouter -m anthropic/claude-sonnet-4 openai/gpt-4o")

elif provider == "requesty":
print(colored("Fetching Requesty models...\n", Colors.CYAN))
models = asyncio.run(get_available_requesty_models(config))

if not models:
log_callback("error", "Failed to fetch Requesty models")
return 1

print(colored(f"Available Requesty Models ({len(models)}):\n", Colors.BOLD))

# Table header
print(f"{'Model ID':<50}")
print("-" * 50)

for model in models[:50]: # Limit to 50 for readability
if isinstance(model, dict):
model_id = model.get("id", "unknown")
else:
model_id = model
print(f"{model_id:<50}")

print()
print(colored("Tip: Use -m to specify models, e.g.:", Colors.YELLOW))
print(" python -m benchmark.cli run -p requesty -m openai/gpt-4o-mini")

elif provider == "openai":
print(colored("Fetching OpenAI-compatible models...\n", Colors.CYAN))
models = asyncio.run(get_available_openai_models(config))
Expand Down Expand Up @@ -1035,10 +1074,10 @@ def create_parser() -> argparse.ArgumentParser:
)
run_parser.add_argument(
"-p", "--provider",
choices=["ollama", "openai", "openrouter", "poe"],
choices=["ollama", "openai", "openrouter", "requesty", "poe"],
default="ollama",
help="Translation provider: 'ollama' (local), 'openai' (OpenAI-compatible), "
"'openrouter' (cloud), or 'poe' (Poe.com unified API)."
"'openrouter' (cloud), 'requesty' (OpenAI-compatible router), or 'poe' (Poe.com unified API)."
)
run_parser.add_argument(
"--pairs",
Expand Down Expand Up @@ -1074,6 +1113,11 @@ def create_parser() -> argparse.ArgumentParser:
help="OpenRouter API key (for evaluation, and translation if using --provider openrouter). "
"Can also be set via OPENROUTER_API_KEY env var."
)
run_parser.add_argument(
"--requesty-key",
help="Requesty API key (for translation if using --provider requesty). "
"Can also be set via REQUESTY_API_KEY env var."
)
run_parser.add_argument(
"--evaluator-provider",
choices=["openrouter", "poe"],
Expand Down Expand Up @@ -1152,7 +1196,7 @@ def create_parser() -> argparse.ArgumentParser:
models_parser = subparsers.add_parser("models", help="List available models for benchmarking")
models_parser.add_argument(
"-p", "--provider",
choices=["ollama", "openai", "openrouter", "poe"],
choices=["ollama", "openai", "openrouter", "requesty", "poe"],
default="ollama",
help="Provider to list models for (default: ollama)"
)
Expand All @@ -1168,6 +1212,10 @@ def create_parser() -> argparse.ArgumentParser:
"--openrouter-key",
help="OpenRouter API key (required for listing OpenRouter models)"
)
models_parser.add_argument(
"--requesty-key",
help="Requesty API key (required for listing Requesty models)"
)
models_parser.add_argument(
"--poe-key",
help="Poe API key (overrides POE_API_KEY env)"
Expand Down
33 changes: 30 additions & 3 deletions benchmark/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,22 @@ class OpenRouterConfig:
site_name: str = "TranslateBookWithLLM Benchmark"


@dataclass
class RequestyConfig:
"""Configuration for Requesty (OpenAI-compatible) provider."""

api_key: Optional[str] = field(
default_factory=lambda: os.getenv("REQUESTY_API_KEY")
)
endpoint: str = "https://router.requesty.ai/v1/chat/completions"
default_model: str = "openai/gpt-4o-mini"
timeout: int = 120

# Request headers
site_url: str = "https://github.com/yourusername/TranslateBookWithLLM"
site_name: str = "TranslateBookWithLLM Benchmark"


@dataclass
class OpenAICompatibleConfig:
"""Configuration for OpenAI-compatible translation provider."""
Expand Down Expand Up @@ -153,13 +169,14 @@ class BenchmarkConfig:
ollama: OllamaConfig = field(default_factory=OllamaConfig)
openai: OpenAICompatibleConfig = field(default_factory=OpenAICompatibleConfig)
openrouter: OpenRouterConfig = field(default_factory=OpenRouterConfig)
requesty: RequestyConfig = field(default_factory=RequestyConfig)
poe: PoeConfig = field(default_factory=PoeConfig)
paths: PathConfig = field(default_factory=PathConfig)

# Benchmark settings
source_language: str = "English"

# Translation provider ("ollama", "openai", or "openrouter")
# Translation provider ("ollama", "openai", "openrouter", or "requesty")
translation_provider: str = "ollama"

# Evaluator provider ("openrouter" or "poe")
Expand All @@ -178,6 +195,7 @@ def from_env(cls) -> "BenchmarkConfig":
def from_cli_args(
cls,
openrouter_key: Optional[str] = None,
requesty_key: Optional[str] = None,
openai_key: Optional[str] = None,
openai_endpoint: Optional[str] = None,
evaluator_model: Optional[str] = None,
Expand All @@ -193,6 +211,9 @@ def from_cli_args(
if openrouter_key:
config.openrouter.api_key = openrouter_key

if requesty_key:
config.requesty.api_key = requesty_key

if openai_key:
config.openai.api_key = openai_key

Expand Down Expand Up @@ -249,6 +270,12 @@ def validate(self) -> list[str]:
"Set OPENROUTER_API_KEY in .env or use --openrouter-key"
)

if self.translation_provider == "requesty" and not self.requesty.api_key:
errors.append(
"Requesty API key not configured. Required for translation. "
"Set REQUESTY_API_KEY in .env or use --requesty-key"
)

if self.translation_provider == "openai" and not self.openai.endpoint:
errors.append(
"OpenAI-compatible endpoint not configured. Required for translation. "
Expand All @@ -275,10 +302,10 @@ def validate(self) -> list[str]:
)

# Validate translation provider
if self.translation_provider not in ("ollama", "openai", "openrouter", "poe"):
if self.translation_provider not in ("ollama", "openai", "openrouter", "requesty", "poe"):
errors.append(
f"Invalid translation provider: {self.translation_provider}. "
"Must be 'ollama', 'openai', 'openrouter', or 'poe'"
"Must be 'ollama', 'openai', 'openrouter', 'requesty', or 'poe'"
)

return errors
Expand Down
12 changes: 11 additions & 1 deletion benchmark/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
code_to_language_name,
test_ollama_connection, get_available_ollama_models,
test_openai_translation_connection, get_available_openai_models,
test_openrouter_translation_connection, get_available_openrouter_models
test_openrouter_translation_connection, get_available_openrouter_models,
test_requesty_translation_connection, get_available_requesty_models
)
from benchmark.evaluator import (
TranslationEvaluator, test_openrouter_connection, test_poe_connection
Expand Down Expand Up @@ -149,6 +150,12 @@ async def validate_setup(self, evaluate: bool = True) -> tuple[bool, list[str]]:
errors.append(f"OpenRouter (translation): {or_trans_msg}")
else:
self._log("info", f"OpenRouter (translation): {or_trans_msg}")
elif self.config.translation_provider == "requesty":
rq_trans_ok, rq_trans_msg = await test_requesty_translation_connection(self.config)
if not rq_trans_ok:
errors.append(f"Requesty (translation): {rq_trans_msg}")
else:
self._log("info", f"Requesty (translation): {rq_trans_msg}")
elif self.config.translation_provider == "openai":
openai_ok, openai_msg = await test_openai_translation_connection(self.config)
if not openai_ok:
Expand Down Expand Up @@ -441,6 +448,9 @@ async def quick_benchmark(
if config.translation_provider == "openrouter":
provider_models = await get_available_openrouter_models(config)
models = [m["id"] if isinstance(m, dict) else m for m in provider_models]
elif config.translation_provider == "requesty":
provider_models = await get_available_requesty_models(config)
models = [m["id"] if isinstance(m, dict) else m for m in provider_models]
elif config.translation_provider == "openai":
provider_models = await get_available_openai_models(config)
models = [m["id"] if isinstance(m, dict) else m for m in provider_models]
Expand Down
2 changes: 1 addition & 1 deletion benchmark/schemas/model.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"properties": {
"id": { "type": "string", "minLength": 1 },
"provider": {
"enum": ["ollama", "openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"]
"enum": ["ollama", "openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"]
},
"context_window": { "type": "integer", "minimum": 0 },
"released_at": { "type": "string", "format": "date" },
Expand Down
2 changes: 1 addition & 1 deletion benchmark/schemas/submission.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"additionalProperties": false,
"properties": {
"provider": {
"enum": ["ollama", "openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"]
"enum": ["ollama", "openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"]
},
"id": { "type": "string", "minLength": 1 },
"context_window": { "type": "integer", "minimum": 0 },
Expand Down
2 changes: 1 addition & 1 deletion benchmark/schemas/translations.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"additionalProperties": false,
"properties": {
"provider": {
"enum": ["ollama", "openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"]
"enum": ["ollama", "openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"]
},
"id": { "type": "string", "minLength": 1 },
"context_window": { "type": "integer", "minimum": 0 },
Expand Down
Loading