From d4763c797a083c749cb6904e5eaf890fa1fd8df0 Mon Sep 17 00:00:00 2001 From: Thibault Jaigu Date: Sun, 5 Jul 2026 09:10:33 +0100 Subject: [PATCH] Add Requesty as a provider --- .env.example | 5 + benchmark/aggregator.py | 2 +- benchmark/cli.py | 54 ++++++++++- benchmark/config.py | 33 ++++++- benchmark/runner.py | 12 ++- benchmark/schemas/model.schema.json | 2 +- benchmark/schemas/submission.schema.json | 2 +- benchmark/schemas/translations.schema.json | 2 +- benchmark/translator.py | 104 +++++++++++++++++++++ docs/BENCHMARK.md | 10 +- docs/BENCHMARK_WORKFLOW.md | 2 +- 11 files changed, 214 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index e37c3122..379295ab 100644 --- a/.env.example +++ b/.env.example @@ -81,6 +81,11 @@ OPENROUTER_API_KEY= OPENROUTER_MODEL=anthropic/claude-4.5-haiku # See all models at https://openrouter.ai/models (text-only models are auto-filtered in the UI) +# Requesty Settings (OpenAI-compatible router, provider/model naming) +# Your Requesty API key (from https://app.requesty.ai/api-keys) +REQUESTY_API_KEY= +REQUESTY_MODEL=openai/gpt-4o-mini + # Mistral AI Settings # Your Mistral API key (from https://console.mistral.ai) MISTRAL_API_KEY= diff --git a/benchmark/aggregator.py b/benchmark/aggregator.py index f3b7bb01..03a58016 100644 --- a/benchmark/aggregator.py +++ b/benchmark/aggregator.py @@ -34,7 +34,7 @@ ) -CLOUD_PROVIDERS = {"openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"} +CLOUD_PROVIDERS = {"openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"} @dataclass diff --git a/benchmark/cli.py b/benchmark/cli.py index 3844ef7a..233c303f 100644 --- a/benchmark/cli.py +++ b/benchmark/cli.py @@ -38,6 +38,7 @@ from benchmark.translator import ( get_available_ollama_models, get_available_openrouter_models, + get_available_requesty_models, get_available_openai_models, ) @@ -98,6 +99,7 @@ def cmd_run(args: argparse.Namespace) -> int: evaluator_provider = getattr(args, 'evaluator_provider', DEFAULT_EVALUATOR_PROVIDER) config = BenchmarkConfig.from_cli_args( openrouter_key=args.openrouter_key, + requesty_key=getattr(args, 'requesty_key', None), openai_key=args.openai_key, openai_endpoint=args.openai_endpoint, poe_key=args.poe_key, @@ -129,6 +131,14 @@ def cmd_run(args: argparse.Namespace) -> int: # Extract model IDs models = [m["id"] if isinstance(m, dict) else m for m in models_data[:10]] print(colored(f"Found {len(models_data)} models. Using top 10: {', '.join(models[:3])}...", Colors.GREEN)) + elif provider == "requesty": + print(colored("Fetching available Requesty models...", Colors.CYAN)) + models_data = asyncio.run(get_available_requesty_models(config)) + if not models_data: + log_callback("error", "No Requesty models available.") + return 1 + models = [m["id"] if isinstance(m, dict) else m for m in models_data[:10]] + print(colored(f"Found {len(models_data)} models. Using top 10: {', '.join(models[:3])}...", Colors.GREEN)) elif provider == "openai": print(colored("Fetching available OpenAI-compatible models...", Colors.CYAN)) models_data = asyncio.run(get_available_openai_models(config)) @@ -370,6 +380,9 @@ def _fetch_model_ids(provider: str, config: BenchmarkConfig) -> list[str]: if provider == "openrouter": models = asyncio.run(get_available_openrouter_models(config)) or [] return [m.get("id") if isinstance(m, dict) else m for m in models] + if provider == "requesty": + models = asyncio.run(get_available_requesty_models(config)) or [] + return [m.get("id") if isinstance(m, dict) else m for m in models] if provider == "openai": models = asyncio.run(get_available_openai_models(config)) or [] return [m.get("id") if isinstance(m, dict) else m for m in models] @@ -415,6 +428,7 @@ def cmd_models(args: argparse.Namespace) -> int: """List available models for benchmarking, or validate a specific id.""" config = BenchmarkConfig.from_cli_args( openrouter_key=args.openrouter_key, + requesty_key=getattr(args, "requesty_key", None), openai_key=args.openai_key, openai_endpoint=args.openai_endpoint, poe_key=getattr(args, "poe_key", None), @@ -474,6 +488,31 @@ def cmd_models(args: argparse.Namespace) -> int: print(colored("Tip: Use -m to specify models, e.g.:", Colors.YELLOW)) print(" python -m benchmark.cli run -p openrouter -m anthropic/claude-sonnet-4 openai/gpt-4o") + elif provider == "requesty": + print(colored("Fetching Requesty models...\n", Colors.CYAN)) + models = asyncio.run(get_available_requesty_models(config)) + + if not models: + log_callback("error", "Failed to fetch Requesty models") + return 1 + + print(colored(f"Available Requesty Models ({len(models)}):\n", Colors.BOLD)) + + # Table header + print(f"{'Model ID':<50}") + print("-" * 50) + + for model in models[:50]: # Limit to 50 for readability + if isinstance(model, dict): + model_id = model.get("id", "unknown") + else: + model_id = model + print(f"{model_id:<50}") + + print() + print(colored("Tip: Use -m to specify models, e.g.:", Colors.YELLOW)) + print(" python -m benchmark.cli run -p requesty -m openai/gpt-4o-mini") + elif provider == "openai": print(colored("Fetching OpenAI-compatible models...\n", Colors.CYAN)) models = asyncio.run(get_available_openai_models(config)) @@ -1035,10 +1074,10 @@ def create_parser() -> argparse.ArgumentParser: ) run_parser.add_argument( "-p", "--provider", - choices=["ollama", "openai", "openrouter", "poe"], + choices=["ollama", "openai", "openrouter", "requesty", "poe"], default="ollama", help="Translation provider: 'ollama' (local), 'openai' (OpenAI-compatible), " - "'openrouter' (cloud), or 'poe' (Poe.com unified API)." + "'openrouter' (cloud), 'requesty' (OpenAI-compatible router), or 'poe' (Poe.com unified API)." ) run_parser.add_argument( "--pairs", @@ -1074,6 +1113,11 @@ def create_parser() -> argparse.ArgumentParser: help="OpenRouter API key (for evaluation, and translation if using --provider openrouter). " "Can also be set via OPENROUTER_API_KEY env var." ) + run_parser.add_argument( + "--requesty-key", + help="Requesty API key (for translation if using --provider requesty). " + "Can also be set via REQUESTY_API_KEY env var." + ) run_parser.add_argument( "--evaluator-provider", choices=["openrouter", "poe"], @@ -1152,7 +1196,7 @@ def create_parser() -> argparse.ArgumentParser: models_parser = subparsers.add_parser("models", help="List available models for benchmarking") models_parser.add_argument( "-p", "--provider", - choices=["ollama", "openai", "openrouter", "poe"], + choices=["ollama", "openai", "openrouter", "requesty", "poe"], default="ollama", help="Provider to list models for (default: ollama)" ) @@ -1168,6 +1212,10 @@ def create_parser() -> argparse.ArgumentParser: "--openrouter-key", help="OpenRouter API key (required for listing OpenRouter models)" ) + models_parser.add_argument( + "--requesty-key", + help="Requesty API key (required for listing Requesty models)" + ) models_parser.add_argument( "--poe-key", help="Poe API key (overrides POE_API_KEY env)" diff --git a/benchmark/config.py b/benchmark/config.py index 72208e0c..93f61d54 100644 --- a/benchmark/config.py +++ b/benchmark/config.py @@ -71,6 +71,22 @@ class OpenRouterConfig: site_name: str = "TranslateBookWithLLM Benchmark" +@dataclass +class RequestyConfig: + """Configuration for Requesty (OpenAI-compatible) provider.""" + + api_key: Optional[str] = field( + default_factory=lambda: os.getenv("REQUESTY_API_KEY") + ) + endpoint: str = "https://router.requesty.ai/v1/chat/completions" + default_model: str = "openai/gpt-4o-mini" + timeout: int = 120 + + # Request headers + site_url: str = "https://github.com/yourusername/TranslateBookWithLLM" + site_name: str = "TranslateBookWithLLM Benchmark" + + @dataclass class OpenAICompatibleConfig: """Configuration for OpenAI-compatible translation provider.""" @@ -153,13 +169,14 @@ class BenchmarkConfig: ollama: OllamaConfig = field(default_factory=OllamaConfig) openai: OpenAICompatibleConfig = field(default_factory=OpenAICompatibleConfig) openrouter: OpenRouterConfig = field(default_factory=OpenRouterConfig) + requesty: RequestyConfig = field(default_factory=RequestyConfig) poe: PoeConfig = field(default_factory=PoeConfig) paths: PathConfig = field(default_factory=PathConfig) # Benchmark settings source_language: str = "English" - # Translation provider ("ollama", "openai", or "openrouter") + # Translation provider ("ollama", "openai", "openrouter", or "requesty") translation_provider: str = "ollama" # Evaluator provider ("openrouter" or "poe") @@ -178,6 +195,7 @@ def from_env(cls) -> "BenchmarkConfig": def from_cli_args( cls, openrouter_key: Optional[str] = None, + requesty_key: Optional[str] = None, openai_key: Optional[str] = None, openai_endpoint: Optional[str] = None, evaluator_model: Optional[str] = None, @@ -193,6 +211,9 @@ def from_cli_args( if openrouter_key: config.openrouter.api_key = openrouter_key + if requesty_key: + config.requesty.api_key = requesty_key + if openai_key: config.openai.api_key = openai_key @@ -249,6 +270,12 @@ def validate(self) -> list[str]: "Set OPENROUTER_API_KEY in .env or use --openrouter-key" ) + if self.translation_provider == "requesty" and not self.requesty.api_key: + errors.append( + "Requesty API key not configured. Required for translation. " + "Set REQUESTY_API_KEY in .env or use --requesty-key" + ) + if self.translation_provider == "openai" and not self.openai.endpoint: errors.append( "OpenAI-compatible endpoint not configured. Required for translation. " @@ -275,10 +302,10 @@ def validate(self) -> list[str]: ) # Validate translation provider - if self.translation_provider not in ("ollama", "openai", "openrouter", "poe"): + if self.translation_provider not in ("ollama", "openai", "openrouter", "requesty", "poe"): errors.append( f"Invalid translation provider: {self.translation_provider}. " - "Must be 'ollama', 'openai', 'openrouter', or 'poe'" + "Must be 'ollama', 'openai', 'openrouter', 'requesty', or 'poe'" ) return errors diff --git a/benchmark/runner.py b/benchmark/runner.py index 058d11be..72d7805e 100644 --- a/benchmark/runner.py +++ b/benchmark/runner.py @@ -28,7 +28,8 @@ code_to_language_name, test_ollama_connection, get_available_ollama_models, test_openai_translation_connection, get_available_openai_models, - test_openrouter_translation_connection, get_available_openrouter_models + test_openrouter_translation_connection, get_available_openrouter_models, + test_requesty_translation_connection, get_available_requesty_models ) from benchmark.evaluator import ( TranslationEvaluator, test_openrouter_connection, test_poe_connection @@ -149,6 +150,12 @@ async def validate_setup(self, evaluate: bool = True) -> tuple[bool, list[str]]: errors.append(f"OpenRouter (translation): {or_trans_msg}") else: self._log("info", f"OpenRouter (translation): {or_trans_msg}") + elif self.config.translation_provider == "requesty": + rq_trans_ok, rq_trans_msg = await test_requesty_translation_connection(self.config) + if not rq_trans_ok: + errors.append(f"Requesty (translation): {rq_trans_msg}") + else: + self._log("info", f"Requesty (translation): {rq_trans_msg}") elif self.config.translation_provider == "openai": openai_ok, openai_msg = await test_openai_translation_connection(self.config) if not openai_ok: @@ -441,6 +448,9 @@ async def quick_benchmark( if config.translation_provider == "openrouter": provider_models = await get_available_openrouter_models(config) models = [m["id"] if isinstance(m, dict) else m for m in provider_models] + elif config.translation_provider == "requesty": + provider_models = await get_available_requesty_models(config) + models = [m["id"] if isinstance(m, dict) else m for m in provider_models] elif config.translation_provider == "openai": provider_models = await get_available_openai_models(config) models = [m["id"] if isinstance(m, dict) else m for m in provider_models] diff --git a/benchmark/schemas/model.schema.json b/benchmark/schemas/model.schema.json index dca97cbf..c8322da3 100644 --- a/benchmark/schemas/model.schema.json +++ b/benchmark/schemas/model.schema.json @@ -8,7 +8,7 @@ "properties": { "id": { "type": "string", "minLength": 1 }, "provider": { - "enum": ["ollama", "openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"] + "enum": ["ollama", "openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"] }, "context_window": { "type": "integer", "minimum": 0 }, "released_at": { "type": "string", "format": "date" }, diff --git a/benchmark/schemas/submission.schema.json b/benchmark/schemas/submission.schema.json index 8f4bf699..97a1b959 100644 --- a/benchmark/schemas/submission.schema.json +++ b/benchmark/schemas/submission.schema.json @@ -39,7 +39,7 @@ "additionalProperties": false, "properties": { "provider": { - "enum": ["ollama", "openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"] + "enum": ["ollama", "openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"] }, "id": { "type": "string", "minLength": 1 }, "context_window": { "type": "integer", "minimum": 0 }, diff --git a/benchmark/schemas/translations.schema.json b/benchmark/schemas/translations.schema.json index f9840dfd..c9edf621 100644 --- a/benchmark/schemas/translations.schema.json +++ b/benchmark/schemas/translations.schema.json @@ -14,7 +14,7 @@ "additionalProperties": false, "properties": { "provider": { - "enum": ["ollama", "openai", "openrouter", "gemini", "mistral", "deepseek", "poe", "nim"] + "enum": ["ollama", "openai", "openrouter", "requesty", "gemini", "mistral", "deepseek", "poe", "nim"] }, "id": { "type": "string", "minLength": 1 }, "context_window": { "type": "integer", "minimum": 0 }, diff --git a/benchmark/translator.py b/benchmark/translator.py index 31825cde..2a747acd 100644 --- a/benchmark/translator.py +++ b/benchmark/translator.py @@ -78,6 +78,7 @@ def _provider_label(self) -> str: "ollama": "Ollama", "openai": "OpenAI-compatible provider", "openrouter": "OpenRouter", + "requesty": "Requesty", "poe": "Poe", } return labels.get(self.provider_type, self.provider_type) @@ -100,6 +101,17 @@ def _get_provider(self, model: str) -> LLMProvider: api_key=self.config.openrouter.api_key, model=model ) + elif self.provider_type == "requesty": + if not self.config.requesty.api_key: + raise ValueError("Requesty API key is required for translation. " + "Set REQUESTY_API_KEY in .env or use --requesty-key") + self._providers[model] = OpenAICompatibleProvider( + api_endpoint=self.config.requesty.endpoint, + api_key=self.config.requesty.api_key, + model=model, + log_callback=lambda level, msg: self._log(level, msg), + provider_name="requesty" + ) elif self.provider_type == "poe": if not self.config.poe.api_key: raise ValueError("Poe API key is required for translation. " @@ -519,3 +531,95 @@ async def test_openrouter_translation_connection(config: BenchmarkConfig) -> tup except Exception as e: return False, f"OpenRouter connection test failed: {e}" + + +async def _fetch_requesty_models(config: BenchmarkConfig) -> list[dict]: + """ + Fetch and filter models from the Requesty (OpenAI-compatible) endpoint. + + Args: + config: Benchmark configuration + + Returns: + List of model dicts with id, name, and owned_by fields. + Empty list if the endpoint is unreachable or returns no models. + """ + import httpx + + base_url = config.requesty.endpoint.replace("/chat/completions", "").rstrip("/") + headers = {} + if config.requesty.api_key: + headers["Authorization"] = f"Bearer {config.requesty.api_key}" + + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get(f"{base_url}/models", headers=headers) + response.raise_for_status() + + data = response.json() + models = [] + for model in data.get("data", []): + model_id = model.get("id", "") + if not model_id: + continue + if "embedding" in model_id.lower() or "whisper" in model_id.lower(): + continue + models.append({ + "id": model_id, + "name": model_id, + "owned_by": model.get("owned_by", "unknown"), + }) + + models.sort(key=lambda item: item["name"].lower()) + return models + + +async def get_available_requesty_models(config: BenchmarkConfig) -> list[dict]: + """ + Get list of available Requesty models. + + Args: + config: Benchmark configuration + + Returns: + List of model dicts with id and name. Empty list on failure. + """ + if not config.requesty.api_key: + print("⚠️ Requesty API key not configured. Returning default model.") + return [{"id": config.requesty.default_model, "name": config.requesty.default_model}] + + try: + return await _fetch_requesty_models(config) + except Exception as e: + print(f"⚠️ Failed to fetch Requesty models: {e}") + return [{"id": config.requesty.default_model, "name": config.requesty.default_model}] + + +async def test_requesty_translation_connection(config: BenchmarkConfig) -> tuple[bool, str]: + """ + Test if Requesty is accessible for translation. + + Args: + config: Benchmark configuration + + Returns: + Tuple of (success, message) + """ + import httpx + + if not config.requesty.api_key: + return False, "Requesty API key not configured" + + try: + models = await _fetch_requesty_models(config) + if not models: + return False, "No Requesty models available" + + model_names = [m["id"] if isinstance(m, dict) else m for m in models[:5]] + return True, f"Requesty connected. Available models: {', '.join(model_names)}..." + + except httpx.ConnectError: + return False, f"Cannot connect to Requesty endpoint at {config.requesty.endpoint}" + except httpx.HTTPStatusError as e: + return False, f"Requesty HTTP error: {e.response.status_code}" + except Exception as e: + return False, f"Requesty connection test failed: {e}" diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index 4ff0caa6..0e48bd35 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -29,8 +29,9 @@ python -m benchmark.cli wiki-publish ## Prerequisites -1. **Translation backend**: Ollama, OpenAI-compatible endpoint, or OpenRouter +1. **Translation backend**: Ollama, OpenAI-compatible endpoint, OpenRouter, or Requesty 2. **OpenRouter API key** for translation evaluation (get one at [openrouter.ai](https://openrouter.ai)) + - For Requesty translation, get a key at [app.requesty.ai/api-keys](https://app.requesty.ai/api-keys) ## CLI Commands @@ -49,10 +50,11 @@ python -m benchmark.cli run [OPTIONS] | `-m, --models MODEL [MODEL ...]` | Specific provider models to test. If omitted, auto-detects available models from the selected provider | | `-l, --languages CODE [CODE ...]` | Language codes to test (e.g., `fr de ja zh`). Default: quick test set (19 languages) | | `--full` | Run full benchmark with all 40+ languages | -| `-p, --provider {ollama,openai,openrouter}` | Translation backend to benchmark | +| `-p, --provider {ollama,openai,openrouter,requesty}` | Translation backend to benchmark | | `--openai-key KEY` | API key for OpenAI-compatible translation backends | | `--openai-endpoint URL` | OpenAI-compatible endpoint or `/v1` base URL | | `--openrouter-key KEY` | OpenRouter API key (can also use `OPENROUTER_API_KEY` env var) | +| `--requesty-key KEY` | Requesty API key (can also use `REQUESTY_API_KEY` env var) | | `--evaluator MODEL` | OpenRouter model for evaluation (default: `anthropic/claude-haiku-4.5`) | | `--ollama-endpoint URL` | Custom Ollama endpoint (default: from `.env` or `http://localhost:11434/api/generate`) | | `--resume RUN_ID` | Resume an interrupted benchmark run | @@ -242,6 +244,10 @@ REQUEST_TIMEOUT=900 # OpenRouter Configuration (for evaluation) OPENROUTER_API_KEY=your_key_here +# Requesty Configuration (OpenAI-compatible translation provider) +# Get a key at https://app.requesty.ai/api-keys +REQUESTY_API_KEY=your_key_here + # Wiki Publishing WIKI_REPO_URL=https://github.com/your-username/TranslateBookWithLLM.wiki.git ``` diff --git a/docs/BENCHMARK_WORKFLOW.md b/docs/BENCHMARK_WORKFLOW.md index 5d94bd52..f87db855 100644 --- a/docs/BENCHMARK_WORKFLOW.md +++ b/docs/BENCHMARK_WORKFLOW.md @@ -53,7 +53,7 @@ Each skill confirms via `AskUserQuestion` before any user-visible action ```bash python -m benchmark.cli run \ - -p \ + -p \ -m \ --no-evaluate \ --pair-set