diff --git a/.fern/metadata.json b/.fern/metadata.json index 62289a3..a2b4d2c 100644 --- a/.fern/metadata.json +++ b/.fern/metadata.json @@ -14,5 +14,5 @@ }, "exclude_types_from_init_exports": true }, - "originGitCommit": "26314a9d3199f5e135302c3c3645b6bbb8f556d4" + "originGitCommit": "0073fa0ae670f77e4987407b8cca1cf179d5f90b" } \ No newline at end of file diff --git a/.fern/replay.lock b/.fern/replay.lock index 699a092..82f3c76 100644 --- a/.fern/replay.lock +++ b/.fern/replay.lock @@ -72,14 +72,28 @@ generations: cli_version: unknown generator_versions: fernapi/fern-python-sdk: 4.37.0 -current_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b + - commit_sha: 88a3914f38e882be10f7a0a6cba18e6f5ec98259 + tree_hash: 0315c486fe5d65313c32c2127af67aaa39f8acf7 + timestamp: 2026-09-24T03:09:07.591Z + cli_version: unknown + generator_versions: + fernapi/fern-python-sdk: 4.37.0 + - commit_sha: b1f9cb3bf3fd8da2262807fd603dee3741187918 + tree_hash: e378b36a4bb43fd42f28703a58814ae759fe9434 + timestamp: 2026-10-09T14:05:05.278Z + cli_version: unknown + generator_versions: + fernapi/fern-python-sdk: 4.37.0 +current_generation: b1f9cb3bf3fd8da2262807fd603dee3741187918 +forgotten_hashes: + - sha256:19124603cb0683406e750ffda0725786727208c0f7ddb481c256a0990b2c5585 patches: - id: patch-13d8e068 content_hash: sha256:97d879ab169016b9abaf1866427a8ebc45af608f69e063a07e5126c9613c329b original_commit: 13d8e0683ffa08bcfc9a381eb9604adf5abdbe0b original_message: "chore: ignore local venv and planning docs" original_author: plutoless - base_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b + base_generation: b1f9cb3bf3fd8da2262807fd603dee3741187918 files: - .gitignore patch_content: | @@ -108,7 +122,7 @@ patches: original_commit: a065088b4c45e2ca7e1bb91b1a899b8444800121 original_message: "fix(vendors): send Speechmatics STT key instead of api_key" original_author: digitallysavvy - base_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b + base_generation: b1f9cb3bf3fd8da2262807fd603dee3741187918 files: - src/agora_agent/types/speechmatics_asr_params.py patch_content: | @@ -251,7 +265,7 @@ patches: original_commit: 33a436b3bdb6f2a6fe4d8987b6536c2f0ac2862e original_message: "fix: address v2.8.0 release review findings" original_author: digitallysavvy - base_generation: f7e9b5d993665a16b43eb1d7d442b118fbeffa4b + base_generation: b1f9cb3bf3fd8da2262807fd603dee3741187918 files: - src/agora_agent/agents/types/start_agents_request_properties_advanced_features.py patch_content: | @@ -312,3 +326,4 @@ patches: frozen = True smart_union = True extra = pydantic.Extra.allow + user_owned: true diff --git a/changelog.md b/changelog.md index 7542307..b7c42fc 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/). +## [Unreleased] + +### Added + +- **RTZR global STT** — Added the global-only `RtzrSTT` AgentKit vendor with typed credentials, recognition options, vendor catalog registration, and generated request serialization. +- **Speak parameters** — Exposed the generated `SpeakConfig`/`ParametersSpeak` types through `SessionParams` so `parameters.speak.batch` can be configured with `Agent.with_parameters()`. + +### Changed + +- **Sarvam TTS v2.14 parameters** — Added `speech_sample_rate`, `enable_preprocessing`, `model`, and flattened `additional_params` support. The legacy `sample_rate` input remains supported as a deprecated alias. + - **Generated SDK alignment** — Updated AgentKit exports, global vendor routing, request validation, avatar sample-rate handling, documentation, and feature-focused regression tests for the latest `engine-v2.14` generated API. + +- **Gemini TTS production routing** — Synchronous and asynchronous `GeminiTTS` sessions, including raw configurations, now use the configured regional production endpoint without the `gemini-live` preview gate. The implementation is registered in the global TTS catalog, while package-root and historical preview imports remain aliases with the same v2.11.0 options, defaults, validation, optional `style`, and `tts.params` wire fields. Compatibility with the generated request schema no longer depends on preview routing. +- **Gemini TTS generated schema support** — The production vendor now uses the generated Gemini TTS shape and accepts `additional_params` and `skip_patterns` while keeping preview-era calls compatible. + ## [v2.12.0] — 2026-10-07 ### Added diff --git a/docs/concepts/vendors.md b/docs/concepts/vendors.md index 13f6cd5..e0c4918 100644 --- a/docs/concepts/vendors.md +++ b/docs/concepts/vendors.md @@ -60,6 +60,7 @@ Used with `agent.with_tts()`. Each TTS vendor produces audio at a specific sampl | `OpenAITTS` | OpenAI | `voice` for Agora-managed global `tts-1`; `api_key`, `model`, `base_url`, `voice` for BYOK | 24000 Hz (fixed) | | `CartesiaTTS` | Cartesia | `api_key`, `voice_id`, `model_id` | 8000–48000 Hz | | `GoogleTTS` | Google Cloud | `key`, `voice_name` | — | +| `GeminiTTS` | Google Gemini TTS | `api_key`; model and voice have defaults | — | | `AmazonTTS` | Amazon Polly | `access_key`, `secret_key`, `region`, `voice_id`, `engine` | — | | `HumeAITTS` | Hume AI | `key`, `voice_id`, `provider` | — | | `RimeTTS` | Rime | `model_id`; BYOK also requires `key` and `speaker`, managed requires `base_url` | — | @@ -125,6 +126,7 @@ top-level `asr.keywords`. Both vendors also accept `additional_params`, serializ | `AssemblyAISTT` | AssemblyAI | `api_key`, `language` | | `AresSTT` | Ares | — (all optional) | | `SarvamSTT` | Sarvam | `api_key`, `language` | +| `RtzrSTT` | RTZR | `client_id`, `client_secret` | | `XaiSTT` | xAI | `api_key` | ### CN STT Vendors diff --git a/docs/guides/avatars.md b/docs/guides/avatars.md index 9e085ff..3f3c5bc 100644 --- a/docs/guides/avatars.md +++ b/docs/guides/avatars.md @@ -86,6 +86,25 @@ session.say('Hello! I am your visual assistant.') session.stop() ``` +## Anam Avatar + +Anam supports portrait models such as Cara 4 through `avatar_model`. Set +`video_width` and `video_height` together for a custom resolution, or omit both +to use the model default: + +```python +from agora_agent import AnamAvatar + +avatar = AnamAvatar( + api_key="your-anam-key", + avatar_id="your-avatar-id", + avatar_model="cara_mk4", + video_width=720, + video_height=1280, +) +agent = agent.with_avatar(avatar) +``` + ## Generic Avatar `GenericAvatar` supports custom avatar providers. `agora_appid`, `agora_channel`, and `agora_token` are optional when using `AgentSession.start()`. diff --git a/docs/guides/preview-endpoint.md b/docs/guides/preview-endpoint.md index dcb2a59..44fe0d4 100644 --- a/docs/guides/preview-endpoint.md +++ b/docs/guides/preview-endpoint.md @@ -10,7 +10,7 @@ Some providers may be released through a preview gateway before their production and `AsyncAgentSession` detect registered preview providers from the resolved start request and route the entire session automatically. -OpenAI GPT Live, Gemini STT, and Gemini Live have graduated to the production gateway and use the normal regional +OpenAI GPT Live, Gemini STT, Gemini Live, and Gemini TTS have graduated to the production gateway and use the normal regional endpoint. Existing imports from `agora_agent.agentkit.preview` remain supported as compatibility aliases. ```python @@ -24,7 +24,7 @@ session = ( agent_id = session.start() ``` -Preview providers use the preview base URL and `agora-feature` gate. Sessions using `GeminiSTT`, `GeminiLive`, or GPT +Preview providers use the preview base URL and `agora-feature` gate. Sessions using `GeminiSTT`, `GeminiLive`, `GeminiTTS`, or GPT Live use the client's normal production regional endpoint without that header. Use the single production `GeminiLive(api_key=..., model=...)` class with `with_mllm`. The model IDs are @@ -69,23 +69,26 @@ Add routing tests for both synchronous and asynchronous sessions when registerin the preview base URL, exact feature header, all lifecycle requests, caller-header precedence, and that the original client remains configured for production. -## Gemini 3.8 Flash TTS preview +## Gemini 3.8 Flash TTS production migration `GeminiTTS` emits `tts.vendor = "gemini"` with `api_key`, `model`, `voice`, and optional `style` inside `tts.params`. It defaults to `gemini-3.8-flash-tts` -and `Puck`. Model names are sent unchanged; there is no automatic fallback -or model rewriting. -Model strings remain open for preview rollout changes. Blank keys are rejected. - -AgentSession detects the TTS vendor from the resolved request body, including -handwritten configs, and uses the existing preview host with -`agora-feature: gemini-live` throughout the session lifecycle. Use the retained -session for stop/say/interrupt; the shared client remains on its normal route. -Gemini ASR alone still uses the production route. No sample-rate or avatar -compatibility is assumed by this preview provider. +and `Puck`. Model names are sent unchanged; there is no automatic fallback or +model rewriting. +Model strings remain open for future model IDs. Blank keys are rejected. + +Existing v2.11.0 calls now use the client's configured regional production +endpoint throughout both synchronous and asynchronous session lifecycles. +Handwritten Gemini TTS configs follow the same route, and no preview feature +header is added. `GeminiTTS` and `GeminiTTSModels` now live in `agentkit.vendors.tts` +and are registered in the global vendor catalog. Imports from the package root, +`agora_agent.agentkit.preview`, and `agora_agent.agentkit.preview.gemini_tts` +remain aliases to the same production objects. `skip_patterns` maps to the +generated top-level TTS field. `additional_params` is merged into `tts.params`, +with named options taking precedence. The provider does not expose a configurable sample rate. ```python -from agora_agent.agentkit.preview import GeminiTTS, GeminiTTSModels +from agora_agent import GeminiTTS, GeminiTTSModels agent.with_tts(GeminiTTS( api_key=google_api_key, diff --git a/docs/guides/regional-routing.md b/docs/guides/regional-routing.md index ff6e4cc..f97e6c5 100644 --- a/docs/guides/regional-routing.md +++ b/docs/guides/regional-routing.md @@ -40,7 +40,7 @@ If you omit `with_stt()`, AgentKit uses `FengmingSTT` by default for `Area.CN` c | Client area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | |---|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GeminiSTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GeminiSTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GeminiTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global client example: diff --git a/docs/reference/agent.md b/docs/reference/agent.md index 68e46f1..f4e8aed 100644 --- a/docs/reference/agent.md +++ b/docs/reference/agent.md @@ -178,6 +178,7 @@ Enable or disable MCP and inline REST tool invocation by setting `advanced_featu ### `with_parameters(parameters: SessionParams) -> Agent` Set session parameters (silence config, farewell config, data channel, audio scenario, etc.). +Use `SpeakConfig(batch=False)` in the `speak` field to control sentence segmentation for speak requests. ### `with_audio_scenario(audio_scenario: ParametersAudioScenario) -> Agent` diff --git a/docs/reference/vendors.md b/docs/reference/vendors.md index 959690b..57a8058 100644 --- a/docs/reference/vendors.md +++ b/docs/reference/vendors.md @@ -20,7 +20,7 @@ Construct vendors directly from `agora_agent`, then bind a client with `Agent(cl | Area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | |---|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GeminiSTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GeminiSTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GeminiTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global example: @@ -272,6 +272,25 @@ The SDK also includes named helpers for the remaining Agora-supported LLM provid | `sample_rate` | `int` | No | `None` | Sample rate: 8000–48000 Hz | | `skip_patterns` | `List[int]` | No | `None` | Skip patterns | +### `GeminiTTS` + +Uses the configured regional production endpoint for synchronous and asynchronous +sessions. Package-root and historical preview imports alias the production class. +Credentials and options serialize inside `tts.params`, with `tts.vendor = "gemini"`. + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Google Gemini API key | +| `model` | `str` | No | `GeminiTTSModels.FLASH_38` | Model identifier, sent verbatim | +| `voice` | `str` | No | `Puck` | Voice name | +| `style` | `Optional[str]` | No | `None` | Natural-language speaking instruction | +| `additional_params` | `Optional[Dict[str, Any]]` | No | `None` | Provider-specific parameters merged into `tts.params`; named options take precedence | +| `skip_patterns` | `Optional[List[int]]` | No | `None` | Patterns to skip in TTS output | + +Blank credentials, model, or voice are rejected. Existing v2.11.0 calls and raw +configs remain compatible without a preview feature header. No sample rate +option is exposed. + ### `GoogleTTS` | Parameter | Type | Required | Default | Description | @@ -446,7 +465,11 @@ AgentKit serializes `credential_mode` at the top level of the Rime TTS configura | `pitch` | `float` | No | `None` | Pitch adjustment | | `pace` | `float` | No | `None` | Speed of speech | | `loudness` | `float` | No | `None` | Volume level | -| `sample_rate` | `int` | No | `None` | Audio sample rate | +| `speech_sample_rate` | `int` | No | `None` | Output speech sample rate in Hz | +| `enable_preprocessing` | `bool` | No | `None` | Normalize English words and numeric entities before synthesis | +| `model` | `str` | No | `None` | Sarvam TTS model; defaults to `bulbul:v3` | +| `sample_rate` | `int` | No | `None` | Deprecated alias for `speech_sample_rate` | +| `additional_params` | `Dict[str, Any]` | No | `None` | Additional Sarvam TTS parameters, flattened into `params` | | `skip_patterns` | `List[int]` | No | `None` | Skip patterns | ### `SmallestAITTS` @@ -636,6 +659,26 @@ stt = GeminiSTT( | `language` | `str` | No | `None` | Language code for speech recognition | | `additional_params` | `Dict[str, Any]` | No | `None` | Additional xAI STT parameters | +### `RtzrSTT` + +Global-only RTZR speech-to-text provider (`asr.vendor: "rtzr"`). + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `client_id` | `str` | Yes | — | RTZR client ID | +| `client_secret` | `str` | Yes | — | RTZR client secret | +| `api_base` | `str` | No | `None` | RTZR API base URL | +| `model_name` | `str` | No | `None` | RTZR recognition model name | +| `language` | `str` | No | `None` | RTZR recognition language code | +| `sample_rate` | `int` | No | `None` | Input audio sample rate in Hz | +| `encoding` | `str` | No | `None` | Input audio encoding | +| `use_itn` | `bool` | No | `None` | Enable inverse text normalization | +| `use_disfluency_filter` | `bool` | No | `None` | Filter disfluencies | +| `use_profanity_filter` | `bool` | No | `None` | Filter profanity | +| `use_punctuation` | `bool` | No | `None` | Add punctuation to recognized text | +| `keywords` | `List[str]` | No | `None` | Keywords that improve recognition accuracy | +| `additional_params` | `Dict[str, Any]` | No | `None` | Additional RTZR parameters | + ### `SmallestAISTT` Global-only Smallest AI speech-to-text provider (`asr.vendor: "smallestai"`). @@ -1111,6 +1154,9 @@ Same options as `HeyGenAvatar`, but serializes `vendor: "liveavatar"`. `agora_to |---|---|---|---|---| | `api_key` | `str` | Yes | — | Anam API key | | `avatar_id` | `str` | Yes | — | Anam avatar ID | +| `avatar_model` | `str` | No | `None` | Anam model, such as `cara_mk4` for Cara 4 portrait mode | +| `video_width` | `int` | No | `None` | Output width in pixels; set together with `video_height` | +| `video_height` | `int` | No | `None` | Output height in pixels; set together with `video_width` | | `enable` | `bool` | No | `True` | Enable or disable the avatar | ### `GenericAvatar` diff --git a/src/agora_agent/__init__.py b/src/agora_agent/__init__.py index f731ff6..1a29ed4 100644 --- a/src/agora_agent/__init__.py +++ b/src/agora_agent/__init__.py @@ -72,6 +72,9 @@ GeminiTTSModels, OpenAIGPTLive, OpenAISTT, + RtzrSTT, + SpeakConfig, + ParametersSpeak, OpenAITTS, RimeTTS, SarvamSTT, diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index 0844a5c..0e658e8 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -55,6 +55,8 @@ FarewellConfig, ParametersDataChannel, ParametersAudioScenario, + SpeakConfig, + ParametersSpeak, InterruptionConfig, InterruptionMode, MllmTurnDetectionConfig, @@ -155,7 +157,6 @@ OpenAITtsPresetModels, normalize_preset_input, ) -from .preview import GeminiTTS, GeminiTTSModels from .vendors import ( GEMINI_MLLM_DEFAULT_MODEL, AkoolAvatar, @@ -185,6 +186,8 @@ Gemini, GeminiSTT, GeminiSTTModels, + GeminiTTS, + GeminiTTSModels, GeminiLive, GeminiLiveModels, GenericAvatar, @@ -211,6 +214,7 @@ OpenAIGPTLive, OpenAISampleRate, OpenAISTT, + RtzrSTT, OpenAITTS, GenericTTS, RimeTTS, @@ -309,6 +313,8 @@ "FarewellConfig", "ParametersDataChannel", "ParametersAudioScenario", + "SpeakConfig", + "ParametersSpeak", "InterruptionConfig", "InterruptionMode", "MllmTurnDetectionConfig", @@ -455,6 +461,7 @@ "MicrosoftSTT", "MicrosoftCNSTT", "OpenAISTT", + "RtzrSTT", "GeminiSTT", "GeminiSTTModels", "GoogleSTT", diff --git a/src/agora_agent/agentkit/agent.py b/src/agora_agent/agentkit/agent.py index 5eb5659..34be68c 100644 --- a/src/agora_agent/agentkit/agent.py +++ b/src/agora_agent/agentkit/agent.py @@ -47,6 +47,7 @@ from ..agents.types.start_agents_request_properties_parameters_farewell_config import StartAgentsRequestPropertiesParametersFarewellConfig from ..agents.types.start_agents_request_properties_parameters_data_channel import StartAgentsRequestPropertiesParametersDataChannel from ..agents.types.start_agents_request_properties_parameters_audio_scenario import StartAgentsRequestPropertiesParametersAudioScenario +from ..agents.types.start_agents_request_properties_parameters_speak import StartAgentsRequestPropertiesParametersSpeak from ..agents.types.start_agents_request_properties_interruption import StartAgentsRequestPropertiesInterruption from ..agents.types.start_agents_request_properties_interruption_mode import StartAgentsRequestPropertiesInterruptionMode from ..agents.types.start_agents_request_properties_geofence import StartAgentsRequestPropertiesGeofence @@ -140,6 +141,8 @@ FarewellConfig = StartAgentsRequestPropertiesParametersFarewellConfig ParametersDataChannel = StartAgentsRequestPropertiesParametersDataChannel ParametersAudioScenario = StartAgentsRequestPropertiesParametersAudioScenario +SpeakConfig = StartAgentsRequestPropertiesParametersSpeak +ParametersSpeak = SpeakConfig InterruptionConfig = StartAgentsRequestPropertiesInterruption InterruptionMode = StartAgentsRequestPropertiesInterruptionMode MllmTurnDetectionConfig = MllmTurnDetection @@ -160,6 +163,7 @@ class SessionParamsInput(typing_extensions.TypedDict, total=False): + speak: StartAgentsRequestPropertiesParametersSpeak silence_config: StartAgentsRequestPropertiesParametersSilenceConfig farewell_config: StartAgentsRequestPropertiesParametersFarewellConfig data_channel: StartAgentsRequestPropertiesParametersDataChannel diff --git a/src/agora_agent/agentkit/agent_session.py b/src/agora_agent/agentkit/agent_session.py index 18a2bac..b1171db 100644 --- a/src/agora_agent/agentkit/agent_session.py +++ b/src/agora_agent/agentkit/agent_session.py @@ -260,6 +260,7 @@ def _validate_avatar_config(self) -> None: sample_rate = ( tts_params.get("sample_rate") or tts_params.get("sample_rate_hertz") + or tts_params.get("speech_sample_rate") or tts_params.get("samplingRate") ) if isinstance(sample_rate, int): diff --git a/src/agora_agent/agentkit/avatar_types.py b/src/agora_agent/agentkit/avatar_types.py index 577803f..ad52941 100644 --- a/src/agora_agent/agentkit/avatar_types.py +++ b/src/agora_agent/agentkit/avatar_types.py @@ -102,6 +102,10 @@ def validate_avatar_config( raise ValueError("Anam avatar requires api_key") if not params.get("avatar_id"): raise ValueError("Anam avatar requires avatar_id") + has_video_width = params.get("video_width") is not None + has_video_height = params.get("video_height") is not None + if has_video_width != has_video_height: + raise ValueError("Anam avatar requires video_width and video_height together") elif is_generic_avatar(config): params = config.get("params", {}) if not params.get("api_key"): diff --git a/src/agora_agent/agentkit/presets.py b/src/agora_agent/agentkit/presets.py index f160cee..ce76a2c 100644 --- a/src/agora_agent/agentkit/presets.py +++ b/src/agora_agent/agentkit/presets.py @@ -108,7 +108,7 @@ def infer_asr_preset(asr: typing.Optional[typing.Dict[str, typing.Any]]) -> typi if not asr or asr.get("vendor") != "deepgram": return None params = asr.get("params") or {} - if params.get("key"): + if params.get("api_key"): return None return _DEEPGRAM_MODEL_TO_PRESET.get(_normalize_model_name(params.get("model")) or "") diff --git a/src/agora_agent/agentkit/preview/__init__.py b/src/agora_agent/agentkit/preview/__init__.py index 55b1675..62f089d 100644 --- a/src/agora_agent/agentkit/preview/__init__.py +++ b/src/agora_agent/agentkit/preview/__init__.py @@ -1,8 +1,4 @@ -"""Preview endpoint support. - -Temporary package: delete it when these providers ship on the production -gateway. See ``client.py`` for the routing and gate header. -""" +"""Preview routing helpers and legacy aliases for production vendors.""" from .client import ( PREVIEW_API_BASE_URL, @@ -13,7 +9,7 @@ create_preview_session_clients, required_preview_features, ) -from .gemini_tts import GeminiTTS, GeminiTTSModels +from ..vendors.tts import GeminiTTS, GeminiTTSModels from .vendors import ( GEMINI_MLLM_DEFAULT_MODEL, GEMINI_PREVIEW_MLLM_URL, diff --git a/src/agora_agent/agentkit/preview/client.py b/src/agora_agent/agentkit/preview/client.py index e917b1c..1cd30d6 100644 --- a/src/agora_agent/agentkit/preview/client.py +++ b/src/agora_agent/agentkit/preview/client.py @@ -5,9 +5,9 @@ for preview traffic while the caller's ``Agora`` / ``AsyncAgora`` client stays on its production endpoint. -Everything under ``agentkit/preview/`` is temporary. When a provider ships on -the production gateway, remove its preview registration and move its class into -the corresponding production vendor module. +Preview registrations are temporary. When a provider ships on the production +gateway, remove its registration and move its class into the corresponding +production vendor module, retaining historical imports as aliases. """ from __future__ import annotations @@ -37,7 +37,7 @@ class PreviewFeatures: vendors on the preview endpoint. """ - #: Gemini TTS preview gate. Gemini ASR and Live use production. + #: Legacy Gemini gate retained for compatibility. ASR, Live, and TTS use production. GEMINI_LIVE = "gemini-live" LIVE_MODELS = "live-models" @@ -131,10 +131,10 @@ def create_preview_session_clients( raise TypeError("Unsupported Agora client wrapper") -#: ASR vendors served only by the preview endpoint. +#: Vendors served only by the preview endpoint, keyed by category. _PREVIEW_FEATURES_BY_CATEGORY: typing.Dict[str, typing.Dict[str, PreviewFeature]] = { "asr": {}, - "tts": {"gemini": PreviewFeatures.GEMINI_LIVE}, + "tts": {}, "mllm": {}, } diff --git a/src/agora_agent/agentkit/preview/gemini_tts.py b/src/agora_agent/agentkit/preview/gemini_tts.py index fbf0282..9cc3b2e 100644 --- a/src/agora_agent/agentkit/preview/gemini_tts.py +++ b/src/agora_agent/agentkit/preview/gemini_tts.py @@ -1,35 +1,5 @@ -"""Gemini TTS preview; reuses the gemini-live session gate.""" +"""Legacy preview import aliases for the production Gemini TTS implementation.""" -from typing import Any, Dict, Optional +from ..vendors.tts import GeminiTTS, GeminiTTSModels -from ..vendors.base import BaseTTS -from pydantic import ConfigDict, Field, field_validator - - -class GeminiTTSModels: - """Gemini 3.8 Flash TTS preview model.""" - - FLASH_38 = "gemini-3.8-flash-tts" - - -class GeminiTTS(BaseTTS): - """Preview-only TTS; model names are sent verbatim for rollout flexibility.""" - - model_config = ConfigDict(extra="forbid") - api_key: str = Field(..., repr=False) - model: str = GeminiTTSModels.FLASH_38 - voice: str = "Puck" - style: Optional[str] = None - - @field_validator("api_key", "model", "voice") - @classmethod - def require_nonblank(cls, value: str) -> str: - if not value.strip(): - raise ValueError("GeminiTTS requires a nonblank value") - return value - - def to_config(self) -> Dict[str, Any]: - params: Dict[str, Any] = {"api_key": self.api_key, "model": self.model, "voice": self.voice} - if self.style is not None: - params["style"] = self.style - return {"vendor": "gemini", "params": params} +__all__ = ["GeminiTTS", "GeminiTTSModels"] diff --git a/src/agora_agent/agentkit/regional_agent.py b/src/agora_agent/agentkit/regional_agent.py index 8658154..e5ba3fc 100644 --- a/src/agora_agent/agentkit/regional_agent.py +++ b/src/agora_agent/agentkit/regional_agent.py @@ -34,6 +34,7 @@ GoogleSTT, MicrosoftSTT, OpenAISTT, + RtzrSTT, SarvamSTT, SmallestAISTT, SpeechmaticsSTT, @@ -57,6 +58,7 @@ DeepgramTTS, ElevenLabsTTS, FishAudioTTS, + GeminiTTS, GoogleTTS, GradiumTTS, HumeAITTS, @@ -91,10 +93,12 @@ AssemblyAISTT, SpeechmaticsSTT, SarvamSTT, + RtzrSTT, SmallestAISTT, XaiSTT, ] GlobalTTS = typing.Union[ + GeminiTTS, MicrosoftTTS, ElevenLabsTTS, MiniMaxTTS, diff --git a/src/agora_agent/agentkit/vendors/__init__.py b/src/agora_agent/agentkit/vendors/__init__.py index 6cd22b6..89bf147 100644 --- a/src/agora_agent/agentkit/vendors/__init__.py +++ b/src/agora_agent/agentkit/vendors/__init__.py @@ -58,6 +58,7 @@ GoogleSTT, MicrosoftSTT, OpenAISTT, + RtzrSTT, SarvamSTT, SmallestAISTT, SpeechmaticsSTT, @@ -69,6 +70,8 @@ DeepgramTTS, ElevenLabsTTS, FishAudioTTS, + GeminiTTS, + GeminiTTSModels, GoogleTTS, GradiumTTS, HumeAITTS, @@ -112,6 +115,8 @@ "CartesiaTTS", "DeepgramTTS", "GoogleTTS", + "GeminiTTS", + "GeminiTTSModels", "GradiumTTS", "AmazonTTS", "HumeAITTS", @@ -137,6 +142,7 @@ "MicrosoftSTT", "MicrosoftCNSTT", "OpenAISTT", + "RtzrSTT", "GeminiSTT", "GeminiSTTModels", "GoogleSTT", diff --git a/src/agora_agent/agentkit/vendors/avatar.py b/src/agora_agent/agentkit/vendors/avatar.py index 83952e3..01d86f9 100644 --- a/src/agora_agent/agentkit/vendors/avatar.py +++ b/src/agora_agent/agentkit/vendors/avatar.py @@ -2,7 +2,7 @@ from typing import Any, Dict, Optional from .base import BaseAvatar -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator LIVEAVATAR_SAMPLE_RATE = 24000 HEYGEN_SAMPLE_RATE = LIVEAVATAR_SAMPLE_RATE @@ -199,9 +199,30 @@ class AnamAvatarOptions(BaseModel): api_key: str = Field(..., description="Anam API key") avatar_id: str = Field(..., description="Anam avatar ID") + avatar_model: Optional[str] = Field( + default=None, + description="Anam avatar model, such as cara_mk4 for Cara 4 portrait mode", + ) + video_width: Optional[int] = Field( + default=None, + description="Output video width in pixels; set together with video_height", + ) + video_height: Optional[int] = Field( + default=None, + description="Output video height in pixels; set together with video_width", + ) enable: Optional[bool] = Field(default=None, description="Enable avatar (default: true)") additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional vendor-specific parameters") + @model_validator(mode="after") + def _validate_video_dimensions(self) -> "AnamAvatarOptions": + additional_params = self.additional_params or {} + has_video_width = self.video_width is not None or additional_params.get("video_width") is not None + has_video_height = self.video_height is not None or additional_params.get("video_height") is not None + if has_video_width != has_video_height: + raise ValueError("Anam avatar requires video_width and video_height together") + return self + class AnamAvatar(AnamAvatarOptions, BaseAvatar): @property @@ -209,13 +230,24 @@ def required_sample_rate(self) -> int: return 0 def to_config(self) -> Dict[str, Any]: - params: Dict[str, Any] = { - "api_key": self.api_key, - "avatar_id": self.avatar_id, - } - - if self.additional_params is not None: - params = {**self.additional_params, **params} + params: Dict[str, Any] = dict(self.additional_params or {}) + params.update( + { + "api_key": self.api_key, + "avatar_id": self.avatar_id, + } + ) + if self.avatar_model is not None: + params["avatar_model"] = self.avatar_model + if self.video_width is not None: + params["video_width"] = self.video_width + if self.video_height is not None: + params["video_height"] = self.video_height + + has_video_width = params.get("video_width") is not None + has_video_height = params.get("video_height") is not None + if has_video_width != has_video_height: + raise ValueError("Anam avatar requires video_width and video_height together") enable = self.enable if self.enable is not None else True return {"enable": enable, "vendor": "anam", "params": params} diff --git a/src/agora_agent/agentkit/vendors/catalog.py b/src/agora_agent/agentkit/vendors/catalog.py index ebfb918..7ee986b 100644 --- a/src/agora_agent/agentkit/vendors/catalog.py +++ b/src/agora_agent/agentkit/vendors/catalog.py @@ -26,6 +26,7 @@ GoogleSTT, MicrosoftSTT, OpenAISTT, + RtzrSTT, SarvamSTT, SmallestAISTT, SpeechmaticsSTT, @@ -37,6 +38,7 @@ DeepgramTTS, ElevenLabsTTS, FishAudioTTS, + GeminiTTS, GoogleTTS, GradiumTTS, HumeAITTS, @@ -83,6 +85,7 @@ def __init__( "assemblyai": AssemblyAISTT, "speechmatics": SpeechmaticsSTT, "sarvam": SarvamSTT, + "rtzr": RtzrSTT, "smallestai": SmallestAISTT, "xai": XaiSTT, }, @@ -106,6 +109,7 @@ def __init__( "xai": XaiGrok, }, tts={ + "gemini": GeminiTTS, "microsoft": MicrosoftTTS, "elevenlabs": ElevenLabsTTS, "minimax": MiniMaxTTS, diff --git a/src/agora_agent/agentkit/vendors/namespaces.py b/src/agora_agent/agentkit/vendors/namespaces.py index 782c5aa..b3d982b 100644 --- a/src/agora_agent/agentkit/vendors/namespaces.py +++ b/src/agora_agent/agentkit/vendors/namespaces.py @@ -23,6 +23,7 @@ GoogleSTT, MicrosoftSTT, OpenAISTT, + RtzrSTT, SarvamSTT, SmallestAISTT, SpeechmaticsSTT, @@ -34,6 +35,7 @@ DeepgramTTS, ElevenLabsTTS, FishAudioTTS, + GeminiTTS, GoogleTTS, GradiumTTS, HumeAITTS, @@ -62,6 +64,7 @@ class GlobalSTTVendors: assemblyai = AssemblyAISTT speechmatics = SpeechmaticsSTT sarvam = SarvamSTT + rtzr = RtzrSTT smallestai = SmallestAISTT xai = XaiSTT @@ -108,6 +111,7 @@ class CNMLLMVendors: class GlobalTTSVendors: + gemini = GeminiTTS microsoft = MicrosoftTTS elevenlabs = ElevenLabsTTS minimax = MiniMaxTTS diff --git a/src/agora_agent/agentkit/vendors/region.py b/src/agora_agent/agentkit/vendors/region.py index 495c8bd..d4944fc 100644 --- a/src/agora_agent/agentkit/vendors/region.py +++ b/src/agora_agent/agentkit/vendors/region.py @@ -28,6 +28,7 @@ "assemblyai", "speechmatics", "sarvam", + "rtzr", "smallestai", "xai", ) @@ -43,6 +44,7 @@ "generic", ) GLOBAL_TTS_VENDORS: typing.Tuple[str, ...] = ( + "gemini", "microsoft", "elevenlabs", "minimax", diff --git a/src/agora_agent/agentkit/vendors/stt.py b/src/agora_agent/agentkit/vendors/stt.py index f05fe79..ffeb9d1 100644 --- a/src/agora_agent/agentkit/vendors/stt.py +++ b/src/agora_agent/agentkit/vendors/stt.py @@ -97,7 +97,7 @@ def to_config(self) -> Dict[str, Any]: params: Dict[str, Any] = dict(self.additional_params or {}) if self.api_key is not None: - params["key"] = self.api_key + params["api_key"] = self.api_key if self.model is not None: params["model"] = self.model if self.language is not None: @@ -115,6 +115,50 @@ def to_config(self) -> Dict[str, Any]: return config +class RtzrSTTOptions(BaseModel): + model_config = ConfigDict(extra="forbid") + + client_id: str = Field(..., min_length=1, description="RTZR client ID") + client_secret: str = Field(..., min_length=1, description="RTZR client secret") + api_base: Optional[str] = Field(default=None, description="RTZR API base URL") + model_name: Optional[str] = Field(default=None, description="RTZR recognition model name") + language: Optional[str] = Field(default=None, description="RTZR recognition language code") + sample_rate: Optional[int] = Field(default=None, gt=0, description="Input audio sample rate in Hz") + encoding: Optional[str] = Field(default=None, description="Input audio encoding") + use_itn: Optional[bool] = Field(default=None, description="Enable inverse text normalization") + use_disfluency_filter: Optional[bool] = Field(default=None, description="Filter disfluencies such as stuttering") + use_profanity_filter: Optional[bool] = Field(default=None, description="Filter profanity") + use_punctuation: Optional[bool] = Field(default=None, description="Add punctuation to recognized text") + keywords: Optional[List[str]] = Field(default=None, description="Keywords that improve recognition accuracy") + additional_params: Optional[Dict[str, Any]] = Field(default=None) + + +class RtzrSTT(RtzrSTTOptions, BaseSTT): + """RTZR streaming speech-to-text provider.""" + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + params.update({"client_id": self.client_id, "client_secret": self.client_secret}) + + for name in ( + "api_base", + "model_name", + "language", + "sample_rate", + "encoding", + "use_itn", + "use_disfluency_filter", + "use_profanity_filter", + "use_punctuation", + "keywords", + ): + value = getattr(self, name) + if value is not None: + params[name] = value + + return {"vendor": "rtzr", "params": params} + + class MicrosoftSTTOptions(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/src/agora_agent/agentkit/vendors/tts.py b/src/agora_agent/agentkit/vendors/tts.py index 0d8d3ff..ba9f22f 100644 --- a/src/agora_agent/agentkit/vendors/tts.py +++ b/src/agora_agent/agentkit/vendors/tts.py @@ -1,3 +1,4 @@ +import warnings from typing import Any, Dict, List, Literal, Optional from urllib.parse import urlsplit @@ -7,6 +8,45 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +class GeminiTTSModels: + """Gemini 3.8 Flash TTS model.""" + + FLASH_38 = "gemini-3.8-flash-tts" + + +class GeminiTTS(BaseTTS): + """Production Gemini TTS; preview-era options and wire fields remain compatible.""" + + model_config = ConfigDict(extra="forbid") + api_key: str = Field(..., repr=False) + model: str = GeminiTTSModels.FLASH_38 + voice: str = "Puck" + style: Optional[str] = None + additional_params: Optional[Dict[str, Any]] = None + skip_patterns: Optional[List[int]] = None + + @field_validator("api_key", "model", "voice") + @classmethod + def require_nonblank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("GeminiTTS requires a nonblank value") + return value + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + params.update({ + "api_key": self.api_key, + "model": self.model, + "voice": self.voice, + }) + if self.style is not None: + params["style"] = self.style + result: Dict[str, Any] = {"vendor": "gemini", "params": params} + if self.skip_patterns is not None: + result["skip_patterns"] = self.skip_patterns + return result + + class ElevenLabsTTSOptions(BaseModel): model_config = ConfigDict(extra="forbid") @@ -568,9 +608,38 @@ class SarvamTTSOptions(BaseModel): pitch: Optional[float] = Field(default=None, description="Pitch adjustment") pace: Optional[float] = Field(default=None, description="Speed of speech") loudness: Optional[float] = Field(default=None, description="Volume level") - sample_rate: Optional[int] = Field(default=None, description="Audio sample rate in Hz") + speech_sample_rate: Optional[int] = Field(default=None, gt=0, description="Output speech sample rate in Hz") + enable_preprocessing: Optional[bool] = Field( + default=None, + description="Normalize English words and numeric entities before synthesis", + ) + model: Optional[str] = Field(default=None, description="Sarvam TTS model") + sample_rate: Optional[int] = Field( + default=None, + gt=0, + description="Deprecated alias for speech_sample_rate", + ) + additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional Sarvam TTS parameters") skip_patterns: Optional[List[int]] = Field(default=None) + @model_validator(mode="before") + @classmethod + def _warn_deprecated_sample_rate(cls, values: Any) -> Any: + if isinstance(values, dict) and "sample_rate" in values: + warnings.warn( + "SarvamTTS.sample_rate is deprecated; use speech_sample_rate instead.", + DeprecationWarning, + stacklevel=2, + ) + return values + + @model_validator(mode="after") + def _validate_sample_rate_alias(self) -> "SarvamTTSOptions": + legacy_sample_rate = self.__dict__.get("sample_rate") + if self.speech_sample_rate is not None and legacy_sample_rate is not None: + raise ValueError("SarvamTTS accepts only one of speech_sample_rate or sample_rate") + return self + class SarvamTTS(SarvamTTSOptions, BaseTTS): @property @@ -578,19 +647,26 @@ def resolved_sample_rate(self) -> Optional[int]: return None def to_config(self) -> Dict[str, Any]: - params: Dict[str, Any] = { + params: Dict[str, Any] = dict(self.additional_params or {}) + params.update({ "api_subscription_key": self.key, "speaker": self.speaker, "target_language_code": self.target_language_code, - } + }) if self.pitch is not None: params["pitch"] = self.pitch if self.pace is not None: params["pace"] = self.pace if self.loudness is not None: params["loudness"] = self.loudness - if self.sample_rate is not None: - params["sample_rate"] = self.sample_rate + legacy_sample_rate = self.__dict__.get("sample_rate") + speech_sample_rate = self.speech_sample_rate if self.speech_sample_rate is not None else legacy_sample_rate + if speech_sample_rate is not None: + params["speech_sample_rate"] = speech_sample_rate + if self.enable_preprocessing is not None: + params["enable_preprocessing"] = self.enable_preprocessing + if self.model is not None: + params["model"] = self.model result: Dict[str, Any] = {"vendor": "sarvam", "params": params} if self.skip_patterns is not None: diff --git a/src/agora_agent/agents/types/start_agents_request_properties_parameters.py b/src/agora_agent/agents/types/start_agents_request_properties_parameters.py index 33bb237..d3153e9 100644 --- a/src/agora_agent/agents/types/start_agents_request_properties_parameters.py +++ b/src/agora_agent/agents/types/start_agents_request_properties_parameters.py @@ -15,6 +15,7 @@ from .start_agents_request_properties_parameters_silence_config import ( StartAgentsRequestPropertiesParametersSilenceConfig, ) +from .start_agents_request_properties_parameters_speak import StartAgentsRequestPropertiesParametersSpeak class StartAgentsRequestPropertiesParameters(UncheckedBaseModel): @@ -22,6 +23,11 @@ class StartAgentsRequestPropertiesParameters(UncheckedBaseModel): Agent configuration parameters. """ + speak: typing.Optional[StartAgentsRequestPropertiesParametersSpeak] = pydantic.Field(default=None) + """ + Settings for the agent's speak behavior. + """ + silence_config: typing.Optional[StartAgentsRequestPropertiesParametersSilenceConfig] = pydantic.Field(default=None) """ Settings related to agent silence behavior. Does not apply when you integrate a `mllm`. diff --git a/src/agora_agent/agents/types/start_agents_request_properties_parameters_speak.py b/src/agora_agent/agents/types/start_agents_request_properties_parameters_speak.py new file mode 100644 index 0000000..95003bc --- /dev/null +++ b/src/agora_agent/agents/types/start_agents_request_properties_parameters_speak.py @@ -0,0 +1,29 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import IS_PYDANTIC_V2 +from ...core.unchecked_base_model import UncheckedBaseModel + + +class StartAgentsRequestPropertiesParametersSpeak(UncheckedBaseModel): + """ + Settings for the agent's speak behavior. + """ + + batch: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to skip sentence segmentation for speak requests: + - `false`: Skip sentence segmentation. + - Omitted or `true`: Preserve sentence segmentation. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/core/client_wrapper.py b/src/agora_agent/core/client_wrapper.py index ad3aa18..4f480c9 100644 --- a/src/agora_agent/core/client_wrapper.py +++ b/src/agora_agent/core/client_wrapper.py @@ -26,10 +26,10 @@ def __init__( def get_headers(self) -> typing.Dict[str, str]: headers: typing.Dict[str, str] = { - "User-Agent": "agora-agents/v2.12.0", + "User-Agent": "agora-agents/v2.13.0", "X-Fern-Language": "Python", "X-Fern-SDK-Name": "agora-agents", - "X-Fern-SDK-Version": "v2.12.0", + "X-Fern-SDK-Version": "v2.13.0", **(self.get_custom_headers() or {}), } headers["Authorization"] = httpx.BasicAuth(self._get_username(), self._get_password())._auth_header diff --git a/src/agora_agent/types/asr.py b/src/agora_agent/types/asr.py index 838ea54..dc5461d 100644 --- a/src/agora_agent/types/asr.py +++ b/src/agora_agent/types/asr.py @@ -18,6 +18,7 @@ from .google_asr_params import GoogleAsrParams from .microsoft_asr_params import MicrosoftAsrParams from .open_ai_asr_params import OpenAiAsrParams +from .rtzr_asr_params import RtzrAsrParams from .sarvam_asr_params import SarvamAsrParams from .smallest_ai_asr_params import SmallestAiAsrParams from .speechmatics_asr_params import SpeechmaticsAsrParams @@ -210,6 +211,21 @@ class Config: extra = pydantic.Extra.allow +class Asr_Rtzr(UncheckedBaseModel): + vendor: typing.Literal["rtzr"] = "rtzr" + language: typing.Optional[AsrLanguage] = None + params: RtzrAsrParams + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + class Asr_Xai(UncheckedBaseModel): vendor: typing.Literal["xai"] = "xai" language: typing.Optional[AsrLanguage] = None @@ -299,6 +315,7 @@ class Config: Asr_Assemblyai, Asr_Speechmatics, Asr_Sarvam, + Asr_Rtzr, Asr_Xai, Asr_Xfyun, Asr_XfyunBigmodel, diff --git a/src/agora_agent/types/deepgram_asr_params.py b/src/agora_agent/types/deepgram_asr_params.py index 6688333..e1f3656 100644 --- a/src/agora_agent/types/deepgram_asr_params.py +++ b/src/agora_agent/types/deepgram_asr_params.py @@ -17,7 +17,7 @@ class DeepgramAsrParams(UncheckedBaseModel): WebSocket URL for Deepgram's streaming API """ - key: str = pydantic.Field() + api_key: str = pydantic.Field() """ Deepgram API key """ diff --git a/src/agora_agent/types/gemini_tts.py b/src/agora_agent/types/gemini_tts.py new file mode 100644 index 0000000..c58a273 --- /dev/null +++ b/src/agora_agent/types/gemini_tts.py @@ -0,0 +1,29 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel +from .gemini_tts_params import GeminiTtsParams + + +class GeminiTts(UncheckedBaseModel): + """ + Gemini Text-to-Speech configuration. + """ + + params: GeminiTtsParams + skip_patterns: typing.Optional[typing.List[int]] = pydantic.Field(default=None) + """ + Controls whether the TTS module skips bracketed content when reading LLM response text. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/gemini_tts_params.py b/src/agora_agent/types/gemini_tts_params.py new file mode 100644 index 0000000..70983fb --- /dev/null +++ b/src/agora_agent/types/gemini_tts_params.py @@ -0,0 +1,42 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel + + +class GeminiTtsParams(UncheckedBaseModel): + """ + Gemini TTS configuration parameters. + """ + + api_key: str = pydantic.Field() + """ + Gemini API key. + """ + + model: str = pydantic.Field() + """ + Gemini TTS model name. + """ + + voice: str = pydantic.Field() + """ + Gemini voice name. + """ + + style: typing.Optional[str] = pydantic.Field(default=None) + """ + Style instruction for the generated speech. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/rtzr_asr.py b/src/agora_agent/types/rtzr_asr.py new file mode 100644 index 0000000..e49023d --- /dev/null +++ b/src/agora_agent/types/rtzr_asr.py @@ -0,0 +1,27 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel +from .asr_language import AsrLanguage +from .rtzr_asr_params import RtzrAsrParams + + +class RtzrAsr(UncheckedBaseModel): + """ + RTZR ASR configuration. + """ + + language: typing.Optional[AsrLanguage] = None + params: RtzrAsrParams + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/rtzr_asr_params.py b/src/agora_agent/types/rtzr_asr_params.py new file mode 100644 index 0000000..40f2baa --- /dev/null +++ b/src/agora_agent/types/rtzr_asr_params.py @@ -0,0 +1,82 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +from ..core.unchecked_base_model import UncheckedBaseModel + + +class RtzrAsrParams(UncheckedBaseModel): + """ + RTZR ASR configuration parameters. + """ + + client_id: str = pydantic.Field() + """ + RTZR client ID. + """ + + client_secret: str = pydantic.Field() + """ + RTZR client secret. + """ + + api_base: typing.Optional[str] = pydantic.Field(default=None) + """ + RTZR API base URL. + """ + + model_name: typing.Optional[str] = pydantic.Field(default=None) + """ + RTZR recognition model name. + """ + + language: typing.Optional[str] = pydantic.Field(default=None) + """ + RTZR recognition language code. Defaults to Korean (`ko`). + """ + + sample_rate: typing.Optional[int] = pydantic.Field(default=None) + """ + Input audio sample rate in Hz. + """ + + encoding: typing.Optional[str] = pydantic.Field(default=None) + """ + Input audio encoding. + """ + + use_itn: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to enable inverse text normalization. + """ + + use_disfluency_filter: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to filter disfluencies such as stuttering. + """ + + use_profanity_filter: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to filter profanity. + """ + + use_punctuation: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to add punctuation to the recognized text. + """ + + keywords: typing.Optional[typing.List[str]] = pydantic.Field(default=None) + """ + Keywords to improve recognition accuracy. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/agora_agent/types/sarvam_tts_params.py b/src/agora_agent/types/sarvam_tts_params.py index 855299f..d544dea 100644 --- a/src/agora_agent/types/sarvam_tts_params.py +++ b/src/agora_agent/types/sarvam_tts_params.py @@ -20,32 +20,42 @@ class SarvamTtsParams(UncheckedBaseModel): speaker: str = pydantic.Field() """ - Voice ID (e.g., anushka, abhilash, karun, hitesh, manisha, vidya, arya) + Speaker voice to use. """ target_language_code: SarvamTtsParamsTargetLanguageCode = pydantic.Field() """ - Target language code (e.g., en-IN) + Target language code in BCP-47 format (e.g., `hi-IN`, `bn-IN`, `en-IN`). """ pitch: typing.Optional[float] = pydantic.Field(default=None) """ - Pitch adjustment for the voice + Pitch control for the `bulbul:v2` model. """ pace: typing.Optional[float] = pydantic.Field(default=None) """ - Speed of speech + Speech speed. Defaults to `1.0`. """ loudness: typing.Optional[float] = pydantic.Field(default=None) """ - Volume level of the speech + Audio loudness control for the `bulbul:v2` model. """ - sample_rate: typing.Optional[float] = pydantic.Field(default=None) + speech_sample_rate: typing.Optional[int] = pydantic.Field(default=None) """ - Audio sample rate in Hz + Output speech sample rate in Hz. Defaults to `24000`. + """ + + enable_preprocessing: typing.Optional[bool] = pydantic.Field(default=None) + """ + Whether to normalize English words and numeric entities. Defaults to `false`. + """ + + model: typing.Optional[str] = pydantic.Field(default=None) + """ + TTS model to use. Defaults to `bulbul:v3`. """ if IS_PYDANTIC_V2: diff --git a/src/agora_agent/types/tts.py b/src/agora_agent/types/tts.py index ce737ad..e83d139 100644 --- a/src/agora_agent/types/tts.py +++ b/src/agora_agent/types/tts.py @@ -16,6 +16,7 @@ from .deepgram_tts_params import DeepgramTtsParams from .eleven_labs_tts_params import ElevenLabsTtsParams from .fish_audio_tts_params import FishAudioTtsParams +from .gemini_tts_params import GeminiTtsParams from .generic_http_tts_params import GenericHttpTtsParams from .google_tts_params import GoogleTtsParams from .gradium_tts_params import GradiumTtsParams @@ -216,6 +217,21 @@ class Config: extra = pydantic.Extra.allow +class Tts_Gemini(UncheckedBaseModel): + vendor: typing.Literal["gemini"] = "gemini" + params: GeminiTtsParams + skip_patterns: typing.Optional[typing.List[int]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + class Tts_Amazon(UncheckedBaseModel): vendor: typing.Literal["amazon"] = "amazon" params: AmazonTtsParams @@ -412,6 +428,7 @@ class Config: Tts_Rime, Tts_Fishaudio, Tts_Google, + Tts_Gemini, Tts_Amazon, Tts_Sarvam, Tts_GenericHttp, diff --git a/tests/custom/test_agentkit_agent.py b/tests/custom/test_agentkit_agent.py index d7cf47e..6d9cd5d 100644 --- a/tests/custom/test_agentkit_agent.py +++ b/tests/custom/test_agentkit_agent.py @@ -6,6 +6,7 @@ LlmStyle, MllmConfig, MllmVendor, + SpeakConfig, SttConfig, SttVendor, TtsConfig, @@ -85,6 +86,21 @@ def test_with_audio_scenario_preserves_existing_parameters(): assert _parameter(agent.config, "audio_scenario") == "chorus" +def test_with_parameters_serializes_speak_settings(): + agent = Agent(test_client()).with_parameters({"speak": SpeakConfig(batch=False)}) + + properties = agent.to_properties( + channel="room", + agent_uid="1", + remote_uids=["100"], + token="token", + skip_vendor_validation=True, + ) + + assert properties.parameters.speak is not None + assert properties.parameters.speak.batch is False + + def test_enable_rtm_defaults_data_channel_to_rtm(): properties = Agent(test_client(), advanced_features={"enable_rtm": True}).to_properties( channel="room", diff --git a/tests/custom/test_anam_avatar.py b/tests/custom/test_anam_avatar.py index a4fa395..4cb12ef 100644 --- a/tests/custom/test_anam_avatar.py +++ b/tests/custom/test_anam_avatar.py @@ -22,11 +22,48 @@ def test_anam_avatar_to_config_shape() -> None: assert is_anam_avatar(config) +def test_anam_avatar_serializes_portrait_options_and_overrides_additional_params() -> None: + config = AnamAvatar( + api_key="anam-key", + avatar_id="anam-avatar", + avatar_model="cara_mk4", + video_width=720, + video_height=1280, + additional_params={ + "avatar_model": "overridden-model", + "video_width": 1, + "video_height": 2, + }, + ).to_config() + + assert config["params"] == { + "api_key": "anam-key", + "avatar_id": "anam-avatar", + "avatar_model": "cara_mk4", + "video_width": 720, + "video_height": 1280, + } + + +@pytest.mark.parametrize("kwargs", [{"video_width": 720}, {"video_height": 1280}]) +def test_anam_avatar_requires_video_dimensions_as_pair(kwargs: dict) -> None: + with pytest.raises(ValueError, match="Anam avatar requires video_width and video_height together"): + AnamAvatar(api_key="anam-key", avatar_id="anam-avatar", **kwargs) + + @pytest.mark.parametrize( ("params", "message"), [ ({}, "Anam avatar requires api_key"), ({"api_key": "key"}, "Anam avatar requires avatar_id"), + ( + {"api_key": "key", "avatar_id": "avatar", "video_width": 720}, + "Anam avatar requires video_width and video_height together", + ), + ( + {"api_key": "key", "avatar_id": "avatar", "video_height": 1280}, + "Anam avatar requires video_width and video_height together", + ), ], ) def test_validate_avatar_config_rejects_incomplete_anam( diff --git a/tests/custom/test_gemini_tts.py b/tests/custom/test_gemini_tts.py index c3a491a..64f8437 100644 --- a/tests/custom/test_gemini_tts.py +++ b/tests/custom/test_gemini_tts.py @@ -1,20 +1,31 @@ import json +from typing import get_args import httpx import pytest from agora_agent import Agent, Agora, Area, AsyncAgora, Gemini, GeminiSTT, GeminiTTS, GeminiTTSModels from agora_agent.agentkit.preview import PREVIEW_API_BASE_URL, required_preview_features +from agora_agent.agentkit.preview import GeminiTTS as PreviewGeminiTTS +from agora_agent.agentkit.preview.gemini_tts import GeminiTTS as LegacyGeminiTTS +from agora_agent.agentkit.regional_agent import GlobalTTS +from agora_agent.agentkit.vendors import GeminiTTS as VendorGeminiTTS +from agora_agent.agentkit.vendors.base import BaseSTT, BaseTTS +from agora_agent.agentkit.vendors.catalog import GLOBAL_VENDOR_NAMESPACE +from agora_agent.agentkit.vendors.namespaces import GlobalTTSVendors +from agora_agent.agentkit.vendors.region import CN_TTS_VENDORS, GLOBAL_TTS_VENDORS +from agora_agent.agentkit.vendors.tts import GeminiTTS as ProductionGeminiTTS +from agora_agent.types.tts import Tts_Gemini -MODELS = [GeminiTTSModels.FLASH_38] +MODELS = [GeminiTTSModels.FLASH_38, "future-tts-model"] -def make_agent(client, model): +def make_agent(client, model, tts_type=GeminiTTS): return ( Agent(client=client) .with_stt(GeminiSTT(api_key="test-key")) .with_llm(Gemini(api_key="test-key", model="gemini-3.6-flash")) - .with_tts(GeminiTTS(api_key="test-key", model=model, voice="Puck", style="warm and reassuring")) + .with_tts(tts_type(api_key="test-key", model=model, voice="Puck", style="warm and reassuring")) ) @@ -26,58 +37,85 @@ def handle(request): return httpx.MockTransport(handle) -def assert_requests(requests, model): - assert len(requests) == 4 +def assert_requests(requests, model, production_url): + assert len(requests) == 9 assert json.loads(requests[0].content)["properties"]["tts"] == { "vendor": "gemini", "params": {"api_key": "test-key", "model": model, "voice": "Puck", "style": "warm and reassuring"}, } for request in requests: - assert str(request.url).startswith(PREVIEW_API_BASE_URL) - assert request.headers["agora-feature"] == "gemini-live" + assert str(request.url).startswith(production_url + "/") + assert "agora-feature" not in request.headers assert request.headers["x-custom"] == "kept" @pytest.mark.parametrize("model", MODELS) -def test_sync_lifecycle(model): +@pytest.mark.parametrize("area", [Area.US, Area.EU, Area.AP]) +@pytest.mark.parametrize( + "tts_type", [GeminiTTS, PreviewGeminiTTS, LegacyGeminiTTS], ids=["root", "preview", "legacy-module"] +) +def test_sync_lifecycle(model, area, tts_type): requests = [] with httpx.Client(transport=transport(requests)) as http_client: client = Agora( - area=Area.US, + area=area, app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client, - headers={"agora-feature": "", "x-custom": "kept"}, + headers={"x-custom": "kept"}, ) production_url = client.get_current_url() - session = make_agent(client, model).create_session(channel="test", agent_uid="1", remote_uids=["100"]) + session = make_agent(client, model, tts_type).create_session(channel="test", agent_uid="1", remote_uids=["100"]) session.start() session.say("hello") session.interrupt() + session.think("think") + session.update({}) + session.get_history() + session.get_info() + session.get_turns() session.stop() - assert_requests(requests, model) + assert_requests(requests, model, production_url) assert client.get_current_url() == production_url - assert client._client_wrapper.get_custom_headers()["agora-feature"] == "" + client.agents.get("0" * 32, "agent-1") + assert str(requests[-1].url).startswith(production_url + "/") + assert "agora-feature" not in requests[-1].headers @pytest.mark.asyncio @pytest.mark.parametrize("model", MODELS) -async def test_async_lifecycle(model): +@pytest.mark.parametrize("area", [Area.US, Area.EU, Area.AP]) +@pytest.mark.parametrize( + "tts_type", [GeminiTTS, PreviewGeminiTTS, LegacyGeminiTTS], ids=["root", "preview", "legacy-module"] +) +async def test_async_lifecycle(model, area, tts_type): requests = [] async with httpx.AsyncClient(transport=transport(requests)) as http_client: client = AsyncAgora( - area=Area.US, + area=area, app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client, - headers={"agora-feature": "", "x-custom": "kept"}, + headers={"x-custom": "kept"}, + ) + production_url = client.get_current_url() + session = make_agent(client, model, tts_type).create_async_session( + channel="test", agent_uid="1", remote_uids=["100"] ) - session = make_agent(client, model).create_async_session(channel="test", agent_uid="1", remote_uids=["100"]) await session.start() await session.say("hello") await session.interrupt() + await session.think("think") + await session.update({}) + await session.get_history() + await session.get_info() + await session.get_turns() await session.stop() - assert_requests(requests, model) + assert_requests(requests, model, production_url) + assert client.get_current_url() == production_url + await client.agents.get("0" * 32, "agent-1") + assert str(requests[-1].url).startswith(production_url + "/") + assert "agora-feature" not in requests[-1].headers def test_defaults_and_raw_detection(): @@ -85,23 +123,37 @@ def test_defaults_and_raw_detection(): "vendor": "gemini", "params": {"api_key": "test-key", "model": "gemini-3.8-flash-tts", "voice": "Puck"}, } - assert required_preview_features({"tts": {"vendor": "gemini", "params": {"model": "future-model"}}}) == [ - "gemini-live" - ] - assert set(required_preview_features({"tts": {"vendor": "gemini"}, "mllm": {"vendor": "openai_gpt_live"}})) == { - "gemini-live", - } + assert required_preview_features({"tts": {"vendor": "gemini", "params": {"model": "future-model"}}}) == [] + assert required_preview_features({"tts": {"vendor": "gemini"}, "mllm": {"vendor": "openai_gpt_live"}}) == [] assert required_preview_features({"asr": {"vendor": "gemini"}, "tts": {"vendor": "google"}}) == [] +def test_additional_params_are_merged_before_named_options(): + additional_params = {"temperature": 0.7, "style": "overridden", "voice": "overridden"} + config = GeminiTTS( + api_key="test-key", style="warm", additional_params=additional_params, skip_patterns=[1] + ).to_config() + assert config["params"] == { + "temperature": 0.7, + "api_key": "test-key", + "model": "gemini-3.8-flash-tts", + "voice": "Puck", + "style": "warm", + } + assert config["skip_patterns"] == [1] + assert additional_params == {"temperature": 0.7, "style": "overridden", "voice": "overridden"} + generated = Tts_Gemini.model_validate(config) + assert generated.params.style == "warm" + assert generated.params.temperature == 0.7 + + @pytest.mark.parametrize("field", ["api_key", "model", "voice"]) def test_rejects_blank_fields(field): with pytest.raises(ValueError): GeminiTTS(**{**{"api_key": "test-key"}, field: " "}) -@pytest.mark.parametrize("header", ["agora-feature", "Agora-Feature", "AGORA-FEATURE"]) -def test_preview_gate_survives_raw_per_call_headers(header): +def test_production_raw_calls_preserve_per_call_headers(): requests = [] with httpx.Client(transport=transport(requests)) as http_client: client = Agora( @@ -109,24 +161,24 @@ def test_preview_gate_survives_raw_per_call_headers(header): app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client, - headers={header: "caller-value"}, + headers={"x-custom": "caller-value"}, ) session = make_agent(client, MODELS[0]).create_session(channel="test", agent_uid="1", remote_uids=["100"]) session.start() - headers = {header: "", "x-custom": "preserved"} + headers = {"x-request": "preserved"} session.raw.get("0" * 32, "agent-1", request_options={"additional_headers": headers}) for req in requests: - assert req.headers.get_list("agora-feature") == ["gemini-live"] - assert requests[-1].headers["x-custom"] == "preserved" - assert headers[header] == "" - assert client._client_wrapper.get_custom_headers()[header] == "caller-value" + assert "agora-feature" not in req.headers + assert str(req.url).startswith(client.get_current_url() + "/") + assert requests[-1].headers["x-request"] == "preserved" + assert headers == {"x-request": "preserved"} client.agents.get("0" * 32, "agent-1") assert not str(requests[-1].url).startswith(PREVIEW_API_BASE_URL) - assert requests[-1].headers["agora-feature"] == "caller-value" + assert requests[-1].headers["x-custom"] == "caller-value" @pytest.mark.asyncio -async def test_async_preview_gate_survives_raw_per_call_headers(): +async def test_async_production_raw_calls_preserve_per_call_headers(): requests = [] async with httpx.AsyncClient(transport=transport(requests)) as http_client: client = AsyncAgora( @@ -134,13 +186,95 @@ async def test_async_preview_gate_survives_raw_per_call_headers(): app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client, - headers={"Agora-Feature": "caller-value"}, + headers={"x-custom": "caller-value"}, ) session = make_agent(client, MODELS[0]).create_async_session(channel="test", agent_uid="1", remote_uids=["100"]) await session.start() - await session.raw.get("0" * 32, "agent-1", request_options={"additional_headers": {"AGORA-FEATURE": ""}}) + await session.raw.get("0" * 32, "agent-1", request_options={"additional_headers": {"x-request": "preserved"}}) for req in requests: - assert req.headers.get_list("agora-feature") == ["gemini-live"] + assert "agora-feature" not in req.headers + assert str(req.url).startswith(client.get_current_url() + "/") + assert requests[-1].headers["x-request"] == "preserved" await client.agents.get("0" * 32, "agent-1") assert not str(requests[-1].url).startswith(PREVIEW_API_BASE_URL) - assert requests[-1].headers["agora-feature"] == "caller-value" + assert requests[-1].headers["x-custom"] == "caller-value" + + +def test_preview_imports_alias_production_and_register_in_global_catalog(): + from agora_agent.agentkit.preview import GeminiTTSModels as PreviewModels + from agora_agent.agentkit.preview.gemini_tts import GeminiTTSModels as LegacyModels + + assert GeminiTTS is VendorGeminiTTS is ProductionGeminiTTS is PreviewGeminiTTS is LegacyGeminiTTS + assert GeminiTTSModels is PreviewModels is LegacyModels + assert GLOBAL_VENDOR_NAMESPACE.tts["gemini"] is GeminiTTS + assert GlobalTTSVendors.gemini is GeminiTTS + assert GeminiTTS in get_args(GlobalTTS) + assert "gemini" in GLOBAL_TTS_VENDORS + assert "gemini" not in CN_TTS_VENDORS + + +def test_raw_config_starts_on_production_and_drops_none(): + config = { + "vendor": "gemini", + "params": {"api_key": "test-key", "model": "future-tts-model", "voice": "Puck", "style": None}, + } + + class RawGeminiTTS(BaseTTS): + def to_config(self): + return config + + requests = [] + with httpx.Client(transport=transport(requests)) as http_client: + client = Agora(area=Area.EU, app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client) + session = make_agent(client, MODELS[0]).with_tts(RawGeminiTTS()).create_session( + channel="raw", agent_uid="1", remote_uids=["100"] + ) + session.start() + session.stop() + assert json.loads(requests[0].content)["properties"]["tts"] == { + "vendor": "gemini", + "params": {"api_key": "test-key", "model": "future-tts-model", "voice": "Puck"}, + } + assert config["params"]["style"] is None + for request in requests: + assert str(request.url).startswith(client.get_current_url() + "/") + assert "agora-feature" not in request.headers + + +def test_gemini_tts_schema_compatibility_still_validates_other_fields(): + class InvalidASR(BaseSTT): + def to_config(self): + return {"vendor": "gemini", "params": {"api_key": "test-key"}} + + requests = [] + with httpx.Client(transport=transport(requests)) as http_client: + client = Agora(area=Area.US, app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client) + session = make_agent(client, MODELS[0]).with_stt(InvalidASR()).create_session( + channel="invalid", agent_uid="1", remote_uids=["100"] + ) + with pytest.raises(ValueError): + session.start() + assert requests == [] + + +def test_gemini_tts_can_share_a_session_with_a_registered_preview_provider(monkeypatch): + from agora_agent.agentkit.preview import client as preview_client + + monkeypatch.setitem(preview_client._PREVIEW_FEATURES_BY_CATEGORY["asr"], "future_asr", "future-asr") + + class PreviewASR(BaseSTT): + def to_config(self): + return {"vendor": "future_asr", "params": {"model": "future-model"}} + + requests = [] + with httpx.Client(transport=transport(requests)) as http_client: + client = Agora(area=Area.US, app_id="0" * 32, app_certificate="1" * 32, httpx_client=http_client) + session = make_agent(client, MODELS[0]).with_stt(PreviewASR()).create_session( + channel="mixed", agent_uid="1", remote_uids=["100"] + ) + session.start() + session.stop() + assert json.loads(requests[0].content)["properties"]["tts"]["vendor"] == "gemini" + for request in requests: + assert str(request.url).startswith(PREVIEW_API_BASE_URL) + assert request.headers["agora-feature"] == "future-asr" diff --git a/tests/custom/test_regional_vendors.py b/tests/custom/test_regional_vendors.py index 92ff328..10516a4 100644 --- a/tests/custom/test_regional_vendors.py +++ b/tests/custom/test_regional_vendors.py @@ -13,6 +13,7 @@ MiniMaxTTS, MistralTTS, OpenAI, + RtzrSTT, SmallestAISTT, SmallestAITTS, SpatiusAvatar, @@ -229,6 +230,14 @@ def test_xai_grok_remains_mllm_vendor() -> None: assert agent.mllm is not None and agent.mllm["vendor"] == "xai" +def test_rtzr_is_registered_as_a_global_stt_vendor() -> None: + assert "rtzr" in GLOBAL_ASR_VENDORS + assert "rtzr" not in CN_ASR_VENDORS + assert GLOBAL_VENDOR_NAMESPACE.asr["rtzr"] is RtzrSTT + assert GlobalSTTVendors.rtzr is RtzrSTT + assert RtzrSTT in get_args(GlobalSTT) + + def test_smallest_ai_is_registered_as_global_only() -> None: assert "smallestai" in GLOBAL_ASR_VENDORS assert "smallestai" in GLOBAL_TTS_VENDORS diff --git a/tests/custom/test_request_body.py b/tests/custom/test_request_body.py index f5bab70..3106870 100644 --- a/tests/custom/test_request_body.py +++ b/tests/custom/test_request_body.py @@ -323,7 +323,7 @@ def test_byok_pipeline_full_properties_shape() -> None: # ASR asr = props["asr"] assert asr["vendor"] == "deepgram" - assert asr["params"]["key"] == "dg-key" + assert asr["params"]["api_key"] == "dg-key" assert asr["params"]["model"] == "nova-2" assert asr["params"]["language"] == "en" @@ -630,7 +630,7 @@ def test_6b_tts_preset_with_byok_llm_and_asr() -> None: properties = dump(call["properties"]) # BYOK ASR: key and model both retained (nothing stripped for BYOK path) - assert properties["asr"]["params"]["key"] == "dg-key" + assert properties["asr"]["params"]["api_key"] == "dg-key" assert properties["asr"]["params"]["model"] == "nova-2" # BYOK LLM key retained assert properties["llm"]["api_key"] == "openai-key" @@ -779,7 +779,7 @@ def test_byok_deepgram_stt_params() -> None: ) props = build_properties(agent, allow_missing={"llm", "tts"}) assert props["asr"]["vendor"] == "deepgram" - assert props["asr"]["params"]["key"] == "dg-key" + assert props["asr"]["params"]["api_key"] == "dg-key" assert props["asr"]["params"]["model"] == "nova-2" assert props["asr"]["params"]["language"] == "en" @@ -1163,7 +1163,7 @@ def test_byok_sarvam_tts_params() -> None: key="sarvam-key", speaker="anushka", target_language_code="en-IN", - sample_rate=24000, + speech_sample_rate=24000, ) ) props = build_properties(agent, allow_missing={"asr", "llm"}) diff --git a/tests/custom/test_stt_language.py b/tests/custom/test_stt_language.py index 27ce63f..9e2b8be 100644 --- a/tests/custom/test_stt_language.py +++ b/tests/custom/test_stt_language.py @@ -11,6 +11,7 @@ MicrosoftSTT, OpenAI, OpenAISTT, + RtzrSTT, SarvamSTT, SpeechmaticsSTT, TurnDetectionConfig, @@ -110,13 +111,13 @@ def test_stt_vendor_params_match_documented_shapes() -> None: DeepgramSTT(model="enhanced") assert DeepgramSTT(api_key="dg-key", language="en").to_config()["params"] == { - "key": "dg-key", + "api_key": "dg-key", "language": "en", } - # api_key → wire key "key"; keyterm passes through unchanged + # api_key uses the generated Deepgram ASR field name; keyterm passes through unchanged assert DeepgramSTT(api_key="dg-key", model="nova-3", language="en", keyterm="term").to_config()["params"] == { - "key": "dg-key", + "api_key": "dg-key", "model": "nova-3", "language": "en", "keyterm": "term", @@ -188,6 +189,55 @@ def test_stt_vendor_params_match_documented_shapes() -> None: } +def test_rtzr_stt_serializes_generated_params() -> None: + assert RtzrSTT( + client_id="client-id", + client_secret="client-secret", + api_base="https://rtzr.example.com", + model_name="general", + language="ko", + sample_rate=16000, + encoding="pcm_s16le", + use_itn=True, + use_disfluency_filter=False, + use_profanity_filter=True, + use_punctuation=True, + keywords=["Agora"], + ).to_config() == { + "vendor": "rtzr", + "params": { + "client_id": "client-id", + "client_secret": "client-secret", + "api_base": "https://rtzr.example.com", + "model_name": "general", + "language": "ko", + "sample_rate": 16000, + "encoding": "pcm_s16le", + "use_itn": True, + "use_disfluency_filter": False, + "use_profanity_filter": True, + "use_punctuation": True, + "keywords": ["Agora"], + }, + } + + +def test_rtzr_stt_reaches_generated_request_properties() -> None: + props = properties( + base_agent().with_stt(RtzrSTT(client_id="client-id", client_secret="client-secret", language="ko")) + ) + + assert props["asr"] == { + "vendor": "rtzr", + "language": "en-US", + "params": { + "client_id": "client-id", + "client_secret": "client-secret", + "language": "ko", + }, + } + + def test_assemblyai_params_stay_nested_and_asr_language_comes_from_turn_detection() -> None: props = properties( Agent(test_client(), turn_detection=TurnDetectionConfig(language="fr-FR")) diff --git a/tests/custom/test_tts_vendors.py b/tests/custom/test_tts_vendors.py index 43388bb..6abf094 100644 --- a/tests/custom/test_tts_vendors.py +++ b/tests/custom/test_tts_vendors.py @@ -198,11 +198,22 @@ def test_tts_vendor_params_match_generated_core_shapes() -> None: "language_boost": "auto", } - assert SarvamTTS(key="sarvam-key", speaker="anushka", target_language_code="en-IN", sample_rate=24000).to_config()["params"] == { + assert SarvamTTS( + key="sarvam-key", + speaker="anushka", + target_language_code="en-IN", + speech_sample_rate=24000, + enable_preprocessing=True, + model="bulbul:v3", + additional_params={"custom_option": "value", "model": "overridden"}, + ).to_config()["params"] == { "api_subscription_key": "sarvam-key", "speaker": "anushka", "target_language_code": "en-IN", - "sample_rate": 24000, + "speech_sample_rate": 24000, + "enable_preprocessing": True, + "model": "bulbul:v3", + "custom_option": "value", } assert MurfTTS(