diff --git a/sdk/python/agentfield/agent_ai.py b/sdk/python/agentfield/agent_ai.py index 2dc9ae4a2..5c5621d47 100644 --- a/sdk/python/agentfield/agent_ai.py +++ b/sdk/python/agentfield/agent_ai.py @@ -402,7 +402,7 @@ def _openrouter_provider(self): @property def _minimax_provider(self): - """Lazy-initialized MiniMax provider for video generation.""" + """Lazy-initialized MiniMax media provider.""" if self._minimax_provider_instance is None: from agentfield.media_providers import MiniMaxProvider @@ -1443,16 +1443,21 @@ async def ai_with_audio( """ AI method optimized for audio output generation. - Automatically detects the model type and uses the appropriate LiteLLM function: + Automatically detects the model type and uses the appropriate provider: + - For MiniMax TTS models (minimax/speech-2.8-hd, etc.): Uses MiniMax T2A - For TTS models (tts-1, tts-1-hd, gpt-4o-mini-tts): Uses litellm.speech() - For audio-capable chat models (gpt-4o-audio-preview): Uses litellm.completion() with audio modalities Args: *args: Input arguments (text prompts, etc.) - voice: Voice to use for audio generation (alloy, echo, fable, onyx, nova, shimmer) - format: Audio format (wav, mp3, etc.) + voice: Voice identifier. OpenAI voices include alloy, echo, fable, + onyx, nova, and shimmer. MiniMax models require a MiniMax voice ID. + format: Audio format. MiniMax supports mp3, wav, flac, and pcm. model: Model to use (defaults to tts-1) - **kwargs: Additional parameters + **kwargs: Additional parameters. MiniMax supports output_format + ("hex" or "url"), language_boost, voice_setting, + pronunciation_dict, audio_setting, voice_modify, and + subtitle_enable. Streaming is not supported. Returns: MultimodalResponse with audio content @@ -1460,6 +1465,13 @@ async def ai_with_audio( Example: audio_result = await agent.ai_with_audio("Say hello warmly", voice="alloy") audio_result.audio.save("greeting.wav") + + minimax_result = await agent.ai_with_audio( + "Say hello warmly", + model="minimax/speech-2.8-hd", + voice="English_Graceful_Lady", + format="mp3", + ) """ # Use TTS model as default (more reliable than gpt-4o-audio-preview) if model is None: @@ -1467,6 +1479,19 @@ async def ai_with_audio( self.agent.ai_config.audio_model ) # Use configured audio model (defaults to tts-1) + if model.startswith("minimax/"): + provider = self._media_router.resolve(model, "audio") + text_input = " ".join(str(arg) for arg in args if isinstance(arg, str)) + if not text_input: + text_input = "Hello, this is a test audio message." + return await provider.generate_audio( + text=text_input, + model=model, + voice=voice, + format=format, + **kwargs, + ) + # Try media router for fal models try: provider = self._media_router.resolve(model, "audio") @@ -1919,14 +1944,22 @@ async def ai_generate_audio( Supported Providers: - LiteLLM: OpenAI TTS models like "tts-1", "tts-1-hd", "gpt-4o-mini-tts" - Fal.ai: TTS models like "fal-ai/kokoro/..." (custom deployments) + - MiniMax: TTS models prefixed with "minimax/", such as + "minimax/speech-2.8-hd" Args: text: Text to convert to speech model: TTS model to use (defaults to AIConfig.audio_model, typically "tts-1") - voice: Voice to use ("alloy", "echo", "fable", "onyx", "nova", "shimmer") - format: Audio format ("wav", "mp3", "opus", "aac", "flac", "pcm") + voice: Voice identifier. OpenAI voices include "alloy", "echo", + "fable", "onyx", "nova", and "shimmer". MiniMax models + require a MiniMax voice ID instead of the default "alloy". + format: Audio format. MiniMax supports "mp3", "wav", "flac", and + "pcm". speed: Speech speed multiplier (0.25 to 4.0) - **kwargs: Provider-specific parameters + **kwargs: Provider-specific parameters. MiniMax supports + output_format ("hex" or "url"), language_boost, + voice_setting, pronunciation_dict, audio_setting, + voice_modify, and subtitle_enable. Streaming is not supported. Returns: MultimodalResponse: Response object with .audio containing AudioOutput. @@ -1949,6 +1982,16 @@ async def ai_generate_audio( format="mp3" ) + # MiniMax TTS with a MiniMax voice ID + result = await app.ai_generate_audio( + "Welcome to the presentation.", + model="minimax/speech-2.8-hd", + voice="English_Graceful_Lady", + format="mp3", + language_boost="English", + output_format="hex", + ) + # Adjust speech speed result = await app.ai_generate_audio( "This is spoken slowly.", diff --git a/sdk/python/agentfield/media_providers.py b/sdk/python/agentfield/media_providers.py index 7b889220d..15b66a58a 100644 --- a/sdk/python/agentfield/media_providers.py +++ b/sdk/python/agentfield/media_providers.py @@ -3,7 +3,7 @@ Provides a unified interface for different media generation backends: - Fal.ai (Flux, SDXL, Whisper, TTS, Video models) -- MiniMax (Video generation) +- MiniMax (Media generation) - OpenRouter (via LiteLLM) - OpenAI DALL-E (via LiteLLM) - Future: ElevenLabs, Replicate, etc. @@ -650,7 +650,7 @@ async def transcribe_audio( class MiniMaxProvider(MediaProvider): - """MiniMax provider for asynchronous video generation.""" + """MiniMax media generation provider.""" def __init__( self, @@ -671,7 +671,7 @@ def name(self) -> str: @property def supported_modalities(self) -> List[str]: - return ["video", "music"] + return ["video", "music", "audio"] async def generate_image( self, @@ -691,9 +691,136 @@ async def generate_audio( format: str = "wav", *, system: Optional[str] = None, + stream: bool = False, + language_boost: Optional[str] = None, + output_format: str = "hex", + voice_setting: Optional[Dict[str, Any]] = None, + pronunciation_dict: Optional[Dict[str, Any]] = None, + audio_setting: Optional[Dict[str, Any]] = None, + voice_modify: Optional[Dict[str, Any]] = None, + subtitle_enable: Optional[bool] = None, **kwargs, ) -> MultimodalResponse: - raise NotImplementedError("minimax does not support audio generation") + """Generate speech via the MiniMax t2a_v2 endpoint.""" + import base64 + import os + + import aiohttp + + api_key = self._api_key or os.environ.get("MINIMAX_API_KEY") + if not api_key: + raise ValueError( + "MiniMax API key required. Set MINIMAX_API_KEY or pass api_key " + "to MiniMaxProvider." + ) + if stream: + raise ValueError("MiniMax streaming TTS is not supported by generate_audio") + if output_format not in {"hex", "url"}: + raise ValueError("MiniMax audio output_format must be hex or url") + + audio_options = dict(audio_setting or {}) + audio_format = audio_options.get("format", format) + if audio_format not in {"mp3", "wav", "flac", "pcm"}: + raise ValueError("MiniMax audio format must be mp3, wav, flac, or pcm") + audio_options["format"] = audio_format + + voice_options = dict(voice_setting or {}) + speed = kwargs.pop("speed", None) + if voice != "alloy": + voice_options.setdefault("voice_id", voice) + if speed is not None and voice_options: + voice_options.setdefault("speed", speed) + elif speed not in (None, 1.0): + raise ValueError( + "MiniMax speed requires a voice or voice_setting with voice_id" + ) + if voice_options and not voice_options.get("voice_id"): + raise ValueError("MiniMax voice_setting requires voice_id") + + send_model = self._strip_prefix(model or "speech-2.8-hd") + if not send_model: + raise ValueError("MiniMax audio generation requires a model") + + body: Dict[str, Any] = { + "model": send_model, + "text": text, + "stream": False, + "output_format": output_format, + "audio_setting": audio_options, + } + optional_fields = { + "language_boost": language_boost, + "voice_setting": voice_options or None, + "pronunciation_dict": pronunciation_dict, + "voice_modify": voice_modify, + "subtitle_enable": subtitle_enable, + } + body.update( + {key: value for key, value in optional_fields.items() if value is not None} + ) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + timeout = aiohttp.ClientTimeout(total=120.0) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post( + f"{self._base_url}/t2a_v2", + headers=headers, + json=body, + ) as response: + if response.status >= 400: + detail = await response.text() + raise RuntimeError( + f"MiniMax audio generation failed ({response.status}): " + f"{detail[:500]}" + ) + data = await response.json() + + if not isinstance(data, dict): + raise RuntimeError("MiniMax audio generation returned an invalid response") + base_resp = data.get("base_resp") or {} + if not isinstance(base_resp, dict): + raise RuntimeError("MiniMax audio generation returned an invalid response") + status_code = base_resp.get("status_code") + if status_code not in (None, 0): + status_msg = base_resp.get("status_msg") or "unknown error" + raise RuntimeError( + f"MiniMax audio generation failed ({status_code}): {status_msg}" + ) + + response_data = data.get("data") or {} + if not isinstance(response_data, dict): + raise RuntimeError("MiniMax audio generation returned no audio") + status = response_data.get("status") + if status not in (None, 2): + raise RuntimeError("MiniMax audio generation did not complete") + audio_value = response_data.get("audio") + if not isinstance(audio_value, str) or not audio_value: + raise RuntimeError("MiniMax audio generation returned no audio") + + if output_format == "url": + output = AudioOutput(data=None, format=audio_format, url=audio_value) + else: + try: + audio_bytes = bytes.fromhex(audio_value) + except ValueError as exc: + raise RuntimeError( + "MiniMax audio generation returned invalid hex audio" + ) from exc + output = AudioOutput( + data=base64.b64encode(audio_bytes).decode("ascii"), + format=audio_format, + url=None, + ) + return MultimodalResponse( + text=text, + audio=output, + images=[], + files=[], + raw_response=data, + ) async def generate_music( self, diff --git a/sdk/python/tests/test_minimax_audio.py b/sdk/python/tests/test_minimax_audio.py new file mode 100644 index 000000000..654893de9 --- /dev/null +++ b/sdk/python/tests/test_minimax_audio.py @@ -0,0 +1,256 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from agentfield.agent_ai import AgentAI +from agentfield.media_providers import ( + MINIMAX_CN_BASE_URL, + MINIMAX_GLOBAL_BASE_URL, + MiniMaxProvider, +) +from agentfield.media_router import MediaRouter + + +class FakeResponse: + def __init__(self, payload, status=200): + self.payload = payload + self.status = status + + async def json(self): + if isinstance(self.payload, Exception): + raise self.payload + return self.payload + + async def text(self): + return str(self.payload) + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + +class CaptureSession: + def __init__(self, response): + self.response = response + self.request = None + + def post(self, url, **kwargs): + self.request = (url, kwargs) + return self.response + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + +@pytest.mark.asyncio +async def test_minimax_audio_hex_output_uses_global_endpoint_and_request_fields( + monkeypatch, +): + audio_bytes = b"generated-speech" + payload = { + "data": {"audio": audio_bytes.hex(), "status": 2}, + "extra_info": {"audio_format": "mp3"}, + "base_resp": {"status_code": 0}, + } + session = CaptureSession(FakeResponse(payload)) + monkeypatch.setattr("aiohttp.ClientSession", lambda **kwargs: session) + provider = MiniMaxProvider(api_key="unit-value") + + result = await provider.generate_audio( + text="Read this sentence clearly", + model="minimax/speech-2.8-turbo", + voice="English_Graceful_Lady", + format="mp3", + speed=1.25, + language_boost="English", + output_format="hex", + pronunciation_dict={"tone": ["read/reed"]}, + audio_setting={"sample_rate": 32000}, + voice_modify={"pitch": 10}, + subtitle_enable=True, + ) + + assert session.request[0] == f"{MINIMAX_GLOBAL_BASE_URL}/t2a_v2" + assert session.request[1]["json"] == { + "model": "speech-2.8-turbo", + "text": "Read this sentence clearly", + "stream": False, + "output_format": "hex", + "audio_setting": {"sample_rate": 32000, "format": "mp3"}, + "language_boost": "English", + "voice_setting": { + "voice_id": "English_Graceful_Lady", + "speed": 1.25, + }, + "pronunciation_dict": {"tone": ["read/reed"]}, + "voice_modify": {"pitch": 10}, + "subtitle_enable": True, + } + assert session.request[1]["headers"]["Authorization"] == "Bearer unit-value" + assert result.audio.format == "mp3" + assert result.audio.url is None + assert result.audio.get_bytes() == audio_bytes + assert result.raw_response["data"]["status"] == 2 + + +@pytest.mark.asyncio +async def test_minimax_audio_url_output_uses_cn_endpoint_and_default_model(monkeypatch): + audio_url = "https://example.test/generated.flac" + session = CaptureSession( + FakeResponse( + { + "data": {"audio": audio_url, "status": 2}, + "base_resp": {"status_code": 0}, + } + ) + ) + monkeypatch.setattr("aiohttp.ClientSession", lambda **kwargs: session) + provider = MiniMaxProvider(api_key="unit-value", base_url=MINIMAX_CN_BASE_URL) + + result = await provider.generate_audio( + text="Generate regional speech", + format="flac", + output_format="url", + voice_setting={"voice_id": "regional-voice"}, + ) + + assert session.request[0] == f"{MINIMAX_CN_BASE_URL}/t2a_v2" + assert session.request[1]["json"] == { + "model": "speech-2.8-hd", + "text": "Generate regional speech", + "stream": False, + "output_format": "url", + "audio_setting": {"format": "flac"}, + "voice_setting": {"voice_id": "regional-voice"}, + } + assert result.audio.data is None + assert result.audio.url == audio_url + assert result.audio.format == "flac" + + +@pytest.mark.parametrize( + ("api_key", "kwargs", "response", "error", "message"), + [ + (None, {}, None, ValueError, "API key required"), + ("unit-value", {"format": "aac"}, None, ValueError, "format must be"), + ("unit-value", {"stream": True}, None, ValueError, "streaming TTS"), + ( + "unit-value", + {"output_format": "base64"}, + None, + ValueError, + "output_format must be", + ), + ("unit-value", {"speed": 1.1}, None, ValueError, "speed requires a voice"), + ( + "unit-value", + {"voice_setting": {"speed": 1.1}}, + None, + ValueError, + "voice_setting requires voice_id", + ), + ("unit-value", {"model": "minimax/"}, None, ValueError, "requires a model"), + ( + "unit-value", + {}, + FakeResponse({}, status=500), + RuntimeError, + "failed \\(500\\)", + ), + ( + "unit-value", + {}, + FakeResponse(ValueError("malformed JSON")), + ValueError, + "malformed JSON", + ), + ("unit-value", {}, FakeResponse([]), RuntimeError, "invalid response"), + ( + "unit-value", + {}, + FakeResponse( + { + "base_resp": { + "status_code": 1004, + "status_msg": "authentication failed", + } + } + ), + RuntimeError, + "authentication failed", + ), + ( + "unit-value", + {}, + FakeResponse({"data": {"audio": "00", "status": 1}}), + RuntimeError, + "did not complete", + ), + ( + "unit-value", + {}, + FakeResponse({"data": {"status": 2}}), + RuntimeError, + "returned no audio", + ), + ( + "unit-value", + {}, + FakeResponse({"data": {"audio": "not-hex", "status": 2}}), + RuntimeError, + "invalid hex audio", + ), + ], +) +@pytest.mark.asyncio +async def test_minimax_audio_validates_inputs_and_api_errors( + monkeypatch, api_key, kwargs, response, error, message +): + monkeypatch.delenv("MINIMAX_API_KEY", raising=False) + provider = MiniMaxProvider(api_key=api_key) + if response is not None: + session = CaptureSession(response) + monkeypatch.setattr("aiohttp.ClientSession", lambda **kwargs: session) + with pytest.raises(error, match=message): + await provider.generate_audio("Audio", **kwargs) + + +@pytest.mark.asyncio +async def test_agent_ai_routes_minimax_audio_models(): + agent = SimpleNamespace( + ai_config=SimpleNamespace(audio_model="minimax/speech-2.8-hd") + ) + ai = AgentAI(agent) + generate_audio = AsyncMock(return_value="generated-audio") + provider = SimpleNamespace( + name="minimax", + supported_modalities=["audio"], + generate_audio=generate_audio, + ) + router = MediaRouter() + router.register("minimax/", provider) + ai._media_router_instance = router + + result = await ai.ai_generate_audio( + "Route this speech", + voice="target-voice", + format="pcm", + speed=1.1, + language_boost="English", + ) + + assert result == "generated-audio" + generate_audio.assert_awaited_once_with( + text="Route this speech", + model="minimax/speech-2.8-hd", + voice="target-voice", + format="pcm", + speed=1.1, + language_boost="English", + ) diff --git a/sdk/python/tests/test_minimax_video.py b/sdk/python/tests/test_minimax_video.py index 427156255..ecf46ddd1 100644 --- a/sdk/python/tests/test_minimax_video.py +++ b/sdk/python/tests/test_minimax_video.py @@ -180,8 +180,6 @@ async def test_minimax_video_validates_credentials_model_and_duration(monkeypatc ) with pytest.raises(NotImplementedError, match="image generation"): await provider.generate_image("An image") - with pytest.raises(NotImplementedError, match="audio generation"): - await provider.generate_audio("Audio") @pytest.mark.asyncio @@ -251,4 +249,4 @@ def test_minimax_provider_configuration_and_registry(): base_url=MINIMAX_GLOBAL_BASE_URL, ) assert isinstance(provider, MiniMaxProvider) - assert provider.supported_modalities == ["video", "music"] + assert provider.supported_modalities == ["video", "music", "audio"]