diff --git a/.env.example b/.env.example index 38c493c8..1d8a24e3 100644 --- a/.env.example +++ b/.env.example @@ -529,6 +529,10 @@ EVA_EN_USER_M=your_elevenlabs_agent_id_for_default_user_m #d csv_list #v EVA_RECORD_IDS= +#i Comma-separated record IDs to skip (applied after EVA_RECORD_IDS). Empty = skip none. +#d csv_list +#v EVA_EXCLUDE_RECORD_IDS= + #i Logging verbosity. #d enum #e DEBUG,INFO,WARNING,ERROR,CRITICAL diff --git a/README.md b/README.md index ae86d0b6..c774268c 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ EVA_DOMAIN=airline EVA_MAX_CONCURRENT_CONVERSATIONS=5 EVA_DEBUG=false # Run only 1 record for testing when enabled EVA_RECORD_IDS=1.2.1,1.2.2 # Run specific records only (remove to run all records) +EVA_EXCLUDE_RECORD_IDS=6.1.1 # Skip specific records (applied after EVA_RECORD_IDS) # User Simulator Configuration EVA_USER_SIMULATOR__PROVIDER=elevenlabs # elevenlabs | openai_realtime diff --git a/Readme.google.md b/Readme.google.md new file mode 100644 index 00000000..26bf4acb --- /dev/null +++ b/Readme.google.md @@ -0,0 +1,15 @@ +## Evaluate all scenarios and get overall scores + +When running against AIS: + +``` +GOOGLE_GENAI_USE_VERTEXAI=0 python main.py --debug +``` + +When running against Vertex: + +``` +GOOGLE_GENAI_USE_VERTEXAI=1 python main.py --debug +``` + +Always use `--debug` to test as it executes a single case run only. diff --git a/apps/analysis.py b/apps/analysis.py index 30e5636f..306daec5 100644 --- a/apps/analysis.py +++ b/apps/analysis.py @@ -78,12 +78,19 @@ def _build_metric_group_map() -> dict[str, str]: "Other": "#AAAAAA", } -_NON_NORMALIZED_METRICS = {"response_speed", "tool_call_validity__num_tool_calls"} +_NON_NORMALIZED_METRICS = { + "response_speed", + "tool_call_validity__num_tool_calls", + "time_to_completion", + "turns_to_completion", +} # Axis title + hover suffix for non-normalized metrics. Sub-metrics fall back to their parent's entry. _NON_NORMALIZED_UNITS: dict[str, tuple[str, str]] = { "response_speed": ("Seconds", "s"), "tool_call_validity__num_tool_calls": ("Count", ""), + "time_to_completion": ("Seconds", "s"), + "turns_to_completion": ("Turns", ""), } diff --git a/docker_build.sh b/docker_build.sh new file mode 100755 index 00000000..be0f005a --- /dev/null +++ b/docker_build.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +TAG="${1:-latest}" +PUSH="${2:-}" +shift 2 2>/dev/null || true + +GIT_COMMIT_SHA=$(git rev-parse HEAD) +GIT_BRANCH=$(git branch --show-current) +GIT_DIRTY=$([[ -n $(git status --porcelain) ]] && echo true || echo false) +GIT_DIFF_HASH=$(git diff | shasum -a 256 | cut -c1-12) + +IMAGE="registry.console.elementai.com/snow.core_llm/eva:$TAG" + +# Use BuildKit — the legacy builder ("Step X/Y" output) does not cache +# multi-stage builds well and re-runs apt-get / re-copies layers needlessly. +export DOCKER_BUILDKIT=1 + +docker build \ + --platform linux/amd64 \ + --build-arg GIT_COMMIT_SHA="$GIT_COMMIT_SHA" \ + --build-arg GIT_BRANCH="$GIT_BRANCH" \ + --build-arg GIT_DIRTY="$GIT_DIRTY" \ + --build-arg GIT_DIFF_HASH="$GIT_DIFF_HASH" \ + -t "$IMAGE" \ + "$@" . + +if [[ "$PUSH" == "--push" ]]; then + docker push "$IMAGE" +fi diff --git a/docs/metrics/README.md b/docs/metrics/README.md index 277a1e7d..24bd4270 100644 --- a/docs/metrics/README.md +++ b/docs/metrics/README.md @@ -2,7 +2,7 @@ ## Overview -The EVA metrics system provides comprehensive evaluation of voice assistant conversations. The system includes 15 metrics organized into four categories, each answering a different question about the conversation. +The EVA metrics system provides comprehensive evaluation of voice assistant conversations. The system includes 17 metrics organized into four categories, each answering a different question about the conversation. ## Why These Categories? @@ -48,7 +48,7 @@ Measures the quality of the user's conversational experience: | [`conciseness`](conciseness.md) | Judge | Language Model | Whether responses are appropriately concise for voice (1-3) | | [`conversation_progression`](conversation_progression.md) | Judge | Language Model | Whether assistant moves conversation forward without repetition (1-3) | -### Diagnostic (7 metrics) +### Diagnostic (9 metrics) Metrics that help isolate root causes of failures. These provide signals for understanding what went wrong, but are not directly used in final evaluation scores. @@ -57,6 +57,8 @@ Metrics that help isolate root causes of failures. These provide signals for und | [`tts_fidelity`](tts_fidelity.md) | Audio Judge (Gemini) | Speech Synthesis | Whether assistant speech audio matches intended text (0-1). **Opt-in** — excluded from the default run; enable via `--metrics tts_fidelity`. | | [`authentication_success`](authentication_success.md) | Deterministic | Speech Recognition, Language Model | Whether get_reservation was called successfully (0-1) | | [`response_speed`](response_speed.md) | Deterministic | VAD, Pipeline | Latency between user utterance end and assistant response start (seconds) | +| [`time_to_completion`](time_to_completion.md) | Deterministic | Pipeline | Wall-clock time to complete the task, successful runs only (seconds, lower is better) | +| [`turns_to_completion`](turns_to_completion.md) | Deterministic | Language Model | Number of turns to complete the task, successful runs only (count, lower is better) | | [`speakability`](speakability.md) | Judge | Language Model | Whether text is voice-friendly and appropriate for TTS (0-1) | | [`stt_wer`](stt_wer.md) | Deterministic | Speech Recognition | Speech-to-Text Word Error Rate using jiwer (0.0+) | | [`tool_call_validity`](tool_call_validity.md) | Deterministic | Language Model | Fraction of tool calls with correctly formatted parameters (0.0-1.0) | diff --git a/docs/metrics/time_to_completion.md b/docs/metrics/time_to_completion.md new file mode 100644 index 00000000..7d0d6a7c --- /dev/null +++ b/docs/metrics/time_to_completion.md @@ -0,0 +1,79 @@ +# Time to Completion + +> **Diagnostic Metric**: How long did a successful conversation take? Reported only for conversations that completed the task, so it answers "when the agent succeeds, how long does it take?" — not directly used in final pass/fail scores. + +## Overview + +Deterministic metric that reports the total wall-clock duration (in seconds) of a conversation, but **only for conversations where the task was actually completed**. Mixing in the durations of failed conversations would make the value meaningless as an efficiency signal, so unsuccessful conversations are skipped. + +Task completion is determined using the exact same criteria as [`task_completion`](task_completion.md): the session must be authenticated correctly and the final scenario database state must match the expected state (SHA-256 hash comparison). + +### Capabilities Measured + +- **Pipeline**: End-to-end wall-clock efficiency of the full system in reaching the correct outcome. Not attributable to a single model capability. + +## How It Works + +### Evaluation Method + +- **Type**: Deterministic (reads `duration_seconds`) +- **Granularity**: Conversation-level + +### Input Data + +Uses the following MetricContext fields: +- `expected_scenario_db`, `final_scenario_db`, `final_scenario_db_hash`: used to determine whether the task was completed (same as `task_completion`). +- `duration_seconds`: total conversation duration. + +### Scoring + +- **Scale**: Seconds (lower is better) +- **Normalization**: None. Raw duration in seconds is not meaningfully normalizable to a 0-1 scale. +- **Skipped when**: the task was not completed, or no valid duration was recorded. Skipped records are excluded from the run-level efficiency aggregate. + +## Example Output + +```json +{ + "name": "time_to_completion", + "score": 42.5, + "normalized_score": null, + "details": { + "task_completed": true, + "duration_seconds": 42.5, + "reason": "Task completed — reporting total conversation duration" + } +} +``` + +When the task was not completed: + +```json +{ + "name": "time_to_completion", + "score": null, + "normalized_score": null, + "skipped": true, + "details": { + "task_completed": false, + "reason": "Final database state differs from expected state" + } +} +``` + +## Summary Aggregation + +The run-level `metrics_summary.json` includes an `overall_scores.efficiency.time_to_completion` block with the mean/min/max duration across successful conversations, alongside the per-metric aggregate under `per_metric.time_to_completion`. + +## Related Metrics + +- [turns_to_completion.md](turns_to_completion.md) - Number of turns (rather than seconds) to complete the task +- [task_completion.md](task_completion.md) - The binary completion check that gates this metric +- [response_speed.md](response_speed.md) - Per-turn response latency (not total conversation time) + +## Implementation Details + +- **File**: `src/eva/metrics/diagnostic/time_to_completion.py` +- **Class**: `TimeToCompletionMetric` +- **Base Class**: `CodeMetric` +- **Configuration**: None (deterministic computation) diff --git a/docs/metrics/turns_to_completion.md b/docs/metrics/turns_to_completion.md new file mode 100644 index 00000000..bebe7d57 --- /dev/null +++ b/docs/metrics/turns_to_completion.md @@ -0,0 +1,78 @@ +# Turns to Completion + +> **Diagnostic Metric**: How many turns did a successful conversation take? Reported only for conversations that completed the task, so it answers "when the agent succeeds, how many turns does it take?" — not directly used in final pass/fail scores. + +## Overview + +Deterministic metric that reports the total number of conversation turns, but **only for conversations where the task was actually completed**. Mixing in the turn counts of failed conversations would make the value meaningless as an efficiency signal, so unsuccessful conversations are skipped. + +Task completion is determined using the exact same criteria as [`task_completion`](task_completion.md): the session must be authenticated correctly and the final scenario database state must match the expected state (SHA-256 hash comparison). + +### Capabilities Measured + +- **Language Model**: How efficiently the agent drives the conversation to the correct outcome (fewer back-and-forth turns for the same result is better). + +## How It Works + +### Evaluation Method + +- **Type**: Deterministic (reads `num_turns`) +- **Granularity**: Conversation-level + +### Input Data + +Uses the following MetricContext fields: +- `expected_scenario_db`, `final_scenario_db`, `final_scenario_db_hash`: used to determine whether the task was completed (same as `task_completion`). +- `num_turns`: total number of conversation turns. + +### Scoring + +- **Scale**: Turn count (lower is better) +- **Normalization**: None. Raw turn count is not meaningfully normalizable to a 0-1 scale. +- **Skipped when**: the task was not completed, or no valid turn count was recorded. Skipped records are excluded from the run-level efficiency aggregate. + +## Example Output + +```json +{ + "name": "turns_to_completion", + "score": 8.0, + "normalized_score": null, + "details": { + "task_completed": true, + "num_turns": 8, + "reason": "Task completed — reporting total conversation turns" + } +} +``` + +When the task was not completed: + +```json +{ + "name": "turns_to_completion", + "score": null, + "normalized_score": null, + "skipped": true, + "details": { + "task_completed": false, + "reason": "Authentication failed — session mismatch on keys: ['user_id']" + } +} +``` + +## Summary Aggregation + +The run-level `metrics_summary.json` includes an `overall_scores.efficiency.turns_to_completion` block with the mean/min/max turn count across successful conversations, alongside the per-metric aggregate under `per_metric.turns_to_completion`. + +## Related Metrics + +- [time_to_completion.md](time_to_completion.md) - Wall-clock time (rather than turns) to complete the task +- [task_completion.md](task_completion.md) - The binary completion check that gates this metric + +## Implementation Details + +- **File**: `src/eva/metrics/diagnostic/turns_to_completion.py` +- **Class**: `TurnsToCompletionMetric` +- **Base Class**: `CodeMetric` +- **Configuration**: None (deterministic computation) diff --git a/docs/refactor-backend-migration.md b/docs/refactor-backend-migration.md new file mode 100644 index 00000000..693e2bbe --- /dev/null +++ b/docs/refactor-backend-migration.md @@ -0,0 +1,154 @@ +# Backend/Role migration — working notes + +Branch-only scratch notes for the multi-backend migration. Not a spec; the spec +is `docs/refactor-step1.md`. This file is a running log so context survives +conversation compaction. Delete before the final merge if desired. + +## Goal / shape + +Split the two duplicated provider stacks (assistant-side `AbstractAssistantServer` +subclasses, user-side `AbstractUserSimulator` subclasses) into: + +- **`Backend`** (`eva.backend.base.Backend`): role-agnostic, provider-specific + adapter. One per provider integration. Uniform `open/send/receive/close` + + `trigger_response`. Emits **normalized** `BackendEvent`s (no raw provider + events leak to roles). +- **`AssistantRole`** / **`UserRole`** (`eva.role.*`): exactly ONE concrete + generic class each. Holds a `Backend`; owns role-common concerns (prompt, + tools, logging, recording, transport to counterparty). Any backend works with + either role. +- **`BackendFactory`**: single concrete class (no ABC — only one factory), + worker-owned, stateless, lazy per-provider imports. `create(name, config)` + returns a `Backend` for a migrated provider, else `None`. + +## Dispatch (post loose-finalization) + +No boolean gate, **no config-type gate**. Both sides route to the Role/Backend +path **iff `create()` returns a backend** (None ⇒ legacy fallback), dispatching +purely on the provider name: + +- Assistant (`_start_assistant`): `if backend := _BACKEND_FACTORY.create(framework, args)`. +- User (`_start_user_simulator`): dump the user-sim config wholesale + (`sim.model_dump()`) into a caller blob, read voice from the dumped dict (not + attribute access — no coupling to a concrete config type), then + `if backend := _BACKEND_FACTORY.create(sim.provider, args)` ⇒ UserRole, else + legacy. No `isinstance` — any config whose provider is factory-backed uses the + new path. + +Migrating a provider = add a lazy-import branch in `BackendFactory.create` (+ if +it's a user-sim provider, allow its name in the user-sim config `provider`). It +then auto-routes; the legacy branch in `_get_server_class` / +`create_user_simulator` becomes dead (harmless fallback until a later sweep). + +### Backend config validation = construction (single source of truth) +No backend-specific credential/field validation in `config.py`. A native-S2S +backend is validated by **constructing it via the `BackendFactory`** — the +backend's own `__init__` checks required fields/keys (api_key with env fallback, +model, accent rejection). This runs in `orchestrator.preflight._preflight_backends`: +- **Always runs** (cheap, no network) — `--no-preflight` only skips the live + model probes, never this. +- Covers the **S2S assistant framework** (when `pipeline_type == S2S`) and the + **user-sim provider**. Non-factory providers (`create()` → None: ElevenLabs + Conversational AI, cascade pipecat) are skipped — validated by the live probes / + legacy paths, not here. So it validates exactly the migrated backends; it does + NOT hard-fail legacy (revisit if we want that forcing function later). +- A misconfigured factory backend (missing key/model) → `PreflightError`. +- Removed from `config.py`: the S2S `_validate_service_params` case (assistant) + and `_check_s2s_simulator_credentials` (user). Trade-off: a missing S2S key is + now a `PreflightError`, not a pydantic `ValidationError`. STT/TTS/LLM/audio-LLM + param validation stays in config (not backends). + +### User-sim config +`OpenAIRealtimeSimulatorConfig` → **`S2SSimulatorConfig`** (generic native-S2S +config), `provider: Literal["openai_realtime", "grok_voice"]`. Grok is +OpenAI-Realtime-compatible so it shares the config; defaults target OpenAI, Grok +users override `model` + voices. Per-provider API key comes from the env +(OPENAI_API_KEY / XAI_API_KEY) — fallback lives in the backend, with a fail-fast +mirror in `RunConfig._check_s2s_simulator_credentials` +(`_S2S_PROVIDER_API_KEY_ENV` map). Accent-perturbation validator generalized to +"native S2S" (still ElevenLabs-only). + +## Migration status + +- [x] `openai_realtime` — assistant + user. Baseline, tested end-to-end. +- [x] `grok_voice` — **assistant + caller DONE** (`src/eva/backend/grok_voice.py`, + in factory). Thin subclass of `OpenAIRealtimeBackend` (xAI is + OpenAI-Realtime-compatible), mirroring `grok_voice_server.py`. Works as a + caller for free (inherits the OpenAI backend's pcmu/send/trigger_response + surface) once the user-sim config accepts `provider="grok_voice"` (see + user-sim config below). api_key falls back to `XAI_API_KEY`. +- [~] `gemini_live` — assistant was implemented then **parked** to avoid a + conflict with a separate in-flight Gemini change. Working file saved at + `output/tmp/gemini_live.py` (git-ignored) to restore later. When resuming: + re-add the file, its factory branch, and re-plumb `language` into the + assistant `backend_args` (Gemini's `language_code`). Caller still deferred + (manual turn-taking / no VAD speech-boundary events don't map onto + UserRole's gating). +- [ ] `elevenlabs`, cascade/pipecat — later. + +### Grok as caller — native-VAD path (manual not supported) +Root cause (proven via backend-tagged DEBUG logs): **xAI ignores manual turn-taking** +(`create_response:false` AND `interrupt_response:false`) — it always auto-responds via +its own VAD. In the freeze runs our code never called `trigger_response` (0 +`caller_response_created`) yet grok emitted `response.created`; it also wedged when its +VAD re-fired `speech_started` on still-streaming audio right at `response.created`. + +So the caller now branches on a capability instead of forcing manual on everyone: +- `BackendCapabilities.supports_manual_response` — True (OpenAI: honors manual gating), + False (Grok: native VAD only). +- `UserRole`: manual-capable → today's "respond now" gating (OpenAI **unchanged**); + native-VAD → **no manual trigger**, we just consume grok's auto + `OUTPUT_TURN_STARTED`/`AUDIO_OUTPUT`/`TURN_END` (symmetric with the assistant). +- `GrokVoiceBackend` widens the caller VAD silence (`GROK_CALLER_MIN_SILENCE_MS=1200`, + only when `manual_turn_taking` is in config, i.e. caller use) so its native VAD doesn't + segment mid-turn and auto-respond into still-arriving audio (the turn-0 wedge). Also + drops inbound audio while `responding` as defense. Manual/`interrupt_response` config is + moot for grok (ignored), so it inherits the role-declared values. +- Tuning knob: `GROK_CALLER_MIN_SILENCE_MS`. If grok still auto-responds mid-turn, raise + it; if it's too laggy, lower it. If widening proves insufficient, the fallback is the + clean-turn feeding-gate (stop feeding at `speech_stopped` until `TURN_END`). + +### Grok — implementation notes +- `GrokVoiceBackend(OpenAIRealtimeBackend)` overrides only: default `base_url` + (x.ai), default `voice` (`eve`), api_key **required** (no OPENAI_API_KEY + fallback), and buffered input transcription. Everything else inherited. +- xAI fires `input_audio_transcription.completed` repeatedly with growing text; + the backend buffers it on the session and emits ONE final input `TRANSCRIPT` + at the turn boundary (next `speech_started` / `response.done`) — matching the + old server's deferred-transcript flush. Without this the role would append + multiple progressive user-input records. +- Generic seam added to `OpenAIRealtimeBackend`: `_SESSION_CLS` class attr that + `open()` instantiates, so a subclass can carry extra per-turn session state + (Grok's `pending_input_transcript`). No behavior change for OpenAI. +- The `grok_voice_server.py` docstring claims it overrides `_build_session_config` + to drop `transcription.model`, but the current class does NOT — so no + transcription-selector change was needed (matched actual code, not the docstring). + +## Known looseness (intentional, revisit as we migrate) + +- **User caller-config assembly is still OpenAI-Realtime-shaped** in + `_start_user_simulator` (`male_voice`/`female_voice`, `CALLER_BACKEND_DEFAULTS`). + The real generic top-level user JSON isn't designed yet; isinstance-narrow to + `OpenAIRealtimeSimulatorConfig` for now. +- **Cascade/pipeline configs** not adapted to the backend-args dump style yet. +- `AssistantRole` may grow features not currently in the S2S path as we migrate + cascade. + +## Behavior deltas vs legacy (all intentional, sub-macro) + +- `end_call` args logged as dict vs string. +- `connected` event drops OpenAI-specific labels. +- 24k-fixed mulaw converters (noted inline in role/user.py). +- `FrameworkLogWriter` S2S methods restored in `observers.py` (was a real `main` + regression from commit 4bf4881, bundled into this refactor). + +## Key files + +- `src/eva/backend/base.py` — `Backend` ABC, `BackendEvent`/`BackendEventType`, + `BackendSession`, `ToolCallRequest`/`ToolCallResult`. +- `src/eva/backend/openai_realtime.py` — reference backend + normalizer. +- `src/eva/backend/factory.py` / `default_factory.py` — factory. +- `src/eva/role/{base,assistant,user}.py` — roles. +- `src/eva/orchestrator/worker.py` — dispatch (`_start_assistant`, + `_start_user_simulator`, `_run_conversation`). +- `tests/unit/test_openai_realtime_backend.py` — backend unit tests (no network). diff --git a/src/eva/assistant/gemini_live_server.py b/src/eva/assistant/gemini_live_server.py index 377386a9..aeaac77b 100644 --- a/src/eva/assistant/gemini_live_server.py +++ b/src/eva/assistant/gemini_live_server.py @@ -138,7 +138,7 @@ def _agent_tools_to_gemini(agent: AgentConfig) -> list[types.Tool] | None: name=tool.function_name, description=f"{tool.name}: {tool.description}", parameters=params_schema, - behavior=types.Behavior.BLOCKING, + # behavior=types.Behavior.BLOCKING, ) ) @@ -188,11 +188,64 @@ def __init__( # Gemini model name from s2s_params or default s2s_params = self.pipeline_config.s2s_params or {} self._model = s2s_params["model"] + # Optional Vertex endpoint override; when unset the SDK uses its default. + self._endpoint = s2s_params.get("endpoint") + # Optional Vertex API version override (e.g. "v1beta1", "v1"). Live/S2S + # preview models are Vertex-only and may require a specific version; when + # unset the SDK's default is used. + self._api_version = s2s_params.get("api_version") + # Optional FunctionResponse scheduling ("WHEN_IDLE" | "INTERRUPT" | + # "SILENT"). Newer Live models (e.g. gemini-3.5-flash-live-preview) do + # NOT support a scheduling field and close the socket with 1007 if one is + # set, so it is OMITTED by default; older models can opt back in. + self._fc_scheduling = s2s_params.get("function_response_scheduling") self._voice = s2s_params.get("voice", "Kore") # s2s_params["language_code"] takes precedence; fall back to EVA_LANGUAGE self._language_code = s2s_params.get("language_code") or self.language self._api_key = s2s_params.get("api_key", "") + # Vertex project/location resolution. Accept both the google-genai names + # (GOOGLE_CLOUD_*) and the LiteLLM/Vertex names (VERTEXAI_*) so a single + # set of credentials works for both the judge and the Live server. + self._vertex_project = ( + s2s_params.get("project") or os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("VERTEXAI_PROJECT") + ) + # Gemini Live/S2S requires a REGIONAL endpoint; "global" (fine for the + # text judge) is not supported for bidiGenerateContent, so it is never + # used here — an explicit region wins, otherwise we fall back to a region. + location = ( + s2s_params.get("location") or os.environ.get("GOOGLE_CLOUD_LOCATION") or os.environ.get("VERTEXAI_LOCATION") + ) + if not location or location == "global": + if location == "global": + logger.warning( + "Gemini Live does not support location='global'; using 'us-central1' instead. " + "Set s2s_params['location'] or GOOGLE_CLOUD_LOCATION to the region the model is enabled in." + ) + location = "us-central1" + self._vertex_location = location + + # Thinking config: controls Gemini's internal reasoning budget. + # Accepts a dict with optional keys: + # "thinking_budget": int (0=disabled, -1=auto, or token count) + # "include_thoughts": bool + # "thinking_level": str ("MINIMAL", "LOW", "MEDIUM", "HIGH") + # Example: {"thinking_budget": 1024, "include_thoughts": false} + # If not set, defaults to ThinkingConfig() (model-dependent defaults). + thinking_raw = s2s_params.get("thinking_config", {}) + if isinstance(thinking_raw, dict) and thinking_raw: + tc_kwargs: dict[str, Any] = {} + if "thinking_budget" in thinking_raw: + tc_kwargs["thinking_budget"] = int(thinking_raw["thinking_budget"]) + if "include_thoughts" in thinking_raw: + tc_kwargs["include_thoughts"] = bool(thinking_raw["include_thoughts"]) + if "thinking_level" in thinking_raw: + tc_kwargs["thinking_level"] = thinking_raw["thinking_level"] + self._thinking_config = types.ThinkingConfig(**tc_kwargs) + logger.info(f"Thinking config: {tc_kwargs}") + else: + self._thinking_config = types.ThinkingConfig() + self._system_prompt = self._build_system_prompt() # Build Gemini tools @@ -269,17 +322,54 @@ async def _shutdown(self) -> None: # ------------------------------------------------------------------ def _create_genai_client(self) -> genai.Client: - """Create a google-genai Client using Vertex AI or API key.""" + """Create a google-genai Client for Vertex AI or the Developer API. + + Vertex-only models (e.g. gemini-*-live-preview) must route through + aiplatform.googleapis.com with ``vertexai=True``; a Developer API key + (AIza…) would send them to generativelanguage.googleapis.com/v1beta, + where they 404. We therefore prefer Vertex whenever a project is + resolvable (or GOOGLE_GENAI_USE_VERTEXAI is set) and ignore any + Developer API key in that mode. + """ + flag = os.environ.get("GOOGLE_GENAI_USE_VERTEXAI") + if flag is not None: + use_vertex = flag.strip().lower() in ("1", "true", "yes") + else: + use_vertex = bool(self._vertex_project) + + if use_vertex: + if not self._vertex_project: + raise ValueError( + "Vertex mode requested but no project found. Set GOOGLE_CLOUD_PROJECT / " + "VERTEXAI_PROJECT or s2s_params['project']." + ) + http_kwargs: dict[str, Any] = {} + if self._endpoint: + http_kwargs["base_url"] = f"wss://{self._endpoint}" + if self._api_version: + http_kwargs["api_version"] = self._api_version + http_options = types.HttpOptions(**http_kwargs) if http_kwargs else None + + if self._api_key: + logger.warning( + "Ignoring s2s_params api_key in Vertex mode (Vertex uses ADC / service-account " + "credentials via GOOGLE_APPLICATION_CREDENTIALS)." + ) + logger.info( + f"Using Vertex AI (project={self._vertex_project}, location={self._vertex_location}, " + f"api_version={self._api_version or 'sdk-default'})" + ) + return genai.Client( + vertexai=True, + project=self._vertex_project, + location=self._vertex_location, + http_options=http_options, + ) + if self._api_key: - logger.info("Using Gemini API key for authentication") + logger.info("Using Gemini Developer API key for authentication") return genai.Client(api_key=self._api_key) - project = os.environ.get("GOOGLE_CLOUD_PROJECT") - location = os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1") - if project: - logger.info(f"Using Vertex AI (project={project}, location={location})") - return genai.Client(vertexai=True, project=project, location=location) - # Fallback: let the SDK resolve credentials (e.g. ADC) logger.warning(msg="No explicit credentials; relying on google-genai default resolution") return genai.Client() @@ -312,6 +402,7 @@ def _build_live_config(self) -> types.LiveConnectConfig: ), "input_audio_transcription": types.AudioTranscriptionConfig(), "output_audio_transcription": types.AudioTranscriptionConfig(), + "thinking_config": self._thinking_config, } if self._gemini_tools: config_kwargs["tools"] = self._gemini_tools @@ -608,16 +699,18 @@ async def _process_gemini_events() -> None: f"Tool result: {tool_name} -> {json.dumps(result, ensure_ascii=False)}" ) - # Send result back to Gemini + # Send result back to Gemini. Only set the + # scheduling field when explicitly configured + # — newer Live models reject it (1007). + fr_kwargs: dict[str, Any] = { + "id": fc.id, + "name": fc.name, + "response": result, + } + if self._fc_scheduling: + fr_kwargs["scheduling"] = types.FunctionResponseScheduling[self._fc_scheduling] await session.send_tool_response( - function_responses=[ - types.FunctionResponse( - id=fc.id, - name=fc.name, - response=result, - scheduling=types.FunctionResponseScheduling.WHEN_IDLE, - ) - ] + function_responses=[types.FunctionResponse(**fr_kwargs)] ) # --- Usage metadata --- diff --git a/src/eva/backend/__init__.py b/src/eva/backend/__init__.py index b9513915..aeb94b61 100644 --- a/src/eva/backend/__init__.py +++ b/src/eva/backend/__init__.py @@ -1,13 +1,19 @@ -"""Provider-agnostic ``Backend`` abstraction (design-only, Step 1 of the refactor). +"""Provider-agnostic ``Backend`` abstraction (see ``docs/refactor-step1.md``). -This package defines the contracts described in ``docs/refactor-step1.md``: -pure API/session objects (``Backend``) that know nothing about role -(assistant vs. user), plus a factory to construct them. Nothing in this -package is wired into the existing ``eva.assistant`` / ``eva.user_simulator`` -code yet -- these are new, additive, currently-unused types. +This package defines pure API/session objects (``Backend``) that know nothing +about role (assistant vs. user), plus a ``BackendFactory`` to construct them. +The worker builds a backend per conversation and drives it through an +``AssistantRole`` / ``UserRole`` for every provider the factory supports. """ -from eva.backend.base import Backend, BackendEvent, BackendEventType, ToolCallRequest, ToolCallResult +from eva.backend.base import ( + Backend, + BackendEvent, + BackendEventType, + BackendSession, + ToolCallRequest, + ToolCallResult, +) from eva.backend.capabilities import BackendCapabilities from eva.backend.factory import BackendFactory @@ -17,6 +23,7 @@ "BackendEvent", "BackendEventType", "BackendFactory", + "BackendSession", "ToolCallRequest", "ToolCallResult", ] diff --git a/src/eva/backend/base.py b/src/eva/backend/base.py index 63bbd278..71b8446b 100644 --- a/src/eva/backend/base.py +++ b/src/eva/backend/base.py @@ -1,8 +1,9 @@ """Abstract ``Backend`` contract: pure API/session exchange, no role knowledge. -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). This module -defines shapes, not behavior -- every method body is a stub. Nothing in -``eva.assistant`` or ``eva.user_simulator`` depends on this yet. +This is the live ``Backend`` contract -- +implemented by ``eva.backend.openai_realtime`` and driven by ``AssistantRole`` +/ ``UserRole``. The worker builds one per conversation via ``BackendFactory`` +for every provider the factory supports. A ``Backend`` wraps exactly one provider integration (OpenAI Realtime, Gemini Live, ElevenLabs Agents, a cascade STT->LLM->TTS pipeline, ...) and exposes a @@ -25,11 +26,12 @@ from __future__ import annotations +import os from abc import ABC, abstractmethod from collections.abc import AsyncIterator from dataclasses import dataclass, field from enum import StrEnum -from typing import Any +from typing import Any, ClassVar from eva.backend.capabilities import BackendCapabilities @@ -37,11 +39,16 @@ class BackendEventType(StrEnum): """Kinds of events a ``Backend`` can surface via ``receive()``. - Not every ``Backend`` implementation will emit every event type -- a thin, + Every event is normalized: typed fields (``audio`` / ``transcript`` / + ``tool_call_request`` / ``error``) plus normalized ``metadata`` scalars. + Backends never surface raw provider event objects -- all provider-specific + parsing happens inside the backend, so a ``Role`` consuming these stays + fully provider-agnostic. + + Not every ``Backend`` implementation emits every event type -- a thin, end-to-end backend (e.g. ElevenLabs Agents) may only ever emit ``AUDIO_OUTPUT``, ``TRANSCRIPT``, ``TURN_END``, and ``ERROR``, because it - has no separable tool-calling seam of its own that the caller can observe - (tool calls, if any, happen inside the provider and are not surfaced). + has no separable tool-calling seam of its own that the caller can observe. Consumers must treat unhandled event types as ignorable, not as errors. """ @@ -50,24 +57,45 @@ class BackendEventType(StrEnum): simulated user's speech, depending on which role's Backend this is).""" TRANSCRIPT = "transcript" - """A (possibly partial) transcript of something spoken -- either the - backend's own output or, for backends that provide it, the other party's - input as heard by this backend's ASR.""" + """A finalized transcript of something spoken. ``transcript`` holds the + text; ``metadata`` carries normalized descriptors: ``stream`` is + ``"input"`` (what the backend heard from the inbound party) or ``"output"`` + (what the backend's own model said), and for input transcripts + ``metadata["failed"] = True`` marks a transcription failure (empty text). + These are normalized scalars, not raw provider payloads.""" TOOL_CALL_REQUEST = "tool_call_request" - """The backend's model wants to invoke a tool. Only emitted by backends - that expose a separable tool-calling seam (native S2S realtime APIs, - cascade LLM backends). The owning ``Role`` is responsible for executing - the tool and returning the result via ``send(tool_result=...)`` -- the - ``Backend`` never executes tools itself (see docs/refactor-step1.md, - "tool execution stays role-side").""" + """The backend's model wants to invoke a tool. The owning ``Role`` is responsible for executing + the tool and returning the result via ``send(tool_result=...)``""" TURN_END = "turn_end" - """The backend's model has finished its current turn (end-of-utterance / - end-of-response signal).""" + """The backend's model finished a response turn. informs whether turn was cancelled, interrupted, etc.""" + + INPUT_SPEECH_STARTED = "input_speech_started" + """The backend's VAD detected that the *inbound* party (whoever is talking + *to* this backend's model) started speaking. Role-agnostic: for an + ``AssistantRole`` backend the inbound party is the caller, for a + ``UserRole`` backend it is the assistant. Emitted only by backends whose + provider surfaces input-side voice-activity boundaries (native S2S realtime + APIs).""" + + INPUT_SPEECH_STOPPED = "input_speech_stopped" + """The backend's VAD detected that the inbound party stopped speaking. The + end-of-speech counterpart to ``INPUT_SPEECH_STARTED`` (see its docstring).""" + + OUTPUT_TURN_STARTED = "output_turn_started" + """The backend's model began a response turn. Needed by a manually-sequencing + consumer (e.g. a ``UserRole`` gating replies) to know a response is now in + flight; consumers that don't care ignore it.""" + + OUTPUT_AUDIO_DONE = "output_audio_done" + """The backend's model finished emitting output audio for the current turn + (the audio stream is drained, distinct from ``TURN_END`` which also covers + the text/tool bookkeeping). Lets a consumer that paces or gates on playout + flush trailing output; ignorable otherwise.""" ERROR = "error" - """A provider-level error occurred (connection drop, API error, etc.).""" + """A provider-level error occurred (connection drop, API error, etc.)""" @dataclass @@ -121,33 +149,44 @@ class BackendEvent: tool_call_request: ToolCallRequest | None = None error: str | None = None metadata: dict[str, Any] = field(default_factory=dict) - """Provider-specific extras (e.g. raw event name, timestamps) that don't - warrant a first-class field. Consumers should not rely on specific keys - being present across providers. - - Convention (not enforced by this contract): a backend that proactively - re-engages after a dropped user turn (the turn-end fallback; see - ``AssistantRole``'s ``turn_end_fallback_seconds`` and the shipped - ``eva.assistant.pipeline.fallback``) tags the ``AUDIO_OUTPUT``/ - ``TRANSCRIPT`` event it emits for that turn so callers can distinguish a - fallback nudge from an ordinary model turn (e.g. for audit logging and so - downstream metrics can zero it). The shipped feature records the transcript - marker with ``message_type="turn_fallback"``; a backend surfacing the same - turn here should carry an equivalent flag in ``metadata`` (e.g. - ``metadata["turn_fallback"] = True``). This is *not* a new event type -- a - nudge is just an ordinary turn from the backend's model, triggered by the - backend noticing that a user turn was never detected within the fallback - window rather than by new input; it flows through the same ``receive()`` - surface as anything else.""" + """Normalized descriptors for this event -- plain scalars/dicts, never a raw + provider event object. Which keys are present depends on ``event_type`` and + is documented on each ``BackendEventType`` member (e.g. ``stream`` / + ``failed`` for ``TRANSCRIPT``; ``cancelled`` / ``interrupted`` / + ``has_function_calls`` / ``usage`` for ``TURN_END``; ``code`` for + ``ERROR``). A ``Role`` consumes these normalized fields only, so it stays + provider-agnostic: all provider-specific event parsing happens inside the + backend before emission.""" + + +class BackendSession: + """Opaque handle to one live provider session, returned by ``Backend.open``. + + The ``Backend`` itself is stateless beyond its construction config (model, + key, endpoint): *all* per-exchange state -- the live connection, any + provider-side accumulators -- lives on the session handle, not on the + backend. The caller (a ``Role``, or later a mediator) holds this handle and + passes it back into ``send`` / ``receive`` / ``close``. Concrete backends + subclass this with whatever they need to carry; consumers treat it as + opaque and never introspect it. + + Keeping session state off the backend is deliberate (see + docs/refactor-step1.md discussion): one ``Backend`` instance can then serve + many independent sessions/conversations concurrently, and no exchange data + is smuggled into the backend object. + """ class Backend(ABC): - """Pure API/session exchange with one provider. No role knowledge. + """Stateless adapter to one provider's API. No role knowledge, no session state. - Lifecycle: ``open()`` establishes the session, ``send()`` pushes audio / - text / tool results to the provider, ``receive()`` yields events back, - and ``close()`` tears the session down. A ``Role`` (see - ``eva.role.base``) owns one ``Backend`` instance and drives it. + Lifecycle: ``open()`` establishes a session and returns a + ``BackendSession`` handle, ``send()`` pushes audio / text / tool results to + the provider on a given session, ``receive()`` yields events back for a + session, and ``close()`` tears a session down. The backend holds only its + construction config; the caller holds the session handle. A ``Role`` (see + ``eva.role.base``) is given a ``Backend`` instance (constructed by the + worker via a ``BackendFactory`` the worker owns) and drives it. Implementations are expected to fall along a spectrum: @@ -168,8 +207,67 @@ class Backend(ABC): Symmetry: this contract says nothing about which side dials out and which side is dialed into -- see the module docstring. + + Construction/config: every concrete backend is built from a single flat + ``config`` dict (assembled by the worker/factory) and validates it in its own + ``__init__`` -- there is no per-provider pydantic schema, so config validation + lives entirely here and can be run cheaply/early (see + ``orchestrator.preflight``). To keep that validation *identical* across + providers, the shared pieces of the config contract live on this base: + + - ``api_key``: every backend resolves it the same way via + ``_resolve_api_key`` -- explicit ``config['api_key']`` first, else the + provider's ``_API_KEY_ENV`` environment variable. + - ``speaker_id``: the single, provider-agnostic speaker identifier (an OpenAI/ + Grok *voice* name, or an ElevenLabs *agent id*) -- one key name so callers + learn it once. Backends read it with ``_require`` (mandatory) or + ``config.get("speaker_id", )`` (optional with a provider default). + + ``model`` and any provider-specific extras are read directly by each backend; + the extras are optional and need no shared naming. + + Validation is *cumulative*: the shared helpers below collect problems into an + ``errors`` list instead of raising on the first one, and the backend calls + ``_raise_config_errors(errors)`` once at the end -- so a caller who got two + fields wrong sees both at once rather than fixing-and-rerunning. """ + _API_KEY_ENV: ClassVar[str | None] = None + """Environment variable the api_key falls back to when ``config['api_key']`` is + absent. Each concrete backend sets it (``OPENAI_API_KEY`` / ``XAI_API_KEY`` / + ``ELEVENLABS_API_KEY``); ``None`` disables the fallback (key must be explicit).""" + + @classmethod + def _resolve_api_key(cls, config: dict[str, Any], errors: list[str]) -> str: + """Resolve the api_key uniformly: explicit config, else ``_API_KEY_ENV``. + + Records a problem in ``errors`` (returning ``""``) rather than raising, so + api-key validation aggregates with the rest -- the single api-key path + every backend shares. + """ + key = config.get("api_key") or (os.environ.get(cls._API_KEY_ENV) if cls._API_KEY_ENV else None) + if not key: + hint = "config['api_key']" + (f" or {cls._API_KEY_ENV}" if cls._API_KEY_ENV else "") + errors.append(f"missing api_key ({hint})") + return "" + return str(key) + + @classmethod + def _require(cls, config: dict[str, Any], key: str, errors: list[str]) -> str: + """Return a required non-empty ``config[key]``, or record a problem in ``errors``.""" + value = config.get(key) + if not value: + errors.append(f"missing '{key}' (config['{key}'])") + return "" + return str(value) + + @classmethod + def _raise_config_errors(cls, errors: list[str]) -> None: + """Raise a single ``ValueError`` listing every collected config problem, if any.""" + if errors: + joined = "\n".join(f" - {e}" for e in errors) + raise ValueError(f"{cls.__name__} config invalid:\n{joined}") + @property @abstractmethod def capabilities(self) -> BackendCapabilities: @@ -180,39 +278,51 @@ def capabilities(self) -> BackendCapabilities: """ ... + @property + def input_sample_rate(self) -> int: + """Sample rate (Hz) of PCM the backend expects via ``send(audio=...)``. + + Lets a role convert/record counterparty audio without knowing the + provider. Defaults to 24 kHz (the common realtime rate); backends on a + different rate override. Describes the session's audio format, not turn + state. + """ + return 24000 + + @property + def output_sample_rate(self) -> int: + """Sample rate (Hz) of PCM carried in ``AUDIO_OUTPUT`` events. + + See ``input_sample_rate``; defaults to 24 kHz, overridden per backend. + """ + return 24000 + @abstractmethod - async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None, config: dict[str, Any]) -> None: - """Establish the provider session. + async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None) -> BackendSession: + """Establish a provider session and return its opaque handle. + + The role supplies only the two things that are genuinely its own -- the + prompt and the tool catalog. All provider-specific session shaping + (model, voice, sample rate, turn-detection, audio formats, ...) is the + backend's own construction config, injected by the worker via the + ``BackendFactory``; the role neither builds nor sees it. That is what + keeps a single generic ``Role`` usable with any backend. Args: - system_prompt: Fully-built system prompt for this session, as - assembled by the owning ``Role`` (``Role.build_prompt()``). - A thin end-to-end backend still receives this even if it - maps it onto a different provider concept (e.g. ElevenLabs - agent overrides). - tools: Tool schemas to expose to the provider's model, in - whatever wire format the concrete backend needs to translate - from the agent's tool definitions. ``None`` or ``[]`` for - backends/roles that don't expose tool calling (e.g. a - ``UserRole`` that only needs an ``end_call`` tool would still - pass that single tool here; a backend with no tool-calling - seam at all may simply ignore this argument). - config: Provider-specific configuration blob (model name, voice, - sample rate, turn-detection parameters, etc.). Deliberately - untyped here -- each concrete ``Backend`` defines and - validates its own config shape; the abstract contract does - not prescribe one, since a native S2S config and a cascade - config share little structure. An ``AssistantRole`` backend - configured for the turn-end fallback (see - ``AssistantRole.turn_end_fallback_seconds``) reads its - threshold from this blob (e.g. a - ``config["turn_end_fallback_seconds"]`` key) the same way -- - the fallback needs no dedicated typed parameter or new - ``Backend`` method, since the resulting nudge is just an - ordinary outbound turn (see ``BackendEvent.metadata``). - - Must be safe to call exactly once per ``Backend`` instance. Must not - block on the other party being ready to exchange data -- readiness to + system_prompt: Fully-built system prompt/instructions for this + session (the role's ``build_prompt()`` output). A thin + end-to-end backend still receives this even if it maps it onto a + different provider concept (e.g. ElevenLabs agent overrides). + tools: Provider-agnostic tool specs -- a list of + ``{"name", "description", "parameters"}`` dicts -- which the + backend translates into its provider's tool-schema wire format. + ``None`` or ``[]`` for roles that expose no tools; a backend + with no tool-calling seam may ignore this argument. + + Each call returns a fresh, independent ``BackendSession``; because the + backend carries no session state, a single ``Backend`` instance may be + opened many times (e.g. one session per conversation). Must not block + on the other party being ready to exchange data -- readiness to *accept* traffic is enough (mirrors today's ``AbstractAssistantServer.start()`` contract: non-blocking, returns once ready). @@ -222,14 +332,16 @@ async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None, @abstractmethod async def send( self, + session: BackendSession, *, audio: bytes | None = None, text: str | None = None, tool_result: ToolCallResult | None = None, ) -> None: - """Push data to the provider. Exactly one of the keyword args is set. + """Push data to the provider on ``session``. Exactly one kwarg is set. Args: + session: The handle returned by ``open()`` for this exchange. audio: Raw input audio chunk (format/sample-rate is whatever this backend's ``open(config=...)`` declared; format conversion is the caller's responsibility via the shared audio utilities, @@ -252,8 +364,8 @@ async def send( ... @abstractmethod - def receive(self) -> AsyncIterator[BackendEvent]: - """Yield events from the provider as they arrive. + def receive(self, session: BackendSession) -> AsyncIterator[BackendEvent]: + """Yield events from the provider on ``session`` as they arrive. The single, symmetric inbound stream for both "network-server-like" and "client-like" backends. Must be an async generator (or return an @@ -265,12 +377,26 @@ def receive(self) -> AsyncIterator[BackendEvent]: """ ... + async def trigger_response(self, session: BackendSession) -> None: + """Ask the provider to generate a response now, with no new input. + + Only meaningful for backends whose turn detection is configured *not* + to auto-create responses, so the caller sequences replies itself (e.g. + a ``UserRole`` that gates when the simulated caller speaks). Backends + with no such control -- thin end-to-end providers, or any backend where + responses are always driven by input/tool-results -- leave this as the + default ``NotImplementedError``; consult ``capabilities`` / provider + docs before calling. Not abstract, so those backends need not implement + it. + """ + raise NotImplementedError(f"{type(self).__name__} does not support trigger_response()") + @abstractmethod - async def close(self) -> None: - """Tear down the provider session. + async def close(self, session: BackendSession) -> None: + """Tear down the given provider ``session``. - Must be safe to call even if ``open()`` was never called or the - session already ended on its own (idempotent). Concrete backends are + Must be safe to call even if the session already ended on its own + (idempotent). Concrete backends are responsible for their own provider-specific teardown (closing websockets, cancelling tasks, flushing buffers); this method does not itself define audio/output persistence -- that remains a ``Role`` diff --git a/src/eva/backend/capabilities.py b/src/eva/backend/capabilities.py index 7561872b..849f9be9 100644 --- a/src/eva/backend/capabilities.py +++ b/src/eva/backend/capabilities.py @@ -1,9 +1,8 @@ """Capability flags describing what a ``Backend`` implementation can do. -These flags exist so that a future mediator/turn-taking layer can branch on -backend shape without every ``Backend`` implementation needing to expose the -same granular seams. Per docs/refactor-step1.md, they are declared now but -must remain UNUSED in this step -- no turn-taking logic should read them yet. +These flags let a role / turn-taking layer branch on backend shape without every +``Backend`` implementation exposing the same granular seams. Informational for now, +reserved for the later mediator/interruption work. """ from dataclasses import dataclass @@ -15,7 +14,6 @@ class BackendCapabilities: Each concrete ``Backend`` subclass sets these once (typically as a class attribute or constructed in ``__init__``) to describe its streaming shape. - They are informational only in this step -- nothing consumes them yet. Attributes: emits_continuous_audio: True if the backend produces a continuous diff --git a/src/eva/backend/elevenlabs.py b/src/eva/backend/elevenlabs.py new file mode 100644 index 00000000..4d6dd099 --- /dev/null +++ b/src/eva/backend/elevenlabs.py @@ -0,0 +1,375 @@ +"""ElevenLabs ``Backend``: end-to-end Conversational AI behind the role contract. + +ElevenLabs Agents are a *thin, end-to-end* provider (see ``eva.backend.base``): +the agent owns ASR, dialogue policy, turn-taking, and TTS server-side, so this +backend exposes far fewer knobs than the OpenAI Realtime family. It carries the +same shared config core -- ``model`` (a metrics label here), ``api_key`` (with an +``ELEVENLABS_API_KEY`` env fallback) -- plus the one shared *speaker* field: +``speaker_id`` (an ElevenLabs agent id here, the realtime backends' voice analog). +Everything else (VAD / transcription / formats / reasoning) lives inside the +ElevenLabs agent and has no config surface here. + +Adapting the SDK to the ``Backend`` contract requires bridging three shape +differences; all of them are absorbed here so ``AssistantRole`` stays generic +and unchanged: + +- **callbacks -> event stream**: the SDK is callback-driven + (agent-response / user-transcript / end-session). Each callback pushes a + normalized ``BackendEvent`` onto an internal queue that ``receive()`` drains. +- **audio rate**: the SDK speaks 8 kHz mulaw in / 16 kHz PCM out, but the role's + Twilio transport is uniformly 24 kHz PCM. The backend declares 24 kHz in/out + and converts internally (24 kHz PCM -> 8 kHz mulaw for the SDK; 16 kHz PCM -> + 24 kHz PCM for ``AUDIO_OUTPUT``), so the role's audio path is untouched. +- **tool execution**: the SDK runs tools itself via ``ClientTools`` handlers. + To keep tool execution role-side (matching OpenAI/Grok), each handler emits a + ``TOOL_CALL_REQUEST`` event and awaits a future the role resolves via + ``send(tool_result=...)`` -- so the ElevenLabs agent's tool call round-trips + through the same ``AssistantRole`` seam as every other backend. + +Greeting: ``AssistantRole`` triggers the opening line with +``send(text="Say: ''")`` right after ``open()``. ElevenLabs greets +from an ``initial_message`` dynamic variable set at session start, so the actual +``start_session()`` is deferred to that first ``send(text=...)`` -- the greeting +is unwrapped and injected as the dynamic variable, matching the legacy server. +""" + +from __future__ import annotations + +import asyncio +import audioop +import json +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any, ClassVar + +import httpx +from elevenlabs.client import ElevenLabs +from elevenlabs.conversational_ai.conversation import ( + AsyncConversation, + ClientTools, + ConversationInitiationData, +) + +from eva.assistant.elevenlabs_audio_interface import TwilioAudioBridge +from eva.backend.base import ( + Backend, + BackendEvent, + BackendEventType, + BackendSession, + ToolCallRequest, + ToolCallResult, +) +from eva.backend.capabilities import BackendCapabilities +from eva.utils.audio_utils import pcm16_24k_to_mulaw_8k +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +DEFAULT_MODEL = "elevenlabs" +ROLE_SAMPLE_RATE = 24000 # the rate AssistantRole's Twilio transport works in +ELEVENLABS_OUTPUT_RATE = 16000 # the SDK delivers assistant audio at 16 kHz + + +def _pcm16_24k_to_mulaw_8k(pcm_24k: bytes) -> bytes: + """Convert the role's 24 kHz PCM input into the SDK's 8 kHz mulaw.""" + return pcm16_24k_to_mulaw_8k(pcm_24k) + + +def _pcm16_16k_to_pcm16_24k(pcm_16k: bytes) -> bytes: + """Resample the SDK's 16 kHz PCM output up to the role's 24 kHz.""" + pcm_24k, _ = audioop.ratecv(pcm_16k, 2, 1, ELEVENLABS_OUTPUT_RATE, ROLE_SAMPLE_RATE, None) + return pcm_24k + + +def _unwrap_greeting(text: str) -> str: + """Strip the role's ``Say: ''`` wrapper down to the raw greeting.""" + prefix = "Say: '" + if text.startswith(prefix) and text.endswith("'"): + return text[len(prefix) : -1] + return text + + +@dataclass +class ElevenLabsSession(BackendSession): + """Live state for one ElevenLabs Conversational AI session. + + Holds the SDK client + audio bridge, the queue that ``receive()`` drains, the + pending tool-call futures (resolved by the role via ``send(tool_result=...)``), + and the background task pumping SDK output audio into ``AUDIO_OUTPUT`` events. + The ``AsyncConversation`` itself is created lazily on the first + ``send(text=...)`` (see the module docstring on greeting/deferred start). + """ + + client: Any + bridge: Any + system_prompt: str + client_tools: Any + events: asyncio.Queue[BackendEvent] = field(default_factory=asyncio.Queue) + pending_tools: dict[str, asyncio.Future[ToolCallResult]] = field(default_factory=dict) + conversation: Any = None + output_task: asyncio.Task[None] | None = None + ended: asyncio.Event = field(default_factory=asyncio.Event) + + +class ElevenLabsBackend(Backend): + """One ElevenLabs Conversational AI session behind the ``Backend`` contract. + + Recognized ``config`` keys: + + - ``model`` (optional, default ``"elevenlabs"``): a metrics label only -- + ElevenLabs selects the actual model inside the agent. + - ``api_key`` (optional): falls back to the ``ELEVENLABS_API_KEY`` env var. + - ``speaker_id`` (**required**): the shared speaker-identifier key (an ElevenLabs + *agent id* here, the realtime backends' voice-name analog). + """ + + _API_KEY_ENV: ClassVar[str] = "ELEVENLABS_API_KEY" + + # End-to-end provider: continuous audio, no first-class streaming-interruption + # seam exposed to us, and it does not own the role's playout clock. + _CAPABILITIES = BackendCapabilities( + emits_continuous_audio=True, supports_streaming_interruption=False, owns_playout_clock=False + ) + + def __init__(self, *, config: dict[str, Any]) -> None: + errors: list[str] = [] + self._api_key = self._resolve_api_key(config, errors) + self._agent_id: str = self._require(config, "speaker_id", errors) + self._raise_config_errors(errors) + self._model: str = config.get("model") or DEFAULT_MODEL + + @property + def capabilities(self) -> BackendCapabilities: + return self._CAPABILITIES + + @property + def input_sample_rate(self) -> int: + # Declared at the role's rate; the SDK's 8 kHz mulaw conversion is internal. + return ROLE_SAMPLE_RATE + + @property + def output_sample_rate(self) -> int: + # AUDIO_OUTPUT is resampled up to the role's rate (from the SDK's 16 kHz). + return ROLE_SAMPLE_RATE + + # ── Session lifecycle ───────────────────────────────────────────── + + async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None) -> ElevenLabsSession: + """Build the client, audio bridge, and tool bridge; defer session start. + + The ElevenLabs ``AsyncConversation`` is created on the first + ``send(text=...)`` (the greeting), which supplies the ``initial_message`` + dynamic variable the agent greets from. + """ + client = ElevenLabs(api_key=self._api_key, timeout=30.0, httpx_client=httpx.Client(verify=False, timeout=30.0)) + session = ElevenLabsSession( + client=client, + bridge=TwilioAudioBridge(), + system_prompt=system_prompt, + client_tools=None, + ) + session.client_tools = self._build_client_tools(session, tools) + logger.info(f"ElevenLabs backend prepared (agent_id={self._agent_id})") + return session + + def _build_client_tools(self, session: ElevenLabsSession, tools: list[dict[str, Any]] | None) -> Any: + """Register each generic tool spec as an SDK ``ClientTool`` that bridges to the role. + + Each handler emits a ``TOOL_CALL_REQUEST`` event and blocks on a future the + role resolves via ``send(tool_result=...)``, so tool execution stays role-side. + + The ``ClientTools`` is bound to THIS (the caller's) event loop. Without a + ``loop``, the SDK spins its own loop in a separate thread and runs handlers + there -- our handler would then touch the main-loop ``session.events`` queue and + the ``send(tool_result=...)`` future cross-loop, raise, and the SDK would report + the tool as failed (``is_error``) to the agent. Binding it to the running loop + keeps the whole tool round-trip on one loop. + """ + if not tools: + return None + client_tools = ClientTools(loop=asyncio.get_running_loop()) + for spec in tools: + name = spec["name"] + + async def _handle(parameters: dict[str, Any], _name: str = name) -> str: + return await self._bridge_tool_call(session, _name, parameters) + + client_tools.register(name, _handle, is_async=True) + return client_tools + + async def _bridge_tool_call(self, session: ElevenLabsSession, name: str, parameters: dict[str, Any]) -> str: + """Surface an SDK tool call as a role event; await and return the role's result.""" + # The SDK injects tool_call_id into parameters; use it to correlate the result. + call_id = str(parameters.get("tool_call_id") or f"{name}-{len(session.pending_tools)}") + arguments = {k: v for k, v in parameters.items() if k != "tool_call_id"} + future: asyncio.Future[ToolCallResult] = asyncio.get_running_loop().create_future() + session.pending_tools[call_id] = future + await session.events.put( + BackendEvent( + event_type=BackendEventType.TOOL_CALL_REQUEST, + tool_call_request=ToolCallRequest(call_id=call_id, name=name, arguments=arguments), + ) + ) + result = await future + return json.dumps(result.result, ensure_ascii=False) if isinstance(result.result, dict) else str(result.result) + + async def send( + self, + session: BackendSession, + *, + audio: bytes | None = None, + text: str | None = None, + tool_result: ToolCallResult | None = None, + ) -> None: + """Push audio / the greeting / a tool result (exactly one).""" + provided = [x is not None for x in (audio, text, tool_result)] + if sum(provided) != 1: + raise ValueError("send() requires exactly one of audio, text, tool_result") + s = self._session(session) + + if audio is not None: + # 24 kHz PCM from the role -> 8 kHz mulaw for the SDK's audio interface. + await s.bridge.feed_user_audio(_pcm16_24k_to_mulaw_8k(audio)) + return + + if text is not None: + # The greeting: start the deferred session with it as initial_message. + await self._start_session(s, greeting=_unwrap_greeting(text)) + return + + assert tool_result is not None # exactly-one check above + future = s.pending_tools.pop(tool_result.call_id, None) + if future is not None and not future.done(): + future.set_result(tool_result) + + async def _start_session(self, session: ElevenLabsSession, *, greeting: str) -> None: + """Create and start the ElevenLabs conversation, then pump its output audio.""" + if session.conversation is not None: + return + conv_config = ConversationInitiationData( + dynamic_variables={"system_prompt": session.system_prompt, "initial_message": greeting}, + ) + + # The SDK expects awaitable callbacks; each just enqueues a normalized event. + async def _on_agent_response(text: str) -> None: + self._enqueue(session, self._agent_response_event(text)) + + async def _on_agent_response_correction(original: str, corrected: str) -> None: + self._enqueue(session, self._correction_event(corrected)) + + async def _on_user_transcript(text: str) -> None: + self._enqueue(session, self._user_transcript_event(text)) + + async def _on_end_session() -> None: + session.ended.set() + + session.conversation = AsyncConversation( + session.client, + self._agent_id, + requires_auth=True, + audio_interface=session.bridge, + config=conv_config, + client_tools=session.client_tools, + callback_agent_response=_on_agent_response, + callback_agent_response_correction=_on_agent_response_correction, + callback_user_transcript=_on_user_transcript, + callback_end_session=_on_end_session, + ) + await session.conversation.start_session() + session.output_task = asyncio.create_task(self._pump_output(session)) + logger.info("ElevenLabs conversation session started") + + @staticmethod + def _enqueue(session: ElevenLabsSession, event: BackendEvent) -> None: + """Push a normalized event from an SDK callback onto the receive queue.""" + session.events.put_nowait(event) + + @staticmethod + def _agent_response_event(text: str) -> BackendEvent: + """A completed assistant turn -> ``TURN_END`` (thin provider: no tool/usage metadata).""" + return BackendEvent( + event_type=BackendEventType.TURN_END, + transcript=(text or "").strip(), + metadata={"interrupted": False, "cancelled": False, "has_function_calls": False, "usage": None}, + ) + + @staticmethod + def _correction_event(corrected: str) -> BackendEvent: + """An interruption-corrected assistant turn -> interrupted ``TURN_END``.""" + return BackendEvent( + event_type=BackendEventType.TURN_END, + transcript=(corrected or "").strip(), + metadata={"interrupted": True, "cancelled": False, "has_function_calls": False, "usage": None}, + ) + + @staticmethod + def _user_transcript_event(text: str) -> BackendEvent: + """A finalized user transcript -> input ``TRANSCRIPT``.""" + return BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript=(text or "").strip(), + metadata={"stream": "input", "final": True}, + ) + + async def _pump_output(self, session: ElevenLabsSession) -> None: + """Drain the bridge's 16 kHz PCM output and emit 24 kHz ``AUDIO_OUTPUT`` events.""" + try: + while not session.ended.is_set(): + pcm_16k = await session.bridge.get_output_audio(timeout=1.0) + if not pcm_16k or len(pcm_16k) < 4: + continue + session.events.put_nowait( + BackendEvent(event_type=BackendEventType.AUDIO_OUTPUT, audio=_pcm16_16k_to_pcm16_24k(pcm_16k)) + ) + except asyncio.CancelledError: + pass + except Exception as e: # noqa: BLE001 -- background task must not crash the session + logger.error(f"ElevenLabs output pump error: {e}", exc_info=True) + + async def receive(self, session: BackendSession) -> AsyncIterator[BackendEvent]: + """Yield normalized events until the SDK signals end-of-session.""" + s = self._session(session) + while True: + if s.ended.is_set() and s.events.empty(): + return + try: + event = await asyncio.wait_for(s.events.get(), timeout=0.5) + except TimeoutError: + continue + yield event + + async def close(self, session: BackendSession) -> None: + """Tear down the SDK session and background pump. Idempotent.""" + s = self._session(session) + s.ended.set() + if s.output_task is not None: + s.output_task.cancel() + try: + await s.output_task + except asyncio.CancelledError: + pass + s.output_task = None + # Unblock any tool handler still awaiting a result so the SDK task can exit. + for future in s.pending_tools.values(): + if not future.done(): + future.cancel() + s.pending_tools.clear() + if s.conversation is not None: + try: + await s.conversation.end_session() + await s.conversation.wait_for_session_end() + except Exception as e: # noqa: BLE001 + logger.warning(f"Error ending ElevenLabs session: {e}") + finally: + s.conversation = None + if s.client is not None: + try: + s.client = None + except Exception as e: # noqa: BLE001 + logger.debug(f"Error closing ElevenLabs client: {e}") + + @staticmethod + def _session(session: BackendSession) -> ElevenLabsSession: + if not isinstance(session, ElevenLabsSession): + raise TypeError(f"expected ElevenLabsSession, got {type(session).__name__}") + return session diff --git a/src/eva/backend/factory.py b/src/eva/backend/factory.py index d0fa61b2..b8ccce9e 100644 --- a/src/eva/backend/factory.py +++ b/src/eva/backend/factory.py @@ -1,55 +1,69 @@ -"""Factory interface for constructing ``Backend`` instances by name. - -DESIGN ONLY (Step 1 of the refactor). Mirrors the shape of today's -``eva.user_simulator.factory.create_user_simulator`` (lazy per-provider -imports keyed off config type) but is provider-and-role-agnostic: the same -factory is meant to be usable to build a backend for either an -``AssistantRole`` or a ``UserRole``, since a ``Backend`` has no role -knowledge (that's the whole point of the split -- see docs/refactor-step1.md, -"lets any backend act as either role"). +"""Factory that constructs ``Backend`` instances by provider name. + +Mirrors the shape of ``eva.user_simulator.factory.create_user_simulator`` (lazy +per-provider imports keyed off a provider name) but is +provider-and-role-agnostic: the same factory builds a backend for either an +``AssistantRole`` or a ``UserRole``, since a ``Backend`` has no role knowledge +(that's the whole point of the split -- see docs/refactor-step1.md, "lets any +backend act as either role"). + +A single concrete class -- there is no abstract base, since there is only ever +one factory. ``create`` returns ``None`` for a provider that has not been +migrated onto the ``Backend`` contract; the worker uses that as the signal to +fall back to the legacy server/simulator for that provider. """ from __future__ import annotations -from abc import ABC, abstractmethod from typing import Any from eva.backend.base import Backend -class BackendFactory(ABC): +class BackendFactory: """Constructs a ``Backend`` for a named provider from a config blob. - A concrete implementation is expected to hold (or look up) a registry - mapping provider name -> ``Backend`` subclass, analogous to today's - ``create_user_simulator`` / assistant-server construction in - ``orchestrator/runner.py``, and to import each provider module lazily so - that unused providers' SDKs need not be installed/imported. + Providers are added as their backends are migrated onto the ``Backend`` + contract: add a lazy-import branch in ``create``. Each provider's SDK is + imported only when that provider is selected, so unused providers need not + be importable. ``create`` returning ``None`` is the single signal for "not a + native backend" -- there is no parallel list of supported names to maintain. """ - @abstractmethod - def create(self, name: str, config: dict[str, Any]) -> Backend: - """Construct and return a not-yet-opened ``Backend``. + def create(self, name: str, config: dict[str, Any]) -> Backend | None: + """Construct a not-yet-opened ``Backend``, or ``None`` if not yet migrated. Args: name: Provider identifier (e.g. ``"openai_realtime"``, - ``"gemini_live"``, ``"elevenlabs"``, ``"cascade"``). The set - of valid names is defined by the concrete factory's registry, - not by this interface. + ``"gemini_live"``, ``"elevenlabs"``, ``"cascade"``). config: Provider-specific configuration understood by that - backend's ``open()`` (see ``Backend.open``). This factory - does not validate the shape of ``config`` beyond dispatching - on ``name`` -- each ``Backend`` subclass is responsible for - validating its own config. + backend. This factory does not validate the shape of + ``config`` beyond dispatching on ``name`` -- each ``Backend`` + subclass validates its own config and assembles its own + provider session (the caller hand-builds no provider JSON). Returns: - A constructed ``Backend`` instance. The returned backend has not - had ``open()`` called on it yet -- construction and session - establishment are separate steps so a ``Role`` can construct its - backend early (e.g. at record setup) and open the session later - (e.g. once the other party is ready). - - Raises: - ValueError: if ``name`` does not match a known provider. + A constructed ``Backend`` for a migrated provider, not yet + ``open()``ed (construction and session establishment are separate + steps, so a ``Role`` can build its backend early and open the + session later). ``None`` if ``name`` is not a migrated provider -- + the assistant path treats that as an error (unported = unusable); + the (unported) legacy server survives only as reference. """ - ... + if name == "openai_realtime": + from eva.backend.openai_realtime import OpenAIRealtimeBackend + + return OpenAIRealtimeBackend(config=config) + if name == "grok_voice": + from eva.backend.grok_voice import GrokVoiceBackend + + return GrokVoiceBackend(config=config) + if name == "elevenlabs": + from eva.backend.elevenlabs import ElevenLabsBackend + + return ElevenLabsBackend(config=config) + if name == "gemini_live": + from eva.backend.gemini_live import GeminiLiveBackend + + return GeminiLiveBackend(config=config) + return None diff --git a/src/eva/backend/gemini_live.py b/src/eva/backend/gemini_live.py new file mode 100644 index 00000000..d9aefcd5 --- /dev/null +++ b/src/eva/backend/gemini_live.py @@ -0,0 +1,553 @@ +"""Gemini Live ``Backend``: normalizing adapter for one Google Gemini Live session. + +Wraps a single Gemini Live session (google-genai ``client.aio.live``) behind the +role-agnostic ``Backend`` contract (see ``eva.backend.base``), mirroring +``eva.backend.openai_realtime``. All provider-specific work lives here so roles +stay generic: + +- session lifecycle: connect / stream audio in / events out / tool results / close; +- ``LiveConnectConfig`` assembly (voice, VAD, transcription, language) from the + worker-supplied flat config -- roles never build provider config; +- tool-schema translation: generic ``{name, description, parameters}`` specs -> + Gemini ``types.Tool`` / ``FunctionDeclaration``; +- **full event normalization**: every ``LiveServerMessage`` is parsed here and + surfaced as a clean ``BackendEvent``. Per-turn bookkeeping (output-transcript + accumulation, tool-call-name lookup for responses, token usage) lives on the + ``GeminiLiveSession`` handle so the backend object stays stateless. + +Config: shares the assistant backends' core keys -- ``model`` (required), +``speaker_id`` (a Gemini voice name here, default ``"Kore"``). ``api_key`` is +OPTIONAL for Gemini (unlike the OpenAI family): absent, it falls back to +``GOOGLE_API_KEY``, then to Vertex AI (``GOOGLE_CLOUD_PROJECT`` / +``GOOGLE_CLOUD_LOCATION``), then to google-genai default credential resolution +(ADC) -- so it does not use the base ``_resolve_api_key`` (which requires a key). +``language`` (or ``language_code``) sets speech-synthesis language; the worker +passes the run language through ``backend_args``. + +Scope note (docs/refactor-backend-migration.md): this backend faithfully ports +the **assistant** path from ``eva.assistant.gemini_live_server`` (automatic-VAD +turn-taking). Two assistant-side deltas vs the OpenAI backend, both benign: +user-input transcripts carry no speech-start timestamp (Gemini surfaces no input +speech-boundary event, so ``TRANSCRIPT`` stream=input has ts=None downstream), +and 24 kHz role input is resampled to Gemini's 16 kHz (mulaw for a pcmu caller). +""" + +from __future__ import annotations + +import audioop +import os +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any, ClassVar + +from google import genai +from google.genai import types + +from eva.backend.base import ( + Backend, + BackendEvent, + BackendEventType, + BackendSession, + ToolCallRequest, + ToolCallResult, +) +from eva.backend.capabilities import BackendCapabilities +from eva.utils.audio_utils import mulaw_8k_to_pcm16_16k +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +DEFAULT_SAMPLE_RATE = 24000 +PCMU_SAMPLE_RATE = 8000 +GEMINI_INPUT_SAMPLE_RATE = 16000 # Gemini Live expects 16 kHz PCM input +DEFAULT_VOICE = "Kore" + + +@dataclass +class GeminiLiveSession(BackendSession): + """Live state for one Gemini Live session. + + Carries the genai client, the entered live-connection context manager + the + live session, and the per-turn accumulators the normalizer needs (whether a + model turn is in flight, accumulated output-transcript text, tool-call name + lookup for ``send_tool_response``, latest usage, input resampler state). All + session state, deliberately off the stateless backend object. + """ + + client: genai.Client + conn_cm: Any + live: Any + in_model_turn: bool = False + output_transcript_parts: list[str] = field(default_factory=list) + has_function_calls: bool = False + tool_names: dict[str, str] = field(default_factory=dict) + usage: dict[str, int] | None = None + resampler_state: Any = None + + +class GeminiLiveBackend(Backend): + """One Gemini Live session behind the role-agnostic ``Backend`` contract. + + Construction is cheap and network-free (client + connection are created in + ``open()``). Takes a single flat ``config`` and assembles the + ``LiveConnectConfig`` itself -- the caller never hand-builds provider JSON. + Recognized keys (see the module docstring for auth/language details): + + - ``model`` (required). ``speaker_id`` (default ``"Kore"``): the Gemini voice. + - ``api_key`` (optional): else ``GOOGLE_API_KEY`` env, else Vertex AI / ADC. + - ``language`` / ``language_code``: speech-synthesis language. + - ``output_sample_rate`` (default 24000). ``input_format``: ``"pcm"`` + (default; role sends ``output_sample_rate`` PCM16, resampled to 16 kHz) or + ``"pcmu"`` (telephony mulaw, converted to 16 kHz). + - ``vad_settings``: turn-detection tunables (``silence_duration_ms``). + - ``accent``: rejected if set (realized via ElevenLabs agent IDs). + + Vertex-only extras (for Vertex-preview Live models, e.g. ``*-live-preview``): + - ``project`` / ``location``: Vertex project + region (else ``GOOGLE_CLOUD_*`` + / ``VERTEXAI_*`` env). A resolvable project (or ``GOOGLE_GENAI_USE_VERTEXAI``) + routes through Vertex and ignores ``api_key``; ``"global"`` is forced to a + region since Live requires a regional endpoint. + - ``endpoint`` / ``api_version``: optional Vertex endpoint / API-version overrides. + - ``function_response_scheduling``: ``"WHEN_IDLE"`` | ``"INTERRUPT"`` | + ``"SILENT"``. Omitted by default (newer Live models reject it with 1007). + - ``thinking_config``: dict with optional ``thinking_budget`` (int), + ``include_thoughts`` (bool), ``thinking_level`` (str). + """ + + # api_key is optional (Vertex/ADC fallback), so it is looked up inline rather + # than via the base _resolve_api_key (which requires a key). + _API_KEY_ENV: ClassVar[str] = "GOOGLE_API_KEY" + + _CAPABILITIES = BackendCapabilities( + emits_continuous_audio=True, + supports_streaming_interruption=True, + owns_playout_clock=False, + ) + + def __init__(self, *, config: dict[str, Any]) -> None: + errors: list[str] = [] + self._model = self._require(config, "model", errors) + if config.get("accent") is not None: + errors.append("accent variants are not supported (accents are realized via ElevenLabs agents)") + self._raise_config_errors(errors) + + self._api_key = config.get("api_key") or os.environ.get(self._API_KEY_ENV) or "" + self._voice = config.get("speaker_id", DEFAULT_VOICE) + self._language_code = config.get("language_code") or config.get("language") + self._input_format: str = config.get("input_format", "pcm") + self._output_sample_rate = int(config.get("output_sample_rate", DEFAULT_SAMPLE_RATE)) + vad = config.get("vad_settings") or {} + self._silence_duration_ms = int(vad.get("silence_duration_ms", 200)) + + # Optional Vertex endpoint / API-version overrides; when unset the SDK + # uses its defaults. Live/S2S preview models are Vertex-only and may + # require a specific api_version (e.g. "v1beta1"). + self._endpoint = config.get("endpoint") + self._api_version = config.get("api_version") + + # Optional FunctionResponse scheduling ("WHEN_IDLE" | "INTERRUPT" | + # "SILENT"). Newer Live models (e.g. gemini-3.5-flash-live-preview) do + # NOT support a scheduling field and close the socket with 1007 if one is + # set, so it is OMITTED by default; older models can opt back in. + self._fc_scheduling = config.get("function_response_scheduling") + + # Vertex project/location resolution. Accept both the google-genai names + # (GOOGLE_CLOUD_*) and the LiteLLM/Vertex names (VERTEXAI_*) so a single + # set of credentials works for both the judge and the Live server. + self._vertex_project = ( + config.get("project") or os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("VERTEXAI_PROJECT") + ) + # Gemini Live/S2S requires a REGIONAL endpoint; "global" (fine for the + # text judge) is not supported for bidiGenerateContent, so it is never + # used here — an explicit region wins, otherwise fall back to a region. + location = ( + config.get("location") or os.environ.get("GOOGLE_CLOUD_LOCATION") or os.environ.get("VERTEXAI_LOCATION") + ) + if not location or location == "global": + if location == "global": + logger.warning( + "Gemini Live does not support location='global'; using 'us-central1' instead. " + "Set s2s_params['location'] or GOOGLE_CLOUD_LOCATION to the region the model is enabled in." + ) + location = "us-central1" + self._vertex_location = location + + # Thinking config: controls Gemini's internal reasoning budget. Accepts a + # dict with optional keys "thinking_budget" (int), "include_thoughts" + # (bool), "thinking_level" (str). Unset -> model-dependent defaults. + self._thinking_config = self._build_thinking_config(config.get("thinking_config", {})) + + @property + def capabilities(self) -> BackendCapabilities: + return self._CAPABILITIES + + @property + def output_sample_rate(self) -> int: + """Sample rate (Hz) of ``AUDIO_OUTPUT`` payloads (Gemini outputs 24 kHz).""" + return self._output_sample_rate + + @property + def input_sample_rate(self) -> int: + """Sample rate (Hz) the role sends via ``send(audio=...)`` (converted to 16 kHz internally).""" + return PCMU_SAMPLE_RATE if self._input_format == "pcmu" else self._output_sample_rate + + # ── Client / config assembly ────────────────────────────────────── + + @staticmethod + def _build_thinking_config(thinking_raw: Any) -> types.ThinkingConfig: + """Build a ``ThinkingConfig`` from the optional flat ``thinking_config`` dict.""" + if isinstance(thinking_raw, dict) and thinking_raw: + tc_kwargs: dict[str, Any] = {} + if "thinking_budget" in thinking_raw: + tc_kwargs["thinking_budget"] = int(thinking_raw["thinking_budget"]) + if "include_thoughts" in thinking_raw: + tc_kwargs["include_thoughts"] = bool(thinking_raw["include_thoughts"]) + if "thinking_level" in thinking_raw: + tc_kwargs["thinking_level"] = thinking_raw["thinking_level"] + logger.info(f"Thinking config: {tc_kwargs}") + return types.ThinkingConfig(**tc_kwargs) + return types.ThinkingConfig() + + def _create_client(self) -> genai.Client: + """Create a google-genai Client for Vertex AI or the Developer API. + + Vertex-only models (e.g. gemini-*-live-preview) must route through + aiplatform.googleapis.com with ``vertexai=True``; a Developer API key + (AIza…) would send them to generativelanguage.googleapis.com/v1beta, + where they 404. We therefore prefer Vertex whenever a project is + resolvable (or GOOGLE_GENAI_USE_VERTEXAI is set) and ignore any + Developer API key in that mode. Otherwise fall back to the Developer API + key, then to google-genai default credential resolution (ADC). + """ + flag = os.environ.get("GOOGLE_GENAI_USE_VERTEXAI") + if flag is not None: + use_vertex = flag.strip().lower() in ("1", "true", "yes") + else: + use_vertex = bool(self._vertex_project) + + if use_vertex: + if not self._vertex_project: + raise ValueError( + "Vertex mode requested but no project found. Set GOOGLE_CLOUD_PROJECT / " + "VERTEXAI_PROJECT or s2s_params['project']." + ) + http_kwargs: dict[str, Any] = {} + if self._endpoint: + http_kwargs["base_url"] = f"wss://{self._endpoint}" + if self._api_version: + http_kwargs["api_version"] = self._api_version + http_options = types.HttpOptions(**http_kwargs) if http_kwargs else None + + if self._api_key: + logger.warning( + "Ignoring api_key in Vertex mode (Vertex uses ADC / service-account " + "credentials via GOOGLE_APPLICATION_CREDENTIALS)." + ) + logger.info( + f"Using Vertex AI (project={self._vertex_project}, location={self._vertex_location}, " + f"api_version={self._api_version or 'sdk-default'})" + ) + return genai.Client( + vertexai=True, + project=self._vertex_project, + location=self._vertex_location, + http_options=http_options, + ) + + if self._api_key: + logger.info("Using Gemini Developer API key for authentication") + return genai.Client(api_key=self._api_key) + logger.warning("No explicit Gemini credentials; relying on google-genai default resolution") + return genai.Client() + + def _build_live_config(self, system_prompt: str, tools: list[dict[str, Any]] | None) -> types.LiveConnectConfig: + """Build the ``LiveConnectConfig`` for the session from flat config + role prompt/tools.""" + config_kwargs: dict[str, Any] = { + "response_modalities": [types.Modality.AUDIO], + "system_instruction": system_prompt, + "speech_config": types.SpeechConfig( + voice_config=types.VoiceConfig(prebuilt_voice_config=types.PrebuiltVoiceConfig(voice_name=self._voice)), + language_code=self._language_code, + ), + "realtime_input_config": types.RealtimeInputConfig( + automatic_activity_detection=types.AutomaticActivityDetection( + disabled=False, + start_of_speech_sensitivity=types.StartSensitivity.START_SENSITIVITY_LOW, + end_of_speech_sensitivity=types.EndSensitivity.END_SENSITIVITY_LOW, + silence_duration_ms=self._silence_duration_ms, + ), + activity_handling=types.ActivityHandling.START_OF_ACTIVITY_INTERRUPTS, + ), + "input_audio_transcription": types.AudioTranscriptionConfig(), + "output_audio_transcription": types.AudioTranscriptionConfig(), + "thinking_config": self._thinking_config, + } + gemini_tools = self._format_tools(tools) + if gemini_tools: + config_kwargs["tools"] = gemini_tools + return types.LiveConnectConfig(**config_kwargs) + + @staticmethod + def _json_schema_type(python_type: str) -> str: + """Map Python/EVA type names to Gemini Schema type strings.""" + mapping = { + "string": "STRING", + "str": "STRING", + "integer": "INTEGER", + "int": "INTEGER", + "number": "NUMBER", + "float": "NUMBER", + "boolean": "BOOLEAN", + "bool": "BOOLEAN", + "array": "ARRAY", + "list": "ARRAY", + "object": "OBJECT", + "dict": "OBJECT", + } + return mapping.get(python_type.lower(), "STRING") + + @classmethod + def _convert_schema_properties(cls, props: dict[str, Any]) -> dict[str, types.Schema]: + """Recursively convert JSON-Schema property dicts to Gemini ``Schema`` objects.""" + result: dict[str, types.Schema] = {} + for name, defn in props.items(): + if not isinstance(defn, dict): + result[name] = types.Schema(type=types.Type.STRING) + continue + schema_type = cls._json_schema_type(defn.get("type", "string")) + kwargs: dict[str, Any] = {"type": types.Type(schema_type)} + if "description" in defn: + kwargs["description"] = defn["description"] + if "enum" in defn: + kwargs["enum"] = defn["enum"] + if schema_type == "OBJECT" and "properties" in defn: + kwargs["properties"] = cls._convert_schema_properties(defn["properties"]) + if schema_type == "ARRAY" and "items" in defn: + items = defn["items"] + if isinstance(items, dict): + item_kwargs: dict[str, Any] = { + "type": types.Type(cls._json_schema_type(items.get("type", "string"))) + } + if "properties" in items: + item_kwargs["properties"] = cls._convert_schema_properties(items["properties"]) + kwargs["items"] = types.Schema(**item_kwargs) + else: + kwargs["items"] = types.Schema(type=types.Type.STRING) + result[name] = types.Schema(**kwargs) + return result + + @classmethod + def _format_tools(cls, tools: list[dict[str, Any]] | None) -> list[types.Tool] | None: + """Translate generic ``{name, description, parameters}`` specs to Gemini tools.""" + declarations: list[types.FunctionDeclaration] = [] + for tool in tools or []: + params = tool.get("parameters") or {} + properties = cls._convert_schema_properties(params.get("properties", {})) + required = params.get("required") or None + declarations.append( + types.FunctionDeclaration( + name=tool["name"], + description=tool["description"], + parameters=types.Schema(type=types.Type.OBJECT, properties=properties, required=required), + behavior=types.Behavior.BLOCKING, + ) + ) + if not declarations: + return None + return [types.Tool(function_declarations=declarations)] + + # ── Session lifecycle ───────────────────────────────────────────── + + async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None) -> GeminiLiveSession: + """Connect and configure a new Gemini Live session; return its handle.""" + client = self._create_client() + live_config = self._build_live_config(system_prompt, tools) + conn_cm = client.aio.live.connect(model=self._model, config=live_config) + live = await conn_cm.__aenter__() + logger.info(f"Gemini Live session opened (model={self._model})") + return GeminiLiveSession(client=client, conn_cm=conn_cm, live=live) + + async def send( + self, + session: BackendSession, + *, + audio: bytes | None = None, + text: str | None = None, + tool_result: ToolCallResult | None = None, + ) -> None: + """Push audio / a text turn / a tool result to ``session`` (exactly one).""" + provided = [x is not None for x in (audio, text, tool_result)] + if sum(provided) != 1: + raise ValueError("send() requires exactly one of audio, text, tool_result") + s = self._session(session) + + if audio is not None: + pcm_16k = self._to_gemini_input(s, audio) + await s.live.send_realtime_input(audio=types.Blob(data=pcm_16k, mime_type="audio/pcm;rate=16000")) + return + + if text is not None: + await s.live.send_realtime_input(text=text) + return + + assert tool_result is not None # exactly-one check above + # Only set the scheduling field when explicitly configured — newer Live + # models reject it and close the socket with 1007. + fr_kwargs: dict[str, Any] = { + "id": tool_result.call_id, + "name": s.tool_names.get(tool_result.call_id), + "response": tool_result.result, + } + if self._fc_scheduling: + fr_kwargs["scheduling"] = types.FunctionResponseScheduling[self._fc_scheduling] + await s.live.send_tool_response(function_responses=[types.FunctionResponse(**fr_kwargs)]) + + def _to_gemini_input(self, session: GeminiLiveSession, audio: bytes) -> bytes: + """Convert role-supplied input audio to Gemini's 16 kHz PCM16.""" + if self._input_format == "pcmu": + return mulaw_8k_to_pcm16_16k(audio) + pcm_16k, session.resampler_state = audioop.ratecv( + audio, 2, 1, self._output_sample_rate, GEMINI_INPUT_SAMPLE_RATE, session.resampler_state + ) + return pcm_16k + + async def receive(self, session: BackendSession) -> AsyncIterator[BackendEvent]: + """Yield normalized events from ``session``'s connection until it ends. + + Uses the live session's manual receive (``_receive``) rather than the + public ``receive()`` iterator, which returns after ``turn_complete`` and + would close the session between model turns (mirrors the old server). + """ + s = self._session(session) + while True: + try: + response = await s.live._receive() + except Exception as e: + logger.debug(f"Gemini Live receive ended: {e}") + return + if response is None: + continue + for be in self._map_response(s, response): + yield be + + @staticmethod + def _session(session: BackendSession) -> GeminiLiveSession: + if not isinstance(session, GeminiLiveSession): + raise TypeError(f"expected GeminiLiveSession, got {type(session).__name__}") + return session + + def _map_response(self, session: GeminiLiveSession, response: Any) -> list[BackendEvent]: + """Normalize one Gemini ``LiveServerMessage`` into zero or more clean ``BackendEvent``s. + + Stateful (accumulates output transcript / turn flags / usage on + ``session``). Pure of I/O, so unit-testable with a fake session + message. + """ + out: list[BackendEvent] = [] + sc = getattr(response, "server_content", None) + + if sc is not None: + model_turn = getattr(sc, "model_turn", None) + if model_turn: + if not session.in_model_turn: + session.in_model_turn = True + session.output_transcript_parts = [] + out.append(BackendEvent(event_type=BackendEventType.OUTPUT_TURN_STARTED)) + for part in getattr(model_turn, "parts", None) or []: + inline = getattr(part, "inline_data", None) + data = getattr(inline, "data", None) if inline is not None else None + if data and len(data) >= 6: + out.append(BackendEvent(event_type=BackendEventType.AUDIO_OUTPUT, audio=bytes(data))) + + input_tx = getattr(sc, "input_transcription", None) + if input_tx is not None: + text = (getattr(input_tx, "text", "") or "").strip() + if text: + out.append( + BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript=text, + metadata={"stream": "input", "final": True}, + ) + ) + + output_tx = getattr(sc, "output_transcription", None) + if output_tx is not None: + chunk = (getattr(output_tx, "text", "") or "").strip() + if chunk: + session.output_transcript_parts.append(chunk) + + if getattr(sc, "interrupted", False): + out.append(self._turn_end_event(session, interrupted=True)) + self._reset_turn(session) + elif getattr(sc, "turn_complete", False): + partial = " ".join(session.output_transcript_parts).strip() + if partial: + out.append( + BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript=partial, + metadata={"stream": "output", "final": True}, + ) + ) + out.append(BackendEvent(event_type=BackendEventType.OUTPUT_AUDIO_DONE)) + out.append(self._turn_end_event(session, interrupted=False)) + self._reset_turn(session) + + tool_call = getattr(response, "tool_call", None) + if tool_call is not None: + for fc in getattr(tool_call, "function_calls", None) or []: + session.has_function_calls = True + call_id = getattr(fc, "id", "") or "" + name = getattr(fc, "name", "") or "" + session.tool_names[call_id] = name + out.append( + BackendEvent( + event_type=BackendEventType.TOOL_CALL_REQUEST, + tool_call_request=ToolCallRequest( + call_id=call_id, + name=name, + arguments=dict(fc.args) if getattr(fc, "args", None) else {}, + ), + ) + ) + + usage_metadata = getattr(response, "usage_metadata", None) + if usage_metadata is not None: + prompt_tokens = getattr(usage_metadata, "prompt_token_count", 0) or 0 + completion_tokens = getattr(usage_metadata, "candidates_token_count", 0) or 0 + if prompt_tokens or completion_tokens: + session.usage = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens} + + return out + + @staticmethod + def _turn_end_event(session: GeminiLiveSession, *, interrupted: bool) -> BackendEvent: + return BackendEvent( + event_type=BackendEventType.TURN_END, + transcript=" ".join(session.output_transcript_parts).strip(), + metadata={ + "interrupted": interrupted, + "cancelled": False, + "has_function_calls": session.has_function_calls, + "usage": session.usage, + }, + ) + + @staticmethod + def _reset_turn(session: GeminiLiveSession) -> None: + session.in_model_turn = False + session.output_transcript_parts = [] + session.has_function_calls = False + session.usage = None + + async def close(self, session: BackendSession) -> None: + """Tear down ``session``. Idempotent: safe to call more than once.""" + s = self._session(session) + if s.conn_cm is not None: + try: + await s.conn_cm.__aexit__(None, None, None) + except Exception as e: + logger.debug(f"Error closing Gemini Live connection: {e}") + finally: + s.conn_cm = None + s.live = None diff --git a/src/eva/backend/grok_voice.py b/src/eva/backend/grok_voice.py new file mode 100644 index 00000000..56f684eb --- /dev/null +++ b/src/eva/backend/grok_voice.py @@ -0,0 +1,105 @@ +"""Grok Voice ``Backend``: xAI's voice realtime API (OpenAI Realtime-compatible). + +xAI's voice realtime API is event-compatible with OpenAI's Realtime API +(https://docs.x.ai/developers/model-capabilities/audio/voice-agent), so this +backend subclasses ``OpenAIRealtimeBackend`` and overrides only what differs -- +mirroring how ``eva.assistant.grok_voice_server.GrokVoiceAssistantServer`` +subclasses the OpenAI Realtime server: + +- endpoint: point the client at ``https://api.x.ai/v1`` (default ``base_url``); +- default voice: xAI's built-in voices (``eve``/``ara``/``rex``/``sal``/``leo``); +- api key: falls back to ``XAI_API_KEY`` (not ``OPENAI_API_KEY``, which would be + the wrong key for x.ai) when not supplied in config; +- input transcription: xAI fires + ``conversation.item.input_audio_transcription.completed`` multiple times per + turn, each carrying the *cumulative* text so far rather than a delta. Only the + latest cumulative is buffered on the session and emitted as ONE input + ``TRANSCRIPT`` at the turn boundary (next ``speech_started`` / ``response.done``), + matching the legacy ``grok_voice_server``'s deferred flush -- so the role logs a + single user turn instead of one per fragment. + +Everything else -- session assembly, audio, tool round-trip, interruption, +usage, the rest of the event normalization -- is inherited unchanged. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, ClassVar + +from eva.backend.base import BackendEvent, BackendEventType +from eva.backend.openai_realtime import OpenAIRealtimeBackend, OpenAIRealtimeSession +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +XAI_REALTIME_BASE_URL = "https://api.x.ai/v1" +DEFAULT_VOICE = "eve" + + +@dataclass +class GrokVoiceSession(OpenAIRealtimeSession): + """OpenAI Realtime session state plus xAI's buffered cumulative input transcript. + + xAI streams ``input_audio_transcription.completed`` repeatedly, each carrying the + *cumulative* text so far (it does NOT emit incremental ``.delta`` events like + OpenAI). Emitting each ``completed`` directly makes the role log one user turn per + fragment. Instead ``pending_input_transcript`` holds the latest cumulative text, + which is flushed as a single input ``TRANSCRIPT`` at the turn boundary. + """ + + pending_input_transcript: str = "" + + +class GrokVoiceBackend(OpenAIRealtimeBackend): + """xAI Grok voice realtime behind the role-agnostic ``Backend`` contract.""" + + _SESSION_CLS: ClassVar[type[OpenAIRealtimeSession]] = GrokVoiceSession + _API_KEY_ENV: ClassVar[str] = "XAI_API_KEY" + + # Turn boundaries at which the buffered cumulative input transcript is flushed + # (mirrors the legacy server flushing on _on_speech_started / _on_response_done). + _FLUSH_ON: ClassVar[tuple[str, ...]] = ("input_audio_buffer.speech_started", "response.done") + + def __init__(self, *, config: dict[str, Any]) -> None: + # base_url / speaker_id are xAI defaults any explicit config overrides; api_key + # falls back to XAI_API_KEY in the parent (via _API_KEY_ENV). + merged = {"base_url": XAI_REALTIME_BASE_URL, "speaker_id": DEFAULT_VOICE, **config} + super().__init__(config=merged) + + @staticmethod + def _map_event(session: OpenAIRealtimeSession, event: Any) -> list[BackendEvent]: + """Normalize one xAI event, buffering its cumulative input transcript. + + ``input_audio_transcription.completed`` is cumulative (xAI re-sends the whole + text, no ``.delta`` events), so it is buffered rather than emitted; the buffer + is flushed as one input ``TRANSCRIPT`` just before the parent's events at a turn + boundary (``_FLUSH_ON``). Everything else defers to ``OpenAIRealtimeBackend``. + """ + etype = getattr(event, "type", "") + if etype == "conversation.item.input_audio_transcription.completed": + text = (getattr(event, "transcript", "") or "").strip() + if isinstance(session, GrokVoiceSession) and text: + session.pending_input_transcript = text + return [] + + events: list[BackendEvent] = [] + if etype in GrokVoiceBackend._FLUSH_ON: + events.extend(GrokVoiceBackend._flush_input_transcript(session)) + events.extend(OpenAIRealtimeBackend._map_event(session, event)) + return events + + @staticmethod + def _flush_input_transcript(session: OpenAIRealtimeSession) -> list[BackendEvent]: + """Emit the buffered cumulative input transcript once, then clear it.""" + if not isinstance(session, GrokVoiceSession) or not session.pending_input_transcript: + return [] + text = session.pending_input_transcript + session.pending_input_transcript = "" + return [ + BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript=text, + metadata={"stream": "input", "final": True}, + ) + ] diff --git a/src/eva/backend/openai_realtime.py b/src/eva/backend/openai_realtime.py new file mode 100644 index 00000000..07ca0b45 --- /dev/null +++ b/src/eva/backend/openai_realtime.py @@ -0,0 +1,475 @@ +"""OpenAI Realtime ``Backend``: normalizing adapter for one provider session. + +Wraps a single OpenAI Realtime API session behind the role-agnostic ``Backend`` +contract (see ``eva.backend.base``). It knows nothing about whether it drives +an ``AssistantRole`` or a ``UserRole``; both use the *same* backend and differ +only in the ``session_config`` the worker constructs it with and in how they +interpret the clean events it emits. + +Responsibilities (all provider-specific work lives here, so roles stay +generic): +- session lifecycle: connect / ``session.update`` / stream audio in / events + out / tool results / close; +- session-config assembly (voice, VAD, formats, transcription) from the + worker-supplied ``session_config`` -- roles never build provider config; +- tool-schema translation: generic ``{name, description, parameters}`` specs + -> OpenAI Realtime ``session.tools`` shape; +- **full event normalization**: every provider event is parsed here and + surfaced as a clean ``BackendEvent`` (typed fields + normalized ``metadata`` + scalars). Roles never see a raw OpenAI event. Provider-specific bookkeeping + (output-transcript accumulation, final-text selection, interruption + detection, token-usage extraction) happens here, with per-turn state carried + on the ``OpenAIRealtimeSession`` handle (the backend object stays stateless). + +Not covered here (role concerns): the transport to the counterparty (the +Twilio WS server / audio-bridge client), audio format conversion for that +transport, recording, prompt building, and tool execution. +""" + +from __future__ import annotations + +import base64 +import json +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any, ClassVar + +from openai import AsyncOpenAI + +from eva.backend.base import ( + Backend, + BackendEvent, + BackendEventType, + BackendSession, + ToolCallRequest, + ToolCallResult, +) +from eva.backend.capabilities import BackendCapabilities +from eva.utils.logging import get_logger + +logger = get_logger(__name__) + +DEFAULT_SAMPLE_RATE = 24000 +PCMU_SAMPLE_RATE = 8000 + + +@dataclass +class OpenAIRealtimeSession(BackendSession): + """Live state for one OpenAI Realtime session. + + Carries the SDK client, the entered realtime-connection context manager + + connection, and the per-turn accumulators the normalizer needs (output + transcript parts, whether a response is in flight, whether it produced tool + calls). All of this is session state, deliberately off the backend object. + """ + + client: AsyncOpenAI + conn_cm: Any + conn: Any + responding: bool = False + output_transcript_parts: list[str] = field(default_factory=list) + output_transcript_done: str = "" + has_function_calls: bool = False + + +class OpenAIRealtimeBackend(Backend): + """One OpenAI Realtime session behind the role-agnostic ``Backend`` contract. + + Construction is cheap and network-free (client + connection are created in + ``open()``), matching ``BackendFactory.create``'s "not-yet-opened" contract. + + Takes a single flat ``config`` of "config things" and assembles the + OpenAI ``session.update`` structure itself -- the caller (worker via the + factory) never hand-builds the provider JSON. Recognized keys: + + - ``model`` (required). ``api_key`` (optional): falls back to the + ``OPENAI_API_KEY`` env var if not provided. ``base_url`` (optional). + - ``accent``: if set, rejected -- this backend can't honor accents (they + are realized via ElevenLabs agent IDs). Fails loud, mirroring the old + ``OpenAIRealtimeUserSimulator`` guard, now backend-side. + - ``speaker_id`` (default ``"marin"``): the shared speaker-identifier key (an + OpenAI voice name here), mapped onto the provider's ``voice`` field. + - ``output_sample_rate`` (default 24000). + - ``input_format``: ``"pcm"`` (default) or ``"pcmu"``. + - ``vad_settings``: turn-detection tunables (``type`` / ``threshold`` / + ``prefix_padding_ms`` / ``silence_duration_ms``), defaults applied. Named + to match EVA's ``s2s_params["vad_settings"]`` so an assistant can pass its + provider params straight through. + - ``transcription_model`` (default ``"whisper-1"``), + ``transcription_language`` (optional). + - ``reasoning_effort`` (optional), ``parallel_tool_calls`` (optional). + """ + + _CAPABILITIES = BackendCapabilities( + emits_continuous_audio=True, supports_streaming_interruption=True, owns_playout_clock=False + ) + + # Session handle class ``open()`` instantiates. Subclasses for API-compatible + # providers (e.g. Grok Voice) override this to carry extra per-turn state. + _SESSION_CLS: ClassVar[type[OpenAIRealtimeSession]] = OpenAIRealtimeSession + + # Env var the api_key falls back to when not supplied in config. Subclasses + # for other OpenAI-compatible providers override it (e.g. Grok -> XAI_API_KEY). + _API_KEY_ENV: ClassVar[str] = "OPENAI_API_KEY" + + def __init__(self, *, config: dict[str, Any]) -> None: + errors: list[str] = [] + self._api_key = self._resolve_api_key(config, errors) + if config.get("accent") is not None: + errors.append("accent variants are not supported (accents are realized via ElevenLabs agents)") + self._model: str = self._require(config, "model", errors) + self._raise_config_errors(errors) + self._base_url = config.get("base_url") + self._input_format: str = config.get("input_format", "pcm") + self._output_sample_rate = int(config.get("output_sample_rate", DEFAULT_SAMPLE_RATE)) + self._session_config = self._assemble_session_config(config) + + @property + def capabilities(self) -> BackendCapabilities: + return self._CAPABILITIES + + @property + def output_sample_rate(self) -> int: + """Sample rate (Hz) of ``AUDIO_OUTPUT`` payloads.""" + return self._output_sample_rate + + @property + def input_sample_rate(self) -> int: + """Sample rate (Hz) the session expects for ``send(audio=...)`` input.""" + return PCMU_SAMPLE_RATE if self._input_format == "pcmu" else self._output_sample_rate + + def _assemble_session_config(self, config: dict[str, Any]) -> dict[str, Any]: + """Build the OpenAI session-shaping block (minus type/instructions/tools) from flat config.""" + input_fmt: dict[str, Any] = ( + {"type": "audio/pcmu"} + if self._input_format == "pcmu" + else {"type": "audio/pcm", "rate": self._output_sample_rate} + ) + vad = config.get("vad_settings") or {} + turn_detection = { + "type": vad.get("type", "server_vad"), + "threshold": vad.get("threshold", 0.5), + "prefix_padding_ms": vad.get("prefix_padding_ms", 300), + "silence_duration_ms": vad.get("silence_duration_ms", 200), + } + transcription: dict[str, Any] = {"model": config.get("transcription_model", "whisper-1")} + if config.get("transcription_language"): + transcription["language"] = config["transcription_language"] + + session_config: dict[str, Any] = { + "output_modalities": ["audio"], + "audio": { + "output": { + "voice": config.get("speaker_id", "marin"), + "format": {"type": "audio/pcm", "rate": self._output_sample_rate}, + }, + "input": {"format": input_fmt, "turn_detection": turn_detection, "transcription": transcription}, + }, + } + if config.get("reasoning_effort"): + session_config["reasoning"] = {"effort": config["reasoning_effort"]} + if config.get("parallel_tool_calls") is not None: + session_config["parallel_tool_calls"] = config["parallel_tool_calls"] + return session_config + + # ── Session lifecycle ───────────────────────────────────────────── + + async def open(self, *, system_prompt: str, tools: list[dict[str, Any]] | None) -> OpenAIRealtimeSession: + """Connect and configure a new OpenAI Realtime session; return its handle.""" + client_kwargs: dict[str, Any] = {"api_key": self._api_key} + if self._base_url is not None: + client_kwargs["base_url"] = self._base_url + client = AsyncOpenAI(**client_kwargs) + + conn_cm = client.realtime.connect(model=self._model) + conn = await conn_cm.__aenter__() + + session_update = self._build_session_update(system_prompt, tools) + await conn.session.update(session=session_update) # type: ignore[arg-type] + logger.info(f"OpenAI Realtime session opened (model={self._model})") + return self._SESSION_CLS(client=client, conn_cm=conn_cm, conn=conn) + + def _build_session_update(self, system_prompt: str, tools: list[dict[str, Any]] | None) -> dict[str, Any]: + """Finalize the ``session.update`` payload for ``open()``. + + Takes the session-shaping block assembled at construction and stamps the + per-open fields the backend owns (``type`` / ``instructions`` / + ``tools``), translating generic tool specs to the provider shape. + """ + session_update: dict[str, Any] = dict(self._session_config) + session_update["type"] = "realtime" + session_update["instructions"] = system_prompt + session_update["tools"] = self._format_tools(tools) + return session_update + + @staticmethod + def _format_tools(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + """Translate generic ``{name, description, parameters}`` specs to OpenAI schema.""" + return [ + { + "type": "function", + "name": tool["name"], + "description": tool["description"], + "parameters": tool["parameters"], + } + for tool in (tools or []) + ] + + async def send( + self, + session: BackendSession, + *, + audio: bytes | None = None, + text: str | None = None, + tool_result: ToolCallResult | None = None, + ) -> None: + """Push audio / a text turn / a tool result to ``session`` (exactly one).""" + provided = [x is not None for x in (audio, text, tool_result)] + if sum(provided) != 1: + raise ValueError("send() requires exactly one of audio, text, tool_result") + conn = self._conn(session) + + if audio is not None: + await conn.input_audio_buffer.append(audio=base64.b64encode(audio).decode("ascii")) + return + + if text is not None: + await conn.conversation.item.create( + item={"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]} + ) + await conn.response.create() + return + + assert tool_result is not None # exactly-one check above + await conn.conversation.item.create( + item={ + "type": "function_call_output", + "call_id": tool_result.call_id, + "output": json.dumps(tool_result.result, ensure_ascii=False), + } + ) + await conn.response.create() + + async def receive(self, session: BackendSession) -> AsyncIterator[BackendEvent]: + """Yield normalized events from ``session``'s connection until it ends.""" + s = self._session(session) + async for event in s.conn: + for be in self._map_event(s, event): + yield be + + @staticmethod + def _session(session: BackendSession) -> OpenAIRealtimeSession: + if not isinstance(session, OpenAIRealtimeSession): + raise TypeError(f"expected OpenAIRealtimeSession, got {type(session).__name__}") + return session + + @classmethod + def _conn(cls, session: BackendSession) -> Any: + return cls._session(session).conn + + @staticmethod + def _map_event(session: OpenAIRealtimeSession, event: Any) -> list[BackendEvent]: + """Normalize one provider event into zero or more clean ``BackendEvent``s. + + Stateful (accumulates output transcript / turn flags on ``session``) so + it can surface a fully-selected final transcript and detect + interruption without the role touching raw provider data. Pure of I/O, + so unit-testable with a fake session + event. + """ + event_type = getattr(event, "type", "") + out: list[BackendEvent] = [] + + match event_type: + case "response.created": + session.responding = True + session.output_transcript_parts = [] + session.output_transcript_done = "" + session.has_function_calls = False + out.append(BackendEvent(event_type=BackendEventType.OUTPUT_TURN_STARTED)) + + case "response.output_audio.delta": + delta_b64 = getattr(event, "delta", "") or "" + if delta_b64: + out.append( + BackendEvent(event_type=BackendEventType.AUDIO_OUTPUT, audio=base64.b64decode(delta_b64)) + ) + + case "response.output_audio_transcript.delta": + session.output_transcript_parts.append(getattr(event, "delta", "") or "") + + case "response.output_audio_transcript.done": + done = (getattr(event, "transcript", "") or "").strip() + session.output_transcript_done = done + text = done or "".join(session.output_transcript_parts).strip() + out.append( + BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript=text, + metadata={"stream": "output", "final": True}, + ) + ) + + case "conversation.item.input_audio_transcription.completed": + transcript = (getattr(event, "transcript", "") or "").strip() + if transcript: + out.append( + BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript=transcript, + metadata={"stream": "input", "final": True}, + ) + ) + + case "conversation.item.input_audio_transcription.failed": + out.append( + BackendEvent( + event_type=BackendEventType.TRANSCRIPT, + transcript="", + metadata={"stream": "input", "failed": True}, + ) + ) + + case "input_audio_buffer.speech_started": + # If the inbound party barges in over a response that has already + # produced text, flush that partial as an interrupted turn before + # signaling the speech start (mirrors the old flush-then-new-turn order). + if session.responding and session.output_transcript_parts: + partial = "".join(session.output_transcript_parts) + out.append( + BackendEvent( + event_type=BackendEventType.TURN_END, + transcript=partial, + metadata={ + "interrupted": True, + "cancelled": False, + "has_function_calls": session.has_function_calls, + "usage": None, + }, + ) + ) + session.responding = False + session.output_transcript_parts = [] + session.output_transcript_done = "" + out.append(BackendEvent(event_type=BackendEventType.INPUT_SPEECH_STARTED)) + + case "input_audio_buffer.speech_stopped": + out.append(BackendEvent(event_type=BackendEventType.INPUT_SPEECH_STOPPED)) + + case "response.function_call_arguments.done": + session.has_function_calls = True + arguments_str = getattr(event, "arguments", "{}") or "{}" + try: + arguments = json.loads(arguments_str) + except json.JSONDecodeError: + arguments = {} + out.append( + BackendEvent( + event_type=BackendEventType.TOOL_CALL_REQUEST, + tool_call_request=ToolCallRequest( + call_id=getattr(event, "call_id", "") or "", + name=getattr(event, "name", "") or "", + arguments=arguments, + ), + ) + ) + + case "response.output_audio.done": + out.append(BackendEvent(event_type=BackendEventType.OUTPUT_AUDIO_DONE)) + + case "response.done": + response = getattr(event, "response", None) + cancelled = bool(response and getattr(response, "status", None) == "cancelled") + final_text = ( + session.output_transcript_done + or "".join(session.output_transcript_parts).strip() + or OpenAIRealtimeBackend._extract_response_text(event) + ) + has_fc = OpenAIRealtimeBackend._response_has_function_calls(event) or session.has_function_calls + out.append( + BackendEvent( + event_type=BackendEventType.TURN_END, + transcript=final_text, + metadata={ + "cancelled": cancelled, + "interrupted": False, + "has_function_calls": has_fc, + "usage": OpenAIRealtimeBackend._extract_usage(response), + }, + ) + ) + session.responding = False + session.output_transcript_parts = [] + session.output_transcript_done = "" + session.has_function_calls = False + + case "error": + error_data = getattr(event, "error", None) + code = getattr(error_data, "code", None) if error_data is not None else None + out.append( + BackendEvent( + event_type=BackendEventType.ERROR, + error=str(error_data) if error_data is not None else "unknown error", + metadata={"code": code}, + ) + ) + + case _: + # session.created/updated, interim transcription deltas, etc.: no + # cross-role meaning -> drop. + pass + + return out + + @staticmethod + def _extract_usage(response: Any) -> dict[str, int] | None: + if not response: + return None + usage = getattr(response, "usage", None) + if not usage: + return None + return { + "prompt_tokens": getattr(usage, "input_tokens", 0) or 0, + "completion_tokens": getattr(usage, "output_tokens", 0) or 0, + } + + @staticmethod + def _response_has_function_calls(event: Any) -> bool: + response = getattr(event, "response", None) + if not response: + return False + output_items = getattr(response, "output", None) or [] + return any(getattr(item, "type", "") == "function_call" for item in output_items) + + @staticmethod + def _extract_response_text(event: Any) -> str: + response = getattr(event, "response", None) + if not response: + return "" + output_items = getattr(response, "output", None) or [] + text_parts: list[str] = [] + for item in output_items: + for part in getattr(item, "content", None) or []: + if getattr(part, "type", "") in ("audio", "text"): + transcript = getattr(part, "transcript", None) or getattr(part, "text", None) or "" + if transcript: + text_parts.append(transcript) + return "".join(text_parts).strip() + + async def close(self, session: BackendSession) -> None: + """Tear down ``session``. Idempotent: safe to call more than once.""" + s = self._session(session) + if s.conn_cm is not None: + try: + await s.conn_cm.__aexit__(None, None, None) + except Exception as e: + logger.debug(f"Error closing OpenAI Realtime connection: {e}") + finally: + s.conn_cm = None + s.conn = None + if s.client is not None: + try: + await s.client.close() + except Exception as e: + logger.debug(f"Error closing OpenAI client: {e}") + finally: + s.client = None # type: ignore[assignment] diff --git a/src/eva/metrics/accuracy/task_completion.py b/src/eva/metrics/accuracy/task_completion.py index 2ca780b8..34267ffc 100644 --- a/src/eva/metrics/accuracy/task_completion.py +++ b/src/eva/metrics/accuracy/task_completion.py @@ -55,7 +55,8 @@ async def compute(self, context: MetricContext) -> MetricScore: """ details: dict = {"match": False, "auth_success": True} - # Require auth success — if session mismatches, task cannot be complete + # Require auth success — if session mismatches, task cannot be complete. + # Kept in sync with eva.metrics.diagnostic.task_completion_utils.is_task_completed. auth_mismatches = compute_session_auth_mismatches(context.expected_scenario_db, context.final_scenario_db) if auth_mismatches: details["auth_success"] = False diff --git a/src/eva/metrics/aggregation.py b/src/eva/metrics/aggregation.py index 2752f061..ca606188 100644 --- a/src/eva/metrics/aggregation.py +++ b/src/eva/metrics/aggregation.py @@ -32,6 +32,13 @@ class EVACompositeDefinition: # ── Composite definitions ──────────────────────────────────────────── +# Diagnostic efficiency metrics reported (as raw means) in the run-level summary +# alongside the EVA composites. These are lower-is-better raw values (not 0-1 +# normalized) and are only recorded for conversations that completed the task, so +# they answer "when the agent succeeds, how long / how many turns does it take?". +EFFICIENCY_METRICS: list[str] = ["time_to_completion", "turns_to_completion"] + + EVA_COMPOSITES: list[EVACompositeDefinition] = [ EVACompositeDefinition( name="EVA-A_pass", @@ -264,6 +271,12 @@ def compute_run_level_aggregates( result[comp.name] = entry + # Efficiency diagnostics (time/turns to completion): raw means over the + # records where the metric was recorded (i.e. the task was completed). + efficiency = _compute_efficiency_aggregates(all_metrics) + if efficiency: + result["efficiency"] = efficiency + # pass_k for aggregate metrics if multi-trial if num_draws > 1: pass_k_data = _compute_aggregate_pass_k(all_metrics, num_draws, composites, seed=seed) @@ -273,6 +286,46 @@ def compute_run_level_aggregates( return result +def _compute_efficiency_aggregates(all_metrics: dict[str, RecordMetrics]) -> dict: + """Aggregate completion-gated efficiency metrics for the run-level summary. + + For each metric in ``EFFICIENCY_METRICS``, computes the mean/min/max raw + score across the records where it was recorded (skipped/None records are + excluded — those are conversations that did not complete the task, so their + time/turn counts are not meaningful "to-completion" values). + + Returns: + Dict mapping metric name to {mean, min, max, count, total_records}. + Empty dict if no efficiency metrics were recorded. + """ + total_records = len(all_metrics) + result: dict = {} + + for name in EFFICIENCY_METRICS: + values: list[float] = [] + for record_metrics in all_metrics.values(): + score = record_metrics.metrics.get(name) + if score is None or score.error or score.skipped: + continue + value = score.score + if value is not None: + values.append(value) + + if not values: + continue + + result[name] = { + "mean": round(sum(values) / len(values), 4), + "min": round(min(values), 4), + "max": round(max(values), 4), + "count": len(values), + "total_records": total_records, + "higher_is_better": False, + } + + return result + + def _compute_aggregate_pass_k( all_metrics: dict[str, RecordMetrics], num_draws: int, diff --git a/src/eva/metrics/diagnostic/__init__.py b/src/eva/metrics/diagnostic/__init__.py index cfbb5af9..2162bb23 100644 --- a/src/eva/metrics/diagnostic/__init__.py +++ b/src/eva/metrics/diagnostic/__init__.py @@ -6,9 +6,11 @@ from . import response_speed # noqa from . import speakability # noqa from . import stt_wer # noqa +from . import time_to_completion # noqa from . import tool_call_validity # noqa from . import transcription_accuracy_key_entities # noqa from . import tts_fidelity # noqa +from . import turns_to_completion # noqa __all__ = [ "authentication_success", @@ -17,7 +19,9 @@ "response_speed", "speakability", "stt_wer", + "time_to_completion", "tool_call_validity", "transcription_accuracy_key_entities", "tts_fidelity", + "turns_to_completion", ] diff --git a/src/eva/metrics/diagnostic/task_completion_utils.py b/src/eva/metrics/diagnostic/task_completion_utils.py new file mode 100644 index 00000000..a83d1235 --- /dev/null +++ b/src/eva/metrics/diagnostic/task_completion_utils.py @@ -0,0 +1,39 @@ +"""Shared helper for determining whether a task was completed. + +This mirrors the logic of the ``task_completion`` accuracy metric so that +efficiency diagnostics (time/turns to completion) agree with it exactly: a task +is completed only if the session is authenticated correctly and the final +scenario database state matches the expected state (SHA-256 hash comparison). +""" + +from eva.metrics.base import MetricContext +from eva.metrics.diagnostic.authentication_success import compute_session_auth_mismatches +from eva.utils.hash_utils import get_dict_hash + + +def is_task_completed(context: MetricContext) -> tuple[bool, str]: + """Return whether the task was completed and a human-readable reason. + + Uses the same criteria as the ``task_completion`` accuracy metric: + 1. The session must be authenticated correctly (no session mismatches). + 2. The final scenario DB hash must equal the expected scenario DB hash. + + Args: + context: Metric context containing scenario DB states and hashes. + + Returns: + (completed, reason) where ``completed`` is True only when both checks + pass, and ``reason`` explains the outcome. + """ + # Require auth success — if the session mismatches, the task cannot be complete. + auth_mismatches = compute_session_auth_mismatches(context.expected_scenario_db, context.final_scenario_db) + if auth_mismatches: + return False, f"Authentication failed — session mismatch on keys: {list(auth_mismatches)}" + + expected_hash = get_dict_hash(context.expected_scenario_db) + actual_hash = context.final_scenario_db_hash + + if expected_hash == actual_hash: + return True, "Final database state matches expected state exactly" + + return False, "Final database state differs from expected state" diff --git a/src/eva/metrics/diagnostic/time_to_completion.py b/src/eva/metrics/diagnostic/time_to_completion.py new file mode 100644 index 00000000..6d6bd788 --- /dev/null +++ b/src/eva/metrics/diagnostic/time_to_completion.py @@ -0,0 +1,77 @@ +"""Time-to-completion diagnostic metric. + +Measures the total wall-clock time (in seconds) a conversation took, reported +only for conversations where the task was actually completed. This makes the +value a meaningful "how long did it take to succeed" signal rather than mixing +in the durations of failed conversations. + +Task completion is determined the same way as the ``task_completion`` accuracy +metric: the session must be authenticated correctly and the final scenario +database state must match the expected state (SHA-256 hash comparison). + +Diagnostic metric — reported for benchmarking, not used in final pass/fail +scores. Lower is better. +""" + +from eva.metrics.base import CodeMetric, MetricContext +from eva.metrics.diagnostic.task_completion_utils import is_task_completed +from eva.metrics.registry import register_metric +from eva.models.results import MetricScore + + +@register_metric +class TimeToCompletionMetric(CodeMetric): + """Wall-clock time (seconds) to complete the task. + + Reports ``context.duration_seconds`` when the task was completed, and is + skipped otherwise so that efficiency stats only aggregate over successful + conversations. + + Score: total conversation duration in seconds (lower is better). + """ + + name = "time_to_completion" + category = "diagnostic" + description = "Diagnostic metric: wall-clock time in seconds to complete the task (successful runs only)" + exclude_from_pass_at_k = True + higher_is_better = False # Score is time in seconds — lower is better. + + async def compute(self, context: MetricContext) -> MetricScore: + try: + completed, reason = is_task_completed(context) + + if not completed: + return MetricScore( + name=self.name, + score=None, + normalized_score=None, + skipped=True, + details={"task_completed": False, "reason": reason}, + ) + + duration = context.duration_seconds + if duration is None or duration <= 0: + return MetricScore( + name=self.name, + score=None, + normalized_score=None, + skipped=True, + details={ + "task_completed": True, + "reason": f"No valid duration recorded (duration_seconds={duration})", + }, + ) + + return MetricScore( + name=self.name, + score=round(duration, 3), + normalized_score=None, + details={ + "task_completed": True, + "duration_seconds": round(duration, 3), + "reason": "Task completed — reporting total conversation duration", + }, + ) + + except Exception as e: + return self._handle_error(e, context) diff --git a/src/eva/metrics/diagnostic/turns_to_completion.py b/src/eva/metrics/diagnostic/turns_to_completion.py new file mode 100644 index 00000000..32701ecb --- /dev/null +++ b/src/eva/metrics/diagnostic/turns_to_completion.py @@ -0,0 +1,77 @@ +"""Turns-to-completion diagnostic metric. + +Measures how many conversation turns a task took, reported only for +conversations where the task was actually completed. This makes the value a +meaningful "how many turns did it take to succeed" signal rather than mixing in +the turn counts of failed conversations. + +Task completion is determined the same way as the ``task_completion`` accuracy +metric: the session must be authenticated correctly and the final scenario +database state must match the expected state (SHA-256 hash comparison). + +Diagnostic metric — reported for benchmarking, not used in final pass/fail +scores. Lower is better. +""" + +from eva.metrics.base import CodeMetric, MetricContext +from eva.metrics.diagnostic.task_completion_utils import is_task_completed +from eva.metrics.registry import register_metric +from eva.models.results import MetricScore + + +@register_metric +class TurnsToCompletionMetric(CodeMetric): + """Number of conversation turns to complete the task. + + Reports ``context.num_turns`` (total conversation turns) when the task was + completed, and is skipped otherwise so that efficiency stats only aggregate + over successful conversations. + + Score: total number of conversation turns (lower is better). + """ + + name = "turns_to_completion" + category = "diagnostic" + description = "Diagnostic metric: number of conversation turns to complete the task (successful runs only)" + exclude_from_pass_at_k = True + higher_is_better = False # Score is a turn count — lower is better. + + async def compute(self, context: MetricContext) -> MetricScore: + try: + completed, reason = is_task_completed(context) + + if not completed: + return MetricScore( + name=self.name, + score=None, + normalized_score=None, + skipped=True, + details={"task_completed": False, "reason": reason}, + ) + + num_turns = context.num_turns + if not num_turns or num_turns <= 0: + return MetricScore( + name=self.name, + score=None, + normalized_score=None, + skipped=True, + details={ + "task_completed": True, + "reason": f"No valid turn count recorded (num_turns={num_turns})", + }, + ) + + return MetricScore( + name=self.name, + score=float(num_turns), + normalized_score=None, + details={ + "task_completed": True, + "num_turns": num_turns, + "reason": "Task completed — reporting total conversation turns", + }, + ) + + except Exception as e: + return self._handle_error(e, context) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 315bf4eb..66cf9603 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -56,51 +56,6 @@ def get_model_alias_from_params(params: dict[str, Any]) -> str: return params.get("alias") or params["model"] -_elevenlabs_agent_cache: dict[str, dict[str, str]] = {} - - -def _fetch_elevenlabs_agent_models(s2s_params: dict[str, Any]) -> dict[str, str]: - """Fetch STT, LLM, and TTS model names from the ElevenLabs agent API. - - Results are cached per agent ID so repeated calls (e.g. run_id generation) - don't hit the API multiple times. - """ - agent_id = s2s_params.get("assistant_agent_id", "") - if not agent_id: - logger.warning("No assistant_agent_id in s2s_params, cannot fetch ElevenLabs agent models") - return {"stt": "unknown", "llm": "unknown", "tts": "unknown"} - - if agent_id in _elevenlabs_agent_cache: - return _elevenlabs_agent_cache[agent_id] - - try: - from elevenlabs.client import ElevenLabs - - client = ElevenLabs(api_key=s2s_params.get("api_key")) - agent = client.conversational_ai.agents.get(agent_id=agent_id) - cc = agent.conversation_config - - stt = "unknown" - if cc.asr and cc.asr.provider: - stt = cc.asr.provider - - llm = "unknown" - if cc.agent and cc.agent.prompt and cc.agent.prompt.llm: - llm = cc.agent.prompt.llm - - tts = "unknown" - if cc.tts and cc.tts.model_id: - tts = cc.tts.model_id - - result = {"stt": stt, "llm": llm, "tts": tts} - _elevenlabs_agent_cache[agent_id] = result - logger.info(f"Fetched ElevenLabs agent models: {result}") - return result - except Exception as e: - logger.warning(f"Failed to fetch ElevenLabs agent models: {e}") - return {"stt": "unknown", "llm": "unknown", "tts": "unknown"} - - class ModelConfig(BaseModel): """Flat model configuration covering all pipeline modes. @@ -248,13 +203,12 @@ def pipeline_parts(self) -> dict[str, str]: "tts": get_model_alias_from_params(self.tts_params), } case PipelineType.S2S: - if self.s2s == "elevenlabs": - # hardcoded for now. Models are set on the agent UI - return { - "s2s": get_model_alias_from_params(self.s2s_params) or self.s2s, - **_fetch_elevenlabs_agent_models(self.s2s_params), - } - return {"s2s": get_model_alias_from_params(self.s2s_params)} + # Every native S2S backend is labeled uniformly by its model/alias, + # falling back to the provider name. ElevenLabs is no longer special- + # cased: its agent's internal STT/LLM/TTS are an agent-UI detail we + # don't fetch at config time, and its model is optional (so fall back). + alias = self.s2s_params.get("alias") or self.s2s_params.get("model") or self.s2s + return {"s2s": alias} case PipelineType.CASCADE: return { "stt": get_model_alias_from_params(self.stt_params), @@ -639,6 +593,10 @@ class ModelDeployment(DeploymentTypedDict): None, description="Specific record IDs to run", ) + exclude_record_ids: list[str] | None = Field( + None, + description="Specific record IDs to skip (applied after record_ids)", + ) # Execution max_concurrent_conversations: int = Field( @@ -766,8 +724,12 @@ def _check_companion_services(self) -> "RunConfig": errors.extend( self._validate_service_params("AUDIO_LLM", self.model.audio_llm, self.model.audio_llm_params) ) - case PipelineType.S2S: - errors.extend(self._validate_service_params("S2S", self.model.s2s, self.model.s2s_params)) + # S2S is intentionally not validated here: an S2S run uses a native backend, + # and that backend's construction (via the BackendFactory, exercised in + # orchestrator.preflight) is the single source of truth for its required + # fields/keys. Keeping it out of config avoids duplicating backend knowledge + # (the trade-off: a missing S2S key surfaces as a PreflightError, not a + # pydantic ValidationError). if errors: raise ValidationError.from_exception_data(title=type(self).__name__, line_errors=errors) @@ -851,7 +813,7 @@ def _handle_all_keyword(cls, data: Any): return rest return data - @field_validator("metrics", "record_ids", mode="before") + @field_validator("metrics", "record_ids", "exclude_record_ids", mode="before") @classmethod def _parse_comma_separated(cls, v: Any) -> list[str] | None: """Accept comma-separated strings from env vars.""" diff --git a/src/eva/orchestrator/preflight.py b/src/eva/orchestrator/preflight.py index aedcb903..c2763400 100644 --- a/src/eva/orchestrator/preflight.py +++ b/src/eva/orchestrator/preflight.py @@ -12,8 +12,16 @@ - Probes are conservative: a component is reported as failed only on a definitive error (an exception or an ``ErrorFrame``). Ambiguity — no output, an unexpected-but-benign frame — is treated as a pass, so preflight never blocks an otherwise-valid run. -- S2S live probing is not yet supported (each framework needs its own connect path); - S2S relies on the config-level credential validation in ``RunConfig``. +- The assistant runs on the Role/Backend path ONLY, so the configured framework must be + one the ``BackendFactory`` backs. ``_preflight_backends`` validates it cheaply by + *constructing* the backend (its ``__init__`` checks required fields/keys, no API call) + and rejects a framework the factory doesn't back yet (unported = unusable; the legacy + server survives only as reference). This is the home for that validation (config-side + validation was removed). A live backend-session probe is still not supported. +- The live model probes below (LLM/STT/TTS/audio-LLM) validate a cascade/audio-LLM + *assistant pipeline*. They are dormant while those frameworks are unported (rejected at + ``_preflight_backends``) and light up again once such a provider becomes a native + backend — kept as the reference wiring for that. """ import asyncio @@ -39,11 +47,15 @@ create_tts_service, ) from eva.assistant.services.llm import LiteLLMClient +from eva.backend.factory import BackendFactory from eva.models.config import PipelineType, RunConfig, get_model_alias_from_params from eva.utils.logging import get_logger logger = get_logger(__name__) +# Stateless; shared across preflight calls (see orchestrator.worker for the same pattern). +_BACKEND_FACTORY = BackendFactory() + # 0.3s of 16 kHz mono 16-bit silence — enough to make a streaming STT service open its # connection (and reveal an auth failure) without depending on actual speech content. _SILENCE_SAMPLE_RATE = 16000 @@ -173,19 +185,53 @@ async def _run_preflight(config: RunConfig) -> list[ProbeResult]: probes.append(("AUDIO_LLM", get_model_alias_from_params(model.audio_llm_params), _probe_audio_llm(config))) probes.append(("TTS", get_model_alias_from_params(model.tts_params), _probe_tts(config))) case PipelineType.S2S: - logger.info("Pre-flight: S2S live probe not yet supported") + # Native S2S backends are validated by construction in _preflight_backends; + # there is no live session probe yet, so nothing to probe here. + logger.info("Pre-flight: S2S live probe not yet supported (construction validated)") return [] logger.info(f"Pre-flight: checking {len(probes)} model(s) before the run starts...") return await asyncio.gather(*(_guard(model_type, alias, probe, timeout) for model_type, alias, probe in probes)) +def _assistant_backend_args(config: RunConfig) -> dict[str, Any]: + """The config blob the worker/factory builds a native assistant backend from.""" + return { + **(config.model.s2s_params or {}), + "parallel_tool_calls": config.model.parallel_tool_calls, + } + + +def _preflight_backends(config: RunConfig) -> None: + """Cheap, network-free validation that the assistant's native backend constructs. + + The assistant runs on the Role/Backend path only, so the configured framework MUST + be one the ``BackendFactory`` backs. Constructing it validates required fields/keys + (api_key, model, speaker_id, ...) with no API call — the single source of truth for + backend config. A framework the factory doesn't back yet is unusable (its legacy + server survives only as reference), so it fails here rather than falling through. + The user simulator is on its legacy stack and validated by config, not here. + """ + try: + backend = _BACKEND_FACTORY.create(config.framework, _assistant_backend_args(config)) + except Exception as e: + detail = str(e).strip()[:400] or type(e).__name__ + raise PreflightError(f"assistant framework {config.framework!r} config invalid:\n{detail}") from e + if backend is None: + raise PreflightError( + f"assistant framework {config.framework!r} is not available as a native backend yet." + ) + + async def run_preflight(config: RunConfig) -> None: - """Probe models and raise ``PreflightError`` if any required component fails. + """Validate backends, then probe models; raise ``PreflightError`` on any failure. - No-op when ``config.preflight`` is set or there are no probes to run - (e.g. S2S). Call this immediately before launching simulations. + The cheap, network-free backend-construction check (``_preflight_backends``) ALWAYS + runs -- ``--no-preflight`` only skips the live model probes (the ones that make real + API calls), never the config validation. Call this immediately before launching + simulations. """ + _preflight_backends(config) # always: cheap, no network if not config.preflight: return results = await _run_preflight(config) diff --git a/src/eva/orchestrator/runner.py b/src/eva/orchestrator/runner.py index 57921c9f..7747e037 100644 --- a/src/eva/orchestrator/runner.py +++ b/src/eva/orchestrator/runner.py @@ -95,7 +95,7 @@ def _load_agent_config(self) -> AgentConfig: return agent def _filter_records(self, records: list[EvaluationRecord]) -> list[EvaluationRecord]: - """Filter records based on debug mode or record_ids. + """Filter records based on debug mode, record_ids, and exclude_record_ids. Args: records: All records from dataset @@ -108,10 +108,12 @@ def _filter_records(self, records: list[EvaluationRecord]) -> list[EvaluationRec logger.info("Debug mode enabled: running only 1 record") return records[:1] + filtered = records + # Filter by specific record IDs if provided if self.config.record_ids: logger.info(f"Filtering to specific records: {self.config.record_ids}") - filtered = [r for r in records if r.id in self.config.record_ids] + filtered = [r for r in filtered if r.id in self.config.record_ids] # Warn if some IDs not found found_ids = {r.id for r in filtered} @@ -119,10 +121,21 @@ def _filter_records(self, records: list[EvaluationRecord]) -> list[EvaluationRec if missing_ids: logger.warning(f"Record IDs not found in dataset: {missing_ids}") - return filtered - - # No filtering - return all records - return records + # Exclude specific record IDs if provided (applied after the include filter) + if self.config.exclude_record_ids: + exclude = set(self.config.exclude_record_ids) + logger.info(f"Excluding specific records: {sorted(exclude)}") + before = len(filtered) + filtered = [r for r in filtered if r.id not in exclude] + + # Warn about exclude IDs that never matched anything in the current set + excluded_ids = {r.id for r in records if r.id in exclude} + unknown_exclude_ids = exclude - excluded_ids + if unknown_exclude_ids: + logger.warning(f"Exclude record IDs not found in dataset: {sorted(unknown_exclude_ids)}") + logger.info(f"Excluded {before - len(filtered)} record(s)") + + return filtered async def run(self, records: list[EvaluationRecord]) -> RunResult: """Run all records with validation and reruns. diff --git a/src/eva/orchestrator/worker.py b/src/eva/orchestrator/worker.py index ef5519c4..2705af4c 100644 --- a/src/eva/orchestrator/worker.py +++ b/src/eva/orchestrator/worker.py @@ -7,11 +7,12 @@ from pathlib import Path from typing import Any -from eva.assistant.base_server import AbstractAssistantServer +from eva.backend.factory import BackendFactory from eva.models.agents import AgentConfig from eva.models.config import RunConfig from eva.models.record import EvaluationRecord from eva.models.results import ConversationResult, ErrorDetails, LatencyStats +from eva.role.assistant import AssistantRole from eva.user_simulator.factory import create_user_simulator from eva.utils.culture import resolve_scenario_db, resolve_user_config, resolve_user_goal from eva.utils.error_handler import create_error_details @@ -22,38 +23,12 @@ USER_SIMULATOR_SHUTDOWN_GRACE_SECONDS = 20 - -def _get_server_class(framework: str) -> type[AbstractAssistantServer]: - """Return the server class for the given framework name. - - Uses lazy imports to avoid importing heavy dependencies (pipecat, openai, etc.) - unless the framework is actually selected. - """ - if framework == "pipecat": - from eva.assistant.pipecat_server import PipecatAssistantServer - - return PipecatAssistantServer - elif framework == "openai_realtime": - from eva.assistant.openai_realtime_server import OpenAIRealtimeAssistantServer - - return OpenAIRealtimeAssistantServer - elif framework == "gemini_live": - from eva.assistant.gemini_live_server import GeminiLiveAssistantServer - - return GeminiLiveAssistantServer - elif framework == "elevenlabs": - from eva.assistant.elevenlabs_server import ElevenLabsAssistantServer - - return ElevenLabsAssistantServer - elif framework == "grok_voice": - from eva.assistant.grok_voice_server import GrokVoiceAssistantServer - - return GrokVoiceAssistantServer - else: - raise ValueError( - f"Unknown framework: {framework!r}. " - "Supported: pipecat, openai_realtime, gemini_live, elevenlabs, grok_voice" - ) +# The backend factory is stateless (pure dispatch + lazy per-provider imports), so +# a single shared instance serves every worker. The assistant runs on the Role/Backend +# path ONLY: the factory builds the backend for the configured framework, and a +# framework it doesn't yet back is unusable (see _start_assistant). The legacy +# assistant servers survive as reference but are no longer wired. +_BACKEND_FACTORY = BackendFactory() def _percentile(sorted_data: list[float], p: float) -> float: @@ -116,8 +91,9 @@ def __init__( self.port = port self.output_id = output_id - # Will be set during run - self._assistant_server = None + # Set during run: a Role (for factory-supported providers) or the legacy + # server/simulator (for the rest). + self._assistant_server: AssistantRole | None = None self._user_simulator = None self._conversation_stats: dict[str, Any] = {} self._log_file_handler = None @@ -305,15 +281,30 @@ async def run(self) -> ConversationResult: async def _start_assistant(self) -> None: """Start the assistant server using the configured framework.""" - server_cls = _get_server_class(self.config.framework) resolved_db_path = self._materialize_resolved_scenario_db() - # The turn-end fallback applies to the Pipecat pipelines (cascade + audio-LLM), whose - # turn detection can drop a user turn. S2S servers handle turn-taking natively and - # don't accept this kwarg. - server_kwargs: dict[str, Any] = {} - if self.config.framework == "pipecat": - server_kwargs["turn_end_fallback_time"] = self.config.turn_end_fallback_time - self._assistant_server = server_cls( + + s2s = self.config.model.s2s_params or {} + # `language` is passed through for backends that localize speech synthesis + # (e.g. Gemini's language_code); backends that don't need it ignore it. + backend_args = { + "language": self.config.language, + **s2s, + "parallel_tool_calls": self.config.model.parallel_tool_calls, + } + backend = _BACKEND_FACTORY.create(self.config.framework, backend_args) + if backend is None: + # Assistant runs on the Role/Backend path only. A framework the factory + # doesn't back yet is unusable (its legacy server survives as reference, + # but is no longer wired); add it to the factory to enable it. + raise ValueError( + f"Framework {self.config.framework!r} is not available as a native backend yet." + ) + + # A generic AssistantRole over a factory-built backend, selected by the + # configured framework name; its args are the S2S provider params passed + # through (the backend reads what it needs and assembles its own session). + self._assistant_server = AssistantRole( + backend=backend, current_date_time=self.record.current_date_time, pipeline_config=self.config.model, agent=self.agent, @@ -323,7 +314,6 @@ async def _start_assistant(self) -> None: port=self.port, conversation_id=self.record.id, language=self.config.language, - **server_kwargs, ) await self._assistant_server.start() diff --git a/src/eva/role/__init__.py b/src/eva/role/__init__.py index c005c0a7..5c4e71d4 100644 --- a/src/eva/role/__init__.py +++ b/src/eva/role/__init__.py @@ -1,17 +1,13 @@ -"""Provider-agnostic ``Role`` abstraction (design-only, Step 1 of the refactor). +"""Provider-agnostic ``Role`` abstraction (see ``docs/refactor-step1.md``). -A ``Role`` owns everything that today is duplicated across the assistant and -user-simulator stacks per-provider: prompt construction, tool ownership, and -goal/persona/agent-config data. Each ``Role`` holds exactly one -``eva.backend.Backend`` instance, created at runtime via a -``eva.backend.BackendFactory``. - -Nothing in this package is wired into the existing ``eva.assistant`` / -``eva.user_simulator`` code yet. +A ``Role`` owns everything that was duplicated across the assistant provider +stacks: prompt construction, tool ownership, and agent-config data. Each +``Role`` holds exactly one ``eva.backend.Backend`` instance, created at runtime +via a ``eva.backend.BackendFactory``. Only the assistant side is on this path +for now; the user simulator stays on its legacy stack. """ from eva.role.assistant import AssistantRole from eva.role.base import Role -from eva.role.user import UserRole -__all__ = ["AssistantRole", "Role", "UserRole"] +__all__ = ["AssistantRole", "Role"] diff --git a/src/eva/role/assistant.py b/src/eva/role/assistant.py index 897dba14..9dc42dbb 100644 --- a/src/eva/role/assistant.py +++ b/src/eva/role/assistant.py @@ -1,132 +1,615 @@ -"""``AssistantRole`` contract: the business-side answering role. - -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). Method -bodies are stubs; nothing here is wired into the existing code path yet. - -Plug-in point (where this will eventually replace existing code): - Today the assistant side is a concrete ``AbstractAssistantServer`` - subclass selected by ``eva.orchestrator.worker._get_server_class(framework)`` - (worker.py) and constructed + started inside - ``ConversationWorker._start_assistant()`` (worker.py), which calls - ``server_cls(...).start()``. Its outputs are flushed by - ``ConversationWorker._cleanup()`` via ``server.stop()`` (which internally - calls ``save_outputs()``), and ``get_conversation_stats()`` / - ``get_final_scenario_db()`` are read back in ``ConversationWorker.run()``. - - In a later phase, ``_start_assistant()`` becomes the construction site for - an ``AssistantRole`` (framework string -> ``backend_name`` passed to the - ``BackendFactory``), and the worker drives ``role.run()`` / - ``role.save_outputs()`` / ``role.get_final_scenario_db()`` instead of the - server's own lifecycle methods. The provider-specific server subclasses - collapse into ``Backend`` implementations behind the factory; the - role-agnostic orchestration in ``ConversationWorker`` stays put. This - module is deliberately separate from ``eva.role.user`` so that migration - can land assistant-side first without touching the user-side diff. +"""``AssistantRole``: the business-side answering role (one generic class). + +A single concrete, provider-agnostic role. It holds a ``Backend`` and works +with *any* backend -- swapping the backend swaps the provider. All provider +specifics (session, audio format, event parsing) live in the backend; this +class owns only the role-common concerns shared by every assistant regardless +of provider: + +- the assistant system prompt (built from agent config); +- the agent tool catalog + ``ToolExecutor`` (tool execution stays role-side); +- the ``AuditLog`` and output artifacts (audit_log.json / transcript.jsonl / + scenario DBs / audio WAVs); +- the counterparty transport: a Twilio-framed WebSocket **server** the user + simulator connects to, plus real-time output pacing and audio-track + recording/alignment (all provider-agnostic -- every assistant exposes this + same Twilio WS). + +It consumes only the normalized ``BackendEvent`` stream, so it never touches a +raw provider event. + +Plug-in point: mirrors ``AbstractAssistantServer``'s surface (``start`` / +``stop`` / ``get_conversation_stats`` / ``get_final_scenario_db`` / +``notify_conversation_ending``) so the worker swap is 1:1 -- construct +``AssistantRole(backend=factory.create(...), ...)`` instead of +``server_cls(...)`` and keep every downstream call. The worker takes this path +for every provider the ``BackendFactory`` supports. """ from __future__ import annotations -from abc import abstractmethod +import asyncio +import json +import time +from dataclasses import dataclass +from pathlib import Path from typing import Any -from eva.backend.factory import BackendFactory +import uvicorn +from fastapi import FastAPI, WebSocket, WebSocketDisconnect + +from eva.assistant.agentic.audit_log import AuditLog +from eva.assistant.pipeline.observers import FrameworkLogWriter, MetricsLogWriter +from eva.assistant.tools.tool_executor import ToolExecutor, execute_and_log_tool +from eva.backend.base import Backend, BackendEvent, BackendEventType, ToolCallRequest, ToolCallResult +from eva.models.agents import AgentConfig +from eva.models.config import ModelConfig from eva.role.base import Role +from eva.utils.audio_utils import ( + create_twilio_media_message, + mulaw_8k_to_pcm16_24k, + parse_twilio_media_message, + pcm16_24k_to_mulaw_8k, + pcm16_mix, + save_audio_track, + sync_buffer_to_position, +) +from eva.utils.culture import get_initial_message +from eva.utils.logging import get_logger +from eva.utils.prompt_manager import PromptManager + +logger = get_logger(__name__) + +# Twilio counterparty-transport constants (provider-agnostic). +MULAW_CHUNK_SIZE = 160 # bytes per chunk (20ms at 8kHz mulaw) +MULAW_CHUNK_DURATION_S = 0.02 +# Don't pad the user track to align with the assistant when real user audio +# arrived within this window (the speaking-state flag can go stale under jitter; +# padding then injects a mid-utterance chop). Guard only ever *skips* a pad. +USER_ACTIVE_GUARD_S = 0.3 + + +def _wall_ms() -> str: + """Current wall-clock time as epoch-milliseconds string.""" + return str(int(round(time.time() * 1000))) + + +@dataclass +class _UserTurnRecord: + """State for a single user speech turn (timestamps + transcript flush flag).""" + + speech_started_wall_ms: str = "" + speech_stopped_wall_ms: str = "" + transcript: str = "" + flushed: bool = False + + +@dataclass +class _AssistantTurnState: + """Per-response state the role tracks for recording/metrics/logging.""" + + first_audio_wall_ms: str | None = None + audio_was_streamed: bool = False + responding: bool = False class AssistantRole(Role): - """Role that answers on behalf of the business (today's "assistant server"). - - Carries agent configuration and tool catalog; owns a ``ToolExecutor`` - (constructed by subclasses, not by this contract) to fulfill - ``handle_tool_call_request``. - - Turn-end fallback (self-nudge): the assistant's backstop for a *dropped - user turn*. When VAD / turn detection silently fails to fire for a real - user utterance, the call would otherwise hang until the provider's - inactivity timeout ends it. After the assistant stops speaking, if no user - turn is detected within ``turn_end_fallback_seconds``, the assistant - proactively re-engages with a nudge (acknowledge-and-answer if partial - user speech/audio was captured, otherwise ask the caller to repeat). This - is the seam already shipped as the pipeline-side ``TurnEndFallbackTimer`` - (see ``eva.assistant.pipeline.fallback`` and ``EVA_TURN_END_FALLBACK_TIME``); - it works for both cascade and audio-LLM pipelines. - - Two policies the backend owns, carried over from the shipped feature: - - Give up after a small number of *consecutive* nudges without a real user - turn resetting the count (``MAX_CONSECUTIVE_FALLBACK_NUDGES``), then let - the provider's inactivity backstop end the call. - - Never nudge once the call is ending (a nudge during teardown produces a - phantom assistant turn after the conversation is logically closed). - - Unlike the tool-call/idle-detection seams elsewhere in this contract, the - fallback needs no new ``Role`` method and no new ``Backend`` event type: - the nudge is just an ordinary outbound turn that this role's backend - produces on its own after the timeout, using the same - ``system_prompt``/instructions already established at ``open()`` time (see - ``Backend.open``'s ``config`` docstring). It is surfaced through the normal - ``receive()`` stream and tagged so downstream metrics can identify and zero - it (the shipped feature records the transcript marker with - ``message_type="turn_fallback"``; see ``BackendEvent.metadata``). Whether - the *other* side (a ``UserRole``) needs to do anything special upon - receiving it, versus just treating it as an ordinary assistant turn through - its existing ``run()`` loop, is left open -- see docs/refactor-step1.md - discussion; nothing here requires ``UserRole`` changes to handle it today. - """ + """Generic assistant role. Drives any ``Backend``; owns the Twilio WS transport.""" def __init__( self, *, - backend_factory: BackendFactory, - backend_name: str, - backend_config: dict[str, Any], + backend: Backend, + current_date_time: str, + pipeline_config: ModelConfig, + agent: AgentConfig, agent_config_path: str, scenario_db_path: str, - current_date_time: str, + output_dir: Path, + port: int, + conversation_id: str, + language: str = "en", turn_end_fallback_seconds: float | None = None, ) -> None: - """Initialize the assistant role. - - Args: - backend_factory: Factory used to construct the backend. - backend_name: Key passed to the factory to select a backend. - backend_config: Provider-specific configuration for the backend. - agent_config_path: Path to the agent YAML (role, instructions, - tool schemas) -- mirrors ``AbstractAssistantServer.agent`` / - ``agent_config_path``. - scenario_db_path: Path to the per-record scenario database JSON - consumed by tool execution -- mirrors - ``AbstractAssistantServer.scenario_db_path``. - current_date_time: Current date/time string threaded into both - prompt construction and tool execution (mirrors existing - ``current_date_time`` plumbing throughout the assistant - stack). - turn_end_fallback_seconds: How long after the assistant stops - speaking to wait for a user turn before firing a turn-end - fallback nudge, or ``None`` to disable the fallback entirely - (preserving the old behavior of waiting for the provider's - inactivity timeout). Mirrors the shipped - ``EVA_TURN_END_FALLBACK_TIME`` knob. This is an - ``AssistantRole``-level tuning value, not a - ``BackendCapabilities`` flag (capabilities describe what a - backend *can* do, statically). Wiring it into the constructed - ``self.backend``'s own config (via ``backend_config`` / - ``Backend.open(config=...)``) is left to the concrete - subclass's constructor, same as elsewhere in this contract -- - a ``Role`` does not otherwise reach into backend config after - construction. A backend with no notion of idle timing (e.g. a - thin end-to-end backend that relies on its own provider - backstop) may simply ignore this value. - """ - super().__init__(backend_factory=backend_factory, backend_name=backend_name, backend_config=backend_config) + super().__init__(backend=backend) + self.current_date_time = current_date_time + self.pipeline_config = pipeline_config + self.agent = agent self.agent_config_path = agent_config_path self.scenario_db_path = scenario_db_path - self.current_date_time = current_date_time + self.output_dir = Path(output_dir) + self.port = port + self.conversation_id = conversation_id + self.language = language self.turn_end_fallback_seconds = turn_end_fallback_seconds + self.initial_message = get_initial_message(language) + + # Core components. + self.audit_log = AuditLog() # type: ignore[no-untyped-call] + self.tool_handler = ToolExecutor( + tool_config_path=agent_config_path, + scenario_db_path=scenario_db_path, + tool_module_path=self.agent.tool_module_path, + current_date_time=current_date_time, + ) + + # Recording buffers. Sample rate comes from the backend (provider format). + self._audio_buffer = bytearray() + self.user_audio_buffer = bytearray() + self.assistant_audio_buffer = bytearray() + self._audio_sample_rate = backend.output_sample_rate + + self._fw_log: FrameworkLogWriter | None = None + self._metrics_log: MetricsLogWriter | None = None + + # Server state. + self._app: FastAPI | None = None + self._server: uvicorn.Server | None = None + self._server_task: asyncio.Task[Any] | None = None + self._running = False + + # Prompt + generic tool specs (built once); model name for metrics labels. + self._system_prompt = self.build_prompt() + self._tool_specs = self._build_tool_specs() + self._model = (self.pipeline_config.s2s_params or {}).get("model", "") + + # Per-session/turn state. + self._user_turn: _UserTurnRecord | None = None + self._assistant_turn = _AssistantTurnState() + self._stream_sid = "" + self._user_speaking = False + self._bot_speaking = False + self._audio_interface_speech_start_ts: str | None = None + self._last_user_audio_mono = 0.0 + + # ── Prompt / tool specs (role-owned, provider-agnostic) ─────────── + + def build_prompt(self) -> str: + """Build the assistant system prompt from the agent config.""" + prompt_manager = PromptManager() + prompt = prompt_manager.get_prompt( + "realtime_agent.system_prompt", + agent_personality=self.agent.description, + agent_instructions=self.agent.instructions, + datetime=self.current_date_time, + ) + if self.pipeline_config.pre_tool_speech == "auto": + prompt += "\n\n" + prompt_manager.get_prompt("agent.pre_tool_speech") + return prompt + + def _build_tool_specs(self) -> list[dict[str, Any]]: + """Provider-agnostic tool specs from the agent tools (backend formats to its schema).""" + specs: list[dict[str, Any]] = [] + for tool in self.agent.tools or []: + specs.append( + { + "name": tool.function_name, + "description": f"{tool.name}: {tool.description}", + "parameters": { + "type": "object", + "properties": tool.get_parameter_properties(), + "required": tool.get_required_param_names(), + }, + } + ) + return specs + + # ── Role seams ───────────────────────────────────────────────────── + + async def handle_tool_call_request(self, request: ToolCallRequest) -> ToolCallResult: + """Execute a tool call the backend surfaced and record it in the audit log.""" + result = await execute_and_log_tool(self.tool_handler, self.audit_log, request.name, request.arguments) + return ToolCallResult(call_id=request.call_id, result=result) + + def record_audio(self, source: str, audio_data: bytes) -> None: + """Append PCM16 to the named channel buffer (alignment handled by callers).""" + if source == "user": + self.user_audio_buffer.extend(audio_data) + elif source == "assistant": + self.assistant_audio_buffer.extend(audio_data) + + def notify_conversation_ending(self, reason: str | None = None) -> None: + """No-op: native-S2S turn-taking needs no early-end signal (see AbstractAssistantServer).""" + return None + + def get_conversation_stats(self) -> dict[str, Any]: + return self.audit_log.get_stats() + + def get_initial_scenario_db(self) -> dict[str, Any]: + return self.tool_handler.original_db - @abstractmethod def get_final_scenario_db(self) -> dict[str, Any]: - """Return the (possibly mutated) scenario database state, for metrics. + return self.tool_handler.db + + # ── Server lifecycle ────────────────────────────────────────────── + + async def start(self) -> None: + """Start the FastAPI WebSocket server (non-blocking).""" + if self._running: + logger.warning("Assistant role already running") + return + + self.output_dir.mkdir(parents=True, exist_ok=True) + self._fw_log = FrameworkLogWriter(self.output_dir) + self._metrics_log = MetricsLogWriter(self.output_dir) + + self._app = FastAPI() + + @self._app.websocket("/ws") + async def websocket_endpoint(websocket: WebSocket) -> None: + await websocket.accept() + await self._handle_session(websocket) + + @self._app.websocket("/") + async def websocket_root(websocket: WebSocket) -> None: + await websocket.accept() + await self._handle_session(websocket) + + config = uvicorn.Config(self._app, host="0.0.0.0", port=self.port, log_level="warning", lifespan="off") + self._server = uvicorn.Server(config) + self._running = True + self._server_task = asyncio.create_task(self._server.serve()) + + while not self._server.started: + await asyncio.sleep(0.01) + + logger.info(f"Assistant role started on ws://localhost:{self.port}") + + async def _shutdown(self) -> None: + if not self._running: + return + self._running = False + if self._server: + self._server.should_exit = True + if self._server_task: + try: + await asyncio.wait_for(self._server_task, timeout=5.0) + except TimeoutError: + self._server_task.cancel() + try: + await self._server_task + except asyncio.CancelledError: + pass + except (asyncio.CancelledError, KeyboardInterrupt): + pass + self._server = None + self._server_task = None + logger.info(f"Assistant role stopped on port {self.port}") + + async def stop(self) -> asyncio.Task[None] | None: + """Shut down, extract audio, save outputs (mirrors AbstractAssistantServer.stop).""" + await self._shutdown() + self._ensure_mixed_audio() + + mixed_audio = bytes(self._audio_buffer) + user_audio = bytes(self.user_audio_buffer) + assistant_audio = bytes(self.assistant_audio_buffer) + sample_rate = self._audio_sample_rate + self._audio_buffer.clear() + self.user_audio_buffer.clear() + self.assistant_audio_buffer.clear() + + self.save_outputs() + + if mixed_audio or user_audio or assistant_audio: + return asyncio.create_task( + asyncio.to_thread(self._save_audio_deferred, mixed_audio, user_audio, assistant_audio, sample_rate) + ) + return None + + # ── Output persistence ──────────────────────────────────────────── + + def save_outputs(self) -> None: + self.audit_log.save(self.output_dir / "audit_log.json") + self.audit_log.save_transcript_jsonl(self.output_dir / "transcript.jsonl") + self._save_scenario_dbs() + logger.info(f"Outputs saved to {self.output_dir}") + + def _ensure_mixed_audio(self) -> None: + if self._audio_buffer: + return + if self.user_audio_buffer and self.assistant_audio_buffer: + diff_bytes = abs(len(self.user_audio_buffer) - len(self.assistant_audio_buffer)) + diff_ms = diff_bytes / (2 * self._audio_sample_rate) * 1000 + if diff_ms > 500: + logger.warning( + f"Audio buffer length mismatch: user={len(self.user_audio_buffer)} " + f"assistant={len(self.assistant_audio_buffer)} diff={diff_ms:.0f}ms — mixed recording may be skewed" + ) + self._audio_buffer = bytearray(pcm16_mix(bytes(self.user_audio_buffer), bytes(self.assistant_audio_buffer))) + elif self.user_audio_buffer: + self._audio_buffer = bytearray(self.user_audio_buffer) + elif self.assistant_audio_buffer: + self._audio_buffer = bytearray(self.assistant_audio_buffer) + + def _save_audio_deferred( + self, mixed_audio: bytes, user_audio: bytes, assistant_audio: bytes, sample_rate: int + ) -> None: + save_audio_track(mixed_audio, self.output_dir / "audio_mixed.wav", sample_rate) + save_audio_track(user_audio, self.output_dir / "audio_user.wav", sample_rate) + save_audio_track(assistant_audio, self.output_dir / "audio_assistant.wav", sample_rate) + if mixed_audio or user_audio or assistant_audio: + logger.info(f"Saved audio files to {self.output_dir} ({len(mixed_audio)} bytes mixed)") + + def _save_scenario_dbs(self) -> None: + try: + with open(self.output_dir / "initial_scenario_db.json", "w") as f: + json.dump(self.get_initial_scenario_db(), f, indent=2, sort_keys=True, default=str, ensure_ascii=False) + with open(self.output_dir / "final_scenario_db.json", "w") as f: + json.dump(self.get_final_scenario_db(), f, indent=2, sort_keys=True, default=str, ensure_ascii=False) + logger.info(f"Saved scenario database states to {self.output_dir}") + except Exception as e: + logger.error(f"Error saving scenario database states: {e}", exc_info=True) + raise + + # ── Session handling (Twilio WS <-> backend) ────────────────────── + + async def _handle_session(self, websocket: WebSocket) -> None: + logger.info("Client connected to assistant role") + self._user_turn = None + self._assistant_turn = _AssistantTurnState() + self._stream_sid = self.conversation_id + self._user_speaking = False + self._bot_speaking = False + + session = None + try: + session = await self.backend.open(system_prompt=self._system_prompt, tools=self._tool_specs) + # Trigger the initial greeting. + await self.backend.send(session, text=f"Say: '{self.initial_message}'") + + audio_output_queue: asyncio.Queue[bytes] = asyncio.Queue() + forward_task = asyncio.create_task(self._forward_user_audio(websocket, session)) + receive_task = asyncio.create_task(self._process_backend_events(session, audio_output_queue)) + pacer_task = asyncio.create_task(self._pace_audio_output(websocket, audio_output_queue)) + + done, pending = await asyncio.wait( + [forward_task, receive_task, pacer_task], return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + for task in done: + if task.exception(): + logger.error(f"Session task failed: {task.exception()}") + except Exception as e: + logger.error(f"Assistant session error: {e}", exc_info=True) + finally: + if session is not None: + await self.backend.close(session) + logger.info("Client disconnected from assistant role") + + async def _pace_audio_output(self, websocket: WebSocket, audio_output_queue: asyncio.Queue[bytes]) -> None: + """Drain the output queue and forward chunks to Twilio at real-time rate.""" + next_send_time = time.monotonic() + try: + while True: + try: + chunk = await asyncio.wait_for(audio_output_queue.get(), timeout=1.0) + except TimeoutError: + continue + try: + await websocket.send_text(create_twilio_media_message(self._stream_sid, chunk)) + except Exception as e: + logger.error(f"Error sending audio to Twilio WS: {e}") + return + now = time.monotonic() + if next_send_time <= now: + next_send_time = now + next_send_time += MULAW_CHUNK_DURATION_S + sleep_duration = next_send_time - time.monotonic() + if sleep_duration > 0: + await asyncio.sleep(sleep_duration) + except asyncio.CancelledError: + pass + + async def _forward_user_audio(self, websocket: WebSocket, session: Any) -> None: + """Read Twilio media frames and forward audio to the backend.""" + try: + while True: + raw = await websocket.receive_text() + data = json.loads(raw) + event_type = data.get("event") + + if event_type == "start": + self._stream_sid = data.get("start", {}).get("streamSid", self.conversation_id) + continue + if event_type == "stop": + break + if event_type == "user_speech_start": + self._audio_interface_speech_start_ts = data.get("timestamp_ms") + continue + if event_type != "media": + continue + + mulaw_bytes = parse_twilio_media_message(raw) + if mulaw_bytes is None: + continue + + # Twilio 8kHz mulaw -> backend PCM (24kHz converters; the only backend + # today is 24k — a different-rate backend would need rate-generic utils). + pcm = mulaw_8k_to_pcm16_24k(mulaw_bytes) + if not self._bot_speaking: + sync_buffer_to_position(self.assistant_audio_buffer, len(self.user_audio_buffer)) + self.record_audio("user", pcm) + self._last_user_audio_mono = time.monotonic() + + await self.backend.send(session, audio=pcm) + except WebSocketDisconnect: + logger.debug("Twilio WebSocket disconnected") + except asyncio.CancelledError: + pass + except Exception as e: + logger.error(f"Error forwarding user audio: {e}", exc_info=True) + + async def _process_backend_events(self, session: Any, audio_output_queue: asyncio.Queue[bytes]) -> None: + """Consume normalized backend events and produce audit/transcript/audio + tool results.""" + try: + async for event in self.backend.receive(session): + try: + await self._handle_backend_event(event, session, audio_output_queue) + except Exception as e: + logger.error(f"Error handling event {event.event_type}: {e}", exc_info=True) + except asyncio.CancelledError: + pass + except Exception as e: + logger.error(f"Error in backend event loop: {e}", exc_info=True) + + async def _handle_backend_event( + self, event: BackendEvent, session: Any, audio_output_queue: asyncio.Queue[bytes] + ) -> None: + match event.event_type: + case BackendEventType.INPUT_SPEECH_STARTED: + self._on_speech_started() + case BackendEventType.INPUT_SPEECH_STOPPED: + self._on_speech_stopped() + case BackendEventType.TRANSCRIPT: + self._on_transcript(event) + case BackendEventType.AUDIO_OUTPUT: + await self._on_audio_output(event.audio or b"", audio_output_queue) + case BackendEventType.TURN_END: + self._on_turn_end(event) + case BackendEventType.TOOL_CALL_REQUEST: + await self._on_tool_call(event, session) + case BackendEventType.ERROR: + logger.error(f"Backend error: {event.error}") + # OUTPUT_TURN_STARTED / OUTPUT_AUDIO_DONE: not needed by the assistant. + + # ── Event handlers (consume only normalized fields) ─────────────── + + def _on_speech_started(self) -> None: + self._user_speaking = True + # Start a new user turn only if the previous one was flushed (preserves the + # original timestamp when VAD fires multiple speech_started per utterance). + if not self._user_turn or self._user_turn.flushed: + start_ts = self._audio_interface_speech_start_ts or _wall_ms() + self._user_turn = _UserTurnRecord(speech_started_wall_ms=start_ts) + if self._fw_log: + self._fw_log.turn_start(timestamp_ms=int(start_ts)) + self._audio_interface_speech_start_ts = None + + def _on_speech_stopped(self) -> None: + self._user_speaking = False + wall = _wall_ms() + if self._user_turn: + self._user_turn.speech_stopped_wall_ms = wall + else: + self._user_turn = _UserTurnRecord(speech_stopped_wall_ms=wall) + + def _on_transcript(self, event: BackendEvent) -> None: + # Assistant records only the inbound (user) transcript; the outbound + # transcript arrives finalized on TURN_END. + if event.metadata.get("stream") != "input": + return + if event.metadata.get("failed"): + if self._user_turn and not self._user_turn.flushed: + ts = self._user_turn.speech_started_wall_ms or None + self.audit_log.append_user_input("[user speech - transcription unavailable]", timestamp_ms=ts) + self._user_turn.flushed = True + return + transcript = (event.transcript or "").strip() + if not transcript: + return + ts = None + if self._user_turn: + ts = self._user_turn.speech_started_wall_ms or None + self._user_turn.transcript = transcript + self._user_turn.flushed = True + self.audit_log.append_user_input(transcript, timestamp_ms=ts) + + async def _on_audio_output(self, pcm16_bytes: bytes, audio_output_queue: asyncio.Queue[bytes]) -> None: + if not pcm16_bytes: + return + if self._assistant_turn.first_audio_wall_ms is None: + self._assistant_turn.first_audio_wall_ms = _wall_ms() + self._assistant_turn.responding = True + self._bot_speaking = True + # Model response latency: user speech end -> first audio chunk. + if self._user_turn and self._user_turn.speech_stopped_wall_ms and self._metrics_log: + latency_ms = int(self._assistant_turn.first_audio_wall_ms) - int(self._user_turn.speech_stopped_wall_ms) + if 0 < latency_ms < 30_000: + self._metrics_log.write_latency("model_response", latency_ms / 1000, self._model) + + # Skip the user-track pad while the user track is actively receiving audio. + user_recently_active = (time.monotonic() - self._last_user_audio_mono) <= USER_ACTIVE_GUARD_S + if not self._user_speaking and not user_recently_active: + sync_buffer_to_position(self.user_audio_buffer, len(self.assistant_audio_buffer)) + self.record_audio("assistant", pcm16_bytes) + self._assistant_turn.audio_was_streamed = True + + try: + mulaw_bytes = pcm16_24k_to_mulaw_8k(pcm16_bytes) + offset = 0 + while offset < len(mulaw_bytes): + await audio_output_queue.put(mulaw_bytes[offset : offset + MULAW_CHUNK_SIZE]) + offset += MULAW_CHUNK_SIZE + except Exception as e: + logger.error(f"Error converting audio for output queue: {e}") + + def _on_turn_end(self, event: BackendEvent) -> None: + meta = event.metadata + usage = meta.get("usage") + if usage and self._metrics_log: + self._metrics_log.write_token_usage( + processor="openai_realtime", + model=self._model, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + ) + + content = (event.transcript or "").strip() + + if meta.get("interrupted"): + if content: + text = content + " [interrupted]" + self.audit_log.append_assistant_output(text, timestamp_ms=self._assistant_turn.first_audio_wall_ms) + if self._fw_log: + self._fw_log.s2s_transcript(text) + self._fw_log.turn_end(was_interrupted=True) + self._reset_assistant_turn() + return + + if meta.get("cancelled"): + self._reset_assistant_turn() + return + + has_fc = bool(meta.get("has_function_calls")) + audio_was_streamed = self._assistant_turn.audio_was_streamed + + # Skip rules (unchanged from the s2s server): tool-call-only, mixed-no-audio, + # audio-without-transcript, and empty turns are not logged as assistant output. + if (not content and has_fc) or (content and not audio_was_streamed and has_fc) or not content: + self._reset_assistant_turn() + return + + timestamp = self._assistant_turn.first_audio_wall_ms or _wall_ms() + self.audit_log.append_assistant_output(content, timestamp_ms=timestamp) + if self._fw_log: + self._fw_log.llm_response(content) + self._fw_log.turn_end(was_interrupted=False) + self._reset_assistant_turn() + + def _reset_assistant_turn(self) -> None: + if self._assistant_turn.first_audio_wall_ms is not None: + self._bot_speaking = False + self._assistant_turn = _AssistantTurnState() - Mirrors ``AbstractAssistantServer.get_final_scenario_db()``. - """ - ... + async def _on_tool_call(self, event: BackendEvent, session: Any) -> None: + request = event.tool_call_request + assert request is not None + logger.info(f"Tool call: {request.name}({json.dumps(request.arguments, ensure_ascii=False)})") + result = await self.handle_tool_call_request(request) + if self._fw_log: + self._fw_log.write( + "tool_call", + { + "frame": "tool_call", + "tool_name": request.name, + "arguments": request.arguments, + "result": result.result, + }, + ) + await self.backend.send(session, tool_result=result) diff --git a/src/eva/role/base.py b/src/eva/role/base.py index a75b4fa1..26298938 100644 --- a/src/eva/role/base.py +++ b/src/eva/role/base.py @@ -1,8 +1,8 @@ """Abstract ``Role`` base contract: prompt/tools/goal ownership over a ``Backend``. -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). Every -method body here is a stub -- this module defines shapes, not behavior, and -is not imported by any existing code path. +See docs/refactor-step1.md. Abstract base shared by the concrete +``AssistantRole`` / ``UserRole``, which the worker constructs for any provider +the ``BackendFactory`` supports. This module holds only the shared ``Role`` base. The two concrete roles live in sibling modules -- ``AssistantRole`` in ``eva.role.assistant`` and @@ -12,40 +12,37 @@ separate files keeps each phase's diff scoped to one role. Design choice -- one ``Role`` base with ``AssistantRole``/``UserRole`` -subclasses, rather than two unrelated ABCs: - Both roles share an identical *control loop* shape: construct a backend - via ``BackendFactory``, ``build_prompt()`` before opening it, drive - ``backend.receive()`` and dispatch tool-call requests to - ``handle_tool_call_request()``, and record recorded audio/transcript for - output. What differs between them is only the *data* they carry (agent - config + tool catalog for the assistant; goal + persona + starting - utterance for the user) and how they decide the conversation is over. - That's a difference in constructor args and a couple of abstract methods, - not in control flow -- so one shared base with two thin subclasses avoids - duplicating the event loop, while still keeping tool-ownership and - prompt-building role-specific via abstract methods. If the two roles' - control loops diverge significantly in a later phase, splitting them - apart is a mechanical extraction of ``Role`` into two ABCs -- nothing - here should make that harder. +subclasses: + The two roles share the *seams* that don't depend on transport direction: + ``build_prompt()`` (instructions handed to the backend at open-time), + ``handle_tool_call_request()`` (role-side tool execution), and + ``record_audio()`` (accumulating audio for output). Those live here. + + Their *lifecycle* does differ today, and deliberately so: the assistant is + a WebSocket **server** the user connects to (passive; ``start()`` / + ``stop()``), while the user is the **driver** that dials in and runs the + conversation to completion (``run()``). That asymmetry is a consequence of + there being no mediator yet (docs/refactor-step1.md keeps ``Backend`` + direction-agnostic precisely so a later mediator can absorb the transport + and re-symmetrize the two roles). Rather than force an ill-fitting uniform + ``run()`` onto the server-shaped assistant, the lifecycle entry points live + on the subclasses (``AssistantRole`` / ``UserRole``); the scaffold + anticipated this ("if the two roles' control loops diverge, splitting is a + mechanical extraction"). When the mediator lands, both sides can converge + on a single driven loop. """ from __future__ import annotations from abc import ABC, abstractmethod -from pathlib import Path -from typing import Any from eva.backend.base import Backend, ToolCallRequest, ToolCallResult -from eva.backend.factory import BackendFactory class Role(ABC): - """Owns prompt, tools/goal, and a runtime-created ``Backend``. + """Owns prompt, tools/goal, and drives a worker-injected ``Backend``. - A ``Role`` is the thing that used to be split across - ``AbstractAssistantServer`` (assistant side) and ``AbstractUserSimulator`` - (user side): everything that is *not* pure provider API exchange lives - here instead of in ``Backend``. In particular: + A ``Role`` controls everything that is *not* pure provider API exchange but is specific to a side of the conversation. - Tool execution stays role-side (per docs/refactor-step1.md): a ``Role`` is responsible for turning a ``ToolCallRequest`` surfaced by its @@ -65,20 +62,22 @@ class declares the seam (``record_audio`` / ``save_outputs``) but does not implement the shared helper itself; that helper is later work. """ - def __init__(self, *, backend_factory: BackendFactory, backend_name: str, backend_config: dict[str, Any]) -> None: - """Construct the role's backend (but do not open its session yet). + def __init__(self, *, backend: Backend) -> None: + """Take the (not-yet-opened) backend the role will drive. + + The role does **not** construct its own backend and knows nothing about + the ``BackendFactory``. The worker owns the factory, calls + ``factory.create(name, config)``, and injects the resulting ``Backend`` + here. This keeps backend selection/configuration a worker concern and + lets the same backend be wired to either role. Args: - backend_factory: Factory used to construct ``self.backend``. - backend_name: Provider name passed through to - ``BackendFactory.create``. - backend_config: Provider-specific config passed through to - ``BackendFactory.create`` (not to be confused with the - ``config`` argument of ``Backend.open``, which is also - provider-specific but may be augmented by the role at - open-time, e.g. with a resolved sample rate). + backend: A constructed, not-yet-opened ``Backend`` (see + ``BackendFactory.create``). The role opens a session on it in + ``run()`` and holds the returned ``BackendSession`` handle; + per-exchange state lives on that handle, not on the backend. """ - self.backend: Backend = backend_factory.create(backend_name, backend_config) + self.backend = backend @abstractmethod def build_prompt(self) -> str: @@ -104,30 +103,6 @@ class docstring). Implementations should log the call/result (e.g. """ ... - @abstractmethod - async def run(self) -> str: - """Drive the conversation for this role until it reaches a terminal state. - - Expected shape (left to subclasses to implement, not prescribed in - detail here since the exact loop depends on the backend's - capabilities -- see ``BackendCapabilities``): - 1. ``await self.backend.open(system_prompt=self.build_prompt(), ...)`` - 2. Iterate ``self.backend.receive()``, dispatching - ``TOOL_CALL_REQUEST`` events to ``handle_tool_call_request`` and - feeding the ``ToolCallResult`` back via - ``self.backend.send(tool_result=...)``. - 3. Record audio/transcript events as they arrive (see - ``record_audio``). - 4. On a terminal event (hangup, timeout, transfer, error), call - ``await self.backend.close()`` and return an end-reason string. - - Returns: - A short end-reason string (e.g. ``"goodbye"``, ``"transfer"``, - ``"timeout"``, ``"error"``) -- mirrors the return contract of - today's ``AbstractUserSimulator.run_conversation()``. - """ - ... - @abstractmethod def record_audio(self, source: str, audio_data: bytes) -> None: """Accumulate a chunk of audio for later persistence. @@ -142,18 +117,3 @@ def record_audio(self, source: str, audio_data: bytes) -> None: audio_data: Raw PCM16 bytes at this role's recording sample rate. """ ... - - @abstractmethod - async def save_outputs(self, output_dir: Path) -> None: - """Persist this role's output artifacts to ``output_dir``. - - For ``AssistantRole`` this covers ``audit_log.json``, - ``transcript.jsonl``, scenario DB snapshots (mirrors - ``AbstractAssistantServer.save_outputs``). For ``UserRole`` this - covers ``user_simulator_events.jsonl`` (mirrors the event logger in - ``AbstractUserSimulator``). Audio WAV files are expected to be - written by the shared audio-recording helper referenced in - ``record_audio``, not necessarily by this method -- exact division of - labor is left to the later implementation phase. - """ - ... diff --git a/src/eva/role/user.py b/src/eva/role/user.py deleted file mode 100644 index 415d8b7a..00000000 --- a/src/eva/role/user.py +++ /dev/null @@ -1,83 +0,0 @@ -"""``UserRole`` contract: the simulated-caller role. - -DESIGN ONLY (Step 1 of the refactor, see docs/refactor-step1.md). Method -bodies are stubs; nothing here is wired into the existing code path yet. - -Plug-in point (where this will eventually replace existing code): - Today the user side is a concrete ``AbstractUserSimulator`` subclass - selected by ``eva.user_simulator.factory.create_user_simulator(config, ...)`` - and constructed inside ``ConversationWorker._start_user_simulator()`` - (worker.py), which passes it ``server_url=f"ws://localhost:{port}/ws"`` - to reach the assistant server. The conversation is driven by - ``ConversationWorker._run_conversation()`` calling - ``user_simulator.run_conversation()``, whose returned end-reason string - becomes the conversation result. - - In a later phase, ``_start_user_simulator()`` becomes the construction - site for a ``UserRole`` (simulator config -> ``backend_name`` + - ``backend_config`` for the ``BackendFactory``), and the worker drives - ``role.run()`` (returning the same end-reason string via - ``get_end_reason()``) instead of ``run_conversation()``. Note the - ``server_url`` handoff is a *transport* detail that today's user side owns - directly; per docs/refactor-step1.md the ``Backend`` contract is kept - direction-agnostic precisely so this WS-connect concern can move into a - ``Backend`` implementation (or, later, a mediator) without the role - caring. This module is deliberately separate from ``eva.role.assistant`` - so the user-side migration can land as its own scoped diff. -""" - -from __future__ import annotations - -from abc import abstractmethod -from typing import Any - -from eva.backend.factory import BackendFactory -from eva.role.base import Role - - -class UserRole(Role): - """Role that simulates the human caller (today's "user simulator"). - - Carries goal/persona instead of agent config/tools -- its tool surface, - if any, is limited to caller-side affordances like ``end_call`` (see - ``END_CALL_DESCRIPTION`` in today's ``eva.user_simulator.base``), not a - business tool catalog. - """ - - def __init__( - self, - *, - backend_factory: BackendFactory, - backend_name: str, - backend_config: dict[str, Any], - goal: dict[str, Any], - persona_config: dict[str, Any], - current_date_time: str, - ) -> None: - """Initialize the user role. - - Args: - backend_factory: Factory used to construct the backend. - backend_name: Key passed to the factory to select a backend. - backend_config: Provider-specific configuration for the backend. - goal: User goal / decision-tree data -- mirrors - ``AbstractUserSimulator.goal``. - persona_config: Persona/voice/behavior configuration -- mirrors - ``AbstractUserSimulator.persona_config``. - current_date_time: Threaded into prompt construction, mirroring - existing plumbing. - """ - super().__init__(backend_factory=backend_factory, backend_name=backend_name, backend_config=backend_config) - self.goal = goal - self.persona_config = persona_config - self.current_date_time = current_date_time - - @abstractmethod - def get_end_reason(self) -> str: - """Return the terminal end-reason for this conversation. - - Mirrors the return value of today's - ``AbstractUserSimulator.run_conversation()`` (``"goodbye"``, - ``"transfer"``, ``"timeout"``, ``"error"``, ...). - """ - ... diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 1f1b72b4..dadf84af 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -74,7 +74,7 @@ "TaskCompletion": { "name": "task_completion", "prompt_hash": null, - "source_hash": "01aed1a552f4", + "source_hash": "8d9872e60b10", "version": "v0.1" }, "ToolCallValidity": { diff --git a/tests/unit/metrics/test_aggregation.py b/tests/unit/metrics/test_aggregation.py index 3f3a2d9a..b7d9d481 100644 --- a/tests/unit/metrics/test_aggregation.py +++ b/tests/unit/metrics/test_aggregation.py @@ -714,3 +714,78 @@ def test_per_metric_seed_propagation(self): assert ( entry_a["mean_ci_lower"] != entry_b["mean_ci_lower"] or entry_a["mean_ci_upper"] != entry_b["mean_ci_upper"] ) + + +class TestEfficiencyAggregates: + def _record(self, record_id: str, **metric_scores: MetricScore) -> RecordMetrics: + return RecordMetrics(record_id=record_id, metrics=dict(metric_scores)) + + def test_efficiency_mean_over_completed_records(self): + """Efficiency block reports mean/min/max over records where the metric was recorded.""" + r1 = self._record( + "1", + time_to_completion=MetricScore(name="time_to_completion", score=10.0), + turns_to_completion=MetricScore(name="turns_to_completion", score=4.0), + ) + r2 = self._record( + "2", + time_to_completion=MetricScore(name="time_to_completion", score=20.0), + turns_to_completion=MetricScore(name="turns_to_completion", score=8.0), + ) + + result = compute_run_level_aggregates({"1": r1, "2": r2}, seed=42) + + assert "efficiency" in result + ttc = result["efficiency"]["time_to_completion"] + assert ttc["mean"] == 15.0 + assert ttc["min"] == 10.0 + assert ttc["max"] == 20.0 + assert ttc["count"] == 2 + assert ttc["total_records"] == 2 + assert ttc["higher_is_better"] is False + + turns = result["efficiency"]["turns_to_completion"] + assert turns["mean"] == 6.0 + assert turns["count"] == 2 + + def test_skipped_and_errored_records_excluded(self): + """Skipped (task not completed) and errored records are excluded from the mean.""" + r1 = self._record( + "1", + time_to_completion=MetricScore(name="time_to_completion", score=10.0), + ) + r2 = self._record( + "2", + time_to_completion=MetricScore(name="time_to_completion", score=None, skipped=True), + ) + r3 = self._record( + "3", + time_to_completion=MetricScore(name="time_to_completion", score=0.0, error="boom"), + ) + + result = compute_run_level_aggregates({"1": r1, "2": r2, "3": r3}, seed=42) + + ttc = result["efficiency"]["time_to_completion"] + assert ttc["mean"] == 10.0 + assert ttc["count"] == 1 + assert ttc["total_records"] == 3 + + def test_no_efficiency_metrics_recorded(self): + """When no efficiency metrics are present, no efficiency block is emitted.""" + r1 = make_record_metrics({"task_completion": 1.0}, record_id="1") + r1.aggregate_metrics = compute_record_aggregates(r1) + + result = compute_run_level_aggregates({"1": r1}, seed=42) + + assert "efficiency" not in result + + def test_all_records_skipped_no_block(self): + """If every efficiency record is skipped, the metric is omitted from the block.""" + r1 = self._record( + "1", + time_to_completion=MetricScore(name="time_to_completion", score=None, skipped=True), + ) + + result = compute_run_level_aggregates({"1": r1}, seed=42) + + assert "efficiency" not in result diff --git a/tests/unit/metrics/test_time_to_completion.py b/tests/unit/metrics/test_time_to_completion.py new file mode 100644 index 00000000..8839feb5 --- /dev/null +++ b/tests/unit/metrics/test_time_to_completion.py @@ -0,0 +1,94 @@ +"""Tests for the TimeToCompletionMetric.""" + +import pytest + +from eva.metrics.diagnostic.time_to_completion import TimeToCompletionMetric +from eva.utils.hash_utils import get_dict_hash + +from .conftest import make_metric_context + + +class TestTimeToCompletionMetric: + def setup_method(self): + self.metric = TimeToCompletionMetric() + + @pytest.mark.asyncio + async def test_completed_task_reports_duration(self): + """When the task completed, reports the total duration in seconds.""" + db = {"reservations": {"ABC": {"status": "confirmed"}}} + ctx = make_metric_context( + expected_scenario_db=db, + final_scenario_db=db, + final_scenario_db_hash=get_dict_hash(db), + duration_seconds=42.5, + ) + + result = await self.metric.compute(ctx) + + assert result.name == "time_to_completion" + assert result.score == pytest.approx(42.5) + assert result.normalized_score is None + assert result.error is None + assert result.skipped is False + assert result.details["task_completed"] is True + assert result.details["duration_seconds"] == pytest.approx(42.5) + + @pytest.mark.asyncio + async def test_incomplete_task_is_skipped(self): + """When the task did not complete (hash mismatch), the metric is skipped.""" + expected_db = {"reservations": {"ABC": {"status": "confirmed"}}} + actual_db = {"reservations": {"ABC": {"status": "cancelled"}}} + ctx = make_metric_context( + expected_scenario_db=expected_db, + final_scenario_db=actual_db, + final_scenario_db_hash=get_dict_hash(actual_db), + duration_seconds=42.5, + ) + + result = await self.metric.compute(ctx) + + assert result.score is None + assert result.normalized_score is None + assert result.error is None + assert result.skipped is True + assert result.details["task_completed"] is False + + @pytest.mark.asyncio + async def test_auth_failure_is_skipped(self): + """Auth failure means the task is not completed, so the metric is skipped.""" + db = {"reservations": {"ABC": {"status": "confirmed"}}} + ctx = make_metric_context( + expected_scenario_db={**db, "session": {"confirmation_number": "ABC", "last_name": "doe"}}, + final_scenario_db={**db, "session": {"confirmation_number": "ABC", "last_name": "wrong"}}, + final_scenario_db_hash=get_dict_hash(db), + duration_seconds=42.5, + ) + + result = await self.metric.compute(ctx) + + assert result.skipped is True + assert result.details["task_completed"] is False + assert "Authentication failed" in result.details["reason"] + + @pytest.mark.asyncio + async def test_completed_but_no_duration_is_skipped(self): + """Completed task with a non-positive duration is skipped rather than scored.""" + db = {"reservations": {"ABC": {"status": "confirmed"}}} + ctx = make_metric_context( + expected_scenario_db=db, + final_scenario_db=db, + final_scenario_db_hash=get_dict_hash(db), + duration_seconds=0.0, + ) + + result = await self.metric.compute(ctx) + + assert result.score is None + assert result.skipped is True + assert result.details["task_completed"] is True + + def test_metric_attributes(self): + assert self.metric.name == "time_to_completion" + assert self.metric.category == "diagnostic" + assert self.metric.exclude_from_pass_at_k is True + assert self.metric.higher_is_better is False diff --git a/tests/unit/metrics/test_turns_to_completion.py b/tests/unit/metrics/test_turns_to_completion.py new file mode 100644 index 00000000..c308a9f7 --- /dev/null +++ b/tests/unit/metrics/test_turns_to_completion.py @@ -0,0 +1,94 @@ +"""Tests for the TurnsToCompletionMetric.""" + +import pytest + +from eva.metrics.diagnostic.turns_to_completion import TurnsToCompletionMetric +from eva.utils.hash_utils import get_dict_hash + +from .conftest import make_metric_context + + +class TestTurnsToCompletionMetric: + def setup_method(self): + self.metric = TurnsToCompletionMetric() + + @pytest.mark.asyncio + async def test_completed_task_reports_turns(self): + """When the task completed, reports the total number of turns.""" + db = {"reservations": {"ABC": {"status": "confirmed"}}} + ctx = make_metric_context( + expected_scenario_db=db, + final_scenario_db=db, + final_scenario_db_hash=get_dict_hash(db), + num_turns=8, + ) + + result = await self.metric.compute(ctx) + + assert result.name == "turns_to_completion" + assert result.score == pytest.approx(8.0) + assert result.normalized_score is None + assert result.error is None + assert result.skipped is False + assert result.details["task_completed"] is True + assert result.details["num_turns"] == 8 + + @pytest.mark.asyncio + async def test_incomplete_task_is_skipped(self): + """When the task did not complete (hash mismatch), the metric is skipped.""" + expected_db = {"reservations": {"ABC": {"status": "confirmed"}}} + actual_db = {"reservations": {"ABC": {"status": "cancelled"}}} + ctx = make_metric_context( + expected_scenario_db=expected_db, + final_scenario_db=actual_db, + final_scenario_db_hash=get_dict_hash(actual_db), + num_turns=8, + ) + + result = await self.metric.compute(ctx) + + assert result.score is None + assert result.normalized_score is None + assert result.error is None + assert result.skipped is True + assert result.details["task_completed"] is False + + @pytest.mark.asyncio + async def test_auth_failure_is_skipped(self): + """Auth failure means the task is not completed, so the metric is skipped.""" + db = {"reservations": {"ABC": {"status": "confirmed"}}} + ctx = make_metric_context( + expected_scenario_db={**db, "session": {"confirmation_number": "ABC", "last_name": "doe"}}, + final_scenario_db={**db, "session": {"confirmation_number": "ABC", "last_name": "wrong"}}, + final_scenario_db_hash=get_dict_hash(db), + num_turns=8, + ) + + result = await self.metric.compute(ctx) + + assert result.skipped is True + assert result.details["task_completed"] is False + assert "Authentication failed" in result.details["reason"] + + @pytest.mark.asyncio + async def test_completed_but_no_turns_is_skipped(self): + """Completed task with zero turns is skipped rather than scored.""" + db = {"reservations": {"ABC": {"status": "confirmed"}}} + ctx = make_metric_context( + expected_scenario_db=db, + final_scenario_db=db, + final_scenario_db_hash=get_dict_hash(db), + num_turns=0, + ) + + result = await self.metric.compute(ctx) + + assert result.score is None + assert result.skipped is True + assert result.details["task_completed"] is True + + def test_metric_attributes(self): + assert self.metric.name == "turns_to_completion" + assert self.metric.category == "diagnostic" + assert self.metric.exclude_from_pass_at_k is True + assert self.metric.higher_is_better is False diff --git a/tests/unit/orchestrator/test_framework_dispatch.py b/tests/unit/orchestrator/test_framework_dispatch.py index 82884115..7bcfed12 100644 --- a/tests/unit/orchestrator/test_framework_dispatch.py +++ b/tests/unit/orchestrator/test_framework_dispatch.py @@ -1,22 +1,38 @@ -"""Verify framework dispatcher returns the right server class.""" - -import pytest - -from eva.assistant.grok_voice_server import GrokVoiceAssistantServer -from eva.assistant.openai_realtime_server import OpenAIRealtimeAssistantServer -from eva.orchestrator.worker import _get_server_class - - -def test_grok_voice_dispatch_returns_grok_class(): - cls = _get_server_class("grok_voice") - assert cls is GrokVoiceAssistantServer - - -def test_grok_voice_is_subclass_of_openai_realtime(): - assert issubclass(GrokVoiceAssistantServer, OpenAIRealtimeAssistantServer) - - -def test_unknown_framework_error_lists_grok_voice(): - with pytest.raises(ValueError) as exc_info: - _get_server_class("nope") - assert "grok_voice" in str(exc_info.value) +"""Assistant framework dispatch: backend-only via the BackendFactory. + +The assistant runs on the Role/Backend path exclusively — the legacy +``_get_server_class`` wiring is gone. A framework the factory backs is usable; +anything else is unported (its legacy server survives only as reference) and the +factory returns ``None`` for it. +""" + +from eva.backend.elevenlabs import ElevenLabsBackend +from eva.backend.factory import BackendFactory +from eva.backend.gemini_live import GeminiLiveBackend +from eva.backend.grok_voice import GrokVoiceBackend +from eva.backend.openai_realtime import OpenAIRealtimeBackend + +_PORTED = { + "openai_realtime": OpenAIRealtimeBackend, + "grok_voice": GrokVoiceBackend, + "elevenlabs": ElevenLabsBackend, + "gemini_live": GeminiLiveBackend, +} + +_MINIMAL_CONFIG = { + "openai_realtime": {"model": "gpt-realtime", "api_key": "k"}, + "grok_voice": {"model": "grok-voice", "api_key": "k"}, + "elevenlabs": {"api_key": "k", "speaker_id": "ag_1"}, + "gemini_live": {"model": "gemini-live-2.5-flash", "api_key": "k"}, +} + + +def test_create_builds_each_ported_backend(): + factory = BackendFactory() + for name, cls in _PORTED.items(): + assert isinstance(factory.create(name, _MINIMAL_CONFIG[name]), cls) + + +def test_create_returns_none_for_unported_frameworks(): + # pipecat cascade is the last framework not yet a native backend -> unusable as assistant. + assert BackendFactory().create("pipecat", {}) is None diff --git a/tests/unit/orchestrator/test_preflight.py b/tests/unit/orchestrator/test_preflight.py index 1510583a..7bfee0dd 100644 --- a/tests/unit/orchestrator/test_preflight.py +++ b/tests/unit/orchestrator/test_preflight.py @@ -120,6 +120,90 @@ async def test_s2s_is_skipped(tmp_path): assert results == [] +# ── Cheap backend-construction validation (_preflight_backends) ────────────── + + +def test_backend_construction_validates_s2s_assistant_missing_key(tmp_path): + # S2S key is no longer validated at config load (removed); construction catches it. + with patch.dict(os.environ, _BASE_ENV, clear=True): # no OPENAI_API_KEY + cfg = RunConfig( + model=ModelConfig(s2s="gpt-realtime", s2s_params={"model": "gpt-realtime"}), + framework="openai_realtime", + output_dir=tmp_path / "out", + run_id="r", + ) + with pytest.raises(PreflightError, match="assistant framework 'openai_realtime'"): + preflight._preflight_backends(cfg) + + +def test_backend_construction_passes_with_key(tmp_path): + with patch.dict(os.environ, _BASE_ENV | {"OPENAI_API_KEY": "k"}, clear=True): + cfg = RunConfig( + model=ModelConfig(s2s="gpt-realtime", s2s_params={"api_key": "k", "model": "gpt-realtime"}), + framework="openai_realtime", + output_dir=tmp_path / "out", + run_id="r", + ) + preflight._preflight_backends(cfg) # no raise + + +def test_backend_construction_rejects_unported_assistant(tmp_path): + # The assistant is backend-only now: a cascade (pipecat) framework isn't a native + # backend, so it's unusable and rejected up front rather than silently skipped. + with patch.dict(os.environ, _BASE_ENV, clear=True): + with pytest.raises(PreflightError, match="not available as a native backend"): + preflight._preflight_backends(_cascade_config(tmp_path)) + + +def _elevenlabs_config(tmp_path, s2s_params) -> RunConfig: + with patch.dict(os.environ, _BASE_ENV, clear=True): + return RunConfig( + model=ModelConfig(s2s="elevenlabs", s2s_params=s2s_params), + framework="elevenlabs", + output_dir=tmp_path / "out", + run_id="r", + ) + + +def test_backend_construction_validates_elevenlabs_missing_agent_id(tmp_path): + # ElevenLabs is CASCADE-typed internally but factory-backed -> construction validates it. + cfg = _elevenlabs_config(tmp_path, {"api_key": "k"}) # no speaker_id + with patch.dict(os.environ, _BASE_ENV, clear=True): + with pytest.raises(PreflightError, match="assistant framework 'elevenlabs'"): + preflight._preflight_backends(cfg) + + +def test_backend_construction_passes_for_elevenlabs_with_agent_id(tmp_path): + cfg = _elevenlabs_config(tmp_path, {"api_key": "k", "speaker_id": "ag_1"}) + with patch.dict(os.environ, _BASE_ENV, clear=True): + preflight._preflight_backends(cfg) # no raise + + +@pytest.mark.asyncio +async def test_elevenlabs_skips_live_probes(tmp_path): + # ElevenLabs' pipeline_type property is S2S, so _run_preflight skips live model + # probes (native backends are validated by construction, not a live session probe). + cfg = _elevenlabs_config(tmp_path, {"api_key": "k", "speaker_id": "ag_1"}) + results = await _run_preflight(cfg) + assert results == [] + + +@pytest.mark.asyncio +async def test_backend_construction_runs_even_when_preflight_disabled(tmp_path): + # --no-preflight skips only the live model probes; the cheap S2S-assistant construction + # check still runs (S2S key isn't validated at config load). + with patch.dict(os.environ, _BASE_ENV, clear=True): # no OPENAI_API_KEY + cfg = RunConfig( + model=ModelConfig(s2s="gpt-realtime", s2s_params={"model": "gpt-realtime"}), + framework="openai_realtime", + output_dir=tmp_path / "out", + run_id="r", + ) + cfg.preflight = False + with pytest.raises(PreflightError, match="assistant framework 'openai_realtime'"): + await run_preflight(cfg) + + @pytest.mark.asyncio async def test_guard_times_out(): async def hang(): @@ -131,10 +215,15 @@ async def hang(): assert "timed out" in result.detail +# These exercise run_preflight's or-raise wrapper over the live model probes. The +# cascade probe path is only reachable once a cascade provider is a native backend +# (it's rejected at _preflight_backends until then), so the backend gate is stubbed +# to isolate the wrapper/probe behavior. @pytest.mark.asyncio async def test_or_raise_passes_when_all_ok(tmp_path): cfg = _cascade_config(tmp_path) with ( + patch.object(preflight, "_preflight_backends", lambda c: None), patch.object(preflight, "create_stt_service", return_value=_GoodSTT()) as stt, patch.object(preflight, "create_tts_service", return_value=_GoodTTS()) as tts, patch.object(preflight, "LiteLLMClient") as llm, @@ -150,6 +239,7 @@ async def test_or_raise_passes_when_all_ok(tmp_path): async def test_or_raise_raises_on_failure(tmp_path): cfg = _cascade_config(tmp_path) with ( + patch.object(preflight, "_preflight_backends", lambda c: None), patch.object(preflight, "create_stt_service", return_value=_GoodSTT()), patch.object(preflight, "create_tts_service", return_value=_GoodTTS()), patch.object(preflight, "LiteLLMClient") as llm, @@ -161,8 +251,15 @@ async def test_or_raise_raises_on_failure(tmp_path): @pytest.mark.asyncio async def test_or_raise_noop_when_disabled(tmp_path): - cfg = _cascade_config(tmp_path) + # Disabled preflight skips the live probes (but backend construction still runs). + with patch.dict(os.environ, _BASE_ENV | {"OPENAI_API_KEY": "k"}, clear=True): + cfg = RunConfig( + model=ModelConfig(s2s="gpt-realtime", s2s_params={"api_key": "k", "model": "gpt-realtime"}), + framework="openai_realtime", + output_dir=tmp_path / "out", + run_id="r", + ) cfg.preflight = False - with patch.object(preflight, "run_preflight") as probe: + with patch.object(preflight, "_run_preflight") as probe: await run_preflight(cfg) probe.assert_not_called() diff --git a/tests/unit/orchestrator/test_runner.py b/tests/unit/orchestrator/test_runner.py index 89751b5c..d0486eed 100644 --- a/tests/unit/orchestrator/test_runner.py +++ b/tests/unit/orchestrator/test_runner.py @@ -91,6 +91,51 @@ def test_missing_ids_still_returns_found(self, tmp_path): assert len(filtered) == 1 assert filtered[0].id == "rec-0" + def test_exclude_record_ids_filter(self, tmp_path): + """Excluding specific record IDs removes them from the full set.""" + config = _make_config(tmp_path) + config = config.model_copy(update={"exclude_record_ids": ["rec-1", "rec-3"]}) + runner = _make_runner(config) + + records = [_make_record(f"rec-{i}") for i in range(5)] + filtered = runner._filter_records(records) + + assert {r.id for r in filtered} == {"rec-0", "rec-2", "rec-4"} + + def test_exclude_applied_after_record_ids(self, tmp_path): + """exclude_record_ids is applied on top of the record_ids include filter.""" + config = _make_config(tmp_path) + config = config.model_copy(update={"record_ids": ["rec-1", "rec-2", "rec-3"], "exclude_record_ids": ["rec-2"]}) + runner = _make_runner(config) + + records = [_make_record(f"rec-{i}") for i in range(5)] + filtered = runner._filter_records(records) + + assert {r.id for r in filtered} == {"rec-1", "rec-3"} + + def test_exclude_unknown_id_is_noop(self, tmp_path): + """Excluding an ID not in the dataset leaves all records intact.""" + config = _make_config(tmp_path) + config = config.model_copy(update={"exclude_record_ids": ["rec-99"]}) + runner = _make_runner(config) + + records = [_make_record(f"rec-{i}") for i in range(3)] + filtered = runner._filter_records(records) + + assert len(filtered) == 3 + + def test_debug_takes_precedence_over_exclude(self, tmp_path): + """Debug mode short-circuits before exclusion is applied.""" + config = _make_config(tmp_path) + config = config.model_copy(update={"debug": True, "exclude_record_ids": ["rec-0"]}) + runner = _make_runner(config) + + records = [_make_record(f"rec-{i}") for i in range(5)] + filtered = runner._filter_records(records) + + assert len(filtered) == 1 + assert filtered[0].id == "rec-0" + class TestArchiveFailedAttempt: def test_moves_record_dir_to_archive(self, tmp_path): diff --git a/tests/unit/orchestrator/test_worker.py b/tests/unit/orchestrator/test_worker.py index e5a0366d..7c2d45df 100644 --- a/tests/unit/orchestrator/test_worker.py +++ b/tests/unit/orchestrator/test_worker.py @@ -6,6 +6,7 @@ import pytest +from eva.models.config import ElevenLabsSimulatorConfig from eva.orchestrator.worker import USER_SIMULATOR_SHUTDOWN_GRACE_SECONDS, ConversationWorker, _percentile @@ -222,8 +223,10 @@ async def test_raises_when_simulator_not_initialized(self, tmp_path): class TestUserSimulatorSelection: @pytest.mark.asyncio async def test_worker_uses_configured_factory_and_timeout(self, tmp_path, monkeypatch): + # A non-factory (legacy) provider falls through to create_user_simulator; a + # factory-backed provider (openai_realtime/grok_voice) takes the Role/Backend path. worker = _make_worker(tmp_path) - worker.config.user_simulator = MagicMock(provider="openai_realtime") + worker.config.user_simulator = ElevenLabsSimulatorConfig() worker.config.perturbation = None worker.config.language = "en" worker.config.conversation_time_limit_seconds = 60 diff --git a/tests/unit/test_elevenlabs_backend.py b/tests/unit/test_elevenlabs_backend.py new file mode 100644 index 00000000..141eb604 --- /dev/null +++ b/tests/unit/test_elevenlabs_backend.py @@ -0,0 +1,183 @@ +"""Unit tests for ElevenLabsBackend (no network, no live SDK session). + +Covers the deterministic surfaces: config/api-key validation with the +``ELEVENLABS_API_KEY`` fallback, factory dispatch, sample-rate/capability +exposure, the SDK-callback -> normalized ``BackendEvent`` mapping, the +greeting-unwrap, and the role-side tool-call bridge (a ``ClientTools`` handler +surfaces a ``TOOL_CALL_REQUEST`` and blocks until the role resolves it via +``send(tool_result=...)``). The live ``AsyncConversation`` (open/start over a +real WS) is out of scope. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from eva.backend.base import BackendEventType, ToolCallResult +from eva.backend.elevenlabs import ( + ElevenLabsBackend, + ElevenLabsSession, + _unwrap_greeting, +) +from eva.backend.factory import BackendFactory + + +def _backend(**overrides) -> ElevenLabsBackend: + config = {"model": "elevenlabs", "api_key": "el-key", "speaker_id": "ag_1", **overrides} + return ElevenLabsBackend(config=config) + + +def _session() -> ElevenLabsSession: + return ElevenLabsSession(client=None, bridge=None, system_prompt="p", client_tools=None) + + +def test_requires_api_key(monkeypatch): + monkeypatch.delenv("ELEVENLABS_API_KEY", raising=False) + with pytest.raises(ValueError, match="ELEVENLABS_API_KEY"): + ElevenLabsBackend(config={"speaker_id": "ag_1"}) + + +def test_api_key_falls_back_to_env(monkeypatch): + monkeypatch.setenv("ELEVENLABS_API_KEY", "env-key") + b = ElevenLabsBackend(config={"speaker_id": "ag_1"}) + assert b._api_key == "env-key" + + +def test_requires_speaker_id(): + with pytest.raises(ValueError, match="speaker_id"): + ElevenLabsBackend(config={"api_key": "k"}) + + +def test_multiple_config_errors_are_cumulative(monkeypatch): + # Missing api_key AND speaker_id -> both reported in one raise, not just the first. + monkeypatch.delenv("ELEVENLABS_API_KEY", raising=False) + with pytest.raises(ValueError) as exc: + ElevenLabsBackend(config={}) + message = str(exc.value) + assert "api_key" in message + assert "speaker_id" in message + + +def test_model_defaults_to_elevenlabs(): + assert ElevenLabsBackend(config={"api_key": "k", "speaker_id": "a"})._model == "elevenlabs" + assert _backend(model="custom-label")._model == "custom-label" + + +def test_sample_rates_are_role_rate(): + b = _backend() + # The SDK's 8k/16k rates are hidden; the role sees a uniform 24 kHz in/out. + assert b.input_sample_rate == 24000 + assert b.output_sample_rate == 24000 + + +def test_capabilities(): + caps = _backend().capabilities + assert caps.emits_continuous_audio is True + assert caps.supports_streaming_interruption is False + assert caps.owns_playout_clock is False + + +def test_factory_dispatch(): + b = BackendFactory().create("elevenlabs", {"api_key": "k", "speaker_id": "ag_1"}) + assert isinstance(b, ElevenLabsBackend) + + +# ── Greeting unwrap ────────────────────────────────────────────────────── + + +def test_unwrap_greeting_strips_role_wrapper(): + assert _unwrap_greeting("Say: 'Hello there!'") == "Hello there!" + + +def test_unwrap_greeting_passthrough_when_unwrapped(): + assert _unwrap_greeting("Hello there!") == "Hello there!" + + +# ── Callback -> normalized event mapping ───────────────────────────────── + + +def test_agent_response_event_is_turn_end(): + be = ElevenLabsBackend._agent_response_event(" all done ") + assert be.event_type == BackendEventType.TURN_END + assert be.transcript == "all done" + assert be.metadata == {"interrupted": False, "cancelled": False, "has_function_calls": False, "usage": None} + + +def test_correction_event_is_interrupted_turn_end(): + be = ElevenLabsBackend._correction_event("partial reply") + assert be.event_type == BackendEventType.TURN_END + assert be.transcript == "partial reply" + assert be.metadata["interrupted"] is True + + +def test_user_transcript_event_is_input_transcript(): + be = ElevenLabsBackend._user_transcript_event("hi there") + assert be.event_type == BackendEventType.TRANSCRIPT + assert be.transcript == "hi there" + assert be.metadata == {"stream": "input", "final": True} + + +# ── Tool-call bridge (role-side execution) ─────────────────────────────── + + +@pytest.mark.asyncio +async def test_client_tools_bound_to_running_loop(): + # The SDK must run tool handlers on OUR loop, not a separate thread loop -- otherwise + # the handler touches the main-loop event queue / result future cross-loop, raises, + # and the SDK reports the tool as failed to the agent. + b, s = _backend(), _session() + client_tools = b._build_client_tools(s, [{"name": "get_reservation", "description": "d", "parameters": {}}]) + assert client_tools is not None + assert client_tools._custom_loop is asyncio.get_running_loop() + + +@pytest.mark.asyncio +async def test_tool_call_bridges_to_role_and_awaits_result(): + b, s = _backend(), _session() + + async def _role_resolves() -> None: + # Wait for the handler to enqueue the request, then resolve it like the role. + event = await s.events.get() + assert event.event_type == BackendEventType.TOOL_CALL_REQUEST + req = event.tool_call_request + assert req.name == "get_reservation" + assert req.arguments == {"confirmation_number": "ABC"} # tool_call_id stripped + await b.send(s, tool_result=ToolCallResult(call_id=req.call_id, result={"status": "ok"})) + + resolver = asyncio.create_task(_role_resolves()) + out = await b._bridge_tool_call(s, "get_reservation", {"confirmation_number": "ABC", "tool_call_id": "call-42"}) + await resolver + assert out == '{"status": "ok"}' + assert s.pending_tools == {} # popped on resolution + + +@pytest.mark.asyncio +async def test_send_requires_exactly_one_arg(): + b, s = _backend(), _session() + with pytest.raises(ValueError): + await b.send(s) + with pytest.raises(ValueError): + await b.send(s, audio=b"x", text="y") + + +@pytest.mark.asyncio +async def test_send_wrong_session_type_raises(): + with pytest.raises(TypeError): + await _backend().send(object(), tool_result=ToolCallResult(call_id="c", result={})) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_receive_returns_when_ended_and_drained(): + b, s = _backend(), _session() + s.ended.set() + events = [e async for e in b.receive(s)] + assert events == [] + + +@pytest.mark.asyncio +async def test_close_is_idempotent_and_network_free(): + b, s = _backend(), _session() + await b.close(s) + await b.close(s) diff --git a/tests/unit/test_gemini_live_backend.py b/tests/unit/test_gemini_live_backend.py new file mode 100644 index 00000000..2a6e7eaa --- /dev/null +++ b/tests/unit/test_gemini_live_backend.py @@ -0,0 +1,341 @@ +"""Unit tests for the Gemini Live ``Backend`` (no network). + +Covers the pure surfaces: config validation (model required, accent rejected, +cumulative errors, optional api_key), sample-rate/capability exposure, tool-schema +translation, the stateful ``LiveServerMessage`` -> ``BackendEvent`` mapping +(audio, transcripts, turn completion, interruption, tool calls, usage), factory +dispatch, and ``send()`` validation. The live session is out of scope here. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from eva.backend.base import BackendEventType, ToolCallResult +from eva.backend.factory import BackendFactory +from eva.backend.gemini_live import DEFAULT_VOICE, GeminiLiveBackend, GeminiLiveSession + + +def _backend(**overrides) -> GeminiLiveBackend: + config = {"model": "gemini-live-2.5-flash", "api_key": "k", **overrides} + return GeminiLiveBackend(config=config) + + +def _session() -> GeminiLiveSession: + return GeminiLiveSession(client=None, conn_cm=None, live=None) # type: ignore[arg-type] + + +_BACKEND = _backend() + + +def _map_response(session, response): + return _BACKEND._map_response(session, response) + + +# ── Config validation ──────────────────────────────────────────────────── + + +def test_requires_model(): + with pytest.raises(ValueError, match="model"): + GeminiLiveBackend(config={"api_key": "k"}) + + +def test_accent_is_rejected(): + with pytest.raises(ValueError, match="accent"): + GeminiLiveBackend(config={"model": "m", "accent": "british"}) + + +def test_config_errors_are_cumulative(): + with pytest.raises(ValueError) as exc: + GeminiLiveBackend(config={"accent": "british"}) # missing model AND accent set + msg = str(exc.value) + assert "model" in msg and "accent" in msg + + +def test_api_key_is_optional(monkeypatch): + # Unlike the OpenAI family, Gemini may auth via Vertex/ADC -> no api_key needed to construct. + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + b = GeminiLiveBackend(config={"model": "gemini-live-2.5-flash"}) + assert b._api_key == "" + + +def test_api_key_falls_back_to_google_env(monkeypatch): + monkeypatch.setenv("GOOGLE_API_KEY", "env-key") + assert GeminiLiveBackend(config={"model": "m"})._api_key == "env-key" + + +def test_speaker_id_maps_to_voice_with_default(): + assert _backend()._voice == DEFAULT_VOICE + assert _backend(speaker_id="Puck")._voice == "Puck" + + +def test_language_code_precedence(): + assert _backend(language="en")._language_code == "en" + # explicit language_code wins over the run language + assert _backend(language="en", language_code="fr-FR")._language_code == "fr-FR" + + +def test_sample_rates_from_input_format(): + assert _backend().input_sample_rate == 24000 # pcm: role rate + assert _backend().output_sample_rate == 24000 + assert _backend(input_format="pcmu").input_sample_rate == 8000 + + +def test_capabilities(): + caps = _backend().capabilities + assert caps.emits_continuous_audio is True + assert caps.supports_streaming_interruption is True + assert caps.owns_playout_clock is False + + +def test_factory_dispatch(): + b = BackendFactory().create("gemini_live", {"model": "m", "api_key": "k"}) + assert isinstance(b, GeminiLiveBackend) + + +# ── Tool-schema translation ────────────────────────────────────────────── + + +def test_format_tools_translates_to_gemini_declarations(): + tools = [ + { + "name": "get_reservation", + "description": "look up", + "parameters": { + "type": "object", + "properties": {"confirmation_number": {"type": "string", "description": "code"}}, + "required": ["confirmation_number"], + }, + } + ] + (tool,) = GeminiLiveBackend._format_tools(tools) + (decl,) = tool.function_declarations + assert decl.name == "get_reservation" + assert "confirmation_number" in decl.parameters.properties + assert decl.parameters.required == ["confirmation_number"] + + +def test_format_tools_none_becomes_none(): + assert GeminiLiveBackend._format_tools(None) is None + assert GeminiLiveBackend._format_tools([]) is None + + +# ── Event mapping ──────────────────────────────────────────────────────── + + +def _server_content(**kwargs): + return SimpleNamespace(server_content=SimpleNamespace(**kwargs), tool_call=None, usage_metadata=None) + + +def test_model_turn_emits_turn_started_then_audio(): + s = _session() + part = SimpleNamespace(inline_data=SimpleNamespace(data=b"\x00\x01\x02\x03\x04\x05")) + events = _map_response(s, _server_content(model_turn=SimpleNamespace(parts=[part]))) + assert [e.event_type for e in events] == [BackendEventType.OUTPUT_TURN_STARTED, BackendEventType.AUDIO_OUTPUT] + assert events[1].audio == b"\x00\x01\x02\x03\x04\x05" + assert s.in_model_turn is True + # A second model_turn chunk does not re-emit OUTPUT_TURN_STARTED. + events2 = _map_response(s, _server_content(model_turn=SimpleNamespace(parts=[part]))) + assert [e.event_type for e in events2] == [BackendEventType.AUDIO_OUTPUT] + + +def test_input_transcription_emits_input_transcript(): + (be,) = _map_response(_session(), _server_content(input_transcription=SimpleNamespace(text="hi there"))) + assert be.event_type == BackendEventType.TRANSCRIPT + assert be.transcript == "hi there" + assert be.metadata == {"stream": "input", "final": True} + + +def test_output_transcription_accumulates_then_turn_complete_emits(): + s = _session() + assert _map_response(s, _server_content(output_transcription=SimpleNamespace(text="Hello"))) == [] + assert _map_response(s, _server_content(output_transcription=SimpleNamespace(text="there"))) == [] + events = _map_response(s, _server_content(turn_complete=True)) + types_ = [e.event_type for e in events] + assert types_ == [BackendEventType.TRANSCRIPT, BackendEventType.OUTPUT_AUDIO_DONE, BackendEventType.TURN_END] + assert events[0].transcript == "Hello there" + assert events[0].metadata == {"stream": "output", "final": True} + assert events[-1].metadata["interrupted"] is False + assert s.in_model_turn is False # reset + + +def test_interrupted_emits_turn_end_and_resets(): + s = _session() + _map_response(s, _server_content(output_transcription=SimpleNamespace(text="partial"))) + (be,) = _map_response(s, _server_content(interrupted=True)) + assert be.event_type == BackendEventType.TURN_END + assert be.transcript == "partial" + assert be.metadata["interrupted"] is True + assert s.output_transcript_parts == [] + + +def test_tool_call_emits_request_and_records_name(): + s = _session() + fc = SimpleNamespace(id="c1", name="get_reservation", args={"confirmation_number": "ABC"}) + msg = SimpleNamespace(server_content=None, tool_call=SimpleNamespace(function_calls=[fc]), usage_metadata=None) + (be,) = _map_response(s, msg) + assert be.event_type == BackendEventType.TOOL_CALL_REQUEST + assert be.tool_call_request.call_id == "c1" + assert be.tool_call_request.arguments == {"confirmation_number": "ABC"} + assert s.tool_names["c1"] == "get_reservation" + assert s.has_function_calls is True + + +def test_usage_metadata_captured_and_surfaced_on_turn_end(): + s = _session() + usage_msg = SimpleNamespace( + server_content=None, + tool_call=None, + usage_metadata=SimpleNamespace(prompt_token_count=11, candidates_token_count=7), + ) + _map_response(s, usage_msg) + assert s.usage == {"prompt_tokens": 11, "completion_tokens": 7} + events = _map_response(s, _server_content(turn_complete=True)) # [OUTPUT_AUDIO_DONE, TURN_END] + turn_end = events[-1] + assert turn_end.event_type == BackendEventType.TURN_END + assert turn_end.metadata["usage"] == {"prompt_tokens": 11, "completion_tokens": 7} + + +def test_empty_response_yields_nothing(): + assert _map_response(_session(), SimpleNamespace(server_content=None, tool_call=None, usage_metadata=None)) == [] + + +# ── send() validation ──────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_send_requires_exactly_one_arg(): + b, s = _backend(), _session() + with pytest.raises(ValueError): + await b.send(s) + with pytest.raises(ValueError): + await b.send(s, audio=b"x", text="y") + + +@pytest.mark.asyncio +async def test_send_wrong_session_type_raises(): + with pytest.raises(TypeError): + await _backend().send(object(), tool_result=ToolCallResult(call_id="c", result={})) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_close_is_idempotent_and_network_free(): + b, s = _backend(), _session() + await b.close(s) + await b.close(s) + + +# ── Client routing (Vertex vs Developer API) ───────────────────────────── +# +# Vertex-only Live/S2S preview models must route through Vertex AI (vertexai=True); +# a Developer API key would 404 them, so Vertex must win whenever a project is +# resolvable (or GOOGLE_GENAI_USE_VERTEXAI is set) and the key is ignored then. + + +@pytest.fixture +def mock_genai_client(): + with patch("eva.backend.gemini_live.genai.Client") as mock: + yield mock + + +class TestCreateClientRouting: + def test_vertex_when_project_resolvable_ignores_dev_key(self, mock_genai_client, monkeypatch): + monkeypatch.delenv("GOOGLE_GENAI_USE_VERTEXAI", raising=False) + _backend(project="proj-x", api_key="AIzaSyDEVKEY")._create_client() + _, kwargs = mock_genai_client.call_args + assert kwargs["vertexai"] is True + assert kwargs["project"] == "proj-x" + assert kwargs["location"] == "us-central1" + assert "api_key" not in kwargs + + def test_use_vertexai_flag_forces_vertex(self, mock_genai_client, monkeypatch): + monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "1") + _backend(project="proj-x", api_key="AIzaSyDEVKEY")._create_client() + _, kwargs = mock_genai_client.call_args + assert kwargs["vertexai"] is True + + def test_flag_zero_forces_dev_api_when_key_present(self, mock_genai_client, monkeypatch): + monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "0") + _backend(project="proj-x", api_key="AIzaSyDEVKEY")._create_client() + _, kwargs = mock_genai_client.call_args + assert kwargs.get("api_key") == "AIzaSyDEVKEY" + assert "vertexai" not in kwargs + + def test_vertex_flag_without_project_raises(self, mock_genai_client, monkeypatch): + monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "1") + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + monkeypatch.delenv("VERTEXAI_PROJECT", raising=False) + with pytest.raises(ValueError, match="no project found"): + _backend()._create_client() + + def test_endpoint_and_api_version_passed_via_http_options(self, mock_genai_client, monkeypatch): + monkeypatch.delenv("GOOGLE_GENAI_USE_VERTEXAI", raising=False) + _backend( + project="proj-x", + endpoint="us-central1-aiplatform.googleapis.com", + api_version="v1beta1", + )._create_client() + _, kwargs = mock_genai_client.call_args + http_options = kwargs["http_options"] + assert http_options.base_url == "wss://us-central1-aiplatform.googleapis.com" + assert http_options.api_version == "v1beta1" + + def test_dev_api_when_only_key_and_no_project(self, mock_genai_client, monkeypatch): + monkeypatch.delenv("GOOGLE_GENAI_USE_VERTEXAI", raising=False) + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + monkeypatch.delenv("VERTEXAI_PROJECT", raising=False) + _backend(api_key="AIzaSyDEVKEY")._create_client() + _, kwargs = mock_genai_client.call_args + assert kwargs.get("api_key") == "AIzaSyDEVKEY" + assert "vertexai" not in kwargs + + def test_global_location_forced_to_region(self, monkeypatch): + monkeypatch.delenv("GOOGLE_CLOUD_LOCATION", raising=False) + monkeypatch.delenv("VERTEXAI_LOCATION", raising=False) + assert _backend(location="global")._vertex_location == "us-central1" + + +# ── thinking_config / function_response_scheduling ─────────────────────── + + +def test_thinking_config_defaults_to_empty(): + # No thinking_config -> a default ThinkingConfig is still attached to the live config. + cfg = _backend()._build_live_config("sys", None) + assert cfg.thinking_config is not None + + +def test_thinking_config_parses_flat_dict(): + b = _backend(thinking_config={"thinking_budget": 1024, "include_thoughts": False}) + assert b._thinking_config.thinking_budget == 1024 + assert b._thinking_config.include_thoughts is False + + +@pytest.mark.asyncio +async def test_scheduling_omitted_by_default(monkeypatch): + b, s = _backend(), _session() + captured = {} + + async def fake_send(function_responses): + captured["fr"] = function_responses[0] + + s.live = SimpleNamespace(send_tool_response=fake_send) + s.tool_names["c1"] = "get_reservation" + await b.send(s, tool_result=ToolCallResult(call_id="c1", result={"ok": True})) + assert captured["fr"].scheduling is None + + +@pytest.mark.asyncio +async def test_scheduling_set_when_configured(): + b, s = _backend(function_response_scheduling="WHEN_IDLE"), _session() + captured = {} + + async def fake_send(function_responses): + captured["fr"] = function_responses[0] + + s.live = SimpleNamespace(send_tool_response=fake_send) + s.tool_names["c1"] = "get_reservation" + await b.send(s, tool_result=ToolCallResult(call_id="c1", result={"ok": True})) + assert captured["fr"].scheduling is not None diff --git a/tests/unit/test_grok_voice_backend.py b/tests/unit/test_grok_voice_backend.py new file mode 100644 index 00000000..10828fd5 --- /dev/null +++ b/tests/unit/test_grok_voice_backend.py @@ -0,0 +1,138 @@ +"""Unit tests for GrokVoiceBackend (no network). + +Covers what Grok changes vs the OpenAI Realtime backend it subclasses: xAI +defaults, the required api key, factory dispatch, and the buffered/deferred +input-transcription behavior. Everything else is inherited and covered by +``test_openai_realtime_backend``. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from eva.backend.base import BackendEventType +from eva.backend.factory import BackendFactory +from eva.backend.grok_voice import DEFAULT_VOICE, XAI_REALTIME_BASE_URL, GrokVoiceBackend, GrokVoiceSession +from eva.backend.openai_realtime import OpenAIRealtimeSession + + +def _backend(**overrides) -> GrokVoiceBackend: + return GrokVoiceBackend(config={"model": "grok-voice", "api_key": "xai-key", **overrides}) + + +def _session() -> GrokVoiceSession: + return GrokVoiceSession(client=None, conn_cm=None, conn=None) # type: ignore[arg-type] + + +def _map(session, event): + return GrokVoiceBackend._map_event(session, event) + + +def test_api_key_falls_back_to_xai_env(monkeypatch): + monkeypatch.setenv("XAI_API_KEY", "xai-from-env") + b = GrokVoiceBackend(config={"model": "grok-voice"}) + assert b._api_key == "xai-from-env" + + +def test_openai_env_does_not_satisfy_grok(monkeypatch): + # An OpenAI key is the wrong key for x.ai: it must NOT be used as a fallback. + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "sk-should-not-be-used") + with pytest.raises(ValueError, match="XAI_API_KEY"): + GrokVoiceBackend(config={"model": "grok-voice"}) + + +def test_xai_defaults_applied(): + b = _backend() + assert b._base_url == XAI_REALTIME_BASE_URL + assert b._session_config["audio"]["output"]["voice"] == DEFAULT_VOICE + + +def test_explicit_config_wins_over_defaults(): + b = _backend(base_url="https://custom/v1", speaker_id="ara") + assert b._base_url == "https://custom/v1" + assert b._session_config["audio"]["output"]["voice"] == "ara" + + +def test_open_uses_grok_session_class(): + assert GrokVoiceBackend._SESSION_CLS is GrokVoiceSession + assert issubclass(GrokVoiceSession, OpenAIRealtimeSession) + + +def test_factory_dispatch(): + b = BackendFactory().create("grok_voice", {"model": "grok-voice", "api_key": "xai-key"}) + assert isinstance(b, GrokVoiceBackend) + + +def _completed(text=""): + return SimpleNamespace(type="conversation.item.input_audio_transcription.completed", transcript=text) + + +def _speech_started(): + return SimpleNamespace(type="input_audio_buffer.speech_started") + + +def _response_done(status="completed"): + response = SimpleNamespace(status=status, usage=None, output=[]) + return SimpleNamespace(type="response.done", response=response) + + +def _transcripts(events): + return [e.transcript for e in events if e.event_type == BackendEventType.TRANSCRIPT] + + +def test_cumulative_completed_is_buffered_not_emitted(): + # xAI re-sends the whole cumulative transcript each time; nothing is emitted until + # the turn boundary flush, so the role logs one user turn (not one per fragment). + s = _session() + assert _map(s, _completed("Hi, I need to change my")) == [] + assert _map(s, _completed("Hi, I need to change my flight to March 20")) == [] + assert _map(s, _completed("Hi, I need to change my flight to March 25th.")) == [] + assert s.pending_input_transcript == "Hi, I need to change my flight to March 25th." + + +def test_flush_on_response_done_emits_latest_cumulative_once(): + s = _session() + _map(s, _completed("Hi, I need to change my")) + _map(s, _completed("Hi, I need to change my flight to March 25th.")) + events = _map(s, _response_done()) + # The buffered transcript is flushed once, before the parent's TURN_END. + assert _transcripts(events) == ["Hi, I need to change my flight to March 25th."] + assert events[0].event_type == BackendEventType.TRANSCRIPT + assert events[-1].event_type == BackendEventType.TURN_END + assert s.pending_input_transcript == "" # cleared after flush + + +def test_flush_on_speech_started_emits_and_precedes_speech_signal(): + s = _session() + _map(s, _completed("First utterance.")) + events = _map(s, _speech_started()) + assert _transcripts(events) == ["First utterance."] + assert events[0].event_type == BackendEventType.TRANSCRIPT + assert events[-1].event_type == BackendEventType.INPUT_SPEECH_STARTED + + +def test_flush_without_buffer_emits_no_transcript(): + s = _session() + assert _transcripts(_map(s, _response_done())) == [] + + +def test_flush_is_not_repeated_after_clear(): + s = _session() + _map(s, _completed("only once")) + assert _transcripts(_map(s, _response_done())) == ["only once"] + assert _transcripts(_map(s, _speech_started())) == [] # buffer already cleared + + +def test_empty_completed_buffers_nothing(): + s = _session() + _map(s, _completed("")) + assert s.pending_input_transcript == "" + + +def test_other_events_delegate_to_parent(): + # AUDIO_OUTPUT still normalized by the inherited handler. + (be,) = _map(_session(), SimpleNamespace(type="response.output_audio.delta", delta="AQIDBA==")) + assert be.event_type == BackendEventType.AUDIO_OUTPUT diff --git a/tests/unit/test_openai_realtime_backend.py b/tests/unit/test_openai_realtime_backend.py new file mode 100644 index 00000000..7302e007 --- /dev/null +++ b/tests/unit/test_openai_realtime_backend.py @@ -0,0 +1,292 @@ +"""Unit tests for the OpenAI Realtime ``Backend`` (no network). + +Covers the pure surfaces: ``session.update`` assembly + tool translation, the +stateful provider-event -> normalized ``BackendEvent`` mapping (including +final-transcript selection and interruption), capability flags, sample-rate +exposure, factory dispatch, and ``send()`` validation. The live session +(open/receive over a real connection) is out of scope here. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from eva.backend.base import BackendEventType, ToolCallResult +from eva.backend.factory import BackendFactory +from eva.backend.openai_realtime import OpenAIRealtimeBackend, OpenAIRealtimeSession + + +def _backend(**overrides) -> OpenAIRealtimeBackend: + config = {"model": "gpt-realtime", "api_key": "test-key", "input_format": "pcm", **overrides} + return OpenAIRealtimeBackend(config=config) + + +def _session() -> OpenAIRealtimeSession: + return OpenAIRealtimeSession(client=None, conn_cm=None, conn=None) # type: ignore[arg-type] + + +def _map(session, event): + return OpenAIRealtimeBackend._map_event(session, event) + + +def test_requires_api_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + with pytest.raises(ValueError): + OpenAIRealtimeBackend(config={"model": "gpt-realtime"}) + + +def test_api_key_falls_back_to_env(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + # No api_key in config -> backend picks it up from the environment (no raise). + backend = OpenAIRealtimeBackend(config={"model": "gpt-realtime"}) + assert backend.output_sample_rate == 24000 + + +def test_accent_is_rejected(): + with pytest.raises(ValueError): + OpenAIRealtimeBackend(config={"model": "gpt-realtime", "api_key": "k", "accent": "british"}) + + +def test_capabilities(): + caps = _backend().capabilities + assert caps.emits_continuous_audio is True + assert caps.supports_streaming_interruption is True + assert caps.owns_playout_clock is False + + +def test_sample_rates_from_input_format(): + b = _backend() # pcm + assert b.output_sample_rate == 24000 + assert b.input_sample_rate == 24000 + b2 = _backend(input_format="pcmu") # telephony/caller input + assert b2.output_sample_rate == 24000 + assert b2.input_sample_rate == 8000 + + +def test_assemble_assistant_session_defaults(): + # pcm input, auto turn-taking: no manual create_response fields; whisper, no language. + sc = _backend(speaker_id="marin", vad_settings={})._session_config + assert sc["output_modalities"] == ["audio"] + assert sc["audio"]["output"] == {"voice": "marin", "format": {"type": "audio/pcm", "rate": 24000}} + assert sc["audio"]["input"]["format"] == {"type": "audio/pcm", "rate": 24000} + assert sc["audio"]["input"]["turn_detection"] == { + "type": "server_vad", + "threshold": 0.5, + "prefix_padding_ms": 300, + "silence_duration_ms": 200, + } + assert sc["audio"]["input"]["transcription"] == {"model": "whisper-1"} + + +def test_assemble_session_auto_turn_taking(): + sc = _backend( + speaker_id="ballad", + vad_settings={"threshold": 0.5, "prefix_padding_ms": 300, "silence_duration_ms": 500}, + transcription_language="en", + parallel_tool_calls=False, + )._session_config + assert sc["audio"]["output"]["voice"] == "ballad" + # Auto turn-taking: no create_response/interrupt_response override. + assert sc["audio"]["input"]["turn_detection"] == { + "type": "server_vad", + "threshold": 0.5, + "prefix_padding_ms": 300, + "silence_duration_ms": 500, + } + assert sc["audio"]["input"]["transcription"] == {"model": "whisper-1", "language": "en"} + assert sc["parallel_tool_calls"] is False + + +def test_build_session_update_stamps_owned_fields_and_translates_tools(): + b = _backend() + tools = [{"name": "get_reservation", "description": "look up", "parameters": {"type": "object", "properties": {}}}] + session = b._build_session_update("SYSTEM PROMPT", tools) + + assert session["type"] == "realtime" + assert session["instructions"] == "SYSTEM PROMPT" + assert session["output_modalities"] == ["audio"] + # Generic tool spec -> OpenAI schema (type: function added). + assert session["tools"] == [ + { + "type": "function", + "name": "get_reservation", + "description": "look up", + "parameters": {"type": "object", "properties": {}}, + } + ] + + +def test_build_session_update_none_tools_becomes_empty_list(): + assert _backend()._build_session_update("p", None)["tools"] == [] + + +def test_map_audio_delta(): + (be,) = _map(_session(), SimpleNamespace(type="response.output_audio.delta", delta="AQIDBA==")) + assert be.event_type == BackendEventType.AUDIO_OUTPUT + assert be.audio == b"\x01\x02\x03\x04" + + +def test_map_empty_audio_delta_dropped(): + assert _map(_session(), SimpleNamespace(type="response.output_audio.delta", delta="")) == [] + + +def test_map_output_transcript_delta_accumulates_silently_then_done_emits(): + s = _session() + assert _map(s, SimpleNamespace(type="response.output_audio_transcript.delta", delta="hel")) == [] + assert _map(s, SimpleNamespace(type="response.output_audio_transcript.delta", delta="lo")) == [] + (be,) = _map(s, SimpleNamespace(type="response.output_audio_transcript.done", transcript="hello")) + assert be.event_type == BackendEventType.TRANSCRIPT + assert be.transcript == "hello" + assert be.metadata == {"stream": "output", "final": True} + + +def test_map_input_transcription_completed_and_failed(): + (done,) = _map( + _session(), SimpleNamespace(type="conversation.item.input_audio_transcription.completed", transcript="hi") + ) + assert done.event_type == BackendEventType.TRANSCRIPT + assert done.transcript == "hi" and done.metadata == {"stream": "input", "final": True} + + # Empty completed transcription is dropped. + assert ( + _map(_session(), SimpleNamespace(type="conversation.item.input_audio_transcription.completed", transcript="")) + == [] + ) + + (failed,) = _map(_session(), SimpleNamespace(type="conversation.item.input_audio_transcription.failed", error="x")) + assert failed.event_type == BackendEventType.TRANSCRIPT + assert failed.metadata == {"stream": "input", "failed": True} + + +def test_map_speech_boundaries_no_interruption(): + (be,) = _map(_session(), SimpleNamespace(type="input_audio_buffer.speech_started")) + assert be.event_type == BackendEventType.INPUT_SPEECH_STARTED + (be2,) = _map(_session(), SimpleNamespace(type="input_audio_buffer.speech_stopped")) + assert be2.event_type == BackendEventType.INPUT_SPEECH_STOPPED + + +def test_map_interruption_flushes_partial_turn_then_speech_started(): + s = _session() + _map(s, SimpleNamespace(type="response.created")) + _map(s, SimpleNamespace(type="response.output_audio_transcript.delta", delta="I was sa")) + events = _map(s, SimpleNamespace(type="input_audio_buffer.speech_started")) + # Interrupted TURN_END (with the partial) precedes the speech-started signal. + assert [e.event_type for e in events] == [BackendEventType.TURN_END, BackendEventType.INPUT_SPEECH_STARTED] + turn_end = events[0] + assert turn_end.transcript == "I was sa" + assert turn_end.metadata["interrupted"] is True and turn_end.metadata["cancelled"] is False + # State reset: a second speech_started does not re-flush. + assert [e.event_type for e in _map(s, SimpleNamespace(type="input_audio_buffer.speech_started"))] == [ + BackendEventType.INPUT_SPEECH_STARTED + ] + + +def test_map_function_call(): + s = _session() + (be,) = _map( + s, + SimpleNamespace( + type="response.function_call_arguments.done", + call_id="c1", + name="get_reservation", + arguments='{"n": "ABC"}', + ), + ) + assert be.event_type == BackendEventType.TOOL_CALL_REQUEST + assert be.tool_call_request.call_id == "c1" + assert be.tool_call_request.arguments == {"n": "ABC"} + assert s.has_function_calls is True + + +def test_map_function_call_bad_arguments_becomes_empty(): + (be,) = _map( + _session(), + SimpleNamespace(type="response.function_call_arguments.done", call_id="c", name="f", arguments="nope"), + ) + assert be.tool_call_request.arguments == {} + + +def test_map_output_audio_done(): + (be,) = _map(_session(), SimpleNamespace(type="response.output_audio.done")) + assert be.event_type == BackendEventType.OUTPUT_AUDIO_DONE + + +def test_map_turn_started(): + (be,) = _map(_session(), SimpleNamespace(type="response.created")) + assert be.event_type == BackendEventType.OUTPUT_TURN_STARTED + + +def test_map_response_done_selects_final_transcript_and_usage(): + s = _session() + _map(s, SimpleNamespace(type="response.created")) + _map(s, SimpleNamespace(type="response.output_audio_transcript.done", transcript="all done")) + usage = SimpleNamespace(input_tokens=11, output_tokens=7) + response = SimpleNamespace(status="completed", usage=usage, output=[]) + (be,) = _map(s, SimpleNamespace(type="response.done", response=response)) + assert be.event_type == BackendEventType.TURN_END + assert be.transcript == "all done" + assert be.metadata["cancelled"] is False and be.metadata["interrupted"] is False + assert be.metadata["usage"] == {"prompt_tokens": 11, "completion_tokens": 7} + + +def test_map_response_done_cancelled(): + response = SimpleNamespace(status="cancelled", usage=None, output=[]) + (be,) = _map(_session(), SimpleNamespace(type="response.done", response=response)) + assert be.event_type == BackendEventType.TURN_END + assert be.metadata["cancelled"] is True and be.metadata["usage"] is None + + +def test_map_response_done_has_function_calls_from_output_items(): + response = SimpleNamespace(status="completed", usage=None, output=[SimpleNamespace(type="function_call")]) + (be,) = _map(_session(), SimpleNamespace(type="response.done", response=response)) + assert be.metadata["has_function_calls"] is True + + +def test_map_error_carries_code(): + (be,) = _map( + _session(), SimpleNamespace(type="error", error=SimpleNamespace(code="rate_limit", message="slow down")) + ) + assert be.event_type == BackendEventType.ERROR + assert be.metadata["code"] == "rate_limit" + + +def test_map_unhandled_event_dropped(): + assert _map(_session(), SimpleNamespace(type="session.updated")) == [] + assert _map(_session(), SimpleNamespace(type="conversation.item.created")) == [] + + +@pytest.mark.asyncio +async def test_send_requires_exactly_one_arg(): + b, s = _backend(), _session() + with pytest.raises(ValueError): + await b.send(s) + with pytest.raises(ValueError): + await b.send(s, audio=b"x", text="y") + + +@pytest.mark.asyncio +async def test_send_wrong_session_type_raises(): + with pytest.raises(TypeError): + await _backend().send(object(), tool_result=ToolCallResult(call_id="c", result={})) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_close_is_idempotent_and_network_free(): + b, s = _backend(), _session() + await b.close(s) + await b.close(s) + + +def test_factory_dispatch_builds_backend_from_flat_config(): + backend = BackendFactory().create( + "openai_realtime", {"model": "gpt-realtime", "api_key": "k", "input_format": "pcmu"} + ) + assert isinstance(backend, OpenAIRealtimeBackend) + assert backend.output_sample_rate == 24000 + assert backend.input_sample_rate == 8000 + + +def test_factory_unknown_provider_returns_none(): + assert BackendFactory().create("does_not_exist", {}) is None