diff --git a/.cursor/skills/scaffold-elevenlabs-example/reference.md b/.cursor/skills/scaffold-elevenlabs-example/reference.md index d5ef60dc..b8fd2725 100644 --- a/.cursor/skills/scaffold-elevenlabs-example/reference.md +++ b/.cursor/skills/scaffold-elevenlabs-example/reference.md @@ -22,6 +22,7 @@ Current repo skills: - `/music` - `/setup-api-key` - `/sound-effects` +- `/speech-engine` - `/speech-to-text` - `/text-to-speech` @@ -41,17 +42,18 @@ Ignore the deprecated root `examples/` folder for new work. ## Current example matrix -| Path | Shared template | Prompt sections | Setup extras | -| -------------------------------------- | ---------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | -| `text-to-speech/typescript/quickstart` | `templates/typescript` | `index.ts` | Copies `.env`, preserves `node_modules`, installs with `pnpm` | -| `text-to-speech/python/quickstart` | `templates/python` | `main.py` | Copies `.env`, preserves `.venv`, installs with `pip` | -| `speech-to-text/typescript/quickstart` | `templates/typescript` | `index.ts` | Optional `assets/`, copies `.env`, preserves `node_modules` | -| `speech-to-text/python/quickstart` | `templates/python` | `main.py` | Optional `assets/`, copies `.env`, preserves `.venv` | -| `music/typescript/quickstart` | `templates/typescript` | `index.ts` | Copies `.env`, preserves `node_modules`, installs with `pnpm` | -| `music/nextjs/quickstart` | `templates/nextjs` | `app/api/generate-music/route.ts`, `app/page.tsx` | Adds `@elevenlabs/elevenlabs-js`, copies `.env.local`, preserves `node_modules` and `.next` | -| `speech-to-text/nextjs/realtime` | `templates/nextjs` | `app/api/scribe-token/route.ts`, `app/page.tsx` | Adds `@elevenlabs/react` and `@elevenlabs/elevenlabs-js`, copies `.env.local`, preserves `node_modules` and `.next` | -| `agents/nextjs/quickstart` | `templates/nextjs` | `app/api/agent/route.ts`, `app/api/conversation-token/route.ts`, `app/page.tsx` | Same Next.js setup pattern, removes `@elevenlabs/client` if present | -| `agents/nextjs/guardrails` | `templates/nextjs` | `app/api/agent/route.ts`, `app/api/conversation-token/route.ts`, `app/page.tsx` | Same as quickstart, but prompt targets guardrails and `onGuardrailTriggered` | +| Path | Shared template | Prompt sections | Setup extras | +| -------------------------------------- | ---------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `text-to-speech/typescript/quickstart` | `templates/typescript` | `index.ts` | Copies `.env`, preserves `node_modules`, installs with `pnpm` | +| `text-to-speech/python/quickstart` | `templates/python` | `main.py` | Copies `.env`, preserves `.venv`, installs with `pip` | +| `speech-to-text/typescript/quickstart` | `templates/typescript` | `index.ts` | Optional `assets/`, copies `.env`, preserves `node_modules` | +| `speech-to-text/python/quickstart` | `templates/python` | `main.py` | Optional `assets/`, copies `.env`, preserves `.venv` | +| `music/typescript/quickstart` | `templates/typescript` | `index.ts` | Copies `.env`, preserves `node_modules`, installs with `pnpm` | +| `music/nextjs/quickstart` | `templates/nextjs` | `app/api/generate-music/route.ts`, `app/page.tsx` | Adds `@elevenlabs/elevenlabs-js`, copies `.env.local`, preserves `node_modules` and `.next` | +| `speech-to-text/nextjs/realtime` | `templates/nextjs` | `app/api/scribe-token/route.ts`, `app/page.tsx` | Adds `@elevenlabs/react` and `@elevenlabs/elevenlabs-js`, copies `.env.local`, preserves `node_modules` and `.next` | +| `agents/nextjs/quickstart` | `templates/nextjs` | `app/api/agent/route.ts`, `app/api/conversation-token/route.ts`, `app/page.tsx` | Same Next.js setup pattern, removes `@elevenlabs/client` if present | +| `agents/nextjs/guardrails` | `templates/nextjs` | `app/api/agent/route.ts`, `app/api/conversation-token/route.ts`, `app/page.tsx` | Same as quickstart, but prompt targets guardrails and `onGuardrailTriggered` | +| `speech-engine/nextjs/quickstart` | `templates/nextjs` | `server.mts`, `scripts/create-engine.mts`, `app/api/token/route.ts`, `app/page.tsx`, `lib/assistant.ts` | Adds `openai`, `dotenv`, `tsx`; Speech Engine scripts; `livekit-client` override; copies `.env.example` | ## Runtime setup rules @@ -93,6 +95,7 @@ Ignore the deprecated root `examples/` folder for new work. - CLI transcription or file-based Scribe example: start from the speech-to-text quickstarts. - Realtime microphone UI: start from `speech-to-text/nextjs/realtime`. - Voice agent creation and conversation UI: start from `agents/nextjs/quickstart`. +- Speech Engine WebSocket server plus Next.js WebRTC client: start from `speech-engine/nextjs/quickstart`. - First Expo full-stack example for a product: start from the closest same-product `nextjs` example until a dedicated Expo reference exists. - For specialized agent behavior, start from `agents/nextjs/quickstart` and consult `agents/nextjs/guardrails` only as an existing reference, not as a scaffold mode. diff --git a/README.md b/README.md index bf248693..84b5aa03 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Shared base templates live in `templates/` (Expo, Next.js, Python, TypeScript). - [Sound Effects Quickstart (Python)](sound-effects/python/quickstart/example/README.md) — Generate a sound effect MP3 from a text prompt with the ElevenLabs Python SDK. - [Sound Effects Playground (Next.js)](sound-effects/nextjs/quickstart/example/README.md) — Enter a prompt, generate a sound effect, and play it back in the browser. - [Real-Time Speech-to-Text (Next.js)](speech-to-text/nextjs/realtime/example/README.md) — Live microphone transcription with VAD in a Next.js app. +- [Speech Engine Quickstart (Next.js)](speech-engine/nextjs/quickstart/example/README.md) — Add voice to your own LLM-backed chat agent with a Speech Engine WebSocket server and Next.js client. - [Real-Time Voice Agent (Next.js)](agents/nextjs/quickstart/example/README.md) — Live voice conversations with the ElevenLabs Agents Platform using the React Agents SDK. - [Voice Agent Guardrails Demo (Next.js)](agents/nextjs/guardrails/example/README.md) — Demonstrate custom guardrails and the `guardrail_triggered` client event in a live voice agent. - [Voice Isolator (Next.js)](voice-isolator/nextjs/quickstart/example/README.md) — Record your voice in the browser and remove background noise with the Voice Isolator API. diff --git a/speech-engine/nextjs/quickstart/PROMPT.md b/speech-engine/nextjs/quickstart/PROMPT.md new file mode 100644 index 00000000..68f5b711 --- /dev/null +++ b/speech-engine/nextjs/quickstart/PROMPT.md @@ -0,0 +1,83 @@ +Before writing any code, invoke the `/speech-engine` skill to learn the correct ElevenLabs SDK patterns. + +## 1. `package.json` + +- Add `@elevenlabs/react`, `@elevenlabs/elevenlabs-js`, `openai`, `dotenv`, and `tsx`. +- Add scripts: `speech-engine:create` (`tsx scripts/create-engine.mts`), `speech-engine:enable-first-message` (`tsx scripts/enable-first-message.mts`), `speech-engine:server` (`tsx server.mts`). +- Pin `livekit-client` to `2.16.1` under `pnpm.overrides` for WebRTC stability. + +## 2. `.env.example` + +- Document `ELEVENLABS_API_KEY`, `ELEVENLABS_SPEECH_ENGINE_ID`, `OPENAI_API_KEY`, and `PUBLIC_WS_URL` (`wss://…/ws` from ngrok). + +## 3. `scripts/create-engine.mts` + +- Load env with `dotenv/config`. +- Create a Speech Engine with `elevenlabs.speechEngine.create`, using `speechEngine.wsUrl` from `PUBLIC_WS_URL`. +- Call a shared helper to enable `overrides.firstMessage` on the new resource. +- Print `engineId` and next-step instructions. + +## 4. `scripts/enable-first-message.mts` + +- Update an existing Speech Engine (`ELEVENLABS_SPEECH_ENGINE_ID`) so the client may set `overrides.agent.firstMessage`. + +## 5. `lib/speech-engine-overrides.ts` + +- Export `enableFirstMessageOverride(client, speechEngineId)` using `speechEngine.update` with `overrides: { firstMessage: true }`. + +## 6. `lib/assistant.ts` + +- Shared OpenAI Responses API helpers: `ASSISTANT_INSTRUCTIONS`, `VOICE_FIRST_MESSAGE`, `isChatRole`, `normalizeChatMessages`, `transcriptToChatMessages`, `createAssistantReply(messages, signal)`, and `createAssistantStream(messages, signal)`. +- Map Speech Engine `agent` role to OpenAI `assistant`; keep responses concise. +- Normalize history before sending it to OpenAI: trim content, drop empty messages, cap message count and per-message length. + +## 7. `lib/voice-history.ts` + +- Add a tiny local store that lets the Next.js app hand typed chat history to the standalone Speech Engine server. +- Store normalized `ChatMessage[]` in `.next/cache/voice-history.json` by random `historyId`. +- Export `createVoiceHistory(messages)`, `linkVoiceHistory(conversationId, historyId)`, and `loadVoiceHistory(conversationId)`. +- Keep this demo-only and file-backed; no database or auth layer. + +## 8. `server.mts` + +- Standalone HTTP server on port **3001** with `speechEngine.attach(SPEECH_ENGINE_ID, httpServer, "/ws", { debug: true, … })`. +- `onTranscript`: load any typed chat history linked to `session.conversationId`, append `transcriptToChatMessages(transcript)`, pass `AbortSignal` into `createAssistantStream`, and `session.sendResponse(stream)`. +- Cache loaded initial history per conversation and retry briefly because the browser may link history immediately after the voice conversation is created. +- Log `onInit`, `onClose`, and `onError`. Require `ELEVENLABS_SPEECH_ENGINE_ID` and `OPENAI_API_KEY`. + +## 9. `app/api/chat/route.ts` + +- Secure POST route for regular typed chat. +- Accept `{ messages }`, validate each message as `{ role: "user" | "assistant", content: string }`, cap request history, and call `createAssistantReply(messages, request.signal)`. +- Return `{ message }`; return JSON errors for invalid payloads or OpenAI failures. +- This route and voice mode must use the same `ChatMessage` shape so context can move between modes. + +## 10. `app/api/voice-history/route.ts` + +- POST route that accepts the current typed chat history before voice mode starts. +- Validate and normalize messages with `isChatRole`, call `createVoiceHistory(messages)`, and return `{ historyId }`. +- Return a 400 if no valid messages are provided. + +## 11. `app/api/voice-history/link/route.ts` + +- POST route that accepts `{ historyId, conversationId }` from the browser once the Speech Engine client creates a voice conversation. +- Call `linkVoiceHistory(conversationId, historyId)` and return `{ ok: true }`, or 404 if the history id is unknown. + +## 12. `app/api/token/route.ts` + +- Secure GET route; never expose `ELEVENLABS_API_KEY` to the client. +- Return `{ token }` from `conversationalAi.conversations.getWebrtcToken({ agentId: ELEVENLABS_SPEECH_ENGINE_ID })`. +- Handle missing env and API errors with JSON error responses. + +## 13. `app/page.tsx` + +- Build a single chat UI that works like a normal assistant chat first, then can switch into voice mode without losing context. +- Keep one `messages` state array with `{ id, role: "user" | "assistant", content, channel: "chat" | "voice", pending?, error? }`. +- Text submit path: append the user message, POST the full normalized history to `/api/chat`, replace the pending assistant message with the reply, and keep the composer usable. +- Voice start path: request microphone access, POST current non-pending/non-error chat history to `/api/voice-history`, fetch `/api/token`, then `startSession({ conversationToken: token, overrides: { agent: { firstMessage: VOICE_FIRST_MESSAGE } }, onConversationCreated })`. +- In `onConversationCreated`, link the stored `historyId` to `voiceConversation.getId()` with `/api/voice-history/link`; if useful, also send a contextual update summarizing recent typed chat. +- Voice event path: use `useConversation({ onMessage })` to append or update transcript messages in the same `messages` array, mapping SDK role `agent` to local role `assistant`. +- While voice is connected, allow typed messages too: append them to the same thread and call `conversation.sendUserMessage(content)`; avoid duplicating echoes from `onMessage`. +- The next typed chat after voice mode must include voice transcript messages in the history sent to `/api/chat`. +- Show a scrollable message thread, a text composer, a mic/start voice control, a stop voice control, connection status, mute/unmute when connected, and separate chat/voice error messages. +- Follow `DESIGN.md`; make the page feel like a regular chat app with voice as a mode, not a voice-only demo. diff --git a/speech-engine/nextjs/quickstart/README.md b/speech-engine/nextjs/quickstart/README.md new file mode 100644 index 00000000..4ff82f68 --- /dev/null +++ b/speech-engine/nextjs/quickstart/README.md @@ -0,0 +1,56 @@ +# Speech Engine Quickstart (Next.js) + +Add real-time voice to your own LLM-backed agent: a Speech Engine WebSocket server streams OpenAI responses to ElevenLabs, and a Next.js client starts WebRTC voice sessions. + +## Setup + +1. Copy the environment file and add your credentials: + + ```bash + cp .env.example .env + ``` + + Then edit `.env` and set: + - `ELEVENLABS_API_KEY` — [create one in the dashboard](https://elevenlabs.io/app/settings/api-keys) + - `OPENAI_API_KEY` — for the assistant LLM in the Speech Engine server + - `PUBLIC_WS_URL` — public `wss://` URL for your Speech Engine WebSocket (see step 2) + +2. Expose port **3001** with [ngrok](https://ngrok.com/) (run this before creating the Speech Engine resource): + + ```bash + ngrok http 3001 + ``` + + Set `PUBLIC_WS_URL` to your forwarding URL with `/ws` appended, for example `wss://abc123.ngrok-free.app/ws`. + +3. Install dependencies: + + ```bash + pnpm install + ``` + +4. Create the Speech Engine resource and enable client first-message overrides: + + ```bash + pnpm run speech-engine:create + pnpm run speech-engine:enable-first-message + ``` + + Copy the printed Speech Engine ID into `.env` as `ELEVENLABS_SPEECH_ENGINE_ID`. + +## Run + +Three processes must run together: + +1. **ngrok** — `ngrok http 3001` +2. **Speech Engine server** — `pnpm run speech-engine:server` (port 3001) +3. **Next.js app** — `pnpm run dev` (port 3000) + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## Usage + +- Click **Start conversation** and allow microphone access when prompted. +- Speak naturally; the agent responds with streamed speech. With `debug: true` on the server, transcripts and responses log to the terminal. +- Click **End conversation** to stop the session. +- The agent greets first using `overrides.agent.firstMessage` (requires `speech-engine:enable-first-message` once per Speech Engine ID). diff --git a/speech-engine/nextjs/quickstart/example/.env.example b/speech-engine/nextjs/quickstart/example/.env.example new file mode 100644 index 00000000..507a4f20 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/.env.example @@ -0,0 +1,12 @@ +# Server-only ElevenLabs API key. +ELEVENLABS_API_KEY= + +# Speech Engine ID returned by `pnpm run speech-engine:create`. +ELEVENLABS_SPEECH_ENGINE_ID= + +# Server-only OpenAI API key for assistant responses. +OPENAI_API_KEY= + +# Public WebSocket URL, usually from ngrok, including `/ws`. +# Example: wss://your-subdomain.ngrok.app/ws +PUBLIC_WS_URL= diff --git a/speech-engine/nextjs/quickstart/example/README.md b/speech-engine/nextjs/quickstart/example/README.md new file mode 100644 index 00000000..4ff82f68 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/README.md @@ -0,0 +1,56 @@ +# Speech Engine Quickstart (Next.js) + +Add real-time voice to your own LLM-backed agent: a Speech Engine WebSocket server streams OpenAI responses to ElevenLabs, and a Next.js client starts WebRTC voice sessions. + +## Setup + +1. Copy the environment file and add your credentials: + + ```bash + cp .env.example .env + ``` + + Then edit `.env` and set: + - `ELEVENLABS_API_KEY` — [create one in the dashboard](https://elevenlabs.io/app/settings/api-keys) + - `OPENAI_API_KEY` — for the assistant LLM in the Speech Engine server + - `PUBLIC_WS_URL` — public `wss://` URL for your Speech Engine WebSocket (see step 2) + +2. Expose port **3001** with [ngrok](https://ngrok.com/) (run this before creating the Speech Engine resource): + + ```bash + ngrok http 3001 + ``` + + Set `PUBLIC_WS_URL` to your forwarding URL with `/ws` appended, for example `wss://abc123.ngrok-free.app/ws`. + +3. Install dependencies: + + ```bash + pnpm install + ``` + +4. Create the Speech Engine resource and enable client first-message overrides: + + ```bash + pnpm run speech-engine:create + pnpm run speech-engine:enable-first-message + ``` + + Copy the printed Speech Engine ID into `.env` as `ELEVENLABS_SPEECH_ENGINE_ID`. + +## Run + +Three processes must run together: + +1. **ngrok** — `ngrok http 3001` +2. **Speech Engine server** — `pnpm run speech-engine:server` (port 3001) +3. **Next.js app** — `pnpm run dev` (port 3000) + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## Usage + +- Click **Start conversation** and allow microphone access when prompted. +- Speak naturally; the agent responds with streamed speech. With `debug: true` on the server, transcripts and responses log to the terminal. +- Click **End conversation** to stop the session. +- The agent greets first using `overrides.agent.firstMessage` (requires `speech-engine:enable-first-message` once per Speech Engine ID). diff --git a/speech-engine/nextjs/quickstart/example/app/api/chat/route.ts b/speech-engine/nextjs/quickstart/example/app/api/chat/route.ts new file mode 100644 index 00000000..70a5844e --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/api/chat/route.ts @@ -0,0 +1,82 @@ +import { NextResponse } from "next/server"; + +import { + createAssistantReply, + isChatRole, + type ChatMessage, +} from "@/lib/assistant"; + +export const dynamic = "force-dynamic"; + +const MAX_REQUEST_MESSAGES = 30; + +function parseMessages(value: unknown): ChatMessage[] | null { + if (!Array.isArray(value)) { + return null; + } + + const messages: ChatMessage[] = []; + + for (const item of value) { + if (!item || typeof item !== "object") { + return null; + } + + const { role, content } = item as Record; + + if (!isChatRole(role) || typeof content !== "string") { + return null; + } + + const trimmedContent = content.trim(); + + if (trimmedContent.length > 0) { + messages.push({ + role, + content: trimmedContent, + }); + } + } + + return messages.slice(-MAX_REQUEST_MESSAGES); +} + +export async function POST(request: Request) { + let body: unknown; + + try { + body = await request.json(); + } catch { + return NextResponse.json( + { error: "Request body must be valid JSON." }, + { status: 400 } + ); + } + + const messages = parseMessages( + typeof body === "object" && body !== null + ? (body as Record).messages + : null + ); + + if (!messages || messages.length === 0) { + return NextResponse.json( + { error: "Send at least one user or assistant message." }, + { status: 400 } + ); + } + + try { + const message = await createAssistantReply(messages, request.signal); + + return NextResponse.json({ message }); + } catch (error: unknown) { + const details = + error instanceof Error ? error.message : "Failed to generate a reply."; + + return NextResponse.json( + { error: "Unable to generate a chat response.", details }, + { status: 500 } + ); + } +} diff --git a/speech-engine/nextjs/quickstart/example/app/api/token/route.ts b/speech-engine/nextjs/quickstart/example/app/api/token/route.ts new file mode 100644 index 00000000..f589014c --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/api/token/route.ts @@ -0,0 +1,38 @@ +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; +import { NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +function getRequiredEnv( + name: "ELEVENLABS_API_KEY" | "ELEVENLABS_SPEECH_ENGINE_ID" +) { + const value = process.env[name]?.trim(); + + if (!value) { + throw new Error(`Missing ${name}.`); + } + + return value; +} + +export async function GET() { + try { + const apiKey = getRequiredEnv("ELEVENLABS_API_KEY"); + const speechEngineId = getRequiredEnv("ELEVENLABS_SPEECH_ENGINE_ID"); + const elevenlabs = new ElevenLabsClient({ apiKey }); + const response = + await elevenlabs.conversationalAi.conversations.getWebrtcToken({ + agentId: speechEngineId, + }); + + return NextResponse.json({ token: response.token }); + } catch (error: unknown) { + const details = + error instanceof Error ? error.message : "Failed to create a token."; + + return NextResponse.json( + { error: "Unable to create a conversation token.", details }, + { status: 500 } + ); + } +} diff --git a/speech-engine/nextjs/quickstart/example/app/api/voice-history/link/route.ts b/speech-engine/nextjs/quickstart/example/app/api/voice-history/link/route.ts new file mode 100644 index 00000000..22566afc --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/api/voice-history/link/route.ts @@ -0,0 +1,33 @@ +import { NextResponse } from "next/server"; + +import { linkVoiceHistory } from "@/lib/voice-history"; + +export const dynamic = "force-dynamic"; + +export async function POST(request: Request) { + const body = (await request.json().catch(() => null)) as { + conversationId?: unknown; + historyId?: unknown; + } | null; + + if ( + typeof body?.conversationId !== "string" || + typeof body.historyId !== "string" + ) { + return NextResponse.json( + { error: "Provide a conversationId and historyId." }, + { status: 400 } + ); + } + + const linked = await linkVoiceHistory(body.conversationId, body.historyId); + + if (!linked) { + return NextResponse.json( + { error: "The voice history could not be found." }, + { status: 404 } + ); + } + + return NextResponse.json({ ok: true }); +} diff --git a/speech-engine/nextjs/quickstart/example/app/api/voice-history/route.ts b/speech-engine/nextjs/quickstart/example/app/api/voice-history/route.ts new file mode 100644 index 00000000..00cac579 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/api/voice-history/route.ts @@ -0,0 +1,48 @@ +import { NextResponse } from "next/server"; + +import { isChatRole, type ChatMessage } from "@/lib/assistant"; +import { createVoiceHistory } from "@/lib/voice-history"; + +export const dynamic = "force-dynamic"; + +function parseMessages(input: unknown): ChatMessage[] | null { + if (!Array.isArray(input)) { + return null; + } + + const messages = input + .filter((message): message is ChatMessage => { + if (!message || typeof message !== "object") { + return false; + } + + const { role, content } = message as Partial; + return isChatRole(role) && typeof content === "string"; + }) + .map(message => ({ + role: message.role, + content: message.content.trim(), + })) + .filter(message => message.content.length > 0); + + return messages.length > 0 ? messages : null; +} + +export async function POST(request: Request) { + const body = (await request.json().catch(() => null)) as { + messages?: unknown; + } | null; + + const messages = parseMessages(body?.messages); + + if (!messages) { + return NextResponse.json( + { error: "Provide at least one chat message." }, + { status: 400 } + ); + } + + const historyId = await createVoiceHistory(messages); + + return NextResponse.json({ historyId }); +} diff --git a/speech-engine/nextjs/quickstart/example/app/favicon.ico b/speech-engine/nextjs/quickstart/example/app/favicon.ico new file mode 100644 index 00000000..718d6fea Binary files /dev/null and b/speech-engine/nextjs/quickstart/example/app/favicon.ico differ diff --git a/speech-engine/nextjs/quickstart/example/app/globals.css b/speech-engine/nextjs/quickstart/example/app/globals.css new file mode 100644 index 00000000..d767ad63 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/globals.css @@ -0,0 +1,126 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); + --color-sidebar-ring: var(--sidebar-ring); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar: var(--sidebar); + --color-chart-5: var(--chart-5); + --color-chart-4: var(--chart-4); + --color-chart-3: var(--chart-3); + --color-chart-2: var(--chart-2); + --color-chart-1: var(--chart-1); + --color-ring: var(--ring); + --color-input: var(--input); + --color-border: var(--border); + --color-destructive: var(--destructive); + --color-accent-foreground: var(--accent-foreground); + --color-accent: var(--accent); + --color-muted-foreground: var(--muted-foreground); + --color-muted: var(--muted); + --color-secondary-foreground: var(--secondary-foreground); + --color-secondary: var(--secondary); + --color-primary-foreground: var(--primary-foreground); + --color-primary: var(--primary); + --color-popover-foreground: var(--popover-foreground); + --color-popover: var(--popover); + --color-card-foreground: var(--card-foreground); + --color-card: var(--card); + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.145 0 0); + --card: oklch(1 0 0); + --card-foreground: oklch(0.145 0 0); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.145 0 0); + --primary: oklch(0.205 0 0); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.97 0 0); + --secondary-foreground: oklch(0.205 0 0); + --muted: oklch(0.97 0 0); + --muted-foreground: oklch(0.556 0 0); + --accent: oklch(0.97 0 0); + --accent-foreground: oklch(0.205 0 0); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.922 0 0); + --input: oklch(0.922 0 0); + --ring: oklch(0.708 0 0); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.145 0 0); + --sidebar-primary: oklch(0.205 0 0); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.97 0 0); + --sidebar-accent-foreground: oklch(0.205 0 0); + --sidebar-border: oklch(0.922 0 0); + --sidebar-ring: oklch(0.708 0 0); +} + +.dark { + --background: oklch(0.145 0 0); + --foreground: oklch(0.985 0 0); + --card: oklch(0.205 0 0); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.205 0 0); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.922 0 0); + --primary-foreground: oklch(0.205 0 0); + --secondary: oklch(0.269 0 0); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.269 0 0); + --muted-foreground: oklch(0.708 0 0); + --accent: oklch(0.269 0 0); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.556 0 0); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.205 0 0); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.269 0 0); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.556 0 0); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} diff --git a/speech-engine/nextjs/quickstart/example/app/layout.tsx b/speech-engine/nextjs/quickstart/example/app/layout.tsx new file mode 100644 index 00000000..f7fa87eb --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/layout.tsx @@ -0,0 +1,34 @@ +import type { Metadata } from "next"; +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "Create Next App", + description: "Generated by create next app", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + {children} + + + ); +} diff --git a/speech-engine/nextjs/quickstart/example/app/page.tsx b/speech-engine/nextjs/quickstart/example/app/page.tsx new file mode 100644 index 00000000..e818cae0 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/page.tsx @@ -0,0 +1,561 @@ +"use client"; + +import { ConversationProvider, useConversation } from "@elevenlabs/react"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + type FormEvent, + type KeyboardEvent, +} from "react"; + +import { VOICE_FIRST_MESSAGE } from "@/lib/assistant-constants"; + +type DemoRole = "user" | "assistant"; +type DemoChannel = "chat" | "voice"; + +type DemoMessage = { + id: string; + role: DemoRole; + content: string; + channel: DemoChannel; + pending?: boolean; + error?: boolean; +}; + +type ApiMessage = { + role: DemoRole; + content: string; +}; + +function createMessageId(prefix: string) { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + +function getErrorMessage(error: unknown) { + return error instanceof Error ? error.message : "Something went wrong."; +} + +function roleLabel(role: DemoRole) { + return role === "assistant" ? "Assistant" : "You"; +} + +function toApiMessages(messages: DemoMessage[]): ApiMessage[] { + return messages + .filter(message => !message.pending && !message.error) + .map(message => ({ + role: message.role, + content: message.content, + })); +} + +function buildVoiceContext(messages: DemoMessage[]) { + return toApiMessages(messages) + .slice(-12) + .map(message => `${roleLabel(message.role)}: ${message.content}`) + .join("\n"); +} + +function appendOrUpdateVoiceMessage( + messages: DemoMessage[], + role: DemoRole, + content: string, + eventId?: number +) { + const id = + eventId != null + ? `voice-${role}-${eventId}` + : createMessageId(`voice-${role}`); + const existingIndex = messages.findIndex(message => message.id === id); + + if (existingIndex === -1) { + return [ + ...messages, + { + id, + role, + content, + channel: "voice" as const, + }, + ]; + } + + return messages.map((message, index) => + index === existingIndex ? { ...message, content } : message + ); +} + +async function getConversationToken() { + const response = await fetch("/api/token"); + const data = (await response.json().catch(() => ({}))) as { + token?: string; + error?: string; + details?: string; + }; + + if (!response.ok || !data.token) { + throw new Error( + data.details || data.error || "Failed to get a conversation token." + ); + } + + return data.token; +} + +async function registerVoiceHistory(messages: ApiMessage[]) { + if (messages.length === 0) { + return null; + } + + const response = await fetch("/api/voice-history", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ messages }), + }); + const data = (await response.json().catch(() => ({}))) as { + historyId?: string; + error?: string; + }; + + if (!response.ok || !data.historyId) { + throw new Error(data.error || "Failed to prepare voice history."); + } + + return data.historyId; +} + +async function linkVoiceHistory(historyId: string, conversationId: string) { + const response = await fetch("/api/voice-history/link", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ historyId, conversationId }), + }); + const data = (await response.json().catch(() => ({}))) as { + error?: string; + }; + + if (!response.ok) { + throw new Error(data.error || "Failed to link voice history."); + } +} + +function ConversationView() { + const [messages, setMessages] = useState([]); + const [draft, setDraft] = useState(""); + const [isSending, setIsSending] = useState(false); + const [chatError, setChatError] = useState(null); + const [voiceError, setVoiceError] = useState(null); + const threadRef = useRef(null); + const voiceHistoryIdRef = useRef(null); + const linkedVoiceConversationIdsRef = useRef>(new Set()); + const pendingTypedVoiceMessagesRef = useRef([]); + + const conversation = useConversation({ + onConnect: () => { + setVoiceError(null); + }, + onDisconnect: details => { + if (details.reason === "error") { + setVoiceError(details.message); + return; + } + + if (details.reason === "agent") { + const reason = + "closeReason" in details && details.closeReason + ? details.closeReason + : null; + + if (reason) { + setVoiceError(reason); + } + } + }, + onError: message => { + setVoiceError(message); + }, + onMessage: ({ role, message, event_id }) => { + const content = message.trim(); + + if (!content) { + return; + } + + if (role === "user") { + const pendingIndex = + pendingTypedVoiceMessagesRef.current.indexOf(content); + + if (pendingIndex !== -1) { + pendingTypedVoiceMessagesRef.current.splice(pendingIndex, 1); + return; + } + } + + setMessages(currentMessages => + appendOrUpdateVoiceMessage( + currentMessages, + role === "agent" ? "assistant" : "user", + content, + event_id + ) + ); + }, + }); + + const chatHistory = useMemo(() => toApiMessages(messages), [messages]); + const voiceContext = useMemo(() => buildVoiceContext(messages), [messages]); + const isVoiceConnected = conversation.status === "connected"; + const isVoiceStarting = conversation.status === "connecting"; + const canStartVoice = conversation.status === "disconnected"; + const connectionStatus = isVoiceConnected + ? conversation.mode === "speaking" + ? "Voice connected - speaking" + : "Voice connected - listening" + : isVoiceStarting + ? "Voice connecting" + : "Chat only"; + + useEffect(() => { + const thread = threadRef.current; + + if (!thread) { + return; + } + + thread.scrollTo({ + top: thread.scrollHeight, + behavior: "smooth", + }); + }, [messages]); + + const sendChatMessage = useCallback(async () => { + const content = draft.trim(); + + if (!content || isSending) { + return; + } + + const userMessage: DemoMessage = { + id: createMessageId(isVoiceConnected ? "voice-user-typed" : "chat-user"), + role: "user", + content, + channel: isVoiceConnected ? "voice" : "chat", + }; + + if (isVoiceConnected) { + try { + pendingTypedVoiceMessagesRef.current.push(content); + conversation.sendUserMessage(content); + setMessages(currentMessages => [...currentMessages, userMessage]); + setDraft(""); + setChatError(null); + setVoiceError(null); + } catch (error: unknown) { + pendingTypedVoiceMessagesRef.current = + pendingTypedVoiceMessagesRef.current.filter( + pendingContent => pendingContent !== content + ); + setVoiceError(getErrorMessage(error)); + } + + return; + } + + const pendingAssistantId = createMessageId("chat-assistant"); + const pendingAssistant: DemoMessage = { + id: pendingAssistantId, + role: "assistant", + content: "Thinking...", + channel: "chat", + pending: true, + }; + const nextHistory = [...chatHistory, { role: "user" as const, content }]; + + setMessages(currentMessages => [ + ...currentMessages, + userMessage, + pendingAssistant, + ]); + setDraft(""); + setIsSending(true); + setChatError(null); + + try { + const response = await fetch("/api/chat", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ messages: nextHistory }), + }); + const data = (await response.json().catch(() => ({}))) as { + message?: string; + error?: string; + details?: string; + }; + + if (!response.ok) { + throw new Error(data.details || data.error || "Chat request failed."); + } + + if (!data.message) { + throw new Error("Chat response did not include a message."); + } + + setMessages(currentMessages => + currentMessages.map(message => + message.id === pendingAssistantId + ? { ...message, content: data.message ?? "", pending: false } + : message + ) + ); + } catch (error: unknown) { + const message = getErrorMessage(error); + + setChatError(message); + setMessages(currentMessages => + currentMessages.map(currentMessage => + currentMessage.id === pendingAssistantId + ? { + ...currentMessage, + content: message, + pending: false, + error: true, + } + : currentMessage + ) + ); + } finally { + setIsSending(false); + } + }, [chatHistory, conversation, draft, isSending, isVoiceConnected]); + + const handleSubmit = useCallback( + (event: FormEvent) => { + event.preventDefault(); + void sendChatMessage(); + }, + [sendChatMessage] + ); + + const handleComposerKeyDown = useCallback( + (event: KeyboardEvent) => { + if (event.key === "Enter" && !event.shiftKey) { + event.preventDefault(); + void sendChatMessage(); + } + }, + [sendChatMessage] + ); + + const startVoiceMode = useCallback(async () => { + if (isVoiceConnected || isVoiceStarting) { + return; + } + + try { + if (!navigator.mediaDevices?.getUserMedia) { + throw new Error("Microphone access is not available in this browser."); + } + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + stream.getTracks().forEach(track => track.stop()); + + const historyId = await registerVoiceHistory(chatHistory); + const token = await getConversationToken(); + const context = voiceContext; + + voiceHistoryIdRef.current = historyId; + linkedVoiceConversationIdsRef.current.clear(); + setVoiceError(null); + + await conversation.startSession({ + conversationToken: token, + overrides: { + agent: { + firstMessage: VOICE_FIRST_MESSAGE, + }, + }, + onConversationCreated: voiceConversation => { + const conversationId = voiceConversation.getId(); + const activeHistoryId = voiceHistoryIdRef.current; + + if ( + activeHistoryId && + !linkedVoiceConversationIdsRef.current.has(conversationId) + ) { + void linkVoiceHistory(activeHistoryId, conversationId) + .then(() => { + linkedVoiceConversationIdsRef.current.add(conversationId); + }) + .catch((error: unknown) => { + setVoiceError(getErrorMessage(error)); + }); + } + + if (context) { + voiceConversation.sendContextualUpdate( + `Typed chat before voice mode:\n${context}`, + { contextId: "typed-chat-history" } + ); + } + }, + }); + } catch (error: unknown) { + setVoiceError(getErrorMessage(error)); + } + }, [ + chatHistory, + conversation, + isVoiceConnected, + isVoiceStarting, + voiceContext, + ]); + + const stopVoiceMode = useCallback(() => { + conversation.endSession(); + }, [conversation]); + + return ( +
+
+
+

+ Voice & Chat Agent +

+

+ Live speech-to-text & text-to-speech with Speech Engine. +

+

{connectionStatus}

+
+ +
+
+ {isVoiceConnected ? ( + <> + + + + ) : ( + + )} +
+ + {chatError ? ( +

Chat error: {chatError}

+ ) : null} + + {voiceError ? ( +

Voice error: {voiceError}

+ ) : null} + +
+ {messages.length === 0 ? ( +

+ Start with a typed message, then switch to voice without losing + context. +

+ ) : ( +
+ {messages.map(message => { + const isAssistant = message.role === "assistant"; + + return ( +
+
+

+ {roleLabel(message.role)} + {message.channel === "voice" ? " / voice" : ""} +

+

+ {message.content} +

+
+
+ ); + })} +
+ )} +
+ +
+ +
+