From c101aeacbacf3a81a7a6286ed40cd5f9422039de Mon Sep 17 00:00:00 2001 From: Tadas Petra <60107328+tadaspetra@users.noreply.github.com> Date: Mon, 18 May 2026 18:51:28 -0500 Subject: [PATCH 1/7] working test --- README.md | 1 + speech-engine/nextjs/quickstart/PROMPT.md | 57 ++ speech-engine/nextjs/quickstart/README.md | 67 ++ .../nextjs/quickstart/example/.env.example | 6 + .../nextjs/quickstart/example/README.md | 67 ++ .../quickstart/example/app/api/token/route.ts | 41 ++ .../nextjs/quickstart/example/app/favicon.ico | Bin 0 -> 25931 bytes .../nextjs/quickstart/example/app/globals.css | 126 ++++ .../nextjs/quickstart/example/app/layout.tsx | 34 + .../nextjs/quickstart/example/app/page.tsx | 300 +++++++++ .../nextjs/quickstart/example/components.json | 23 + .../example/components/ui/live-waveform.tsx | 560 ++++++++++++++++ .../quickstart/example/eslint.config.mjs | 18 + .../nextjs/quickstart/example/lib/utils.ts | 6 + .../nextjs/quickstart/example/next.config.ts | 13 + .../nextjs/quickstart/example/package.json | 46 ++ .../quickstart/example/postcss.config.mjs | 7 + .../nextjs/quickstart/example/public/file.svg | 1 + .../quickstart/example/public/globe.svg | 1 + .../nextjs/quickstart/example/public/next.svg | 1 + .../quickstart/example/public/vercel.svg | 1 + .../quickstart/example/public/window.svg | 1 + .../example/scripts/create-engine.mts | 103 +++ .../example/scripts/test-ws-auth.mts | 115 ++++ .../nextjs/quickstart/example/server.mts | 273 ++++++++ .../nextjs/quickstart/example/tsconfig.json | 35 + speech-engine/nextjs/quickstart/setup.sh | 63 ++ test/.gitignore | 41 ++ test/AGENTS.md | 5 + test/CLAUDE.md | 1 + test/README.md | 36 ++ test/app/api/chat/route.ts | 81 +++ test/app/api/token/route.ts | 30 + test/app/api/voice-history/link/route.ts | 33 + test/app/api/voice-history/route.ts | 48 ++ test/app/favicon.ico | Bin 0 -> 25931 bytes test/app/globals.css | 41 ++ test/app/layout.tsx | 33 + test/app/page.tsx | 598 ++++++++++++++++++ test/create-engine.mts | 15 + test/create-server.mts | 72 +++ test/eslint.config.mjs | 18 + test/lib/assistant.ts | 98 +++ test/lib/voice-history.ts | 68 ++ test/next.config.ts | 7 + test/package.json | 36 ++ test/postcss.config.mjs | 7 + test/public/file.svg | 1 + test/public/globe.svg | 1 + test/public/next.svg | 1 + test/public/vercel.svg | 1 + test/public/window.svg | 1 + test/tsconfig.json | 34 + 53 files changed, 3273 insertions(+) create mode 100644 speech-engine/nextjs/quickstart/PROMPT.md create mode 100644 speech-engine/nextjs/quickstart/README.md create mode 100644 speech-engine/nextjs/quickstart/example/.env.example create mode 100644 speech-engine/nextjs/quickstart/example/README.md create mode 100644 speech-engine/nextjs/quickstart/example/app/api/token/route.ts create mode 100644 speech-engine/nextjs/quickstart/example/app/favicon.ico create mode 100644 speech-engine/nextjs/quickstart/example/app/globals.css create mode 100644 speech-engine/nextjs/quickstart/example/app/layout.tsx create mode 100644 speech-engine/nextjs/quickstart/example/app/page.tsx create mode 100644 speech-engine/nextjs/quickstart/example/components.json create mode 100644 speech-engine/nextjs/quickstart/example/components/ui/live-waveform.tsx create mode 100644 speech-engine/nextjs/quickstart/example/eslint.config.mjs create mode 100644 speech-engine/nextjs/quickstart/example/lib/utils.ts create mode 100644 speech-engine/nextjs/quickstart/example/next.config.ts create mode 100644 speech-engine/nextjs/quickstart/example/package.json create mode 100644 speech-engine/nextjs/quickstart/example/postcss.config.mjs create mode 100644 speech-engine/nextjs/quickstart/example/public/file.svg create mode 100644 speech-engine/nextjs/quickstart/example/public/globe.svg create mode 100644 speech-engine/nextjs/quickstart/example/public/next.svg create mode 100644 speech-engine/nextjs/quickstart/example/public/vercel.svg create mode 100644 speech-engine/nextjs/quickstart/example/public/window.svg create mode 100644 speech-engine/nextjs/quickstart/example/scripts/create-engine.mts create mode 100644 speech-engine/nextjs/quickstart/example/scripts/test-ws-auth.mts create mode 100644 speech-engine/nextjs/quickstart/example/server.mts create mode 100644 speech-engine/nextjs/quickstart/example/tsconfig.json create mode 100755 speech-engine/nextjs/quickstart/setup.sh create mode 100644 test/.gitignore create mode 100644 test/AGENTS.md create mode 100644 test/CLAUDE.md create mode 100644 test/README.md create mode 100644 test/app/api/chat/route.ts create mode 100644 test/app/api/token/route.ts create mode 100644 test/app/api/voice-history/link/route.ts create mode 100644 test/app/api/voice-history/route.ts create mode 100644 test/app/favicon.ico create mode 100644 test/app/globals.css create mode 100644 test/app/layout.tsx create mode 100644 test/app/page.tsx create mode 100644 test/create-engine.mts create mode 100644 test/create-server.mts create mode 100644 test/eslint.config.mjs create mode 100644 test/lib/assistant.ts create mode 100644 test/lib/voice-history.ts create mode 100644 test/next.config.ts create mode 100644 test/package.json create mode 100644 test/postcss.config.mjs create mode 100644 test/public/file.svg create mode 100644 test/public/globe.svg create mode 100644 test/public/next.svg create mode 100644 test/public/vercel.svg create mode 100644 test/public/window.svg create mode 100644 test/tsconfig.json diff --git a/README.md b/README.md index bf248693..84b5aa03 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Shared base templates live in `templates/` (Expo, Next.js, Python, TypeScript). - [Sound Effects Quickstart (Python)](sound-effects/python/quickstart/example/README.md) — Generate a sound effect MP3 from a text prompt with the ElevenLabs Python SDK. - [Sound Effects Playground (Next.js)](sound-effects/nextjs/quickstart/example/README.md) — Enter a prompt, generate a sound effect, and play it back in the browser. - [Real-Time Speech-to-Text (Next.js)](speech-to-text/nextjs/realtime/example/README.md) — Live microphone transcription with VAD in a Next.js app. +- [Speech Engine Quickstart (Next.js)](speech-engine/nextjs/quickstart/example/README.md) — Add voice to your own LLM-backed chat agent with a Speech Engine WebSocket server and Next.js client. - [Real-Time Voice Agent (Next.js)](agents/nextjs/quickstart/example/README.md) — Live voice conversations with the ElevenLabs Agents Platform using the React Agents SDK. - [Voice Agent Guardrails Demo (Next.js)](agents/nextjs/guardrails/example/README.md) — Demonstrate custom guardrails and the `guardrail_triggered` client event in a live voice agent. - [Voice Isolator (Next.js)](voice-isolator/nextjs/quickstart/example/README.md) — Record your voice in the browser and remove background noise with the Voice Isolator API. diff --git a/speech-engine/nextjs/quickstart/PROMPT.md b/speech-engine/nextjs/quickstart/PROMPT.md new file mode 100644 index 00000000..edbf1ff3 --- /dev/null +++ b/speech-engine/nextjs/quickstart/PROMPT.md @@ -0,0 +1,57 @@ +Before writing any code, invoke the `/speech-engine` skill to learn the correct ElevenLabs SDK patterns. + +## 1. `package.json` + +- Add `@elevenlabs/react`, `@elevenlabs/elevenlabs-js`, `openai`, and `dotenv`. +- Add `tsx` as a dev dependency. +- Add scripts: `speech-engine:create` runs `scripts/create-engine.mts`; `speech-engine:server` runs `server.mts`. + +## 2. `.env.example` + +- Include `ELEVENLABS_API_KEY`, `ELEVENLABS_SPEECH_ENGINE_ID`, `PUBLIC_WS_URL`, `OPENAI_API_KEY`, and `OPENAI_MODEL=gpt-4o`. + +## 3. `scripts/create-engine.mts` + +Small script that creates a Speech Engine resource. + +- Load env from `.env.local`. +- Validate `ELEVENLABS_API_KEY` and `PUBLIC_WS_URL`. +- Accept `wss://` or `https://` ngrok URLs with `/ws` appended. Normalize `https://` to `wss://` before calling the API. +- Use `ElevenLabsClient` and call `client.speechEngine.create({ name, speechEngine: { wsUrl } })`. +- Print the returned `engineId` or `speechEngineId` clearly so it can be copied to `ELEVENLABS_SPEECH_ENGINE_ID`. + +## 4. `server.mts` + +Node Speech Engine server that listens on port `3001` and attaches the SDK on `/ws`. + +- Load env from `.env.local`. +- Validate `ELEVENLABS_API_KEY`, `ELEVENLABS_SPEECH_ENGINE_ID`, `OPENAI_API_KEY`, and `OPENAI_MODEL`. +- Create a Node HTTP server with `/health`. +- Fetch the resource with `client.speechEngine.get(speechEngineId)`, then call `engine.attach(httpServer, "/ws", callbacks)`. +- In `onTranscript`, call `openai.responses.create` with `stream: true`, concise voice-assistant instructions, and the full transcript mapped to OpenAI roles (`agent` -> `assistant`). +- Pass the provided `AbortSignal` to OpenAI so user interruptions cancel the in-flight response. +- `await session.sendResponse(response)` with the OpenAI stream. +- Wrap `onTranscript` in try/catch; on failure, log the error and send a short fallback string with `session.sendResponse(...)` unless the turn was aborted. +- Log `onInit`, `onClose`, `onDisconnect`, and `onError`; keep `debug: true` for local development. + +## 5. `app/api/token/route.ts` + +Secure GET endpoint that returns a fresh conversation token for `ELEVENLABS_SPEECH_ENGINE_ID`. +Never expose `ELEVENLABS_API_KEY` to the client. + +- Use `ElevenLabsClient`. +- Call `client.conversationalAi.conversations.getWebrtcToken({ agentId: speechEngineId })`. +- Return `{ token }`; return readable JSON errors for missing env and API failures. + +## 6. `app/page.tsx` + +Minimal Speech Engine voice chat page. + +- Wrap the page in `ConversationProvider` and use `useConversation` from `@elevenlabs/react`. +- Show setup reminders: run ngrok for port `3001`, create a Speech Engine with `PUBLIC_WS_URL`, run `pnpm run speech-engine:server`, then run the Next.js app. +- Start voice sessions by requesting microphone access, fetching `/api/token`, and calling `conversation.startSession({ conversationToken: token })`. +- Do not set `overrides.agent.firstMessage` unless the Speech Engine resource explicitly allows that override. +- Track a local starting state so users cannot launch multiple overlapping sessions while WebRTC is connecting. +- Show Start/Stop controls, connection status, and inline errors. +- Use callbacks such as `onConnect`, `onDisconnect`, `onMessage`, and `onError` to render a running transcript when events are available. +- Rely on the SDK's connection-type inference; do not hardcode `connectionType`. diff --git a/speech-engine/nextjs/quickstart/README.md b/speech-engine/nextjs/quickstart/README.md new file mode 100644 index 00000000..b22304ee --- /dev/null +++ b/speech-engine/nextjs/quickstart/README.md @@ -0,0 +1,67 @@ +# Speech Engine Quickstart (Next.js) + +Add voice to your own LLM-backed chat agent with ElevenLabs Speech Engine, a Node WebSocket server, and a Next.js browser client. + +## Setup + +1. Copy the environment file and add your credentials: + + ```bash + cp .env.example .env.local + ``` + + Then edit `.env.local` and set: + - `ELEVENLABS_API_KEY` + - `OPENAI_API_KEY` + - `OPENAI_MODEL` + +2. Install dependencies: + + ```bash + pnpm install + ``` + +3. Expose the Speech Engine server with ngrok: + + ```bash + ngrok http 3001 + ``` + + Copy the forwarding URL into `.env.local` as `PUBLIC_WS_URL` with `/ws` appended, for example `wss://abc123.ngrok.app/ws`. Use `wss://` (the create script accepts `https://` and converts it). + +4. Create a Speech Engine and copy the printed id into `.env.local`: + + ```bash + pnpm run speech-engine:create + ``` + + Set the value as `ELEVENLABS_SPEECH_ENGINE_ID`. If your ngrok URL changes, run this command again and update the id. + +## Run + +Start the Speech Engine server: + +```bash +pnpm run speech-engine:server +``` + +In another terminal, start the Next.js app: + +```bash +pnpm run dev +``` + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## Usage + +- Keep ngrok, `pnpm run speech-engine:server`, and `pnpm run dev` running. +- Click **Start conversation** and allow microphone access. The app fetches a conversation token from the server and starts a WebRTC voice session. +- Once connected, speak naturally. ElevenLabs transcribes your speech, your server streams an OpenAI response back, and ElevenLabs plays the generated voice response in the browser. +- Click **Stop conversation** to end the session. + +## Troubleshooting + +If the browser cannot connect, check the Speech Engine server terminal first. You should see a session log when the browser starts a conversation. If you do not, confirm ngrok is still forwarding to port `3001`, `PUBLIC_WS_URL` includes `/ws`, and `ELEVENLABS_SPEECH_ENGINE_ID` points to an engine created with that URL. If you previously created the Speech Engine with a different ngrok URL, run `pnpm run speech-engine:create` again and update the id. + +When you speak, the Speech Engine server terminal should log `[speech-engine] transcript:` followed by `[speech-engine] response sent for:`. If you see `onTranscript error` instead, fix the reported OpenAI or server issue first. A backend failure can also surface in the browser as a client SDK error about `error_type`. diff --git a/speech-engine/nextjs/quickstart/example/.env.example b/speech-engine/nextjs/quickstart/example/.env.example new file mode 100644 index 00000000..4479ead9 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/.env.example @@ -0,0 +1,6 @@ +ELEVENLABS_API_KEY= +ELEVENLABS_SPEECH_ENGINE_ID= +PUBLIC_WS_URL= +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o +SPEECH_ENGINE_DEV_SKIP_AUTH=true diff --git a/speech-engine/nextjs/quickstart/example/README.md b/speech-engine/nextjs/quickstart/example/README.md new file mode 100644 index 00000000..b22304ee --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/README.md @@ -0,0 +1,67 @@ +# Speech Engine Quickstart (Next.js) + +Add voice to your own LLM-backed chat agent with ElevenLabs Speech Engine, a Node WebSocket server, and a Next.js browser client. + +## Setup + +1. Copy the environment file and add your credentials: + + ```bash + cp .env.example .env.local + ``` + + Then edit `.env.local` and set: + - `ELEVENLABS_API_KEY` + - `OPENAI_API_KEY` + - `OPENAI_MODEL` + +2. Install dependencies: + + ```bash + pnpm install + ``` + +3. Expose the Speech Engine server with ngrok: + + ```bash + ngrok http 3001 + ``` + + Copy the forwarding URL into `.env.local` as `PUBLIC_WS_URL` with `/ws` appended, for example `wss://abc123.ngrok.app/ws`. Use `wss://` (the create script accepts `https://` and converts it). + +4. Create a Speech Engine and copy the printed id into `.env.local`: + + ```bash + pnpm run speech-engine:create + ``` + + Set the value as `ELEVENLABS_SPEECH_ENGINE_ID`. If your ngrok URL changes, run this command again and update the id. + +## Run + +Start the Speech Engine server: + +```bash +pnpm run speech-engine:server +``` + +In another terminal, start the Next.js app: + +```bash +pnpm run dev +``` + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## Usage + +- Keep ngrok, `pnpm run speech-engine:server`, and `pnpm run dev` running. +- Click **Start conversation** and allow microphone access. The app fetches a conversation token from the server and starts a WebRTC voice session. +- Once connected, speak naturally. ElevenLabs transcribes your speech, your server streams an OpenAI response back, and ElevenLabs plays the generated voice response in the browser. +- Click **Stop conversation** to end the session. + +## Troubleshooting + +If the browser cannot connect, check the Speech Engine server terminal first. You should see a session log when the browser starts a conversation. If you do not, confirm ngrok is still forwarding to port `3001`, `PUBLIC_WS_URL` includes `/ws`, and `ELEVENLABS_SPEECH_ENGINE_ID` points to an engine created with that URL. If you previously created the Speech Engine with a different ngrok URL, run `pnpm run speech-engine:create` again and update the id. + +When you speak, the Speech Engine server terminal should log `[speech-engine] transcript:` followed by `[speech-engine] response sent for:`. If you see `onTranscript error` instead, fix the reported OpenAI or server issue first. A backend failure can also surface in the browser as a client SDK error about `error_type`. diff --git a/speech-engine/nextjs/quickstart/example/app/api/token/route.ts b/speech-engine/nextjs/quickstart/example/app/api/token/route.ts new file mode 100644 index 00000000..1b4bf9fb --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/api/token/route.ts @@ -0,0 +1,41 @@ +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; +import { NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + + if (!value) { + throw new Error(`Missing ${name}.`); + } + + return value; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : "Failed to create token."; +} + +export async function GET() { + try { + const client = new ElevenLabsClient({ + apiKey: requireEnv("ELEVENLABS_API_KEY"), + }); + const speechEngineId = requireEnv("ELEVENLABS_SPEECH_ENGINE_ID"); + const response = + await client.conversationalAi.conversations.getWebrtcToken({ + agentId: speechEngineId, + }); + + return NextResponse.json({ token: response.token }); + } catch (error: unknown) { + return NextResponse.json( + { + error: "Unable to create a conversation token.", + details: errorMessage(error), + }, + { status: 500 }, + ); + } +} diff --git a/speech-engine/nextjs/quickstart/example/app/favicon.ico b/speech-engine/nextjs/quickstart/example/app/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..718d6fea4835ec2d246af9800eddb7ffb276240c GIT binary patch literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m literal 0 HcmV?d00001 diff --git a/speech-engine/nextjs/quickstart/example/app/globals.css b/speech-engine/nextjs/quickstart/example/app/globals.css new file mode 100644 index 00000000..d767ad63 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/globals.css @@ -0,0 +1,126 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); + --color-sidebar-ring: var(--sidebar-ring); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar: var(--sidebar); + --color-chart-5: var(--chart-5); + --color-chart-4: var(--chart-4); + --color-chart-3: var(--chart-3); + --color-chart-2: var(--chart-2); + --color-chart-1: var(--chart-1); + --color-ring: var(--ring); + --color-input: var(--input); + --color-border: var(--border); + --color-destructive: var(--destructive); + --color-accent-foreground: var(--accent-foreground); + --color-accent: var(--accent); + --color-muted-foreground: var(--muted-foreground); + --color-muted: var(--muted); + --color-secondary-foreground: var(--secondary-foreground); + --color-secondary: var(--secondary); + --color-primary-foreground: var(--primary-foreground); + --color-primary: var(--primary); + --color-popover-foreground: var(--popover-foreground); + --color-popover: var(--popover); + --color-card-foreground: var(--card-foreground); + --color-card: var(--card); + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.145 0 0); + --card: oklch(1 0 0); + --card-foreground: oklch(0.145 0 0); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.145 0 0); + --primary: oklch(0.205 0 0); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.97 0 0); + --secondary-foreground: oklch(0.205 0 0); + --muted: oklch(0.97 0 0); + --muted-foreground: oklch(0.556 0 0); + --accent: oklch(0.97 0 0); + --accent-foreground: oklch(0.205 0 0); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.922 0 0); + --input: oklch(0.922 0 0); + --ring: oklch(0.708 0 0); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.145 0 0); + --sidebar-primary: oklch(0.205 0 0); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.97 0 0); + --sidebar-accent-foreground: oklch(0.205 0 0); + --sidebar-border: oklch(0.922 0 0); + --sidebar-ring: oklch(0.708 0 0); +} + +.dark { + --background: oklch(0.145 0 0); + --foreground: oklch(0.985 0 0); + --card: oklch(0.205 0 0); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.205 0 0); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.922 0 0); + --primary-foreground: oklch(0.205 0 0); + --secondary: oklch(0.269 0 0); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.269 0 0); + --muted-foreground: oklch(0.708 0 0); + --accent: oklch(0.269 0 0); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.556 0 0); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.205 0 0); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.269 0 0); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.556 0 0); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} diff --git a/speech-engine/nextjs/quickstart/example/app/layout.tsx b/speech-engine/nextjs/quickstart/example/app/layout.tsx new file mode 100644 index 00000000..c4b1390e --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/layout.tsx @@ -0,0 +1,34 @@ +import type { Metadata } from "next"; +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "Speech Engine Quickstart", + description: "Voice chat with ElevenLabs Speech Engine and Next.js", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + {children} + + + ); +} diff --git a/speech-engine/nextjs/quickstart/example/app/page.tsx b/speech-engine/nextjs/quickstart/example/app/page.tsx new file mode 100644 index 00000000..ededa651 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/app/page.tsx @@ -0,0 +1,300 @@ +"use client"; + +import { ConversationProvider, useConversation } from "@elevenlabs/react"; +import { useCallback, useRef, useState } from "react"; + +type TranscriptRole = "assistant" | "system" | "user"; + +type TranscriptEntry = { + id: number; + role: TranscriptRole; + text: string; +}; + +type TokenResponse = { + token?: string; + error?: string; + details?: string; +}; + +function formatError(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + + if (typeof error === "string" && error.trim().length > 0) { + return error; + } + + return "Voice chat failed."; +} + +function normalizeMessage( + value: unknown, +): { role: "assistant" | "user"; text: string } | null { + if (!value || typeof value !== "object") { + return null; + } + + const record = value as Record; + const text = [record.message, record.text, record.content].find( + (candidate): candidate is string => + typeof candidate === "string" && candidate.trim().length > 0, + ); + + if (!text) { + return null; + } + + const source = record.source ?? record.role ?? record.type; + const role = source === "user" ? "user" : "assistant"; + + return { role, text }; +} + +function SpeechEnginePage() { + const [error, setError] = useState(null); + const [isStarting, setIsStarting] = useState(false); + const [transcript, setTranscript] = useState([]); + const nextTranscriptId = useRef(0); + + const appendTranscript = useCallback((role: TranscriptRole, text: string) => { + const trimmedText = text.trim(); + + if (!trimmedText) { + return; + } + + const id = nextTranscriptId.current + 1; + nextTranscriptId.current = id; + + setTranscript((current) => [ + ...current, + { + id, + role, + text: trimmedText, + }, + ]); + }, []); + + const conversation = useConversation({ + onConnect({ conversationId }) { + setError(null); + appendTranscript( + "system", + `Connected to the Speech Engine session: ${conversationId}. Start speaking when you are ready.`, + ); + }, + onDisconnect(details) { + appendTranscript( + "system", + details.reason === "error" + ? `Disconnected: ${details.message}` + : "Disconnected from the Speech Engine session.", + ); + }, + onMessage(message) { + const entry = normalizeMessage(message); + + if (entry) { + appendTranscript(entry.role, entry.text); + } + }, + onError(message) { + const nextError = formatError(message); + + setError(nextError); + appendTranscript("system", nextError); + }, + onDebug(event) { + console.debug("[speech-engine client]", JSON.stringify(event, null, 2)); + }, + }); + + const startConversation = useCallback(async () => { + try { + setError(null); + setIsStarting(true); + conversation.endSession(); + setTranscript([]); + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + stream.getTracks().forEach((track) => track.stop()); + + const response = await fetch("/api/token", { cache: "no-store" }); + const data = (await response.json()) as TokenResponse; + + if (!response.ok || !data.token) { + throw new Error( + data.details ?? data.error ?? "Failed to fetch a conversation token.", + ); + } + + appendTranscript("system", "Starting a new voice session."); + conversation.startSession({ conversationToken: data.token }); + } catch (startError: unknown) { + const nextError = formatError(startError); + + setError(nextError); + appendTranscript("system", nextError); + } finally { + setIsStarting(false); + } + }, [appendTranscript, conversation]); + + const stopConversation = useCallback(() => { + setError(null); + conversation.endSession(); + }, [conversation]); + + const [typedMessage, setTypedMessage] = useState(""); + + const sendTypedMessage = useCallback(() => { + const text = typedMessage.trim(); + + if (!text || conversation.status !== "connected") { + return; + } + + setError(null); + appendTranscript("user", text); + conversation.sendUserMessage(text); + setTypedMessage(""); + }, [appendTranscript, conversation, typedMessage]); + + return ( +
+
+
+

+ Speech Engine Voice Chat +

+

+ Add voice to your own OpenAI-backed agent with ElevenLabs Speech + Engine. +

+
+ +
+

+ Local setup reminders +

+
    +
  1. 1. Run `ngrok http 3001`.
    2. +
  2. + 2. Set `PUBLIC_WS_URL` to your ngrok URL with `/ws` appended. +
    3. +
  3. 3. Run `pnpm run speech-engine:create`.
    4. +
  4. 4. Run `pnpm run speech-engine:server` and `pnpm run dev`.
    5. +
+
+ +
+
+
+

+ Connection status +

+

+ {conversation.status} +

+
+ +
+ + +
+
+ + {error ? ( +

+ {error} +

+ ) : null} + + {conversation.status === "connected" ? ( +
+ setTypedMessage(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + sendTypedMessage(); + } + }} + placeholder="Or type a message to test without the microphone" + type="text" + value={typedMessage} + /> + +
+ ) : null} +
+ +
+
+

Transcript

+

+ {conversation.status} +

+
+ +
+ {transcript.length > 0 ? ( + transcript.map((entry) => ( +
+

+ {entry.role} +

+

{entry.text}

+
+ )) + ) : ( +

+ Start a session to capture conversation events here. +

+ )} +
+
+
+
+ ); +} + +export default function Home() { + return ( + + + + ); +} diff --git a/speech-engine/nextjs/quickstart/example/components.json b/speech-engine/nextjs/quickstart/example/components.json new file mode 100644 index 00000000..f87021ee --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/components.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "", + "css": "app/globals.css", + "baseColor": "neutral", + "cssVariables": true, + "prefix": "" + }, + "iconLibrary": "lucide", + "rtl": false, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "registries": {} +} diff --git a/speech-engine/nextjs/quickstart/example/components/ui/live-waveform.tsx b/speech-engine/nextjs/quickstart/example/components/ui/live-waveform.tsx new file mode 100644 index 00000000..d3533506 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/components/ui/live-waveform.tsx @@ -0,0 +1,560 @@ +"use client"; + +import { useEffect, useRef, type HTMLAttributes } from "react"; + +import { cn } from "@/lib/utils"; + +export type LiveWaveformProps = HTMLAttributes & { + active?: boolean; + processing?: boolean; + deviceId?: string; + barWidth?: number; + barHeight?: number; + barGap?: number; + barRadius?: number; + barColor?: string; + fadeEdges?: boolean; + fadeWidth?: number; + height?: string | number; + sensitivity?: number; + smoothingTimeConstant?: number; + fftSize?: number; + historySize?: number; + updateRate?: number; + mode?: "scrolling" | "static"; + onError?: (error: Error) => void; + onStreamReady?: (stream: MediaStream) => void; + onStreamEnd?: () => void; +}; + +export const LiveWaveform = ({ + active = false, + processing = false, + deviceId, + barWidth = 3, + barGap = 1, + barRadius = 1.5, + barColor, + fadeEdges = true, + fadeWidth = 24, + barHeight: baseBarHeight = 4, + height = 64, + sensitivity = 1, + smoothingTimeConstant = 0.8, + fftSize = 256, + historySize = 60, + updateRate = 30, + mode = "static", + onError, + onStreamReady, + onStreamEnd, + className, + ...props +}: LiveWaveformProps) => { + const canvasRef = useRef(null); + const containerRef = useRef(null); + const historyRef = useRef([]); + const analyserRef = useRef(null); + const audioContextRef = useRef(null); + const streamRef = useRef(null); + const animationRef = useRef(0); + const lastUpdateRef = useRef(0); + const processingAnimationRef = useRef(null); + const lastActiveDataRef = useRef([]); + const transitionProgressRef = useRef(0); + const staticBarsRef = useRef([]); + const needsRedrawRef = useRef(true); + const gradientCacheRef = useRef(null); + const lastWidthRef = useRef(0); + + const heightStyle = typeof height === "number" ? `${height}px` : height; + + // Handle canvas resizing + useEffect(() => { + const canvas = canvasRef.current; + const container = containerRef.current; + if (!canvas || !container) return; + + const resizeObserver = new ResizeObserver(() => { + const rect = container.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + + canvas.width = rect.width * dpr; + canvas.height = rect.height * dpr; + canvas.style.width = `${rect.width}px`; + canvas.style.height = `${rect.height}px`; + + const ctx = canvas.getContext("2d"); + if (ctx) { + ctx.scale(dpr, dpr); + } + + gradientCacheRef.current = null; + lastWidthRef.current = rect.width; + needsRedrawRef.current = true; + }); + + resizeObserver.observe(container); + return () => resizeObserver.disconnect(); + }, []); + + useEffect(() => { + if (processing && !active) { + let time = 0; + transitionProgressRef.current = 0; + + const animateProcessing = () => { + time += 0.03; + transitionProgressRef.current = Math.min( + 1, + transitionProgressRef.current + 0.02 + ); + + const processingData = []; + const barCount = Math.floor( + (containerRef.current?.getBoundingClientRect().width || 200) / + (barWidth + barGap) + ); + + if (mode === "static") { + const halfCount = Math.floor(barCount / 2); + + for (let i = 0; i < barCount; i++) { + const normalizedPosition = (i - halfCount) / halfCount; + const centerWeight = 1 - Math.abs(normalizedPosition) * 0.4; + + const wave1 = Math.sin(time * 1.5 + normalizedPosition * 3) * 0.25; + const wave2 = Math.sin(time * 0.8 - normalizedPosition * 2) * 0.2; + const wave3 = Math.cos(time * 2 + normalizedPosition) * 0.15; + const combinedWave = wave1 + wave2 + wave3; + const processingValue = (0.2 + combinedWave) * centerWeight; + + let finalValue = processingValue; + if ( + lastActiveDataRef.current.length > 0 && + transitionProgressRef.current < 1 + ) { + const lastDataIndex = Math.min( + i, + lastActiveDataRef.current.length - 1 + ); + const lastValue = lastActiveDataRef.current[lastDataIndex] || 0; + finalValue = + lastValue * (1 - transitionProgressRef.current) + + processingValue * transitionProgressRef.current; + } + + processingData.push(Math.max(0.05, Math.min(1, finalValue))); + } + } else { + for (let i = 0; i < barCount; i++) { + const normalizedPosition = (i - barCount / 2) / (barCount / 2); + const centerWeight = 1 - Math.abs(normalizedPosition) * 0.4; + + const wave1 = Math.sin(time * 1.5 + i * 0.15) * 0.25; + const wave2 = Math.sin(time * 0.8 - i * 0.1) * 0.2; + const wave3 = Math.cos(time * 2 + i * 0.05) * 0.15; + const combinedWave = wave1 + wave2 + wave3; + const processingValue = (0.2 + combinedWave) * centerWeight; + + let finalValue = processingValue; + if ( + lastActiveDataRef.current.length > 0 && + transitionProgressRef.current < 1 + ) { + const lastDataIndex = Math.floor( + (i / barCount) * lastActiveDataRef.current.length + ); + const lastValue = lastActiveDataRef.current[lastDataIndex] || 0; + finalValue = + lastValue * (1 - transitionProgressRef.current) + + processingValue * transitionProgressRef.current; + } + + processingData.push(Math.max(0.05, Math.min(1, finalValue))); + } + } + + if (mode === "static") { + staticBarsRef.current = processingData; + } else { + historyRef.current = processingData; + } + + needsRedrawRef.current = true; + processingAnimationRef.current = + requestAnimationFrame(animateProcessing); + }; + + animateProcessing(); + + return () => { + if (processingAnimationRef.current) { + cancelAnimationFrame(processingAnimationRef.current); + } + }; + } else if (!active && !processing) { + const hasData = + mode === "static" + ? staticBarsRef.current.length > 0 + : historyRef.current.length > 0; + + if (hasData) { + let fadeProgress = 0; + const fadeToIdle = () => { + fadeProgress += 0.03; + if (fadeProgress < 1) { + if (mode === "static") { + staticBarsRef.current = staticBarsRef.current.map( + value => value * (1 - fadeProgress) + ); + } else { + historyRef.current = historyRef.current.map( + value => value * (1 - fadeProgress) + ); + } + needsRedrawRef.current = true; + requestAnimationFrame(fadeToIdle); + } else { + if (mode === "static") { + staticBarsRef.current = []; + } else { + historyRef.current = []; + } + } + }; + fadeToIdle(); + } + } + }, [processing, active, barWidth, barGap, mode]); + + // Handle microphone setup and teardown + useEffect(() => { + if (!active) { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + onStreamEnd?.(); + } + if ( + audioContextRef.current && + audioContextRef.current.state !== "closed" + ) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (animationRef.current) { + cancelAnimationFrame(animationRef.current); + animationRef.current = 0; + } + return; + } + + const setupMicrophone = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: deviceId + ? { + deviceId: { exact: deviceId }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + : { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + streamRef.current = stream; + onStreamReady?.(stream); + + const AudioContextConstructor = + window.AudioContext || + (window as unknown as { webkitAudioContext: typeof AudioContext }) + .webkitAudioContext; + const audioContext = new AudioContextConstructor(); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = fftSize; + analyser.smoothingTimeConstant = smoothingTimeConstant; + + const source = audioContext.createMediaStreamSource(stream); + source.connect(analyser); + + audioContextRef.current = audioContext; + analyserRef.current = analyser; + + // Clear history when starting + historyRef.current = []; + } catch (error) { + onError?.(error as Error); + } + }; + + setupMicrophone(); + + return () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + onStreamEnd?.(); + } + if ( + audioContextRef.current && + audioContextRef.current.state !== "closed" + ) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (animationRef.current) { + cancelAnimationFrame(animationRef.current); + animationRef.current = 0; + } + }; + }, [ + active, + deviceId, + fftSize, + smoothingTimeConstant, + onError, + onStreamReady, + onStreamEnd, + ]); + + // Animation loop + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + let rafId: number; + + const animate = (currentTime: number) => { + // Render waveform + const rect = canvas.getBoundingClientRect(); + + // Update audio data if active + if (active && currentTime - lastUpdateRef.current > updateRate) { + lastUpdateRef.current = currentTime; + + if (analyserRef.current) { + const dataArray = new Uint8Array( + analyserRef.current.frequencyBinCount + ); + analyserRef.current.getByteFrequencyData(dataArray); + + if (mode === "static") { + // For static mode, update bars in place + const startFreq = Math.floor(dataArray.length * 0.05); + const endFreq = Math.floor(dataArray.length * 0.4); + const relevantData = dataArray.slice(startFreq, endFreq); + + const barCount = Math.floor(rect.width / (barWidth + barGap)); + const halfCount = Math.floor(barCount / 2); + const newBars: number[] = []; + + // Mirror the data for symmetric display + for (let i = halfCount - 1; i >= 0; i--) { + const dataIndex = Math.floor( + (i / halfCount) * relevantData.length + ); + const value = Math.min( + 1, + (relevantData[dataIndex] / 255) * sensitivity + ); + newBars.push(Math.max(0.05, value)); + } + + for (let i = 0; i < halfCount; i++) { + const dataIndex = Math.floor( + (i / halfCount) * relevantData.length + ); + const value = Math.min( + 1, + (relevantData[dataIndex] / 255) * sensitivity + ); + newBars.push(Math.max(0.05, value)); + } + + staticBarsRef.current = newBars; + lastActiveDataRef.current = newBars; + } else { + // Scrolling mode - original behavior + let sum = 0; + const startFreq = Math.floor(dataArray.length * 0.05); + const endFreq = Math.floor(dataArray.length * 0.4); + const relevantData = dataArray.slice(startFreq, endFreq); + + for (let i = 0; i < relevantData.length; i++) { + sum += relevantData[i]; + } + const average = (sum / relevantData.length / 255) * sensitivity; + + // Add to history + historyRef.current.push(Math.min(1, Math.max(0.05, average))); + lastActiveDataRef.current = [...historyRef.current]; + + // Maintain history size + if (historyRef.current.length > historySize) { + historyRef.current.shift(); + } + } + needsRedrawRef.current = true; + } + } + + // Only redraw if needed + if (!needsRedrawRef.current && !active) { + rafId = requestAnimationFrame(animate); + return; + } + + needsRedrawRef.current = active; + ctx.clearRect(0, 0, rect.width, rect.height); + + const computedBarColor = + barColor || + (() => { + const style = getComputedStyle(canvas); + // Try to get the computed color value directly + const color = style.color; + return color || "#000"; + })(); + + const step = barWidth + barGap; + const barCount = Math.floor(rect.width / step); + const centerY = rect.height / 2; + + // Draw bars based on mode + if (mode === "static") { + // Static mode - bars in fixed positions + const dataToRender = processing + ? staticBarsRef.current + : active + ? staticBarsRef.current + : staticBarsRef.current.length > 0 + ? staticBarsRef.current + : []; + + for (let i = 0; i < barCount && i < dataToRender.length; i++) { + const value = dataToRender[i] || 0.1; + const x = i * step; + const barHeight = Math.max(baseBarHeight, value * rect.height * 0.8); + const y = centerY - barHeight / 2; + + ctx.fillStyle = computedBarColor; + ctx.globalAlpha = 0.4 + value * 0.6; + + if (barRadius > 0) { + ctx.beginPath(); + ctx.roundRect(x, y, barWidth, barHeight, barRadius); + ctx.fill(); + } else { + ctx.fillRect(x, y, barWidth, barHeight); + } + } + } else { + // Scrolling mode - original behavior + for (let i = 0; i < barCount && i < historyRef.current.length; i++) { + const dataIndex = historyRef.current.length - 1 - i; + const value = historyRef.current[dataIndex] || 0.1; + const x = rect.width - (i + 1) * step; + const barHeight = Math.max(baseBarHeight, value * rect.height * 0.8); + const y = centerY - barHeight / 2; + + ctx.fillStyle = computedBarColor; + ctx.globalAlpha = 0.4 + value * 0.6; + + if (barRadius > 0) { + ctx.beginPath(); + ctx.roundRect(x, y, barWidth, barHeight, barRadius); + ctx.fill(); + } else { + ctx.fillRect(x, y, barWidth, barHeight); + } + } + } + + // Apply edge fading + if (fadeEdges && fadeWidth > 0 && rect.width > 0) { + // Cache gradient if width hasn't changed + if (!gradientCacheRef.current || lastWidthRef.current !== rect.width) { + const gradient = ctx.createLinearGradient(0, 0, rect.width, 0); + const fadePercent = Math.min(0.3, fadeWidth / rect.width); + + // destination-out: removes destination where source alpha is high + // We want: fade edges out, keep center solid + // Left edge: start opaque (1) = remove, fade to transparent (0) = keep + gradient.addColorStop(0, "rgba(255,255,255,1)"); + gradient.addColorStop(fadePercent, "rgba(255,255,255,0)"); + // Center stays transparent = keep everything + gradient.addColorStop(1 - fadePercent, "rgba(255,255,255,0)"); + // Right edge: fade from transparent (0) = keep to opaque (1) = remove + gradient.addColorStop(1, "rgba(255,255,255,1)"); + + gradientCacheRef.current = gradient; + lastWidthRef.current = rect.width; + } + + ctx.globalCompositeOperation = "destination-out"; + ctx.fillStyle = gradientCacheRef.current; + ctx.fillRect(0, 0, rect.width, rect.height); + ctx.globalCompositeOperation = "source-over"; + } + + ctx.globalAlpha = 1; + + rafId = requestAnimationFrame(animate); + }; + + rafId = requestAnimationFrame(animate); + + return () => { + if (rafId) { + cancelAnimationFrame(rafId); + } + }; + }, [ + active, + processing, + sensitivity, + updateRate, + historySize, + barWidth, + baseBarHeight, + barGap, + barRadius, + barColor, + fadeEdges, + fadeWidth, + mode, + ]); + + return ( +
+ {!active && !processing && ( +
+ )} +
+ ); +}; diff --git a/speech-engine/nextjs/quickstart/example/eslint.config.mjs b/speech-engine/nextjs/quickstart/example/eslint.config.mjs new file mode 100644 index 00000000..05e726d1 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/eslint.config.mjs @@ -0,0 +1,18 @@ +import { defineConfig, globalIgnores } from "eslint/config"; +import nextVitals from "eslint-config-next/core-web-vitals"; +import nextTs from "eslint-config-next/typescript"; + +const eslintConfig = defineConfig([ + ...nextVitals, + ...nextTs, + // Override default ignores of eslint-config-next. + globalIgnores([ + // Default ignores of eslint-config-next: + ".next/**", + "out/**", + "build/**", + "next-env.d.ts", + ]), +]); + +export default eslintConfig; diff --git a/speech-engine/nextjs/quickstart/example/lib/utils.ts b/speech-engine/nextjs/quickstart/example/lib/utils.ts new file mode 100644 index 00000000..a5ef1935 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/lib/utils.ts @@ -0,0 +1,6 @@ +import { clsx, type ClassValue } from "clsx"; +import { twMerge } from "tailwind-merge"; + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} diff --git a/speech-engine/nextjs/quickstart/example/next.config.ts b/speech-engine/nextjs/quickstart/example/next.config.ts new file mode 100644 index 00000000..fff67474 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/next.config.ts @@ -0,0 +1,13 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { NextConfig } from "next"; + +const projectRoot = path.dirname(fileURLToPath(import.meta.url)); + +const nextConfig: NextConfig = { + turbopack: { + root: projectRoot, + }, +}; + +export default nextConfig; diff --git a/speech-engine/nextjs/quickstart/example/package.json b/speech-engine/nextjs/quickstart/example/package.json new file mode 100644 index 00000000..db20e79a --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/package.json @@ -0,0 +1,46 @@ +{ + "name": "speech-engine-quickstart", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "eslint", + "speech-engine:create": "tsx scripts/create-engine.mts", + "speech-engine:server": "tsx server.mts", + "speech-engine:test-ws": "tsx scripts/test-ws-auth.mts" + }, + "dependencies": { + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "lucide-react": "^0.575.0", + "next": "16.1.6", + "radix-ui": "^1.4.3", + "react": "19.2.3", + "react-dom": "19.2.3", + "tailwind-merge": "^3.5.0", + "@elevenlabs/react": "^1.6.0", + "@elevenlabs/elevenlabs-js": "^2.47.0", + "openai": "^6.37.0", + "dotenv": "^17.4.2" + }, + "devDependencies": { + "@tailwindcss/postcss": "^4", + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "eslint": "^9", + "eslint-config-next": "16.1.6", + "shadcn": "^3.8.5", + "tailwindcss": "^4", + "tw-animate-css": "^1.4.0", + "typescript": "^5", + "tsx": "^4.22.0" + }, + "pnpm": { + "overrides": { + "livekit-client": "2.16.1" + } + } +} diff --git a/speech-engine/nextjs/quickstart/example/postcss.config.mjs b/speech-engine/nextjs/quickstart/example/postcss.config.mjs new file mode 100644 index 00000000..61e36849 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/postcss.config.mjs @@ -0,0 +1,7 @@ +const config = { + plugins: { + "@tailwindcss/postcss": {}, + }, +}; + +export default config; diff --git a/speech-engine/nextjs/quickstart/example/public/file.svg b/speech-engine/nextjs/quickstart/example/public/file.svg new file mode 100644 index 00000000..004145cd --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/public/file.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/speech-engine/nextjs/quickstart/example/public/globe.svg b/speech-engine/nextjs/quickstart/example/public/globe.svg new file mode 100644 index 00000000..567f17b0 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/public/globe.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/speech-engine/nextjs/quickstart/example/public/next.svg b/speech-engine/nextjs/quickstart/example/public/next.svg new file mode 100644 index 00000000..5174b28c --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/public/next.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/speech-engine/nextjs/quickstart/example/public/vercel.svg b/speech-engine/nextjs/quickstart/example/public/vercel.svg new file mode 100644 index 00000000..77053960 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/public/vercel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/speech-engine/nextjs/quickstart/example/public/window.svg b/speech-engine/nextjs/quickstart/example/public/window.svg new file mode 100644 index 00000000..b2b2a44f --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/public/window.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/speech-engine/nextjs/quickstart/example/scripts/create-engine.mts b/speech-engine/nextjs/quickstart/example/scripts/create-engine.mts new file mode 100644 index 00000000..dbad21fd --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/scripts/create-engine.mts @@ -0,0 +1,103 @@ +import { resolve as resolvePath } from "node:path"; + +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; +import dotenv from "dotenv"; + +const envPath = resolvePath(process.cwd(), ".env.local"); + +dotenv.config({ path: envPath }); + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + + if (!value) { + throw new Error(`Missing ${name} in ${envPath}.`); + } + + return value; +} + +function normalizeWebSocketUrl(url: string): string { + const trimmed = url.trim(); + + if (trimmed.startsWith("wss://")) { + return trimmed; + } + + if (trimmed.startsWith("https://")) { + return `wss://${trimmed.slice("https://".length)}`; + } + + if (trimmed.startsWith("http://")) { + return `ws://${trimmed.slice("http://".length)}`; + } + + throw new Error( + "PUBLIC_WS_URL must start with wss://, https://, ws://, or http://.", + ); +} + +function readSpeechEngineId(engine: unknown): string { + if (!engine || typeof engine !== "object") { + throw new Error("Speech Engine create response did not include an id."); + } + + const record = engine as Record; + const id = record.engineId ?? record.speechEngineId ?? record.speech_engine_id; + + if (typeof id !== "string" || id.length === 0) { + throw new Error("Speech Engine create response did not include an id."); + } + + return id; +} + +async function main() { + const apiKey = requireEnv("ELEVENLABS_API_KEY"); + const wsUrl = normalizeWebSocketUrl(requireEnv("PUBLIC_WS_URL")); + const client = new ElevenLabsClient({ apiKey }); + + console.log(`Creating Speech Engine with WebSocket URL: ${wsUrl}`); + + const engine = await client.speechEngine.create({ + name: "Speech Engine Quickstart", + speechEngine: { + wsUrl, + }, + }); + const speechEngineId = readSpeechEngineId(engine); + + const engines = await client.speechEngine.list(); + const duplicates = engines.speechEngines.filter( + (entry) => entry.speechEngineId !== speechEngineId, + ); + + console.log("Speech Engine created successfully."); + console.log(`Speech Engine ID: ${speechEngineId}`); + console.log(""); + console.log(`Copy this into .env.local:`); + console.log(`ELEVENLABS_SPEECH_ENGINE_ID=${speechEngineId}`); + + if (duplicates.length > 0) { + console.log(""); + console.warn( + `Warning: ${duplicates.length} other Speech Engine resource(s) still exist.`, + ); + console.warn( + "Delete unused engines in the ElevenLabs dashboard or via the API.", + ); + console.warn( + "Multiple engines pointing at the same ws_url can cause upstream connection failures.", + ); + } +} + +main().catch((error: unknown) => { + const message = + error instanceof Error ? error.message : "Failed to create Speech Engine."; + + console.error(message); + process.exitCode = 1; +}); + +export {}; diff --git a/speech-engine/nextjs/quickstart/example/scripts/test-ws-auth.mts b/speech-engine/nextjs/quickstart/example/scripts/test-ws-auth.mts new file mode 100644 index 00000000..fa0dd712 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/scripts/test-ws-auth.mts @@ -0,0 +1,115 @@ +import crypto from "node:crypto"; +import { createRequire } from "node:module"; +import { resolve } from "node:path"; + +import dotenv from "dotenv"; + +const require = createRequire(import.meta.url); +const WebSocket = require("ws") as typeof import("ws"); + +const envPath = resolve(process.cwd(), ".env.local"); + +dotenv.config({ path: envPath, quiet: true }); + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + + if (!value) { + throw new Error(`Missing ${name} in ${envPath}.`); + } + + return value; +} + +function base64Url(value: string): string { + return Buffer.from(value).toString("base64url"); +} + +function signJwt(apiKey: string, agentId: string, conversationId: string): string { + const secret = crypto.createHash("sha256").update(apiKey.trim()).digest(); + const now = Math.floor(Date.now() / 1000); + const header = base64Url(JSON.stringify({ alg: "HS256", typ: "JWT" })); + const payload = base64Url( + JSON.stringify({ + iss: "https://api.elevenlabs.io/convai/speech-engine", + sub: "convai_speech_engine_upstream", + agent_id: agentId, + conversation_id: conversationId, + iat: now, + exp: now + 600, + }), + ); + const signature = crypto + .createHmac("sha256", secret) + .update(`${header}.${payload}`) + .digest("base64url"); + + return `${header}.${payload}.${signature}`; +} + +async function main() { + const apiKey = requireEnv("ELEVENLABS_API_KEY"); + const engineId = requireEnv("ELEVENLABS_SPEECH_ENGINE_ID"); + const agentId = engineId.replace(/^seng_/, "agent_"); + const conversationId = "conv_test_local"; + const token = signJwt(apiKey, agentId, conversationId); + + await new Promise((resolvePromise, reject) => { + const ws = new WebSocket("ws://localhost:3001/ws", { + headers: { + "X-Elevenlabs-Speech-Engine-Authorization": `Bearer ${token}`, + }, + }); + + let sawFinalResponse = false; + + ws.on("open", () => { + console.log("WebSocket connected with auth."); + + ws.send(JSON.stringify({ type: "init", conversation_id: conversationId })); + ws.send( + JSON.stringify({ + type: "user_transcript", + event_id: 1, + user_transcript: [{ role: "user", content: "What is 2 plus 2?" }], + }), + ); + }); + + ws.on("message", (data) => { + const message = JSON.parse(data.toString()) as { + type?: string; + text?: string; + is_final?: boolean; + }; + + console.log("Received:", JSON.stringify(message).slice(0, 240)); + + if (message.type === "agent_response" && message.is_final) { + sawFinalResponse = true; + ws.close(); + resolvePromise(); + } + }); + + ws.on("error", reject); + + setTimeout(() => { + if (!sawFinalResponse) { + reject(new Error("Timed out waiting for a final agent response.")); + } + }, 20_000); + }); + + console.log("Speech Engine server responded successfully."); +} + +main().catch((error: unknown) => { + const message = + error instanceof Error ? error.message : "Speech Engine WS test failed."; + + console.error(message); + process.exitCode = 1; +}); + +export {}; diff --git a/speech-engine/nextjs/quickstart/example/server.mts b/speech-engine/nextjs/quickstart/example/server.mts new file mode 100644 index 00000000..a135db12 --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/server.mts @@ -0,0 +1,273 @@ +import { createRequire } from "node:module"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { resolve as resolvePath } from "node:path"; +import type { Duplex } from "node:stream"; + +import { + ElevenLabsClient, + SpeechEngineResource, + type SpeechEngineCallbacks, +} from "@elevenlabs/elevenlabs-js"; +import dotenv from "dotenv"; +import OpenAI from "openai"; + +const require = createRequire(import.meta.url); +const { Server: WebSocketServer } = require("ws") as typeof import("ws"); + +const envPath = resolvePath(process.cwd(), ".env.local"); +const port = 3001; +const wsPath = "/ws"; + +dotenv.config({ path: envPath }); + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + + if (!value) { + throw new Error(`Missing ${name} in ${envPath}.`); + } + + return value; +} + +function isAbortError(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === "AbortError" || error.message.includes("aborted")) + ); +} + +function wireSpeechEngineCallbacks( + engine: SpeechEngineResource, + session: ReturnType, + handler: SpeechEngineCallbacks, +) { + const { onInit, onTranscript, onClose, onDisconnect, onError } = handler; + + if (onInit) { + session.on("init", (conversationId) => { + onInit.call(session, conversationId, session); + }); + } + + if (onTranscript) { + session.on("user_transcript", (transcript, signal) => { + Promise.resolve(onTranscript.call(session, transcript, signal, session)).catch( + (error: unknown) => { + if (isAbortError(error)) { + return; + } + + const nextError = + error instanceof Error ? error : new Error(String(error)); + + if (onError) { + onError.call(session, nextError, session); + } + }, + ); + }); + } + + if (onClose) { + session.on("close", () => { + onClose.call(session, session); + }); + } + + if (onDisconnect) { + session.on("disconnected", () => { + onDisconnect.call(session, session); + }); + } + + if (onError) { + session.on("error", (error) => { + onError.call(session, error, session); + }); + } + + void engine; +} + +function attachSpeechEngine( + engine: SpeechEngineResource, + httpServer: Server, + path: string, + handler: SpeechEngineCallbacks, + devSkipAuth: boolean, +) { + const debug = handler.debug ?? false; + const log = debug + ? (...args: unknown[]) => console.log("[SpeechEngine]", ...args) + : () => {}; + + const wss = new WebSocketServer({ noServer: true }); + + const upgradeListener = async ( + request: IncomingMessage, + socket: Duplex, + head: Buffer, + ) => { + const url = new URL(request.url ?? "/", `http://${request.headers.host}`); + + log(`upgrade request: ${request.method} ${url.pathname}`); + + if (url.pathname !== path) { + log(`path mismatch: expected ${path}, got ${url.pathname} — skipping`); + return; + } + + const authHeader = request.headers["x-elevenlabs-speech-engine-authorization"]; + console.log("request headers:", request.headers); + const hasAuth = Boolean( + Array.isArray(authHeader) ? authHeader[0] : authHeader, + ); + + + if (!(await engine.verifyRequest(request))) { + log("rejected connection — invalid X-Elevenlabs-Speech-Engine-Authorization header"); + socket.destroy(); + return; + } + + + log("upgrading connection to WebSocket"); + + wss.handleUpgrade(request, socket, head, (webSocket) => { + log("WebSocket connection established"); + wss.emit("connection", webSocket); + }); + }; + + httpServer.on("upgrade", upgradeListener); + + wss.on("connection", (webSocket) => { + log("creating new session"); + const session = engine.createSession(webSocket, { debug }); + wireSpeechEngineCallbacks(engine, session, handler); + }); + + log(`listening for WebSocket upgrades on ${path}`); +} + +const speechEngineId = requireEnv("ELEVENLABS_SPEECH_ENGINE_ID"); +const devSkipAuth = process.env.SPEECH_ENGINE_DEV_SKIP_AUTH === "true"; +const elevenlabs = new ElevenLabsClient({ + apiKey: requireEnv("ELEVENLABS_API_KEY"), +}); +const openai = new OpenAI({ + apiKey: requireEnv("OPENAI_API_KEY"), +}); +const model = requireEnv("OPENAI_MODEL"); + +const httpServer = createServer((request, response) => { + if (request.url === "/health") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ ok: true, devSkipAuth })); + return; + } + + response.writeHead(404, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "Not found" })); +}); + +const speechEngineCallbacks: SpeechEngineCallbacks = { + debug: true, + + onInit(conversationId) { + console.log("[speech-engine] session started:", conversationId); + }, + + async onTranscript(transcript, signal, session) { + console.log( + "[speech-engine] transcript payload:", + JSON.stringify(transcript), + ); + + const latestMessage = transcript.at(-1)?.content?.trim() ?? ""; + + if (transcript.length === 0 || !latestMessage) { + console.warn("[speech-engine] received empty transcript, sending fallback."); + + if (!signal.aborted) { + session.sendResponse("I didn't catch that. Could you say that again?"); + } + + return; + } + + console.log("[speech-engine] transcript:", latestMessage); + + try { + const response = await openai.responses.create( + { + model, + instructions: + "You are a helpful voice assistant. Keep responses concise and conversational.", + input: transcript.map((message) => ({ + role: message.role === "agent" ? "assistant" : "user", + content: message.content, + })), + stream: true, + }, + { signal }, + ); + + session.sendResponse(response); + console.log("[speech-engine] response streaming for:", latestMessage); + } catch (error: unknown) { + if (signal.aborted) { + console.log("[speech-engine] response aborted for:", latestMessage); + return; + } + + const message = + error instanceof Error ? error.message : "Failed to generate a response."; + + console.error("[speech-engine] onTranscript error:", message); + + try { + session.sendResponse( + "Sorry, I ran into an error generating a response. Please try again.", + ); + } catch (fallbackError: unknown) { + console.error( + "[speech-engine] failed to send fallback response:", + fallbackError instanceof Error ? fallbackError.message : fallbackError, + ); + } + } + }, + + onClose(session) { + console.log("[speech-engine] session ended:", session.conversationId); + }, + + onDisconnect(session) { + console.log("[speech-engine] session disconnected:", session.conversationId); + }, + + onError(error) { + console.error("[speech-engine] error:", error); + }, +}; + +const engine = await elevenlabs.speechEngine.get(speechEngineId); + +attachSpeechEngine(engine, httpServer, wsPath, speechEngineCallbacks, devSkipAuth); + +httpServer.listen(port, () => { + console.log(`Speech Engine server listening on http://localhost:${port}`); + console.log(`WebSocket endpoint: ws://localhost:${port}${wsPath}`); + console.log(`Health check: http://localhost:${port}/health`); + console.log(`Speech Engine ID: ${speechEngineId}`); + + if (devSkipAuth) { + console.warn( + "SPEECH_ENGINE_DEV_SKIP_AUTH=true — accepting upstream WebSocket connections without JWT.", + ); + } +}); + +export {}; diff --git a/speech-engine/nextjs/quickstart/example/tsconfig.json b/speech-engine/nextjs/quickstart/example/tsconfig.json new file mode 100644 index 00000000..163963ff --- /dev/null +++ b/speech-engine/nextjs/quickstart/example/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "moduleDetection": "force", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + } + }, + "include": [ + "next-env.d.ts", + "**/*.ts", + "**/*.tsx", + ".next/types/**/*.ts", + ".next/dev/types/**/*.ts", + "**/*.mts" + ], + "exclude": ["node_modules"] +} diff --git a/speech-engine/nextjs/quickstart/setup.sh b/speech-engine/nextjs/quickstart/setup.sh new file mode 100755 index 00000000..22868529 --- /dev/null +++ b/speech-engine/nextjs/quickstart/setup.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$DIR/../../.." && pwd)" +cd "$DIR" + +# Clean example/ but preserve cache directories for speed. +if [ -d example ]; then + find example -mindepth 1 -maxdepth 1 ! -name node_modules ! -name .next -exec rm -rf {} + +fi +mkdir -p example + +# Copy shared template structure. +rsync -a \ + --exclude node_modules --exclude .next \ + --exclude pnpm-lock.yaml --exclude package-lock.json \ + --exclude example \ + "$REPO_ROOT/templates/nextjs/" example/ + +# Copy project-specific README. +cp README.md example/README.md + +# Add ElevenLabs dependencies, fetching latest versions at setup time. +cd example +export REACT_VER=$(npm view @elevenlabs/react version) +export ELEVENLABS_VER=$(npm view @elevenlabs/elevenlabs-js version) +export OPENAI_VER=$(npm view openai version) +export DOTENV_VER=$(npm view dotenv version) +export TSX_VER=$(npm view tsx version) +node -e " + const pkg = JSON.parse(require('fs').readFileSync('package.json', 'utf8')); + pkg.name = 'speech-engine-quickstart'; + pkg.dependencies['@elevenlabs/react'] = '^' + process.env.REACT_VER; + pkg.dependencies['@elevenlabs/elevenlabs-js'] = '^' + process.env.ELEVENLABS_VER; + pkg.dependencies.openai = '^' + process.env.OPENAI_VER; + pkg.dependencies.dotenv = '^' + process.env.DOTENV_VER; + pkg.devDependencies.tsx = '^' + process.env.TSX_VER; + delete pkg.dependencies['@elevenlabs/client']; + pkg.pnpm = pkg.pnpm || {}; + pkg.pnpm.overrides = pkg.pnpm.overrides || {}; + pkg.pnpm.overrides['livekit-client'] = '2.16.1'; + pkg.scripts['speech-engine:create'] = 'tsx scripts/create-engine.mts'; + pkg.scripts['speech-engine:server'] = 'tsx server.mts'; + require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n'); +" + +mkdir -p app/api/token +mkdir -p scripts + +cat > .env.example <<'EOF' +ELEVENLABS_API_KEY= +ELEVENLABS_SPEECH_ENGINE_ID= +PUBLIC_WS_URL= +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o +EOF + +if [ -f "$DIR/.env" ]; then + cp "$DIR/.env" .env.local +fi + +pnpm install --config.confirmModulesPurge=false diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..5ef6a520 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,41 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/test/AGENTS.md b/test/AGENTS.md new file mode 100644 index 00000000..8bd0e390 --- /dev/null +++ b/test/AGENTS.md @@ -0,0 +1,5 @@ + +# This is NOT the Next.js you know + +This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices. + diff --git a/test/CLAUDE.md b/test/CLAUDE.md new file mode 100644 index 00000000..43c994c2 --- /dev/null +++ b/test/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..e215bc4c --- /dev/null +++ b/test/README.md @@ -0,0 +1,36 @@ +This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. + +This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. diff --git a/test/app/api/chat/route.ts b/test/app/api/chat/route.ts new file mode 100644 index 00000000..41d75227 --- /dev/null +++ b/test/app/api/chat/route.ts @@ -0,0 +1,81 @@ +import { NextResponse } from "next/server"; +import { + createAssistantReply, + isChatRole, + type ChatMessage, +} from "@/lib/assistant"; + +export const dynamic = "force-dynamic"; + +const MAX_REQUEST_MESSAGES = 30; + +function parseMessages(value: unknown): ChatMessage[] | null { + if (!Array.isArray(value)) { + return null; + } + + const messages: ChatMessage[] = []; + + for (const item of value) { + if (!item || typeof item !== "object") { + return null; + } + + const { role, content } = item as Record; + + if (!isChatRole(role) || typeof content !== "string") { + return null; + } + + const trimmedContent = content.trim(); + + if (trimmedContent.length > 0) { + messages.push({ + role, + content: trimmedContent, + }); + } + } + + return messages.slice(-MAX_REQUEST_MESSAGES); +} + +export async function POST(request: Request) { + let body: unknown; + + try { + body = await request.json(); + } catch { + return NextResponse.json( + { error: "Request body must be valid JSON." }, + { status: 400 }, + ); + } + + const messages = parseMessages( + typeof body === "object" && body !== null + ? (body as Record).messages + : null, + ); + + if (!messages || messages.length === 0) { + return NextResponse.json( + { error: "Send at least one user or assistant message." }, + { status: 400 }, + ); + } + + try { + const message = await createAssistantReply(messages, request.signal); + + return NextResponse.json({ message }); + } catch (error: unknown) { + const details = + error instanceof Error ? error.message : "Failed to generate a reply."; + + return NextResponse.json( + { error: "Unable to generate a chat response.", details }, + { status: 500 }, + ); + } +} diff --git a/test/app/api/token/route.ts b/test/app/api/token/route.ts new file mode 100644 index 00000000..59b93ab3 --- /dev/null +++ b/test/app/api/token/route.ts @@ -0,0 +1,30 @@ +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; +import { NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +const SPEECH_ENGINE_ID = + process.env.ELEVENLABS_SPEECH_ENGINE_ID ?? "seng_0801kry8xv4nf43vb65vas7b064x"; + +const elevenlabs = new ElevenLabsClient({ + apiKey: process.env.ELEVENLABS_API_KEY, +}); + +export async function GET() { + try { + const response = + await elevenlabs.conversationalAi.conversations.getSignedUrl({ + agentId: SPEECH_ENGINE_ID, + }); + + return NextResponse.json({ signedUrl: response.signedUrl }); + } catch (error: unknown) { + const details = + error instanceof Error ? error.message : "Failed to create signed URL."; + + return NextResponse.json( + { error: "Unable to create a conversation signed URL.", details }, + { status: 500 }, + ); + } +} diff --git a/test/app/api/voice-history/link/route.ts b/test/app/api/voice-history/link/route.ts new file mode 100644 index 00000000..34cc5e79 --- /dev/null +++ b/test/app/api/voice-history/link/route.ts @@ -0,0 +1,33 @@ +import { NextResponse } from "next/server"; + +import { linkVoiceHistory } from "@/lib/voice-history"; + +export const dynamic = "force-dynamic"; + +export async function POST(request: Request) { + const body = (await request.json().catch(() => null)) as { + conversationId?: unknown; + historyId?: unknown; + } | null; + + if ( + typeof body?.conversationId !== "string" || + typeof body.historyId !== "string" + ) { + return NextResponse.json( + { error: "Provide a conversationId and historyId." }, + { status: 400 }, + ); + } + + const linked = await linkVoiceHistory(body.conversationId, body.historyId); + + if (!linked) { + return NextResponse.json( + { error: "The voice history could not be found." }, + { status: 404 }, + ); + } + + return NextResponse.json({ ok: true }); +} diff --git a/test/app/api/voice-history/route.ts b/test/app/api/voice-history/route.ts new file mode 100644 index 00000000..816607ff --- /dev/null +++ b/test/app/api/voice-history/route.ts @@ -0,0 +1,48 @@ +import { NextResponse } from "next/server"; + +import { isChatRole, type ChatMessage } from "@/lib/assistant"; +import { createVoiceHistory } from "@/lib/voice-history"; + +export const dynamic = "force-dynamic"; + +function parseMessages(input: unknown): ChatMessage[] | null { + if (!Array.isArray(input)) { + return null; + } + + const messages = input + .filter((message): message is ChatMessage => { + if (!message || typeof message !== "object") { + return false; + } + + const { role, content } = message as Partial; + return isChatRole(role) && typeof content === "string"; + }) + .map((message) => ({ + role: message.role, + content: message.content.trim(), + })) + .filter((message) => message.content.length > 0); + + return messages.length > 0 ? messages : null; +} + +export async function POST(request: Request) { + const body = (await request.json().catch(() => null)) as { + messages?: unknown; + } | null; + + const messages = parseMessages(body?.messages); + + if (!messages) { + return NextResponse.json( + { error: "Provide at least one chat message." }, + { status: 400 }, + ); + } + + const historyId = await createVoiceHistory(messages); + + return NextResponse.json({ historyId }); +} diff --git a/test/app/favicon.ico b/test/app/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..718d6fea4835ec2d246af9800eddb7ffb276240c GIT binary patch literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m literal 0 HcmV?d00001 diff --git a/test/app/globals.css b/test/app/globals.css new file mode 100644 index 00000000..2e227a32 --- /dev/null +++ b/test/app/globals.css @@ -0,0 +1,41 @@ +@import "tailwindcss"; + +:root { + --background: #ffffff; + --foreground: #171717; +} + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); +} + +@media (prefers-color-scheme: dark) { + :root { + --background: #0a0a0a; + --foreground: #ededed; + } +} + +body { + background: var(--background); + color: var(--foreground); + font-family: Arial, Helvetica, sans-serif; +} + +html, +body { + height: 100%; + overflow: hidden; +} + +[data-chat-thread] { + scrollbar-width: none; + -ms-overflow-style: none; +} + +[data-chat-thread]::-webkit-scrollbar { + display: none; +} diff --git a/test/app/layout.tsx b/test/app/layout.tsx new file mode 100644 index 00000000..f4c109bc --- /dev/null +++ b/test/app/layout.tsx @@ -0,0 +1,33 @@ +import type { Metadata } from "next"; +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "Chat and Voice Demo", + description: "Typed chat with a handoff into voice mode.", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + {children} + + ); +} diff --git a/test/app/page.tsx b/test/app/page.tsx new file mode 100644 index 00000000..c52c5f66 --- /dev/null +++ b/test/app/page.tsx @@ -0,0 +1,598 @@ +"use client"; + +import { ConversationProvider, useConversation } from "@elevenlabs/react"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + type FormEvent, + type KeyboardEvent, + type SVGProps, +} from "react"; + +type DemoRole = "user" | "assistant"; +type DemoChannel = "chat" | "voice"; + +type DemoMessage = { + id: string; + role: DemoRole; + content: string; + channel: DemoChannel; + pending?: boolean; + error?: boolean; +}; + +type ApiMessage = { + role: DemoRole; + content: string; +}; + +const WELCOME_MESSAGE_ID = "welcome"; + +const INITIAL_MESSAGES: DemoMessage[] = [ + { + id: WELCOME_MESSAGE_ID, + role: "assistant", + content: "Ready when you are.", + channel: "chat", + }, +]; + +async function getSignedUrl(): Promise { + const response = await fetch("/api/token"); + + if (!response.ok) { + throw Error("Failed to get conversation signed URL"); + } + + const data = (await response.json()) as { signedUrl?: string }; + + if (!data.signedUrl) { + throw Error("Token response did not include a signed URL"); + } + + return data.signedUrl; +} + +async function registerVoiceHistory(messages: ApiMessage[]) { + if (messages.length === 0) { + return null; + } + + const response = await fetch("/api/voice-history", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ messages }), + }); + const data = (await response.json().catch(() => ({}))) as { + historyId?: string; + error?: string; + }; + + if (!response.ok || !data.historyId) { + throw new Error(data.error || "Failed to prepare voice history."); + } + + return data.historyId; +} + +async function linkVoiceHistory(historyId: string, conversationId: string) { + const response = await fetch("/api/voice-history/link", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ historyId, conversationId }), + }); + + if (!response.ok) { + const data = (await response.json().catch(() => ({}))) as { + error?: string; + }; + throw new Error(data.error || "Failed to link voice history."); + } +} + +function createMessageId(prefix: string) { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + +function getErrorMessage(error: unknown) { + return error instanceof Error ? error.message : "Something went wrong."; +} + +function roleLabel(role: DemoRole) { + return role === "assistant" ? "Assistant" : "You"; +} + +function toApiMessages(messages: DemoMessage[]): ApiMessage[] { + return messages + .filter( + (message) => + message.id !== WELCOME_MESSAGE_ID && !message.pending && !message.error, + ) + .map((message) => ({ + role: message.role, + content: message.content, + })); +} + +function buildVoiceContext(messages: DemoMessage[]) { + return toApiMessages(messages) + .slice(-12) + .map((message) => `${roleLabel(message.role)}: ${message.content}`) + .join("\n"); +} + +function appendOrUpdateVoiceMessage( + messages: DemoMessage[], + role: DemoRole, + content: string, + eventId?: number, +) { + const id = eventId + ? `voice-${role}-${eventId}` + : createMessageId(`voice-${role}`); + const existingIndex = messages.findIndex((message) => message.id === id); + + if (existingIndex === -1) { + return [ + ...messages, + { + id, + role, + content, + channel: "voice" as const, + }, + ]; + } + + return messages.map((message, index) => + index === existingIndex ? { ...message, content } : message, + ); +} + +function SendIcon(props: SVGProps) { + return ( + + ); +} + +function MicIcon(props: SVGProps) { + return ( + + ); +} + +function StopIcon(props: SVGProps) { + return ( + + ); +} + +function ConversationView() { + const [messages, setMessages] = useState(INITIAL_MESSAGES); + const [draft, setDraft] = useState(""); + const [isSending, setIsSending] = useState(false); + const [chatError, setChatError] = useState(null); + const [voiceError, setVoiceError] = useState(null); + const threadRef = useRef(null); + const voiceHistoryIdRef = useRef(null); + const linkedVoiceConversationIdsRef = useRef>(new Set()); + const pendingTypedVoiceMessagesRef = useRef([]); + + const linkActiveVoiceHistory = useCallback(async (conversationId?: string) => { + const historyId = voiceHistoryIdRef.current; + + if ( + !historyId || + !conversationId || + linkedVoiceConversationIdsRef.current.has(conversationId) + ) { + return; + } + + await linkVoiceHistory(historyId, conversationId); + linkedVoiceConversationIdsRef.current.add(conversationId); + }, []); + + const conversation = useConversation({ + onConnect: ({ conversationId }) => { + setVoiceError(null); + void linkActiveVoiceHistory(conversationId).catch((error: unknown) => { + setVoiceError(getErrorMessage(error)); + }); + }, + onError: (message) => setVoiceError(message), + onMessage: ({ role, message, event_id }) => { + const content = message.trim(); + + if (!content) { + return; + } + + if (role === "user") { + const pendingIndex = + pendingTypedVoiceMessagesRef.current.indexOf(content); + + if (pendingIndex !== -1) { + pendingTypedVoiceMessagesRef.current.splice(pendingIndex, 1); + return; + } + } + + setMessages((currentMessages) => + appendOrUpdateVoiceMessage( + currentMessages, + role === "agent" ? "assistant" : "user", + content, + event_id, + ), + ); + }, + }); + + const chatHistory = useMemo(() => toApiMessages(messages), [messages]); + const voiceContext = useMemo(() => buildVoiceContext(messages), [messages]); + const isVoiceConnected = conversation.status === "connected"; + const isVoiceStarting = conversation.status === "connecting"; + const canStartVoice = conversation.status === "disconnected"; + const voiceStatusLabel = + conversation.status === "connected" ? conversation.mode : conversation.status; + const headerStatus = isVoiceConnected + ? voiceStatusLabel + : isVoiceStarting + ? "connecting" + : "chat"; + + useEffect(() => { + const thread = threadRef.current; + + if (!thread) { + return; + } + + thread.scrollTo({ + top: thread.scrollHeight, + behavior: "smooth", + }); + }, [messages]); + + const sendChatMessage = useCallback(async () => { + const content = draft.trim(); + + if (!content || isSending) { + return; + } + + const userMessage: DemoMessage = { + id: createMessageId(isVoiceConnected ? "voice-user-typed" : "chat-user"), + role: "user", + content, + channel: isVoiceConnected ? "voice" : "chat", + }; + + if (isVoiceConnected) { + try { + pendingTypedVoiceMessagesRef.current.push(content); + conversation.sendUserMessage(content); + setMessages((currentMessages) => [...currentMessages, userMessage]); + setDraft(""); + setChatError(null); + setVoiceError(null); + } catch (error: unknown) { + pendingTypedVoiceMessagesRef.current = + pendingTypedVoiceMessagesRef.current.filter( + (pendingContent) => pendingContent !== content, + ); + setVoiceError(getErrorMessage(error)); + } + + return; + } + + const pendingAssistantId = createMessageId("chat-assistant"); + const pendingAssistant: DemoMessage = { + id: pendingAssistantId, + role: "assistant", + content: "Thinking...", + channel: "chat", + pending: true, + }; + const nextHistory = [...chatHistory, { role: "user" as const, content }]; + + setMessages((currentMessages) => [ + ...currentMessages, + userMessage, + pendingAssistant, + ]); + setDraft(""); + setIsSending(true); + setChatError(null); + + try { + const response = await fetch("/api/chat", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ messages: nextHistory }), + }); + const data = (await response.json().catch(() => ({}))) as { + message?: string; + error?: string; + details?: string; + }; + + if (!response.ok) { + throw new Error(data.details || data.error || "Chat request failed."); + } + + if (!data.message) { + throw new Error("Chat response did not include a message."); + } + + setMessages((currentMessages) => + currentMessages.map((message) => + message.id === pendingAssistantId + ? { ...message, content: data.message ?? "", pending: false } + : message, + ), + ); + } catch (error: unknown) { + const message = getErrorMessage(error); + + setChatError(message); + setMessages((currentMessages) => + currentMessages.map((currentMessage) => + currentMessage.id === pendingAssistantId + ? { + ...currentMessage, + content: message, + pending: false, + error: true, + } + : currentMessage, + ), + ); + } finally { + setIsSending(false); + } + }, [chatHistory, conversation, draft, isSending, isVoiceConnected]); + + const handleSubmit = useCallback( + (event: FormEvent) => { + event.preventDefault(); + void sendChatMessage(); + }, + [sendChatMessage], + ); + + const handleComposerKeyDown = useCallback( + (event: KeyboardEvent) => { + if (event.key === "Enter" && !event.shiftKey) { + event.preventDefault(); + void sendChatMessage(); + } + }, + [sendChatMessage], + ); + + const startVoiceMode = useCallback(async () => { + if (isVoiceConnected || isVoiceStarting) { + return; + } + + try { + if (!navigator.mediaDevices?.getUserMedia) { + throw new Error("Microphone access is not available in this browser."); + } + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + stream.getTracks().forEach((track) => track.stop()); + + const historyId = await registerVoiceHistory(chatHistory); + const signedUrl = await getSignedUrl(); + const context = voiceContext; + + voiceHistoryIdRef.current = historyId; + linkedVoiceConversationIdsRef.current.clear(); + + conversation.startSession({ + signedUrl, + onConversationCreated: (voiceConversation) => { + void linkActiveVoiceHistory(voiceConversation.getId()).catch( + (error: unknown) => { + setVoiceError(getErrorMessage(error)); + }, + ); + + if (context) { + voiceConversation.sendContextualUpdate( + `Typed chat before voice mode:\n${context}`, + { contextId: "typed-chat-history" }, + ); + } + }, + }); + setVoiceError(null); + } catch (error: unknown) { + setVoiceError(getErrorMessage(error)); + } + }, [ + chatHistory, + conversation, + isVoiceConnected, + isVoiceStarting, + linkActiveVoiceHistory, + voiceContext, + ]); + + const stopVoiceMode = useCallback(() => { + conversation.endSession(); + }, [conversation]); + + return ( +
+
+
+
+

Assistant

+

{headerStatus}

+
+ +
+ {isVoiceConnected ? ( + <> + + + + ) : ( + <> + {isVoiceStarting ? ( + Connecting + ) : null} + + + )} +
+
+ + {(chatError || voiceError) && ( +
+ {chatError || voiceError} +
+ )} + +
+
+ {messages.map((message) => { + const isAssistant = message.role === "assistant"; + + return ( +
+
+

+ {roleLabel(message.role)} + {message.channel === "voice" ? " / voice" : ""} +

+

{message.content}

+
+
+ ); + })} +
+
+ +
+
+ +