Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
HEYGEN_API_KEY="your Heygen API key"
NEXT_PUBLIC_BASE_API_URL=https://api.heygen.com
SUPABASE_URL="your Supabase URL"
SUPABASE_SERVICE_ROLE_KEY="your Supabase service role key"
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ Feel free to play around with the existing code and please leave any feedback fo

5. (Optional) If you would like to use the OpenAI features, enter your OpenAI Api Key in the `.env` file.

6. Run `npm run dev`
6. Add your Supabase credentials to `.env` as `SUPABASE_URL` and either `SUPABASE_SERVICE_ROLE_KEY` or `SUPABASE_ANON_KEY`.

7. Run `npm run dev`

### Starting sessions

Expand Down
19 changes: 19 additions & 0 deletions app/api/create-session/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { supabase } from '@/app/lib/supabase';

export async function POST(request: Request) {
try {
const { name, email, language } = await request.json();
const { data, error } = await supabase
.from('sessions')
.insert({ name, email, language })
.select('id')
.single();

if (error) throw error;

return Response.json({ sessionId: data.id });
} catch (err) {
console.error('Error creating session:', err);
return new Response('Failed to create session', { status: 500 });
}
}
17 changes: 17 additions & 0 deletions app/api/save-message/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { supabase } from '@/app/lib/supabase';

export async function POST(request: Request) {
try {
const { sessionId, content } = await request.json();
const { error } = await supabase
.from('messages')
.insert({ session_id: sessionId, content });

if (error) throw error;

return Response.json({ success: true });
} catch (err) {
console.error('Error saving message:', err);
return new Response('Failed to save message', { status: 500 });
}
}
6 changes: 6 additions & 0 deletions app/lib/supabase.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { createClient } from '@supabase/supabase-js';

export const supabase = createClient(
process.env.SUPABASE_URL!,
process.env.SUPABASE_SERVICE_ROLE_KEY!,
);
27 changes: 23 additions & 4 deletions components/AvatarConfig/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,23 @@ import { Field } from "./Field";

import { AVATARS, STT_LANGUAGE_LIST } from "@/app/lib/constants";

interface AvatarConfigValues extends StartAvatarRequest {
userName: string;
userEmail: string;
}

interface AvatarConfigProps {
onConfigChange: (config: StartAvatarRequest) => void;
config: StartAvatarRequest;
onConfigChange: (config: AvatarConfigValues) => void;
config: AvatarConfigValues;
}

export const AvatarConfig: React.FC<AvatarConfigProps> = ({
onConfigChange,
config,
}) => {
const onChange = <T extends keyof StartAvatarRequest>(
const onChange = <T extends keyof AvatarConfigValues>(
key: T,
value: StartAvatarRequest[T],
value: AvatarConfigValues[T],
) => {
onConfigChange({ ...config, [key]: value });
};
Expand Down Expand Up @@ -96,6 +101,20 @@ export const AvatarConfig: React.FC<AvatarConfigProps> = ({
/>
</Field>
)}
<Field label="Name">
<Input
placeholder="Enter your name"
value={config.userName}
onChange={(value) => onChange("userName", value)}
/>
</Field>
<Field label="Email">
<Input
placeholder="Enter your email"
value={config.userEmail}
onChange={(value) => onChange("userEmail", value)}
/>
</Field>
<Field label="Language">
<Select
isSelected={(option) => option.value === config.language}
Expand Down
30 changes: 27 additions & 3 deletions components/InteractiveAvatar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,22 @@ import { AvatarVideo } from "./AvatarSession/AvatarVideo";
import { useStreamingAvatarSession } from "./logic/useStreamingAvatarSession";
import { AvatarControls } from "./AvatarSession/AvatarControls";
import { useVoiceChat } from "./logic/useVoiceChat";
import { StreamingAvatarProvider, StreamingAvatarSessionState } from "./logic";
import {
StreamingAvatarProvider,
StreamingAvatarSessionState,
useStreamingAvatarContext,
} from "./logic";
import { LoadingIcon } from "./Icons";
import { MessageHistory } from "./AvatarSession/MessageHistory";

import { AVATARS } from "@/app/lib/constants";

const DEFAULT_CONFIG: StartAvatarRequest = {
interface AvatarSessionConfig extends StartAvatarRequest {
userName: string;
userEmail: string;
}

const DEFAULT_CONFIG: AvatarSessionConfig = {
quality: AvatarQuality.Low,
avatarName: AVATARS[0].avatar_id,
knowledgeId: undefined,
Expand All @@ -36,14 +45,17 @@ const DEFAULT_CONFIG: StartAvatarRequest = {
sttSettings: {
provider: STTProvider.DEEPGRAM,
},
userName: "",
userEmail: "",
};

function InteractiveAvatar() {
const { initAvatar, startAvatar, stopAvatar, sessionState, stream } =
useStreamingAvatarSession();
const { startVoiceChat } = useVoiceChat();
const { setSessionId } = useStreamingAvatarContext();

const [config, setConfig] = useState<StartAvatarRequest>(DEFAULT_CONFIG);
const [config, setConfig] = useState<AvatarSessionConfig>(DEFAULT_CONFIG);

const mediaStream = useRef<HTMLVideoElement>(null);

Expand All @@ -65,6 +77,18 @@ function InteractiveAvatar() {

const startSessionV2 = useMemoizedFn(async (isVoiceChat: boolean) => {
try {
const sessionRes = await fetch('/api/create-session', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: config.userName,
email: config.userEmail,
language: config.language,
}),
});
const { sessionId } = await sessionRes.json();
setSessionId(sessionId);

const newToken = await fetchAccessToken();
const avatar = initAvatar(newToken);

Expand Down
47 changes: 41 additions & 6 deletions components/logic/context.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import StreamingAvatar, {
StreamingTalkingMessageEvent,
UserTalkingMessageEvent,
} from "@heygen/streaming-avatar";
import React, { useRef, useState } from "react";
import React, { useEffect, useRef, useState } from "react";

export enum StreamingAvatarSessionState {
INACTIVE = "inactive",
Expand All @@ -26,6 +26,9 @@ type StreamingAvatarContextProps = {
avatarRef: React.MutableRefObject<StreamingAvatar | null>;
basePath?: string;

sessionId: string | null;
setSessionId: (id: string | null) => void;

isMuted: boolean;
setIsMuted: (isMuted: boolean) => void;
isVoiceChatLoading: boolean;
Expand All @@ -50,7 +53,7 @@ type StreamingAvatarContextProps = {
}: {
detail: StreamingTalkingMessageEvent;
}) => void;
handleEndMessage: () => void;
handleEndMessage: () => Promise<void>;

isListening: boolean;
setIsListening: (isListening: boolean) => void;
Expand All @@ -66,6 +69,9 @@ type StreamingAvatarContextProps = {
const StreamingAvatarContext = React.createContext<StreamingAvatarContextProps>(
{
avatarRef: { current: null },
basePath: undefined,
sessionId: null,
setSessionId: () => {},
isMuted: true,
setIsMuted: () => {},
isVoiceChatLoading: false,
Expand All @@ -80,7 +86,7 @@ const StreamingAvatarContext = React.createContext<StreamingAvatarContextProps>(
clearMessages: () => {},
handleUserTalkingMessage: () => {},
handleStreamingTalkingMessage: () => {},
handleEndMessage: () => {},
handleEndMessage: async () => {},
isListening: false,
setIsListening: () => {},
isUserTalking: false,
Expand Down Expand Up @@ -121,7 +127,9 @@ const useStreamingAvatarVoiceChatState = () => {
};
};

const useStreamingAvatarMessageState = () => {
const useStreamingAvatarMessageState = (
sessionIdRef: React.MutableRefObject<string | null>,
) => {
const [messages, setMessages] = useState<Message[]>([]);
const currentSenderRef = useRef<MessageSender | null>(null);

Expand Down Expand Up @@ -177,7 +185,27 @@ const useStreamingAvatarMessageState = () => {
}
};

const handleEndMessage = () => {
const handleEndMessage = async () => {
if (
currentSenderRef.current === MessageSender.CLIENT &&
sessionIdRef.current
) {
const last = messages[messages.length - 1];
if (last) {
try {
await fetch('/api/save-message', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sessionId: sessionIdRef.current,
content: last.content,
}),
});
} catch (err) {
console.error('Failed to save message', err);
}
}
}
currentSenderRef.current = null;
};

Expand Down Expand Up @@ -229,7 +257,12 @@ export const StreamingAvatarProvider = ({
const avatarRef = React.useRef<StreamingAvatar>(null);
const voiceChatState = useStreamingAvatarVoiceChatState();
const sessionState = useStreamingAvatarSessionState();
const messageState = useStreamingAvatarMessageState();
const [sessionId, setSessionId] = useState<string | null>(null);
const sessionIdRef = useRef<string | null>(null);
useEffect(() => {
sessionIdRef.current = sessionId;
}, [sessionId]);
const messageState = useStreamingAvatarMessageState(sessionIdRef);
const listeningState = useStreamingAvatarListeningState();
const talkingState = useStreamingAvatarTalkingState();
const connectionQualityState = useStreamingAvatarConnectionQualityState();
Expand All @@ -239,6 +272,8 @@ export const StreamingAvatarProvider = ({
value={{
avatarRef,
basePath,
sessionId,
setSessionId,
...voiceChatState,
...sessionState,
...messageState,
Expand Down
Loading