"use client"; import { getToolName, isToolUIPart, TextPart } from "ai"; import { DEFAULT_VOICE_TOOLS, UIMessageWithCompleted } from "lib/ai/speech"; import { OPENAI_VOICE, useOpenAIVoiceChat as OpenAIVoiceChat, } from "lib/ai/speech/open-ai/use-voice-chat.openai"; import { cn, groupBy, isNull } from "lib/utils"; import { CheckIcon, Loader, MicIcon, MicOffIcon, PhoneIcon, Settings2Icon, TriangleAlertIcon, XIcon, MessagesSquareIcon, MessageSquareMoreIcon, WrenchIcon, ChevronRight, } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { safe } from "ts-safe"; import { Alert, AlertDescription, AlertTitle } from "ui/alert"; import { Button } from "ui/button"; import { Drawer, DrawerContent, DrawerPortal, DrawerTitle } from "ui/drawer"; import { DropdownMenu, DropdownMenuContent, DropdownMenuGroup, DropdownMenuItem, DropdownMenuPortal, DropdownMenuSub, DropdownMenuSubContent, DropdownMenuSubTrigger, DropdownMenuTrigger, } from "ui/dropdown-menu"; import { GeminiIcon } from "ui/gemini-icon"; import { MessageLoading } from "ui/message-loading"; import { OpenAIIcon } from "ui/openai-icon"; import { Tooltip, TooltipContent, TooltipTrigger } from "ui/tooltip"; import { ToolMessagePart } from "./message-parts"; import { EnabledTools, EnabledToolsDropdown } from "./enabled-tools-dropdown"; import { appStore } from "@/app/store"; import { useShallow } from "zustand/shallow"; import { useTranslations } from "next-intl"; import { Dialog, DialogContent, DialogTitle, DialogTrigger } from "ui/dialog"; import JsonView from "ui/json-view"; import { isShortcutEvent, Shortcuts } from "lib/keyboard-shortcuts"; import { useAgent } from "@/hooks/queries/use-agent"; import { ChatMention } from "app-types/chat"; import { Avatar, AvatarFallback, AvatarImage } from "ui/avatar"; const prependTools: EnabledTools[] = [ { groupName: "Browser", tools: DEFAULT_VOICE_TOOLS.map((tool) => ({ name: tool.name, description: tool.description, })), }, ]; export function ChatBotVoice() { const t = useTranslations("Chat"); const [ agentId, appStoreMutate, voiceChat, model, allowedMcpServers, mcpList, ] = appStore( useShallow((state) => [ state.voiceChat.agentId, state.mutate, state.voiceChat, state.chatModel, state.allowedMcpServers, state.mcpList, ]), ); const { agent } = useAgent(agentId); const [isClosing, setIsClosing] = useState(false); const startAudio = useRef(null); const [useCompactView, setUseCompactView] = useState(true); const toolMentions = useMemo(() => { if (!agentId) { if (!allowedMcpServers) return []; return mcpList .filter((v) => { return ( v.id in allowedMcpServers && allowedMcpServers[v.id]?.tools?.length ); }) .flatMap((v) => { const tools = allowedMcpServers[v.id].tools; return tools.map((tool) => { const toolInfo = v.toolInfo.find((t) => t.name === tool); const mention: ChatMention = { type: "mcpTool", serverName: v.name, serverId: v.id, name: tool, description: toolInfo?.description ?? "", }; return mention; }); }); } return ( agent?.instructions.mentions?.filter((v) => v.type === "mcpTool") ?? [] ); }, [agentId, agent, mcpList, allowedMcpServers]); const { isListening, isAssistantSpeaking, isLoading, isActive, isUserSpeaking, messages, error, start, startListening, stop, stopListening, } = OpenAIVoiceChat({ toolMentions, agentId, ...voiceChat.options.providerOptions, }); const startWithSound = useCallback(() => { if (!startAudio.current) { startAudio.current = new Audio("/sounds/start_voice.ogg"); } start().then(() => { startAudio.current?.play().catch(() => {}); }); }, [start]); const endVoiceChat = useCallback(async () => { setIsClosing(true); await safe(() => stop()); setIsClosing(false); appStoreMutate({ voiceChat: { ...voiceChat, isOpen: false, }, }); }, [messages, model]); const statusMessage = useMemo(() => { if (isLoading) { return (

{t("VoiceChat.preparing")}

); } if (!isActive) return (

{t("VoiceChat.startVoiceChat")}

); if (!isListening) return (

{t("VoiceChat.yourMicIsOff")}

); if (!isAssistantSpeaking && messages.length === 0) { return (

{t("VoiceChat.readyWhenYouAreJustStartTalking")}

); } if (isUserSpeaking && useCompactView) { return ; } if (!isAssistantSpeaking && !isUserSpeaking) { return (

{t("VoiceChat.readyWhenYouAreJustStartTalking")}

); } }, [ isAssistantSpeaking, isUserSpeaking, isActive, isLoading, isListening, messages.length, useCompactView, ]); const mcpTools = useMemo(() => { const mcpMentions = toolMentions.filter( (v) => v.type === "mcpTool", ) as Extract[]; const groupByServer = groupBy(mcpMentions, "serverName"); return Object.entries(groupByServer).map(([serverName, tools]) => { return { groupName: serverName, tools: tools.map((v) => ({ name: v.name, description: v.description, })), }; }); }, [toolMentions]); const tools = useMemo(() => { return [...prependTools, ...mcpTools]; }, [prependTools, mcpTools]); useEffect(() => { return () => { if (isActive) { stop(); } }; }, [voiceChat.options, isActive]); useEffect(() => { if (voiceChat.isOpen) { // startWithSound(); } else if (isActive) { stop(); } }, [voiceChat.isOpen]); useEffect(() => { if (!voiceChat.isOpen && !isNull(voiceChat.agentId)) { appStoreMutate((prev) => ({ voiceChat: { ...prev.voiceChat, agentId: undefined, }, })); } }, [voiceChat.isOpen]); useEffect(() => { if (error && isActive) { toast.error(error.message); stop(); } }, [error]); useEffect(() => { if (voiceChat.isOpen) return; const handleKeyDown = (e: KeyboardEvent) => { const isVoiceChatEvent = isShortcutEvent(e, Shortcuts.toggleVoiceChat); if (isVoiceChatEvent) { e.preventDefault(); e.stopPropagation(); appStoreMutate((prev) => ({ voiceChat: { ...prev.voiceChat, isOpen: true, agentId: undefined, }, })); } }; window.addEventListener("keydown", handleKeyDown); return () => window.removeEventListener("keydown", handleKeyDown); }, [voiceChat.isOpen]); return (
{agent && (
{agent.name.slice(0, 1)}
{agent.name}
{agent.description && (
{agent.description}
)}
)} {useCompactView ? t("VoiceChat.compactDisplayMode") : t("VoiceChat.conversationDisplayMode")} Open AI {Object.entries(OPENAI_VOICE).map( ([key, value]) => ( appStoreMutate({ voiceChat: { ...voiceChat, options: { provider: "openai", providerOptions: { voice: value, }, }, }, }) } key={key} > {key} {value === voiceChat.options.providerOptions ?.voice && ( )} ), )} Gemini
Not Implemented Yet
{error ? (
Error {error.message}

{t("VoiceChat.pleaseCloseTheVoiceChatAndTryAgain")}

) : null} {isLoading ? (
) : (
{useCompactView ? ( ) : ( )}
)}
{statusMessage}
{!isActive ? t("VoiceChat.startConversation") : isListening ? t("VoiceChat.closeMic") : t("VoiceChat.openMic")}

{t("VoiceChat.endConversation")}

); } function ConversationView({ messages, }: { messages: UIMessageWithCompleted[] }) { const ref = useRef(null); useEffect(() => { if (ref.current) { ref.current.scrollTo({ top: ref.current.scrollHeight, behavior: "smooth", }); } }, [messages.length]); return (
{messages.map((message) => (
{!message.completed ? ( ) : ( message.parts.map((part, index) => { if (part.type === "text") { if (!part.text) { return ( ); } return (

{(part.text || "...") ?.trim() .split(" ") .map((word, wordIndex) => ( {word}{" "} ))}

); } else if (isToolUIPart(part)) { return ( ); } return

{part.type} unknown part

; }) )}
))}
); } function CompactMessageView({ messages, }: { messages: UIMessageWithCompleted[]; }) { const { toolParts, textPart } = useMemo(() => { const toolParts = messages .filter((msg) => msg.parts.some(isToolUIPart)) .map((msg) => msg.parts.find(isToolUIPart)); const textPart = messages.findLast((msg) => msg.role === "assistant") ?.parts[0] as TextPart; return { toolParts, textPart }; }, [messages]); return (
{toolParts.map((toolPart, index) => { const isExecuting = toolPart?.state.startsWith("input"); if (!toolPart) return null; return (
{getToolName(toolPart)}
Inputs
Outputs
); })}
{/* Current Message - Prominent */} {textPart && (

{textPart.text?.split(" ").map((word, wordIndex) => ( {word}{" "} ))}

)}
); }