import { useEffect, useMemo, useRef, useState } from 'react'; import type { DemoLocale, DemoMode, ProgressStep, ProgressStepStatus, TraceItem } from './types'; import VoiceFeatureDemo from './VoiceFeatureDemo'; import { sendVoiceAssistantMessage } from './liveApi'; import type { LiveVoiceToolUi } from './liveApi'; type VoiceStatus = 'idle' | 'connecting' | 'ready' | 'listening' | 'processing' | 'error'; type VoiceRole = 'assistant' | 'user' | 'system'; type VoiceToolCard = | { kind: 'product_updated'; title: string; description: string; product: { name: string; price: string; stock: string; category: string; }; actions: string[]; } | { kind: 'toolkit'; title: string; description: string; items: string[]; actions: string[]; } | { kind: 'stats'; title: string; description: string; stats: Array<{ label: string; value: string }>; actions: string[]; }; type VoiceEntry = { id: string; role: VoiceRole; text?: string; card?: VoiceToolCard; timestamp: string; }; type WorkflowStepTemplate = { id: string; label: string; runningDetail: string; doneDetail: string; kind: TraceItem['kind']; delayMs: number; }; type VoiceDemoPanelProps = { mode: DemoMode; locale: DemoLocale; onWorkingChange: (working: boolean) => void; onWorkflowInit: (steps: ProgressStep[]) => void; onStepStatus: (stepId: string, status: ProgressStepStatus, detail?: string) => void; onTrace: (item: TraceItem) => void; onClearPanels: () => void; queuedPrompt: string | null; onConsumeQueuedPrompt: () => void; }; function uiActionLabels(ui: LiveVoiceToolUi | undefined): string[] { if (!ui || !Array.isArray(ui.actions)) return []; return ui.actions .map((action) => (action && typeof action.label === 'string' ? action.label : '')) .filter(Boolean) .slice(0, 4); } function mapLiveUiToCard(ui: LiveVoiceToolUi | undefined, locale: DemoLocale): VoiceToolCard | undefined { if (!ui) return undefined; const kind = typeof ui.kind === 'string' ? ui.kind : ''; const fallbackTitle = locale === 'zh-TW' ? '工具結果' : 'Tool Result'; const title = typeof ui.title === 'string' ? ui.title : fallbackTitle; const description = typeof ui.description === 'string' ? ui.description : ''; const actions = uiActionLabels(ui); if (kind === 'product_created' || kind === 'product_updated') { const product = typeof ui.product === 'object' && ui.product ? (ui.product as Record) : {}; return { kind: 'product_updated', title, description, product: { name: typeof product.name === 'string' ? product.name : locale === 'zh-TW' ? '未命名產品' : 'Unnamed product', price: typeof product.price === 'number' ? `NT$ ${product.price}${typeof product.unit === 'string' && product.unit ? ` / ${product.unit}` : ''}` : '-', stock: typeof product.stock === 'number' ? String(product.stock) : '-', category: typeof product.category === 'string' ? product.category : '-', }, actions: actions.length ? actions : locale === 'zh-TW' ? ['查看產品'] : ['Open product'], }; } if (kind === 'stats' || kind === 'marketplace_stats') { const items = Array.isArray(ui.items) ? ui.items : []; const stats = items .map((item) => { if (typeof item !== 'object' || !item) return null; const row = item as Record; if (typeof row.label !== 'string') return null; return { label: row.label, value: typeof row.value === 'number' ? String(row.value) : typeof row.value === 'string' ? row.value : '-', }; }) .filter((row): row is { label: string; value: string } => Boolean(row)) .slice(0, 6); return { kind: 'stats', title, description, stats, actions: actions.length ? actions : locale === 'zh-TW' ? ['查看詳情'] : ['View details'], }; } const items = Array.isArray(ui.items) ? ui.items : []; const textItems = items .map((item) => { if (typeof item !== 'object' || !item) return ''; const row = item as Record; if (typeof row.title === 'string') return row.title; if (typeof row.name === 'string') return row.name; if (typeof row.label === 'string') return row.label; return ''; }) .filter(Boolean) .slice(0, 5); if (!textItems.length && !description) return undefined; return { kind: 'toolkit', title, description: description || (locale === 'zh-TW' ? '已收到後端工具輸出。' : 'Backend tool output received.'), items: textItems.length ? textItems : [locale === 'zh-TW' ? '已完成即時工具執行' : 'Live tool execution completed'], actions: actions.length ? actions : locale === 'zh-TW' ? ['下一步'] : ['Next step'], }; } const COPY = { en: { title: 'Voice Assistant', launcher: 'AI Voice Assistant', status: { idle: 'Idle', connecting: 'Connecting', ready: 'Ready', listening: 'Listening', processing: 'Thinking', error: 'Retry needed', }, listeningTitle: 'Listening', listeningDesc: 'Capturing your voice command…', thinkingTitle: 'Thinking', thinkingDesc: 'Planning tools and preparing response…', readyMessage: 'Voice assistant is ready. Tap the mic and start speaking.', reconnect: 'Reconnect', startTutorial: 'Start Voice Walkthrough', open: 'Open', close: 'Close', micStart: 'Start', micStop: 'Stop', tutorialRequest: 'Start a voice walkthrough for this page with key actions.', clearSession: 'Clear Session', roleLabels: { assistant: 'assistant', user: 'user', system: 'system', }, }, 'zh-TW': { title: '語音助理', launcher: 'AI 語音助理', status: { idle: '待命', connecting: '連線中', ready: '已就緒', listening: '聆聽中', processing: '思考中', error: '需要重試', }, listeningTitle: '聆聽中', listeningDesc: '正在接收你的語音指令…', thinkingTitle: '思考中', thinkingDesc: '正在規劃工具並準備回覆…', readyMessage: '語音助理已就緒,按下麥克風開始說話。', reconnect: '重新連線', startTutorial: '開始語音導覽', open: '開啟', close: '關閉', micStart: '開始', micStop: '停止', tutorialRequest: '請開始這個頁面的語音導覽,並說明主要操作。', clearSession: '清除對話', roleLabels: { assistant: '助理', user: '使用者', system: '系統', }, }, } as const; const VOICE_SAMPLE_INPUTS = { en: [ 'Find products below 200 and show the best options.', 'Update egg stock to 48 boxes and add pickup details.', 'Show my dashboard stats for this week.', ], 'zh-TW': ['幫我找 200 以下的雞蛋', '把雞蛋庫存改成 48 箱並補上取貨資訊', '顯示本週營運統計'], } as const; const now = () => new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); const sleep = (ms: number) => new Promise((resolve) => { setTimeout(resolve, ms); }); function makeId(prefix: string): string { return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`; } function buildTrace(kind: TraceItem['kind'], title: string, detail: string, status: TraceItem['status'], payload?: Record): TraceItem { return { id: makeId('trace'), kind, title, detail, status, payload, timestamp: now(), }; } function buildTurnSteps(locale: DemoLocale): WorkflowStepTemplate[] { if (locale === 'zh-TW') { return [ { id: 'voice-step-1', label: '語音內容標準化', runningDetail: '整理語句、判斷語系與指令意圖。', doneDetail: '語音轉寫與意圖解析完成。', kind: 'validator', delayMs: 540, }, { id: 'voice-step-2', label: '工具路由規劃', runningDetail: '選擇最適合的產品/市集工具。', doneDetail: '已決定工具與參數。', kind: 'planner', delayMs: 700, }, { id: 'voice-step-3', label: '執行工具', runningDetail: '執行動作並收集結構化結果。', doneDetail: '工具執行完成並回傳資料。', kind: 'tool', delayMs: 900, }, { id: 'voice-step-4', label: '回覆整理', runningDetail: '彙整語音摘要與卡片結果。', doneDetail: '最終語音回覆已完成。', kind: 'renderer', delayMs: 420, }, ]; } return [ { id: 'voice-step-1', label: 'Normalize transcript', runningDetail: 'Cleaning transcript and intent hints.', doneDetail: 'Transcript and intent parsing complete.', kind: 'validator', delayMs: 540, }, { id: 'voice-step-2', label: 'Plan tool route', runningDetail: 'Selecting the best action workflow.', doneDetail: 'Tool and arguments prepared.', kind: 'planner', delayMs: 700, }, { id: 'voice-step-3', label: 'Execute tool', runningDetail: 'Running action and collecting structured output.', doneDetail: 'Tool execution returned successfully.', kind: 'tool', delayMs: 900, }, { id: 'voice-step-4', label: 'Render voice response', runningDetail: 'Composing concise spoken summary and cards.', doneDetail: 'Final voice response prepared.', kind: 'renderer', delayMs: 420, }, ]; } function buildTutorialSteps(locale: DemoLocale): WorkflowStepTemplate[] { if (locale === 'zh-TW') { return [ { id: 'voice-tutorial-1', label: '分析頁面脈絡', runningDetail: '讀取目前頁面重點與導覽目標。', doneDetail: '頁面脈絡分析完成。', kind: 'planner', delayMs: 620, }, { id: 'voice-tutorial-2', label: '規劃導覽節奏', runningDetail: '安排導覽順序與下一步提示。', doneDetail: '導覽腳本已生成。', kind: 'renderer', delayMs: 780, }, ]; } return [ { id: 'voice-tutorial-1', label: 'Analyze page context', runningDetail: 'Reading current page scope and key tasks.', doneDetail: 'Page context resolved.', kind: 'planner', delayMs: 620, }, { id: 'voice-tutorial-2', label: 'Build walkthrough script', runningDetail: 'Preparing guided route and next actions.', doneDetail: 'Walkthrough output is ready.', kind: 'renderer', delayMs: 780, }, ]; } function buildToolCard(locale: DemoLocale): VoiceToolCard { if (locale === 'zh-TW') { return { kind: 'product_updated', title: '產品更新完成', description: '已套用語音指令並更新產品資料。', product: { name: '放牧雞蛋', price: 'NT$ 118 / 盒', stock: '48', category: '蛋品', }, actions: ['查看產品', '繼續修改', '設定取貨地址'], }; } return { kind: 'product_updated', title: 'Product Updated', description: 'Your spoken command has been applied to the product record.', product: { name: 'Free-range Eggs', price: 'NT$ 118 / box', stock: '48', category: 'Eggs', }, actions: ['Open product', 'Edit again', 'Set pickup address'], }; } function buildTutorialCard(locale: DemoLocale): VoiceToolCard { if (locale === 'zh-TW') { return { kind: 'toolkit', title: '頁面語音導覽', description: '這是本頁建議的操作路徑。', items: ['先查看目前庫存與價格', '接著更新產品圖片或文案', '最後確認店家與取貨設定'], actions: ['開始第一步', '跳到產品管理', '查看分析'], }; } return { kind: 'toolkit', title: 'Page Voice Walkthrough', description: 'Recommended sequence for this screen.', items: ['Review current stock and pricing', 'Update product image or copy', 'Confirm store and pickup settings'], actions: ['Start step 1', 'Go to products', 'Open analytics'], }; } function buildStatsCard(locale: DemoLocale): VoiceToolCard { if (locale === 'zh-TW') { return { kind: 'stats', title: '語音摘要統計', description: '依目前資料整理的重點指標。', stats: [ { label: '產品數', value: '42' }, { label: '店家數', value: '18' }, { label: '本週互動', value: '136' }, ], actions: ['查看詳情', '切換市集模式'], }; } return { kind: 'stats', title: 'Voice Summary Stats', description: 'Key indicators assembled from current data.', stats: [ { label: 'Products', value: '42' }, { label: 'Stores', value: '18' }, { label: 'Weekly interactions', value: '136' }, ], actions: ['View details', 'Switch market mode'], }; } export default function VoiceDemoPanel({ mode, locale, onWorkingChange, onWorkflowInit, onStepStatus, onTrace, onClearPanels, queuedPrompt, onConsumeQueuedPrompt, }: VoiceDemoPanelProps) { const copy = COPY[locale]; const [isOpen, setIsOpen] = useState(false); const [status, setStatus] = useState('idle'); const [isRecording, setIsRecording] = useState(false); const [micLevel, setMicLevel] = useState(0); const [entries, setEntries] = useState([]); const threadRef = useRef(null); const micIntervalRef = useRef(null); const autoStopRef = useRef(null); const runRef = useRef(0); const mountedRef = useRef(true); const activity = useMemo(() => { if (status === 'listening') return { title: copy.listeningTitle, desc: copy.listeningDesc, style: 'listening' as const }; if (status === 'processing' || status === 'connecting') return { title: copy.thinkingTitle, desc: copy.thinkingDesc, style: 'thinking' as const }; return null; }, [copy.listeningDesc, copy.listeningTitle, copy.thinkingDesc, copy.thinkingTitle, status]); const appendEntry = (entry: Omit) => { setEntries((prev) => [...prev, { ...entry, id: makeId('voice-entry'), timestamp: now() }]); }; const stopMicSimulation = () => { if (micIntervalRef.current !== null) { window.clearInterval(micIntervalRef.current); micIntervalRef.current = null; } if (autoStopRef.current !== null) { window.clearTimeout(autoStopRef.current); autoStopRef.current = null; } setMicLevel(0); setIsRecording(false); }; const runWorkflow = async (steps: WorkflowStepTemplate[], outcome: { text: string; card?: VoiceToolCard }) => { runRef.current += 1; const runId = runRef.current; onWorkingChange(true); setStatus('processing'); const initialized: ProgressStep[] = steps.map((step) => ({ id: step.id, label: step.label, detail: step.runningDetail, status: 'pending', })); onWorkflowInit(initialized); for (const step of steps) { if (!mountedRef.current || runRef.current !== runId) return; onStepStatus(step.id, 'in_progress', step.runningDetail); onTrace(buildTrace(step.kind, step.label, step.runningDetail, 'running')); await sleep(step.delayMs); if (!mountedRef.current || runRef.current !== runId) return; onStepStatus(step.id, 'completed', step.doneDetail); onTrace( buildTrace(step.kind, step.label, step.doneDetail, 'ok', step.kind === 'tool' ? { route: 'voice_tool', latencyMs: Math.floor(640 + Math.random() * 280) } : undefined) ); } if (!mountedRef.current || runRef.current !== runId) return; appendEntry({ role: 'assistant', text: outcome.text }); if (outcome.card) appendEntry({ role: 'assistant', card: outcome.card }); setStatus('ready'); onWorkingChange(false); }; const openAssistant = async () => { if (isOpen) return; setIsOpen(true); setStatus('connecting'); onWorkingChange(true); const stepId = 'voice-connect'; onWorkflowInit([ { id: stepId, label: locale === 'zh-TW' ? '建立語音工作階段' : 'Establish voice session', detail: locale === 'zh-TW' ? '初始化語音介面與音訊狀態。' : 'Initializing voice runtime and audio state.', status: 'pending', }, ]); onStepStatus( stepId, 'in_progress', locale === 'zh-TW' ? '正在連線語音流程…' : 'Connecting voice workflow…' ); onTrace( buildTrace( 'planner', locale === 'zh-TW' ? '語音初始化' : 'Voice bootstrap', locale === 'zh-TW' ? '建立前端語音示範工作階段。' : 'Creating frontend voice demo session.', 'running' ) ); await sleep(540); if (!mountedRef.current) return; onStepStatus( stepId, 'completed', locale === 'zh-TW' ? '語音工作階段已就緒。' : 'Voice session is ready.' ); onTrace( buildTrace( 'planner', locale === 'zh-TW' ? '語音初始化' : 'Voice bootstrap', locale === 'zh-TW' ? '語音介面可開始收音。' : 'Voice interface is now ready for capture.', 'ok' ) ); setStatus('ready'); onWorkingChange(false); appendEntry({ role: 'system', text: copy.readyMessage }); }; const closeAssistant = () => { stopMicSimulation(); runRef.current += 1; setStatus('idle'); setIsOpen(false); onWorkingChange(false); }; const runLiveVoiceTurn = async (transcript: string) => { runRef.current += 1; const runId = runRef.current; onWorkingChange(true); setStatus('processing'); const steps = buildTurnSteps(locale); onWorkflowInit( steps.map((step) => ({ id: step.id, label: step.label, detail: step.runningDetail, status: 'pending', })) ); try { onStepStatus(steps[0].id, 'in_progress', steps[0].runningDetail); onTrace(buildTrace(steps[0].kind, steps[0].label, steps[0].runningDetail, 'running')); await sleep(160); if (!mountedRef.current || runRef.current !== runId) return; onStepStatus(steps[0].id, 'completed', steps[0].doneDetail); onTrace(buildTrace(steps[0].kind, steps[0].label, steps[0].doneDetail, 'ok')); onStepStatus(steps[1].id, 'in_progress', steps[1].runningDetail); onTrace(buildTrace(steps[1].kind, steps[1].label, steps[1].runningDetail, 'running')); const response = await sendVoiceAssistantMessage({ message: transcript, locale, pageRoute: '/hf-demo/voice', pageContext: locale === 'zh-TW' ? 'Hugging Face Voice Agent Demo' : 'Hugging Face Voice Agent Demo', }); if (!mountedRef.current || runRef.current !== runId) return; onStepStatus(steps[1].id, 'completed', steps[1].doneDetail); onTrace( buildTrace(steps[1].kind, steps[1].label, steps[1].doneDetail, 'ok', { messages: response.messages?.length ?? 0, uiKind: response.ui?.kind || null, }) ); onStepStatus(steps[2].id, 'in_progress', steps[2].runningDetail); onTrace(buildTrace(steps[2].kind, steps[2].label, steps[2].runningDetail, 'running')); const assistantTexts = (response.messages || []) .filter((message) => message.role === 'assistant' && message.text) .map((message) => message.text); if (assistantTexts.length) { for (const text of assistantTexts) { appendEntry({ role: 'assistant', text }); } } else { appendEntry({ role: 'assistant', text: locale === 'zh-TW' ? '已完成後端語音工具流程。' : 'Live voice tool flow completed.', }); } const liveCard = mapLiveUiToCard(response.ui, locale); if (liveCard) { appendEntry({ role: 'assistant', card: liveCard }); } onStepStatus(steps[2].id, 'completed', steps[2].doneDetail); onTrace( buildTrace(steps[2].kind, steps[2].label, steps[2].doneDetail, 'ok', { card: liveCard?.kind || null, }) ); onStepStatus(steps[3].id, 'in_progress', steps[3].runningDetail); onTrace(buildTrace(steps[3].kind, steps[3].label, steps[3].runningDetail, 'running')); await sleep(120); if (!mountedRef.current || runRef.current !== runId) return; onStepStatus(steps[3].id, 'completed', steps[3].doneDetail); onTrace(buildTrace(steps[3].kind, steps[3].label, steps[3].doneDetail, 'ok')); setStatus('ready'); onWorkingChange(false); } catch (error) { const message = error instanceof Error ? error.message : locale === 'zh-TW' ? '語音流程發生錯誤。' : 'Voice flow failed.'; onStepStatus(steps[1].id, 'error', message); onTrace(buildTrace('tool', locale === 'zh-TW' ? '語音錯誤' : 'Voice error', message, 'error')); setStatus('error'); onWorkingChange(false); } }; const simulateVoiceTurn = async (transcript?: string) => { const sample = transcript || VOICE_SAMPLE_INPUTS[locale][Math.floor(Math.random() * VOICE_SAMPLE_INPUTS[locale].length)]; appendEntry({ role: 'user', text: sample }); if (mode === 'live') { await runLiveVoiceTurn(sample); return; } const steps = buildTurnSteps(locale); const summary = locale === 'zh-TW' ? `我已處理「${sample}」,並完成對應工具操作。要不要繼續更新圖片或取貨設定?` : `I processed "${sample}" and finished the tool workflow. Want to update images or pickup details next?`; const card = sample.includes('統計') || sample.toLowerCase().includes('stats') ? buildStatsCard(locale) : buildToolCard(locale); await runWorkflow(steps, { text: summary, card }); }; const stopRecording = async (forcedTranscript?: string) => { if (!isRecording) return; stopMicSimulation(); await simulateVoiceTurn(forcedTranscript); }; const startRecording = () => { if (isRecording || status === 'processing' || status === 'connecting') return; setStatus('listening'); setIsRecording(true); micIntervalRef.current = window.setInterval(() => { setMicLevel(0.18 + Math.random() * 0.82); }, 120); autoStopRef.current = window.setTimeout(() => { void stopRecording(); }, 2800); }; const toggleMic = () => { if (isRecording) { void stopRecording(); return; } startRecording(); }; const handleTutorial = async () => { if (!isOpen) { await openAssistant(); } appendEntry({ role: 'user', text: copy.tutorialRequest }); if (mode === 'live') { await runLiveVoiceTurn(copy.tutorialRequest); return; } await runWorkflow(buildTutorialSteps(locale), { text: locale === 'zh-TW' ? '已完成語音導覽摘要。我先帶你看重點,再引導下一步操作。' : 'Voice walkthrough summary is ready. I will guide the key sections step by step.', card: buildTutorialCard(locale), }); }; useEffect(() => { if (!queuedPrompt) return; const applyQueuedPrompt = async () => { onConsumeQueuedPrompt(); if (!isOpen) { await openAssistant(); } await simulateVoiceTurn(queuedPrompt); }; void applyQueuedPrompt(); }, [queuedPrompt]); useEffect(() => { if (!threadRef.current) return; threadRef.current.scrollTo({ top: threadRef.current.scrollHeight, behavior: 'smooth' }); }, [activity, entries, isRecording, micLevel]); useEffect(() => { mountedRef.current = true; return () => { mountedRef.current = false; stopMicSimulation(); onWorkingChange(false); }; }, []); const clearSession = () => { setEntries([]); onClearPanels(); }; const renderCard = (card: VoiceToolCard) => { if (card.kind === 'product_updated') { return (

{card.title}

{card.description}

{locale === 'zh-TW' ? '品項' : 'Product'} {card.product.name}
{locale === 'zh-TW' ? '價格' : 'Price'} {card.product.price}
{locale === 'zh-TW' ? '庫存' : 'Stock'} {card.product.stock}
{locale === 'zh-TW' ? '分類' : 'Category'} {card.product.category}
{card.actions.map((action) => ( ))}
); } if (card.kind === 'stats') { return (

{card.title}

{card.description}

{card.stats.map((stat) => (
{stat.label} {stat.value}
))}
{card.actions.map((action) => ( ))}
); } return (

{card.title}

{card.description}

    {card.items.map((item) => (
  • {item}
  • ))}
{card.actions.map((action) => ( ))}
); }; const isDemoBusy = status === 'connecting' || status === 'listening' || status === 'processing'; return (
{isOpen ? (

{copy.title}

{copy.status[status]}
{entries.length === 0 ? (

{locale === 'zh-TW' ? '等待語音輸入…' : 'Waiting for voice input…'}

) : null} {entries.map((entry) => (
{copy.roleLabels[entry.role]}
{entry.text ?

{entry.text}

: null} {entry.card ? renderCard(entry.card) : null}
))}
{activity ? (

{activity.title}

{activity.desc}

) : null}
) : (
)}
); }