import { Loader2, Mic, MicOff, Navigation, ShieldOff, Volume2, X, } from 'lucide-react'; import { useCallback, useEffect, useRef, useState } from 'react'; const T = { bg: '#0d0d0d', surface: 'rgba(255,255,255,0.025)', border: 'rgba(255,255,255,0.08)', accent: '#c9b787', text: '#f5f5f5', textDim: '#8a8a8a', textMuted: '#5e5e5e', mono: 'ui-monospace, SFMono-Regular, Menlo, monospace', red: '#ef4444', green: '#22c55e', }; // --------------------------------------------------------------------------- // Navigation intent parsing // --------------------------------------------------------------------------- /** * Maps spoken artifact/section names to their URL paths. * External app paths trigger a full page navigation (window.location.href). * Internal Command paths use wouter navigation. */ const ARTIFACT_ROUTE_MAP: Array<{ keywords: string[]; path: string; label: string; external: boolean }> = [ // External ecosystem apps { keywords: ['aegis', 'sentra', 'defense', 'cyber', 'cyber resilience', 'intelligence command', 'cyber command'], path: '/sentra/', label: 'Sentra', external: true }, { keywords: ['vessels', 'maritime', 'fleet', 'ship'], path: '/vessels/', label: 'SEXTANT', external: true }, { keywords: ['terra', 'real estate'], path: '/terra/', label: 'DOMAINE', external: true }, { keywords: ['counsel', 'legal', 'matter command'], path: '/counsel/', label: 'Counsel', external: true }, { keywords: ['pulse', 'briefing', 'executive briefing'], path: '/pulse/', label: 'Pulse', external: true }, { keywords: ['a11oy', 'continuum', 'brand orchestration'], path: '/a11oy/', label: 'A11oy', external: true }, { keywords: ['holdings', 'szl holdings', 'portfolio dashboard'], path: '/szl-holdings/', label: 'SZL Holdings', external: true }, { keywords: ['lexicon', 'license intelligence', 'license catalog'], path: '/governance/lexicon', label: 'LEXICON', external: false }, // Internal Command routes { keywords: ['strategy', 'strategic overview'], path: '/strategy', label: 'Strategy', external: false }, { keywords: ['operations', 'live operations', 'ops'], path: '/operations/live', label: 'Live Operations', external: false }, { keywords: ['agents', 'agent monitor', 'agent runtime'], path: '/agents', label: 'Agents', external: false }, { keywords: ['omnia', 'world model'], path: '/omnia', label: 'OMNIA', external: false }, { keywords: ['cognitive', 'cognitive loop'], path: '/cognitive', label: 'Cognitive', external: false }, { keywords: ['infrastructure', 'global fabric'], path: '/infrastructure', label: 'Infrastructure', external: false }, { keywords: ['approvals', 'hitl', 'approval queue'], path: '/substrate/approvals', label: 'Approvals', external: false }, { keywords: ['evolution', 'evolution runtime'], path: '/evolution', label: 'Evolution', external: false }, { keywords: ['forge', 'tool intelligence'], path: '/operations/forge', label: 'Forge', external: false }, ]; // --------------------------------------------------------------------------- // Approval intent detection // --------------------------------------------------------------------------- const APPROVAL_INTENT_WORDS = [ 'approve', 'approved', 'confirm', 'confirmed', 'authorize', 'authorise', 'proceed', 'yes proceed', 'go ahead', 'i approve', 'i confirm', ]; interface ApprovalResult { actionId: string; actionLabel: string; transcript: string; approved: boolean; reason?: string; } function detectApprovalIntent(userText: string): { detected: boolean; actionLabel: string } { const lower = userText.toLowerCase(); const detected = APPROVAL_INTENT_WORDS.some((w) => lower.includes(w)); const match = lower.match(/(?:approve|confirm|authorize)\s+(?:the\s+)?(.{3,60}?)(?:\s|$)/); const actionLabel = match ? match[1].trim() : 'pending action'; return { detected, actionLabel }; } /** Navigation trigger phrases the assistant will produce */ const NAV_TRIGGER_PATTERNS = [ /navigating\s+(?:you\s+)?to\s+(.+?)(?:\.|$)/i, /taking\s+you\s+to\s+(.+?)(?:\.|$)/i, /opening\s+(.+?)(?:\.|$)/i, /switching\s+to\s+(.+?)(?:\.|$)/i, /going\s+to\s+(.+?)(?:\.|$)/i, /i(?:'ll|'m going to)\s+navigate\s+(?:you\s+)?to\s+(.+?)(?:\.|$)/i, ]; interface NavigationIntent { path: string; label: string; external: boolean; } function parseNavigationIntent(text: string): NavigationIntent | null { const lowerText = text.toLowerCase(); // Check for explicit navigation phrases for (const pattern of NAV_TRIGGER_PATTERNS) { const match = lowerText.match(pattern); if (match) { const target = match[1]?.trim() ?? ''; for (const route of ARTIFACT_ROUTE_MAP) { if (route.keywords.some((kw) => target.includes(kw))) { return { path: route.path, label: route.label, external: route.external }; } } } } // Fallback: check if any artifact keyword appears in a navigation context const navContextWords = ['navigate', 'go', 'open', 'switch', 'take you', 'heading']; const hasNavContext = navContextWords.some((w) => lowerText.includes(w)); if (hasNavContext) { for (const route of ARTIFACT_ROUTE_MAP) { if (route.keywords.some((kw) => lowerText.includes(kw))) { return { path: route.path, label: route.label, external: route.external }; } } } return null; } // --------------------------------------------------------------------------- // Component // --------------------------------------------------------------------------- interface Message { id: string; role: 'user' | 'assistant'; content: string; timestamp: Date; navigationIntent?: NavigationIntent; } function getApiBase() { const base = import.meta.env.BASE_URL?.replace(/\/$/, '') || ''; return `${base}/api`; } async function blobToBase64(blob: Blob): Promise { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => { const result = reader.result as string; resolve(result.split(',')[1] ?? ''); }; reader.onerror = reject; reader.readAsDataURL(blob); }); } interface VoiceCommandPanelProps { open: boolean; onClose: () => void; /** Called when the assistant produces a navigation intent. Path may be external or internal. */ onNavigate?: (path: string, external: boolean) => void; } export function VoiceCommandPanel({ open, onClose, onNavigate }: VoiceCommandPanelProps) { const [messages, setMessages] = useState([]); const [privacyMode, setPrivacyMode] = useState(false); const [userTranscript, setUserTranscript] = useState(''); const [isRecording, setIsRecording] = useState(false); const [isProcessing, setIsProcessing] = useState(false); const [isPlaying, setIsPlaying] = useState(false); const [error, setError] = useState(null); const [pendingNavIntent, setPendingNavIntent] = useState(null); const [lastApproval, setLastApproval] = useState(null); const scrollRef = useRef(null); const mediaRecorderRef = useRef(null); const audioChunksRef = useRef([]); const audioRef = useRef(null); const objectUrlRef = useRef(null); const abortRef = useRef(null); const conversationIdRef = useRef(null); /** Real pending approvals fetched from the DB on panel open — used for voice approvals. */ const pendingApprovalsRef = useRef>([]); // Create a conversation session and fetch pending approvals when the panel opens useEffect(() => { if (!open) return; setMessages([]); setPendingNavIntent(null); conversationIdRef.current = null; pendingApprovalsRef.current = []; const abort = new AbortController(); // Create conversation session (non-fatal if fails) fetch(`${getApiBase()}/openai/conversations`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, credentials: 'include', signal: abort.signal, }) .then((r) => r.json()) .then((data: { id?: string | number }) => { if (data.id) conversationIdRef.current = String(data.id); }) .catch(() => { /* non-fatal */ }); // Fetch real pending approvals so voice-approval calls use actual DB IDs fetch(`${getApiBase()}/approvals?status=pending&limit=20`, { credentials: 'include', signal: abort.signal, }) .then((r) => r.ok ? r.json() : null) .then((resp: { data?: Array<{ id: number; title: string }> } | Array<{ id: number; title: string }> | null) => { // Handles both sendSuccess shapes: raw array OR { data, meta } envelope const list: Array<{ id: number; title: string }> = Array.isArray(resp) ? resp : (resp as { data?: Array<{ id: number; title: string }> })?.data ?? []; pendingApprovalsRef.current = list.map((a) => ({ id: a.id, title: a.title })); }) .catch(() => { /* non-fatal — approval list unavailable */ }); return () => abort.abort(); }, [open]); useEffect(() => { if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; } }, [messages, userTranscript]); useEffect(() => { return () => { mediaRecorderRef.current?.stop(); audioRef.current?.pause(); if (objectUrlRef.current) URL.revokeObjectURL(objectUrlRef.current); abortRef.current?.abort(); }; }, []); const stopAudio = useCallback(() => { abortRef.current?.abort(); audioRef.current?.pause(); if (objectUrlRef.current) { URL.revokeObjectURL(objectUrlRef.current); objectUrlRef.current = null; } setIsPlaying(false); }, []); const executeNavigation = useCallback( (intent: NavigationIntent) => { if (onNavigate) { onNavigate(intent.path, intent.external); } else if (intent.external) { window.location.href = intent.path; } setPendingNavIntent(null); onClose(); }, [onNavigate, onClose], ); const handleMicPress = useCallback(async () => { if (privacyMode) return; if (isRecording) { setIsRecording(false); mediaRecorderRef.current?.stop(); return; } setError(null); setPendingNavIntent(null); audioChunksRef.current = []; let stream: MediaStream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); } catch { setError('Microphone access denied. Check browser permissions.'); return; } const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') ? 'audio/webm;codecs=opus' : 'audio/webm'; const recorder = new MediaRecorder(stream, { mimeType }); mediaRecorderRef.current = recorder; recorder.ondataavailable = (e) => { if (e.data.size > 0) audioChunksRef.current.push(e.data); }; recorder.onstop = async () => { stream.getTracks().forEach((t) => t.stop()); const blob = new Blob(audioChunksRef.current, { type: mimeType }); audioChunksRef.current = []; mediaRecorderRef.current = null; if (blob.size < 1000) return; setIsProcessing(true); stopAudio(); try { const base64 = await blobToBase64(blob); const abort = new AbortController(); abortRef.current = abort; const body: Record = { audio: base64 }; if (conversationIdRef.current) body.conversationId = conversationIdRef.current; const res = await fetch(`${getApiBase()}/openai/voice-query`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), credentials: 'include', signal: abort.signal, }); if (!res.ok) throw new Error('Voice query failed'); const data = await res.json() as { userTranscript?: string; assistantTranscript?: string; audioBase64?: string; provenance?: { conversationId?: string }; }; // Update conversationId if server assigned/echoed one if (data.provenance?.conversationId && !conversationIdRef.current) { conversationIdRef.current = data.provenance.conversationId; } const userText = data.userTranscript?.trim() || userTranscript || 'Voice query'; const assistantText = data.assistantTranscript?.trim() || 'No response received.'; // Check for approval intent in the user's transcript const { detected: hasApprovalIntent, actionLabel } = detectApprovalIntent(userText); if (hasApprovalIntent && pendingApprovalsRef.current.length > 0) { // Match the spoken action label against real pending approvals; default to first pending const matched = pendingApprovalsRef.current.find( (a) => actionLabel && a.title.toLowerCase().includes(actionLabel.toLowerCase()), ) ?? pendingApprovalsRef.current[0]; fetch(`${getApiBase()}/openai/voice-approval`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ pendingActionId: matched.id, transcript: userText, ...(conversationIdRef.current ? { conversationId: conversationIdRef.current } : {}), }), credentials: 'include', }) .then((r) => r.json()) .then((result: { approved?: boolean; reason?: string }) => { setLastApproval({ actionId: String(matched.id), actionLabel: matched.title, transcript: userText, approved: result.approved ?? false, reason: result.reason, }); // Remove approved item from local ref to prevent re-submission if (result.approved) { pendingApprovalsRef.current = pendingApprovalsRef.current.filter( (a) => a.id !== matched.id, ); } setTimeout(() => setLastApproval(null), 8000); }) .catch(() => {/* non-fatal */}); } else if (hasApprovalIntent && pendingApprovalsRef.current.length === 0) { // Intent detected but no real pending approvals available setLastApproval({ actionId: 'none', actionLabel, transcript: userText, approved: false, reason: 'No pending approvals in queue — nothing to approve', }); setTimeout(() => setLastApproval(null), 6000); } // Check for navigation intent const navIntent = parseNavigationIntent(assistantText); setUserTranscript(''); setMessages((prev) => [ ...prev, { id: `user-${Date.now()}`, role: 'user', content: userText, timestamp: new Date(), }, { id: `asst-${Date.now()}`, role: 'assistant', content: assistantText, timestamp: new Date(), navigationIntent: navIntent ?? undefined, }, ]); if (navIntent) { setPendingNavIntent(navIntent); } if (data.audioBase64 && !abort.signal.aborted) { const audioBytes = Uint8Array.from(atob(data.audioBase64), (c) => c.charCodeAt(0)); const audioBlob = new Blob([audioBytes], { type: 'audio/mpeg' }); const url = URL.createObjectURL(audioBlob); objectUrlRef.current = url; const audio = new Audio(url); audioRef.current = audio; setIsPlaying(true); audio.onended = () => { setIsPlaying(false); URL.revokeObjectURL(url); objectUrlRef.current = null; // Auto-execute navigation after audio finishes (if still pending) if (navIntent) { setPendingNavIntent((current) => { if (current) executeNavigation(current); return null; }); } }; audio.onerror = () => setIsPlaying(false); abort.signal.addEventListener('abort', () => audio.pause()); audio.play().catch(() => setIsPlaying(false)); } else if (navIntent) { // No audio — navigate immediately after a short delay setTimeout(() => executeNavigation(navIntent), 800); } } catch (err) { if ((err as Error).name !== 'AbortError') { setError('Voice processing failed. Please try again.'); } } finally { setIsProcessing(false); } }; recorder.start(); setIsRecording(true); setUserTranscript('Recording…'); }, [privacyMode, isRecording, userTranscript, stopAudio, executeNavigation]); // When privacy mode is enabled mid-recording, immediately stop the mic useEffect(() => { if (privacyMode && isRecording) { mediaRecorderRef.current?.stop(); setIsRecording(false); setUserTranscript(''); } }, [privacyMode, isRecording]); const handleClose = useCallback(() => { if (isRecording) { mediaRecorderRef.current?.stop(); setIsRecording(false); } stopAudio(); onClose(); }, [isRecording, stopAudio, onClose]); const isBusy = isProcessing || isPlaying; if (!open) return null; return (
{/* Header */}
Command Voice
{/* Voice approval confirmation banner */} {lastApproval && (
{lastApproval.approved ? `✓ Voice approval recorded — "${lastApproval.actionLabel}"` : `✗ Approval not confirmed — rephrase with "approve" or "confirm"`}
)} {/* Privacy mode banner */} {privacyMode && (
Privacy mode — microphone muted
)} {/* Pending navigation intent banner */} {pendingNavIntent && (
Navigate to {pendingNavIntent.label}?
)} {/* Message thread */}
{messages.length === 0 && !userTranscript && !isProcessing && (
Speak to navigate, query, or approve actions
"Show me active threats" · "Navigate to PARAGON" · "Read my briefing"
)} {messages.map((msg) => (
{msg.content} {msg.navigationIntent && (
Route: {msg.navigationIntent.label}
)}
{msg.role === 'user' ? 'You' : 'Command'} {' · '} {msg.timestamp.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}
))} {userTranscript && (
{userTranscript}
)} {isProcessing && (
Processing voice query…
)} {error && (
{error}
)}
{/* Controls */}
{isPlaying && (
Speaking…
)} {!isBusy && ( )} {isProcessing && !isPlaying && ( )}
{isRecording ? 'Recording — tap mic to send' : isProcessing ? 'Processing your query…' : isPlaying ? 'Command is responding…' : pendingNavIntent ? `Ready to navigate to ${pendingNavIntent.label}` : 'Tap mic to speak'}
); }