import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { createTranscriptionSocket, getTranscriptionSocketUrl, type TranscriptSocketMessage } from '../lib/websocket' type VoiceSessionState = 'idle' | 'connecting' | 'recording' | 'stopping' | 'error' type UseVoiceSessionOptions = { onFallback?: () => void onRecordingStarted?: () => void } const CHUNK_INTERVAL_MS = 2500 const VOICE_DETECTED_THRESHOLD = 0.08 const MEDIA_RECORDER_MIME_TYPES = [ 'audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', ] export function useVoiceSession({ onFallback, onRecordingStarted }: UseVoiceSessionOptions) { const socketRef = useRef(null) const recorderRef = useRef(null) const streamRef = useRef(null) const audioContextRef = useRef(null) const analyserRef = useRef(null) const levelRafRef = useRef(0) const sessionIdRef = useRef(0) const transcriptRef = useRef('') const voiceDetectedLoggedRef = useRef(false) const lastAudioLevelLogAtRef = useRef(0) const [state, setState] = useState('idle') const [transcript, setTranscript] = useState('') const [audioLevel, setAudioLevel] = useState(0) const [hasDetectedVoice, setHasDetectedVoice] = useState(false) const [isTranscribingChunk, setIsTranscribingChunk] = useState(false) const [lastVoiceDetectedAt, setLastVoiceDetectedAt] = useState(null) const [lastTranscriptReceivedAt, setLastTranscriptReceivedAt] = useState(null) const [generationStatus, setGenerationStatus] = useState(null) const [errorMessage, setErrorMessage] = useState(null) const cleanup = useCallback(() => { window.cancelAnimationFrame(levelRafRef.current) levelRafRef.current = 0 analyserRef.current = null audioContextRef.current?.close().catch(() => undefined) audioContextRef.current = null recorderRef.current = null socketRef.current?.close() socketRef.current = null streamRef.current?.getTracks().forEach((track) => track.stop()) streamRef.current = null setAudioLevel(0) setIsTranscribingChunk(false) }, []) const reset = useCallback(() => { cleanup() setState('idle') sessionIdRef.current += 1 transcriptRef.current = '' setTranscript('') setAudioLevel(0) setHasDetectedVoice(false) voiceDetectedLoggedRef.current = false lastAudioLevelLogAtRef.current = 0 setIsTranscribingChunk(false) setLastVoiceDetectedAt(null) setLastTranscriptReceivedAt(null) setGenerationStatus(null) setErrorMessage(null) }, [cleanup]) const stop = useCallback(async (finalizeMs = 1600) => { setState('stopping') const recorder = recorderRef.current if (recorder && recorder.state !== 'inactive') { recorder.requestData() recorder.stop() } streamRef.current?.getTracks().forEach((track) => track.stop()) streamRef.current = null window.cancelAnimationFrame(levelRafRef.current) levelRafRef.current = 0 setAudioLevel(0) console.debug('stop recording') await wait(finalizeMs) const finalTranscript = transcriptRef.current sessionIdRef.current += 1 cleanup() setState('idle') return finalTranscript }, [cleanup]) const startAudioLevelDetection = useCallback(async (stream: MediaStream, sessionId: number) => { const audioWindow = window as Window & { webkitAudioContext?: typeof AudioContext } const AudioContextClass = window.AudioContext || audioWindow.webkitAudioContext if (!AudioContextClass) { console.info('REAL_AUDIO_CONTEXT_UNAVAILABLE') return } const audioContext = new AudioContextClass() const analyser = audioContext.createAnalyser() analyser.fftSize = 1024 audioContext.createMediaStreamSource(stream).connect(analyser) audioContextRef.current = audioContext analyserRef.current = analyser console.info('REAL_AUDIO_CONTEXT_STATE', { state: audioContext.state }) if (audioContext.state === 'suspended') { try { await audioContext.resume() console.info('REAL_AUDIO_CONTEXT_RESUMED', { state: audioContext.state }) } catch (error) { console.info('REAL_AUDIO_CONTEXT_RESUME_FAILED', getErrorMessage(error)) } } const samples = new Uint8Array(analyser.fftSize) const tick = () => { if (sessionIdRef.current !== sessionId || !analyserRef.current) return analyserRef.current.getByteTimeDomainData(samples) let sum = 0 for (let index = 0; index < samples.length; index += 1) { const normalized = (samples[index] - 128) / 128 sum += normalized * normalized } const rms = Math.sqrt(sum / samples.length) const level = Math.min(1, Math.max(0, rms * 4)) setAudioLevel(level) const now = Date.now() if (now - lastAudioLevelLogAtRef.current > 1000) { lastAudioLevelLogAtRef.current = now console.info('REAL_AUDIO_LEVEL_SAMPLE', { audioContextState: audioContext.state, level, rms, }) } if (level >= VOICE_DETECTED_THRESHOLD) { if (!voiceDetectedLoggedRef.current) { console.debug('voice detected threshold reached') voiceDetectedLoggedRef.current = true } setHasDetectedVoice(true) setLastVoiceDetectedAt(Date.now()) } levelRafRef.current = window.requestAnimationFrame(tick) } tick() }, []) const start = useCallback(async () => { console.info('REAL_RECORDING_ENV', getRecordingEnvironment()) if (state === 'connecting' || state === 'recording') { console.info('REAL_RECORDING_START_SKIPPED', { state }) return false } if (!navigator.mediaDevices?.getUserMedia) { const message = 'Microphone capture is not available in this browser or context.' console.info('REAL_RECORDING_ERROR', message) console.error('[recording] microphone capture is not available in this browser or context') setErrorMessage(message) setState('error') return false } if (typeof MediaRecorder === 'undefined') { const message = 'MediaRecorder is not supported in this browser.' console.info('REAL_RECORDING_ERROR', message) console.error('[recording] MediaRecorder is not supported in this browser') setErrorMessage(message) setState('error') return false } setState('connecting') transcriptRef.current = '' setTranscript('') setAudioLevel(0) setHasDetectedVoice(false) voiceDetectedLoggedRef.current = false setIsTranscribingChunk(false) setLastVoiceDetectedAt(null) setLastTranscriptReceivedAt(null) setGenerationStatus(null) setErrorMessage(null) const sessionId = sessionIdRef.current + 1 sessionIdRef.current = sessionId try { let stream: MediaStream try { console.info('REAL_MIC_REQUEST_START') stream = await navigator.mediaDevices.getUserMedia({ audio: true }) } catch (error) { const message = describeMicrophoneError(error) console.info('REAL_RECORDING_ERROR', message) console.error(`[recording] ${message}`, error) throw new RecordingStartError(message) } console.info('REAL_MIC_STREAM_ACQUIRED', { audioTracks: stream.getAudioTracks().map((track) => ({ enabled: track.enabled, id: track.id, label: track.label, muted: track.muted, readyState: track.readyState, settings: track.getSettings(), })), }) stream.getAudioTracks().forEach((track) => { track.addEventListener('mute', () => { console.info('REAL_AUDIO_TRACK_MUTED', { id: track.id, label: track.label, readyState: track.readyState, }) }) track.addEventListener('unmute', () => { console.info('REAL_AUDIO_TRACK_UNMUTED', { id: track.id, label: track.label, readyState: track.readyState, }) }) track.addEventListener('ended', () => { console.info('REAL_AUDIO_TRACK_ENDED', { id: track.id, label: track.label, readyState: track.readyState, }) }) }) console.debug('microphone stream acquired') const socketUrl = getTranscriptionSocketUrl() console.info('REAL_WS_CONNECTING', { url: socketUrl }) console.debug(`connecting transcription WebSocket: ${socketUrl}`) const socket = createTranscriptionSocket() streamRef.current = stream socketRef.current = socket await startAudioLevelDetection(stream, sessionId) await new Promise((resolve, reject) => { let didOpen = false const cleanupStartupListeners = () => { socket.removeEventListener('open', handleOpen) socket.removeEventListener('error', handleErrorBeforeOpen) socket.removeEventListener('close', handleCloseBeforeOpen) } const handleOpen = () => { didOpen = true cleanupStartupListeners() console.info('REAL_WS_CONNECTED', { url: socketUrl }) resolve() } const handleErrorBeforeOpen = (event: Event) => { if (didOpen) return cleanupStartupListeners() const message = `Transcription WebSocket unavailable at ${socketUrl}` console.info('REAL_RECORDING_ERROR', message) console.error('[recording] WebSocket connection failed before open', { url: socketUrl, event }) reject(new RecordingStartError(message)) } const handleCloseBeforeOpen = (event: CloseEvent) => { if (didOpen) return cleanupStartupListeners() const message = event.reason ? `Transcription WebSocket closed before recording started: ${event.reason}` : `Transcription WebSocket unavailable at ${socketUrl}` console.info('REAL_WS_CLOSED_BEFORE_OPEN', { code: event.code, reason: event.reason, wasClean: event.wasClean, url: socketUrl, }) console.info('REAL_RECORDING_ERROR', message) console.error('[recording] WebSocket closed before open', { url: socketUrl, event }) reject(new RecordingStartError(message)) } socket.addEventListener('open', handleOpen) socket.addEventListener('error', handleErrorBeforeOpen) socket.addEventListener('close', handleCloseBeforeOpen) }) socket.addEventListener('close', (event) => { console.info('REAL_WS_CLOSED_NORMAL', { code: event.code, reason: event.reason, wasClean: event.wasClean, url: socketUrl, }) }) socket.addEventListener('message', (event) => { if (sessionIdRef.current !== sessionId) return const data = parseSocketMessage(event.data) if (!data) return if (data.type === 'error') { const message = data.message ?? data.detail ?? 'Backend reported an unknown transcription error.' console.error(`[recording] backend error message: ${message}`) setErrorMessage(message) setState('error') return } if (data.type === 'transcript' && data.text.trim()) { console.info('REAL_TRANSCRIPT_RECEIVED', { chars: data.text.length, source: data.source, }) console.debug('transcript message received') setIsTranscribingChunk(false) setLastTranscriptReceivedAt(Date.now()) setTranscript((current) => { const merged = mergeTranscriptSegment(current, data.text) transcriptRef.current = merged console.debug('transcript state updated') return merged }) } if (data.type === 'generation_status') { setGenerationStatus(data.status) } }) const mimeType = chooseMediaRecorderMimeType() console.info('REAL_MEDIA_RECORDER_CREATED', { mimeType: mimeType || '' }) const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined) recorderRef.current = recorder recorder.addEventListener('dataavailable', (event) => { console.info('REAL_AUDIO_CHUNK_AVAILABLE', { size: event.data.size, type: event.data.type, socketReadyState: socket.readyState, }) if (event.data.size > 0 && socket.readyState === WebSocket.OPEN) { console.info('REAL_AUDIO_CHUNK_SENT', { size: event.data.size }) console.debug('audio chunk sent') setIsTranscribingChunk(true) socket.send(event.data) } }) recorder.start(CHUNK_INTERVAL_MS) console.info('REAL_MEDIA_RECORDER_STARTED', { intervalMs: CHUNK_INTERVAL_MS, mimeType: recorder.mimeType || '', state: recorder.state, }) setState('recording') onRecordingStarted?.() return true } catch (error) { const message = error instanceof RecordingStartError ? error.message : `Recording failed: ${getErrorMessage(error)}` console.info('REAL_RECORDING_ERROR', message) console.error('[recording] start failed', error) setErrorMessage(message) cleanup() setState('error') return false } }, [cleanup, onFallback, onRecordingStarted, startAudioLevelDetection, state]) useEffect( () => () => { sessionIdRef.current += 1 cleanup() }, [cleanup], ) const transcriptWords = useMemo( () => transcript.split(/\s+/).filter(Boolean), [transcript], ) const transcriptText = useMemo(() => transcript.trim(), [transcript]) return { generationStatus, errorMessage, audioLevel, hasDetectedVoice, isTranscribingChunk, isRecording: state === 'recording', isStarting: state === 'connecting', lastTranscriptReceivedAt, lastVoiceDetectedAt, state, transcriptText, transcriptWords, hasTranscript: transcriptWords.length > 0, reset, start, stop, } } export function mergeTranscriptSegment(currentTranscript: string, newSegment: string) { const current = normalizeWhitespace(currentTranscript) const incoming = normalizeWhitespace(newSegment) if (!incoming) return current if (!current) return incoming if (canonicalText(current) === canonicalText(incoming)) return current const currentWords = splitWords(current) const incomingWords = splitWords(incoming) const currentCanonical = currentWords.map(canonicalWord) const incomingCanonical = incomingWords.map(canonicalWord) if (startsWithWords(incomingCanonical, currentCanonical) && currentCanonical.length >= 3) { return incoming } if (startsWithWords(currentCanonical, incomingCanonical) && incomingCanonical.length >= 3) { return current } const overlap = findWordOverlap(currentCanonical, incomingCanonical) if (overlap >= 4) { return normalizeWhitespace(`${current} ${incomingWords.slice(overlap).join(' ')}`) } return normalizeWhitespace(`${current} ${incoming}`) } function normalizeWhitespace(value: string) { return value.trim().replace(/\s+/g, ' ') } function splitWords(value: string) { return normalizeWhitespace(value).split(/\s+/).filter(Boolean) } function canonicalText(value: string) { return splitWords(value).map(canonicalWord).join(' ') } function canonicalWord(value: string) { return value.toLowerCase().replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '') } function startsWithWords(words: string[], prefix: string[]) { if (prefix.length > words.length) return false return prefix.every((word, index) => word === words[index]) } function findWordOverlap(currentWords: string[], incomingWords: string[]) { const max = Math.min(currentWords.length, incomingWords.length) for (let size = max; size > 0; size -= 1) { const currentSuffix = currentWords.slice(currentWords.length - size) const incomingPrefix = incomingWords.slice(0, size) if (currentSuffix.every((word, index) => word === incomingPrefix[index])) { return size } } return 0 } function wait(ms: number) { return new Promise((resolve) => { window.setTimeout(resolve, ms) }) } class RecordingStartError extends Error { constructor(message: string) { super(message) this.name = 'RecordingStartError' } } function getRecordingEnvironment() { return { href: window.location.href, origin: window.location.origin, isSecureContext: window.isSecureContext, hasMediaDevices: Boolean(navigator.mediaDevices), hasGetUserMedia: Boolean(navigator.mediaDevices?.getUserMedia), hasMediaRecorder: typeof MediaRecorder !== 'undefined', documentHasFocus: document.hasFocus(), isTopWindow: getIsTopWindow(), } } function getIsTopWindow() { try { return window.top === window.self } catch (error) { return `blocked: ${getErrorMessage(error)}` } } function chooseMediaRecorderMimeType() { if (typeof MediaRecorder === 'undefined' || typeof MediaRecorder.isTypeSupported !== 'function') { return '' } return MEDIA_RECORDER_MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)) ?? '' } function describeMicrophoneError(error: unknown) { if (error instanceof DOMException) { const details = `${error.name}: ${error.message || 'No browser message provided.'}` if (error.name === 'NotAllowedError' || error.name === 'SecurityError') { return `Microphone permission denied (${details})` } if (error.name === 'NotFoundError') { return `No microphone device was found (${details})` } if (error.name === 'NotReadableError') { return `Microphone is unavailable, possibly because another app is using it (${details})` } return `Microphone capture failed (${details})` } return `Microphone capture failed: ${getErrorMessage(error)}` } function getErrorMessage(error: unknown) { return error instanceof Error ? error.message : String(error) } function parseSocketMessage(data: unknown): TranscriptSocketMessage | null { if (typeof data !== 'string') return null try { const parsed = JSON.parse(data) as TranscriptSocketMessage if (parsed.type === 'transcript' || parsed.type === 'generation_status' || parsed.type === 'error') { return parsed } } catch { return null } return null }