Spaces:
Sleeping
Sleeping
| import { useCallback, useEffect, useMemo, useRef, useState } from 'react' | |
| import { createTranscriptionSocket, getTranscriptionSocketUrl, type TranscriptSocketMessage } from '../lib/websocket' | |
| type VoiceSessionState = 'idle' | 'connecting' | 'recording' | 'stopping' | 'error' | |
| type UseVoiceSessionOptions = { | |
| onFallback?: () => void | |
| onRecordingStarted?: () => void | |
| } | |
| const CHUNK_INTERVAL_MS = 2500 | |
| const VOICE_DETECTED_THRESHOLD = 0.08 | |
| const MEDIA_RECORDER_MIME_TYPES = [ | |
| 'audio/webm;codecs=opus', | |
| 'audio/webm', | |
| 'audio/mp4', | |
| ] | |
| export function useVoiceSession({ onFallback, onRecordingStarted }: UseVoiceSessionOptions) { | |
| const socketRef = useRef<WebSocket | null>(null) | |
| const recorderRef = useRef<MediaRecorder | null>(null) | |
| const streamRef = useRef<MediaStream | null>(null) | |
| const audioContextRef = useRef<AudioContext | null>(null) | |
| const analyserRef = useRef<AnalyserNode | null>(null) | |
| const levelRafRef = useRef(0) | |
| const sessionIdRef = useRef(0) | |
| const transcriptRef = useRef('') | |
| const voiceDetectedLoggedRef = useRef(false) | |
| const lastAudioLevelLogAtRef = useRef(0) | |
| const [state, setState] = useState<VoiceSessionState>('idle') | |
| const [transcript, setTranscript] = useState('') | |
| const [audioLevel, setAudioLevel] = useState(0) | |
| const [hasDetectedVoice, setHasDetectedVoice] = useState(false) | |
| const [isTranscribingChunk, setIsTranscribingChunk] = useState(false) | |
| const [lastVoiceDetectedAt, setLastVoiceDetectedAt] = useState<number | null>(null) | |
| const [lastTranscriptReceivedAt, setLastTranscriptReceivedAt] = useState<number | null>(null) | |
| const [generationStatus, setGenerationStatus] = useState<string | null>(null) | |
| const [errorMessage, setErrorMessage] = useState<string | null>(null) | |
| const cleanup = useCallback(() => { | |
| window.cancelAnimationFrame(levelRafRef.current) | |
| levelRafRef.current = 0 | |
| analyserRef.current = null | |
| audioContextRef.current?.close().catch(() => undefined) | |
| audioContextRef.current = null | |
| recorderRef.current = null | |
| socketRef.current?.close() | |
| socketRef.current = null | |
| streamRef.current?.getTracks().forEach((track) => track.stop()) | |
| streamRef.current = null | |
| setAudioLevel(0) | |
| setIsTranscribingChunk(false) | |
| }, []) | |
| const reset = useCallback(() => { | |
| cleanup() | |
| setState('idle') | |
| sessionIdRef.current += 1 | |
| transcriptRef.current = '' | |
| setTranscript('') | |
| setAudioLevel(0) | |
| setHasDetectedVoice(false) | |
| voiceDetectedLoggedRef.current = false | |
| lastAudioLevelLogAtRef.current = 0 | |
| setIsTranscribingChunk(false) | |
| setLastVoiceDetectedAt(null) | |
| setLastTranscriptReceivedAt(null) | |
| setGenerationStatus(null) | |
| setErrorMessage(null) | |
| }, [cleanup]) | |
| const stop = useCallback(async (finalizeMs = 1600) => { | |
| setState('stopping') | |
| const recorder = recorderRef.current | |
| if (recorder && recorder.state !== 'inactive') { | |
| recorder.requestData() | |
| recorder.stop() | |
| } | |
| streamRef.current?.getTracks().forEach((track) => track.stop()) | |
| streamRef.current = null | |
| window.cancelAnimationFrame(levelRafRef.current) | |
| levelRafRef.current = 0 | |
| setAudioLevel(0) | |
| console.debug('stop recording') | |
| await wait(finalizeMs) | |
| const finalTranscript = transcriptRef.current | |
| sessionIdRef.current += 1 | |
| cleanup() | |
| setState('idle') | |
| return finalTranscript | |
| }, [cleanup]) | |
| const startAudioLevelDetection = useCallback(async (stream: MediaStream, sessionId: number) => { | |
| const audioWindow = window as Window & { | |
| webkitAudioContext?: typeof AudioContext | |
| } | |
| const AudioContextClass = window.AudioContext || audioWindow.webkitAudioContext | |
| if (!AudioContextClass) { | |
| console.info('REAL_AUDIO_CONTEXT_UNAVAILABLE') | |
| return | |
| } | |
| const audioContext = new AudioContextClass() | |
| const analyser = audioContext.createAnalyser() | |
| analyser.fftSize = 1024 | |
| audioContext.createMediaStreamSource(stream).connect(analyser) | |
| audioContextRef.current = audioContext | |
| analyserRef.current = analyser | |
| console.info('REAL_AUDIO_CONTEXT_STATE', { state: audioContext.state }) | |
| if (audioContext.state === 'suspended') { | |
| try { | |
| await audioContext.resume() | |
| console.info('REAL_AUDIO_CONTEXT_RESUMED', { state: audioContext.state }) | |
| } catch (error) { | |
| console.info('REAL_AUDIO_CONTEXT_RESUME_FAILED', getErrorMessage(error)) | |
| } | |
| } | |
| const samples = new Uint8Array(analyser.fftSize) | |
| const tick = () => { | |
| if (sessionIdRef.current !== sessionId || !analyserRef.current) return | |
| analyserRef.current.getByteTimeDomainData(samples) | |
| let sum = 0 | |
| for (let index = 0; index < samples.length; index += 1) { | |
| const normalized = (samples[index] - 128) / 128 | |
| sum += normalized * normalized | |
| } | |
| const rms = Math.sqrt(sum / samples.length) | |
| const level = Math.min(1, Math.max(0, rms * 4)) | |
| setAudioLevel(level) | |
| const now = Date.now() | |
| if (now - lastAudioLevelLogAtRef.current > 1000) { | |
| lastAudioLevelLogAtRef.current = now | |
| console.info('REAL_AUDIO_LEVEL_SAMPLE', { | |
| audioContextState: audioContext.state, | |
| level, | |
| rms, | |
| }) | |
| } | |
| if (level >= VOICE_DETECTED_THRESHOLD) { | |
| if (!voiceDetectedLoggedRef.current) { | |
| console.debug('voice detected threshold reached') | |
| voiceDetectedLoggedRef.current = true | |
| } | |
| setHasDetectedVoice(true) | |
| setLastVoiceDetectedAt(Date.now()) | |
| } | |
| levelRafRef.current = window.requestAnimationFrame(tick) | |
| } | |
| tick() | |
| }, []) | |
| const start = useCallback(async () => { | |
| console.info('REAL_RECORDING_ENV', getRecordingEnvironment()) | |
| if (state === 'connecting' || state === 'recording') { | |
| console.info('REAL_RECORDING_START_SKIPPED', { state }) | |
| return false | |
| } | |
| if (!navigator.mediaDevices?.getUserMedia) { | |
| const message = 'Microphone capture is not available in this browser or context.' | |
| console.info('REAL_RECORDING_ERROR', message) | |
| console.error('[recording] microphone capture is not available in this browser or context') | |
| setErrorMessage(message) | |
| setState('error') | |
| return false | |
| } | |
| if (typeof MediaRecorder === 'undefined') { | |
| const message = 'MediaRecorder is not supported in this browser.' | |
| console.info('REAL_RECORDING_ERROR', message) | |
| console.error('[recording] MediaRecorder is not supported in this browser') | |
| setErrorMessage(message) | |
| setState('error') | |
| return false | |
| } | |
| setState('connecting') | |
| transcriptRef.current = '' | |
| setTranscript('') | |
| setAudioLevel(0) | |
| setHasDetectedVoice(false) | |
| voiceDetectedLoggedRef.current = false | |
| setIsTranscribingChunk(false) | |
| setLastVoiceDetectedAt(null) | |
| setLastTranscriptReceivedAt(null) | |
| setGenerationStatus(null) | |
| setErrorMessage(null) | |
| const sessionId = sessionIdRef.current + 1 | |
| sessionIdRef.current = sessionId | |
| try { | |
| let stream: MediaStream | |
| try { | |
| console.info('REAL_MIC_REQUEST_START') | |
| stream = await navigator.mediaDevices.getUserMedia({ audio: true }) | |
| } catch (error) { | |
| const message = describeMicrophoneError(error) | |
| console.info('REAL_RECORDING_ERROR', message) | |
| console.error(`[recording] ${message}`, error) | |
| throw new RecordingStartError(message) | |
| } | |
| console.info('REAL_MIC_STREAM_ACQUIRED', { | |
| audioTracks: stream.getAudioTracks().map((track) => ({ | |
| enabled: track.enabled, | |
| id: track.id, | |
| label: track.label, | |
| muted: track.muted, | |
| readyState: track.readyState, | |
| settings: track.getSettings(), | |
| })), | |
| }) | |
| stream.getAudioTracks().forEach((track) => { | |
| track.addEventListener('mute', () => { | |
| console.info('REAL_AUDIO_TRACK_MUTED', { | |
| id: track.id, | |
| label: track.label, | |
| readyState: track.readyState, | |
| }) | |
| }) | |
| track.addEventListener('unmute', () => { | |
| console.info('REAL_AUDIO_TRACK_UNMUTED', { | |
| id: track.id, | |
| label: track.label, | |
| readyState: track.readyState, | |
| }) | |
| }) | |
| track.addEventListener('ended', () => { | |
| console.info('REAL_AUDIO_TRACK_ENDED', { | |
| id: track.id, | |
| label: track.label, | |
| readyState: track.readyState, | |
| }) | |
| }) | |
| }) | |
| console.debug('microphone stream acquired') | |
| const socketUrl = getTranscriptionSocketUrl() | |
| console.info('REAL_WS_CONNECTING', { url: socketUrl }) | |
| console.debug(`connecting transcription WebSocket: ${socketUrl}`) | |
| const socket = createTranscriptionSocket() | |
| streamRef.current = stream | |
| socketRef.current = socket | |
| await startAudioLevelDetection(stream, sessionId) | |
| await new Promise<void>((resolve, reject) => { | |
| let didOpen = false | |
| const cleanupStartupListeners = () => { | |
| socket.removeEventListener('open', handleOpen) | |
| socket.removeEventListener('error', handleErrorBeforeOpen) | |
| socket.removeEventListener('close', handleCloseBeforeOpen) | |
| } | |
| const handleOpen = () => { | |
| didOpen = true | |
| cleanupStartupListeners() | |
| console.info('REAL_WS_CONNECTED', { url: socketUrl }) | |
| resolve() | |
| } | |
| const handleErrorBeforeOpen = (event: Event) => { | |
| if (didOpen) return | |
| cleanupStartupListeners() | |
| const message = `Transcription WebSocket unavailable at ${socketUrl}` | |
| console.info('REAL_RECORDING_ERROR', message) | |
| console.error('[recording] WebSocket connection failed before open', { url: socketUrl, event }) | |
| reject(new RecordingStartError(message)) | |
| } | |
| const handleCloseBeforeOpen = (event: CloseEvent) => { | |
| if (didOpen) return | |
| cleanupStartupListeners() | |
| const message = event.reason | |
| ? `Transcription WebSocket closed before recording started: ${event.reason}` | |
| : `Transcription WebSocket unavailable at ${socketUrl}` | |
| console.info('REAL_WS_CLOSED_BEFORE_OPEN', { | |
| code: event.code, | |
| reason: event.reason, | |
| wasClean: event.wasClean, | |
| url: socketUrl, | |
| }) | |
| console.info('REAL_RECORDING_ERROR', message) | |
| console.error('[recording] WebSocket closed before open', { url: socketUrl, event }) | |
| reject(new RecordingStartError(message)) | |
| } | |
| socket.addEventListener('open', handleOpen) | |
| socket.addEventListener('error', handleErrorBeforeOpen) | |
| socket.addEventListener('close', handleCloseBeforeOpen) | |
| }) | |
| socket.addEventListener('close', (event) => { | |
| console.info('REAL_WS_CLOSED_NORMAL', { | |
| code: event.code, | |
| reason: event.reason, | |
| wasClean: event.wasClean, | |
| url: socketUrl, | |
| }) | |
| }) | |
| socket.addEventListener('message', (event) => { | |
| if (sessionIdRef.current !== sessionId) return | |
| const data = parseSocketMessage(event.data) | |
| if (!data) return | |
| if (data.type === 'error') { | |
| const message = data.message ?? data.detail ?? 'Backend reported an unknown transcription error.' | |
| console.error(`[recording] backend error message: ${message}`) | |
| setErrorMessage(message) | |
| setState('error') | |
| return | |
| } | |
| if (data.type === 'transcript' && data.text.trim()) { | |
| console.info('REAL_TRANSCRIPT_RECEIVED', { | |
| chars: data.text.length, | |
| source: data.source, | |
| }) | |
| console.debug('transcript message received') | |
| setIsTranscribingChunk(false) | |
| setLastTranscriptReceivedAt(Date.now()) | |
| setTranscript((current) => { | |
| const merged = mergeTranscriptSegment(current, data.text) | |
| transcriptRef.current = merged | |
| console.debug('transcript state updated') | |
| return merged | |
| }) | |
| } | |
| if (data.type === 'generation_status') { | |
| setGenerationStatus(data.status) | |
| } | |
| }) | |
| const mimeType = chooseMediaRecorderMimeType() | |
| console.info('REAL_MEDIA_RECORDER_CREATED', { mimeType: mimeType || '<browser-default>' }) | |
| const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined) | |
| recorderRef.current = recorder | |
| recorder.addEventListener('dataavailable', (event) => { | |
| console.info('REAL_AUDIO_CHUNK_AVAILABLE', { | |
| size: event.data.size, | |
| type: event.data.type, | |
| socketReadyState: socket.readyState, | |
| }) | |
| if (event.data.size > 0 && socket.readyState === WebSocket.OPEN) { | |
| console.info('REAL_AUDIO_CHUNK_SENT', { size: event.data.size }) | |
| console.debug('audio chunk sent') | |
| setIsTranscribingChunk(true) | |
| socket.send(event.data) | |
| } | |
| }) | |
| recorder.start(CHUNK_INTERVAL_MS) | |
| console.info('REAL_MEDIA_RECORDER_STARTED', { | |
| intervalMs: CHUNK_INTERVAL_MS, | |
| mimeType: recorder.mimeType || '<browser-default>', | |
| state: recorder.state, | |
| }) | |
| setState('recording') | |
| onRecordingStarted?.() | |
| return true | |
| } catch (error) { | |
| const message = error instanceof RecordingStartError | |
| ? error.message | |
| : `Recording failed: ${getErrorMessage(error)}` | |
| console.info('REAL_RECORDING_ERROR', message) | |
| console.error('[recording] start failed', error) | |
| setErrorMessage(message) | |
| cleanup() | |
| setState('error') | |
| return false | |
| } | |
| }, [cleanup, onFallback, onRecordingStarted, startAudioLevelDetection, state]) | |
| useEffect( | |
| () => () => { | |
| sessionIdRef.current += 1 | |
| cleanup() | |
| }, | |
| [cleanup], | |
| ) | |
| const transcriptWords = useMemo( | |
| () => transcript.split(/\s+/).filter(Boolean), | |
| [transcript], | |
| ) | |
| const transcriptText = useMemo(() => transcript.trim(), [transcript]) | |
| return { | |
| generationStatus, | |
| errorMessage, | |
| audioLevel, | |
| hasDetectedVoice, | |
| isTranscribingChunk, | |
| isRecording: state === 'recording', | |
| isStarting: state === 'connecting', | |
| lastTranscriptReceivedAt, | |
| lastVoiceDetectedAt, | |
| state, | |
| transcriptText, | |
| transcriptWords, | |
| hasTranscript: transcriptWords.length > 0, | |
| reset, | |
| start, | |
| stop, | |
| } | |
| } | |
| export function mergeTranscriptSegment(currentTranscript: string, newSegment: string) { | |
| const current = normalizeWhitespace(currentTranscript) | |
| const incoming = normalizeWhitespace(newSegment) | |
| if (!incoming) return current | |
| if (!current) return incoming | |
| if (canonicalText(current) === canonicalText(incoming)) return current | |
| const currentWords = splitWords(current) | |
| const incomingWords = splitWords(incoming) | |
| const currentCanonical = currentWords.map(canonicalWord) | |
| const incomingCanonical = incomingWords.map(canonicalWord) | |
| if (startsWithWords(incomingCanonical, currentCanonical) && currentCanonical.length >= 3) { | |
| return incoming | |
| } | |
| if (startsWithWords(currentCanonical, incomingCanonical) && incomingCanonical.length >= 3) { | |
| return current | |
| } | |
| const overlap = findWordOverlap(currentCanonical, incomingCanonical) | |
| if (overlap >= 4) { | |
| return normalizeWhitespace(`${current} ${incomingWords.slice(overlap).join(' ')}`) | |
| } | |
| return normalizeWhitespace(`${current} ${incoming}`) | |
| } | |
| function normalizeWhitespace(value: string) { | |
| return value.trim().replace(/\s+/g, ' ') | |
| } | |
| function splitWords(value: string) { | |
| return normalizeWhitespace(value).split(/\s+/).filter(Boolean) | |
| } | |
| function canonicalText(value: string) { | |
| return splitWords(value).map(canonicalWord).join(' ') | |
| } | |
| function canonicalWord(value: string) { | |
| return value.toLowerCase().replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '') | |
| } | |
| function startsWithWords(words: string[], prefix: string[]) { | |
| if (prefix.length > words.length) return false | |
| return prefix.every((word, index) => word === words[index]) | |
| } | |
| function findWordOverlap(currentWords: string[], incomingWords: string[]) { | |
| const max = Math.min(currentWords.length, incomingWords.length) | |
| for (let size = max; size > 0; size -= 1) { | |
| const currentSuffix = currentWords.slice(currentWords.length - size) | |
| const incomingPrefix = incomingWords.slice(0, size) | |
| if (currentSuffix.every((word, index) => word === incomingPrefix[index])) { | |
| return size | |
| } | |
| } | |
| return 0 | |
| } | |
| function wait(ms: number) { | |
| return new Promise<void>((resolve) => { | |
| window.setTimeout(resolve, ms) | |
| }) | |
| } | |
| class RecordingStartError extends Error { | |
| constructor(message: string) { | |
| super(message) | |
| this.name = 'RecordingStartError' | |
| } | |
| } | |
| function getRecordingEnvironment() { | |
| return { | |
| href: window.location.href, | |
| origin: window.location.origin, | |
| isSecureContext: window.isSecureContext, | |
| hasMediaDevices: Boolean(navigator.mediaDevices), | |
| hasGetUserMedia: Boolean(navigator.mediaDevices?.getUserMedia), | |
| hasMediaRecorder: typeof MediaRecorder !== 'undefined', | |
| documentHasFocus: document.hasFocus(), | |
| isTopWindow: getIsTopWindow(), | |
| } | |
| } | |
| function getIsTopWindow() { | |
| try { | |
| return window.top === window.self | |
| } catch (error) { | |
| return `blocked: ${getErrorMessage(error)}` | |
| } | |
| } | |
| function chooseMediaRecorderMimeType() { | |
| if (typeof MediaRecorder === 'undefined' || typeof MediaRecorder.isTypeSupported !== 'function') { | |
| return '' | |
| } | |
| return MEDIA_RECORDER_MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)) ?? '' | |
| } | |
| function describeMicrophoneError(error: unknown) { | |
| if (error instanceof DOMException) { | |
| const details = `${error.name}: ${error.message || 'No browser message provided.'}` | |
| if (error.name === 'NotAllowedError' || error.name === 'SecurityError') { | |
| return `Microphone permission denied (${details})` | |
| } | |
| if (error.name === 'NotFoundError') { | |
| return `No microphone device was found (${details})` | |
| } | |
| if (error.name === 'NotReadableError') { | |
| return `Microphone is unavailable, possibly because another app is using it (${details})` | |
| } | |
| return `Microphone capture failed (${details})` | |
| } | |
| return `Microphone capture failed: ${getErrorMessage(error)}` | |
| } | |
| function getErrorMessage(error: unknown) { | |
| return error instanceof Error ? error.message : String(error) | |
| } | |
| function parseSocketMessage(data: unknown): TranscriptSocketMessage | null { | |
| if (typeof data !== 'string') return null | |
| try { | |
| const parsed = JSON.parse(data) as TranscriptSocketMessage | |
| if (parsed.type === 'transcript' || parsed.type === 'generation_status' || parsed.type === 'error') { | |
| return parsed | |
| } | |
| } catch { | |
| return null | |
| } | |
| return null | |
| } | |