"use client"; /** * useLiveConversation — full-duplex voice mode with barge-in. * * KI-044 (2026-05-14) — PCM pre-roll via AudioWorklet. * KI-057 (2026-05-15) — Noise-robust VAD + flush-on-stop. * KI-060 (2026-05-15) — Silence-end window lengthened (40 → 90 frames, * ~640 ms → ~1.5 s) so natural mid-sentence pauses don't auto-submit. * KI-159 (2026-05-15) — Early-close on stable silence. If the user has * already spoken ≥3× minUtteranceMs (~1.2 s) and silence has accumulated * to half the silenceEndFrames window (~1.5 s), close the segment NOW. * Prevents notification dings / transient background noise mid-pause * from re-triggering `speechLike`, zeroing the silence counter, and * extending the segment until either the full 3 s window or the 18 s * max-cap fires — by which point the real words are buried in a bloated * blob that Sarvam STT either drops or mis-transcribes. * * Why KI-057 was needed * -------------------------------------------------------------------- * Real user feedback after KI-044 shipped: "Background noise continues * to play a big issue, even random noise without people speaking or * just ambient noise keeps the live listening on, and it keeps * processing on something and not moving on. Also, if an entire series * of things have been said and I turn off the live chat, should that * not auto submit?" * * Two failure modes: * 1. Static `rmsThreshold: 18` triggered on HVAC / fan / traffic. * Once triggered, `silenceEndFrames: 40` (~640 ms of silence) * never accumulated because ambient noise kept the meter above * threshold — segment never closed, "Hearing you…" stuck on. * 2. Toggling Live OFF mid-utterance ran tearDown() which silently * dropped `speechBufferRef` — user's words were lost. * * Fixes layered in (defaults — overridable via opts): * A. Adaptive noise floor. While not recording, EMA the ambient * energy. Effective threshold = max(noise_floor * 2 + 4, * cfg.rmsThreshold). HVAC keeps the bar high. * B. Voice-band spectral gate. Require ≥35% of FFT energy to live * in bins 2-22 (~190-2150 Hz at 48 kHz) — the voiced-speech band. * Broadband noise fails this even when loud. * C. `speechStartFrames: 3` (~48 ms) so a single click/clack * doesn't open a segment. * D. Hard cap: `maxUtteranceMs: 18 s`. If a segment runs that long * without silence-end firing, force-close it. Prevents the * "noise pinned the meter open forever" state. * E. Post-utterance cooldown (700 ms). After we close + dispatch a * segment, suppress new triggers — even with echoCancellation, * the bot's TTS attack transient sometimes bleeds in. * F. Flush-on-teardown. If the user toggles Live OFF while a * capture is in progress and the duration meets minUtteranceMs, * encode + fire `onUtterance` once before tearing down — so * whatever they were saying gets submitted. * * KI-044 still applies — see below. * * Current implementation: * - Single getUserMedia stream + AudioContext stay open while Live is on. * - An AudioWorkletNode taps the raw PCM from the source — every render * quantum (128 samples) is posted back to the main thread as Float32. * - The main thread keeps a circular preroll buffer (~300 ms / 4800 * samples at 16 kHz) when no utterance is in progress. * - When VAD fires speech-start, the preroll is snapshotted into the * active utterance buffer and subsequent samples are appended. * - When VAD fires silence-end, we encode the full utterance (preroll + * speech + small post-roll) as a 16-bit PCM WAV (Sarvam Saarika's * native format) and post it to `onUtterance`. * - VAD itself still runs off the AnalyserNode (separate path) so its * sensitivity tuning is independent from the PCM capture rate. * * Result: the user's first phoneme is in the blob. No more "ello". * * Push-to-talk path (page.tsx::startRecording) is unaffected — PTT * recording starts when the user clicks, the input is already primed. */ import { useCallback, useEffect, useRef, useState } from "react"; export type LiveConversationOptions = { onUtterance: (blob: Blob, abort: AbortController) => Promise; onSpeechStart?: () => void; onSpeechEnd?: () => void; rmsThreshold?: number; speechStartFrames?: number; silenceEndFrames?: number; // KI-165 (2026-05-15) — caller-owned signal indicating a typed-text chat // request is currently in flight. When true, voice captures that close // during this window are silently discarded (no /api/transcribe call, // no UI mutation). Prevents the "text typed → background notification // dings → empty voice capture clobbers the typed-text response" UX bug. // Caller flips this ref true at the start of its text send() and false // in the finally; the voice hook reads it inside endSpeechCapture. isTextRequestPendingRef?: React.MutableRefObject; }; export type LiveConversationState = { live: boolean; recording: boolean; micPermissionDenied: boolean; setLive: (v: boolean) => void; inflightAbortRef: React.MutableRefObject; }; const DEFAULTS = { // KI-113 raised from 18 → 26 to reject ambient noise. // KI-139 (2026-05-15) — backed off to 18 because voice-forensics agent // proved 26 sat ABOVE typical speech avg on consumer mics (especially // built-ins with active noise gate that pin noiseFloor to 0). VAD never // opened → green pill rendered → zero audio posted. rmsThreshold: 18, // KI-113 — raised 3 → 5 (~80 ms sustained). Single clicks / cutlery / // typing transients no longer flip the gate. Preroll buffer (KI-044) // still captures the first phoneme via the 300 ms look-back. speechStartFrames: 5, // KI-060/064/115 (2026-05-15) — silence-end window tuning. // v1 (KI-057): 40 (~640 ms) — too tight; users said pause→submit. // v2 (KI-060): 90 (~1.5 s) — still cut "Hi, I'm looking to buy a // new insurance ..." before "policy". // v3 (KI-064): 120 (~2 s) — covers a normal thinking pause between // phrases. // v4 (KI-115): 180 (~3 s) — user reported the 2s window still cut // mid-thought pauses ("um", "let me think", etc.). 3 s is the // pause length where most speakers genuinely consider the // utterance complete. Trade: +1 s tail latency before bot // responds, accepted to kill the "submit on pause" UX bug. silenceEndFrames: 180, minUtteranceMs: 400, // KI-044 — How much pre-trigger PCM we keep in the rolling buffer. // 300 ms is generous; covers the ~80 ms VAD latency + ~100 ms of // user onset before the first detectable frame, with margin. prerollMs: 300, // KI-057 — hard cap. If silence-end never fires (e.g. continuous // ambient noise pinned the meter open), force-close the segment. maxUtteranceMs: 18000, // KI-057 — suppress new triggers for this long after we close a // segment. Avoids bot's TTS attack transient bleeding through even // with echoCancellation on. postUtteranceCooldownMs: 700, // KI-113 raised to 0.50, KI-134 backed off to 0.35. // KI-139 (2026-05-15) — voice-forensics agent measured live bundle on // user's actual hardware: voiceProp sits at 0.25–0.33 on quiet laptop // mics with NS enabled. 0.35 still gates the user out. 0.20 puts the // floor well below voiced-speech minimum and only rejects pure tones // (constant whine of HVAC, traffic). This is the deepest pushback // — if HVAC noise creeps in, KI-140 will add a /api/transcribe round // trip that detects empty responses and surfaces "couldn't hear you". voiceBandMinProp: 0.20, // KI-165 (2026-05-15) — minimum genuinely-voiced frames required before // we'll submit a captured segment. A frame ≈ 1 raf tick (~16 ms). 8 frames // ≈ 130 ms of audio that actually cleared the voice-band + threshold gate. // Anything shorter is almost certainly a notification ding / cough / chair // creak that briefly cleared `speechLike` for the speechStartFrames burst // and then died — we must not POST that to /api/transcribe + clobber the // chat pane. minVoicedFrames: 8, }; // AudioWorklet processor source — inlined as a Blob URL so we don't need // a separate static asset route. Runs on the audio thread; posts each // 128-sample mono Float32Array back to the main thread. const WORKLET_SOURCE = ` class PCMCaptureProcessor extends AudioWorkletProcessor { process(inputs) { const input = inputs[0]; if (input && input[0]) { // Clone the buffer so it survives the transfer; the original is // a view onto the audio thread's internal buffer. this.port.postMessage(input[0].slice(0)); } return true; } } registerProcessor('pcm-capture', PCMCaptureProcessor); `; // Encode Float32 samples as a 16-bit PCM WAV file (mono). Returns a Blob // suitable for `` upload to /api/transcribe. function encodeWAV(samples: Float32Array, sampleRate: number): Blob { const headerSize = 44; const dataSize = samples.length * 2; // 16-bit const buffer = new ArrayBuffer(headerSize + dataSize); const view = new DataView(buffer); const writeString = (offset: number, str: string) => { for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i)); }; writeString(0, "RIFF"); view.setUint32(4, 36 + dataSize, true); writeString(8, "WAVE"); writeString(12, "fmt "); view.setUint32(16, 16, true); // PCM chunk size view.setUint16(20, 1, true); // PCM format view.setUint16(22, 1, true); // mono view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 2, true); // byte rate view.setUint16(32, 2, true); // block align view.setUint16(34, 16, true); // bits per sample writeString(36, "data"); view.setUint32(40, dataSize, true); let offset = headerSize; for (let i = 0; i < samples.length; i++, offset += 2) { const s = Math.max(-1, Math.min(1, samples[i])); view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); } return new Blob([buffer], { type: "audio/wav" }); } export function useLiveConversation(opts: LiveConversationOptions): LiveConversationState { const [live, setLive] = useState(false); const [recording, setRecording] = useState(false); const [micPermissionDenied, setMicPermissionDenied] = useState(false); const streamRef = useRef(null); const audioCtxRef = useRef(null); const analyserRef = useRef(null); const sourceRef = useRef(null); const workletRef = useRef(null); const workletUrlRef = useRef(null); const sampleRateRef = useRef(48000); // KI-044 — sample-level capture buffers const prerollRef = useRef([]); const speechBufferRef = useRef([]); const recordingRef = useRef(false); const rafIdRef = useRef(null); const inflightAbortRef = useRef(null); const recStartTsRef = useRef(0); // KI-165 (2026-05-15) — count VAD frames that genuinely cleared the // voice-band + threshold gate while a capture is in progress. Used by // endSpeechCapture / flush-on-stop to discard captures that opened on a // notification ding / cough but never accumulated real speech. Reset on // every beginSpeechCapture so each segment is judged on its own merits. const voicedFramesRef = useRef(0); // KI-165 (2026-05-15) — caller-owned flag indicating a typed-text chat // request is currently awaiting its response. Voice captures closed // during this window are discarded silently. const isTextRequestPendingRef = opts.isTextRequestPendingRef; // KI-057 — adaptive noise floor (EMA of ambient avg while idle). const noiseFloorRef = useRef(0); // KI-057 — gates "did the bot just stop talking?" cooldown. const lastUtteranceEndedAtRef = useRef(0); // KI-141 (2026-05-15) — TTS-playback awareness for reliable barge-in. // When the bot's