InsuranceBot / frontend /src /lib /useLiveConversation.ts
rohitsar567's picture
fix(parser+admin+voice): KI-161 + KI-164 + KI-165 bundle
0a02616
Raw
History Blame
36.6 kB
"use client";
/**
* useLiveConversation β€” full-duplex voice mode with barge-in.
*
* KI-044 (2026-05-14) β€” PCM pre-roll via AudioWorklet.
* KI-057 (2026-05-15) β€” Noise-robust VAD + flush-on-stop.
* KI-060 (2026-05-15) β€” Silence-end window lengthened (40 β†’ 90 frames,
* ~640 ms β†’ ~1.5 s) so natural mid-sentence pauses don't auto-submit.
* KI-159 (2026-05-15) β€” Early-close on stable silence. If the user has
* already spoken β‰₯3Γ— minUtteranceMs (~1.2 s) and silence has accumulated
* to half the silenceEndFrames window (~1.5 s), close the segment NOW.
* Prevents notification dings / transient background noise mid-pause
* from re-triggering `speechLike`, zeroing the silence counter, and
* extending the segment until either the full 3 s window or the 18 s
* max-cap fires β€” by which point the real words are buried in a bloated
* blob that Sarvam STT either drops or mis-transcribes.
*
* Why KI-057 was needed
* --------------------------------------------------------------------
* Real user feedback after KI-044 shipped: "Background noise continues
* to play a big issue, even random noise without people speaking or
* just ambient noise keeps the live listening on, and it keeps
* processing on something and not moving on. Also, if an entire series
* of things have been said and I turn off the live chat, should that
* not auto submit?"
*
* Two failure modes:
* 1. Static `rmsThreshold: 18` triggered on HVAC / fan / traffic.
* Once triggered, `silenceEndFrames: 40` (~640 ms of silence)
* never accumulated because ambient noise kept the meter above
* threshold β€” segment never closed, "Hearing you…" stuck on.
* 2. Toggling Live OFF mid-utterance ran tearDown() which silently
* dropped `speechBufferRef` β€” user's words were lost.
*
* Fixes layered in (defaults β€” overridable via opts):
* A. Adaptive noise floor. While not recording, EMA the ambient
* energy. Effective threshold = max(noise_floor * 2 + 4,
* cfg.rmsThreshold). HVAC keeps the bar high.
* B. Voice-band spectral gate. Require β‰₯35% of FFT energy to live
* in bins 2-22 (~190-2150 Hz at 48 kHz) β€” the voiced-speech band.
* Broadband noise fails this even when loud.
* C. `speechStartFrames: 3` (~48 ms) so a single click/clack
* doesn't open a segment.
* D. Hard cap: `maxUtteranceMs: 18 s`. If a segment runs that long
* without silence-end firing, force-close it. Prevents the
* "noise pinned the meter open forever" state.
* E. Post-utterance cooldown (700 ms). After we close + dispatch a
* segment, suppress new triggers β€” even with echoCancellation,
* the bot's TTS attack transient sometimes bleeds in.
* F. Flush-on-teardown. If the user toggles Live OFF while a
* capture is in progress and the duration meets minUtteranceMs,
* encode + fire `onUtterance` once before tearing down β€” so
* whatever they were saying gets submitted.
*
* KI-044 still applies β€” see below.
*
* Current implementation:
* - Single getUserMedia stream + AudioContext stay open while Live is on.
* - An AudioWorkletNode taps the raw PCM from the source β€” every render
* quantum (128 samples) is posted back to the main thread as Float32.
* - The main thread keeps a circular preroll buffer (~300 ms / 4800
* samples at 16 kHz) when no utterance is in progress.
* - When VAD fires speech-start, the preroll is snapshotted into the
* active utterance buffer and subsequent samples are appended.
* - When VAD fires silence-end, we encode the full utterance (preroll +
* speech + small post-roll) as a 16-bit PCM WAV (Sarvam Saarika's
* native format) and post it to `onUtterance`.
* - VAD itself still runs off the AnalyserNode (separate path) so its
* sensitivity tuning is independent from the PCM capture rate.
*
* Result: the user's first phoneme is in the blob. No more "ello".
*
* Push-to-talk path (page.tsx::startRecording) is unaffected β€” PTT
* recording starts when the user clicks, the input is already primed.
*/
import { useCallback, useEffect, useRef, useState } from "react";
export type LiveConversationOptions = {
onUtterance: (blob: Blob, abort: AbortController) => Promise<void>;
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
rmsThreshold?: number;
speechStartFrames?: number;
silenceEndFrames?: number;
// KI-165 (2026-05-15) β€” caller-owned signal indicating a typed-text chat
// request is currently in flight. When true, voice captures that close
// during this window are silently discarded (no /api/transcribe call,
// no UI mutation). Prevents the "text typed β†’ background notification
// dings β†’ empty voice capture clobbers the typed-text response" UX bug.
// Caller flips this ref true at the start of its text send() and false
// in the finally; the voice hook reads it inside endSpeechCapture.
isTextRequestPendingRef?: React.MutableRefObject<boolean>;
};
export type LiveConversationState = {
live: boolean;
recording: boolean;
micPermissionDenied: boolean;
setLive: (v: boolean) => void;
inflightAbortRef: React.MutableRefObject<AbortController | null>;
};
const DEFAULTS = {
// KI-113 raised from 18 β†’ 26 to reject ambient noise.
// KI-139 (2026-05-15) β€” backed off to 18 because voice-forensics agent
// proved 26 sat ABOVE typical speech avg on consumer mics (especially
// built-ins with active noise gate that pin noiseFloor to 0). VAD never
// opened β†’ green pill rendered β†’ zero audio posted.
rmsThreshold: 18,
// KI-113 β€” raised 3 β†’ 5 (~80 ms sustained). Single clicks / cutlery /
// typing transients no longer flip the gate. Preroll buffer (KI-044)
// still captures the first phoneme via the 300 ms look-back.
speechStartFrames: 5,
// KI-060/064/115 (2026-05-15) β€” silence-end window tuning.
// v1 (KI-057): 40 (~640 ms) — too tight; users said pause→submit.
// v2 (KI-060): 90 (~1.5 s) β€” still cut "Hi, I'm looking to buy a
// new insurance ..." before "policy".
// v3 (KI-064): 120 (~2 s) β€” covers a normal thinking pause between
// phrases.
// v4 (KI-115): 180 (~3 s) β€” user reported the 2s window still cut
// mid-thought pauses ("um", "let me think", etc.). 3 s is the
// pause length where most speakers genuinely consider the
// utterance complete. Trade: +1 s tail latency before bot
// responds, accepted to kill the "submit on pause" UX bug.
silenceEndFrames: 180,
minUtteranceMs: 400,
// KI-044 β€” How much pre-trigger PCM we keep in the rolling buffer.
// 300 ms is generous; covers the ~80 ms VAD latency + ~100 ms of
// user onset before the first detectable frame, with margin.
prerollMs: 300,
// KI-057 β€” hard cap. If silence-end never fires (e.g. continuous
// ambient noise pinned the meter open), force-close the segment.
maxUtteranceMs: 18000,
// KI-057 β€” suppress new triggers for this long after we close a
// segment. Avoids bot's TTS attack transient bleeding through even
// with echoCancellation on.
postUtteranceCooldownMs: 700,
// KI-113 raised to 0.50, KI-134 backed off to 0.35.
// KI-139 (2026-05-15) β€” voice-forensics agent measured live bundle on
// user's actual hardware: voiceProp sits at 0.25–0.33 on quiet laptop
// mics with NS enabled. 0.35 still gates the user out. 0.20 puts the
// floor well below voiced-speech minimum and only rejects pure tones
// (constant whine of HVAC, traffic). This is the deepest pushback
// β€” if HVAC noise creeps in, KI-140 will add a /api/transcribe round
// trip that detects empty responses and surfaces "couldn't hear you".
voiceBandMinProp: 0.20,
// KI-165 (2026-05-15) β€” minimum genuinely-voiced frames required before
// we'll submit a captured segment. A frame β‰ˆ 1 raf tick (~16 ms). 8 frames
// β‰ˆ 130 ms of audio that actually cleared the voice-band + threshold gate.
// Anything shorter is almost certainly a notification ding / cough / chair
// creak that briefly cleared `speechLike` for the speechStartFrames burst
// and then died β€” we must not POST that to /api/transcribe + clobber the
// chat pane.
minVoicedFrames: 8,
};
// AudioWorklet processor source β€” inlined as a Blob URL so we don't need
// a separate static asset route. Runs on the audio thread; posts each
// 128-sample mono Float32Array back to the main thread.
const WORKLET_SOURCE = `
class PCMCaptureProcessor extends AudioWorkletProcessor {
process(inputs) {
const input = inputs[0];
if (input && input[0]) {
// Clone the buffer so it survives the transfer; the original is
// a view onto the audio thread's internal buffer.
this.port.postMessage(input[0].slice(0));
}
return true;
}
}
registerProcessor('pcm-capture', PCMCaptureProcessor);
`;
// Encode Float32 samples as a 16-bit PCM WAV file (mono). Returns a Blob
// suitable for `<input type=file>` upload to /api/transcribe.
function encodeWAV(samples: Float32Array, sampleRate: number): Blob {
const headerSize = 44;
const dataSize = samples.length * 2; // 16-bit
const buffer = new ArrayBuffer(headerSize + dataSize);
const view = new DataView(buffer);
const writeString = (offset: number, str: string) => {
for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
};
writeString(0, "RIFF");
view.setUint32(4, 36 + dataSize, true);
writeString(8, "WAVE");
writeString(12, "fmt ");
view.setUint32(16, 16, true); // PCM chunk size
view.setUint16(20, 1, true); // PCM format
view.setUint16(22, 1, true); // mono
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * 2, true); // byte rate
view.setUint16(32, 2, true); // block align
view.setUint16(34, 16, true); // bits per sample
writeString(36, "data");
view.setUint32(40, dataSize, true);
let offset = headerSize;
for (let i = 0; i < samples.length; i++, offset += 2) {
const s = Math.max(-1, Math.min(1, samples[i]));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
}
return new Blob([buffer], { type: "audio/wav" });
}
export function useLiveConversation(opts: LiveConversationOptions): LiveConversationState {
const [live, setLive] = useState(false);
const [recording, setRecording] = useState(false);
const [micPermissionDenied, setMicPermissionDenied] = useState(false);
const streamRef = useRef<MediaStream | null>(null);
const audioCtxRef = useRef<AudioContext | null>(null);
const analyserRef = useRef<AnalyserNode | null>(null);
const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
const workletRef = useRef<AudioWorkletNode | null>(null);
const workletUrlRef = useRef<string | null>(null);
const sampleRateRef = useRef<number>(48000);
// KI-044 β€” sample-level capture buffers
const prerollRef = useRef<Float32Array[]>([]);
const speechBufferRef = useRef<Float32Array[]>([]);
const recordingRef = useRef(false);
const rafIdRef = useRef<number | null>(null);
const inflightAbortRef = useRef<AbortController | null>(null);
const recStartTsRef = useRef<number>(0);
// KI-165 (2026-05-15) β€” count VAD frames that genuinely cleared the
// voice-band + threshold gate while a capture is in progress. Used by
// endSpeechCapture / flush-on-stop to discard captures that opened on a
// notification ding / cough but never accumulated real speech. Reset on
// every beginSpeechCapture so each segment is judged on its own merits.
const voicedFramesRef = useRef<number>(0);
// KI-165 (2026-05-15) β€” caller-owned flag indicating a typed-text chat
// request is currently awaiting its response. Voice captures closed
// during this window are discarded silently.
const isTextRequestPendingRef = opts.isTextRequestPendingRef;
// KI-057 β€” adaptive noise floor (EMA of ambient avg while idle).
const noiseFloorRef = useRef<number>(0);
// KI-057 β€” gates "did the bot just stop talking?" cooldown.
const lastUtteranceEndedAtRef = useRef<number>(0);
// KI-141 (2026-05-15) β€” TTS-playback awareness for reliable barge-in.
// When the bot's <audio> element is playing, the mic re-captures the
// speaker output (echoCancellation is imperfect). The noiseFloor EMA
// would otherwise learn the bot's voice level and pull effectiveThreshold
// up to bot-loudness, making user voice unable to clear the gate. We
// (a) freeze noise-floor learning, (b) bypass the post-utterance cooldown
// (cooldown only makes sense AFTER bot finishes), and (c) drop the
// speech-start frame count so barge-in fires in ~30 ms instead of ~80 ms.
const ttsPlayingRef = useRef<boolean>(false);
const ttsAudioElementsRef = useRef<Set<HTMLAudioElement>>(new Set());
const onUtteranceRef = useRef(opts.onUtterance);
const onSpeechStartRef = useRef(opts.onSpeechStart);
const onSpeechEndRef = useRef(opts.onSpeechEnd);
useEffect(() => {
onUtteranceRef.current = opts.onUtterance;
onSpeechStartRef.current = opts.onSpeechStart;
onSpeechEndRef.current = opts.onSpeechEnd;
}, [opts.onUtterance, opts.onSpeechStart, opts.onSpeechEnd]);
const cfg = {
rmsThreshold: opts.rmsThreshold ?? DEFAULTS.rmsThreshold,
speechStartFrames: opts.speechStartFrames ?? DEFAULTS.speechStartFrames,
silenceEndFrames: opts.silenceEndFrames ?? DEFAULTS.silenceEndFrames,
minUtteranceMs: DEFAULTS.minUtteranceMs,
prerollMs: DEFAULTS.prerollMs,
maxUtteranceMs: DEFAULTS.maxUtteranceMs,
postUtteranceCooldownMs: DEFAULTS.postUtteranceCooldownMs,
voiceBandMinProp: DEFAULTS.voiceBandMinProp,
};
const interruptBotAudio = useCallback(() => {
if (typeof document !== "undefined") {
document.querySelectorAll("audio").forEach((a) => {
try {
a.pause();
a.currentTime = a.duration || 0;
} catch {}
});
}
// KI-141 β€” clearing TTS-playing immediately on barge-in so the VAD
// resumes idle-mode noise-floor learning even before the `pause`
// event fires on the (now stopped) <audio> element.
ttsPlayingRef.current = false;
// KI-141 β€” anchor cooldown to barge-in moment so the next 700 ms
// suppresses any residual decay tail / echo from the just-paused TTS.
lastUtteranceEndedAtRef.current = Date.now();
}, []);
// KI-141 (2026-05-15) β€” TTS-playback observer.
// Watch every <audio> element in the document for play/pause/ended so
// the VAD knows when the bot is currently speaking. This is the signal
// that flips the VAD into "barge-in mode" (no cooldown, faster start,
// frozen noise floor). MutationObserver picks up new <audio> elements
// as Message components mount.
useEffect(() => {
if (!live || typeof document === "undefined") return;
const tracked = ttsAudioElementsRef.current;
const refreshPlayingState = () => {
let anyPlaying = false;
tracked.forEach((a) => {
if (!a.paused && !a.ended && a.currentTime > 0) anyPlaying = true;
});
ttsPlayingRef.current = anyPlaying;
};
const onPlay = () => {
ttsPlayingRef.current = true;
};
const onPauseOrEnded = () => {
refreshPlayingState();
// KI-141 β€” anchor the post-utterance cooldown to the moment TTS
// actually finished, not to when the user's previous segment closed.
// This is what the 700 ms cooldown was always meant to gate: the
// bot's tail decay / echo bleeding back into the mic.
if (!ttsPlayingRef.current) {
lastUtteranceEndedAtRef.current = Date.now();
}
};
const attach = (a: HTMLAudioElement) => {
if (tracked.has(a)) return;
tracked.add(a);
a.addEventListener("play", onPlay);
a.addEventListener("playing", onPlay);
a.addEventListener("pause", onPauseOrEnded);
a.addEventListener("ended", onPauseOrEnded);
a.addEventListener("emptied", onPauseOrEnded);
// If the element is already playing when we attach, capture that.
if (!a.paused && !a.ended) ttsPlayingRef.current = true;
};
const detach = (a: HTMLAudioElement) => {
a.removeEventListener("play", onPlay);
a.removeEventListener("playing", onPlay);
a.removeEventListener("pause", onPauseOrEnded);
a.removeEventListener("ended", onPauseOrEnded);
a.removeEventListener("emptied", onPauseOrEnded);
tracked.delete(a);
};
// Attach to anything already in the DOM.
document.querySelectorAll("audio").forEach((el) => attach(el as HTMLAudioElement));
const observer = new MutationObserver((mutations) => {
for (const m of mutations) {
m.addedNodes.forEach((n) => {
if (n instanceof HTMLAudioElement) attach(n);
else if (n instanceof Element) {
n.querySelectorAll("audio").forEach((el) => attach(el as HTMLAudioElement));
}
});
m.removedNodes.forEach((n) => {
if (n instanceof HTMLAudioElement) detach(n);
else if (n instanceof Element) {
n.querySelectorAll("audio").forEach((el) => detach(el as HTMLAudioElement));
}
});
}
refreshPlayingState();
});
observer.observe(document.body, { childList: true, subtree: true });
return () => {
observer.disconnect();
tracked.forEach((a) => detach(a));
tracked.clear();
ttsPlayingRef.current = false;
};
}, [live]);
// KI-044 β€” open speech capture: snapshot the preroll into speechBuffer,
// flag recording, fire callbacks. The PCM keeps flowing via the worklet
// port; we just toggle where it lands.
const beginSpeechCapture = useCallback(() => {
speechBufferRef.current = [...prerollRef.current];
prerollRef.current = [];
recordingRef.current = true;
recStartTsRef.current = Date.now();
// KI-165 β€” reset the voiced-frame counter for this segment. The frames
// that triggered speechStart (1 burst of speechStartFrames) are
// intentionally NOT pre-counted; we want endSpeechCapture's >= 8-frame
// floor to mean "8 frames of *sustained* voiced energy DURING capture",
// not "the trigger burst was long enough" β€” a notification ding can
// easily produce 5 frames of broadband energy that clears voiceBandMinProp.
voicedFramesRef.current = 0;
setRecording(true);
onSpeechStartRef.current?.();
}, []);
// KI-044 β€” close speech capture: encode WAV, run guards, fire onUtterance.
// KI-057 β€” also anchors the post-utterance cooldown.
const endSpeechCapture = useCallback(async () => {
if (!recordingRef.current) return;
recordingRef.current = false;
setRecording(false);
lastUtteranceEndedAtRef.current = Date.now();
const durationMs = Date.now() - (recStartTsRef.current || Date.now());
const chunks = speechBufferRef.current;
speechBufferRef.current = [];
const voicedFrames = voicedFramesRef.current;
voicedFramesRef.current = 0;
if (chunks.length === 0) return;
if (durationMs < cfg.minUtteranceMs) {
// eslint-disable-next-line no-console
console.debug("[live-mode] dropped short utterance", durationMs, "ms");
return;
}
// KI-165 (2026-05-15) β€” discard captures with too few genuinely-voiced
// frames. Notification dings / Mac camera screenshot clicks / chair
// creaks can briefly clear the voice-band threshold for the trigger
// burst (speechStartFrames) but never accumulate real speech. Without
// this guard, we POST a 1.5s WAV of mostly silence to /api/transcribe,
// get back an empty string, but still flap UI state (voicePhase,
// isProcessing) and β€” most damagingly β€” race with an in-flight typed
// text response.
if (voicedFrames < DEFAULTS.minVoicedFrames) {
// eslint-disable-next-line no-console
console.debug(
"[live-mode] discarded near-empty capture (KI-165)",
{ voicedFrames, minRequired: DEFAULTS.minVoicedFrames, durationMs },
);
return;
}
// KI-165 (2026-05-15) β€” if the user typed a message and that chat
// request is still in flight, the voice path silently discards this
// capture. Text wins; voice never touches chat state during a typed
// turn. Prevents the "type β†’ mac notif ding opens mic β†’ empty STT
// response clobbers the typed-text response" UX bug.
if (isTextRequestPendingRef?.current) {
// eslint-disable-next-line no-console
console.debug(
"[live-mode] discarded capture: text request in flight (KI-165)",
{ voicedFrames, durationMs },
);
return;
}
// Concatenate Float32Array chunks
let totalSamples = 0;
for (const c of chunks) totalSamples += c.length;
const merged = new Float32Array(totalSamples);
let offset = 0;
for (const c of chunks) {
merged.set(c, offset);
offset += c.length;
}
const wav = encodeWAV(merged, sampleRateRef.current);
if (wav.size < 3000) return; // floor (matches prior heuristic)
onSpeechEndRef.current?.();
const abort = new AbortController();
inflightAbortRef.current = abort;
try {
await onUtteranceRef.current(wav, abort);
} catch (e) {
const name = (e as { name?: string })?.name;
if (name !== "AbortError") {
// eslint-disable-next-line no-console
console.error("[live-mode] utterance handler failed:", e);
}
} finally {
if (inflightAbortRef.current === abort) {
inflightAbortRef.current = null;
}
}
}, [cfg.minUtteranceMs]);
// VAD loop β€” runs while `live` is true.
// KI-057 β€” adaptive threshold + voice-band gate + max-utterance cap.
const tickVAD = useCallback(() => {
if (!analyserRef.current) return;
const a = analyserRef.current;
const buf = new Uint8Array(a.frequencyBinCount);
let loud = 0;
let quiet = 0;
// Voice band: bins 2-22 at fftSize=512 cover ~190-2150 Hz at 48 kHz β€”
// where voiced speech lives. Capped at bin count for safety.
const voiceBandStart = 2;
const voiceBandEnd = Math.min(22, buf.length - 1);
const loop = () => {
if (!analyserRef.current) return;
a.getByteFrequencyData(buf);
let sum = 0;
let voiceSum = 0;
for (let i = 0; i < buf.length; i++) {
sum += buf[i];
if (i >= voiceBandStart && i <= voiceBandEnd) voiceSum += buf[i];
}
const avg = sum / buf.length;
const voiceProp = sum > 0 ? voiceSum / sum : 0;
// KI-057/114 β€” adaptive threshold. Floor at cfg.rmsThreshold so
// genuinely quiet rooms don't open the gate too low. KI-114 raised
// the multiplier 2.0β†’2.5 and the offset 4β†’6 so DISTANT speech
// (which sits just above ambient because high frequencies attenuate
// with distance) is rejected. Close-in speech still clears the gate
// because the speaker's directly-radiated energy is ~10-20Γ— ambient.
// KI-139 (2026-05-15) β€” noise-floor multiplier 2.5 β†’ 1.8. On built-in
// mics with active noise gate, noiseFloor EMAs to 0 β†’ effectiveThreshold
// pinned at max(rmsThreshold, 6). User's voice avg sits 18-22 then
// never crosses. 1.8 keeps headroom for HVAC (which pins at 5-7) but
// lets quiet voice through.
const effectiveThreshold = Math.max(
cfg.rmsThreshold,
noiseFloorRef.current * 1.8 + 6,
);
// KI-057 β€” suppress new triggers right after we closed a segment
// (bot's TTS onset can bleed in via the mic loopback).
// KI-141 β€” but DO NOT suppress during TTS playback itself; that's
// exactly when barge-in must work. Cooldown is only meaningful for
// the brief window after bot speech ends.
const ttsPlaying = ttsPlayingRef.current;
const cooldownActive =
!ttsPlaying &&
Date.now() - lastUtteranceEndedAtRef.current < cfg.postUtteranceCooldownMs;
// KI-141 β€” barge-in must be SNAPPY. During TTS, 2 frames (~30 ms) is
// enough to confirm voice and pause the bot; the longer 5-frame gate
// is only needed in idle mode where it rejects click/clack transients.
const startFrames = ttsPlaying ? 2 : cfg.speechStartFrames;
const speechLike =
avg > effectiveThreshold &&
voiceProp >= cfg.voiceBandMinProp &&
!cooldownActive;
if (speechLike) {
loud++;
quiet = 0;
if (loud >= startFrames && !recordingRef.current) {
// Barge in: kill bot audio + cancel in-flight chat + begin capture.
interruptBotAudio();
if (inflightAbortRef.current) {
try { inflightAbortRef.current.abort(); } catch {}
inflightAbortRef.current = null;
}
beginSpeechCapture();
}
// KI-165 (2026-05-15) β€” count voiced frames during capture. Used by
// endSpeechCapture to discard segments that opened on a transient
// (notification ding) but never accumulated real speech.
if (recordingRef.current) {
voicedFramesRef.current++;
}
} else {
quiet++;
loud = 0;
// KI-057 β€” only learn the noise floor while idle, so ongoing
// speech doesn't poison the EMA.
// KI-141 β€” also freeze noise-floor learning while TTS is playing.
// The bot's voice bleeding through the speakers would otherwise be
// EMA'd into the floor, pulling effectiveThreshold up to bot loudness
// β€” at which point the user's voice can't clear it. Holding the
// pre-TTS noise floor keeps the gate at room-ambient level where
// user speech reliably crosses.
if (!recordingRef.current && !ttsPlaying) {
noiseFloorRef.current =
noiseFloorRef.current === 0
? avg
: noiseFloorRef.current * 0.95 + avg * 0.05;
}
if (quiet === cfg.silenceEndFrames && recordingRef.current) {
void endSpeechCapture();
}
// KI-159 (2026-05-15) β€” early-close on stable silence after enough
// captured speech. Protects against transient noise bursts (e.g. a
// notification ding mid-pause) that would otherwise re-trigger
// speechLike, zero `quiet`, and extend the segment until either the
// full silenceEndFrames (180 β‰ˆ 3 s) accumulates AGAIN or the
// maxUtteranceMs (18 s) hard-cap fires β€” by which point the real
// words are buried in a bloated blob that Sarvam STT mis-transcribes
// or returns empty for.
//
// Trigger: half the silence window AND we already have 3Γ— the
// minUtteranceMs (~1.2 s) of captured speech. Submit the user's
// words IMMEDIATELY at the first stable pause, before any noise can
// contaminate the segment.
else if (
recordingRef.current &&
quiet >= Math.floor(cfg.silenceEndFrames / 2) &&
recStartTsRef.current > 0 &&
Date.now() - recStartTsRef.current >= cfg.minUtteranceMs * 3
) {
// eslint-disable-next-line no-console
console.debug(
"[live-mode] early-close on stable silence (KI-159)",
{ quiet, durationMs: Date.now() - recStartTsRef.current },
);
void endSpeechCapture();
}
}
// KI-057 β€” max-utterance cap. If recording has run too long
// without silence-end firing, force-close it. Prevents the
// "noise pinned the meter open" stuck state.
if (
recordingRef.current &&
recStartTsRef.current > 0 &&
Date.now() - recStartTsRef.current > cfg.maxUtteranceMs
) {
// eslint-disable-next-line no-console
console.debug("[live-mode] force-closing at max-utterance cap");
void endSpeechCapture();
}
rafIdRef.current = requestAnimationFrame(loop);
};
rafIdRef.current = requestAnimationFrame(loop);
}, [
cfg.rmsThreshold,
cfg.silenceEndFrames,
cfg.speechStartFrames,
cfg.maxUtteranceMs,
cfg.postUtteranceCooldownMs,
cfg.voiceBandMinProp,
interruptBotAudio,
beginSpeechCapture,
endSpeechCapture,
]);
useEffect(() => {
let cancelled = false;
const tearDown = () => {
if (rafIdRef.current !== null) {
cancelAnimationFrame(rafIdRef.current);
rafIdRef.current = null;
}
// KI-057 β€” flush a mid-utterance capture before dropping refs.
// If the user toggled Live OFF while speaking, encode + fire
// onUtterance once (fire-and-forget) so their words still land.
// KI-165 (2026-05-15) β€” honor the same voiced-frames + text-in-flight
// guards here so toggling Live OFF mid-noise-burst doesn't also
// submit garbage.
if (
recordingRef.current &&
speechBufferRef.current.length > 0 &&
voicedFramesRef.current >= DEFAULTS.minVoicedFrames &&
!isTextRequestPendingRef?.current
) {
const durationMs = Date.now() - (recStartTsRef.current || Date.now());
if (durationMs >= DEFAULTS.minUtteranceMs) {
let total = 0;
for (const c of speechBufferRef.current) total += c.length;
const merged = new Float32Array(total);
let off = 0;
for (const c of speechBufferRef.current) {
merged.set(c, off);
off += c.length;
}
const wav = encodeWAV(merged, sampleRateRef.current);
if (wav.size >= 3000) {
const handler = onUtteranceRef.current;
const abort = new AbortController();
// Fire-and-forget. The page handler is independent of Live
// being on, so the response will still render in the chat
// pane after teardown completes.
try {
handler(wav, abort).catch((e) => {
const name = (e as { name?: string })?.name;
if (name !== "AbortError") {
// eslint-disable-next-line no-console
console.error("[live-mode] flush-on-stop failed:", e);
}
});
} catch {}
}
}
}
recordingRef.current = false;
speechBufferRef.current = [];
prerollRef.current = [];
noiseFloorRef.current = 0;
recStartTsRef.current = 0;
voicedFramesRef.current = 0;
if (workletRef.current) {
try { workletRef.current.disconnect(); } catch {}
workletRef.current = null;
}
if (workletUrlRef.current) {
try { URL.revokeObjectURL(workletUrlRef.current); } catch {}
workletUrlRef.current = null;
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((t) => t.stop());
streamRef.current = null;
}
if (sourceRef.current) {
try { sourceRef.current.disconnect(); } catch {}
sourceRef.current = null;
}
analyserRef.current = null;
if (audioCtxRef.current) {
audioCtxRef.current.close().catch(() => {});
audioCtxRef.current = null;
}
};
if (!live) {
tearDown();
return;
}
(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
if (cancelled) {
stream.getTracks().forEach((t) => t.stop());
return;
}
streamRef.current = stream;
const AudioCtx =
(window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext ||
window.AudioContext;
const ctx = new AudioCtx();
audioCtxRef.current = ctx;
// KI-134 (2026-05-15) β€” Chrome/Safari autoplay policy starts new
// AudioContexts in state='suspended' when the click that toggled
// Voice on has already been consumed by React state propagation.
// Without resume(), the worklet's process() never runs, no PCM
// frames arrive, and the green pill renders forever with zero
// audio posted. This is THE canonical "voice on but nothing
// happens" trap. Resume + bail visibly if the context refuses.
if (ctx.state === "suspended") {
try {
await ctx.resume();
} catch (e) {
// eslint-disable-next-line no-console
console.error("[live-mode] AudioContext resume failed", e);
setMicPermissionDenied(true);
setLive(false);
return;
}
}
// KI-139 (2026-05-15) β€” Safari iOS resume() can return without
// throwing yet leave state at "suspended" β€” silent rejection of the
// autoplay-policy unlock. Treat anything other than "running" as a
// failure and surface to the user.
if (ctx.state !== "running") {
// eslint-disable-next-line no-console
console.error("[live-mode] AudioContext state stuck at", ctx.state, "β€” giving up");
setMicPermissionDenied(true);
setLive(false);
return;
}
sampleRateRef.current = ctx.sampleRate;
const source = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 512;
analyser.smoothingTimeConstant = 0.5;
source.connect(analyser);
sourceRef.current = source;
analyserRef.current = analyser;
// KI-044 β€” register the inline PCM-capture worklet + tap the source.
const blob = new Blob([WORKLET_SOURCE], { type: "application/javascript" });
const url = URL.createObjectURL(blob);
workletUrlRef.current = url;
try {
await ctx.audioWorklet.addModule(url);
const node = new AudioWorkletNode(ctx, "pcm-capture");
workletRef.current = node;
const prerollSamplesCap = Math.ceil((cfg.prerollMs / 1000) * ctx.sampleRate);
node.port.onmessage = (ev: MessageEvent<Float32Array>) => {
const chunk = ev.data;
if (recordingRef.current) {
speechBufferRef.current.push(chunk);
} else {
prerollRef.current.push(chunk);
// Trim oldest chunks to keep total length under prerollSamplesCap.
let total = 0;
for (const c of prerollRef.current) total += c.length;
while (total > prerollSamplesCap && prerollRef.current.length > 1) {
total -= prerollRef.current[0].length;
prerollRef.current.shift();
}
}
};
source.connect(node);
// Worklet's process() only runs while the node is connected to a
// destination (directly or via the graph). But we DON'T want the
// user's mic playing back through speakers β€” route via a zero-gain
// GainNode so the graph stays "live" but output is silent.
const silentSink = ctx.createGain();
silentSink.gain.value = 0;
node.connect(silentSink);
silentSink.connect(ctx.destination);
} catch (e) {
// KI-131 (2026-05-15) β€” previously, a worklet-setup failure left
// setMicPermissionDenied=false and live.live=true so the UI pill
// stayed green ("Voice on β€” just speak") while no PCM frames were
// ever delivered β€” silent functional break. Now flip both states
// so the pill correctly switches to "πŸ”‡ Mic blocked" and the user
// gets a visible signal that voice is broken on their device.
// eslint-disable-next-line no-console
console.error("[live-mode] AudioWorklet setup failed", e);
setMicPermissionDenied(true);
setLive(false);
return;
}
setMicPermissionDenied(false);
tickVAD();
} catch (e) {
// eslint-disable-next-line no-console
console.error("[live-mode] mic permission denied or unavailable", e);
setMicPermissionDenied(true);
setLive(false);
}
})();
return () => {
cancelled = true;
tearDown();
};
}, [live, tickVAD, cfg.prerollMs]);
return {
live,
recording,
micPermissionDenied,
setLive,
inflightAbortRef,
};
}