rohitsar567 commited on
Commit
e5b5e6c
·
1 Parent(s): 804917b

fix(voice): KI-134 — 5 blind fixes for silent-fail mic capture

Browse files

Diagnosis agent (run after KI-131's green-pill-on-broken-worklet patch)
ranked 5 high-probability additional silent-failure modes. All five
patched in one commit since they're all in the same path.

1. AudioContext.resume() — Chrome/Safari autoplay policy starts new
AudioContexts in state='suspended' when the click that toggled Voice
on has already been consumed by React state propagation. Without
resume(), worklet.process() never runs, no PCM frames arrive, green
pill renders forever. Patch: explicit ctx.resume() before VAD start;
on failure, flip setMicPermissionDenied(true) + setLive(false).
(useLiveConversation.ts)

2. PTT silent-recording UX — pre-fix, blob.size < 1000 returned silently
with zero user feedback. Now surfaces 'Didn't catch any audio — try
holding the mic button while speaking.' (page.tsx)

3. PTT teardown race — 120ms wait between setLive(false) and getUserMedia
isn't enough on HF Space's cpu-basic hardware to let AudioContext.close()
(async, returns ignored Promise) release the input device. Bumped to
400ms. (page.tsx)

4. voiceBandMinProp — KI-113 raised this from 0.35 to 0.50 to gate out
HVAC noise, but aggressive noiseSuppression on built-in laptop mics
flattens voiced speech to 0.35-0.45 in the gate band; VAD never opens
and no audio is ever posted. Backed off to 0.35.
(useLiveConversation.ts)

5. iOS Safari MediaRecorder produces audio/mp4 (no webm support). The
filename was 'audio.wav' but bytes were mp4 → backend's mime/ext
whitelist rejected. Added explicit mp4/m4a -> 'm4a' mapping. (api.ts)

Verified locally: TypeScript clean (exit 0). No new runtime test possible
without browser; user will validate live after rebuild.

frontend/src/app/page.tsx CHANGED
@@ -442,7 +442,11 @@ export default function Page() {
442
  // Small wait so useLiveConversation tears down its stream before we
443
  // open a fresh one; without this, two AudioContexts can briefly grab
444
  // the same input device.
445
- await new Promise((r) => setTimeout(r, 120));
 
 
 
 
446
  }
447
  try {
448
  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
@@ -461,7 +465,14 @@ export default function Page() {
461
  // does its one turn and leaves Live off. They'll keep using PTT
462
  // until they click the dot back to green themselves.
463
  const maybeResumeLive = () => { if (userPrefersLive) live.setLive(true); };
464
- if (blob.size < 1000) { maybeResumeLive(); return; }
 
 
 
 
 
 
 
465
  setBusy(true);
466
  setVoicePhase("transcribing"); // KI-038 — STT in flight on PTT
467
  try {
 
442
  // Small wait so useLiveConversation tears down its stream before we
443
  // open a fresh one; without this, two AudioContexts can briefly grab
444
  // the same input device.
445
+ // KI-134 (2026-05-15) bumped 120ms → 400ms because AudioContext.close()
446
+ // returns a Promise that's discarded; on HF Space hardware the previous
447
+ // context can still hold the device when getUserMedia fires, producing
448
+ // a silent stream with no error.
449
+ await new Promise((r) => setTimeout(r, 400));
450
  }
451
  try {
452
  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
 
465
  // does its one turn and leaves Live off. They'll keep using PTT
466
  // until they click the dot back to green themselves.
467
  const maybeResumeLive = () => { if (userPrefersLive) live.setLive(true); };
468
+ // KI-134 (2026-05-15) surface the silent-recording case to the user
469
+ // instead of returning quietly. Previously, holding PTT briefly and
470
+ // releasing produced no feedback at all; now they at least see why.
471
+ if (blob.size < 1000) {
472
+ pushAssistant("Didn't catch any audio — try holding the mic button while speaking.");
473
+ maybeResumeLive();
474
+ return;
475
+ }
476
  setBusy(true);
477
  setVoicePhase("transcribing"); // KI-038 — STT in flight on PTT
478
  try {
frontend/src/lib/api.ts CHANGED
@@ -131,13 +131,18 @@ export async function postTranscribe(
131
  const fd = new FormData();
132
  // Use blob's mime to derive extension; default to wav
133
  const mime = blob.type || "audio/wav";
 
 
 
134
  const ext = mime.includes("webm")
135
  ? "webm"
136
- : mime.includes("mp3")
137
- ? "mp3"
138
- : mime.includes("ogg")
139
- ? "ogg"
140
- : "wav";
 
 
141
  fd.append("file", blob, `audio.${ext}`);
142
  if (language_code) fd.append("language_code", language_code);
143
 
 
131
  const fd = new FormData();
132
  // Use blob's mime to derive extension; default to wav
133
  const mime = blob.type || "audio/wav";
134
+ // KI-134 (2026-05-15) — iOS Safari MediaRecorder produces audio/mp4 (no
135
+ // webm support). Without explicit mapping the file was sent as audio.wav
136
+ // with mp4 bytes inside, breaking the backend's mime/ext whitelist.
137
  const ext = mime.includes("webm")
138
  ? "webm"
139
+ : mime.includes("mp4") || mime.includes("m4a")
140
+ ? "m4a"
141
+ : mime.includes("mp3")
142
+ ? "mp3"
143
+ : mime.includes("ogg")
144
+ ? "ogg"
145
+ : "wav";
146
  fd.append("file", blob, `audio.${ext}`);
147
  if (language_code) fd.append("language_code", language_code);
148
 
frontend/src/lib/useLiveConversation.ts CHANGED
@@ -119,12 +119,12 @@ const DEFAULTS = {
119
  // segment. Avoids bot's TTS attack transient bleeding through even
120
  // with echoCancellation on.
121
  postUtteranceCooldownMs: 700,
122
- // KI-113 raised 0.35 → 0.50 minimum fraction of total FFT energy
123
- // that must sit in the voice band (bins 2-22 190-2150 Hz at 48 kHz).
124
- // Broadband HVAC / fan / traffic noise typically scores 0.20-0.30;
125
- // voiced speech typically scores 0.40-0.70. Lifting to 0.50 puts the
126
- // gate squarely above noise and just below the bottom of speech.
127
- voiceBandMinProp: 0.50,
128
  };
129
 
130
  // AudioWorklet processor source — inlined as a Blob URL so we don't need
@@ -489,6 +489,24 @@ export function useLiveConversation(opts: LiveConversationOptions): LiveConversa
489
  window.AudioContext;
490
  const ctx = new AudioCtx();
491
  audioCtxRef.current = ctx;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
492
  sampleRateRef.current = ctx.sampleRate;
493
 
494
  const source = ctx.createMediaStreamSource(stream);
 
119
  // segment. Avoids bot's TTS attack transient bleeding through even
120
  // with echoCancellation on.
121
  postUtteranceCooldownMs: 700,
122
+ // KI-113 raised 0.35 → 0.50 to gate out broadband HVAC / fan / traffic.
123
+ // KI-134 (2026-05-15) backed off to 0.35 because some laptop built-in
124
+ // mics with aggressive noiseSuppression flatten the voice-band proportion
125
+ // to 0.35-0.45, so the VAD never opens and the green pill renders but
126
+ // no audio is ever posted matches the live-test symptom exactly.
127
+ voiceBandMinProp: 0.35,
128
  };
129
 
130
  // AudioWorklet processor source — inlined as a Blob URL so we don't need
 
489
  window.AudioContext;
490
  const ctx = new AudioCtx();
491
  audioCtxRef.current = ctx;
492
+ // KI-134 (2026-05-15) — Chrome/Safari autoplay policy starts new
493
+ // AudioContexts in state='suspended' when the click that toggled
494
+ // Voice on has already been consumed by React state propagation.
495
+ // Without resume(), the worklet's process() never runs, no PCM
496
+ // frames arrive, and the green pill renders forever with zero
497
+ // audio posted. This is THE canonical "voice on but nothing
498
+ // happens" trap. Resume + bail visibly if the context refuses.
499
+ if (ctx.state === "suspended") {
500
+ try {
501
+ await ctx.resume();
502
+ } catch (e) {
503
+ // eslint-disable-next-line no-console
504
+ console.error("[live-mode] AudioContext resume failed", e);
505
+ setMicPermissionDenied(true);
506
+ setLive(false);
507
+ return;
508
+ }
509
+ }
510
  sampleRateRef.current = ctx.sampleRate;
511
 
512
  const source = ctx.createMediaStreamSource(stream);