rohitsar567 Claude Opus 4.7 (1M context) commited on
Commit
52ecbc9
·
1 Parent(s): 897727d

fix(voice): KI-188 — pause SpeechRecognition during TTS to kill echo

Browse files

User report (persistent across KI-185): bot's TTS still being
transcribed as user input ("perfect days to get started Rohit" was
echo of bot's "perfect age to get started, Rohit").

Why KI-185 (AEC constraints on getUserMedia) didn't fix it:
SpeechRecognition uses its OWN internal mic pipeline that bypasses
JS-configured constraints on other getUserMedia streams. AEC applied
to MediaRecorder doesn't help SpeechRecognition's interim transcripts
that drive the visible input + auto-submit.

Fix: MutationObserver on document.body watches for <audio> elements;
when ANY <audio> is playing, set isTtsPlayingRef=true and call
recognition.abort(). Heartbeat (KI-173) + visibility revival (KI-174)
now check !isTtsPlayingRef.current before starting. When TTS ends,
trigger immediate safeStart() so recognition resumes within ~50ms.

Trade-off: live "barge-in by just speaking" disabled during TTS.
Push-to-talk button still works (uses MediaRecorder, not Speech-
Recognition). KI-189 follow-up will add DIY VAD-based barge-in on
the AEC'd MediaRecorder stream so live-speak barge-in returns.

Verification: npx tsc --noEmit clean (exit 0).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. frontend/src/lib/useStreamingVoice.ts +107 -0
frontend/src/lib/useStreamingVoice.ts CHANGED
@@ -131,6 +131,14 @@ export function useStreamingVoice(
131
  const wantRunningRef = useRef(false); // mirrors `enabled` for handler closures
132
  const restartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
133
  const errorBackoffRef = useRef(0);
 
 
 
 
 
 
 
 
134
 
135
  // ----------------------------------------------------------------------
136
  // KI-168 PHASE 2 — Sarvam authoritative-transcript layer.
@@ -544,6 +552,7 @@ export function useStreamingVoice(
544
  if (
545
  wantRunningRef.current
546
  && !isTextRequestPendingRef.current
 
547
  && restartTimerRef.current === null
548
  ) {
549
  safeStart();
@@ -552,6 +561,103 @@ export function useStreamingVoice(
552
  return () => clearInterval(tick);
553
  }, [enabled, isSupported, isTextRequestPendingRef, safeStart]);
554
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
555
  // KI-174 (2026-05-15) — immediate-revival on visibility/focus changes.
556
  // User reported: "sometimes when I go away from clicking the text box,
557
  // it seems to not input my voice anymore. I have to restart the whole
@@ -571,6 +677,7 @@ export function useStreamingVoice(
571
  if (
572
  wantRunningRef.current
573
  && !isTextRequestPendingRef.current
 
574
  && document.visibilityState === "visible"
575
  ) {
576
  console.debug("[useStreamingVoice] revival trigger=" + trigger);
 
131
  const wantRunningRef = useRef(false); // mirrors `enabled` for handler closures
132
  const restartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
133
  const errorBackoffRef = useRef(0);
134
+ // KI-188 (2026-05-15) — TTS-playback gate. Web Speech API has its own
135
+ // internal mic pipeline that bypasses our getUserMedia AEC constraints,
136
+ // so SpeechRecognition transcribes the bot's TTS audio bleeding from
137
+ // speakers as user input ("echo loop"). The only reliable fix from JS
138
+ // is to abort recognition while ANY <audio> in the DOM is playing.
139
+ // Tracked via a MutationObserver + per-element play/pause/ended hooks.
140
+ const isTtsPlayingRef = useRef(false);
141
+ const ttsAudioElementsRef = useRef<Set<HTMLAudioElement>>(new Set());
142
 
143
  // ----------------------------------------------------------------------
144
  // KI-168 PHASE 2 — Sarvam authoritative-transcript layer.
 
552
  if (
553
  wantRunningRef.current
554
  && !isTextRequestPendingRef.current
555
+ && !isTtsPlayingRef.current // KI-188 — block revival during TTS playback
556
  && restartTimerRef.current === null
557
  ) {
558
  safeStart();
 
561
  return () => clearInterval(tick);
562
  }, [enabled, isSupported, isTextRequestPendingRef, safeStart]);
563
 
564
+ // KI-188 (2026-05-15) — TTS playback gate. Browser Web Speech API has
565
+ // its own internal mic pipeline that bypasses our getUserMedia AEC
566
+ // constraints (KI-185), so SpeechRecognition transcribes the bot's TTS
567
+ // audio bleeding from speakers as if it were user input. The visible
568
+ // echo "perfect days to get started Rohit" was echo of bot's TTS
569
+ // "perfect age to get started, Rohit". The only reliable JS-level fix
570
+ // is to ABORT recognition while ANY <audio> element in the DOM is
571
+ // playing, then revive via the heartbeat (KI-173) the moment all
572
+ // audio ends.
573
+ //
574
+ // Trade-off: live "barge-in by just speaking" is disabled DURING TTS.
575
+ // Push-to-talk still works (it uses MediaRecorder, not SpeechRecognition).
576
+ useEffect(() => {
577
+ if (!enabled || !isSupported) return;
578
+ if (typeof document === "undefined") return;
579
+
580
+ const updateTtsState = () => {
581
+ let anyPlaying = false;
582
+ ttsAudioElementsRef.current.forEach((el) => {
583
+ if (!el.paused && !el.ended) anyPlaying = true;
584
+ });
585
+ const wasPlaying = isTtsPlayingRef.current;
586
+ isTtsPlayingRef.current = anyPlaying;
587
+ if (anyPlaying && !wasPlaying) {
588
+ // TTS just started — abort any in-flight recognition so it stops
589
+ // transcribing the bot voice.
590
+ console.debug("[useStreamingVoice] KI-188 TTS started — pausing recognition");
591
+ const rec = recognitionRef.current;
592
+ if (rec) {
593
+ try { rec.abort(); } catch { /* ignore */ }
594
+ }
595
+ } else if (!anyPlaying && wasPlaying) {
596
+ // TTS just ended — let the heartbeat/visibility listeners revive.
597
+ // Trigger immediately too so the user doesn't wait ~4s.
598
+ console.debug("[useStreamingVoice] KI-188 TTS ended — resuming recognition");
599
+ if (wantRunningRef.current && !isTextRequestPendingRef.current) {
600
+ safeStart();
601
+ }
602
+ }
603
+ };
604
+
605
+ const watchAudio = (el: HTMLAudioElement) => {
606
+ if (ttsAudioElementsRef.current.has(el)) return;
607
+ ttsAudioElementsRef.current.add(el);
608
+ el.addEventListener("play", updateTtsState);
609
+ el.addEventListener("playing", updateTtsState);
610
+ el.addEventListener("pause", updateTtsState);
611
+ el.addEventListener("ended", updateTtsState);
612
+ // Initial check (handles audio that was already playing on mount)
613
+ updateTtsState();
614
+ };
615
+
616
+ const unwatchAudio = (el: HTMLAudioElement) => {
617
+ if (!ttsAudioElementsRef.current.has(el)) return;
618
+ el.removeEventListener("play", updateTtsState);
619
+ el.removeEventListener("playing", updateTtsState);
620
+ el.removeEventListener("pause", updateTtsState);
621
+ el.removeEventListener("ended", updateTtsState);
622
+ ttsAudioElementsRef.current.delete(el);
623
+ updateTtsState();
624
+ };
625
+
626
+ // Initial scan
627
+ document.querySelectorAll("audio").forEach((el) => watchAudio(el as HTMLAudioElement));
628
+
629
+ // Watch the whole document for new <audio> elements
630
+ const observer = new MutationObserver((mutations) => {
631
+ mutations.forEach((m) => {
632
+ m.addedNodes.forEach((n) => {
633
+ if (n instanceof HTMLElement) {
634
+ if (n.tagName === "AUDIO") watchAudio(n as HTMLAudioElement);
635
+ n.querySelectorAll?.("audio").forEach((el) => watchAudio(el as HTMLAudioElement));
636
+ }
637
+ });
638
+ m.removedNodes.forEach((n) => {
639
+ if (n instanceof HTMLElement) {
640
+ if (n.tagName === "AUDIO") unwatchAudio(n as HTMLAudioElement);
641
+ n.querySelectorAll?.("audio").forEach((el) => unwatchAudio(el as HTMLAudioElement));
642
+ }
643
+ });
644
+ });
645
+ });
646
+ observer.observe(document.body, { childList: true, subtree: true });
647
+
648
+ return () => {
649
+ observer.disconnect();
650
+ ttsAudioElementsRef.current.forEach((el) => {
651
+ el.removeEventListener("play", updateTtsState);
652
+ el.removeEventListener("playing", updateTtsState);
653
+ el.removeEventListener("pause", updateTtsState);
654
+ el.removeEventListener("ended", updateTtsState);
655
+ });
656
+ ttsAudioElementsRef.current.clear();
657
+ isTtsPlayingRef.current = false;
658
+ };
659
+ }, [enabled, isSupported, isTextRequestPendingRef, safeStart]);
660
+
661
  // KI-174 (2026-05-15) — immediate-revival on visibility/focus changes.
662
  // User reported: "sometimes when I go away from clicking the text box,
663
  // it seems to not input my voice anymore. I have to restart the whole
 
677
  if (
678
  wantRunningRef.current
679
  && !isTextRequestPendingRef.current
680
+ && !isTtsPlayingRef.current // KI-188 — block revival during TTS
681
  && document.visibilityState === "visible"
682
  ) {
683
  console.debug("[useStreamingVoice] revival trigger=" + trigger);