rohitsar567 Claude Opus 4.7 (1M context) commited on
Commit
3d06a80
·
1 Parent(s): 65ba46c

fix(voice): KI-030 — bot TTS now plays via in-DOM audio so barge-in works

Browse files

User report: "I can't speak and cut off the bot, it goes through its full
read out without stopping." Root cause: bot replies were played via
`new Audio(audioUrl).play()` (page.tsx:329, :401) — those create
HTMLAudioElement instances that are NOT attached to the DOM tree.
useLiveConversation's barge-in handler does
`document.querySelectorAll("audio").forEach(a => a.pause())` — and
querySelectorAll only returns DOM-attached elements. So when the user
spoke over the bot, the VAD-driven barge-in dutifully paused… nothing.
The detached audio kept reading the full TTS through to the end.

Fix:
• Removed both detached `new Audio(audioUrl).play()` sites.
• Message component now owns the playback via a ref'd
`<audio controls>` element + a mount-only useEffect that calls
`.play()` on it. Same element is what the user sees (so the
play-bar reflects real playback state too — single source of truth).
• barge-in's querySelectorAll picks up the in-DOM element and pauses
it instantly when VAD detects user speech.
• On chat-history rehydrate from localStorage, audioUrl is already
stripped (see persist effect ~line 109), so old replies don't
re-autoplay on page reload.

Net effect: speaking over the bot in Live mode now actually interrupts
its TTS reply mid-sentence, as designed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files changed (1) hide show
  1. frontend/src/app/page.tsx +40 -14
frontend/src/app/page.tsx CHANGED
@@ -325,15 +325,12 @@ export default function Page() {
325
  latencyMs: res.latency_ms,
326
  blocked: res.blocked,
327
  });
328
- if (audioUrl) {
329
- const audio = new Audio(audioUrl);
330
- audio.play().catch(() => {
331
- /* autoplay blocked — user can click the inline audio player to hear it */
332
- });
333
- }
334
- // KI-027 — Hands-free auto-reopen loop removed. Live mode provides the
335
- // continuous-conversation behavior; push-to-talk is now strictly
336
- // one-shot per click.
337
  } catch (e: unknown) {
338
  const err = e as { name?: string; message?: string };
339
  if (err?.name === "AbortError") {
@@ -397,10 +394,9 @@ export default function Page() {
397
  latencyMs: res.latency_ms,
398
  blocked: res.blocked,
399
  });
400
- if (audioUrl) {
401
- const audio = new Audio(audioUrl);
402
- audio.play().catch(() => {});
403
- }
404
  } catch (e: unknown) {
405
  const name = (e as { name?: string })?.name;
406
  if (name === "AbortError") return; // user barged in; intentional
@@ -1367,13 +1363,43 @@ function stripInlineCitations(text: string): string {
1367
  function Message({ m }: { m: DisplayMessage }) {
1368
  const isUser = m.role === "user";
1369
  const displayContent = isUser ? m.content : stripInlineCitations(m.content);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1370
  return (
1371
  <div className={`flex animate-fade-up ${isUser ? "justify-end" : "justify-start"}`}>
1372
  <div className={`max-w-[85%] sm:max-w-[75%] rounded-2xl px-4 py-3 ${
1373
  isUser ? "bg-[var(--primary)] text-[var(--primary-foreground)]" : "bg-[var(--card)] border border-[var(--border)]"
1374
  }`}>
1375
  <div className="text-sm sm:text-base whitespace-pre-wrap leading-relaxed">{displayContent}</div>
1376
- {m.audioUrl && <audio controls src={m.audioUrl} className="mt-2 w-full max-w-xs" style={{ height: 32 }} />}
 
 
 
 
 
 
 
 
1377
  {!isUser && m.citations && m.citations.length > 0 && (
1378
  <PolicyChipsFromCitations citations={m.citations} />
1379
  )}
 
325
  latencyMs: res.latency_ms,
326
  blocked: res.blocked,
327
  });
328
+ // KI-030 (2026-05-14) — playback moved into the in-DOM <audio> element
329
+ // owned by the Message component (autoplay on mount). Detached
330
+ // `new Audio()` instances were invisible to
331
+ // useLiveConversation.interruptBotAudio()'s document.querySelectorAll
332
+ // pause, which broke barge-in. Now the DOM audio handles playback +
333
+ // can be paused by querySelectorAll when the user speaks over the bot.
 
 
 
334
  } catch (e: unknown) {
335
  const err = e as { name?: string; message?: string };
336
  if (err?.name === "AbortError") {
 
394
  latencyMs: res.latency_ms,
395
  blocked: res.blocked,
396
  });
397
+ // KI-030 — playback handled by the in-DOM <audio> in Message component
398
+ // (autoplay-on-mount); needed so barge-in's querySelectorAll("audio")
399
+ // can pause it. See comment in the typed-send branch.
 
400
  } catch (e: unknown) {
401
  const name = (e as { name?: string })?.name;
402
  if (name === "AbortError") return; // user barged in; intentional
 
1363
  function Message({ m }: { m: DisplayMessage }) {
1364
  const isUser = m.role === "user";
1365
  const displayContent = isUser ? m.content : stripInlineCitations(m.content);
1366
+ const audioRef = useRef<HTMLAudioElement | null>(null);
1367
+
1368
+ // KI-030 — Auto-play the bot's TTS reply when the message first mounts.
1369
+ // Replaces the old detached `new Audio(url).play()` approach which created
1370
+ // an element OUTSIDE the DOM tree — that element couldn't be found by the
1371
+ // `document.querySelectorAll("audio")` call in useLiveConversation's
1372
+ // barge-in handler, so the bot kept reading the full TTS even when the
1373
+ // user spoke over it. Now playback lives on a DOM audio element, which
1374
+ // querySelectorAll DOES find — so saying anything during the bot's reply
1375
+ // pauses it instantly.
1376
+ // Played only on mount (one-shot) so chat-history rehydration doesn't
1377
+ // replay every old reply. (audioUrl is also stripped from localStorage on
1378
+ // persist, so old messages don't have URLs to replay anyway.)
1379
+ useEffect(() => {
1380
+ if (m.audioUrl && audioRef.current) {
1381
+ audioRef.current.play().catch(() => {
1382
+ /* autoplay blocked — user can click the inline control to listen */
1383
+ });
1384
+ }
1385
+ // eslint-disable-next-line react-hooks/exhaustive-deps
1386
+ }, []);
1387
+
1388
  return (
1389
  <div className={`flex animate-fade-up ${isUser ? "justify-end" : "justify-start"}`}>
1390
  <div className={`max-w-[85%] sm:max-w-[75%] rounded-2xl px-4 py-3 ${
1391
  isUser ? "bg-[var(--primary)] text-[var(--primary-foreground)]" : "bg-[var(--card)] border border-[var(--border)]"
1392
  }`}>
1393
  <div className="text-sm sm:text-base whitespace-pre-wrap leading-relaxed">{displayContent}</div>
1394
+ {m.audioUrl && (
1395
+ <audio
1396
+ ref={audioRef}
1397
+ controls
1398
+ src={m.audioUrl}
1399
+ className="mt-2 w-full max-w-xs"
1400
+ style={{ height: 32 }}
1401
+ />
1402
+ )}
1403
  {!isUser && m.citations && m.citations.length > 0 && (
1404
  <PolicyChipsFromCitations citations={m.citations} />
1405
  )}