Spaces:
Sleeping
fix(voice): KI-030 — bot TTS now plays via in-DOM audio so barge-in works
Browse filesUser report: "I can't speak and cut off the bot, it goes through its full
read out without stopping." Root cause: bot replies were played via
`new Audio(audioUrl).play()` (page.tsx:329, :401) — those create
HTMLAudioElement instances that are NOT attached to the DOM tree.
useLiveConversation's barge-in handler does
`document.querySelectorAll("audio").forEach(a => a.pause())` — and
querySelectorAll only returns DOM-attached elements. So when the user
spoke over the bot, the VAD-driven barge-in dutifully paused… nothing.
The detached audio kept reading the full TTS through to the end.
Fix:
• Removed both detached `new Audio(audioUrl).play()` sites.
• Message component now owns the playback via a ref'd
`<audio controls>` element + a mount-only useEffect that calls
`.play()` on it. Same element is what the user sees (so the
play-bar reflects real playback state too — single source of truth).
• barge-in's querySelectorAll picks up the in-DOM element and pauses
it instantly when VAD detects user speech.
• On chat-history rehydrate from localStorage, audioUrl is already
stripped (see persist effect ~line 109), so old replies don't
re-autoplay on page reload.
Net effect: speaking over the bot in Live mode now actually interrupts
its TTS reply mid-sentence, as designed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- frontend/src/app/page.tsx +40 -14
|
@@ -325,15 +325,12 @@ export default function Page() {
|
|
| 325 |
latencyMs: res.latency_ms,
|
| 326 |
blocked: res.blocked,
|
| 327 |
});
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
// KI-027 — Hands-free auto-reopen loop removed. Live mode provides the
|
| 335 |
-
// continuous-conversation behavior; push-to-talk is now strictly
|
| 336 |
-
// one-shot per click.
|
| 337 |
} catch (e: unknown) {
|
| 338 |
const err = e as { name?: string; message?: string };
|
| 339 |
if (err?.name === "AbortError") {
|
|
@@ -397,10 +394,9 @@ export default function Page() {
|
|
| 397 |
latencyMs: res.latency_ms,
|
| 398 |
blocked: res.blocked,
|
| 399 |
});
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
}
|
| 404 |
} catch (e: unknown) {
|
| 405 |
const name = (e as { name?: string })?.name;
|
| 406 |
if (name === "AbortError") return; // user barged in; intentional
|
|
@@ -1367,13 +1363,43 @@ function stripInlineCitations(text: string): string {
|
|
| 1367 |
function Message({ m }: { m: DisplayMessage }) {
|
| 1368 |
const isUser = m.role === "user";
|
| 1369 |
const displayContent = isUser ? m.content : stripInlineCitations(m.content);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1370 |
return (
|
| 1371 |
<div className={`flex animate-fade-up ${isUser ? "justify-end" : "justify-start"}`}>
|
| 1372 |
<div className={`max-w-[85%] sm:max-w-[75%] rounded-2xl px-4 py-3 ${
|
| 1373 |
isUser ? "bg-[var(--primary)] text-[var(--primary-foreground)]" : "bg-[var(--card)] border border-[var(--border)]"
|
| 1374 |
}`}>
|
| 1375 |
<div className="text-sm sm:text-base whitespace-pre-wrap leading-relaxed">{displayContent}</div>
|
| 1376 |
-
{m.audioUrl &&
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1377 |
{!isUser && m.citations && m.citations.length > 0 && (
|
| 1378 |
<PolicyChipsFromCitations citations={m.citations} />
|
| 1379 |
)}
|
|
|
|
| 325 |
latencyMs: res.latency_ms,
|
| 326 |
blocked: res.blocked,
|
| 327 |
});
|
| 328 |
+
// KI-030 (2026-05-14) — playback moved into the in-DOM <audio> element
|
| 329 |
+
// owned by the Message component (autoplay on mount). Detached
|
| 330 |
+
// `new Audio()` instances were invisible to
|
| 331 |
+
// useLiveConversation.interruptBotAudio()'s document.querySelectorAll
|
| 332 |
+
// pause, which broke barge-in. Now the DOM audio handles playback +
|
| 333 |
+
// can be paused by querySelectorAll when the user speaks over the bot.
|
|
|
|
|
|
|
|
|
|
| 334 |
} catch (e: unknown) {
|
| 335 |
const err = e as { name?: string; message?: string };
|
| 336 |
if (err?.name === "AbortError") {
|
|
|
|
| 394 |
latencyMs: res.latency_ms,
|
| 395 |
blocked: res.blocked,
|
| 396 |
});
|
| 397 |
+
// KI-030 — playback handled by the in-DOM <audio> in Message component
|
| 398 |
+
// (autoplay-on-mount); needed so barge-in's querySelectorAll("audio")
|
| 399 |
+
// can pause it. See comment in the typed-send branch.
|
|
|
|
| 400 |
} catch (e: unknown) {
|
| 401 |
const name = (e as { name?: string })?.name;
|
| 402 |
if (name === "AbortError") return; // user barged in; intentional
|
|
|
|
| 1363 |
function Message({ m }: { m: DisplayMessage }) {
|
| 1364 |
const isUser = m.role === "user";
|
| 1365 |
const displayContent = isUser ? m.content : stripInlineCitations(m.content);
|
| 1366 |
+
const audioRef = useRef<HTMLAudioElement | null>(null);
|
| 1367 |
+
|
| 1368 |
+
// KI-030 — Auto-play the bot's TTS reply when the message first mounts.
|
| 1369 |
+
// Replaces the old detached `new Audio(url).play()` approach which created
|
| 1370 |
+
// an element OUTSIDE the DOM tree — that element couldn't be found by the
|
| 1371 |
+
// `document.querySelectorAll("audio")` call in useLiveConversation's
|
| 1372 |
+
// barge-in handler, so the bot kept reading the full TTS even when the
|
| 1373 |
+
// user spoke over it. Now playback lives on a DOM audio element, which
|
| 1374 |
+
// querySelectorAll DOES find — so saying anything during the bot's reply
|
| 1375 |
+
// pauses it instantly.
|
| 1376 |
+
// Played only on mount (one-shot) so chat-history rehydration doesn't
|
| 1377 |
+
// replay every old reply. (audioUrl is also stripped from localStorage on
|
| 1378 |
+
// persist, so old messages don't have URLs to replay anyway.)
|
| 1379 |
+
useEffect(() => {
|
| 1380 |
+
if (m.audioUrl && audioRef.current) {
|
| 1381 |
+
audioRef.current.play().catch(() => {
|
| 1382 |
+
/* autoplay blocked — user can click the inline control to listen */
|
| 1383 |
+
});
|
| 1384 |
+
}
|
| 1385 |
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 1386 |
+
}, []);
|
| 1387 |
+
|
| 1388 |
return (
|
| 1389 |
<div className={`flex animate-fade-up ${isUser ? "justify-end" : "justify-start"}`}>
|
| 1390 |
<div className={`max-w-[85%] sm:max-w-[75%] rounded-2xl px-4 py-3 ${
|
| 1391 |
isUser ? "bg-[var(--primary)] text-[var(--primary-foreground)]" : "bg-[var(--card)] border border-[var(--border)]"
|
| 1392 |
}`}>
|
| 1393 |
<div className="text-sm sm:text-base whitespace-pre-wrap leading-relaxed">{displayContent}</div>
|
| 1394 |
+
{m.audioUrl && (
|
| 1395 |
+
<audio
|
| 1396 |
+
ref={audioRef}
|
| 1397 |
+
controls
|
| 1398 |
+
src={m.audioUrl}
|
| 1399 |
+
className="mt-2 w-full max-w-xs"
|
| 1400 |
+
style={{ height: 32 }}
|
| 1401 |
+
/>
|
| 1402 |
+
)}
|
| 1403 |
{!isUser && m.citations && m.citations.length > 0 && (
|
| 1404 |
<PolicyChipsFromCitations citations={m.citations} />
|
| 1405 |
)}
|