feat(voice): full-duplex Live conversation mode with VAD barge-in (D-025)

User can now speak ANY time — including while the bot is mid-sentence —
and the bot stops speaking immediately and listens. No button press.

New file: frontend/src/lib/useLiveConversation.ts
- Continuous getUserMedia stream (echo cancellation, noise suppression,
auto gain control all enabled).
- AudioContext + AnalyserNode RMS-based VAD on a requestAnimationFrame
loop. Thresholds: 28/255 byte FFT magnitude floor, 5 frames (~80ms)
of loudness to confirm speech, 40 frames (~640ms) of silence to end.
- On speech-start: pause every <audio> element (kills bot TTS mid-
sentence), abort the in-flight /api/chat fetch via AbortController,
start a MediaRecorder.
- On utterance-end: stop recorder, call onUtterance(blob, abort).
- Caller-managed inflightAbortRef so the page can register fetches
that VAD should cancel on barge-in.

api.ts: postChat() and postTranscribe() now accept an optional
AbortSignal so the live-mode handler can plumb the VAD abort signal
end-to-end. Existing PTT callers unaffected.

page.tsx: import the hook, add a Go Live button next to Voice reply +
Hands-free in the chat-input footer. Status indicator: green pulse =
listening for speech, red pulse = actively recording an utterance.
useEffect rebinds the onUtterance closure on relevant state changes
(messages, sessionId, ttsLang, view_context inputs) so live-mode turns
flow into the same pushUser → postChat → pushAssistant path the existing
send() uses.

Closes Issue 1 from the 2026-05-14 live-bot review.

v1 limitations (planned for v1.1):
- Conservative RMS threshold; quiet speakers may need it lowered.
Future: Silero VAD via @ricky0123 /vad-web for accuracy.
- No background-noise calibration on entry.
- TTS isn't streaming yet — full reply plays before next utterance
window opens. Streaming TTS makes barge-in feel even more conversational.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

frontend/src/app/page.tsx

@@ -29,6 +29,7 @@ import {
   UserProfile,
 } from "@/lib/api";
 import { translate, UILang, StringKey, GLOSSARY } from "@/lib/i18n";
+import { useLiveConversation } from "@/lib/useLiveConversation";
 
 type DisplayMessage = ChatMessage & {
   id: string;
@@ -127,6 +128,17 @@ export default function Page() {
   const fileInputRef = useRef<HTMLInputElement>(null);
   const scrollRef = useRef<HTMLDivElement>(null);
 
+  // Live-conversation mode (full-duplex VAD barge-in). Defined further down
+  // in the component body so it can reference `sessionId`, `messages`, etc.
+  // via closure when the onUtterance handler fires. See useLiveConversation.ts.
+  const liveOnUtteranceRef = useRef<((blob: Blob, abort: AbortController) => Promise<void>) | null>(null);
+  const live = useLiveConversation({
+    onUtterance: async (blob, abort) => {
+      const fn = liveOnUtteranceRef.current;
+      if (fn) await fn(blob, abort);
+    },
+  });
+
   useEffect(() => {
     getHealth()
       .then((h) => setHealth({ status: h.status, missing: h.missing_keys }))
@@ -248,6 +260,58 @@ export default function Page() {
     }
   }
 
+  // Live-conversation onUtterance binding. Rebinds when relevant state changes
+  // so the latest sessionId / history / view are captured in the closure.
+  useEffect(() => {
+    liveOnUtteranceRef.current = async (blob, abort) => {
+      try {
+        const transcribed = await postTranscribe(blob, ttsLang, abort.signal);
+        const text = (transcribed.text || "").trim();
+        if (text.length < 2) return;
+
+        pushUser(text);
+        const history: ChatMessage[] = messages.map((m) => ({ role: m.role, content: m.content }));
+        const active_view: "chat" | "marketplace" | "profile" | "premium" | "policy_detail" =
+          openPolicy ? "policy_detail" :
+          showMarketplace ? "marketplace" :
+          showProfile ? "profile" :
+          showPremium ? "premium" :
+          "chat";
+
+        const res = await postChat({
+          user_text: text,
+          session_id: sessionId,
+          chat_history: history,
+          return_audio: true,
+          tts_language_code: ttsLang,
+          view_context: { active_view, active_policy_id: openPolicy?.policy_id },
+          signal: abort.signal,
+        });
+        setSessionId(res.session_id);
+        getProfileCompleteness(res.session_id)
+          .then(setProfileCompleteness)
+          .catch(() => {});
+        const audioUrl = res.audio_base64 ? audioBlobURLFromBase64(res.audio_base64) : undefined;
+        pushAssistant(res.reply_text, {
+          citations: res.citations,
+          audioUrl,
+          brain: res.brain_used,
+          latencyMs: res.latency_ms,
+          blocked: res.blocked,
+        });
+        if (audioUrl) {
+          const audio = new Audio(audioUrl);
+          audio.play().catch(() => {});
+        }
+      } catch (e: unknown) {
+        const name = (e as { name?: string })?.name;
+        if (name === "AbortError") return; // user barged in; intentional
+        // eslint-disable-next-line no-console
+        console.error("[live mode] turn failed:", e);
+      }
+    };
+  }, [messages, sessionId, ttsLang, openPolicy, showMarketplace, showProfile, showPremium]);
+
   async function startRecording() {
     try {
       const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
@@ -561,6 +625,25 @@ export default function Page() {
               <label className="flex items-center gap-1.5 cursor-pointer" title="Hands-free voice — auto-submits when you stop speaking">
                 <input type="checkbox" checked={handsFree} onChange={(e) => setHandsFree(e.target.checked)} className="w-3.5 h-3.5 accent-[var(--primary)]" /> Hands-free
               </label>
+              {/* Live conversation — full-duplex with VAD barge-in. Speak any
+                  time, even while the bot is still talking, and it stops mid-
+                  sentence and listens to you. No button press needed. */}
+              <button
+                type="button"
+                onClick={() => live.setLive(!live.live)}
+                className={`flex items-center gap-1.5 px-2 py-0.5 rounded-full border text-xs font-medium transition ${
+                  live.live
+                    ? "bg-red-500/15 border-red-400 text-red-600"
+                    : "border-[var(--border)] hover:border-[var(--primary)] hover:text-[var(--primary)]"
+                }`}
+                title="Live conversation: continuous mic, interrupt the bot by speaking"
+              >
+                <span className={`inline-block w-2 h-2 rounded-full ${live.live ? (live.recording ? "bg-red-500 animate-pulse" : "bg-green-500 animate-pulse") : "bg-gray-400"}`} />
+                {live.live ? (live.recording ? "Listening…" : "Live ✓") : "Go Live"}
+              </button>
+              {live.micPermissionDenied && (
+                <span className="text-red-500" title="Browser blocked microphone access — check site permissions">mic blocked</span>
+              )}
               <label className="flex items-center gap-1.5">
                 Lang:
                 <select value={ttsLang} onChange={(e) => setTtsLang(e.target.value as "en-IN" | "hi-IN")} className="bg-transparent border border-[var(--border)] rounded px-1.5 py-0.5">

frontend/src/lib/api.ts

@@ -56,6 +56,7 @@ export async function postChat(args: {
   return_audio?: boolean;
   tts_language_code?: string;
   view_context?: ViewContext;
+  signal?: AbortSignal;
 }): Promise<ChatResponse> {
   const resp = await fetch(`${BACKEND_URL}/api/chat`, {
     method: "POST",
@@ -70,6 +71,7 @@ export async function postChat(args: {
       tts_language_code: args.tts_language_code ?? "en-IN",
       view_context: args.view_context,
     }),
+    signal: args.signal,
   });
   if (!resp.ok) {
     const t = await resp.text();
@@ -81,6 +83,7 @@ export async function postChat(args: {
 export async function postTranscribe(
   blob: Blob,
   language_code?: string,
+  signal?: AbortSignal,
 ): Promise<{ text: string; language_code?: string; latency_ms: number }> {
   const fd = new FormData();
   // Use blob's mime to derive extension; default to wav
@@ -98,6 +101,7 @@ export async function postTranscribe(
   const resp = await fetch(`${BACKEND_URL}/api/transcribe`, {
     method: "POST",
     body: fd,
+    signal,
   });
   if (!resp.ok) {
     const t = await resp.text();

frontend/src/lib/useLiveConversation.ts

@@ -0,0 +1,267 @@
+"use client";
+
+/**
+ * useLiveConversation — full-duplex voice mode with barge-in.
+ *
+ * Differences from the existing push-to-talk + Hands-free toggle:
+ *
+ *   - Mic is OPEN continuously while live mode is on (single getUserMedia
+ *     stream, not opened/closed per turn).
+ *   - VAD (RMS on AnalyserNode + frame counters) detects when the user
+ *     starts and stops speaking, with no button press.
+ *   - When VAD detects speech start, we immediately:
+ *       * pause + clear every <audio> currently playing (kill the bot's
+ *         in-progress TTS reply mid-sentence),
+ *       * abort the in-flight /api/chat fetch via AbortController,
+ *       * start a new MediaRecorder for the user's utterance.
+ *   - When VAD detects silence for >600ms after speech, we stop the
+ *     recorder and POST the blob to the supplied `onUtterance` handler
+ *     (which the page wires to transcribe → chat).
+ *
+ * The hook returns an AbortController slot the caller assigns to its
+ * own in-flight fetches so barge-in can cancel them.
+ */
+
+import { useCallback, useEffect, useRef, useState } from "react";
+
+export type LiveConversationOptions = {
+  /** Called when the user finishes an utterance — pass it the audio blob. */
+  onUtterance: (blob: Blob, abort: AbortController) => Promise<void>;
+  /** Called when VAD detects speech start (so the UI can show "listening…"). */
+  onSpeechStart?: () => void;
+  /** Called when VAD detects speech end (so the UI can show "thinking…"). */
+  onSpeechEnd?: () => void;
+  /** RMS threshold above which we declare "speech". Tune in browser. */
+  rmsThreshold?: number;
+  /** Consecutive loud frames needed to start recording (debounce). */
+  speechStartFrames?: number;
+  /** Consecutive quiet frames needed to stop recording (~16 ms/frame). */
+  silenceEndFrames?: number;
+};
+
+export type LiveConversationState = {
+  live: boolean;
+  recording: boolean;
+  micPermissionDenied: boolean;
+  setLive: (v: boolean) => void;
+  /** Caller-managed abort slot for in-flight fetches; VAD aborts it on speech. */
+  inflightAbortRef: React.MutableRefObject<AbortController | null>;
+};
+
+const DEFAULTS = {
+  rmsThreshold: 28, // 0-255 byte FFT magnitude floor; tune in browser
+  speechStartFrames: 5, // ~80 ms of consistent loudness to declare speech
+  silenceEndFrames: 40, // ~640 ms of silence to declare utterance end
+};
+
+export function useLiveConversation(opts: LiveConversationOptions): LiveConversationState {
+  const [live, setLive] = useState(false);
+  const [recording, setRecording] = useState(false);
+  const [micPermissionDenied, setMicPermissionDenied] = useState(false);
+
+  const streamRef = useRef<MediaStream | null>(null);
+  const audioCtxRef = useRef<AudioContext | null>(null);
+  const analyserRef = useRef<AnalyserNode | null>(null);
+  const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
+  const recorderRef = useRef<MediaRecorder | null>(null);
+  const chunksRef = useRef<Blob[]>([]);
+  const recordingRef = useRef(false); // sync ref for VAD loop
+  const rafIdRef = useRef<number | null>(null);
+  const inflightAbortRef = useRef<AbortController | null>(null);
+
+  const onUtteranceRef = useRef(opts.onUtterance);
+  const onSpeechStartRef = useRef(opts.onSpeechStart);
+  const onSpeechEndRef = useRef(opts.onSpeechEnd);
+  useEffect(() => {
+    onUtteranceRef.current = opts.onUtterance;
+    onSpeechStartRef.current = opts.onSpeechStart;
+    onSpeechEndRef.current = opts.onSpeechEnd;
+  }, [opts.onUtterance, opts.onSpeechStart, opts.onSpeechEnd]);
+
+  const cfg = {
+    rmsThreshold: opts.rmsThreshold ?? DEFAULTS.rmsThreshold,
+    speechStartFrames: opts.speechStartFrames ?? DEFAULTS.speechStartFrames,
+    silenceEndFrames: opts.silenceEndFrames ?? DEFAULTS.silenceEndFrames,
+  };
+
+  const stopRecording = useCallback(() => {
+    if (recorderRef.current && recorderRef.current.state !== "inactive") {
+      try { recorderRef.current.stop(); } catch {}
+    }
+  }, []);
+
+  const interruptBotAudio = useCallback(() => {
+    // Pause + reset every audio element in the DOM. Bot replies use plain
+    // <audio> elements; killing src forces the loaded buffer to drop.
+    if (typeof document !== "undefined") {
+      document.querySelectorAll("audio").forEach((a) => {
+        try {
+          a.pause();
+          // Don't blank src — let the existing buffer GC but leave the
+          // element so the chat history scroll position doesn't jump.
+          a.currentTime = a.duration || 0;
+        } catch {}
+      });
+    }
+  }, []);
+
+  const startRecording = useCallback(() => {
+    if (!streamRef.current) return;
+    const mime = MediaRecorder.isTypeSupported("audio/webm")
+      ? "audio/webm"
+      : "";
+    const rec = mime
+      ? new MediaRecorder(streamRef.current, { mimeType: mime })
+      : new MediaRecorder(streamRef.current);
+    chunksRef.current = [];
+    rec.ondataavailable = (e) => {
+      if (e.data && e.data.size > 0) chunksRef.current.push(e.data);
+    };
+    rec.onstop = async () => {
+      recordingRef.current = false;
+      setRecording(false);
+      if (chunksRef.current.length === 0) return;
+      const blob = new Blob(chunksRef.current, {
+        type: rec.mimeType || "audio/webm",
+      });
+      // Reject blobs that are almost certainly silence or VAD false-trips.
+      if (blob.size < 3000) return;
+      onSpeechEndRef.current?.();
+      const abort = new AbortController();
+      inflightAbortRef.current = abort;
+      try {
+        await onUtteranceRef.current(blob, abort);
+      } catch (e) {
+        const name = (e as { name?: string })?.name;
+        if (name !== "AbortError") {
+          // surface to console; the UI's existing error toast will fire too
+          // eslint-disable-next-line no-console
+          console.error("[live-mode] utterance handler failed:", e);
+        }
+      } finally {
+        if (inflightAbortRef.current === abort) {
+          inflightAbortRef.current = null;
+        }
+      }
+    };
+    recorderRef.current = rec;
+    recordingRef.current = true;
+    setRecording(true);
+    onSpeechStartRef.current?.();
+    rec.start();
+  }, []);
+
+  // VAD loop — runs while `live` is true.
+  const tickVAD = useCallback(() => {
+    if (!analyserRef.current) return;
+    const a = analyserRef.current;
+    const buf = new Uint8Array(a.frequencyBinCount);
+    let loud = 0;
+    let quiet = 0;
+    const loop = () => {
+      if (!analyserRef.current) return;
+      a.getByteFrequencyData(buf);
+      let sum = 0;
+      for (let i = 0; i < buf.length; i++) sum += buf[i];
+      const avg = sum / buf.length;
+
+      if (avg > cfg.rmsThreshold) {
+        loud++;
+        quiet = 0;
+        if (loud === cfg.speechStartFrames && !recordingRef.current) {
+          // Barge in: kill bot audio + cancel in-flight chat + start recording.
+          interruptBotAudio();
+          if (inflightAbortRef.current) {
+            try { inflightAbortRef.current.abort(); } catch {}
+            inflightAbortRef.current = null;
+          }
+          startRecording();
+        }
+      } else {
+        quiet++;
+        loud = 0;
+        if (quiet === cfg.silenceEndFrames && recordingRef.current) {
+          stopRecording();
+        }
+      }
+
+      rafIdRef.current = requestAnimationFrame(loop);
+    };
+    rafIdRef.current = requestAnimationFrame(loop);
+  }, [cfg.rmsThreshold, cfg.silenceEndFrames, cfg.speechStartFrames, interruptBotAudio, startRecording, stopRecording]);
+
+  useEffect(() => {
+    let cancelled = false;
+
+    const tearDown = () => {
+      if (rafIdRef.current !== null) {
+        cancelAnimationFrame(rafIdRef.current);
+        rafIdRef.current = null;
+      }
+      stopRecording();
+      if (streamRef.current) {
+        streamRef.current.getTracks().forEach((t) => t.stop());
+        streamRef.current = null;
+      }
+      if (sourceRef.current) {
+        try { sourceRef.current.disconnect(); } catch {}
+        sourceRef.current = null;
+      }
+      analyserRef.current = null;
+      if (audioCtxRef.current) {
+        audioCtxRef.current.close().catch(() => {});
+        audioCtxRef.current = null;
+      }
+    };
+
+    if (!live) {
+      tearDown();
+      return;
+    }
+
+    (async () => {
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
+        });
+        if (cancelled) {
+          stream.getTracks().forEach((t) => t.stop());
+          return;
+        }
+        streamRef.current = stream;
+        const AudioCtx =
+          (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext ||
+          window.AudioContext;
+        const ctx = new AudioCtx();
+        audioCtxRef.current = ctx;
+        const source = ctx.createMediaStreamSource(stream);
+        const analyser = ctx.createAnalyser();
+        analyser.fftSize = 512;
+        analyser.smoothingTimeConstant = 0.5;
+        source.connect(analyser);
+        sourceRef.current = source;
+        analyserRef.current = analyser;
+        setMicPermissionDenied(false);
+        tickVAD();
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.error("[live-mode] mic permission denied or unavailable", e);
+        setMicPermissionDenied(true);
+        setLive(false);
+      }
+    })();
+
+    return () => {
+      cancelled = true;
+      tearDown();
+    };
+  }, [live, stopRecording, tickVAD]);
+
+  return {
+    live,
+    recording,
+    micPermissionDenied,
+    setLive,
+    inflightAbortRef,
+  };
+}