applied VAD

requirements.txt

@@ -7,6 +7,7 @@ httpx>=0.27.0
 structlog>=24.0.0
 azure-identity>=1.15.0
 azure-ai-projects==1.0.0b10
+silero-vad-lite>=0.2.0
 streamlit>=1.35.0
 pytest>=8.0.0
 pytest-asyncio>=0.23.0

src/app/api/routes.py

@@ -14,6 +14,7 @@ from ..core.errors import SpeechError, ValidationError
 from ..core.logging import get_logger
 from ..services.pipeline import VoicePipeline
 from ..services.stt import SpeechToTextService
+from ..services.vad import SileroVADStream
 from ..utils.audio import encode_base64
 
 router = APIRouter()
@@ -109,6 +110,91 @@ async def voice_stream(websocket: WebSocket) -> None:
     frames_sent: int | None = None
     avg_rms: float | None = None
     llm_provider: str | None = None
+    vad_stream: SileroVADStream | None = None
+    segment_processing = False
+
+    async def _finalize_segment() -> None:
+        nonlocal stt_session, segment_processing, vad_stream
+        if stt_session is None:
+            raise ValidationError(
+                code="stt_not_started", message="STT session not started."
+            )
+        if not buffer:
+            return
+        segment_processing = True
+        try:
+            stt_result = await anyio.to_thread.run_sync(stt_session.finish)
+        except SpeechError as exc:
+            if exc.code in {"stt_empty", "stt_no_match"}:
+                try:
+                    stt_result = await anyio.to_thread.run_sync(
+                        SpeechToTextService().transcribe,
+                        bytes(buffer),
+                        None,
+                        content_type,
+                    )
+                except SpeechError as exc_fallback:
+                    if exc_fallback.code in {"stt_empty", "stt_no_match"}:
+                        await websocket.send_json(
+                            {
+                                "event": "result",
+                                "transcript": "",
+                                "reply_text": NO_MATCH_REPLY,
+                                "audio_format": "wav",
+                                "reply_audio_base64": None,
+                                "timings_ms": {"stt": 0, "llm": 0, "tts": 0, "total": 0},
+                            }
+                        )
+                        buffer.clear()
+                        stt_session = SpeechToTextService().start_streaming(
+                            end_silence_ms=1200, initial_silence_ms=5000
+                        )
+                        vad_stream = SileroVADStream()
+                        return
+                    raise
+            else:
+                raise
+
+        await websocket.send_json(
+            {"event": "transcript", "transcript": stt_result.transcript}
+        )
+        pipeline = VoicePipeline()
+        result = await pipeline.run(
+            audio_bytes=bytes(buffer),
+            filename=None,
+            content_type=content_type,
+            prompt=prompt,
+            return_audio=return_audio,
+            transcript_override=stt_result.transcript,
+            language_override=stt_result.language,
+            llm_provider=llm_provider,
+        )
+        response_body = {
+            "event": "result",
+            "transcript": result.transcript,
+            "reply_text": result.reply_text,
+            "audio_format": "wav",
+            "reply_audio_base64": None,
+            "timings_ms": result.timings_ms,
+        }
+        log.info(
+            "voice_stream_complete",
+            bytes_received=len(buffer),
+            timings_ms=result.timings_ms,
+            return_audio=return_audio,
+            content_type=content_type,
+            frames_sent=frames_sent,
+            avg_rms=avg_rms,
+        )
+        await websocket.send_json(response_body)
+        if result.reply_audio and return_audio:
+            await websocket.send_bytes(result.reply_audio)
+        buffer.clear()
+        stt_session = SpeechToTextService().start_streaming(
+            end_silence_ms=1200, initial_silence_ms=5000
+        )
+        vad_stream = SileroVADStream()
+        segment_processing = False
 
     try:
         while True:
@@ -123,6 +209,10 @@ async def voice_stream(websocket: WebSocket) -> None:
                 if stt_session is not None:
                     stt_session.write(chunk)
                 buffer.extend(chunk)
+                if vad_stream is not None and not segment_processing:
+                    decision = vad_stream.update(chunk)
+                    if decision.speech_ended:
+                        await _finalize_segment()
                 if len(buffer) > MAX_FILE_SIZE_BYTES:
                     raise ValidationError(
                         code="file_too_large", message="Stream exceeds 15MB limit."
@@ -156,6 +246,7 @@ async def voice_stream(websocket: WebSocket) -> None:
                     stt_session = SpeechToTextService().start_streaming(
                         end_silence_ms=1200, initial_silence_ms=5000
                     )
+                    vad_stream = SileroVADStream()
                     continue
 
                 if event == "stop":
@@ -183,81 +274,12 @@ async def voice_stream(websocket: WebSocket) -> None:
                                 "LLM provider must be 'foundry_agent' or 'azure_openai'."
                             ),
                         )
-                    try:
-                        stt_result = await anyio.to_thread.run_sync(
-                            stt_session.finish
-                        )
-                    except SpeechError as exc:
-                        if exc.code in {"stt_empty", "stt_no_match"}:
-                            try:
-                                stt_result = await anyio.to_thread.run_sync(
-                                    SpeechToTextService().transcribe,
-                                    bytes(buffer),
-                                    None,
-                                    content_type,
-                                )
-                            except SpeechError as exc_fallback:
-                                if exc_fallback.code in {"stt_empty", "stt_no_match"}:
-                                    await websocket.send_json(
-                                        {
-                                            "event": "result",
-                                            "transcript": "",
-                                            "reply_text": NO_MATCH_REPLY,
-                                            "audio_format": "wav",
-                                            "reply_audio_base64": None,
-                                            "timings_ms": {"stt": 0, "llm": 0, "tts": 0, "total": 0},
-                                        }
-                                    )
-                                    buffer.clear()
-                                    break
-                                raise
-                        else:
-                            raise
-                    await websocket.send_json(
-                        {"event": "transcript", "transcript": stt_result.transcript}
-                    )
-                    pipeline = VoicePipeline()
-                    result = await pipeline.run(
-                        audio_bytes=bytes(buffer),
-                        filename=None,
-                        content_type=content_type,
-                        prompt=prompt,
-                        return_audio=return_audio,
-                        transcript_override=stt_result.transcript,
-                        language_override=stt_result.language,
-                        llm_provider=llm_provider,
-                    )
-                    response_body = {
-                        "event": "result",
-                        "transcript": result.transcript,
-                        "reply_text": result.reply_text,
-                        "audio_format": "wav",
-                        "reply_audio_base64": None,
-                        "timings_ms": result.timings_ms,
-                    }
-                    log.info(
-                        "voice_stream_complete",
-                        bytes_received=len(buffer),
-                        timings_ms=result.timings_ms,
-                        return_audio=return_audio,
-                        content_type=content_type,
-                        frames_sent=frames_sent,
-                        avg_rms=avg_rms,
-                    )
-                    await websocket.send_json(response_body)
-                    if result.reply_audio and return_audio:
-                        await websocket.send_bytes(result.reply_audio)
-                    buffer.clear()
+                    await _finalize_segment()
                     break
 
                 if event == "segment_end":
                     if not buffer:
                         continue
-                    if stt_session is None:
-                        raise ValidationError(
-                            code="stt_not_started",
-                            message="STT session not started.",
-                        )
                     prompt = payload.get("prompt", prompt)
                     return_audio = payload.get("return_audio", return_audio)
                     llm_provider = payload.get("llm_provider", llm_provider)
@@ -273,77 +295,21 @@ async def voice_stream(websocket: WebSocket) -> None:
                                 "LLM provider must be 'foundry_agent' or 'azure_openai'."
                             ),
                         )
-                    try:
-                        stt_result = await anyio.to_thread.run_sync(
-                            stt_session.finish
+                    if vad_stream is not None and not vad_stream.has_speech():
+                        await websocket.send_json(
+                            {
+                                "event": "result",
+                                "transcript": "",
+                                "reply_text": NO_MATCH_REPLY,
+                                "audio_format": "wav",
+                                "reply_audio_base64": None,
+                                "timings_ms": {"stt": 0, "llm": 0, "tts": 0, "total": 0},
+                            }
                         )
-                    except SpeechError as exc:
-                        if exc.code in {"stt_empty", "stt_no_match"}:
-                            try:
-                                stt_result = await anyio.to_thread.run_sync(
-                                    SpeechToTextService().transcribe,
-                                    bytes(buffer),
-                                    None,
-                                    content_type,
-                                )
-                            except SpeechError as exc_fallback:
-                                if exc_fallback.code in {"stt_empty", "stt_no_match"}:
-                                    await websocket.send_json(
-                                        {
-                                            "event": "result",
-                                            "transcript": "",
-                                            "reply_text": NO_MATCH_REPLY,
-                                            "audio_format": "wav",
-                                            "reply_audio_base64": None,
-                                            "timings_ms": {"stt": 0, "llm": 0, "tts": 0, "total": 0},
-                                        }
-                                    )
-                                    buffer.clear()
-                                    stt_session = SpeechToTextService().start_streaming(
-                                        end_silence_ms=1200, initial_silence_ms=5000
-                                    )
-                                    continue
-                                raise
-                        else:
-                            raise
-                    await websocket.send_json(
-                        {"event": "transcript", "transcript": stt_result.transcript}
-                    )
-                    pipeline = VoicePipeline()
-                    result = await pipeline.run(
-                        audio_bytes=bytes(buffer),
-                        filename=None,
-                        content_type=content_type,
-                        prompt=prompt,
-                        return_audio=return_audio,
-                        transcript_override=stt_result.transcript,
-                        language_override=stt_result.language,
-                        llm_provider=llm_provider,
-                    )
-                    response_body = {
-                        "event": "result",
-                        "transcript": result.transcript,
-                        "reply_text": result.reply_text,
-                        "audio_format": "wav",
-                        "reply_audio_base64": None,
-                        "timings_ms": result.timings_ms,
-                    }
-                    log.info(
-                        "voice_stream_complete",
-                        bytes_received=len(buffer),
-                        timings_ms=result.timings_ms,
-                        return_audio=return_audio,
-                        content_type=content_type,
-                        frames_sent=frames_sent,
-                        avg_rms=avg_rms,
-                    )
-                    await websocket.send_json(response_body)
-                    if result.reply_audio and return_audio:
-                        await websocket.send_bytes(result.reply_audio)
-                    buffer.clear()
-                    stt_session = SpeechToTextService().start_streaming(
-                        end_silence_ms=1200, initial_silence_ms=5000
-                    )
+                        buffer.clear()
+                        vad_stream.reset()
+                        continue
+                    await _finalize_segment()
                     continue
 
                 raise ValidationError(

src/app/services/vad.py

@@ -0,0 +1,109 @@
+"""Voice activity detection using Silero VAD (ONNX)."""
+from __future__ import annotations
+
+from array import array
+from dataclasses import dataclass
+
+from silero_vad_lite import SileroVAD
+
+
+@dataclass
+class VADDecision:
+    speech_started: bool
+    speech_ended: bool
+    speech_ms: int
+    silence_ms: int
+
+
+class SileroVADStream:
+    """Streaming VAD state machine for 16kHz mono PCM."""
+
+    def __init__(
+        self,
+        sample_rate: int = 16000,
+        speech_threshold: float = 0.8,
+        min_speech_ms: int = 600,
+        end_silence_ms: int = 1400,
+        min_speech_frames: int = 2,
+        min_silence_frames: int = 3,
+        prob_smoothing: float = 0.5,
+    ) -> None:
+        self._sample_rate = sample_rate
+        self._frame_samples = 512  # 32ms @ 16kHz
+        self._frame_bytes = self._frame_samples * 2  # int16
+        self._vad = SileroVAD(sample_rate=sample_rate)
+        self._speech_threshold = speech_threshold
+        self._min_speech_ms = min_speech_ms
+        self._end_silence_ms = end_silence_ms
+        self._min_speech_frames = min_speech_frames
+        self._min_silence_frames = min_silence_frames
+        self._prob_smoothing = prob_smoothing
+
+        self._buffer = bytearray()
+        self._in_speech = False
+        self._speech_ms = 0
+        self._silence_ms = 0
+        self._speech_frames = 0
+        self._silence_frames = 0
+        self._prob_ema = 0.0
+
+    def reset(self) -> None:
+        self._buffer.clear()
+        self._in_speech = False
+        self._speech_ms = 0
+        self._silence_ms = 0
+        self._speech_frames = 0
+        self._silence_frames = 0
+        self._prob_ema = 0.0
+
+    def has_speech(self) -> bool:
+        return self._speech_ms >= self._min_speech_ms
+
+    def update(self, pcm_bytes: bytes) -> VADDecision:
+        """Feed PCM bytes and return VAD decision for the latest frames."""
+        self._buffer.extend(pcm_bytes)
+        speech_started = False
+        speech_ended = False
+
+        while len(self._buffer) >= self._frame_bytes:
+            frame = self._buffer[: self._frame_bytes]
+            del self._buffer[: self._frame_bytes]
+
+            samples = array("h", frame)
+            float32 = [s / 32768.0 for s in samples]
+            prob = self._vad.process(float32)
+            self._prob_ema = (
+                self._prob_ema * self._prob_smoothing
+                + prob * (1.0 - self._prob_smoothing)
+            )
+
+            if self._prob_ema >= self._speech_threshold:
+                self._speech_frames += 1
+                self._silence_frames = 0
+                if not self._in_speech and self._speech_frames >= self._min_speech_frames:
+                    speech_started = True
+                    self._in_speech = True
+                    self._speech_ms = 0
+                if self._in_speech:
+                    self._speech_ms += 32
+                    self._silence_ms = 0
+            else:
+                self._silence_frames += 1
+                self._speech_frames = 0
+                if self._in_speech:
+                    self._silence_ms += 32
+                    if (
+                        self._speech_ms >= self._min_speech_ms
+                        and self._silence_ms >= self._end_silence_ms
+                        and self._silence_frames >= self._min_silence_frames
+                    ):
+                        speech_ended = True
+                        self._in_speech = False
+                        self._silence_ms = 0
+
+        return VADDecision(
+            speech_started=speech_started,
+            speech_ended=speech_ended,
+            speech_ms=self._speech_ms,
+            silence_ms=self._silence_ms,
+        )

ui/streamlit_app.py

@@ -939,21 +939,7 @@ html = """
               performance.now() - lastVoiceAt > SILENCE_MS
             ) {
               segmentInFlight = true;
-              sendEnabled = false;
               setState('thinking');
-              const avgRms = rmsCount ? rmsSum / rmsCount : 0;
-              ws.send(JSON.stringify({
-                event: 'segment_end',
-                prompt: 'Answer briefly.',
-                frames_sent: framesSent,
-                avg_rms: avgRms,
-                llm_provider: llmProvider
-              }));
-              framesSent = 0;
-              rmsSum = 0;
-              rmsCount = 0;
-              hadVoice = false;
-              lastVoiceAt = performance.now();
             }
           };
           source.connect(processor);
@@ -993,6 +979,9 @@ html = """
             sendEnabled = !isMuted;
             hadVoice = false;
             lastVoiceAt = performance.now();
+            framesSent = 0;
+            rmsSum = 0;
+            rmsCount = 0;
             if (data.transcript) {
               const last = messages[messages.length - 1];
               if (!last || last.role !== 'user' || last.text !== data.transcript) {
@@ -1019,6 +1008,9 @@ html = """
             sendEnabled = !isMuted;
             hadVoice = false;
             lastVoiceAt = performance.now();
+            framesSent = 0;
+            rmsSum = 0;
+            rmsCount = 0;
             if (isMuted && ws) ws.close();
           }
         };