Upload 10 files

Dockerfile

@@ -0,0 +1,60 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# Dockerfile for HuggingFace Spaces (Docker SDK)
+#
+# Hardware target: CPU Space (free tier) or GPU Space (T4-small recommended).
+# Port 7860 is the HF Spaces standard.
+#
+# Build order:
+#   1. System deps (libsndfile for soundfile, git for pip installs)
+#   2. Python packages (cached layer)
+#   3. App code (invalidates only on source changes)
+#   4. Switch to non-root user (HF Spaces requirement)
+# ─────────────────────────────────────────────────────────────────────────────
+
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# ── System dependencies ───────────────────────────────────────────────────── #
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      libsndfile1 \
+      libgomp1 \
+      git \
+    && rm -rf /var/lib/apt/lists/*
+
+# ── Python dependencies (cached unless requirements.txt changes) ──────────── #
+COPY requirements.txt .
+
+# Install CPU-only torch first (saves ~1 GB over default CUDA wheel).
+# Comment these two lines out and use the plain pip install if you have a GPU Space.
+RUN pip install --no-cache-dir \
+      torch==2.5.1+cpu torchaudio==2.5.1+cpu \
+      --index-url https://download.pytorch.org/whl/cpu
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+# ── Application code ─────────────────────────────────────────────────────── #
+COPY . .
+
+# ── Non-root user for HF Spaces compatibility ─────────────────────────────── #
+RUN useradd -m -u 1000 appuser \
+    && chown -R appuser:appuser /app
+USER 1000
+
+# ── Model cache lives in the user home so HF Spaces can persist it ────────── #
+ENV HF_HOME=/home/appuser/.cache/huggingface
+ENV TRANSFORMERS_CACHE=/home/appuser/.cache/huggingface/hub
+ENV PORT=7860
+
+EXPOSE 7860
+
+# ── Healthcheck ───────────────────────────────────────────────────────────── #
+HEALTHCHECK --interval=60s --timeout=10s --start-period=300s \
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:7860/health')"
+
+CMD ["python", "-m", "uvicorn", "app:app", \
+     "--host", "0.0.0.0", \
+     "--port", "7860", \
+     "--log-level", "info", \
+     "--ws-ping-interval", "30", \
+     "--ws-ping-timeout", "60"]

README.md

@@ -0,0 +1,138 @@
+---
+title: Open-Source Voice Agent
+emoji: 🎤
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+app_port: 7860
+pinned: false
+license: apache-2.0
+---
+
+# 🎤 Open-Source Voice Agent
+
+End-to-end English voice agent built entirely from open-source HuggingFace
+models with a **streaming overlap pipeline** — TTS synthesis for sentence N
+starts the moment sentence N is detected in the LLM token stream, while the
+model continues generating sentences N+1, N+2 … in parallel.
+
+---
+
+## Models
+
+| Stage | Model | Params | Device |
+|-------|-------|--------|--------|
+| VAD   | [silero-vad](https://github.com/snakers4/silero-vad) | 2 MB | CPU |
+| STT   | [openai/whisper-base.en](https://huggingface.co/openai/whisper-base.en) | 74 M | GPU / CPU |
+| LLM   | [HuggingFaceTB/SmolLM2-1.7B-Instruct](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct) | 1.7 B | GPU / CPU |
+| TTS   | [facebook/mms-tts-eng](https://huggingface.co/facebook/mms-tts-eng) | ~430 M | CPU |
+
+> **Lighter CPU-only alternative**: swap STT → `whisper-tiny.en` (39 M) and
+> LLM → `SmolLM2-360M-Instruct` (360 M) in `models.py`.
+
+---
+
+## Architecture
+
+```
+Browser mic  (16 kHz PCM)
+     │
+     ▼  WebSocket binary frames
+Silero VAD  ──────────────────────► discard noise/silence
+     │ speech segment detected
+     ▼
+Whisper base.en  ──────────────────► transcript text
+     │
+     ▼
+SmolLM2-1.7B  (TextIteratorStreamer)
+     │  token stream
+     ▼
+SentenceBuffer  ──► complete sentence
+     │                    │
+     │          MMS-TTS-eng  (CPU executor)   ← overlap: LLM still generating!
+     │                    │
+     │          PCM bytes ──► ws.send_bytes()
+     │                    │
+     ◄────────────────────┘  repeat until stream ends
+     │
+     ▼
+Browser  AudioQueue  ──► AudioContext playback
+```
+
+**Streaming overlap benefit**: while the browser plays sentence 1, the server
+is already synthesising sentence 2. This removes the full TTS latency from the
+inter-sentence gap, giving noticeably more natural turn-taking.
+
+---
+
+## Project structure
+
+```
+sts-pipeline/
+├── app.py                      FastAPI server + WebSocket handler
+├── models.py                   All model loading & inference (STT/LLM/TTS/VAD)
+├── pipeline.py                 Streaming overlap pipeline
+├── sentence_buffer.py          Token-stream → sentence boundary detector
+├── requirements.txt
+├── Dockerfile
+└── static/
+    ├── index.html              Browser UI
+    ├── app.js                  WebSocket client + audio queue
+    └── audio-capture-worklet.js  Mic capture @ 16 kHz (AudioWorklet)
+```
+
+---
+
+## Local development
+
+```bash
+# 1. Clone
+git clone https://huggingface.co/spaces/<your-user>/voice-agent
+cd voice-agent
+
+# 2. Install (GPU)
+pip install -r requirements.txt
+
+# 2b. Install (CPU-only)
+pip install torch==2.5.1+cpu torchaudio==2.5.1+cpu \
+    --index-url https://download.pytorch.org/whl/cpu
+pip install -r requirements.txt
+
+# 3. Run
+python app.py        # or: uvicorn app:app --port 7860
+# Open http://localhost:7860
+```
+
+Models are downloaded automatically on first run (~3 GB total) and cached in
+`~/.cache/huggingface/hub`.
+
+---
+
+## HuggingFace Spaces deployment
+
+1. Create a new Space → **Docker** SDK.
+2. Push this repo as-is.
+3. The Space will build the Docker image and serve on port 7860.
+4. For GPU hardware: remove the CPU-only torch lines in `Dockerfile` and
+   uncomment the plain `pip install -r requirements.txt`.
+
+---
+
+## WebSocket protocol reference
+
+| Direction | Frame type | Payload |
+|-----------|-----------|---------|
+| Client → Server | binary | PCM int16, mono, 16 kHz, 640 B chunks |
+| Client → Server | text | `{"type":"start"}` or `{"type":"stop"}` |
+| Server → Client | binary | PCM int16, mono, 16 kHz (TTS audio) |
+| Server → Client | text | `{"type":"transcript","text":"..."}` |
+| Server → Client | text | `{"type":"agent_start"}` |
+| Server → Client | text | `{"type":"agent_done","text":"...","latency_ms":000}` |
+| Server → Client | text | `{"type":"error","message":"..."}` |
+
+---
+
+## License
+
+Apache 2.0 — all component models carry their own licenses; see their
+respective HuggingFace model cards for terms of use.

app.js

@@ -0,0 +1,336 @@
+/**
+ * app.js
+ * ------
+ * Browser-side voice agent client.
+ *
+ * Flow
+ * ----
+ *  getUserMedia  ──►  AudioWorklet capture (16 kHz int16)
+ *                          │
+ *                    WebSocket.send(binary)
+ *                          │
+ *                    ◄──── Server ─────►
+ *                          │
+ *              WebSocket.onmessage(binary)  ← PCM int16 TTS audio
+ *                          │
+ *                    playback queue  ──►  AudioContext.playBuffer()
+ *
+ * States:  idle → listening → processing → speaking → idle
+ */
+
+'use strict';
+
+// ── Config ──────────────────────────────────────────────────────────────── //
+const SERVER_SR   = 16_000;   // Server sends 16 kHz PCM
+const WS_URL      = (() => {
+  const proto = location.protocol === 'https:' ? 'wss:' : 'ws:';
+  return `${proto}//${location.host}/ws/audio`;
+})();
+
+// ── State ────────────────────────────────────────────────────────────────── //
+let audioCtx        = null;
+let captureNode     = null;
+let micStream       = null;
+let ws              = null;
+let appState        = 'idle';   // idle | listening | processing | speaking
+
+let audioQueue      = [];       // Array<AudioBuffer> pending playback
+let isPlaying       = false;    // Is the drain loop running?
+let conversation    = [];       // [{role, text}] for UI transcript
+
+// ── DOM refs ─────────────────────────────────────────────────────────────── //
+const btnToggle     = document.getElementById('btn-toggle');
+const statusDot     = document.getElementById('status-dot');
+const statusText    = document.getElementById('status-text');
+const canvasEl      = document.getElementById('visualiser');
+const ctx2d         = canvasEl.getContext('2d');
+const transcriptEl  = document.getElementById('transcript');
+const latencyEl     = document.getElementById('latency');
+
+// ── Entry point ───────────────────────────────────────────────────────────── //
+btnToggle.addEventListener('click', () => {
+  if (appState === 'idle') {
+    startSession();
+  } else {
+    stopSession();
+  }
+});
+
+// ── Session start/stop ───────────────────────────────────────────────────── //
+async function startSession () {
+  try {
+    await initAudio();
+    await initWebSocket();
+    setState('listening');
+    btnToggle.textContent = '⏹ Stop';
+    ws.send(JSON.stringify({ type: 'start' }));
+    startVisualiser();
+  } catch (err) {
+    console.error('startSession:', err);
+    setStatus('error', `Mic / WebSocket error: ${err.message}`);
+  }
+}
+
+function stopSession () {
+  if (ws) {
+    ws.send(JSON.stringify({ type: 'stop' }));
+    ws.close();
+  }
+  teardownAudio();
+  setState('idle');
+  btnToggle.textContent = '🎤 Start';
+  stopVisualiser();
+}
+
+// ── Audio setup ───────────────────────────────────────────────────────────── //
+async function initAudio () {
+  // Request mic with constraints that help speech quality
+  micStream = await navigator.mediaDevices.getUserMedia({
+    audio: {
+      channelCount:    1,
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl:  true,
+    },
+  });
+
+  audioCtx = new AudioContext();
+
+  // Load the capture worklet
+  await audioCtx.audioWorklet.addModule('/audio-capture-worklet.js');
+
+  const source  = audioCtx.createMediaStreamSource(micStream);
+  captureNode   = new AudioWorkletNode(audioCtx, 'audio-capture-processor');
+
+  captureNode.port.onmessage = ({ data }) => {
+    if (data.type === 'audio' && ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(data.pcm);           // ArrayBuffer of int16 PCM
+    }
+  };
+
+  source.connect(captureNode);
+  // captureNode intentionally NOT connected to destination (no feedback)
+  captureNode.port.postMessage({ type: 'start' });
+}
+
+function teardownAudio () {
+  if (captureNode) {
+    captureNode.port.postMessage({ type: 'stop' });
+    captureNode.disconnect();
+    captureNode = null;
+  }
+  if (micStream) {
+    micStream.getTracks().forEach(t => t.stop());
+    micStream = null;
+  }
+  if (audioCtx) {
+    audioCtx.close();
+    audioCtx = null;
+  }
+  audioQueue  = [];
+  isPlaying   = false;
+}
+
+// ── WebSocket ───────────────────────────���─────────────────────────────────── //
+function initWebSocket () {
+  return new Promise((resolve, reject) => {
+    ws = new WebSocket(WS_URL);
+    ws.binaryType = 'arraybuffer';
+
+    ws.onopen    = () => { console.log('WS open'); resolve(); };
+    ws.onerror   = (e) => reject(new Error('WebSocket error'));
+    ws.onclose   = () => {
+      if (appState !== 'idle') stopSession();
+    };
+
+    ws.onmessage = (evt) => {
+      if (evt.data instanceof ArrayBuffer) {
+        onAudioChunk(evt.data);       // Binary: PCM audio from TTS
+      } else {
+        onControlMessage(JSON.parse(evt.data));   // Text: JSON status
+      }
+    };
+  });
+}
+
+// ── Inbound control messages ─────────────────────────────────────────────── //
+function onControlMessage (msg) {
+  switch (msg.type) {
+
+    case 'status':
+      setStatus(appState, msg.message);
+      break;
+
+    case 'transcript':
+      appendTranscript('user', msg.text);
+      setState('processing');
+      break;
+
+    case 'agent_start':
+      setState('speaking');
+      break;
+
+    case 'agent_done':
+      appendTranscript('agent', msg.text);
+      if (msg.latency_ms) {
+        latencyEl.textContent = `Latency: ${msg.latency_ms} ms`;
+      }
+      // State returns to 'listening' when the playback queue drains
+      break;
+
+    case 'interrupted':
+      audioQueue  = [];
+      isPlaying   = false;
+      setState('listening');
+      break;
+
+    case 'error':
+      console.error('Server error:', msg.message);
+      setStatus('error', msg.message);
+      setState('listening');
+      break;
+  }
+}
+
+// ── Audio playback queue ─────────────────────────────────────────────────── //
+function onAudioChunk (arrayBuffer) {
+  if (!audioCtx) return;
+
+  // PCM int16, mono, 16 kHz  →  float32 AudioBuffer at AudioContext.sampleRate
+  const int16        = new Int16Array(arrayBuffer);
+  const float32      = new Float32Array(int16.length);
+  for (let i = 0; i < int16.length; i++) float32[i] = int16[i] / 32_768;
+
+  const resampled    = resample(float32, SERVER_SR, audioCtx.sampleRate);
+  const audioBuf     = audioCtx.createBuffer(1, resampled.length, audioCtx.sampleRate);
+  audioBuf.copyToChannel(resampled, 0);
+
+  audioQueue.push(audioBuf);
+  if (!isPlaying) drainQueue();
+}
+
+function drainQueue () {
+  if (audioQueue.length === 0) {
+    isPlaying = false;
+    if (appState === 'speaking') setState('listening');
+    return;
+  }
+
+  isPlaying       = true;
+  const buf       = audioQueue.shift();
+  const src       = audioCtx.createBufferSource();
+  src.buffer      = buf;
+  src.connect(audioCtx.destination);
+  src.onended     = drainQueue;   // chain next buffer immediately
+  src.start();
+}
+
+// ── Resampler (linear interpolation) ─────────────────────────────────────── //
+function resample (input, inSR, outSR) {
+  if (inSR === outSR) return input;
+  const ratio     = inSR / outSR;
+  const outLen    = Math.round(input.length / ratio);
+  const output    = new Float32Array(outLen);
+  for (let i = 0; i < outLen; i++) {
+    const src     = i * ratio;
+    const lo      = Math.floor(src);
+    const hi      = Math.min(lo + 1, input.length - 1);
+    const frac    = src - lo;
+    output[i]     = input[lo] + frac * (input[hi] - input[lo]);
+  }
+  return output;
+}
+
+// ── State management ──────────────────────────────────────────────────────── //
+const STATE_LABELS = {
+  idle:       'Ready',
+  listening:  'Listening…',
+  processing: 'Thinking…',
+  speaking:   'Speaking…',
+};
+
+const STATE_COLORS = {
+  idle:       '#6b7280',
+  listening:  '#22c55e',
+  processing: '#f59e0b',
+  speaking:   '#3b82f6',
+};
+
+function setState (newState) {
+  appState = newState;
+  statusDot.style.background = STATE_COLORS[newState] ?? '#6b7280';
+  statusText.textContent     = STATE_LABELS[newState] ?? newState;
+}
+
+function setStatus (_state, message) {
+  statusText.textContent = message;
+}
+
+// ── Transcript ────────────────────────────────────────────────────────────── //
+function appendTranscript (role, text) {
+  conversation.push({ role, text });
+  const div   = document.createElement('div');
+  div.className = `msg msg-${role}`;
+  div.innerHTML = `<span class="msg-role">${role === 'user' ? 'You' : 'Agent'}</span><span class="msg-text">${escHtml(text)}</span>`;
+  transcriptEl.appendChild(div);
+  transcriptEl.scrollTop = transcriptEl.scrollHeight;
+}
+
+function escHtml (s) {
+  return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+}
+
+// ── Canvas visualiser ─────────────────────────────────────────────────────── //
+let analyser    = null;
+let visRAF      = null;
+
+function startVisualiser () {
+  if (!audioCtx || analyser) return;
+
+  analyser             = audioCtx.createAnalyser();
+  analyser.fftSize     = 256;
+  const source         = audioCtx.createMediaStreamSource(micStream);
+  source.connect(analyser);
+
+  const dataArr        = new Uint8Array(analyser.frequencyBinCount);
+  const W              = canvasEl.width;
+  const H              = canvasEl.height;
+  const BAR_COUNT      = 48;
+  const BAR_W          = W / BAR_COUNT - 1;
+
+  function draw () {
+    visRAF = requestAnimationFrame(draw);
+    analyser.getByteFrequencyData(dataArr);
+
+    ctx2d.clearRect(0, 0, W, H);
+
+    const color = STATE_COLORS[appState] ?? '#6b7280';
+    ctx2d.fillStyle = color + '99';   // 60% opacity
+
+    for (let i = 0; i < BAR_COUNT; i++) {
+      const binIdx = Math.floor(i * dataArr.length / BAR_COUNT);
+      const v      = dataArr[binIdx] / 255;
+      const barH   = Math.max(2, v * H * 0.9);
+      const x      = i * (BAR_W + 1);
+      const y      = (H - barH) / 2;
+      ctx2d.beginPath();
+      ctx2d.roundRect(x, y, BAR_W, barH, 2);
+      ctx2d.fill();
+    }
+  }
+  draw();
+}
+
+function stopVisualiser () {
+  if (visRAF) { cancelAnimationFrame(visRAF); visRAF = null; }
+  if (analyser) { analyser.disconnect(); analyser = null; }
+  ctx2d.clearRect(0, 0, canvasEl.width, canvasEl.height);
+}
+
+// ── Resize canvas ─────────────────────────────────────────────────────────── //
+function resizeCanvas () {
+  canvasEl.width  = canvasEl.offsetWidth;
+  canvasEl.height = canvasEl.offsetHeight;
+}
+window.addEventListener('resize', resizeCanvas);
+resizeCanvas();

app.py

@@ -0,0 +1,280 @@
+"""
+app.py
+------
+FastAPI server for the open-source streaming voice agent.
+
+All project files live in ONE directory — no subdirectories.
+
+Endpoints
+---------
+GET  /                         → index.html
+GET  /app.js                   → app.js
+GET  /audio-capture-worklet.js → audio-capture-worklet.js
+GET  /health                   → JSON status
+WS   /ws/audio                 → bidirectional audio stream
+
+WebSocket protocol
+------------------
+Client → Server:
+  • Binary frames : PCM int16, mono, 16 kHz, 640-byte chunks (20 ms)
+  • Text  frames  : JSON control  {"type": "start"} | {"type": "stop"}
+
+Server → Client:
+  • Binary frames : PCM int16, mono, 16 kHz (TTS output, variable size)
+  • Text  frames  : JSON status
+        {"type": "status",       "message": "..."}
+        {"type": "transcript",   "text": "..."}
+        {"type": "agent_start"}
+        {"type": "agent_done",   "text": "...", "latency_ms": 000}
+        {"type": "error",        "message": "..."}
+        {"type": "interrupted"}
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import uvicorn
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.responses import FileResponse, JSONResponse
+
+from models import STT_SR, ModelManager
+from pipeline import StreamingPipeline, build_initial_conversation
+
+# --------------------------------------------------------------------------- #
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s  %(levelname)-8s  %(name)s: %(message)s",
+    datefmt="%H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+BASE_DIR = Path(__file__).parent   # All files live here
+
+# --------------------------------------------------------------------------- #
+app = FastAPI(title="Open-Source Voice Agent", version="1.0.0")
+
+_models: ModelManager | None = None
+_pipeline: StreamingPipeline | None = None
+
+
+@app.on_event("startup")
+async def on_startup():
+    global _models, _pipeline
+    logger.info("=== Loading models — this may take a few minutes on first run ===")
+    loop = asyncio.get_event_loop()
+    _models = await loop.run_in_executor(None, ModelManager)
+    await loop.run_in_executor(None, _models.warm_up)
+    _pipeline = StreamingPipeline(_models)
+    logger.info("=== Server ready ===")
+
+
+# --------------------------------------------------------------------------- #
+#  Static file routes (flat — all files in same directory as app.py)          #
+# --------------------------------------------------------------------------- #
+@app.get("/")
+async def root():
+    return FileResponse(str(BASE_DIR / "index.html"))
+
+@app.get("/app.js")
+async def serve_appjs():
+    return FileResponse(str(BASE_DIR / "app.js"), media_type="application/javascript")
+
+@app.get("/audio-capture-worklet.js")
+async def serve_worklet():
+    return FileResponse(str(BASE_DIR / "audio-capture-worklet.js"), media_type="application/javascript")
+
+@app.get("/health")
+async def health():
+    return JSONResponse({
+        "status": "ready" if _models else "loading",
+        "device": str(_models.device) if _models else "N/A",
+        "stt": "whisper-base.en",
+        "llm": "SmolLM2-1.7B-Instruct",
+        "tts": "mms-tts-eng",
+    })
+
+
+# --------------------------------------------------------------------------- #
+#  VAD-based speech segmenter                                                  #
+# --------------------------------------------------------------------------- #
+class SpeechSegmenter:
+    """
+    Wraps Silero VADIterator to accumulate PCM and fire when speech ends.
+
+    Incoming audio is 640-byte chunks (320 int16 samples = 20 ms at 16 kHz).
+    Silero needs exactly 512 float32 samples per call, so we buffer the
+    difference and carry any leftover into the next iteration.
+    """
+
+    VAD_CHUNK = 512
+    MIN_SPEECH_S = 0.30
+
+    def __init__(self, models: ModelManager):
+        self._vad = models.vad_iter
+        self._vad.reset_states()
+        self._pre_roll: list[np.ndarray] = []
+        self._speech_buf: list[np.ndarray] = []
+        self._vad_leftover = np.array([], dtype=np.float32)
+        self._speaking = False
+
+    def feed(self, pcm_bytes: bytes) -> np.ndarray | None:
+        audio = (
+            np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32_768.0
+        )
+        self._pre_roll.append(audio)
+        if len(self._pre_roll) > 5:
+            self._pre_roll.pop(0)
+
+        if self._speaking:
+            self._speech_buf.append(audio)
+
+        combined = np.concatenate([self._vad_leftover, audio])
+        i = 0
+        result = None
+
+        while i + self.VAD_CHUNK <= len(combined):
+            chunk = combined[i : i + self.VAD_CHUNK]
+            i += self.VAD_CHUNK
+            try:
+                event = self._vad(
+                    torch.from_numpy(chunk).float(), return_seconds=False
+                )
+            except Exception:
+                event = None
+
+            if event:
+                if "start" in event and not self._speaking:
+                    self._speaking = True
+                    self._speech_buf = list(self._pre_roll)
+                elif "end" in event and self._speaking:
+                    self._speaking = False
+                    if self._speech_buf:
+                        speech = np.concatenate(self._speech_buf)
+                        if len(speech) / STT_SR >= self.MIN_SPEECH_S:
+                            result = speech
+                    self._speech_buf = []
+
+        self._vad_leftover = combined[i:]
+        return result
+
+    def reset(self):
+        self._pre_roll = []
+        self._speech_buf = []
+        self._vad_leftover = np.array([], dtype=np.float32)
+        self._speaking = False
+        self._vad.reset_states()
+
+
+# --------------------------------------------------------------------------- #
+#  WebSocket handler                                                           #
+# --------------------------------------------------------------------------- #
+@app.websocket("/ws/audio")
+async def ws_audio(ws: WebSocket):
+    await ws.accept()
+    logger.info("Client connected: %s", ws.client)
+
+    if _models is None or _pipeline is None:
+        await ws.send_text(json.dumps({
+            "type": "error", "message": "Models still loading, please wait."
+        }))
+        await ws.close()
+        return
+
+    loop = asyncio.get_event_loop()
+    segmenter = SpeechSegmenter(_models)
+    conversation = build_initial_conversation()
+
+    async def _send(obj: dict):
+        await ws.send_text(json.dumps(obj))
+
+    try:
+        while True:
+            msg = await ws.receive()
+
+            # ── control frame ─────────────────────────────────────────── #
+            if "text" in msg and msg["text"]:
+                try:
+                    ctrl = json.loads(msg["text"])
+                except json.JSONDecodeError:
+                    continue
+                if ctrl.get("type") == "start":
+                    segmenter.reset()
+                    _models.vad_reset()
+                    conversation = build_initial_conversation()
+                    logger.info("Session reset by client.")
+                    await _send({"type": "status", "message": "Session started."})
+                elif ctrl.get("type") == "stop":
+                    break
+                continue
+
+            # ── audio frame ───────────────────────────────────────────── #
+            if "bytes" not in msg or not msg["bytes"]:
+                continue
+
+            speech_audio = segmenter.feed(msg["bytes"])
+            if speech_audio is None:
+                continue
+
+            # ── transcribe ────────────────────────────────────────────── #
+            logger.info("Speech: %.2f s → transcribing…", len(speech_audio) / STT_SR)
+            await _send({"type": "status", "message": "Transcribing…"})
+
+            transcript: str = await loop.run_in_executor(
+                None, _models.transcribe, speech_audio
+            )
+
+            if not transcript or len(transcript.strip()) < 2:
+                await _send({"type": "status", "message": "Couldn't hear clearly. Try again."})
+                continue
+
+            logger.info("Transcript: '%s'", transcript)
+            await _send({"type": "transcript", "text": transcript})
+
+            # ── LLM + TTS streaming pipeline ──────────────────────────── #
+            conversation.append({"role": "user", "content": transcript})
+            await _send({"type": "agent_start"})
+
+            t0 = time.perf_counter()
+            try:
+                response_text = await _pipeline.process(conversation, ws, loop)
+            except Exception as exc:
+                logger.error("Pipeline error: %s", exc, exc_info=True)
+                await _send({"type": "error", "message": f"Pipeline error: {exc}"})
+                continue
+
+            latency_ms = int((time.perf_counter() - t0) * 1000)
+            conversation.append({"role": "assistant", "content": response_text})
+            await _send({"type": "agent_done", "text": response_text, "latency_ms": latency_ms})
+            logger.info("Turn done in %d ms.", latency_ms)
+
+    except WebSocketDisconnect:
+        logger.info("Client disconnected: %s", ws.client)
+    except Exception as exc:
+        logger.error("WS error: %s", exc, exc_info=True)
+        try:
+            await _send({"type": "error", "message": str(exc)})
+        except Exception:
+            pass
+    finally:
+        logger.info("WS session closed: %s", ws.client)
+
+
+# --------------------------------------------------------------------------- #
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 7860))
+    uvicorn.run(
+        "app:app",
+        host="0.0.0.0",
+        port=port,
+        log_level="info",
+        ws_ping_interval=30,
+        ws_ping_timeout=60,
+    )

audio-capture-worklet.js

@@ -0,0 +1,71 @@
+/**
+ * audio-capture-worklet.js
+ * ------------------------
+ * Runs in the AudioWorklet thread (dedicated audio thread, zero GC pauses).
+ *
+ * Responsibilities
+ * ----------------
+ * 1. Receive float32 PCM frames from the mic at AudioContext.sampleRate
+ *    (typically 44100 or 48000 Hz).
+ * 2. Downsample to TARGET_SR = 16000 Hz using averaged decimation
+ *    (basic anti-aliasing — good enough for speech).
+ * 3. Accumulate until CHUNK_SAMPLES (320 samples = 20 ms at 16 kHz) are
+ *    ready, then convert to Int16 and post to the main thread.
+ *
+ * Messages posted to main thread
+ * --------------------------------
+ *   { type: 'audio', pcm: ArrayBuffer }   — 640 bytes = 320 int16 samples
+ */
+
+const TARGET_SR      = 16_000;
+const CHUNK_SAMPLES  = 320;   // 20 ms at 16 kHz → 640 bytes as Int16
+
+class AudioCaptureProcessor extends AudioWorkletProcessor {
+  constructor () {
+    super();
+    this._buf    = [];          // Downsampled float32 accumulator
+    this._active = false;
+
+    this.port.onmessage = ({ data }) => {
+      if (data.type === 'start') this._active = true;
+      if (data.type === 'stop')  this._active = false;
+    };
+  }
+
+  process (inputs) {
+    if (!this._active) return true;
+
+    const channel = inputs[0]?.[0];
+    if (!channel || channel.length === 0) return true;
+
+    // ── Downsample: average samples within each output window ───────── //
+    const ratio      = sampleRate / TARGET_SR;   // e.g. 48000/16000 = 3.0
+    const outCount   = Math.floor(channel.length / ratio);
+
+    for (let i = 0; i < outCount; i++) {
+      const srcStart = i * ratio;
+      const srcEnd   = srcStart + ratio;
+      let   sum      = 0;
+      let   n        = 0;
+      for (let j = Math.floor(srcStart); j < Math.min(Math.ceil(srcEnd), channel.length); j++) {
+        sum += channel[j];
+        n++;
+      }
+      this._buf.push(n > 0 ? sum / n : 0);
+    }
+
+    // ── Emit CHUNK_SAMPLES at a time ─────────────────────────────────── //
+    while (this._buf.length >= CHUNK_SAMPLES) {
+      const slice  = this._buf.splice(0, CHUNK_SAMPLES);
+      const int16  = new Int16Array(CHUNK_SAMPLES);
+      for (let i = 0; i < CHUNK_SAMPLES; i++) {
+        int16[i] = Math.max(-32_768, Math.min(32_767, Math.round(slice[i] * 32_767)));
+      }
+      this.port.postMessage({ type: 'audio', pcm: int16.buffer }, [int16.buffer]);
+    }
+
+    return true;  // keep processor alive
+  }
+}
+
+registerProcessor('audio-capture-processor', AudioCaptureProcessor);

index.html

@@ -0,0 +1,335 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8"/>
+<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+<title>Open-Source Voice Agent</title>
+<style>
+/* ── Reset & base ────────────────────────────────────────────────────────── */
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+:root {
+  --bg0:      #0f1117;
+  --bg1:      #161b27;
+  --bg2:      #1e2436;
+  --bg3:      #252d42;
+  --border:   #2e3a52;
+  --text0:    #e8eaf0;
+  --text1:    #9aa3b8;
+  --text2:    #5c677e;
+  --accent:   #4f8ef7;
+  --green:    #22c55e;
+  --amber:    #f59e0b;
+  --blue:     #3b82f6;
+  --red:      #ef4444;
+  --radius:   12px;
+  --font:     'Inter', system-ui, -apple-system, sans-serif;
+}
+
+body {
+  background: var(--bg0);
+  color: var(--text0);
+  font-family: var(--font);
+  min-height: 100vh;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+/* ── Layout ──────────────────────────────────────────────────────────────── */
+.page {
+  width: 100%;
+  max-width: 680px;
+  padding: 24px 16px 48px;
+  display: flex;
+  flex-direction: column;
+  gap: 20px;
+}
+
+/* ── Header ──────────────────────────────────────────────────────────────── */
+.header { text-align: center; padding: 8px 0 4px; }
+.header h1 { font-size: 22px; font-weight: 600; letter-spacing: -0.3px; }
+.header p  { font-size: 13px; color: var(--text1); margin-top: 6px; }
+
+.badges {
+  display: flex; justify-content: center; gap: 8px; flex-wrap: wrap;
+  margin-top: 10px;
+}
+.badge {
+  font-size: 11px; font-weight: 500;
+  padding: 3px 10px;
+  border-radius: 20px;
+  border: 1px solid var(--border);
+  color: var(--text1);
+  background: var(--bg2);
+}
+.badge span { color: var(--accent); }
+
+/* ── Card ────────────────────────────────────────────────────────────────── */
+.card {
+  background: var(--bg1);
+  border: 1px solid var(--border);
+  border-radius: var(--radius);
+  overflow: hidden;
+}
+
+/* ── Visualiser ──────────────────────────────────────────────────────────── */
+.vis-wrap {
+  padding: 0;
+  background: var(--bg0);
+  border-bottom: 1px solid var(--border);
+}
+#visualiser {
+  display: block;
+  width: 100%;
+  height: 90px;
+}
+
+/* ── Controls ────────────────────────────────────────────────────────────── */
+.controls {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  gap: 14px;
+  padding: 24px 20px 20px;
+}
+
+#btn-toggle {
+  width: 120px; height: 120px;
+  border-radius: 50%;
+  border: 2px solid var(--border);
+  background: var(--bg2);
+  color: var(--text0);
+  font-size: 32px;
+  cursor: pointer;
+  transition: background .15s, transform .1s, border-color .15s;
+  display: flex; align-items: center; justify-content: center;
+  flex-direction: column;
+  gap: 4px;
+  line-height: 1;
+}
+#btn-toggle:hover { background: var(--bg3); border-color: var(--accent); }
+#btn-toggle:active { transform: scale(0.95); }
+#btn-toggle.active {
+  background: #1a2440;
+  border-color: var(--blue);
+  box-shadow: 0 0 0 4px rgba(79,142,247,0.15);
+}
+#btn-label { font-size: 11px; color: var(--text1); font-weight: 500; }
+
+/* ── Status bar ──────────────────────────────────────────────────────────── */
+.status-bar {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 10px 20px;
+  background: var(--bg2);
+  border-top: 1px solid var(--border);
+  justify-content: space-between;
+}
+.status-left { display: flex; align-items: center; gap: 8px; }
+#status-dot {
+  width: 8px; height: 8px; border-radius: 50%;
+  background: #6b7280;
+  transition: background .3s;
+  flex-shrink: 0;
+}
+#status-dot.pulse { animation: pulse 1.2s infinite; }
+@keyframes pulse {
+  0%, 100% { opacity: 1; }
+  50% { opacity: 0.4; }
+}
+#status-text { font-size: 13px; color: var(--text1); }
+#latency { font-size: 12px; color: var(--text2); }
+
+/* ── Transcript ──────────────────────────────────────────────────────────── */
+.transcript-header {
+  padding: 12px 16px 8px;
+  font-size: 11px;
+  font-weight: 600;
+  letter-spacing: .06em;
+  text-transform: uppercase;
+  color: var(--text2);
+  border-bottom: 1px solid var(--border);
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+#btn-clear {
+  font-size: 11px; color: var(--text2);
+  background: none; border: none; cursor: pointer;
+  padding: 2px 6px; border-radius: 4px;
+}
+#btn-clear:hover { color: var(--text1); background: var(--bg3); }
+
+#transcript {
+  min-height: 120px;
+  max-height: 320px;
+  overflow-y: auto;
+  padding: 12px 16px;
+  display: flex;
+  flex-direction: column;
+  gap: 10px;
+  scroll-behavior: smooth;
+}
+#transcript:empty::before {
+  content: 'Conversation will appear here…';
+  color: var(--text2);
+  font-size: 13px;
+}
+
+.msg { display: flex; flex-direction: column; gap: 3px; }
+.msg-role {
+  font-size: 10px;
+  font-weight: 600;
+  letter-spacing: .05em;
+  text-transform: uppercase;
+}
+.msg-text { font-size: 14px; line-height: 1.55; color: var(--text0); }
+
+.msg-user   .msg-role { color: var(--green); }
+.msg-agent  .msg-role { color: var(--blue); }
+.msg-user   .msg-text { color: var(--text0); }
+.msg-agent  .msg-text { color: var(--text0); }
+
+/* ── Model info footer ───────────────────────────────────────────────────── */
+.model-grid {
+  display: grid;
+  grid-template-columns: repeat(3, 1fr);
+  gap: 1px;
+  background: var(--border);
+  border: 1px solid var(--border);
+  border-radius: var(--radius);
+  overflow: hidden;
+}
+.model-cell {
+  background: var(--bg1);
+  padding: 12px 14px;
+  text-align: center;
+}
+.model-cell .mc-label {
+  font-size: 10px;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: .06em;
+  color: var(--text2);
+  margin-bottom: 4px;
+}
+.model-cell .mc-name { font-size: 12px; font-weight: 500; color: var(--text0); }
+.model-cell .mc-size { font-size: 11px; color: var(--text2); margin-top: 2px; }
+
+/* ── Scrollbar ───────────────────────────────────────────────────────────── */
+::-webkit-scrollbar { width: 4px; }
+::-webkit-scrollbar-track { background: transparent; }
+::-webkit-scrollbar-thumb { background: var(--bg3); border-radius: 2px; }
+</style>
+</head>
+<body>
+<div class="page">
+
+  <!-- Header -->
+  <div class="header">
+    <h1>🎤 Open-Source Voice Agent</h1>
+    <p>100% open-source models · streaming overlap pipeline · English</p>
+    <div class="badges">
+      <div class="badge"><span>STT</span> Whisper base.en</div>
+      <div class="badge"><span>LLM</span> SmolLM2-1.7B</div>
+      <div class="badge"><span>TTS</span> MMS-TTS-eng</div>
+      <div class="badge"><span>VAD</span> Silero</div>
+    </div>
+  </div>
+
+  <!-- Voice card -->
+  <div class="card">
+    <!-- Waveform visualiser -->
+    <div class="vis-wrap">
+      <canvas id="visualiser"></canvas>
+    </div>
+
+    <!-- Mic button -->
+    <div class="controls">
+      <button id="btn-toggle" aria-label="Start recording">
+        <span id="btn-icon">🎤</span>
+        <span id="btn-label">START</span>
+      </button>
+    </div>
+
+    <!-- Status bar -->
+    <div class="status-bar">
+      <div class="status-left">
+        <div id="status-dot"></div>
+        <span id="status-text">Ready — click Start to begin</span>
+      </div>
+      <span id="latency"></span>
+    </div>
+  </div>
+
+  <!-- Transcript -->
+  <div class="card">
+    <div class="transcript-header">
+      Conversation
+      <button id="btn-clear">Clear</button>
+    </div>
+    <div id="transcript"></div>
+  </div>
+
+  <!-- Model info -->
+  <div class="model-grid">
+    <div class="model-cell">
+      <div class="mc-label">Speech-to-Text</div>
+      <div class="mc-name">Whisper base.en</div>
+      <div class="mc-size">74 M params · GPU/CPU</div>
+    </div>
+    <div class="model-cell">
+      <div class="mc-label">Language Model</div>
+      <div class="mc-name">SmolLM2-1.7B-Instruct</div>
+      <div class="mc-size">1.7 B params · GPU/CPU</div>
+    </div>
+    <div class="model-cell">
+      <div class="mc-label">Text-to-Speech</div>
+      <div class="mc-name">MMS-TTS-eng</div>
+      <div class="mc-size">VITS · 16 kHz · CPU</div>
+    </div>
+  </div>
+
+</div>
+
+<script src="/app.js" defer></script>
+<script>
+  // Wire up the Clear button after DOM is ready
+  document.getElementById('btn-clear').addEventListener('click', () => {
+    document.getElementById('transcript').innerHTML = '';
+  });
+
+  // Reflect active state on button
+  const _setState_orig = window.setState;
+  // We patch setState in app.js via a MutationObserver hack instead:
+  const _btnToggle = document.getElementById('btn-toggle');
+  const _btnIcon   = document.getElementById('btn-icon');
+  const _btnLabel  = document.getElementById('btn-label');
+  const _dot       = document.getElementById('status-dot');
+
+  // Observe #status-dot background changes to mirror active state on button
+  const obs = new MutationObserver(() => {
+    const bg = _dot.style.background;
+    if (bg === '#22c55e' || bg === '#3b82f6') {
+      _btnToggle.classList.add('active');
+      _btnIcon.textContent  = '⏹';
+      _btnLabel.textContent = 'STOP';
+    } else if (bg === '#f59e0b') {
+      _btnToggle.classList.add('active');
+      _btnIcon.textContent  = '💭';
+      _btnLabel.textContent = 'THINKING';
+    } else {
+      _btnToggle.classList.remove('active');
+      _btnIcon.textContent  = '🎤';
+      _btnLabel.textContent = 'START';
+    }
+
+    // Pulse dot when not idle
+    _dot.classList.toggle('pulse', bg !== '#6b7280');
+  });
+  obs.observe(_dot, { attributes: true, attributeFilter: ['style'] });
+</script>
+</body>
+</html>

models.py

@@ -0,0 +1,218 @@
+"""
+models.py
+---------
+Single source of truth for every model used in the pipeline.
+
+Component    Model                                    Params   Device
+-----------  ---------------------------------------  -------  ------
+STT          openai/whisper-base.en                   74 M     GPU/CPU (fp16/fp32)
+LLM          HuggingFaceTB/SmolLM2-1.7B-Instruct     1.7 B    GPU/CPU (fp16/fp32)
+TTS          facebook/mms-tts-eng  (VITS)             ~430 M   CPU     (fp32)
+VAD          silero-vad                               2 MB     CPU
+
+All models are loaded once at startup. Inference methods are synchronous
+(blocking) — call them from a thread-pool executor inside async handlers.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Optional
+
+import numpy as np
+import torch
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    VitsModel,
+    WhisperForConditionalGeneration,
+    WhisperProcessor,
+)
+
+logger = logging.getLogger(__name__)
+
+# --------------------------------------------------------------------------- #
+#  Model IDs  (swap here for lighter / heavier variants)                       #
+# --------------------------------------------------------------------------- #
+STT_MODEL_ID = "openai/whisper-base.en"          # 74 M  — swap tiny.en for CPU-only
+LLM_MODEL_ID = "HuggingFaceTB/SmolLM2-1.7B-Instruct"  # 1.7 B — swap 360M for CPU-only
+TTS_MODEL_ID = "facebook/mms-tts-eng"           # VITS, 16 kHz output
+
+STT_SR = 16_000          # Whisper expects 16 kHz
+TTS_SR = 16_000          # MMS-TTS-ENG native output rate
+MAX_NEW_TOKENS = 256     # Cap LLM response length for voice interaction
+
+
+# --------------------------------------------------------------------------- #
+#  ModelManager                                                                 #
+# --------------------------------------------------------------------------- #
+class ModelManager:
+    """Load and expose all model inference in one place."""
+
+    def __init__(self):
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.compute_dtype = (
+            torch.float16 if self.device == "cuda" else torch.float32
+        )
+        logger.info("Device: %s  |  dtype: %s", self.device, self.compute_dtype)
+
+        # TTS always runs on CPU (saves GPU VRAM for STT + LLM)
+        self.tts_device = "cpu"
+
+        self._load_vad()
+        self._load_stt()
+        self._load_llm()
+        self._load_tts()
+        logger.info("✅  All models ready.")
+
+    # ------------------------------------------------------------------ #
+    #  VAD                                                                 #
+    # ------------------------------------------------------------------ #
+    def _load_vad(self):
+        logger.info("Loading VAD (silero-vad) …")
+        from silero_vad import VADIterator, load_silero_vad  # type: ignore
+
+        vad_model = load_silero_vad()
+        # silence_ms=700 → 700 ms of quiet ends an utterance
+        self.vad_iter = VADIterator(
+            vad_model,
+            sampling_rate=STT_SR,
+            threshold=0.50,
+            min_silence_duration_ms=700,
+            speech_pad_ms=50,
+        )
+        logger.info("VAD ready.")
+
+    def vad_reset(self):
+        """Reset VAD state between conversations."""
+        self.vad_iter.reset_states()
+
+    # ------------------------------------------------------------------ #
+    #  STT — Whisper                                                       #
+    # ------------------------------------------------------------------ #
+    def _load_stt(self):
+        logger.info("Loading STT: %s …", STT_MODEL_ID)
+        self.stt_processor = WhisperProcessor.from_pretrained(STT_MODEL_ID)
+        self.stt_model = WhisperForConditionalGeneration.from_pretrained(
+            STT_MODEL_ID,
+            torch_dtype=self.compute_dtype,
+        ).to(self.device)
+        self.stt_model.eval()
+        logger.info("STT ready on %s.", self.device)
+
+    def transcribe(self, audio_float32: np.ndarray) -> str:
+        """
+        Transcribe a 16 kHz float32 numpy array to text.
+
+        Parameters
+        ----------
+        audio_float32 : np.ndarray  shape (N,), dtype float32, range [-1, 1]
+        """
+        t0 = time.perf_counter()
+
+        inputs = self.stt_processor(
+            audio_float32,
+            sampling_rate=STT_SR,
+            return_tensors="pt",
+        )
+        features = inputs.input_features.to(
+            device=self.device, dtype=self.compute_dtype
+        )
+
+        with torch.no_grad():
+            ids = self.stt_model.generate(features, language="en", task="transcribe")
+
+        text = self.stt_processor.batch_decode(ids, skip_special_tokens=True)[0].strip()
+        logger.info("STT (%.0f ms): '%s'", (time.perf_counter() - t0) * 1000, text)
+        return text
+
+    # ------------------------------------------------------------------ #
+    #  LLM — SmolLM2                                                       #
+    # ------------------------------------------------------------------ #
+    def _load_llm(self):
+        logger.info("Loading LLM: %s …", LLM_MODEL_ID)
+        self.llm_tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID)
+        self.llm = AutoModelForCausalLM.from_pretrained(
+            LLM_MODEL_ID,
+            torch_dtype=self.compute_dtype,
+        ).to(self.device)
+        self.llm.eval()
+        logger.info("LLM ready on %s.", self.device)
+
+    def build_prompt(self, messages: list[dict]) -> str:
+        """Apply the model's chat template and return a formatted string."""
+        return self.llm_tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+
+    def tokenize(self, prompt: str) -> dict:
+        """Tokenize a prompt string → dict of tensors on self.device."""
+        enc = self.llm_tokenizer(prompt, return_tensors="pt")
+        return {k: v.to(self.device) for k, v in enc.items()}
+
+    # ------------------------------------------------------------------ #
+    #  TTS — MMS-TTS-ENG (VITS)                                           #
+    # ------------------------------------------------------------------ #
+    def _load_tts(self):
+        logger.info("Loading TTS: %s …", TTS_MODEL_ID)
+        self.tts_tokenizer = AutoTokenizer.from_pretrained(TTS_MODEL_ID)
+        self.tts_model = VitsModel.from_pretrained(TTS_MODEL_ID)  # fp32 on CPU
+        self.tts_model.eval()
+        self.tts_sample_rate: int = self.tts_model.config.sampling_rate  # 16 000
+        logger.info("TTS ready on CPU (sample_rate=%d Hz).", self.tts_sample_rate)
+
+    def synthesize(self, text: str) -> Optional[bytes]:
+        """
+        Synthesize text → raw PCM bytes (int16, mono, 16 kHz).
+
+        Returns None if text is empty or synthesis fails.
+        """
+        text = text.strip()
+        if not text:
+            return None
+
+        t0 = time.perf_counter()
+
+        try:
+            inputs = self.tts_tokenizer(text, return_tensors="pt")
+            with torch.no_grad():
+                output = self.tts_model(**inputs)
+
+            # waveform: (1, T)  float32 in roughly [-1, 1]
+            wav: np.ndarray = output.waveform.squeeze().cpu().numpy()
+
+            # Convert float32 → int16 PCM
+            wav_int16 = np.clip(wav * 32_767.0, -32_768, 32_767).astype(np.int16)
+            pcm_bytes = wav_int16.tobytes()
+
+            logger.debug(
+                "TTS (%.0f ms): %d chars → %d bytes",
+                (time.perf_counter() - t0) * 1000,
+                len(text),
+                len(pcm_bytes),
+            )
+            return pcm_bytes
+
+        except Exception as exc:
+            logger.error("TTS synthesis failed for %r: %s", text[:60], exc)
+            return None
+
+    # ------------------------------------------------------------------ #
+    #  Convenience                                                         #
+    # ------------------------------------------------------------------ #
+    def warm_up(self):
+        """Run a tiny inference on each model to pre-JIT all kernels."""
+        logger.info("Warming up models …")
+        silence = np.zeros(STT_SR, dtype=np.float32)
+        self.transcribe(silence)
+        dummy_msgs = [
+            {"role": "system", "content": "You are helpful."},
+            {"role": "user", "content": "hi"},
+        ]
+        # Just tokenize, don't run full generation during warm-up
+        self.tokenize(self.build_prompt(dummy_msgs))
+        self.synthesize("Hello.")
+        logger.info("Warm-up complete.")

pipeline.py

@@ -0,0 +1,231 @@
+"""
+pipeline.py
+-----------
+Core streaming overlap pipeline.
+
+Flow
+----
+   Conversation history
+          │
+          ▼
+   SmolLM2 (thread)  ──tokens──►  SentenceBuffer
+                                        │
+                              sentence complete?
+                                        │ yes
+                                        ▼
+                               MMS-TTS (executor)          ← runs while LLM
+                                        │                    keeps generating!
+                                        ▼
+                              ws.send_bytes(pcm)
+                                        │
+                              ◄─────────┘ repeat
+
+The key insight: TTS synthesis for sentence N starts as soon as sentence N is
+complete — the LLM does NOT wait. By the time the browser finishes playing
+sentence N, sentence N+1 is already synthesised and queued, giving near-zero
+inter-sentence gaps and a much lower perceived total latency.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from threading import Thread
+from typing import AsyncGenerator
+
+from fastapi import WebSocket
+from transformers import TextIteratorStreamer  # type: ignore
+
+from models import MAX_NEW_TOKENS, ModelManager
+from sentence_buffer import SentenceBuffer
+
+logger = logging.getLogger(__name__)
+
+# Maximum conversation turns kept in context (system + N user/assistant pairs)
+MAX_HISTORY_TURNS = 6
+SYSTEM_PROMPT = (
+    "You are a helpful, friendly voice assistant. "
+    "Keep every reply concise — two or three short sentences at most. "
+    "Speak naturally; avoid bullet points, markdown, or lists."
+)
+
+
+# --------------------------------------------------------------------------- #
+#  Async LLM token stream                                                      #
+# --------------------------------------------------------------------------- #
+async def _llm_token_stream(
+    models: ModelManager,
+    messages: list[dict],
+    loop: asyncio.AbstractEventLoop,
+) -> AsyncGenerator[str, None]:
+    """
+    Async generator that yields text tokens from SmolLM2 as they are produced.
+
+    Architecture
+    ~~~~~~~~~~~~
+    model.generate()  runs in  gen_thread  (CPU/GPU bound).
+    TextIteratorStreamer bridges the sync world → async Queue via
+    run_coroutine_threadsafe so the FastAPI event loop stays unblocked.
+    """
+    prompt = models.build_prompt(messages)
+    enc = models.tokenize(prompt)
+
+    streamer = TextIteratorStreamer(
+        models.llm_tokenizer,
+        skip_prompt=True,
+        skip_special_tokens=True,
+        timeout=60.0,
+    )
+
+    gen_kwargs = dict(
+        input_ids=enc["input_ids"],
+        attention_mask=enc.get("attention_mask"),
+        streamer=streamer,
+        max_new_tokens=MAX_NEW_TOKENS,
+        do_sample=True,
+        temperature=0.7,
+        repetition_penalty=1.1,
+        pad_token_id=models.llm_tokenizer.eos_token_id,
+    )
+
+    token_q: asyncio.Queue[str | None] = asyncio.Queue()
+
+    # ── thread: start generation + drain streamer into async queue ──────── #
+    def _generate_and_feed():
+        gen_thread = Thread(
+            target=models.llm.generate, kwargs=gen_kwargs, daemon=True
+        )
+        gen_thread.start()
+        try:
+            for tok in streamer:            # blocks until each token is ready
+                asyncio.run_coroutine_threadsafe(token_q.put(tok), loop)
+        except Exception as exc:
+            asyncio.run_coroutine_threadsafe(token_q.put(None), loop)
+            logger.error("LLM stream error: %s", exc)
+            return
+        asyncio.run_coroutine_threadsafe(token_q.put(None), loop)  # sentinel
+
+    feed_thread = Thread(target=_generate_and_feed, daemon=True)
+    feed_thread.start()
+
+    # ── yield tokens from the queue ─────────────────────────────────────── #
+    while True:
+        tok = await token_q.get()
+        if tok is None:
+            break
+        yield tok
+
+
+# --------------------------------------------------------------------------- #
+#  StreamingPipeline                                                           #
+# --------------------------------------------------------------------------- #
+class StreamingPipeline:
+    """
+    Orchestrates the full LLM → SentenceBuffer → TTS → WebSocket pipeline
+    with streaming overlap.
+    """
+
+    def __init__(self, models: ModelManager):
+        self.models = models
+        self._executor = None   # Use default loop executor (ThreadPoolExecutor)
+
+    async def process(
+        self,
+        conversation: list[dict],
+        ws: WebSocket,
+        loop: asyncio.AbstractEventLoop,
+    ) -> str:
+        """
+        Stream an LLM response over `ws` as audio chunks.
+
+        Returns the full text of the assistant turn (for appending to history).
+
+        Parameters
+        ----------
+        conversation : list of {role, content} dicts including the new user turn.
+        ws           : active WebSocket connection to send PCM bytes to.
+        loop         : running event loop (passed for thread-safe queue ops).
+        """
+        # Trim history to avoid context overflow
+        conversation = _trim_history(conversation)
+
+        sentence_buf = SentenceBuffer(min_length=8)
+        full_text = ""
+        t_start = time.perf_counter()
+        ttft_logged = False
+
+        async for token in _llm_token_stream(self.models, conversation, loop):
+            if not ttft_logged:
+                ttft_ms = (time.perf_counter() - t_start) * 1000
+                logger.info("LLM TTFT: %.0f ms", ttft_ms)
+                ttft_logged = True
+
+            full_text += token
+            ready = sentence_buf.add(token)
+
+            for sentence in ready:
+                await self._synth_and_send(sentence, ws)
+
+        # Flush any partial sentence left in buffer
+        tail = sentence_buf.flush()
+        if tail:
+            await self._synth_and_send(tail, ws)
+
+        total_ms = (time.perf_counter() - t_start) * 1000
+        logger.info(
+            "Pipeline done in %.0f ms | response: %d chars", total_ms, len(full_text)
+        )
+        return full_text.strip()
+
+    # ------------------------------------------------------------------ #
+    async def _synth_and_send(self, sentence: str, ws: WebSocket):
+        """
+        Synthesise one sentence and stream PCM bytes over the WebSocket.
+
+        TTS runs in the default thread-pool executor so the event loop
+        remains free to handle other coroutines (e.g. interruption frames
+        arriving from the client) during synthesis.
+        """
+        sentence = sentence.strip()
+        if not sentence:
+            return
+
+        logger.debug("TTS ← '%s'", sentence[:80])
+        t0 = time.perf_counter()
+
+        pcm: bytes | None = await asyncio.get_event_loop().run_in_executor(
+            None, self.models.synthesize, sentence
+        )
+
+        if not pcm:
+            return
+
+        tts_ms = (time.perf_counter() - t0) * 1000
+        logger.info("TTS (%.0f ms): %d bytes for '%s…'", tts_ms, len(pcm), sentence[:40])
+
+        # Send in 4 kB chunks to give the client a smooth receive stream
+        CHUNK = 4096
+        for i in range(0, len(pcm), CHUNK):
+            await ws.send_bytes(pcm[i : i + CHUNK])
+
+
+# --------------------------------------------------------------------------- #
+#  Helpers                                                                     #
+# --------------------------------------------------------------------------- #
+def _trim_history(messages: list[dict]) -> list[dict]:
+    """Keep system message + last MAX_HISTORY_TURNS user/assistant pairs."""
+    system = [m for m in messages if m["role"] == "system"]
+    turns  = [m for m in messages if m["role"] != "system"]
+
+    # Each "turn" = 1 user + 1 assistant message = 2 items
+    max_items = MAX_HISTORY_TURNS * 2
+    if len(turns) > max_items:
+        turns = turns[-max_items:]
+
+    return system + turns
+
+
+def build_initial_conversation() -> list[dict]:
+    """Start a fresh conversation with only the system prompt."""
+    return [{"role": "system", "content": SYSTEM_PROMPT}]

requirements.txt

@@ -0,0 +1,20 @@
+# ── Core framework ──────────────────────────────────────────────────────────
+fastapi==0.115.5
+uvicorn[standard]==0.32.1
+websockets==13.1
+python-multipart==0.0.12
+
+# ── HuggingFace / ML ─────────────────────────────────────────────────────────
+transformers==4.47.0
+torch==2.5.1
+torchaudio==2.5.1
+accelerate==1.2.1
+huggingface_hub==0.26.5
+datasets==3.2.0              # used by warm-up only; can be removed if desired
+
+# ── VAD ──────────────────────────────────────────────────────────────────────
+silero-vad==5.1.2
+
+# ── Audio utilities ──────────────────────────────────────────────────────────
+numpy==1.26.4
+soundfile==0.12.1

sentence_buffer.py

@@ -0,0 +1,129 @@
+"""
+sentence_buffer.py
+------------------
+Accumulates LLM token stream and emits complete sentences for TTS synthesis.
+
+Key design: We must detect REAL sentence boundaries without false-positives on
+abbreviations (Dr., Mr., Inc.), decimal numbers (3.14), ellipses (...) etc.
+Each emitted sentence is immediately handed off to TTS while the LLM continues
+generating — this is the source of the streaming overlap latency savings.
+"""
+
+import re
+from typing import List
+
+# Abbreviations that must NOT trigger sentence split even when followed by a capital letter
+ABBREVIATIONS = {
+    "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "rev", "gen", "sgt",
+    "cpl", "pvt", "capt", "maj", "col", "lt", "cmdr", "adm",
+    "inc", "corp", "ltd", "llc", "co", "dept", "est",
+    "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
+    "vs", "etc", "approx", "max", "min", "avg", "no", "vol", "fig",
+    "st", "ave", "blvd", "rd",
+}
+
+# Pattern: a word ending with a period that looks like an abbreviation
+_ABBREV_RE = re.compile(
+    r'\b(' + '|'.join(re.escape(a) for a in ABBREVIATIONS) + r')\.$',
+    re.IGNORECASE,
+)
+
+# Decimal numbers like 3.14 or $19.99 must not split
+_DECIMAL_RE = re.compile(r'\d+\.$')
+
+# Sentence-ending punctuation
+_SENTENCE_END_RE = re.compile(r'[.!?]+')
+
+
+def _is_false_boundary(text_before_dot: str) -> bool:
+    """Return True if the period at the end of text_before_dot is NOT a sentence end."""
+    stripped = text_before_dot.rstrip()
+    if _ABBREV_RE.search(stripped):
+        return True
+    if _DECIMAL_RE.search(stripped):
+        return True
+    return False
+
+
+class SentenceBuffer:
+    """
+    Feed LLM tokens one at a time; call flush() at stream end.
+
+    Usage
+    -----
+    buf = SentenceBuffer(min_length=12)
+    for token in llm_stream():
+        sentences = buf.add(token)
+        for s in sentences:
+            audio = tts.synthesize(s)   # start immediately
+    final = buf.flush()
+    if final:
+        audio = tts.synthesize(final)
+    """
+
+    def __init__(self, min_length: int = 10):
+        """
+        min_length : int
+            Minimum character count before a boundary is accepted.
+            Avoids firing TTS on single-word fragments like "Hi."
+        """
+        self.min_length = min_length
+        self._buf = ""
+
+    # ------------------------------------------------------------------
+    def add(self, token: str) -> List[str]:
+        """
+        Append a token and return a list of complete sentences ready for TTS.
+        Typically returns [] or a one-element list.
+        """
+        self._buf += token
+        return self._extract()
+
+    def flush(self) -> str:
+        """Return whatever is left in the buffer (end of stream)."""
+        remaining = self._buf.strip()
+        self._buf = ""
+        return remaining
+
+    def reset(self):
+        self._buf = ""
+
+    # ------------------------------------------------------------------
+    def _extract(self) -> List[str]:
+        sentences = []
+
+        while True:
+            match = _SENTENCE_END_RE.search(self._buf)
+            if not match:
+                break
+
+            end_pos = match.end()
+            candidate = self._buf[:end_pos].strip()
+
+            if len(candidate) < self.min_length:
+                break  # Too short — wait for more tokens
+
+            # Check for false boundary
+            before_punct = self._buf[:match.start()]
+            if _is_false_boundary(before_punct):
+                # Advance past this dot and keep scanning
+                self._buf = self._buf[end_pos:]
+                if sentences:
+                    sentences[-1] += candidate  # Merge into previous
+                else:
+                    # Keep in buffer with content that was before
+                    self._buf = candidate + " " + self._buf
+                    break
+                continue
+
+            # Real sentence boundary — check there's enough after the punctuation
+            after = self._buf[end_pos:].lstrip()
+
+            # If the very next char is another sentence-ender, keep accumulating
+            if after and after[0] in '.!?':
+                break
+
+            sentences.append(candidate)
+            self._buf = self._buf[end_pos:].lstrip()
+
+        return sentences