Spaces:
Sleeping
fix(#55+#56+#53/#54-engine): TTS full natural readout + voice warm-stream/pre-roll
Browse files#55 TTS 10s truncation: voice_format._truncate_for_voice(max_words=55)
chopped every advisor reply to ~55 words (~10s) before it reached
Sarvam. Cap removed (max_words now ignored unless >=100000). Latent
Sarvam Bulbul v2 1500-char cap covered: sarvam_tts now char-ceiling
chunks at item/sentence/comma seams, synthesizes sequentially, gapless
WAV concat, propagates any chunk error (no silent partial) β STT-style.
#56 TTS normalization: _normalize_abbreviations + _normalize_ranges β
e.g./i.e./etc. spoken as words; '/' β 'or'; βΉ/L/Cr/K/% and en-dash
ranges expanded; markdown/list numbering stripped. Verified on the exact
6-question pricing message (all 6 questions present, artifacts gone).
#53/#54 ENGINE: useStreamingVoice warm pre-armed mic + 800ms PreRollRing
+ 200ms deliberate-hold gate + beginPushToTalk/endPushToTalk/
consumePreRollChunks API. Keeps the OS mic hot so page.tsx getUserMedia
is ~10-50ms not 200-700ms (latency #54) and the first word is captured
(head-clip #53). page.tsx delegation wiring lands next (separate).
Full pytest gate green (rc=0; 226). tsc/build green (agent-verified).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- backend/providers/sarvam_tts.py +254 -15
- backend/voice_format.py +195 -6
- frontend/src/lib/useStreamingVoice.ts +555 -2
- tests/test_ptt_preroll_warm_stream.py +457 -0
- tests/test_tts_full_natural_readout.py +359 -0
|
@@ -20,7 +20,9 @@ from __future__ import annotations
|
|
| 20 |
import base64
|
| 21 |
import io
|
| 22 |
import logging
|
| 23 |
-
|
|
|
|
|
|
|
| 24 |
|
| 25 |
import httpx
|
| 26 |
|
|
@@ -29,6 +31,187 @@ from backend.providers.base import TTSProvider
|
|
| 29 |
|
| 30 |
logger = logging.getLogger(__name__)
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
# The frontend sends an `X-Preferred-Codec: audio/{wav,mp4,webm}` header
|
| 33 |
# so the chat endpoint can return audio in the codec the user's browser
|
| 34 |
# decodes natively. Sarvam Bulbul's text-to-speech API itself does NOT support
|
|
@@ -193,20 +376,17 @@ class SarvamTTS(TTSProvider):
|
|
| 193 |
)
|
| 194 |
return audio_bytes
|
| 195 |
|
| 196 |
-
async def
|
| 197 |
self,
|
| 198 |
text: str,
|
| 199 |
-
language_code: str
|
| 200 |
-
speaker: Optional[str]
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
"""Like synthesize() but also returns the actual MIME type.
|
| 204 |
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
transcoding failure (missing dep, ffmpeg error), we fall back to raw
|
| 209 |
-
WAV β the frontend already handles this gracefully.
|
| 210 |
"""
|
| 211 |
url = f"{settings.SARVAM_BASE_URL}{settings.SARVAM_TTS_PATH}"
|
| 212 |
body = {
|
|
@@ -224,7 +404,6 @@ class SarvamTTS(TTSProvider):
|
|
| 224 |
"api-subscription-key": self.api_key,
|
| 225 |
"Content-Type": "application/json",
|
| 226 |
}
|
| 227 |
-
|
| 228 |
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
| 229 |
resp = await client.post(url, headers=headers, json=body)
|
| 230 |
resp.raise_for_status()
|
|
@@ -233,9 +412,69 @@ class SarvamTTS(TTSProvider):
|
|
| 233 |
audios = payload.get("audios", [])
|
| 234 |
if not audios:
|
| 235 |
raise RuntimeError(f"Sarvam TTS returned no audio: {payload}")
|
| 236 |
-
|
| 237 |
# Sarvam returns base64-encoded WAV in `audios[0]`
|
| 238 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
|
| 240 |
codec = (preferred_codec or "audio/wav").lower()
|
| 241 |
return _transcode_wav(wav_bytes, codec)
|
|
|
|
| 20 |
import base64
|
| 21 |
import io
|
| 22 |
import logging
|
| 23 |
+
import re
|
| 24 |
+
import wave
|
| 25 |
+
from typing import List, Optional, Tuple
|
| 26 |
|
| 27 |
import httpx
|
| 28 |
|
|
|
|
| 31 |
|
| 32 |
logger = logging.getLogger(__name__)
|
| 33 |
|
| 34 |
+
# ---------------------------------------------------------------------------
|
| 35 |
+
# #55 β Sarvam Bulbul has a HARD per-request character limit:
|
| 36 |
+
# bulbul:v2 -> 1500 chars, bulbul:v3 -> 2500 chars
|
| 37 |
+
# (confirmed from https://docs.sarvam.ai text-to-speech reference).
|
| 38 |
+
#
|
| 39 |
+
# A long advisor reply (e.g. the 6-question pricing intake) exceeds 1500
|
| 40 |
+
# chars once normalized. Sending it whole means Sarvam only voices the
|
| 41 |
+
# leading slice β the exact "stopped in ten seconds" / questions 2-6 never
|
| 42 |
+
# spoken symptom. So, mirroring the STT 30s-chunking house style in
|
| 43 |
+
# providers/sarvam_stt.py, we split the text into <= TTS_CHUNK_CHARS
|
| 44 |
+
# pieces at SENTENCE / NUMBERED-ITEM boundaries (so we never cut a word or
|
| 45 |
+
# a question mid-way), synthesize each chunk sequentially, and concatenate
|
| 46 |
+
# the decoded PCM into ONE gapless WAV returned to the caller. Any HTTP /
|
| 47 |
+
# transport error on ANY chunk is raised LOUDLY (no silent truncation,
|
| 48 |
+
# no partial-audio-with-HTTP-200).
|
| 49 |
+
#
|
| 50 |
+
# Ceiling is set below the documented cap so per-language preprocessing
|
| 51 |
+
# expansion + minor jitter never pushes a chunk back over Sarvam's real
|
| 52 |
+
# limit.
|
| 53 |
+
_TTS_CHAR_LIMIT_BY_MODEL = {
|
| 54 |
+
"bulbul:v2": 1500,
|
| 55 |
+
"bulbul:v3": 2500,
|
| 56 |
+
}
|
| 57 |
+
_TTS_CHAR_LIMIT_DEFAULT = 1500 # safest assumption for unknown models
|
| 58 |
+
_TTS_SAFETY_MARGIN = 200 # headroom under the documented hard cap
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _tts_char_ceiling(model: str) -> int:
|
| 62 |
+
"""Safe per-request char ceiling for the configured Bulbul model."""
|
| 63 |
+
hard = _TTS_CHAR_LIMIT_BY_MODEL.get(
|
| 64 |
+
(model or "").lower(), _TTS_CHAR_LIMIT_DEFAULT
|
| 65 |
+
)
|
| 66 |
+
return max(200, hard - _TTS_SAFETY_MARGIN)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# Split points, longest-context first: paragraph break, then end-of-
|
| 70 |
+
# sentence punctuation, then a numbered-list item boundary ("\n2. "),
|
| 71 |
+
# then comma, then whitespace. We never split inside a word.
|
| 72 |
+
_SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+")
|
| 73 |
+
_NUMBERED_ITEM = re.compile(r"(?=(?:^|\s)\d{1,2}[.)]\s)")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _hard_wrap(piece: str, limit: int) -> List[str]:
|
| 77 |
+
"""Last-resort splitter for a single 'unit' longer than `limit`.
|
| 78 |
+
|
| 79 |
+
Splits on whitespace so a word is never cut; if a single token still
|
| 80 |
+
exceeds `limit` (pathological), it is hard-sliced so synthesis still
|
| 81 |
+
covers it rather than dropping it.
|
| 82 |
+
"""
|
| 83 |
+
out: List[str] = []
|
| 84 |
+
cur = ""
|
| 85 |
+
for tok in piece.split(" "):
|
| 86 |
+
if not tok:
|
| 87 |
+
continue
|
| 88 |
+
cand = tok if not cur else f"{cur} {tok}"
|
| 89 |
+
if len(cand) <= limit:
|
| 90 |
+
cur = cand
|
| 91 |
+
continue
|
| 92 |
+
if cur:
|
| 93 |
+
out.append(cur)
|
| 94 |
+
cur = ""
|
| 95 |
+
if len(tok) <= limit:
|
| 96 |
+
cur = tok
|
| 97 |
+
else:
|
| 98 |
+
for i in range(0, len(tok), limit):
|
| 99 |
+
out.append(tok[i : i + limit])
|
| 100 |
+
if cur:
|
| 101 |
+
out.append(cur)
|
| 102 |
+
return out
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _chunk_text_for_tts(text: str, limit: int) -> List[str]:
|
| 106 |
+
"""Split `text` into <= `limit`-char chunks at natural speech seams.
|
| 107 |
+
|
| 108 |
+
Order of preference for a seam: sentence end -> numbered-list item ->
|
| 109 |
+
comma -> whitespace. A chunk is never cut mid-word, and a numbered
|
| 110 |
+
pricing question is never split across two synthesis calls unless it
|
| 111 |
+
is itself longer than `limit` (then it hard-wraps on whitespace).
|
| 112 |
+
|
| 113 |
+
The concatenation of all chunks (joined with a single space) preserves
|
| 114 |
+
every character of spoken content β nothing is dropped.
|
| 115 |
+
"""
|
| 116 |
+
text = text.strip()
|
| 117 |
+
if not text:
|
| 118 |
+
return []
|
| 119 |
+
if len(text) <= limit:
|
| 120 |
+
return [text]
|
| 121 |
+
|
| 122 |
+
# 1. Coarse units: prefer numbered-item boundaries (keeps each "2. ..."
|
| 123 |
+
# question intact), else fall back to sentence boundaries.
|
| 124 |
+
units = [u for u in _NUMBERED_ITEM.split(text) if u and u.strip()]
|
| 125 |
+
if len(units) <= 1:
|
| 126 |
+
units = [u for u in _SENTENCE_BOUNDARY.split(text) if u and u.strip()]
|
| 127 |
+
if len(units) <= 1:
|
| 128 |
+
units = [text]
|
| 129 |
+
|
| 130 |
+
# 2. Greedily pack units into chunks <= limit. A unit bigger than limit
|
| 131 |
+
# is itself sentence-split, then hard-wrapped as a last resort.
|
| 132 |
+
chunks: List[str] = []
|
| 133 |
+
cur = ""
|
| 134 |
+
for unit in units:
|
| 135 |
+
unit = unit.strip()
|
| 136 |
+
if len(unit) > limit:
|
| 137 |
+
if cur:
|
| 138 |
+
chunks.append(cur)
|
| 139 |
+
cur = ""
|
| 140 |
+
sub_units = [
|
| 141 |
+
s for s in _SENTENCE_BOUNDARY.split(unit) if s and s.strip()
|
| 142 |
+
]
|
| 143 |
+
if len(sub_units) <= 1:
|
| 144 |
+
sub_units = _hard_wrap(unit, limit)
|
| 145 |
+
for su in sub_units:
|
| 146 |
+
su = su.strip()
|
| 147 |
+
if len(su) > limit:
|
| 148 |
+
chunks.extend(_hard_wrap(su, limit))
|
| 149 |
+
continue
|
| 150 |
+
cand = su if not cur else f"{cur} {su}"
|
| 151 |
+
if len(cand) <= limit:
|
| 152 |
+
cur = cand
|
| 153 |
+
else:
|
| 154 |
+
if cur:
|
| 155 |
+
chunks.append(cur)
|
| 156 |
+
cur = su
|
| 157 |
+
continue
|
| 158 |
+
cand = unit if not cur else f"{cur} {unit}"
|
| 159 |
+
if len(cand) <= limit:
|
| 160 |
+
cur = cand
|
| 161 |
+
else:
|
| 162 |
+
if cur:
|
| 163 |
+
chunks.append(cur)
|
| 164 |
+
cur = unit
|
| 165 |
+
if cur:
|
| 166 |
+
chunks.append(cur)
|
| 167 |
+
return [c for c in chunks if c.strip()]
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _concat_wav_bytes(wav_blobs: List[bytes]) -> bytes:
|
| 171 |
+
"""Concatenate multiple PCM WAV blobs into ONE gapless WAV.
|
| 172 |
+
|
| 173 |
+
All chunks come from the same Sarvam call config (same model, speaker,
|
| 174 |
+
sample rate) so the PCM params match; we still assert that so a
|
| 175 |
+
mismatch fails LOUD rather than producing garbled audio. Stdlib `wave`
|
| 176 |
+
only β no pydub/ffmpeg dependency for the core join (pydub remains the
|
| 177 |
+
optional transcoder downstream).
|
| 178 |
+
"""
|
| 179 |
+
if not wav_blobs:
|
| 180 |
+
raise RuntimeError("Sarvam TTS produced no audio chunks to concat")
|
| 181 |
+
if len(wav_blobs) == 1:
|
| 182 |
+
return wav_blobs[0]
|
| 183 |
+
|
| 184 |
+
params = None
|
| 185 |
+
frames: List[bytes] = []
|
| 186 |
+
for idx, blob in enumerate(wav_blobs):
|
| 187 |
+
with wave.open(io.BytesIO(blob), "rb") as w:
|
| 188 |
+
p = w.getparams()
|
| 189 |
+
if params is None:
|
| 190 |
+
params = p
|
| 191 |
+
elif (
|
| 192 |
+
p.nchannels,
|
| 193 |
+
p.sampwidth,
|
| 194 |
+
p.framerate,
|
| 195 |
+
) != (
|
| 196 |
+
params.nchannels,
|
| 197 |
+
params.sampwidth,
|
| 198 |
+
params.framerate,
|
| 199 |
+
):
|
| 200 |
+
raise RuntimeError(
|
| 201 |
+
"Sarvam TTS chunk audio params diverged "
|
| 202 |
+
f"(chunk {idx}: {p} vs {params}) β refusing to "
|
| 203 |
+
"concatenate mismatched PCM"
|
| 204 |
+
)
|
| 205 |
+
frames.append(w.readframes(w.getnframes()))
|
| 206 |
+
|
| 207 |
+
out = io.BytesIO()
|
| 208 |
+
with wave.open(out, "wb") as w:
|
| 209 |
+
w.setnchannels(params.nchannels)
|
| 210 |
+
w.setsampwidth(params.sampwidth)
|
| 211 |
+
w.setframerate(params.framerate)
|
| 212 |
+
w.writeframes(b"".join(frames))
|
| 213 |
+
return out.getvalue()
|
| 214 |
+
|
| 215 |
# The frontend sends an `X-Preferred-Codec: audio/{wav,mp4,webm}` header
|
| 216 |
# so the chat endpoint can return audio in the codec the user's browser
|
| 217 |
# decodes natively. Sarvam Bulbul's text-to-speech API itself does NOT support
|
|
|
|
| 376 |
)
|
| 377 |
return audio_bytes
|
| 378 |
|
| 379 |
+
async def _synthesize_one(
|
| 380 |
self,
|
| 381 |
text: str,
|
| 382 |
+
language_code: str,
|
| 383 |
+
speaker: Optional[str],
|
| 384 |
+
) -> bytes:
|
| 385 |
+
"""POST ONE <= char-limit text chunk to Sarvam, return its WAV bytes.
|
|
|
|
| 386 |
|
| 387 |
+
Raises on transport / HTTP errors (no swallowing) so the caller's
|
| 388 |
+
classifier maps them to a closed tts_error_code β mirrors
|
| 389 |
+
sarvam_stt._transcribe_wav_bytes.
|
|
|
|
|
|
|
| 390 |
"""
|
| 391 |
url = f"{settings.SARVAM_BASE_URL}{settings.SARVAM_TTS_PATH}"
|
| 392 |
body = {
|
|
|
|
| 404 |
"api-subscription-key": self.api_key,
|
| 405 |
"Content-Type": "application/json",
|
| 406 |
}
|
|
|
|
| 407 |
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
| 408 |
resp = await client.post(url, headers=headers, json=body)
|
| 409 |
resp.raise_for_status()
|
|
|
|
| 412 |
audios = payload.get("audios", [])
|
| 413 |
if not audios:
|
| 414 |
raise RuntimeError(f"Sarvam TTS returned no audio: {payload}")
|
|
|
|
| 415 |
# Sarvam returns base64-encoded WAV in `audios[0]`
|
| 416 |
+
return base64.b64decode(audios[0])
|
| 417 |
+
|
| 418 |
+
async def synthesize_with_mime(
|
| 419 |
+
self,
|
| 420 |
+
text: str,
|
| 421 |
+
language_code: str = "en-IN",
|
| 422 |
+
speaker: Optional[str] = None,
|
| 423 |
+
preferred_codec: Optional[str] = None,
|
| 424 |
+
) -> Tuple[bytes, str]:
|
| 425 |
+
"""Like synthesize() but also returns the actual MIME type.
|
| 426 |
+
|
| 427 |
+
#55 FIX: Bulbul has a HARD per-request char limit (v2=1500). A long
|
| 428 |
+
reply is split at sentence / numbered-item seams into <= ceiling
|
| 429 |
+
chunks, each chunk is synthesized sequentially, and the decoded PCM
|
| 430 |
+
is concatenated into ONE gapless WAV β so the COMPLETE reply
|
| 431 |
+
(every numbered question) is spoken, not just the first ~10s.
|
| 432 |
+
This mirrors the STT 30s-chunking house style. Any HTTP / transport
|
| 433 |
+
error on ANY chunk propagates (no silent truncation).
|
| 434 |
+
|
| 435 |
+
`preferred_codec` is one of "audio/wav" | "audio/mp4" | "audio/webm".
|
| 436 |
+
If None or "audio/wav", returns Sarvam's raw WAV unchanged. Any other
|
| 437 |
+
value triggers in-process transcoding via pydub/ffmpeg; on any
|
| 438 |
+
transcoding failure (missing dep, ffmpeg error), we fall back to raw
|
| 439 |
+
WAV β the frontend already handles this gracefully.
|
| 440 |
+
"""
|
| 441 |
+
ceiling = _tts_char_ceiling(self.model)
|
| 442 |
+
chunks = _chunk_text_for_tts(text or "", ceiling)
|
| 443 |
+
|
| 444 |
+
if not chunks:
|
| 445 |
+
raise RuntimeError("Sarvam TTS called with empty text")
|
| 446 |
+
|
| 447 |
+
if len(chunks) == 1:
|
| 448 |
+
# Common case: short reply, single Sarvam call β identical wire
|
| 449 |
+
# behaviour to the pre-fix path.
|
| 450 |
+
wav_bytes = await self._synthesize_one(
|
| 451 |
+
chunks[0], language_code, speaker
|
| 452 |
+
)
|
| 453 |
+
else:
|
| 454 |
+
logger.info(
|
| 455 |
+
"TTS chunked synthesis: %d chars split into %d chunks "
|
| 456 |
+
"(<= %d chars each, model=%s)",
|
| 457 |
+
len(text or ""),
|
| 458 |
+
len(chunks),
|
| 459 |
+
ceiling,
|
| 460 |
+
self.model,
|
| 461 |
+
)
|
| 462 |
+
# Sequential β preserves order so the concatenated audio reads
|
| 463 |
+
# the questions 1..6 in order. Any chunk failure raises and the
|
| 464 |
+
# boundary classifier surfaces a real error_code; we NEVER
|
| 465 |
+
# return a silently-partial readout.
|
| 466 |
+
wav_parts: List[bytes] = []
|
| 467 |
+
for idx, chunk in enumerate(chunks):
|
| 468 |
+
part = await self._synthesize_one(
|
| 469 |
+
chunk, language_code, speaker
|
| 470 |
+
)
|
| 471 |
+
if not part:
|
| 472 |
+
raise RuntimeError(
|
| 473 |
+
f"Sarvam TTS returned empty audio for chunk {idx} "
|
| 474 |
+
f"of {len(chunks)}"
|
| 475 |
+
)
|
| 476 |
+
wav_parts.append(part)
|
| 477 |
+
wav_bytes = _concat_wav_bytes(wav_parts)
|
| 478 |
|
| 479 |
codec = (preferred_codec or "audio/wav").lower()
|
| 480 |
return _transcode_wav(wav_bytes, codec)
|
|
@@ -28,9 +28,15 @@ Into spoken-ready:
|
|
| 28 |
Rules applied (in order):
|
| 29 |
1. Strip [Source: ...] and [Regulation: ...] inline citations
|
| 30 |
2. Strip markdown formatting (** bold, * italic, # headings, > quote, - bullet, 1. number)
|
| 31 |
-
3. Expand
|
| 32 |
-
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
"""
|
| 35 |
|
| 36 |
from __future__ import annotations
|
|
@@ -199,6 +205,162 @@ def _humanize_int(n: int) -> str:
|
|
| 199 |
return f"{n} rupees"
|
| 200 |
|
| 201 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
def _normalize_money(text: str) -> str:
|
| 203 |
"""Turn currency / range shorthand into spoken-language equivalents.
|
| 204 |
|
|
@@ -559,8 +721,27 @@ def strip_cot_preamble(text: str) -> str:
|
|
| 559 |
return cleaned
|
| 560 |
|
| 561 |
|
| 562 |
-
def tts_preprocess(
|
| 563 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 564 |
if not text:
|
| 565 |
return ""
|
| 566 |
# Defense in depth: run the preamble strip again here in case this is
|
|
@@ -568,6 +749,10 @@ def tts_preprocess(text: str, language: str = "en", max_words: int = 60) -> str:
|
|
| 568 |
# cached reply).
|
| 569 |
cleaned = strip_cot_preamble(text)
|
| 570 |
cleaned = _strip_markdown(cleaned)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
# Currency/range shorthand expansion before acronym handling so βΉ5L
|
| 572 |
# becomes "5 lakhs" instead of getting caught by the bare-L acronym
|
| 573 |
# path.
|
|
@@ -578,5 +763,9 @@ def tts_preprocess(text: str, language: str = "en", max_words: int = 60) -> str:
|
|
| 578 |
cleaned = _normalize_numbers(cleaned)
|
| 579 |
cleaned = _expand_acronyms(cleaned, language=language)
|
| 580 |
cleaned = _compress_whitespace(cleaned)
|
| 581 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 582 |
return cleaned
|
|
|
|
| 28 |
Rules applied (in order):
|
| 29 |
1. Strip [Source: ...] and [Regulation: ...] inline citations
|
| 30 |
2. Strip markdown formatting (** bold, * italic, # headings, > quote, - bullet, 1. number)
|
| 31 |
+
3. Expand abbreviations ("e.g." -> "for example") and slashes ("yes / no"
|
| 32 |
+
-> "yes or no"; "βΉ5L / βΉ10L" -> "5 lakh, 10 lakh") so Sarvam Bulbul
|
| 33 |
+
never spells "e.g." letter-by-letter or reads "/" as "by"/"divide"
|
| 34 |
+
4. Expand acronyms common in insurance to pronounceable forms
|
| 35 |
+
5. Compress whitespace
|
| 36 |
+
6. NO word cap β the FULL reply is spoken. Sarvam Bulbul v2's hard
|
| 37 |
+
per-request character limit is handled downstream by chunked
|
| 38 |
+
synthesis in providers/sarvam_tts.py (mirrors the STT 30s-chunking
|
| 39 |
+
house style); we never silently drop trailing questions here.
|
| 40 |
"""
|
| 41 |
|
| 42 |
from __future__ import annotations
|
|
|
|
| 205 |
return f"{n} rupees"
|
| 206 |
|
| 207 |
|
| 208 |
+
# ---- #56: abbreviation + slash normalization ----
|
| 209 |
+
#
|
| 210 |
+
# Sarvam Bulbul reads "e.g." as the letters "E G", and a bare "/" as
|
| 211 |
+
# "by"/"divide"/"slash". Both make a numbered pricing question sound like a
|
| 212 |
+
# robot reading a spreadsheet. We expand these to natural spoken words
|
| 213 |
+
# BEFORE money normalization so "βΉ5L / βΉ10L / βΉ25L / βΉ1Cr" first becomes a
|
| 214 |
+
# comma-separated list ("5L, 10L, 25L, or 1Cr") and the existing money
|
| 215 |
+
# regexes then turn each token into "5 lakh", "10 lakh", etc.
|
| 216 |
+
|
| 217 |
+
# "e.g." / "i.e." / "etc." in any spacing/casing. Order: longest first.
|
| 218 |
+
# `e.g.,` and `e.g.` both collapse to "for example".
|
| 219 |
+
_ABBR_EG = re.compile(r"\be\.\s*g\.\s*,?", re.IGNORECASE)
|
| 220 |
+
_ABBR_IE = re.compile(r"\bi\.\s*e\.\s*,?", re.IGNORECASE)
|
| 221 |
+
_ABBR_ETC = re.compile(r"\b,?\s*etc\.?", re.IGNORECASE)
|
| 222 |
+
_ABBR_VS = re.compile(r"\bvs\.?\b", re.IGNORECASE)
|
| 223 |
+
_ABBR_APPROX = re.compile(r"\bapprox\.?", re.IGNORECASE)
|
| 224 |
+
|
| 225 |
+
# A "/" that joins currency/amount tokens (βΉ5L / βΉ10L / βΉ25L / βΉ1Cr).
|
| 226 |
+
# Convert the whole run into a natural list: "A, B, C, or D". We match a
|
| 227 |
+
# slash-separated run of money-ish tokens (digits + optional βΉ + optional
|
| 228 |
+
# L/Cr/K/lakh/crore/%/word) and re-join with commas + a trailing "or".
|
| 229 |
+
_SLASH_MONEY_RUN = re.compile(
|
| 230 |
+
r"(βΉ?\s*\d[\d,]*(?:\.\d+)?\s*(?:L\b|Cr\b|K\b|lakh|crore|thousand|%)?[+]?)"
|
| 231 |
+
r"(?:\s*/\s*"
|
| 232 |
+
r"(βΉ?\s*\d[\d,]*(?:\.\d+)?\s*(?:L\b|Cr\b|K\b|lakh|crore|thousand|%)?[+]?))+",
|
| 233 |
+
re.IGNORECASE,
|
| 234 |
+
)
|
| 235 |
+
# Generic word/number slash like "parents/siblings", "yes / no",
|
| 236 |
+
# "work/otherwise" β read as "or". Keep things like "24/7" and "and/or"
|
| 237 |
+
# natural; "and/or" -> "and or" is acceptable spoken English.
|
| 238 |
+
_SLASH_GENERIC = re.compile(
|
| 239 |
+
r"(?<=[\w%)\]])\s*/\s*(?=[\w(βΉ])"
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def _normalize_slash_money_run(text: str) -> str:
|
| 244 |
+
"""Turn "βΉ5L / βΉ10L / βΉ25L / βΉ1Cr" into "βΉ5L, βΉ10L, βΉ25L, or βΉ1Cr".
|
| 245 |
+
|
| 246 |
+
The downstream money regexes then expand each token to "5 lakh" etc.
|
| 247 |
+
Result spoken: "5 lakh rupees, 10 lakh rupees, 25 lakh rupees, or 1
|
| 248 |
+
crore rupees" β never a slash read as "by"/"divide".
|
| 249 |
+
"""
|
| 250 |
+
|
| 251 |
+
def _repl(m: "re.Match[str]") -> str:
|
| 252 |
+
run = m.group(0)
|
| 253 |
+
parts = [p.strip() for p in run.split("/") if p.strip()]
|
| 254 |
+
if len(parts) <= 1:
|
| 255 |
+
return run
|
| 256 |
+
if len(parts) == 2:
|
| 257 |
+
return f"{parts[0]}, or {parts[1]}"
|
| 258 |
+
return ", ".join(parts[:-1]) + f", or {parts[-1]}"
|
| 259 |
+
|
| 260 |
+
return _SLASH_MONEY_RUN.sub(_repl, text)
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
# --- #56: numeric ranges + per-unit slashes that the money regexes miss ---
|
| 264 |
+
#
|
| 265 |
+
# Advisor messages use the EN DASH (β, U+2013) for ranges ("10β30%",
|
| 266 |
+
# "βΉ10β15K", "30β50%"). The existing _MONEY_RANGE_* regexes only match an
|
| 267 |
+
# ASCII hyphen, and _normalize_numbers strips β/β to a space, so an
|
| 268 |
+
# un-normalized en-dash range becomes "ten thirty percent" / "rupees ten
|
| 269 |
+
# 15K". Normalize en/em dash ranges to spoken "X to Y β¦" HERE (before
|
| 270 |
+
# money + before _normalize_numbers) so the whole range survives.
|
| 271 |
+
#
|
| 272 |
+
# Order: percent-range, then βΉK-range (with optional "/period"), then
|
| 273 |
+
# βΉL / βΉCr ranges, then a bare numeric range fallback.
|
| 274 |
+
_RANGE_PCT = re.compile(
|
| 275 |
+
r"(\d+(?:\.\d+)?)\s*[ββ-]\s*(\d+(?:\.\d+)?)\s*%",
|
| 276 |
+
)
|
| 277 |
+
# "βΉ10β15K/year" / "βΉ10β15K per year" / "βΉ10β15K". The trailing
|
| 278 |
+
# "/<word>" becomes "per <word>" (a rate), NOT "or".
|
| 279 |
+
_RANGE_K_PERIOD = re.compile(
|
| 280 |
+
r"βΉ?\s*(\d+(?:\.\d+)?)\s*[ββ-]\s*(\d+(?:\.\d+)?)\s*K\s*/\s*([A-Za-z]+)",
|
| 281 |
+
re.IGNORECASE,
|
| 282 |
+
)
|
| 283 |
+
_RANGE_K = re.compile(
|
| 284 |
+
r"βΉ\s*(\d+(?:\.\d+)?)\s*[ββ-]\s*(\d+(?:\.\d+)?)\s*K\b",
|
| 285 |
+
re.IGNORECASE,
|
| 286 |
+
)
|
| 287 |
+
_RANGE_L = re.compile(
|
| 288 |
+
r"βΉ\s*(\d+(?:\.\d+)?)\s*[ββ-]\s*(\d+(?:\.\d+)?)\s*L\b",
|
| 289 |
+
re.IGNORECASE,
|
| 290 |
+
)
|
| 291 |
+
_RANGE_CR = re.compile(
|
| 292 |
+
r"βΉ\s*(\d+(?:\.\d+)?)\s*[ββ-]\s*(\d+(?:\.\d+)?)\s*Cr\b",
|
| 293 |
+
re.IGNORECASE,
|
| 294 |
+
)
|
| 295 |
+
# A unit "/period" rate that is NOT part of a handled money token, e.g.
|
| 296 |
+
# "50K/year" alone, "βΉ500/month". Convert "/word" -> "per word".
|
| 297 |
+
_RATE_SLASH_PERIOD = re.compile(
|
| 298 |
+
r"(?<=[\w%)\]])\s*/\s*(year|yr|month|mo|annum|day|week|claim|policy|person|member|head)\b",
|
| 299 |
+
re.IGNORECASE,
|
| 300 |
+
)
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def _normalize_ranges(text: str) -> str:
|
| 304 |
+
"""Expand en/em-dash numeric ranges + per-period rate slashes.
|
| 305 |
+
|
| 306 |
+
Runs BEFORE _normalize_money and _normalize_numbers so the dash isn't
|
| 307 |
+
stripped to a space mid-range and each side keeps its unit.
|
| 308 |
+
"""
|
| 309 |
+
# Percent ranges first: "10β30%" -> "10 to 30 percent".
|
| 310 |
+
text = _RANGE_PCT.sub(lambda m: f"{m.group(1)} to {m.group(2)} percent", text)
|
| 311 |
+
# "βΉ10β15K/year" -> "10 to 15 thousand rupees per year".
|
| 312 |
+
text = _RANGE_K_PERIOD.sub(
|
| 313 |
+
lambda m: f"{m.group(1)} to {m.group(2)} thousand rupees per {m.group(3)}",
|
| 314 |
+
text,
|
| 315 |
+
)
|
| 316 |
+
# "βΉ10β15K" -> "10 to 15 thousand rupees".
|
| 317 |
+
text = _RANGE_K.sub(
|
| 318 |
+
lambda m: f"{m.group(1)} to {m.group(2)} thousand rupees", text
|
| 319 |
+
)
|
| 320 |
+
# "βΉ5β10L" -> "5 to 10 lakh rupees" (en-dash variant of _MONEY_RANGE_L).
|
| 321 |
+
text = _RANGE_L.sub(
|
| 322 |
+
lambda m: f"{m.group(1)} to {m.group(2)} lakh rupees", text
|
| 323 |
+
)
|
| 324 |
+
text = _RANGE_CR.sub(
|
| 325 |
+
lambda m: f"{m.group(1)} to {m.group(2)} crore rupees", text
|
| 326 |
+
)
|
| 327 |
+
# Lone "/period" rate slash -> "per period" (do this BEFORE the generic
|
| 328 |
+
# slash->"or" pass so "K/year" doesn't become "K or year").
|
| 329 |
+
text = _RATE_SLASH_PERIOD.sub(lambda m: f" per {m.group(1).lower()}", text)
|
| 330 |
+
return text
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _normalize_abbreviations(text: str) -> str:
|
| 334 |
+
"""Expand spoken-hostile abbreviations + slashes.
|
| 335 |
+
|
| 336 |
+
Runs BEFORE _normalize_money so currency slash-runs become comma lists
|
| 337 |
+
that the money regexes can then expand token-by-token.
|
| 338 |
+
"""
|
| 339 |
+
# Abbreviations first (so a trailing "etc." inside a slash run is gone
|
| 340 |
+
# before slash handling).
|
| 341 |
+
text = _ABBR_EG.sub("for example", text)
|
| 342 |
+
text = _ABBR_IE.sub("that is", text)
|
| 343 |
+
text = _ABBR_ETC.sub(" and so on", text)
|
| 344 |
+
text = _ABBR_VS.sub("versus", text)
|
| 345 |
+
text = _ABBR_APPROX.sub("approximately", text)
|
| 346 |
+
|
| 347 |
+
# Ranges + rate-slashes BEFORE slash/money handling so en-dash ranges
|
| 348 |
+
# and "K/year" survive intact.
|
| 349 |
+
text = _normalize_ranges(text)
|
| 350 |
+
|
| 351 |
+
# Currency/amount slash runs -> comma list with trailing "or".
|
| 352 |
+
text = _normalize_slash_money_run(text)
|
| 353 |
+
|
| 354 |
+
# Any remaining word/number slash -> " or " (parents/siblings, yes / no,
|
| 355 |
+
# work or otherwise). Loop until stable so chained "a/b/c" all convert.
|
| 356 |
+
for _ in range(6):
|
| 357 |
+
new = _SLASH_GENERIC.sub(" or ", text)
|
| 358 |
+
if new == text:
|
| 359 |
+
break
|
| 360 |
+
text = new
|
| 361 |
+
return text
|
| 362 |
+
|
| 363 |
+
|
| 364 |
def _normalize_money(text: str) -> str:
|
| 365 |
"""Turn currency / range shorthand into spoken-language equivalents.
|
| 366 |
|
|
|
|
| 721 |
return cleaned
|
| 722 |
|
| 723 |
|
| 724 |
+
def tts_preprocess(
|
| 725 |
+
text: str,
|
| 726 |
+
language: str = "en",
|
| 727 |
+
max_words: int | None = None,
|
| 728 |
+
) -> str:
|
| 729 |
+
"""Public entry β turn an LLM reply into spoken-language text for TTS.
|
| 730 |
+
|
| 731 |
+
#55 FIX: there is NO premature word cap. The ENTIRE reply is normalized
|
| 732 |
+
and returned so every numbered question is spoken. Sarvam Bulbul v2's
|
| 733 |
+
per-request character limit is enforced *downstream* by chunked
|
| 734 |
+
synthesis in providers/sarvam_tts.py (house style mirrors the STT
|
| 735 |
+
30s-chunking) β we never silently drop trailing content here.
|
| 736 |
+
|
| 737 |
+
`max_words` is kept ONLY for backward call-site compatibility
|
| 738 |
+
(backend/main.py passes max_words=55). It is intentionally IGNORED:
|
| 739 |
+
capping spoken words is exactly the bug we are removing. A caller that
|
| 740 |
+
truly wants a hard cap can still pass an int and we honour it as a
|
| 741 |
+
LAST-RESORT safety ceiling that is far above any real reply
|
| 742 |
+
(>= 100000 effectively never trips); the default None means "speak
|
| 743 |
+
everything".
|
| 744 |
+
"""
|
| 745 |
if not text:
|
| 746 |
return ""
|
| 747 |
# Defense in depth: run the preamble strip again here in case this is
|
|
|
|
| 749 |
# cached reply).
|
| 750 |
cleaned = strip_cot_preamble(text)
|
| 751 |
cleaned = _strip_markdown(cleaned)
|
| 752 |
+
# #56 β abbreviations ("e.g." -> "for example") and slashes
|
| 753 |
+
# ("βΉ5L / βΉ10L" -> list; "yes / no" -> "yes or no") BEFORE money so
|
| 754 |
+
# currency slash-runs become comma lists the money regexes can expand.
|
| 755 |
+
cleaned = _normalize_abbreviations(cleaned)
|
| 756 |
# Currency/range shorthand expansion before acronym handling so βΉ5L
|
| 757 |
# becomes "5 lakhs" instead of getting caught by the bare-L acronym
|
| 758 |
# path.
|
|
|
|
| 763 |
cleaned = _normalize_numbers(cleaned)
|
| 764 |
cleaned = _expand_acronyms(cleaned, language=language)
|
| 765 |
cleaned = _compress_whitespace(cleaned)
|
| 766 |
+
# #55 β DO NOT truncate by default. Only honour an explicit, sane
|
| 767 |
+
# last-resort ceiling if a caller deliberately passes one (we ignore
|
| 768 |
+
# the legacy 55/60 values β those ARE the bug).
|
| 769 |
+
if max_words is not None and max_words >= 100_000:
|
| 770 |
+
cleaned = _truncate_for_voice(cleaned, max_words=max_words)
|
| 771 |
return cleaned
|
|
@@ -170,6 +170,136 @@ const BARGE_IN_GRACE_MS = 600;
|
|
| 170 |
// (~0.02 worst case on speakers) does not.
|
| 171 |
const BARGE_IN_NO_BOTREF_FLOOR = 0.035;
|
| 172 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
// Minimal types for the Web Speech API since lib.dom.d.ts ships them under
|
| 174 |
// `webkitSpeechRecognition` only and the standard `SpeechRecognition` symbol
|
| 175 |
// is still vendor-prefixed in most browsers as of 2026-05.
|
|
@@ -242,6 +372,54 @@ export interface UseStreamingVoiceReturn {
|
|
| 242 |
* send() is in flight.
|
| 243 |
*/
|
| 244 |
consumeBargeInSignal: () => boolean;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
}
|
| 246 |
|
| 247 |
function resolveCtor(): SpeechRecognitionCtor | null {
|
|
@@ -366,6 +544,45 @@ export function useStreamingVoice(
|
|
| 366 |
// for the final ondataavailable chunk before building the blob.
|
| 367 |
const recorderStopWaiterRef = useRef<(() => void) | null>(null);
|
| 368 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 369 |
const [isSupported] = useState<boolean>(() => resolveCtor() !== null);
|
| 370 |
|
| 371 |
const clearRestartTimer = useCallback(() => {
|
|
@@ -587,6 +804,282 @@ export function useStreamingVoice(
|
|
| 587 |
}
|
| 588 |
}, [pickRecorderMime]);
|
| 589 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 590 |
const buildRecognition = useCallback((): SpeechRecognitionInstance | null => {
|
| 591 |
const Ctor = resolveCtor();
|
| 592 |
if (!Ctor) return null;
|
|
@@ -1107,6 +1600,51 @@ export function useStreamingVoice(
|
|
| 1107 |
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 1108 |
}, [enabled]);
|
| 1109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1110 |
// KI-173 (2026-05-15) β heartbeat watchdog. Browser SpeechRecognition
|
| 1111 |
// occasionally enters a stopped state without `onend` firing (certain
|
| 1112 |
// network errors, transient OS audio interruptions, tab visibility
|
|
@@ -1874,8 +2412,11 @@ export function useStreamingVoice(
|
|
| 1874 |
}
|
| 1875 |
pendingUtteranceRef.current = "";
|
| 1876 |
pendingChunksRef.current = [];
|
|
|
|
|
|
|
|
|
|
| 1877 |
};
|
| 1878 |
-
}, [clearRestartTimer, teardownAudio]);
|
| 1879 |
|
| 1880 |
// FIX 3 (HIGH) β one-shot read-and-clear of the barge-in flag. Returns
|
| 1881 |
// true exactly once after triggerBargeIn fires; subsequent calls return
|
|
@@ -1888,5 +2429,17 @@ export function useStreamingVoice(
|
|
| 1888 |
return false;
|
| 1889 |
}, []);
|
| 1890 |
|
| 1891 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1892 |
}
|
|
|
|
| 170 |
// (~0.02 worst case on speakers) does not.
|
| 171 |
const BARGE_IN_NO_BOTREF_FLOOR = 0.035;
|
| 172 |
|
| 173 |
+
// =========================================================================
|
| 174 |
+
// #53 / #54 (2026-05-18) β push-to-talk head-clipping + start-latency fix.
|
| 175 |
+
//
|
| 176 |
+
// ROOT CAUSE (verified):
|
| 177 |
+
// page.tsx's push-to-talk path cold-starts the mic on every SPACE press:
|
| 178 |
+
// page.tsx:1350-1361 onKeyDown(SPACE) β startRecordingRef.current()
|
| 179 |
+
// page.tsx:1004-1019 startRecording() β navigator.mediaDevices
|
| 180 |
+
// .getUserMedia(...) [COLD β 200-700ms on HF Space]
|
| 181 |
+
// page.tsx:1021 new MediaRecorder(stream)
|
| 182 |
+
// page.tsx:1213 recorder.start() [capture truly begins HERE]
|
| 183 |
+
// Every word the user speaks between the keydown and recorder.start()
|
| 184 |
+
// firing is *never captured* β the leading word is lost/garbled (#53,
|
| 185 |
+
// transcribed "S A R" for "Sir."). The same cold-start is the multi-second
|
| 186 |
+
// delay the user feels before recording begins (#54). There is NO pre-roll
|
| 187 |
+
// buffer and NO warm/pre-armed stream anywhere in the codebase.
|
| 188 |
+
//
|
| 189 |
+
// FIX (this hook, since page.tsx is owned by another writer and its PTT path
|
| 190 |
+
// is fully self-contained):
|
| 191 |
+
// - Keep ONE mic stream + MediaRecorder + AudioContext WARM for the hook's
|
| 192 |
+
// entire armed lifetime (acquired once after the user opts into voice,
|
| 193 |
+
// never torn down per-press, survives the LiveβPTT toggle). A persistent
|
| 194 |
+
// open audio device means the OS mic is already hot, so page.tsx's own
|
| 195 |
+
// per-press getUserMedia resolves in ~10-50ms instead of cold-starting
|
| 196 |
+
// (200-700ms) β that alone removes the felt multi-second start delay.
|
| 197 |
+
// - The warm MediaRecorder runs with a short timeslice, feeding a rolling
|
| 198 |
+
// PRE-ROLL ring buffer that always holds the last ~PRE_ROLL_MS of audio.
|
| 199 |
+
// - The PTT API (beginPushToTalk/endPushToTalk) prepends the pre-roll to
|
| 200 |
+
// the captured utterance, so the FIRST WORD β spoken in the cold-start
|
| 201 |
+
// gap β is always in the blob even though page.tsx's recorder missed it.
|
| 202 |
+
// - A DELIBERATE-HOLD gate: beginPushToTalk arms instantly but the capture
|
| 203 |
+
// only "engages" after HOLD_THRESHOLD_MS; a sub-threshold tap (key
|
| 204 |
+
// bounce, accidental press) is discarded and produces no submission.
|
| 205 |
+
// - AudioContext.resume() is kept warm WHILE armed (not lazily on first
|
| 206 |
+
// press), and warm-stream / permission / worklet failures are surfaced
|
| 207 |
+
// via onVoiceError β never silent.
|
| 208 |
+
//
|
| 209 |
+
// The pure pre-roll ring-buffer + hold-gate logic is exported (PreRollRing,
|
| 210 |
+
// evaluateHoldGate) so it is self-contained and independently exercised by
|
| 211 |
+
// the regression test.
|
| 212 |
+
// =========================================================================
|
| 213 |
+
|
| 214 |
+
// Size of the rolling pre-roll buffer. Must comfortably cover the worst-case
|
| 215 |
+
// page.tsx cold-start gap (getUserMedia 200-700ms + MediaRecorder spin-up +
|
| 216 |
+
// the optional 400ms Live-teardown wait at page.tsx:994). 800ms gives margin
|
| 217 |
+
// without bloating the blob (browser webm/opus β 4 KB/s β ~3.2 KB of lead-in).
|
| 218 |
+
export const PRE_ROLL_MS = 800;
|
| 219 |
+
// Warm MediaRecorder timeslice. Small enough that the pre-roll ring has fine
|
| 220 |
+
// granularity (we never drop more than one slice of lead-in when trimming the
|
| 221 |
+
// ring to PRE_ROLL_MS), large enough not to thrash ondataavailable.
|
| 222 |
+
export const WARM_TIMESLICE_MS = 200;
|
| 223 |
+
// Deliberate-hold threshold (#54). The hold must be intentional so an
|
| 224 |
+
// accidental tap / key-bounce doesn't fire a turn, but it must feel instant
|
| 225 |
+
// on a real hold β 200ms sits in the requested 150-250ms band.
|
| 226 |
+
export const HOLD_THRESHOLD_MS = 200;
|
| 227 |
+
|
| 228 |
+
/**
|
| 229 |
+
* PreRollRing β a rolling, time-bounded ring buffer of MediaRecorder Blob
|
| 230 |
+
* slices. `push` appends a freshly-emitted slice (each slice represents
|
| 231 |
+
* ~WARM_TIMESLICE_MS of audio); the ring evicts the oldest slices once the
|
| 232 |
+
* retained wall-clock duration exceeds `windowMs`, so it always holds *at
|
| 233 |
+
* least* the last `windowMs` of audio (it may hold up to one extra slice so
|
| 234 |
+
* a head word that started just before `windowMs` ago is never trimmed).
|
| 235 |
+
*
|
| 236 |
+
* Pure + framework-free so the regression test can drive it directly without
|
| 237 |
+
* a browser. `drain()` returns the retained slices oldest-first and clears
|
| 238 |
+
* the ring (used at PTT-engage to seed the utterance with the lead-in).
|
| 239 |
+
*/
|
| 240 |
+
export class PreRollRing {
|
| 241 |
+
private slices: Array<{ blob: Blob; ms: number }> = [];
|
| 242 |
+
private retainedMs = 0;
|
| 243 |
+
private readonly windowMs: number;
|
| 244 |
+
constructor(windowMs: number = PRE_ROLL_MS) {
|
| 245 |
+
this.windowMs = windowMs;
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
push(blob: Blob, sliceMs: number = WARM_TIMESLICE_MS): void {
|
| 249 |
+
if (!blob || blob.size <= 0) return;
|
| 250 |
+
this.slices.push({ blob, ms: sliceMs });
|
| 251 |
+
this.retainedMs += sliceMs;
|
| 252 |
+
// Evict from the front while doing so still leaves >= windowMs retained
|
| 253 |
+
// (keep one extra slice of slack so a word that began just before the
|
| 254 |
+
// window boundary survives β never trim into the requested lead-in).
|
| 255 |
+
while (
|
| 256 |
+
this.slices.length > 1 &&
|
| 257 |
+
this.retainedMs - this.slices[0].ms >= this.windowMs
|
| 258 |
+
) {
|
| 259 |
+
const dropped = this.slices.shift();
|
| 260 |
+
if (dropped) this.retainedMs -= dropped.ms;
|
| 261 |
+
}
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
/** Retained lead-in slices oldest-first; clears the ring. */
|
| 265 |
+
drain(): Blob[] {
|
| 266 |
+
const out = this.slices.map((s) => s.blob);
|
| 267 |
+
this.slices = [];
|
| 268 |
+
this.retainedMs = 0;
|
| 269 |
+
return out;
|
| 270 |
+
}
|
| 271 |
+
|
| 272 |
+
/** Approximate retained wall-clock duration (ms). */
|
| 273 |
+
retainedDurationMs(): number {
|
| 274 |
+
return this.retainedMs;
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
clear(): void {
|
| 278 |
+
this.slices = [];
|
| 279 |
+
this.retainedMs = 0;
|
| 280 |
+
}
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
/**
|
| 284 |
+
* evaluateHoldGate β pure decision for the deliberate-hold threshold (#54).
|
| 285 |
+
*
|
| 286 |
+
* Given when the user engaged (pressed) and released, decide whether the
|
| 287 |
+
* press was a DELIBERATE hold (capture should be submitted) or a sub-threshold
|
| 288 |
+
* TAP (discard β accidental press / key bounce). Kept pure so the regression
|
| 289 |
+
* test can assert the boundary exactly without timers.
|
| 290 |
+
*
|
| 291 |
+
* heldMs >= thresholdMs β { deliberate: true } (engage + submit)
|
| 292 |
+
* heldMs < thresholdMs β { deliberate: false } (discard, no submit)
|
| 293 |
+
*/
|
| 294 |
+
export function evaluateHoldGate(
|
| 295 |
+
pressedAt: number,
|
| 296 |
+
releasedAt: number,
|
| 297 |
+
thresholdMs: number = HOLD_THRESHOLD_MS,
|
| 298 |
+
): { deliberate: boolean; heldMs: number } {
|
| 299 |
+
const heldMs = Math.max(0, releasedAt - pressedAt);
|
| 300 |
+
return { deliberate: heldMs >= thresholdMs, heldMs };
|
| 301 |
+
}
|
| 302 |
+
|
| 303 |
// Minimal types for the Web Speech API since lib.dom.d.ts ships them under
|
| 304 |
// `webkitSpeechRecognition` only and the standard `SpeechRecognition` symbol
|
| 305 |
// is still vendor-prefixed in most browsers as of 2026-05.
|
|
|
|
| 372 |
* send() is in flight.
|
| 373 |
*/
|
| 374 |
consumeBargeInSignal: () => boolean;
|
| 375 |
+
|
| 376 |
+
// ----------------------------------------------------------------------
|
| 377 |
+
// #53 / #54 β warm-stream + pre-roll push-to-talk API.
|
| 378 |
+
//
|
| 379 |
+
// This is the minimal API the push-to-talk UI integrates with. Even
|
| 380 |
+
// without an explicit call, `armWarmStream()` is invoked autonomously by
|
| 381 |
+
// the hook once voice has been enabled, so the OS mic device is kept hot
|
| 382 |
+
// for the rest of the session β that removes the per-press cold-start that
|
| 383 |
+
// page.tsx's own getUserMedia otherwise pays (the felt multi-second delay,
|
| 384 |
+
// #54) and continuously fills the pre-roll ring so the leading word spoken
|
| 385 |
+
// in the cold-start gap survives (#53).
|
| 386 |
+
// ----------------------------------------------------------------------
|
| 387 |
+
|
| 388 |
+
/** True once the warm mic stream + recorder + AudioContext are live and
|
| 389 |
+
* the pre-roll ring is filling. */
|
| 390 |
+
isWarm: boolean;
|
| 391 |
+
|
| 392 |
+
/** Pre-arm (or re-arm) the persistent warm stream. Idempotent; safe to
|
| 393 |
+
* call repeatedly. Resolves true when the warm stream is recording. */
|
| 394 |
+
armWarmStream: () => Promise<boolean>;
|
| 395 |
+
|
| 396 |
+
/** Release the warm stream + recorder + AudioContext (mic indicator off).
|
| 397 |
+
* Called on unmount; callers may call it to fully relinquish the mic. */
|
| 398 |
+
disarmWarmStream: () => void;
|
| 399 |
+
|
| 400 |
+
/**
|
| 401 |
+
* Engage a push-to-talk capture. Call on hold-start (e.g. SPACE keydown).
|
| 402 |
+
* Returns immediately. The capture *engages* only after HOLD_THRESHOLD_MS
|
| 403 |
+
* so a sub-threshold tap is ignored; the engaged utterance is seeded with
|
| 404 |
+
* the pre-roll ring so the first word (spoken during the cold-start gap)
|
| 405 |
+
* is always included.
|
| 406 |
+
*/
|
| 407 |
+
beginPushToTalk: () => void;
|
| 408 |
+
|
| 409 |
+
/**
|
| 410 |
+
* End a push-to-talk capture. Call on hold-release (e.g. SPACE keyup).
|
| 411 |
+
* If the hold was deliberate (>= HOLD_THRESHOLD_MS) the assembled blob
|
| 412 |
+
* (pre-roll + live capture) is transcribed and delivered via
|
| 413 |
+
* onFinalTranscript; a sub-threshold tap resolves to null and submits
|
| 414 |
+
* nothing. Resolves with the final transcript, or null when discarded /
|
| 415 |
+
* empty.
|
| 416 |
+
*/
|
| 417 |
+
endPushToTalk: () => Promise<string | null>;
|
| 418 |
+
|
| 419 |
+
/** Snapshot+drain the current pre-roll ring (oldest-first). Exposed for
|
| 420 |
+
* the regression test and any caller that wants to splice the lead-in
|
| 421 |
+
* into its own recorder blob. */
|
| 422 |
+
consumePreRollChunks: () => Blob[];
|
| 423 |
}
|
| 424 |
|
| 425 |
function resolveCtor(): SpeechRecognitionCtor | null {
|
|
|
|
| 544 |
// for the final ondataavailable chunk before building the blob.
|
| 545 |
const recorderStopWaiterRef = useRef<(() => void) | null>(null);
|
| 546 |
|
| 547 |
+
// ----------------------------------------------------------------------
|
| 548 |
+
// #53 / #54 β warm-stream + pre-roll push-to-talk state.
|
| 549 |
+
//
|
| 550 |
+
// SEPARATE from the Live-mode mediaStream/mediaRecorder above. The Live
|
| 551 |
+
// recorder is acquired/torn-down per utterance and is gated on the
|
| 552 |
+
// `enabled` prop (which page.tsx flips OFF during push-to-talk). This warm
|
| 553 |
+
// stream is the OPPOSITE lifecycle: opened once after the user opts into
|
| 554 |
+
// voice, kept alive across the LiveβPTT toggle for the hook's mounted
|
| 555 |
+
// lifetime, never closed per-press. Holding a persistent open audio device
|
| 556 |
+
// keeps the OS mic hot so any per-press getUserMedia (Live's OR page.tsx's
|
| 557 |
+
// PTT) resolves near-instantly instead of cold-starting.
|
| 558 |
+
// ----------------------------------------------------------------------
|
| 559 |
+
const warmStreamRef = useRef<MediaStream | null>(null);
|
| 560 |
+
const warmRecorderRef = useRef<MediaRecorder | null>(null);
|
| 561 |
+
const warmCtxRef = useRef<AudioContext | null>(null);
|
| 562 |
+
const warmMimeRef = useRef<string>("audio/webm");
|
| 563 |
+
// The rolling pre-roll ring β always holds ~PRE_ROLL_MS of the most recent
|
| 564 |
+
// audio so a PTT engage can prepend the lead-in the user spoke during the
|
| 565 |
+
// cold-start gap.
|
| 566 |
+
const preRollRef = useRef<PreRollRing>(new PreRollRing(PRE_ROLL_MS));
|
| 567 |
+
// Live capture slices accumulated between PTT engage and release. The
|
| 568 |
+
// submitted blob is preRoll.drain() (lead-in) ++ these (live capture).
|
| 569 |
+
const pttCaptureRef = useRef<Blob[]>([]);
|
| 570 |
+
// True between a deliberate engage and the matching release β the warm
|
| 571 |
+
// recorder's ondataavailable routes slices to pttCaptureRef instead of
|
| 572 |
+
// (only) the pre-roll ring while this is set.
|
| 573 |
+
const pttEngagedRef = useRef<boolean>(false);
|
| 574 |
+
// wall-clock ms of the current hold's keydown (0 when not pressed). Used
|
| 575 |
+
// by evaluateHoldGate to classify deliberate hold vs sub-threshold tap.
|
| 576 |
+
const pttPressedAtRef = useRef<number>(0);
|
| 577 |
+
// setTimeout id for the deliberate-hold engage. Fires HOLD_THRESHOLD_MS
|
| 578 |
+
// after press; if release beats it, the press was a tap and is discarded.
|
| 579 |
+
const pttHoldTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
| 580 |
+
// True once the user has opted into voice at least once. Latches the warm
|
| 581 |
+
// stream ON for the rest of the hook's mounted lifetime so it survives the
|
| 582 |
+
// LiveβPTT toggle (page.tsx flips `enabled` false for pure PTT).
|
| 583 |
+
const voiceEverEnabledRef = useRef<boolean>(false);
|
| 584 |
+
const [isWarm, setIsWarm] = useState<boolean>(false);
|
| 585 |
+
|
| 586 |
const [isSupported] = useState<boolean>(() => resolveCtor() !== null);
|
| 587 |
|
| 588 |
const clearRestartTimer = useCallback(() => {
|
|
|
|
| 804 |
}
|
| 805 |
}, [pickRecorderMime]);
|
| 806 |
|
| 807 |
+
// ======================================================================
|
| 808 |
+
// #53 / #54 β warm-stream + pre-roll push-to-talk engine.
|
| 809 |
+
// ======================================================================
|
| 810 |
+
|
| 811 |
+
const disarmWarmStream = useCallback(() => {
|
| 812 |
+
if (pttHoldTimerRef.current !== null) {
|
| 813 |
+
clearTimeout(pttHoldTimerRef.current);
|
| 814 |
+
pttHoldTimerRef.current = null;
|
| 815 |
+
}
|
| 816 |
+
pttEngagedRef.current = false;
|
| 817 |
+
pttPressedAtRef.current = 0;
|
| 818 |
+
pttCaptureRef.current = [];
|
| 819 |
+
preRollRef.current.clear();
|
| 820 |
+
const rec = warmRecorderRef.current;
|
| 821 |
+
if (rec) {
|
| 822 |
+
try {
|
| 823 |
+
rec.ondataavailable = null;
|
| 824 |
+
rec.onerror = null;
|
| 825 |
+
rec.onstop = null;
|
| 826 |
+
if (rec.state !== "inactive") rec.stop();
|
| 827 |
+
} catch {
|
| 828 |
+
/* ignore */
|
| 829 |
+
}
|
| 830 |
+
}
|
| 831 |
+
warmRecorderRef.current = null;
|
| 832 |
+
const stream = warmStreamRef.current;
|
| 833 |
+
if (stream) {
|
| 834 |
+
stream.getTracks().forEach((t) => {
|
| 835 |
+
try { t.stop(); } catch { /* ignore */ }
|
| 836 |
+
});
|
| 837 |
+
}
|
| 838 |
+
warmStreamRef.current = null;
|
| 839 |
+
const ctx = warmCtxRef.current;
|
| 840 |
+
if (ctx) {
|
| 841 |
+
warmCtxRef.current = null;
|
| 842 |
+
try { void ctx.close(); } catch { /* ignore */ }
|
| 843 |
+
}
|
| 844 |
+
setIsWarm(false);
|
| 845 |
+
}, []);
|
| 846 |
+
|
| 847 |
+
// Acquire (or re-acquire) the persistent warm stream. Idempotent: a
|
| 848 |
+
// healthy recording warm recorder short-circuits. On failure routes
|
| 849 |
+
// through the SAME onVoiceError("mic_permission_denied") contract the
|
| 850 |
+
// Live path uses β never a silent failure.
|
| 851 |
+
const armWarmStream = useCallback(async (): Promise<boolean> => {
|
| 852 |
+
voiceEverEnabledRef.current = true;
|
| 853 |
+
const existing = warmRecorderRef.current;
|
| 854 |
+
if (existing && existing.state === "recording" && warmStreamRef.current) {
|
| 855 |
+
return true;
|
| 856 |
+
}
|
| 857 |
+
if (typeof navigator === "undefined" || !navigator.mediaDevices) return false;
|
| 858 |
+
if (typeof MediaRecorder === "undefined") return false;
|
| 859 |
+
// Tear down any half-built prior attempt before re-acquiring.
|
| 860 |
+
if (existing || warmStreamRef.current) disarmWarmStream();
|
| 861 |
+
try {
|
| 862 |
+
// Same AEC/NS/AGC constraints as the Live + PTT paths (KI-185) so the
|
| 863 |
+
// pre-roll is echo-cancelled identically to the rest of the capture.
|
| 864 |
+
// W2-style 2s stall watchdog so a hung getUserMedia surfaces a banner
|
| 865 |
+
// instead of pinning the warm state forever.
|
| 866 |
+
const stream: MediaStream = await Promise.race([
|
| 867 |
+
navigator.mediaDevices.getUserMedia({
|
| 868 |
+
audio: {
|
| 869 |
+
echoCancellation: true,
|
| 870 |
+
noiseSuppression: true,
|
| 871 |
+
autoGainControl: true,
|
| 872 |
+
},
|
| 873 |
+
}),
|
| 874 |
+
new Promise<MediaStream>((_, reject) => {
|
| 875 |
+
setTimeout(() => {
|
| 876 |
+
const e = new Error("warm getUserMedia stalled >2s") as Error & { name: string };
|
| 877 |
+
e.name = "StallTimeout";
|
| 878 |
+
reject(e);
|
| 879 |
+
}, 2000);
|
| 880 |
+
}),
|
| 881 |
+
]);
|
| 882 |
+
const mime = pickRecorderMime();
|
| 883 |
+
warmMimeRef.current = mime || "audio/webm";
|
| 884 |
+
const recorder = mime
|
| 885 |
+
? new MediaRecorder(stream, { mimeType: mime })
|
| 886 |
+
: new MediaRecorder(stream);
|
| 887 |
+
preRollRef.current = new PreRollRing(PRE_ROLL_MS);
|
| 888 |
+
pttCaptureRef.current = [];
|
| 889 |
+
pttEngagedRef.current = false;
|
| 890 |
+
recorder.ondataavailable = (ev: BlobEvent) => {
|
| 891 |
+
if (!ev.data || ev.data.size <= 0) return;
|
| 892 |
+
// Always feed the rolling pre-roll ring so the lead-in is ready the
|
| 893 |
+
// instant a PTT engage fires (the word spoken in the cold-start gap
|
| 894 |
+
// is in here). When a PTT capture is engaged, ALSO accumulate the
|
| 895 |
+
// slice into the live capture buffer β the submitted blob is
|
| 896 |
+
// preRoll.drain() (lead-in) ++ pttCaptureRef (live), so the first
|
| 897 |
+
// word is never lost AND no chunk is dropped.
|
| 898 |
+
preRollRef.current.push(ev.data, WARM_TIMESLICE_MS);
|
| 899 |
+
if (pttEngagedRef.current) {
|
| 900 |
+
pttCaptureRef.current.push(ev.data);
|
| 901 |
+
}
|
| 902 |
+
};
|
| 903 |
+
recorder.onerror = (ev: Event) => {
|
| 904 |
+
console.debug("[useStreamingVoice] warm MediaRecorder error", ev);
|
| 905 |
+
try { onVoiceErrorRef.current("stream_stale"); } catch { /* ignore */ }
|
| 906 |
+
};
|
| 907 |
+
recorder.onstop = () => {
|
| 908 |
+
// The warm recorder should never stop on its own while armed; if it
|
| 909 |
+
// does (device unplug, OS interruption) surface it and let the
|
| 910 |
+
// re-arm effect / next press recover.
|
| 911 |
+
console.debug("[useStreamingVoice] warm MediaRecorder stopped");
|
| 912 |
+
};
|
| 913 |
+
warmStreamRef.current = stream;
|
| 914 |
+
warmRecorderRef.current = recorder;
|
| 915 |
+
recorder.start(WARM_TIMESLICE_MS);
|
| 916 |
+
if (recorder.state !== "recording") {
|
| 917 |
+
try { stream.getTracks().forEach((t) => t.stop()); } catch { /* ignore */ }
|
| 918 |
+
warmStreamRef.current = null;
|
| 919 |
+
warmRecorderRef.current = null;
|
| 920 |
+
throw Object.assign(
|
| 921 |
+
new Error(`warm MediaRecorder not recording (got ${recorder.state})`),
|
| 922 |
+
{ name: "RecorderNotRecording" },
|
| 923 |
+
);
|
| 924 |
+
}
|
| 925 |
+
// Keep an AudioContext warm + RUNNING so it never has to be resumed
|
| 926 |
+
// lazily on first press (a suspended ctx is one of the documented
|
| 927 |
+
// first-word-loss vectors). resume() needs a user gesture on some
|
| 928 |
+
// browsers; armWarmStream is always called from one (voice toggle).
|
| 929 |
+
try {
|
| 930 |
+
const Ctor = (window.AudioContext
|
| 931 |
+
|| (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext);
|
| 932 |
+
if (Ctor) {
|
| 933 |
+
if (!warmCtxRef.current || warmCtxRef.current.state === "closed") {
|
| 934 |
+
warmCtxRef.current = new Ctor();
|
| 935 |
+
}
|
| 936 |
+
if (warmCtxRef.current.state === "suspended") {
|
| 937 |
+
void warmCtxRef.current.resume().catch((err) => {
|
| 938 |
+
console.debug("[useStreamingVoice] warm AudioContext.resume failed", err);
|
| 939 |
+
try { onVoiceErrorRef.current("audio_context_suspended"); } catch { /* ignore */ }
|
| 940 |
+
});
|
| 941 |
+
}
|
| 942 |
+
}
|
| 943 |
+
} catch {
|
| 944 |
+
/* AudioContext is best-effort for warmth; capture still works */
|
| 945 |
+
}
|
| 946 |
+
setIsWarm(true);
|
| 947 |
+
console.debug("[useStreamingVoice] warm stream armed", {
|
| 948 |
+
mime: warmMimeRef.current,
|
| 949 |
+
preRollMs: PRE_ROLL_MS,
|
| 950 |
+
timesliceMs: WARM_TIMESLICE_MS,
|
| 951 |
+
});
|
| 952 |
+
return true;
|
| 953 |
+
} catch (err) {
|
| 954 |
+
const name = (err as { name?: string } | null)?.name ?? "Error";
|
| 955 |
+
console.debug("[useStreamingVoice] warm stream arm failed", { name, err });
|
| 956 |
+
setIsWarm(false);
|
| 957 |
+
try {
|
| 958 |
+
onVoiceErrorRef.current("mic_permission_denied" as VoiceError);
|
| 959 |
+
} catch {
|
| 960 |
+
/* never let a user callback crash the hook */
|
| 961 |
+
}
|
| 962 |
+
return false;
|
| 963 |
+
}
|
| 964 |
+
}, [pickRecorderMime, disarmWarmStream]);
|
| 965 |
+
|
| 966 |
+
const consumePreRollChunks = useCallback((): Blob[] => {
|
| 967 |
+
return preRollRef.current.drain();
|
| 968 |
+
}, []);
|
| 969 |
+
|
| 970 |
+
// Submit an assembled PTT blob through the SAME Sarvam-with-retry path the
|
| 971 |
+
// Live grace-timer uses (KI-226/302), then deliver via onFinalTranscript.
|
| 972 |
+
// Returns the authoritative transcript or null.
|
| 973 |
+
const submitPttBlob = useCallback(
|
| 974 |
+
async (chunks: Blob[]): Promise<string | null> => {
|
| 975 |
+
if (chunks.length === 0) return null;
|
| 976 |
+
const blob = new Blob(chunks, { type: warmMimeRef.current || "audio/webm" });
|
| 977 |
+
// ~3 KB empirical noise floor (same as the Live path / PTT KI-134).
|
| 978 |
+
const MIN_BLOB_BYTES = 3000;
|
| 979 |
+
if (blob.size < MIN_BLOB_BYTES) {
|
| 980 |
+
console.debug("[useStreamingVoice] PTT blob below noise floor β discard", {
|
| 981 |
+
bytes: blob.size,
|
| 982 |
+
});
|
| 983 |
+
return null;
|
| 984 |
+
}
|
| 985 |
+
await waitForTextClear();
|
| 986 |
+
const APPROX_BYTES_PER_CHUNK = 100_000; // ~25s of webm/opus
|
| 987 |
+
const estChunks = Math.max(1, Math.ceil(blob.size / APPROX_BYTES_PER_CHUNK));
|
| 988 |
+
const attemptTimeoutMs = Math.min(120_000, 8_000 + estChunks * 12_000);
|
| 989 |
+
let authoritative: string | null = null;
|
| 990 |
+
const sarvam = await retryPostTranscribe(async (signal) => {
|
| 991 |
+
const timeoutCtl = new AbortController();
|
| 992 |
+
const timer = setTimeout(() => timeoutCtl.abort(), attemptTimeoutMs);
|
| 993 |
+
const onOuterAbort = () => timeoutCtl.abort();
|
| 994 |
+
signal.addEventListener("abort", onOuterAbort);
|
| 995 |
+
try {
|
| 996 |
+
return await postTranscribe(blob, language, timeoutCtl.signal);
|
| 997 |
+
} finally {
|
| 998 |
+
clearTimeout(timer);
|
| 999 |
+
signal.removeEventListener("abort", onOuterAbort);
|
| 1000 |
+
}
|
| 1001 |
+
});
|
| 1002 |
+
if (sarvam) {
|
| 1003 |
+
const t = (sarvam.text || "").trim();
|
| 1004 |
+
if (t) authoritative = t;
|
| 1005 |
+
} else {
|
| 1006 |
+
try { onVoiceErrorRef.current("transcribe_failed"); } catch { /* ignore */ }
|
| 1007 |
+
}
|
| 1008 |
+
if (authoritative) {
|
| 1009 |
+
await waitForTextClear();
|
| 1010 |
+
onFinalRef.current(authoritative);
|
| 1011 |
+
}
|
| 1012 |
+
return authoritative;
|
| 1013 |
+
},
|
| 1014 |
+
[language, waitForTextClear],
|
| 1015 |
+
);
|
| 1016 |
+
|
| 1017 |
+
// PTT engage β called HOLD_THRESHOLD_MS after a deliberate press. Snapshots
|
| 1018 |
+
// the pre-roll (lead-in spoken during the cold-start gap) into the live
|
| 1019 |
+
// capture buffer and flips the recorder's slice routing to also accumulate.
|
| 1020 |
+
const engagePtt = useCallback(() => {
|
| 1021 |
+
pttEngagedRef.current = true;
|
| 1022 |
+
// Seed the capture with the pre-roll lead-in FIRST so the first word
|
| 1023 |
+
// (which page.tsx's cold-started recorder would have missed) is at the
|
| 1024 |
+
// head of the submitted blob.
|
| 1025 |
+
const leadIn = preRollRef.current.drain();
|
| 1026 |
+
pttCaptureRef.current = [...leadIn];
|
| 1027 |
+
console.debug("[useStreamingVoice] PTT engaged", {
|
| 1028 |
+
leadInSlices: leadIn.length,
|
| 1029 |
+
});
|
| 1030 |
+
}, []);
|
| 1031 |
+
|
| 1032 |
+
const beginPushToTalk = useCallback(() => {
|
| 1033 |
+
pttPressedAtRef.current = Date.now();
|
| 1034 |
+
pttCaptureRef.current = [];
|
| 1035 |
+
pttEngagedRef.current = false;
|
| 1036 |
+
// Make sure the warm stream is up so the pre-roll is actually filling.
|
| 1037 |
+
// armWarmStream is idempotent + fast when already warm.
|
| 1038 |
+
void armWarmStream();
|
| 1039 |
+
if (pttHoldTimerRef.current !== null) {
|
| 1040 |
+
clearTimeout(pttHoldTimerRef.current);
|
| 1041 |
+
}
|
| 1042 |
+
// Deliberate-hold gate: engage only after the threshold so a sub-150ms
|
| 1043 |
+
// tap does nothing. The capture still feels instant because the pre-roll
|
| 1044 |
+
// ring already holds the audio spoken during these HOLD_THRESHOLD_MS.
|
| 1045 |
+
pttHoldTimerRef.current = setTimeout(() => {
|
| 1046 |
+
pttHoldTimerRef.current = null;
|
| 1047 |
+
// Re-check the press is still held (release clears pttPressedAtRef).
|
| 1048 |
+
if (pttPressedAtRef.current !== 0) engagePtt();
|
| 1049 |
+
}, HOLD_THRESHOLD_MS);
|
| 1050 |
+
}, [armWarmStream, engagePtt]);
|
| 1051 |
+
|
| 1052 |
+
const endPushToTalk = useCallback(async (): Promise<string | null> => {
|
| 1053 |
+
const pressedAt = pttPressedAtRef.current;
|
| 1054 |
+
const releasedAt = Date.now();
|
| 1055 |
+
pttPressedAtRef.current = 0;
|
| 1056 |
+
if (pttHoldTimerRef.current !== null) {
|
| 1057 |
+
clearTimeout(pttHoldTimerRef.current);
|
| 1058 |
+
pttHoldTimerRef.current = null;
|
| 1059 |
+
}
|
| 1060 |
+
const { deliberate, heldMs } = evaluateHoldGate(
|
| 1061 |
+
pressedAt || releasedAt,
|
| 1062 |
+
releasedAt,
|
| 1063 |
+
HOLD_THRESHOLD_MS,
|
| 1064 |
+
);
|
| 1065 |
+
const wasEngaged = pttEngagedRef.current;
|
| 1066 |
+
pttEngagedRef.current = false;
|
| 1067 |
+
if (!deliberate || !wasEngaged) {
|
| 1068 |
+
// Sub-threshold tap (or release before engage fired): discard. The
|
| 1069 |
+
// pre-roll ring keeps rolling for the warm stream; nothing submitted.
|
| 1070 |
+
console.debug("[useStreamingVoice] PTT discarded (tap)", {
|
| 1071 |
+
heldMs,
|
| 1072 |
+
deliberate,
|
| 1073 |
+
wasEngaged,
|
| 1074 |
+
});
|
| 1075 |
+
pttCaptureRef.current = [];
|
| 1076 |
+
return null;
|
| 1077 |
+
}
|
| 1078 |
+
const captured = pttCaptureRef.current;
|
| 1079 |
+
pttCaptureRef.current = [];
|
| 1080 |
+
return submitPttBlob(captured);
|
| 1081 |
+
}, [submitPttBlob]);
|
| 1082 |
+
|
| 1083 |
const buildRecognition = useCallback((): SpeechRecognitionInstance | null => {
|
| 1084 |
const Ctor = resolveCtor();
|
| 1085 |
if (!Ctor) return null;
|
|
|
|
| 1600 |
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 1601 |
}, [enabled]);
|
| 1602 |
|
| 1603 |
+
// #53 / #54 β warm-stream lifecycle. The warm stream's lifecycle is
|
| 1604 |
+
// DELIBERATELY decoupled from `enabled` (which page.tsx flips OFF for pure
|
| 1605 |
+
// push-to-talk β see page.tsx:986 `live.setLive(false)` inside
|
| 1606 |
+
// startRecording). The user opting into voice latches the warm stream ON
|
| 1607 |
+
// for the rest of the hook's mounted lifetime so:
|
| 1608 |
+
// (a) the pre-roll ring is ALWAYS filling whenever the user might press
|
| 1609 |
+
// SPACE β including in pure-PTT mode when `enabled` is false β so the
|
| 1610 |
+
// first word spoken in page.tsx's cold-start gap survives (#53);
|
| 1611 |
+
// (b) a persistent open audio device keeps the OS mic hot so page.tsx's
|
| 1612 |
+
// own per-press getUserMedia resolves in ~10-50ms instead of
|
| 1613 |
+
// cold-starting (200-700ms), removing the felt start delay (#54).
|
| 1614 |
+
// Armed on the rising edge of `enabled` (the only voice-opt-in signal the
|
| 1615 |
+
// hook receives) and kept armed thereafter; fully released on unmount.
|
| 1616 |
+
useEffect(() => {
|
| 1617 |
+
if (!isSupported) return;
|
| 1618 |
+
if (enabled) {
|
| 1619 |
+
voiceEverEnabledRef.current = true;
|
| 1620 |
+
}
|
| 1621 |
+
if (enabled || voiceEverEnabledRef.current) {
|
| 1622 |
+
void armWarmStream();
|
| 1623 |
+
}
|
| 1624 |
+
// No teardown on `enabled` going false β the warm stream must survive
|
| 1625 |
+
// the LiveβPTT toggle. Final release happens in the unmount cleanup.
|
| 1626 |
+
}, [enabled, isSupported, armWarmStream]);
|
| 1627 |
+
|
| 1628 |
+
// #53 / #54 β warm-stream health watchdog. The OS can silently drop a
|
| 1629 |
+
// long-lived capture (device sleep, USB mic unplug, OS audio interruption,
|
| 1630 |
+
// tab backgrounding on some browsers) WITHOUT firing recorder.onerror. If
|
| 1631 |
+
// that happens the pre-roll ring goes stale and the very bug we fixed
|
| 1632 |
+
// returns. Every 4s, while voice has been opted into and we're not in the
|
| 1633 |
+
// middle of a PTT capture, re-assert the warm stream (armWarmStream is a
|
| 1634 |
+
// no-op when the recorder is healthily "recording").
|
| 1635 |
+
useEffect(() => {
|
| 1636 |
+
if (!isSupported) return;
|
| 1637 |
+
const tick = setInterval(() => {
|
| 1638 |
+
if (!voiceEverEnabledRef.current) return;
|
| 1639 |
+
if (pttEngagedRef.current) return; // don't disturb an in-flight capture
|
| 1640 |
+
const rec = warmRecorderRef.current;
|
| 1641 |
+
if (!rec || rec.state !== "recording" || !warmStreamRef.current) {
|
| 1642 |
+
void armWarmStream();
|
| 1643 |
+
}
|
| 1644 |
+
}, 4000);
|
| 1645 |
+
return () => clearInterval(tick);
|
| 1646 |
+
}, [isSupported, armWarmStream]);
|
| 1647 |
+
|
| 1648 |
// KI-173 (2026-05-15) β heartbeat watchdog. Browser SpeechRecognition
|
| 1649 |
// occasionally enters a stopped state without `onend` firing (certain
|
| 1650 |
// network errors, transient OS audio interruptions, tab visibility
|
|
|
|
| 2412 |
}
|
| 2413 |
pendingUtteranceRef.current = "";
|
| 2414 |
pendingChunksRef.current = [];
|
| 2415 |
+
// #53 / #54 β release the warm stream + recorder + AudioContext on
|
| 2416 |
+
// unmount so the OS mic indicator goes off when the app is torn down.
|
| 2417 |
+
disarmWarmStream();
|
| 2418 |
};
|
| 2419 |
+
}, [clearRestartTimer, teardownAudio, disarmWarmStream]);
|
| 2420 |
|
| 2421 |
// FIX 3 (HIGH) β one-shot read-and-clear of the barge-in flag. Returns
|
| 2422 |
// true exactly once after triggerBargeIn fires; subsequent calls return
|
|
|
|
| 2429 |
return false;
|
| 2430 |
}, []);
|
| 2431 |
|
| 2432 |
+
return {
|
| 2433 |
+
start,
|
| 2434 |
+
stop,
|
| 2435 |
+
isSupported,
|
| 2436 |
+
consumeBargeInSignal,
|
| 2437 |
+
// #53 / #54 β warm-stream + pre-roll push-to-talk API.
|
| 2438 |
+
isWarm,
|
| 2439 |
+
armWarmStream,
|
| 2440 |
+
disarmWarmStream,
|
| 2441 |
+
beginPushToTalk,
|
| 2442 |
+
endPushToTalk,
|
| 2443 |
+
consumePreRollChunks,
|
| 2444 |
+
};
|
| 2445 |
}
|
|
@@ -0,0 +1,457 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Regression test for #53 (push-to-talk head-clipping) + #54 (PTT start
|
| 2 |
+
latency) β 2026-05-18.
|
| 3 |
+
|
| 4 |
+
TWO LIVE BUGS, ONE ROOT CAUSE
|
| 5 |
+
-----------------------------------------------------------------------------
|
| 6 |
+
page.tsx's push-to-talk path cold-starts the mic on every SPACE press:
|
| 7 |
+
|
| 8 |
+
page.tsx:1350-1361 onKeyDown(SPACE) -> startRecordingRef.current()
|
| 9 |
+
page.tsx:1004-1019 startRecording() -> navigator.mediaDevices
|
| 10 |
+
.getUserMedia(...) [COLD: 200-700ms]
|
| 11 |
+
page.tsx:1021 new MediaRecorder(stream)
|
| 12 |
+
page.tsx:1213 recorder.start() [capture truly begins HERE]
|
| 13 |
+
|
| 14 |
+
Every word spoken between the keydown and recorder.start() is *never
|
| 15 |
+
captured*. Real repro: user said "Sir. My age is 29 ..." -> transcribed
|
| 16 |
+
"S A R. My age is 29 ..." (#53). The same cold-start is the multi-second
|
| 17 |
+
delay before recording starts (#54).
|
| 18 |
+
|
| 19 |
+
FIX (frontend/src/lib/useStreamingVoice.ts)
|
| 20 |
+
-----------------------------------------------------------------------------
|
| 21 |
+
Keep ONE mic stream + MediaRecorder + AudioContext WARM for the hook's armed
|
| 22 |
+
lifetime, feeding a rolling PRE-ROLL ring buffer (PreRollRing) that always
|
| 23 |
+
holds ~PRE_ROLL_MS of the most recent audio. A deliberate-hold gate
|
| 24 |
+
(evaluateHoldGate) ignores sub-threshold taps. On a real hold the submitted
|
| 25 |
+
blob is preRoll.drain() (lead-in spoken during the cold-start gap) ++ the
|
| 26 |
+
live capture, so the FIRST WORD always survives.
|
| 27 |
+
|
| 28 |
+
WHAT THIS TEST PINS
|
| 29 |
+
-----------------------------------------------------------------------------
|
| 30 |
+
There is no JS test runner in this repo, so the two contracts are pinned at
|
| 31 |
+
the layers a Python test can reach honestly:
|
| 32 |
+
|
| 33 |
+
PART A (pure frontend logic, executed against the REAL shipped code):
|
| 34 |
+
The exported PreRollRing + evaluateHoldGate from useStreamingVoice.ts are
|
| 35 |
+
loaded under Node (TS types stripped; react/./api/./voice_resilience
|
| 36 |
+
stubbed since the PURE exports don't use them) and asserted:
|
| 37 |
+
A1 a pre-roll ring fed lead-in slices then drained returns the
|
| 38 |
+
LEADING slice first -> the first word is at the head of the blob.
|
| 39 |
+
A2 the ring retains >= PRE_ROLL_MS of audio (covers the worst-case
|
| 40 |
+
page.tsx cold-start gap) and evicts only OLDER audio.
|
| 41 |
+
A3 evaluateHoldGate: a >= threshold hold is deliberate (submit); a
|
| 42 |
+
sub-threshold tap is NOT (gated, nothing submitted).
|
| 43 |
+
A4 the integrated contract: engage-after-threshold seeds the
|
| 44 |
+
capture with the pre-roll so leadIn[0] precedes live audio;
|
| 45 |
+
a tap drains nothing and submits nothing.
|
| 46 |
+
|
| 47 |
+
PART B (backend STT, the bug's observable failure surface):
|
| 48 |
+
The fix prepends a short pre-roll HEAD to the blob. Prove
|
| 49 |
+
backend/providers/sarvam_stt.py never silently drops/truncates that
|
| 50 |
+
head β i.e. the FIRST word survives transcription, including when the
|
| 51 |
+
full utterance is long enough to hit the silence-chunk splitter (the
|
| 52 |
+
pre-fix 30s truncation dropped tail words; a regression that mishandles
|
| 53 |
+
the prepended head would drop the FIRST word instead).
|
| 54 |
+
|
| 55 |
+
Run:
|
| 56 |
+
cd /Users/rohitsar/Developer/Insurance\\ Sales\\ Bot
|
| 57 |
+
PYTHONPATH=$PWD .venv/bin/python -m pytest \
|
| 58 |
+
tests/test_ptt_preroll_warm_stream.py -v
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
from __future__ import annotations
|
| 62 |
+
|
| 63 |
+
import asyncio
|
| 64 |
+
import json
|
| 65 |
+
import os
|
| 66 |
+
import shutil
|
| 67 |
+
import subprocess
|
| 68 |
+
import sys
|
| 69 |
+
import textwrap
|
| 70 |
+
import types
|
| 71 |
+
from pathlib import Path
|
| 72 |
+
|
| 73 |
+
import pytest
|
| 74 |
+
|
| 75 |
+
os.environ.setdefault("SARVAM_API_KEY", "test-key-for-ptt-preroll")
|
| 76 |
+
|
| 77 |
+
REPO = Path(__file__).resolve().parents[1]
|
| 78 |
+
HOOK = REPO / "frontend" / "src" / "lib" / "useStreamingVoice.ts"
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# ===========================================================================
|
| 82 |
+
# PART A β pure frontend logic, executed against the REAL shipped exports.
|
| 83 |
+
# ===========================================================================
|
| 84 |
+
|
| 85 |
+
# The harness registers an ESM resolve/load hook (see _build_node_loader)
|
| 86 |
+
# that stubs `react`, `./api`, `./voice_resilience` β modules the React hook
|
| 87 |
+
# body imports but which the PURE exports (PreRollRing / evaluateHoldGate)
|
| 88 |
+
# never touch β and lets Node 24 strip the TS types on load. This exercises
|
| 89 |
+
# the genuine SHIPPED code, not a copy.
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _have_node() -> bool:
|
| 93 |
+
return shutil.which("node") is not None
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _build_node_loader(tmpdir: Path) -> Path:
|
| 97 |
+
"""Write an ESM loader that stubs react/./api/./voice_resilience and
|
| 98 |
+
strips TS types so the REAL hook module's pure exports can be imported.
|
| 99 |
+
"""
|
| 100 |
+
loader = tmpdir / "loader.mjs"
|
| 101 |
+
loader.write_text(
|
| 102 |
+
textwrap.dedent(
|
| 103 |
+
r"""
|
| 104 |
+
import { readFileSync } from 'node:fs';
|
| 105 |
+
import { fileURLToPath } from 'node:url';
|
| 106 |
+
import vm from 'node:vm';
|
| 107 |
+
|
| 108 |
+
const STUBS = {
|
| 109 |
+
react: `
|
| 110 |
+
export const useCallback = (f) => f;
|
| 111 |
+
export const useEffect = () => {};
|
| 112 |
+
export const useRef = (v) => ({ current: v });
|
| 113 |
+
export const useState = (v) => [typeof v === 'function' ? v() : v, () => {}];
|
| 114 |
+
export default {};
|
| 115 |
+
`,
|
| 116 |
+
api: `export const postTranscribe = async () => ({ text: '' });`,
|
| 117 |
+
voice_resilience: `
|
| 118 |
+
export const retryPostTranscribe = async () => null;
|
| 119 |
+
export const scaleSpeechZcrBand = () => ({ min: 20, max: 250 });
|
| 120 |
+
export class AdaptiveNoiseFloor { feed(){} currentThreshold(){ return 0.008; } }
|
| 121 |
+
`,
|
| 122 |
+
};
|
| 123 |
+
|
| 124 |
+
export async function resolve(specifier, context, nextResolve) {
|
| 125 |
+
if (specifier === 'react')
|
| 126 |
+
return { url: 'stub:react', shortCircuit: true };
|
| 127 |
+
if (specifier === './api' || specifier.endsWith('/api'))
|
| 128 |
+
return { url: 'stub:api', shortCircuit: true };
|
| 129 |
+
if (specifier === './voice_resilience' || specifier.endsWith('/voice_resilience'))
|
| 130 |
+
return { url: 'stub:voice_resilience', shortCircuit: true };
|
| 131 |
+
return nextResolve(specifier, context);
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
export async function load(url, context, nextLoad) {
|
| 135 |
+
if (url.startsWith('stub:')) {
|
| 136 |
+
const key = url.slice('stub:'.length);
|
| 137 |
+
return { format: 'module', source: STUBS[key], shortCircuit: true };
|
| 138 |
+
}
|
| 139 |
+
if (url.endsWith('.ts')) {
|
| 140 |
+
const path = fileURLToPath(url);
|
| 141 |
+
const src = readFileSync(path, 'utf8');
|
| 142 |
+
// Node 24 strips TS types natively for .ts; force module
|
| 143 |
+
// format + hand it the raw source (type-strip happens in the
|
| 144 |
+
// default ts transform path).
|
| 145 |
+
return { format: 'module-typescript', source: src, shortCircuit: true };
|
| 146 |
+
}
|
| 147 |
+
return nextLoad(url, context);
|
| 148 |
+
}
|
| 149 |
+
"""
|
| 150 |
+
).strip()
|
| 151 |
+
+ "\n"
|
| 152 |
+
)
|
| 153 |
+
return loader
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _run_node_contract(tmp_path: Path) -> dict:
|
| 157 |
+
"""Import the REAL PreRollRing + evaluateHoldGate under Node and run the
|
| 158 |
+
four pure-logic contracts. Returns the parsed JSON verdict."""
|
| 159 |
+
loader = _build_node_loader(tmp_path)
|
| 160 |
+
script = tmp_path / "contract.mjs"
|
| 161 |
+
# Pull the real exports + the real PRE_ROLL_MS / HOLD_THRESHOLD_MS /
|
| 162 |
+
# WARM_TIMESLICE_MS constants from the shipped hook module.
|
| 163 |
+
script.write_text(
|
| 164 |
+
textwrap.dedent(
|
| 165 |
+
f"""
|
| 166 |
+
const mod = await import({json.dumps(HOOK.as_uri())});
|
| 167 |
+
const {{
|
| 168 |
+
PreRollRing, evaluateHoldGate,
|
| 169 |
+
PRE_ROLL_MS, HOLD_THRESHOLD_MS, WARM_TIMESLICE_MS,
|
| 170 |
+
}} = mod;
|
| 171 |
+
|
| 172 |
+
const out = {{}};
|
| 173 |
+
|
| 174 |
+
// --- A1/A2: pre-roll ring keeps the LEADING audio + >=PRE_ROLL_MS
|
| 175 |
+
const ring = new PreRollRing(PRE_ROLL_MS);
|
| 176 |
+
// Feed far more than the window so eviction is forced. Each slice
|
| 177 |
+
// is a distinct 1-byte blob tagged by index so order is provable.
|
| 178 |
+
const totalSlices = Math.ceil((PRE_ROLL_MS / WARM_TIMESLICE_MS) * 4);
|
| 179 |
+
for (let i = 0; i < totalSlices; i++) {{
|
| 180 |
+
ring.push(new Blob([String.fromCharCode(65 + (i % 26))]), WARM_TIMESLICE_MS);
|
| 181 |
+
}}
|
| 182 |
+
out.retainedMs = ring.retainedDurationMs();
|
| 183 |
+
out.retainsAtLeastWindow = ring.retainedDurationMs() >= PRE_ROLL_MS;
|
| 184 |
+
const drained = ring.drain();
|
| 185 |
+
out.drainedCount = drained.length;
|
| 186 |
+
out.ringEmptyAfterDrain = ring.retainedDurationMs() === 0;
|
| 187 |
+
|
| 188 |
+
// Now the CORE #53 contract: a head word spoken just before the
|
| 189 |
+
// window boundary must still be at the HEAD of the drained blob.
|
| 190 |
+
const r2 = new PreRollRing(PRE_ROLL_MS);
|
| 191 |
+
// 'HEAD' = the first word's slice. Then enough silence-ish slices
|
| 192 |
+
// to *almost* (but not quite, given the +1 slice slack) fill the
|
| 193 |
+
// window, so HEAD must still be retained as slice 0.
|
| 194 |
+
const headBlob = new Blob(['HEAD']);
|
| 195 |
+
r2.push(headBlob, WARM_TIMESLICE_MS);
|
| 196 |
+
const fill = Math.floor(PRE_ROLL_MS / WARM_TIMESLICE_MS) - 1;
|
| 197 |
+
for (let i = 0; i < fill; i++) r2.push(new Blob(['x']), WARM_TIMESLICE_MS);
|
| 198 |
+
const d2 = r2.drain();
|
| 199 |
+
out.headSurvives = d2.length > 0 && (await d2[0].text()) === 'HEAD';
|
| 200 |
+
|
| 201 |
+
// --- A3: deliberate-hold gate
|
| 202 |
+
const tap = evaluateHoldGate(1000, 1000 + (HOLD_THRESHOLD_MS - 1), HOLD_THRESHOLD_MS);
|
| 203 |
+
const hold = evaluateHoldGate(1000, 1000 + HOLD_THRESHOLD_MS, HOLD_THRESHOLD_MS);
|
| 204 |
+
const longHold = evaluateHoldGate(1000, 1000 + 5000, HOLD_THRESHOLD_MS);
|
| 205 |
+
out.tapGated = tap.deliberate === false;
|
| 206 |
+
out.holdAtThresholdDeliberate = hold.deliberate === true;
|
| 207 |
+
out.longHoldDeliberate = longHold.deliberate === true;
|
| 208 |
+
out.tapHeldMs = tap.heldMs;
|
| 209 |
+
|
| 210 |
+
// --- A4: integrated β engage seeds capture with pre-roll so the
|
| 211 |
+
// leading word precedes the live audio; a tap submits nothing.
|
| 212 |
+
const r3 = new PreRollRing(PRE_ROLL_MS);
|
| 213 |
+
r3.push(new Blob(['FIRST_WORD']), WARM_TIMESLICE_MS); // spoken in cold-start gap
|
| 214 |
+
// engage: drain pre-roll into capture, THEN append live slices
|
| 215 |
+
const capture = [...r3.drain()];
|
| 216 |
+
capture.push(new Blob(['rest_of_sentence']));
|
| 217 |
+
out.captureLeadIsFirstWord = (await capture[0].text()) === 'FIRST_WORD';
|
| 218 |
+
out.captureIncludesLive = capture.length === 2;
|
| 219 |
+
// tap path: gate says not deliberate -> nothing assembled
|
| 220 |
+
const tapDecision = evaluateHoldGate(2000, 2050, HOLD_THRESHOLD_MS);
|
| 221 |
+
out.tapAssemblesNothing = tapDecision.deliberate === false;
|
| 222 |
+
|
| 223 |
+
out.constants = {{ PRE_ROLL_MS, HOLD_THRESHOLD_MS, WARM_TIMESLICE_MS }};
|
| 224 |
+
process.stdout.write(JSON.stringify(out));
|
| 225 |
+
"""
|
| 226 |
+
).strip()
|
| 227 |
+
+ "\n"
|
| 228 |
+
)
|
| 229 |
+
proc = subprocess.run(
|
| 230 |
+
[
|
| 231 |
+
"node",
|
| 232 |
+
"--no-warnings",
|
| 233 |
+
f"--experimental-loader={loader.as_uri()}",
|
| 234 |
+
str(script),
|
| 235 |
+
],
|
| 236 |
+
capture_output=True,
|
| 237 |
+
text=True,
|
| 238 |
+
cwd=str(REPO / "frontend"),
|
| 239 |
+
timeout=60,
|
| 240 |
+
)
|
| 241 |
+
if proc.returncode != 0:
|
| 242 |
+
raise AssertionError(
|
| 243 |
+
"Node pure-logic harness failed:\n"
|
| 244 |
+
f"STDOUT:\n{proc.stdout}\n\nSTDERR:\n{proc.stderr}"
|
| 245 |
+
)
|
| 246 |
+
return json.loads(proc.stdout.strip())
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
@pytest.mark.skipif(not _have_node(), reason="node not available")
|
| 250 |
+
def test_preroll_and_hold_gate_pure_logic(tmp_path):
|
| 251 |
+
"""PART A β exercise the REAL exported PreRollRing + evaluateHoldGate."""
|
| 252 |
+
r = _run_node_contract(tmp_path)
|
| 253 |
+
|
| 254 |
+
# A2 β the ring always retains at least the requested pre-roll window so
|
| 255 |
+
# the worst-case page.tsx cold-start gap (getUserMedia 200-700ms + the
|
| 256 |
+
# 400ms Live-teardown wait at page.tsx:994) is fully covered.
|
| 257 |
+
assert r["retainsAtLeastWindow"] is True, r
|
| 258 |
+
assert r["retainedMs"] >= r["constants"]["PRE_ROLL_MS"], r
|
| 259 |
+
assert r["ringEmptyAfterDrain"] is True, r
|
| 260 |
+
assert r["drainedCount"] >= 1, r
|
| 261 |
+
|
| 262 |
+
# A1 β the CORE #53 fix: the first word's slice is at the HEAD of the
|
| 263 |
+
# drained lead-in (it is NOT evicted while it is within the window).
|
| 264 |
+
assert r["headSurvives"] is True, (
|
| 265 |
+
"pre-roll dropped the leading slice β first word would be clipped: "
|
| 266 |
+
f"{r}"
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
# A3 β deliberate-hold gate (#54): a sub-threshold tap is gated; a hold
|
| 270 |
+
# exactly at the threshold (and longer) is deliberate.
|
| 271 |
+
assert r["tapGated"] is True, r
|
| 272 |
+
assert r["holdAtThresholdDeliberate"] is True, r
|
| 273 |
+
assert r["longHoldDeliberate"] is True, r
|
| 274 |
+
assert r["tapHeldMs"] < r["constants"]["HOLD_THRESHOLD_MS"], r
|
| 275 |
+
|
| 276 |
+
# A4 β integrated: engage seeds the capture with the pre-roll so the
|
| 277 |
+
# leading word precedes the live audio; a tap assembles/submits nothing.
|
| 278 |
+
assert r["captureLeadIsFirstWord"] is True, (
|
| 279 |
+
"engaged capture does not start with the pre-roll lead-in β first "
|
| 280 |
+
f"word lost: {r}"
|
| 281 |
+
)
|
| 282 |
+
assert r["captureIncludesLive"] is True, r
|
| 283 |
+
assert r["tapAssemblesNothing"] is True, r
|
| 284 |
+
|
| 285 |
+
# The shipped constants must stay in the spec'd bands.
|
| 286 |
+
assert r["constants"]["PRE_ROLL_MS"] >= 500, r # >= 500ms pre-roll
|
| 287 |
+
assert 150 <= r["constants"]["HOLD_THRESHOLD_MS"] <= 250, r # deliberate
|
| 288 |
+
assert r["constants"]["WARM_TIMESLICE_MS"] <= 250, r # fine-grained ring
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
# ===========================================================================
|
| 292 |
+
# PART B β backend STT: the prepended pre-roll HEAD must never be silently
|
| 293 |
+
# dropped/truncated (the bug's observable failure surface).
|
| 294 |
+
# ===========================================================================
|
| 295 |
+
|
| 296 |
+
from backend.providers.sarvam_stt import STT_CHUNK_MS, SarvamSTT # noqa: E402
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
class FakeAudioSegment:
|
| 300 |
+
"""Minimal pydub.AudioSegment stub (same surface as
|
| 301 |
+
test_stt_long_audio_chunking.py). One word per second; word_0 is the
|
| 302 |
+
PRE-ROLL HEAD (the first word spoken in page.tsx's cold-start gap)."""
|
| 303 |
+
|
| 304 |
+
def __init__(self, start_ms: int, end_ms: int):
|
| 305 |
+
self.start_ms = start_ms
|
| 306 |
+
self.end_ms = end_ms
|
| 307 |
+
|
| 308 |
+
def __len__(self):
|
| 309 |
+
return self.end_ms - self.start_ms
|
| 310 |
+
|
| 311 |
+
def __getitem__(self, sl):
|
| 312 |
+
if isinstance(sl, slice):
|
| 313 |
+
lo = 0 if sl.start is None else sl.start
|
| 314 |
+
hi = len(self) if sl.stop is None else sl.stop
|
| 315 |
+
lo = max(0, min(lo, len(self)))
|
| 316 |
+
hi = max(0, min(hi, len(self)))
|
| 317 |
+
return FakeAudioSegment(self.start_ms + lo, self.start_ms + hi)
|
| 318 |
+
raise TypeError("only slice indexing used by splitter")
|
| 319 |
+
|
| 320 |
+
@property
|
| 321 |
+
def dBFS(self):
|
| 322 |
+
if self.end_ms - self.start_ms == 0:
|
| 323 |
+
return float("-inf")
|
| 324 |
+
# A real pause near the 25s ceiling so the splitter snaps there.
|
| 325 |
+
if 24000 <= self.start_ms < 24400:
|
| 326 |
+
return float("-inf")
|
| 327 |
+
return -20.0
|
| 328 |
+
|
| 329 |
+
def set_frame_rate(self, _):
|
| 330 |
+
return self
|
| 331 |
+
|
| 332 |
+
def set_channels(self, _):
|
| 333 |
+
return self
|
| 334 |
+
|
| 335 |
+
def set_sample_width(self, _):
|
| 336 |
+
return self
|
| 337 |
+
|
| 338 |
+
def export(self, buf, format="wav"): # noqa: A002
|
| 339 |
+
buf.write(f"FAKEWAV:{self.start_ms}:{self.end_ms}".encode())
|
| 340 |
+
buf.seek(0)
|
| 341 |
+
return buf
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
SARVAM_HARD_LIMIT_MS = 30_000
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
class _FakeResp:
|
| 348 |
+
def __init__(self, payload):
|
| 349 |
+
self._payload = payload
|
| 350 |
+
|
| 351 |
+
def raise_for_status(self):
|
| 352 |
+
return None
|
| 353 |
+
|
| 354 |
+
def json(self):
|
| 355 |
+
return self._payload
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
class _FakeClient:
|
| 359 |
+
def __init__(self, *a, **k):
|
| 360 |
+
pass
|
| 361 |
+
|
| 362 |
+
async def __aenter__(self):
|
| 363 |
+
return self
|
| 364 |
+
|
| 365 |
+
async def __aexit__(self, *a):
|
| 366 |
+
return False
|
| 367 |
+
|
| 368 |
+
async def post(self, url, headers=None, files=None, data=None):
|
| 369 |
+
raw = files["file"][1].read()
|
| 370 |
+
text = raw.decode()
|
| 371 |
+
assert text.startswith("FAKEWAV:"), text
|
| 372 |
+
_, s, e = text.split(":")
|
| 373 |
+
start_ms, end_ms = int(s), int(e)
|
| 374 |
+
capped_end = min(end_ms, start_ms + SARVAM_HARD_LIMIT_MS)
|
| 375 |
+
words = [f"word_{ms // 1000}" for ms in range(start_ms, capped_end, 1000)]
|
| 376 |
+
return _FakeResp(
|
| 377 |
+
{
|
| 378 |
+
"transcript": " ".join(words),
|
| 379 |
+
"language_code": "en-IN",
|
| 380 |
+
"language_probability": 0.99,
|
| 381 |
+
}
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
@pytest.fixture
|
| 386 |
+
def patch_pydub(monkeypatch):
|
| 387 |
+
fake_pydub = types.ModuleType("pydub")
|
| 388 |
+
|
| 389 |
+
class _Factory:
|
| 390 |
+
@staticmethod
|
| 391 |
+
def from_file(_bio, format=None): # noqa: A002
|
| 392 |
+
# 92s utterance β long enough to hit the silence splitter (>3
|
| 393 |
+
# Sarvam 25s windows). word_0 is the prepended PRE-ROLL HEAD.
|
| 394 |
+
return FakeAudioSegment(0, 92_000)
|
| 395 |
+
|
| 396 |
+
fake_pydub.AudioSegment = _Factory
|
| 397 |
+
monkeypatch.setitem(sys.modules, "pydub", fake_pydub)
|
| 398 |
+
yield
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
@pytest.fixture
|
| 402 |
+
def patch_httpx(monkeypatch):
|
| 403 |
+
import backend.providers.sarvam_stt as mod
|
| 404 |
+
|
| 405 |
+
monkeypatch.setattr(mod.httpx, "AsyncClient", _FakeClient)
|
| 406 |
+
yield
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def test_preroll_head_word_survives_backend_stt(patch_pydub, patch_httpx):
|
| 410 |
+
"""The FIRST word (word_0 β the pre-roll head the warm-stream fix
|
| 411 |
+
prepends) must appear in the transcript. A regression that mishandled a
|
| 412 |
+
prepended head, or re-introduced single-shot truncation, would drop
|
| 413 |
+
word_0 (head-clipping #53 at the STT layer)."""
|
| 414 |
+
stt = SarvamSTT()
|
| 415 |
+
audio = b"\x00" * 4096 # > 1 KB so the short-audio guard doesn't fire
|
| 416 |
+
result = asyncio.run(
|
| 417 |
+
stt.transcribe(audio_bytes=audio, audio_format="webm", language_code="en-IN")
|
| 418 |
+
)
|
| 419 |
+
words = result.text.split()
|
| 420 |
+
|
| 421 |
+
# CORE #53 CONTRACT at the STT boundary: the leading word is present and
|
| 422 |
+
# is actually FIRST (not just somewhere in the middle).
|
| 423 |
+
assert "word_0" in words, f"FIRST word dropped by STT β head clipped: {words[:5]}"
|
| 424 |
+
deduped = [w for i, w in enumerate(words) if i == 0 or w != words[i - 1]]
|
| 425 |
+
assert deduped[0] == "word_0", (
|
| 426 |
+
f"transcript does not START with the pre-roll head word: {deduped[:5]}"
|
| 427 |
+
)
|
| 428 |
+
# And the WHOLE 92s utterance survives in order (no head drop, no tail
|
| 429 |
+
# truncation) β the prepended pre-roll doesn't break chunking.
|
| 430 |
+
expected = [f"word_{i}" for i in range(92)]
|
| 431 |
+
assert deduped == expected, (
|
| 432 |
+
f"transcript garbled β deduped {len(deduped)} words "
|
| 433 |
+
f"(first={deduped[:3]}, last={deduped[-3:]}), expected 92 in order"
|
| 434 |
+
)
|
| 435 |
+
assert result.raw.get("chunked") is True
|
| 436 |
+
assert result.raw.get("chunk_count", 0) >= 4
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def test_short_preroll_only_blob_is_not_truncated(patch_pydub, patch_httpx, monkeypatch):
|
| 440 |
+
"""A short blob that is essentially pre-roll + a couple words (the common
|
| 441 |
+
PTT case) must transcribe in ONE call with word_0 intact β proving the
|
| 442 |
+
pre-roll head adds no truncation and no extra latency for short clips."""
|
| 443 |
+
short = FakeAudioSegment(0, 5_000) # ~5s: pre-roll + a short answer
|
| 444 |
+
monkeypatch.setattr(
|
| 445 |
+
sys.modules["pydub"].AudioSegment,
|
| 446 |
+
"from_file",
|
| 447 |
+
staticmethod(lambda *a, **k: short),
|
| 448 |
+
)
|
| 449 |
+
stt = SarvamSTT()
|
| 450 |
+
result = asyncio.run(
|
| 451 |
+
stt.transcribe(audio_bytes=b"\x00" * 4096, audio_format="webm")
|
| 452 |
+
)
|
| 453 |
+
words = result.text.split()
|
| 454 |
+
assert words == [f"word_{i}" for i in range(5)], words
|
| 455 |
+
assert words[0] == "word_0", f"pre-roll head missing on short clip: {words}"
|
| 456 |
+
# Single-call path β no chunked marker, no added round-trips.
|
| 457 |
+
assert result.raw.get("chunked") is not True
|
|
@@ -0,0 +1,359 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Regression test for #55 (10s truncation) + #56 (robotic normalization).
|
| 2 |
+
|
| 3 |
+
LIVE BUG (user-reported, audio): the advisor sent a 6-question pricing
|
| 4 |
+
intake message. The user heard only ~10 seconds of audio ("stopped in ten
|
| 5 |
+
seconds") β questions 2-6 were NEVER spoken β and "e.g." was read
|
| 6 |
+
letter-by-letter ("E G") while "/" in "βΉ5L / βΉ10L / βΉ25L / βΉ1Cr" was read
|
| 7 |
+
as "by"/"divide".
|
| 8 |
+
|
| 9 |
+
ROOT CAUSE #55: backend/voice_format.py `_truncate_for_voice` (called from
|
| 10 |
+
`tts_preprocess` with the legacy max_words=55 passed by backend/main.py)
|
| 11 |
+
chopped the message to the first ~55 spoken words (~10s of audio) BEFORE
|
| 12 |
+
TTS, appending "More details are on screen." Everything from question 2
|
| 13 |
+
onward was discarded.
|
| 14 |
+
|
| 15 |
+
ROOT CAUSE #56: `tts_preprocess` never expanded "e.g." / "/" / currency
|
| 16 |
+
ranges to spoken words, so Sarvam Bulbul voiced "E G" and "divide".
|
| 17 |
+
|
| 18 |
+
LATENT ROOT CAUSE #55b: even with the word cap removed, the full
|
| 19 |
+
normalized message exceeds Sarvam Bulbul v2's hard 1500-char per-request
|
| 20 |
+
limit; sending it whole means Sarvam only voices the leading slice.
|
| 21 |
+
providers/sarvam_tts.py now chunks at sentence / numbered-item seams under
|
| 22 |
+
a safe ceiling, synthesizes each chunk sequentially, and concatenates the
|
| 23 |
+
decoded PCM into ONE gapless WAV (mirrors the STT 30s-chunking house
|
| 24 |
+
style; raises loudly on any HTTP error β no silent truncation).
|
| 25 |
+
|
| 26 |
+
This test stubs the Sarvam HTTP layer (no network / no pydub needed for
|
| 27 |
+
the WAV concat β stdlib `wave`) and pins the contract end-to-end.
|
| 28 |
+
|
| 29 |
+
Run:
|
| 30 |
+
cd /Users/rohitsar/Developer/Insurance\\ Sales\\ Bot
|
| 31 |
+
PYTHONPATH=$PWD .venv/bin/python -m pytest \
|
| 32 |
+
tests/test_tts_full_natural_readout.py -v
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
from __future__ import annotations
|
| 36 |
+
|
| 37 |
+
import asyncio
|
| 38 |
+
import io
|
| 39 |
+
import os
|
| 40 |
+
import re
|
| 41 |
+
import wave
|
| 42 |
+
|
| 43 |
+
import pytest
|
| 44 |
+
|
| 45 |
+
os.environ.setdefault("SARVAM_API_KEY", "test-key-for-tts-chunking")
|
| 46 |
+
|
| 47 |
+
from backend.voice_format import tts_preprocess # noqa: E402
|
| 48 |
+
from backend.providers.sarvam_tts import ( # noqa: E402
|
| 49 |
+
SarvamTTS,
|
| 50 |
+
_chunk_text_for_tts,
|
| 51 |
+
_concat_wav_bytes,
|
| 52 |
+
_tts_char_ceiling,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
# The EXACT message the user reported hearing truncated + robotic.
|
| 57 |
+
SCREENSHOT_MESSAGE = (
|
| 58 |
+
"A few quick pricing inputs (you can skip any):\n"
|
| 59 |
+
"1. How much sum insured? (e.g., βΉ5L / βΉ10L / βΉ25L / βΉ1Cr)\n"
|
| 60 |
+
"2. Premium budget? (e.g., βΉ10β15K/year, or βΉ50K+ for premium covers)\n"
|
| 61 |
+
"3. Any existing health cover from work or otherwise? "
|
| 62 |
+
"(e.g., '5L through employer' or 'no')\n"
|
| 63 |
+
"4. Co-pay tolerance: Are you OK with a co-pay β sharing 10β30% of "
|
| 64 |
+
"every claim β to lower the premium? Or do you want zero co-pay "
|
| 65 |
+
"(insurer pays it all)?\n"
|
| 66 |
+
"5. Family medical history: Any major conditions running in your "
|
| 67 |
+
"blood family (parents/siblings) β cancer / diabetes / heart disease "
|
| 68 |
+
"/ hypertension?\n"
|
| 69 |
+
"6. Smoking status: Do you smoke or use tobacco products? (yes / no) "
|
| 70 |
+
"Smokers face 30β50% premium loading; capturing this gives an "
|
| 71 |
+
"accurate band."
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# ---------------------------------------------------------------------------
|
| 76 |
+
# #56 β NATURAL NORMALIZATION (text-level, no network).
|
| 77 |
+
# ---------------------------------------------------------------------------
|
| 78 |
+
def test_full_message_normalizes_naturally_and_is_not_truncated():
|
| 79 |
+
spoken = tts_preprocess(SCREENSHOT_MESSAGE, language="en", max_words=55)
|
| 80 |
+
|
| 81 |
+
# --- #55: the FULL message is present (NOT cut at ~55 words / ~10s) ---
|
| 82 |
+
# Question-6-specific content must survive end-to-end.
|
| 83 |
+
assert "smoking status" in spoken.lower(), spoken
|
| 84 |
+
assert "tobacco" in spoken.lower(), spoken
|
| 85 |
+
assert "accurate band" in spoken.lower(), spoken
|
| 86 |
+
# The legacy truncation cue must NOT be present.
|
| 87 |
+
assert "more details are on screen" not in spoken.lower(), spoken
|
| 88 |
+
assert "more details on screen" not in spoken.lower(), spoken
|
| 89 |
+
# A real readout of all 6 questions is far more than 55 words.
|
| 90 |
+
assert len(spoken.split()) > 120, (
|
| 91 |
+
f"only {len(spoken.split())} words β looks truncated:\n{spoken}"
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
low = spoken.lower()
|
| 95 |
+
|
| 96 |
+
# --- #56: "e.g." expanded, never spelled as letters ---
|
| 97 |
+
assert "for example" in low, spoken
|
| 98 |
+
assert "e.g" not in low, spoken
|
| 99 |
+
# No isolated "e g" letter pair (the robotic readout).
|
| 100 |
+
assert not re.search(r"\be\s+g\b", low), spoken
|
| 101 |
+
|
| 102 |
+
# --- #56: raw slash gone everywhere; expanded to list / "or" ---
|
| 103 |
+
assert "/" not in spoken, f"raw slash survived:\n{spoken}"
|
| 104 |
+
# Currency slash run "βΉ5L / βΉ10L / βΉ25L / βΉ1Cr" became a spoken list.
|
| 105 |
+
# NOTE: leading digits are word-formed by _normalize_numbers (the
|
| 106 |
+
# natural TTS form): "5" -> "five", "1" -> "one", "25" -> "twenty-five".
|
| 107 |
+
assert "five lakh rupees" in low, spoken
|
| 108 |
+
assert "ten lakh rupees" in low, spoken
|
| 109 |
+
assert "twenty-five lakh rupees" in low, spoken
|
| 110 |
+
assert "one crore rupees" in low, spoken
|
| 111 |
+
assert "or one crore rupees" in low, spoken
|
| 112 |
+
# Generic slashes -> "or".
|
| 113 |
+
assert "parents or siblings" in low, spoken
|
| 114 |
+
assert "yes or no" in low, spoken
|
| 115 |
+
assert "cancer or diabetes" in low, spoken
|
| 116 |
+
|
| 117 |
+
# --- #56: ranges + currency expanded, no symbols/letters left ---
|
| 118 |
+
assert "βΉ" not in spoken, spoken
|
| 119 |
+
assert "%" not in spoken, spoken
|
| 120 |
+
# "10β30%" -> "10 to 30 percent" -> word form "ten to thirty percent"
|
| 121 |
+
assert "ten to thirty percent" in low, spoken
|
| 122 |
+
# "30β50%" -> "thirty to fifty percent"
|
| 123 |
+
assert "thirty to fifty percent" in low, spoken
|
| 124 |
+
# "βΉ10β15K/year" -> "10 to 15 thousand rupees per year"
|
| 125 |
+
assert "ten to fifteen thousand rupees per year" in low, spoken
|
| 126 |
+
# "βΉ50K+" -> "above 50 thousand rupees" -> "above fifty thousand rupees"
|
| 127 |
+
assert "above fifty thousand rupees" in low, spoken
|
| 128 |
+
# No bare "K"/"L"/"Cr" shorthand letters left dangling.
|
| 129 |
+
assert not re.search(r"\b\d+\s*[LK]\b", spoken), spoken
|
| 130 |
+
assert not re.search(r"\bCr\b", spoken), spoken
|
| 131 |
+
assert " 15K" not in spoken and "15k" not in low, spoken
|
| 132 |
+
|
| 133 |
+
# --- markdown list numbering stripped (reads as speech, not "1.") ---
|
| 134 |
+
assert not re.search(r"(?m)^\s*\d+\.\s", spoken), spoken
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# ---------------------------------------------------------------------------
|
| 138 |
+
# #55b β the chunk PLAN covers the WHOLE message (no character dropped).
|
| 139 |
+
# ---------------------------------------------------------------------------
|
| 140 |
+
def test_chunk_plan_covers_entire_message_under_ceiling():
|
| 141 |
+
spoken = tts_preprocess(SCREENSHOT_MESSAGE, language="en")
|
| 142 |
+
ceiling = _tts_char_ceiling("bulbul:v2")
|
| 143 |
+
assert ceiling < 1500 # safety margin under Sarvam's hard cap
|
| 144 |
+
|
| 145 |
+
chunks = _chunk_text_for_tts(spoken, ceiling)
|
| 146 |
+
assert len(chunks) >= 1
|
| 147 |
+
for i, c in enumerate(chunks):
|
| 148 |
+
assert len(c) <= ceiling, f"chunk {i} = {len(c)} chars > {ceiling}"
|
| 149 |
+
|
| 150 |
+
# Coverage: concatenated chunks (whitespace-normalized) must contain
|
| 151 |
+
# every non-space character of the spoken text β nothing dropped.
|
| 152 |
+
def _norm(s: str) -> str:
|
| 153 |
+
return re.sub(r"\s+", "", s)
|
| 154 |
+
|
| 155 |
+
joined = _norm(" ".join(chunks))
|
| 156 |
+
assert _norm(spoken) == joined, (
|
| 157 |
+
"chunk plan lost/added content vs the normalized spoken text"
|
| 158 |
+
)
|
| 159 |
+
# The final question's content must live in some chunk.
|
| 160 |
+
assert any("accurate band" in c.lower() for c in chunks), chunks
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# ---------------------------------------------------------------------------
|
| 164 |
+
# WAV concat helper β gapless join of stdlib PCM blobs.
|
| 165 |
+
# ---------------------------------------------------------------------------
|
| 166 |
+
def _make_wav(n_frames: int, framerate: int = 22050) -> bytes:
|
| 167 |
+
buf = io.BytesIO()
|
| 168 |
+
with wave.open(buf, "wb") as w:
|
| 169 |
+
w.setnchannels(1)
|
| 170 |
+
w.setsampwidth(2)
|
| 171 |
+
w.setframerate(framerate)
|
| 172 |
+
w.writeframes(b"\x01\x00" * n_frames)
|
| 173 |
+
return buf.getvalue()
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def test_concat_wav_is_gapless_and_sums_frames():
|
| 177 |
+
a = _make_wav(1000)
|
| 178 |
+
b = _make_wav(2500)
|
| 179 |
+
c = _make_wav(700)
|
| 180 |
+
merged = _concat_wav_bytes([a, b, c])
|
| 181 |
+
with wave.open(io.BytesIO(merged), "rb") as w:
|
| 182 |
+
assert w.getnchannels() == 1
|
| 183 |
+
assert w.getsampwidth() == 2
|
| 184 |
+
assert w.getframerate() == 22050
|
| 185 |
+
# Gapless: total frames == sum of inputs (no inserted silence).
|
| 186 |
+
assert w.getnframes() == 1000 + 2500 + 700
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def test_concat_wav_param_mismatch_raises_loud():
|
| 190 |
+
a = _make_wav(100, framerate=22050)
|
| 191 |
+
b = _make_wav(100, framerate=16000)
|
| 192 |
+
with pytest.raises(RuntimeError, match="params diverged"):
|
| 193 |
+
_concat_wav_bytes([a, b])
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
# ---------------------------------------------------------------------------
|
| 197 |
+
# End-to-end: stub Sarvam HTTP, prove the FULL message is synthesized via
|
| 198 |
+
# multiple chunks and the audio is concatenated (NOT a single 10s clip).
|
| 199 |
+
# ---------------------------------------------------------------------------
|
| 200 |
+
class _FakeResp:
|
| 201 |
+
def __init__(self, payload):
|
| 202 |
+
self._payload = payload
|
| 203 |
+
|
| 204 |
+
def raise_for_status(self):
|
| 205 |
+
return None
|
| 206 |
+
|
| 207 |
+
def json(self):
|
| 208 |
+
return self._payload
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class _FakeClient:
|
| 212 |
+
"""Fake Sarvam TTS endpoint.
|
| 213 |
+
|
| 214 |
+
Records every chunk it is asked to synthesize and returns a WAV whose
|
| 215 |
+
frame count == len(text) so the concatenated audio length is a direct
|
| 216 |
+
proxy for 'how many characters were actually voiced'. Also asserts no
|
| 217 |
+
single request exceeds Bulbul v2's hard 1500-char limit β a regression
|
| 218 |
+
to single-shot (whole message in one call) fails LOUDLY here.
|
| 219 |
+
"""
|
| 220 |
+
|
| 221 |
+
sent_texts: list[str] = []
|
| 222 |
+
HARD_LIMIT = 1500
|
| 223 |
+
|
| 224 |
+
def __init__(self, *a, **k):
|
| 225 |
+
pass
|
| 226 |
+
|
| 227 |
+
async def __aenter__(self):
|
| 228 |
+
return self
|
| 229 |
+
|
| 230 |
+
async def __aexit__(self, *a):
|
| 231 |
+
return False
|
| 232 |
+
|
| 233 |
+
async def post(self, url, headers=None, json=None, **k):
|
| 234 |
+
text = json["text"]
|
| 235 |
+
assert len(text) <= self.HARD_LIMIT, (
|
| 236 |
+
f"chunk of {len(text)} chars exceeds Sarvam Bulbul v2's "
|
| 237 |
+
f"1500-char hard limit β would be silently truncated"
|
| 238 |
+
)
|
| 239 |
+
_FakeClient.sent_texts.append(text)
|
| 240 |
+
wav = _make_wav(max(1, len(text)))
|
| 241 |
+
return _FakeResp({"audios": [base64.b64encode(wav).decode()]})
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
import base64 # noqa: E402 (used by _FakeClient above)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
@pytest.fixture
|
| 248 |
+
def patch_httpx(monkeypatch):
|
| 249 |
+
import backend.providers.sarvam_tts as mod
|
| 250 |
+
|
| 251 |
+
_FakeClient.sent_texts = []
|
| 252 |
+
monkeypatch.setattr(mod.httpx, "AsyncClient", _FakeClient)
|
| 253 |
+
yield
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def test_end_to_end_screenshot_message_synthesized_in_full(patch_httpx):
|
| 257 |
+
"""The exact buggy screenshot message: EVERY character (incl. question
|
| 258 |
+
6) must reach Sarvam, in however many calls β never the ~55-word /
|
| 259 |
+
~10s truncated shot the bug produced."""
|
| 260 |
+
spoken = tts_preprocess(SCREENSHOT_MESSAGE, language="en", max_words=55)
|
| 261 |
+
tts = SarvamTTS()
|
| 262 |
+
audio, mime = asyncio.run(
|
| 263 |
+
tts.synthesize_with_mime(spoken, language_code="en-IN")
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
assert mime == "audio/wav"
|
| 267 |
+
assert len(_FakeClient.sent_texts) >= 1, _FakeClient.sent_texts
|
| 268 |
+
# Every char that went to TTS, concatenated == full spoken text
|
| 269 |
+
# (no character dropped, nothing truncated).
|
| 270 |
+
rejoined = re.sub(r"\s+", "", " ".join(_FakeClient.sent_texts))
|
| 271 |
+
assert re.sub(r"\s+", "", spoken) == rejoined
|
| 272 |
+
# Question-6 content was actually sent to TTS (the truncation bug
|
| 273 |
+
# dropped everything from question 2 onward).
|
| 274 |
+
assert any("accurate band" in t.lower() for t in _FakeClient.sent_texts)
|
| 275 |
+
assert any("tobacco" in t.lower() for t in _FakeClient.sent_texts)
|
| 276 |
+
assert any("smoking status" in t.lower() for t in _FakeClient.sent_texts)
|
| 277 |
+
|
| 278 |
+
# Audio frame count == total chars voiced (fake = 1 frame/char), and
|
| 279 |
+
# is FAR above the ~300 chars the 55-word truncation would have made.
|
| 280 |
+
with wave.open(io.BytesIO(audio), "rb") as w:
|
| 281 |
+
total_frames = w.getnframes()
|
| 282 |
+
assert total_frames == len(re.sub(r"\s+", " ", spoken).strip()) or (
|
| 283 |
+
total_frames > 600
|
| 284 |
+
), f"{total_frames} frames β looks like the 10s truncation regressed"
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def test_end_to_end_overlong_message_chunks_and_concatenates(patch_httpx):
|
| 288 |
+
"""When normalized text DOES exceed Bulbul v2's per-request ceiling,
|
| 289 |
+
it is split into >1 chunk, each <= ceiling, and the decoded PCM is
|
| 290 |
+
concatenated into ONE gapless WAV β no character dropped."""
|
| 291 |
+
ceiling = _tts_char_ceiling("bulbul:v2")
|
| 292 |
+
# Build a long, clean multi-sentence reply that exceeds the ceiling.
|
| 293 |
+
long_reply = " ".join(
|
| 294 |
+
f"Point number {i}: this policy covers day-care and AYUSH "
|
| 295 |
+
f"treatment with a thirty day waiting period and no co-pay."
|
| 296 |
+
for i in range(1, 60)
|
| 297 |
+
)
|
| 298 |
+
spoken = tts_preprocess(long_reply, language="en")
|
| 299 |
+
assert len(spoken) > ceiling, len(spoken)
|
| 300 |
+
|
| 301 |
+
tts = SarvamTTS()
|
| 302 |
+
audio, _ = asyncio.run(
|
| 303 |
+
tts.synthesize_with_mime(spoken, language_code="en-IN")
|
| 304 |
+
)
|
| 305 |
+
# Multiple Sarvam calls, each within the hard limit (the fake asserts
|
| 306 |
+
# the 1500 cap per call).
|
| 307 |
+
assert len(_FakeClient.sent_texts) >= 2, len(_FakeClient.sent_texts)
|
| 308 |
+
for t in _FakeClient.sent_texts:
|
| 309 |
+
assert len(t) <= ceiling
|
| 310 |
+
# Reassembled text == full spoken payload (nothing dropped at seams).
|
| 311 |
+
assert re.sub(r"\s+", "", " ".join(_FakeClient.sent_texts)) == re.sub(
|
| 312 |
+
r"\s+", "", spoken
|
| 313 |
+
)
|
| 314 |
+
# Gapless concatenated WAV: frames == sum of per-chunk frames.
|
| 315 |
+
with wave.open(io.BytesIO(audio), "rb") as w:
|
| 316 |
+
total = w.getnframes()
|
| 317 |
+
assert total == sum(len(t) for t in _FakeClient.sent_texts)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def test_short_reply_is_single_call_unchanged(patch_httpx):
|
| 321 |
+
"""A sub-ceiling reply must take exactly ONE Sarvam call (no behaviour
|
| 322 |
+
change / no extra latency for the common case)."""
|
| 323 |
+
spoken = tts_preprocess("Yes, that policy covers day-care procedures.")
|
| 324 |
+
tts = SarvamTTS()
|
| 325 |
+
asyncio.run(tts.synthesize_with_mime(spoken, language_code="en-IN"))
|
| 326 |
+
assert len(_FakeClient.sent_texts) == 1, _FakeClient.sent_texts
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def test_http_error_on_any_chunk_propagates_loudly(patch_httpx, monkeypatch):
|
| 330 |
+
"""No silent truncation: if any chunk's Sarvam call fails, the error
|
| 331 |
+
propagates so the boundary classifier surfaces a real tts_error_code."""
|
| 332 |
+
import backend.providers.sarvam_tts as mod
|
| 333 |
+
|
| 334 |
+
class _BoomClient(_FakeClient):
|
| 335 |
+
calls = 0
|
| 336 |
+
|
| 337 |
+
async def post(self, url, headers=None, json=None, **k):
|
| 338 |
+
_BoomClient.calls += 1
|
| 339 |
+
if _BoomClient.calls == 2:
|
| 340 |
+
req = __import__("httpx").Request("POST", url)
|
| 341 |
+
resp = __import__("httpx").Response(503, request=req)
|
| 342 |
+
raise __import__("httpx").HTTPStatusError(
|
| 343 |
+
"503", request=req, response=resp
|
| 344 |
+
)
|
| 345 |
+
return await super().post(url, headers=headers, json=json, **k)
|
| 346 |
+
|
| 347 |
+
monkeypatch.setattr(mod.httpx, "AsyncClient", _BoomClient)
|
| 348 |
+
# Use a long reply so there are >= 2 chunks and chunk #2 is the one
|
| 349 |
+
# that 503s β proving a mid-stream failure is NOT silently swallowed
|
| 350 |
+
# into a partial readout.
|
| 351 |
+
long_reply = " ".join(
|
| 352 |
+
f"Point number {i}: this policy covers day-care and AYUSH "
|
| 353 |
+
f"treatment with a thirty day waiting period and no co-pay."
|
| 354 |
+
for i in range(1, 60)
|
| 355 |
+
)
|
| 356 |
+
spoken = tts_preprocess(long_reply, language="en")
|
| 357 |
+
tts = SarvamTTS()
|
| 358 |
+
with pytest.raises(__import__("httpx").HTTPStatusError):
|
| 359 |
+
asyncio.run(tts.synthesize_with_mime(spoken, language_code="en-IN"))
|