Spaces:
Sleeping
fix(voice+brain+timeout): KI-168 + KI-169 + KI-170 bundle
Browse filesKI-168 β Voice input rebuilt as hybrid streaming flow.
+ frontend/src/lib/useStreamingVoice.ts (NEW)
Web Speech API streams interim transcripts to chat input in real time
(ChatGPT/Claude-voice UX). MediaRecorder runs in parallel; on browser
silence-detect, audio blob POSTs to Sarvam /api/transcribe for the
authoritative final transcript. Replaces input text + auto-submits.
Falls back to Web Speech transcript if Sarvam errors. KI-165 race
guards preserved (isTextRequestPendingRef + skip-if-tiny-blob).
M frontend/src/app/page.tsx
Swapped useLiveConversation β useStreamingVoice. interim β setInput,
final β send(). Voice OFF by default (KI-131) preserved.
Old useLiveConversation.ts left on disk as graveyard.
KI-169 β Sales brain <think>-tag empty-reply retry.
M backend/sales_brain.py
qwen3-next-80b occasionally emits all reasoning inside <think>...</think>
in the JSON reply field, leaving nothing after strip β orchestrator
fell back to persona.py:221's cryptic "I'm thinking through that.
Could you rephrase or ask a follow-up?". Three fixes:
- System prompt: explicit "NEVER include <think> tags in reply" rule
- Post-parse: strip <think>...</think> from inside the reply value
- On empty-after-strip: retry the LLM call ONCE with a stricter
reminder system message. If retry also empty, bubble up
sales_brain::error:empty_reply.
KI-170 β Timeout bumped 25s β 45s.
Live latency samples on qwen3-next-80b + response_format=json_object:
16.3s / 16.6s / 6.1s. 25s ceiling breached periodically. Bumped both
inner (_TIMEOUT_S in sales_brain) and outer (orchestrator wait_for) to
45s. Outer timeout fallback message reworded from "Sorry, the system
is taking too long" β "Sorry, that took longer than expected β could
you say that one more time?".
VERIFICATION:
py_compile clean for sales_brain.py + orchestrator.py.
npx tsc --noEmit clean (frontend).
Live latency probe: brain_used=sales_brain::nim:qwen, replies natural.
DOCS CASCADE: still held per user request. Stash @0 has WS5 doc edits
(CLAUDE.md + README.md + ADR-039) β restoring after live confirmation
of this bundle.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- backend/orchestrator.py +4 -4
- backend/sales_brain.py +76 -4
- frontend/src/app/page.tsx +72 -63
- frontend/src/lib/useStreamingVoice.ts +538 -0
|
@@ -569,16 +569,16 @@ async def handle_turn(
|
|
| 569 |
chat_history=chat_history[-10:],
|
| 570 |
session_id=session_id,
|
| 571 |
),
|
| 572 |
-
timeout=
|
| 573 |
)
|
| 574 |
except asyncio.TimeoutError:
|
| 575 |
# No scripted fallback. Fail loud β let the operator see it.
|
| 576 |
sb_result = SalesBrainResult(
|
| 577 |
-
reply_text="Sorry,
|
| 578 |
captured_updates={},
|
| 579 |
ready_for_recommendations=False,
|
| 580 |
-
brain_used="sales_brain::error:
|
| 581 |
-
error_reason="
|
| 582 |
)
|
| 583 |
|
| 584 |
# Apply captured updates to profile. captured_updates is already
|
|
|
|
| 569 |
chat_history=chat_history[-10:],
|
| 570 |
session_id=session_id,
|
| 571 |
),
|
| 572 |
+
timeout=45.0, # KI-170 β bumped from 25s; qwen3-next-80b + JSON mode regularly lands 15-25s
|
| 573 |
)
|
| 574 |
except asyncio.TimeoutError:
|
| 575 |
# No scripted fallback. Fail loud β let the operator see it.
|
| 576 |
sb_result = SalesBrainResult(
|
| 577 |
+
reply_text="Sorry, that took longer than expected β could you say that one more time?",
|
| 578 |
captured_updates={},
|
| 579 |
ready_for_recommendations=False,
|
| 580 |
+
brain_used="sales_brain::error:timeout_45s",
|
| 581 |
+
error_reason="outer_timeout_45s",
|
| 582 |
)
|
| 583 |
|
| 584 |
# Apply captured updates to profile. captured_updates is already
|
|
@@ -80,10 +80,16 @@ class SalesBrainResult:
|
|
| 80 |
# 25s mirrors fact_find_brain's KI-075 setting β gives NIM cold-start headroom
|
| 81 |
# + leaves room for one chain fallback. The fast-brain chain already has its
|
| 82 |
# own per-link + total-chain budget; this wait_for is a belt-and-braces stop.
|
| 83 |
-
_TIMEOUT_S: float =
|
| 84 |
_MAX_TOKENS: int = 700 # prose + structured JSON object with safety margin
|
| 85 |
_TEMPERATURE: float = 0.6 # mirrors fact_find_brain β conversational warmth
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
# ----------------------------------------------------------------------------
|
| 89 |
# Slot metadata β descriptions surfaced to the LLM so it knows what to ask for
|
|
@@ -139,6 +145,7 @@ CONVERSATION RULES:
|
|
| 139 |
- Indian English is fine β "βΉ", "lakh", "metro/tier-2 city", "BP", "diabetes" all natural.
|
| 140 |
- On health conditions, be straight: hiding a condition lowers premium today but turns into a denied claim later. Encourage honesty without lecturing.
|
| 141 |
- Never use markdown bold/italics β your reply may be read aloud by TTS.
|
|
|
|
| 142 |
|
| 143 |
OUTPUT FORMAT β STRICT:
|
| 144 |
Return a SINGLE JSON object with exactly these three keys:
|
|
@@ -432,6 +439,13 @@ async def drive_sales_brain(
|
|
| 432 |
if not isinstance(reply_text, str):
|
| 433 |
reply_text = str(reply_text)
|
| 434 |
reply_text = reply_text.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
|
| 436 |
captures_raw = parsed.get("captures") or parsed.get("captured") or {}
|
| 437 |
if not isinstance(captures_raw, dict):
|
|
@@ -445,16 +459,74 @@ async def drive_sales_brain(
|
|
| 445 |
# Normalize + validate captures via the deterministic post-processor.
|
| 446 |
captured_updates = normalize_captures(captures_raw, profile)
|
| 447 |
|
| 448 |
-
#
|
| 449 |
-
#
|
|
|
|
| 450 |
if not reply_text:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 451 |
return SalesBrainResult(
|
| 452 |
reply_text="",
|
| 453 |
captured_updates=captured_updates,
|
| 454 |
ready_for_recommendations=ready_for_recommendations,
|
| 455 |
brain_used="sales_brain::error:empty_reply",
|
| 456 |
raw_json=parsed,
|
| 457 |
-
error_reason=f"reply_field_empty from model={served_model}",
|
| 458 |
)
|
| 459 |
|
| 460 |
return SalesBrainResult(
|
|
|
|
| 80 |
# 25s mirrors fact_find_brain's KI-075 setting β gives NIM cold-start headroom
|
| 81 |
# + leaves room for one chain fallback. The fast-brain chain already has its
|
| 82 |
# own per-link + total-chain budget; this wait_for is a belt-and-braces stop.
|
| 83 |
+
_TIMEOUT_S: float = 45.0 # KI-170 β qwen3-next-80b + JSON mode regularly lands 15-25s; 25s breached periodically
|
| 84 |
_MAX_TOKENS: int = 700 # prose + structured JSON object with safety margin
|
| 85 |
_TEMPERATURE: float = 0.6 # mirrors fact_find_brain β conversational warmth
|
| 86 |
|
| 87 |
+
# KI-169 β strip <think>...</think> blocks the LLM may emit inside the JSON
|
| 88 |
+
# reply value (vs. as a preamble before the JSON, which _parse_brain_json
|
| 89 |
+
# already handles).
|
| 90 |
+
import re as _re_for_think
|
| 91 |
+
_REPLY_THINK_BLOCK = _re_for_think.compile(r"<think>.*?</think>", _re_for_think.DOTALL)
|
| 92 |
+
|
| 93 |
|
| 94 |
# ----------------------------------------------------------------------------
|
| 95 |
# Slot metadata β descriptions surfaced to the LLM so it knows what to ask for
|
|
|
|
| 145 |
- Indian English is fine β "βΉ", "lakh", "metro/tier-2 city", "BP", "diabetes" all natural.
|
| 146 |
- On health conditions, be straight: hiding a condition lowers premium today but turns into a denied claim later. Encourage honesty without lecturing.
|
| 147 |
- Never use markdown bold/italics β your reply may be read aloud by TTS.
|
| 148 |
+
- NEVER include <think> tags or chain-of-thought reasoning in the "reply" field. The "reply" field is the EXACT prose shown to the user. Any internal reasoning is forbidden in "reply" β keep it natural and conversational.
|
| 149 |
|
| 150 |
OUTPUT FORMAT β STRICT:
|
| 151 |
Return a SINGLE JSON object with exactly these three keys:
|
|
|
|
| 439 |
if not isinstance(reply_text, str):
|
| 440 |
reply_text = str(reply_text)
|
| 441 |
reply_text = reply_text.strip()
|
| 442 |
+
# KI-169 β strip <think>...</think> blocks the LLM may have emitted INSIDE
|
| 443 |
+
# the reply field value (vs. as a preamble to the JSON, which
|
| 444 |
+
# _parse_brain_json handles separately). qwen3-next-80b occasionally emits
|
| 445 |
+
# all of its reasoning inside <think>...</think> in the reply string,
|
| 446 |
+
# leaving nothing user-facing after a strip downstream.
|
| 447 |
+
if reply_text and "<think>" in reply_text:
|
| 448 |
+
reply_text = _REPLY_THINK_BLOCK.sub("", reply_text).strip()
|
| 449 |
|
| 450 |
captures_raw = parsed.get("captures") or parsed.get("captured") or {}
|
| 451 |
if not isinstance(captures_raw, dict):
|
|
|
|
| 459 |
# Normalize + validate captures via the deterministic post-processor.
|
| 460 |
captured_updates = normalize_captures(captures_raw, profile)
|
| 461 |
|
| 462 |
+
# KI-169 β empty reply text after <think>-strip: retry ONCE with a
|
| 463 |
+
# stricter reminder before failing. The retry adds a system message
|
| 464 |
+
# forbidding <think> tags + reminds the LLM to put prose in "reply".
|
| 465 |
if not reply_text:
|
| 466 |
+
logging.info(
|
| 467 |
+
"sales_brain empty reply after think-strip β retrying once (session=%s, model=%s)",
|
| 468 |
+
session_id, served_model,
|
| 469 |
+
)
|
| 470 |
+
retry_messages = list(messages) + [
|
| 471 |
+
ChatMessage(
|
| 472 |
+
role="system",
|
| 473 |
+
content=(
|
| 474 |
+
"REMINDER: Your previous response had an empty or <think>-only reply. "
|
| 475 |
+
"The 'reply' field MUST contain natural user-facing prose. "
|
| 476 |
+
"Do NOT include any <think> blocks or internal reasoning in 'reply'. "
|
| 477 |
+
"Try again with a clean conversational reply."
|
| 478 |
+
),
|
| 479 |
+
),
|
| 480 |
+
]
|
| 481 |
+
try:
|
| 482 |
+
retry_result = await asyncio.wait_for(
|
| 483 |
+
llm.chat(
|
| 484 |
+
messages=retry_messages,
|
| 485 |
+
temperature=_TEMPERATURE,
|
| 486 |
+
max_tokens=_MAX_TOKENS,
|
| 487 |
+
response_format={"type": "json_object"},
|
| 488 |
+
),
|
| 489 |
+
timeout=_TIMEOUT_S,
|
| 490 |
+
)
|
| 491 |
+
retry_parsed = _parse_brain_json((retry_result.text or "").strip())
|
| 492 |
+
if retry_parsed:
|
| 493 |
+
retry_reply = retry_parsed.get("reply") or ""
|
| 494 |
+
if isinstance(retry_reply, str):
|
| 495 |
+
retry_reply = retry_reply.strip()
|
| 496 |
+
if "<think>" in retry_reply:
|
| 497 |
+
retry_reply = _REPLY_THINK_BLOCK.sub("", retry_reply).strip()
|
| 498 |
+
if retry_reply:
|
| 499 |
+
# Merge new captures from retry on top of first attempt
|
| 500 |
+
retry_captures = retry_parsed.get("captures") or {}
|
| 501 |
+
if isinstance(retry_captures, dict):
|
| 502 |
+
merged = dict(captures_raw)
|
| 503 |
+
merged.update(retry_captures)
|
| 504 |
+
captured_updates = normalize_captures(merged, profile)
|
| 505 |
+
retry_ready = retry_parsed.get("ready_for_recommendations")
|
| 506 |
+
if retry_ready is not None:
|
| 507 |
+
ready_for_recommendations = bool(retry_ready)
|
| 508 |
+
retry_model = getattr(retry_result, "model", served_model) or served_model
|
| 509 |
+
return SalesBrainResult(
|
| 510 |
+
reply_text=retry_reply,
|
| 511 |
+
captured_updates=captured_updates,
|
| 512 |
+
ready_for_recommendations=ready_for_recommendations,
|
| 513 |
+
brain_used=f"sales_brain::nim:{retry_model}::retry",
|
| 514 |
+
raw_json=retry_parsed,
|
| 515 |
+
error_reason=None,
|
| 516 |
+
)
|
| 517 |
+
except (asyncio.TimeoutError, Exception) as retry_exc: # noqa: BLE001
|
| 518 |
+
logging.warning(
|
| 519 |
+
"sales_brain retry also failed (session=%s): %s",
|
| 520 |
+
session_id, type(retry_exc).__name__,
|
| 521 |
+
)
|
| 522 |
+
# Retry failed β bubble up to orchestrator
|
| 523 |
return SalesBrainResult(
|
| 524 |
reply_text="",
|
| 525 |
captured_updates=captured_updates,
|
| 526 |
ready_for_recommendations=ready_for_recommendations,
|
| 527 |
brain_used="sales_brain::error:empty_reply",
|
| 528 |
raw_json=parsed,
|
| 529 |
+
error_reason=f"reply_field_empty from model={served_model} (retry also empty)",
|
| 530 |
)
|
| 531 |
|
| 532 |
return SalesBrainResult(
|
|
@@ -30,7 +30,10 @@ import {
|
|
| 30 |
UserProfile,
|
| 31 |
} from "@/lib/api";
|
| 32 |
import { translate, UILang, StringKey, GLOSSARY } from "@/lib/i18n";
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
type DisplayMessage = ChatMessage & {
|
| 36 |
id: string;
|
|
@@ -145,24 +148,65 @@ export default function Page() {
|
|
| 145 |
const fileInputRef = useRef<HTMLInputElement>(null);
|
| 146 |
const scrollRef = useRef<HTMLDivElement>(null);
|
| 147 |
|
| 148 |
-
//
|
| 149 |
-
//
|
| 150 |
-
//
|
| 151 |
-
|
|
|
|
| 152 |
// KI-165 (2026-05-15) β typed-text request inflight flag, observed by the
|
| 153 |
-
// voice hook so
|
| 154 |
-
//
|
| 155 |
// Set to true at the start of `send()`, reset to false in its finally.
|
| 156 |
// Use a ref (not state) so the voice hook reads the latest value without
|
| 157 |
// re-rendering / re-subscribing.
|
| 158 |
const isTextRequestPendingRef = useRef(false);
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
isTextRequestPendingRef,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
|
| 167 |
useEffect(() => {
|
| 168 |
getHealth()
|
|
@@ -394,59 +438,24 @@ export default function Page() {
|
|
| 394 |
}
|
| 395 |
}
|
| 396 |
|
| 397 |
-
//
|
| 398 |
-
//
|
|
|
|
|
|
|
|
|
|
| 399 |
useEffect(() => {
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
setVoicePhase("thinking"); // KI-038 β STT done; brain in flight now
|
| 409 |
-
const history: ChatMessage[] = messages.map((m) => ({ role: m.role, content: m.content }));
|
| 410 |
-
const active_view: "chat" | "marketplace" | "profile" | "premium" | "policy_detail" =
|
| 411 |
-
openPolicy ? "policy_detail" :
|
| 412 |
-
showMarketplace ? "marketplace" :
|
| 413 |
-
showProfile ? "profile" :
|
| 414 |
-
showPremium ? "premium" :
|
| 415 |
-
"chat";
|
| 416 |
-
|
| 417 |
-
const res = await postChat({
|
| 418 |
-
user_text: text,
|
| 419 |
-
session_id: sessionId,
|
| 420 |
-
chat_history: history,
|
| 421 |
-
return_audio: true,
|
| 422 |
-
tts_language_code: ttsLang,
|
| 423 |
-
view_context: { active_view, active_policy_id: openPolicy?.policy_id },
|
| 424 |
-
signal: abort.signal,
|
| 425 |
-
});
|
| 426 |
-
setSessionId(res.session_id);
|
| 427 |
-
getProfileCompleteness(res.session_id)
|
| 428 |
-
.then(setProfileCompleteness)
|
| 429 |
-
.catch(() => {});
|
| 430 |
-
const audioUrl = res.audio_base64 ? audioBlobURLFromBase64(res.audio_base64) : undefined;
|
| 431 |
-
pushAssistant(res.reply_text, {
|
| 432 |
-
citations: res.citations,
|
| 433 |
-
audioUrl,
|
| 434 |
-
brain: res.brain_used,
|
| 435 |
-
latencyMs: res.latency_ms,
|
| 436 |
-
blocked: res.blocked,
|
| 437 |
-
});
|
| 438 |
-
// KI-030 β playback handled by the in-DOM <audio> in Message component
|
| 439 |
-
// (autoplay-on-mount); needed so barge-in's querySelectorAll("audio")
|
| 440 |
-
// can pause it. See comment in the typed-send branch.
|
| 441 |
-
} catch (e: unknown) {
|
| 442 |
-
const name = (e as { name?: string })?.name;
|
| 443 |
-
if (name === "AbortError") { setVoicePhase(null); return; } // barge-in
|
| 444 |
-
// eslint-disable-next-line no-console
|
| 445 |
-
console.error("[live mode] turn failed:", e);
|
| 446 |
-
} finally {
|
| 447 |
-
setVoicePhase(null); // KI-038 β clear indicator once turn lands
|
| 448 |
-
}
|
| 449 |
};
|
|
|
|
|
|
|
|
|
|
| 450 |
}, [messages, sessionId, ttsLang, openPolicy, showMarketplace, showProfile, showPremium]);
|
| 451 |
|
| 452 |
async function startRecording() {
|
|
|
|
| 30 |
UserProfile,
|
| 31 |
} from "@/lib/api";
|
| 32 |
import { translate, UILang, StringKey, GLOSSARY } from "@/lib/i18n";
|
| 33 |
+
// KI-168 (2026-05-15) β voice path migrated from custom-VAD `useLiveConversation`
|
| 34 |
+
// to native browser SpeechRecognition via `useStreamingVoice`. The old hook
|
| 35 |
+
// remains on disk as a graveyard reference until KI-168 is field-verified.
|
| 36 |
+
import { useStreamingVoice } from "@/lib/useStreamingVoice";
|
| 37 |
|
| 38 |
type DisplayMessage = ChatMessage & {
|
| 39 |
id: string;
|
|
|
|
| 148 |
const fileInputRef = useRef<HTMLInputElement>(null);
|
| 149 |
const scrollRef = useRef<HTMLDivElement>(null);
|
| 150 |
|
| 151 |
+
// KI-168 (2026-05-15) β streaming-voice path replaces the legacy
|
| 152 |
+
// useLiveConversation full-duplex VAD machinery. Interim transcript shows
|
| 153 |
+
// in the chat input as the user speaks; browser silence-detection auto-
|
| 154 |
+
// submits the final transcript through send().
|
| 155 |
+
//
|
| 156 |
// KI-165 (2026-05-15) β typed-text request inflight flag, observed by the
|
| 157 |
+
// voice hook so a transcript that finalises while a typed-text turn is
|
| 158 |
+
// racing is dropped silently instead of clobbering the text response.
|
| 159 |
// Set to true at the start of `send()`, reset to false in its finally.
|
| 160 |
// Use a ref (not state) so the voice hook reads the latest value without
|
| 161 |
// re-rendering / re-subscribing.
|
| 162 |
const isTextRequestPendingRef = useRef(false);
|
| 163 |
+
|
| 164 |
+
// Compatibility surface: the rest of the component (PTT path, UI pill, mic
|
| 165 |
+
// blocked indicator) still references live.live / live.setLive /
|
| 166 |
+
// live.recording / live.micPermissionDenied β preserve that shape so the
|
| 167 |
+
// rename is contained.
|
| 168 |
+
const [voiceEnabled, setVoiceEnabled] = useState(false);
|
| 169 |
+
const [voiceListening, setVoiceListening] = useState(false);
|
| 170 |
+
const [voicePermDenied, setVoicePermDenied] = useState(false);
|
| 171 |
+
// Submit handler β bound to the latest send() via a ref so the hook
|
| 172 |
+
// doesn't need to re-subscribe on every closure change.
|
| 173 |
+
const voiceSubmitRef = useRef<((text: string) => void) | null>(null);
|
| 174 |
+
const streamingVoice = useStreamingVoice({
|
| 175 |
+
enabled: voiceEnabled,
|
| 176 |
+
language: ttsLang,
|
| 177 |
isTextRequestPendingRef,
|
| 178 |
+
onInterimTranscript: (text) => {
|
| 179 |
+
// Show the running transcript in the chat input area as the user speaks.
|
| 180 |
+
setInput(text);
|
| 181 |
+
},
|
| 182 |
+
onFinalTranscript: (text) => {
|
| 183 |
+
// Browser detected end-of-speech β auto-submit through the regular
|
| 184 |
+
// send() path (which clears the input).
|
| 185 |
+
const submit = voiceSubmitRef.current;
|
| 186 |
+
if (submit) submit(text);
|
| 187 |
+
},
|
| 188 |
+
onError: (msg) => {
|
| 189 |
+
// Surface as an inline assistant message + drop the pill into blocked
|
| 190 |
+
// state if it's a permission failure. Match the legacy
|
| 191 |
+
// micPermissionDenied UX so the existing "π Mic blocked" branch fires.
|
| 192 |
+
if (/Mic permission denied|No microphone/i.test(msg)) {
|
| 193 |
+
setVoicePermDenied(true);
|
| 194 |
+
setVoiceEnabled(false);
|
| 195 |
+
}
|
| 196 |
+
setMessages((m) => [
|
| 197 |
+
...m,
|
| 198 |
+
{ id: `sys_${Date.now()}`, role: "assistant", content: msg },
|
| 199 |
+
]);
|
| 200 |
+
},
|
| 201 |
+
onListening: setVoiceListening,
|
| 202 |
});
|
| 203 |
+
// Legacy-shape adapter so the rest of the file's `live.*` refs keep working.
|
| 204 |
+
const live = {
|
| 205 |
+
live: voiceEnabled,
|
| 206 |
+
recording: voiceListening,
|
| 207 |
+
micPermissionDenied: voicePermDenied || !streamingVoice.isSupported,
|
| 208 |
+
setLive: setVoiceEnabled,
|
| 209 |
+
};
|
| 210 |
|
| 211 |
useEffect(() => {
|
| 212 |
getHealth()
|
|
|
|
| 438 |
}
|
| 439 |
}
|
| 440 |
|
| 441 |
+
// KI-168 (2026-05-15) β streaming-voice submit binding. The browser does
|
| 442 |
+
// the STT; we just route the finalised transcript through send() so it
|
| 443 |
+
// shares the typed-text code path (history, view_context, retries, TTS
|
| 444 |
+
// reply, profile completeness refresh, etc.). The input clears as part
|
| 445 |
+
// of send().
|
| 446 |
useEffect(() => {
|
| 447 |
+
voiceSubmitRef.current = (text: string) => {
|
| 448 |
+
const t = text.trim();
|
| 449 |
+
if (t.length < 2) return;
|
| 450 |
+
// Mirror the typed-input flow: drop transcript into the input
|
| 451 |
+
// (so the user sees their final words land in the box for a frame
|
| 452 |
+
// before send() clears it) then submit.
|
| 453 |
+
setInput(t);
|
| 454 |
+
void send(t);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 455 |
};
|
| 456 |
+
// send() reads `messages` / `sessionId` / `ttsLang` / view flags via
|
| 457 |
+
// closure; rebind whenever they change so the latest values are used.
|
| 458 |
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 459 |
}, [messages, sessionId, ttsLang, openPolicy, showMarketplace, showProfile, showPremium]);
|
| 460 |
|
| 461 |
async function startRecording() {
|
|
@@ -0,0 +1,538 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"use client";
|
| 2 |
+
|
| 3 |
+
/**
|
| 4 |
+
* useStreamingVoice β KI-168 (2026-05-15).
|
| 5 |
+
*
|
| 6 |
+
* Replaces the custom AudioWorklet + VAD + WAV-encode + /api/transcribe path
|
| 7 |
+
* (useLiveConversation) with the browser's native Web Speech API. The user
|
| 8 |
+
* sees their words land in the chat input area in real time as they speak,
|
| 9 |
+
* just like ChatGPT / Claude voice mode β and when the browser detects
|
| 10 |
+
* end-of-utterance silence, the final transcript is auto-submitted through
|
| 11 |
+
* the existing send() path.
|
| 12 |
+
*
|
| 13 |
+
* Why this exists
|
| 14 |
+
* -------------------------------------------------------------------------
|
| 15 |
+
* The previous live-mode stack accumulated 12+ KIs of failure modes
|
| 16 |
+
* (KI-044/057/060/064/113/114/115/131/134/139/141/159/165) trying to bolt
|
| 17 |
+
* a reliable VAD onto raw mic PCM. Every fix surfaced a new failure on a
|
| 18 |
+
* different mic / room / browser combo. The native SpeechRecognition API
|
| 19 |
+
* gives us:
|
| 20 |
+
* - browser-grade end-of-speech detection (no rmsThreshold tuning)
|
| 21 |
+
* - streaming interim transcripts (no "where did my words go?" gap)
|
| 22 |
+
* - in-browser STT (no /api/transcribe round-trip latency)
|
| 23 |
+
*
|
| 24 |
+
* Behaviour
|
| 25 |
+
* -------------------------------------------------------------------------
|
| 26 |
+
* - `enabled = true` β recognition.start() runs, mic icon stays live,
|
| 27 |
+
* interim transcript streams into the chat input via onInterimTranscript.
|
| 28 |
+
* - Browser detects ~1.5s silence β onend fires β we hand the final
|
| 29 |
+
* transcript to onFinalTranscript (caller calls send()).
|
| 30 |
+
* - After onend, if `enabled` is still true and no text request is in
|
| 31 |
+
* flight, we restart recognition so the mic stays live (continuous-mode
|
| 32 |
+
* emulation; native `continuous=true` doesn't fire silence-end on most
|
| 33 |
+
* browsers, so we use continuous=false + auto-restart instead).
|
| 34 |
+
* - `enabled = false` β recognition.abort() runs, no callbacks fire.
|
| 35 |
+
*
|
| 36 |
+
* Bot TTS playback is untouched β the page.tsx-owned <audio> elements still
|
| 37 |
+
* play Sarvam-generated audio for assistant replies.
|
| 38 |
+
*/
|
| 39 |
+
|
| 40 |
+
import { useCallback, useEffect, useRef, useState } from "react";
|
| 41 |
+
import { postTranscribe } from "./api";
|
| 42 |
+
|
| 43 |
+
// Minimal types for the Web Speech API since lib.dom.d.ts ships them under
|
| 44 |
+
// `webkitSpeechRecognition` only and the standard `SpeechRecognition` symbol
|
| 45 |
+
// is still vendor-prefixed in most browsers as of 2026-05.
|
| 46 |
+
type SpeechRecognitionAlternative = { transcript: string; confidence: number };
|
| 47 |
+
type SpeechRecognitionResult = {
|
| 48 |
+
isFinal: boolean;
|
| 49 |
+
length: number;
|
| 50 |
+
[index: number]: SpeechRecognitionAlternative;
|
| 51 |
+
};
|
| 52 |
+
type SpeechRecognitionResultList = {
|
| 53 |
+
length: number;
|
| 54 |
+
[index: number]: SpeechRecognitionResult;
|
| 55 |
+
};
|
| 56 |
+
interface SpeechRecognitionEventLike extends Event {
|
| 57 |
+
resultIndex: number;
|
| 58 |
+
results: SpeechRecognitionResultList;
|
| 59 |
+
}
|
| 60 |
+
interface SpeechRecognitionErrorEventLike extends Event {
|
| 61 |
+
error: string;
|
| 62 |
+
message?: string;
|
| 63 |
+
}
|
| 64 |
+
interface SpeechRecognitionInstance extends EventTarget {
|
| 65 |
+
lang: string;
|
| 66 |
+
continuous: boolean;
|
| 67 |
+
interimResults: boolean;
|
| 68 |
+
maxAlternatives: number;
|
| 69 |
+
start: () => void;
|
| 70 |
+
stop: () => void;
|
| 71 |
+
abort: () => void;
|
| 72 |
+
onresult: ((ev: SpeechRecognitionEventLike) => void) | null;
|
| 73 |
+
onerror: ((ev: SpeechRecognitionErrorEventLike) => void) | null;
|
| 74 |
+
onend: ((ev: Event) => void) | null;
|
| 75 |
+
onstart: ((ev: Event) => void) | null;
|
| 76 |
+
}
|
| 77 |
+
type SpeechRecognitionCtor = new () => SpeechRecognitionInstance;
|
| 78 |
+
|
| 79 |
+
export interface UseStreamingVoiceOptions {
|
| 80 |
+
enabled: boolean;
|
| 81 |
+
onInterimTranscript: (text: string) => void;
|
| 82 |
+
onFinalTranscript: (text: string) => void;
|
| 83 |
+
onError: (msg: string) => void;
|
| 84 |
+
onListening: (listening: boolean) => void;
|
| 85 |
+
isTextRequestPendingRef: React.MutableRefObject<boolean>;
|
| 86 |
+
language?: string;
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
export interface UseStreamingVoiceReturn {
|
| 90 |
+
start: () => void;
|
| 91 |
+
stop: () => void;
|
| 92 |
+
isSupported: boolean;
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
function resolveCtor(): SpeechRecognitionCtor | null {
|
| 96 |
+
if (typeof window === "undefined") return null;
|
| 97 |
+
const w = window as unknown as {
|
| 98 |
+
SpeechRecognition?: SpeechRecognitionCtor;
|
| 99 |
+
webkitSpeechRecognition?: SpeechRecognitionCtor;
|
| 100 |
+
};
|
| 101 |
+
return w.SpeechRecognition ?? w.webkitSpeechRecognition ?? null;
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
export function useStreamingVoice(
|
| 105 |
+
opts: UseStreamingVoiceOptions,
|
| 106 |
+
): UseStreamingVoiceReturn {
|
| 107 |
+
const {
|
| 108 |
+
enabled,
|
| 109 |
+
onInterimTranscript,
|
| 110 |
+
onFinalTranscript,
|
| 111 |
+
onError,
|
| 112 |
+
onListening,
|
| 113 |
+
isTextRequestPendingRef,
|
| 114 |
+
language = "en-IN",
|
| 115 |
+
} = opts;
|
| 116 |
+
|
| 117 |
+
// Keep latest callback refs so the recognition handlers always call the
|
| 118 |
+
// freshest closure without re-binding the recognition instance on every
|
| 119 |
+
// render (re-binding mid-utterance loses interim results).
|
| 120 |
+
const onInterimRef = useRef(onInterimTranscript);
|
| 121 |
+
const onFinalRef = useRef(onFinalTranscript);
|
| 122 |
+
const onErrorRef = useRef(onError);
|
| 123 |
+
const onListeningRef = useRef(onListening);
|
| 124 |
+
useEffect(() => { onInterimRef.current = onInterimTranscript; }, [onInterimTranscript]);
|
| 125 |
+
useEffect(() => { onFinalRef.current = onFinalTranscript; }, [onFinalTranscript]);
|
| 126 |
+
useEffect(() => { onErrorRef.current = onError; }, [onError]);
|
| 127 |
+
useEffect(() => { onListeningRef.current = onListening; }, [onListening]);
|
| 128 |
+
|
| 129 |
+
const recognitionRef = useRef<SpeechRecognitionInstance | null>(null);
|
| 130 |
+
const finalsRef = useRef<string[]>([]);
|
| 131 |
+
const wantRunningRef = useRef(false); // mirrors `enabled` for handler closures
|
| 132 |
+
const restartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
| 133 |
+
const errorBackoffRef = useRef(0);
|
| 134 |
+
|
| 135 |
+
// ----------------------------------------------------------------------
|
| 136 |
+
// KI-168 PHASE 2 β Sarvam authoritative-transcript layer.
|
| 137 |
+
// We run a MediaRecorder in parallel with SpeechRecognition. When the
|
| 138 |
+
// browser detects end-of-utterance silence (recognition.onend), we
|
| 139 |
+
// already have the raw audio chunks in memory. Send them to the backend
|
| 140 |
+
// /api/transcribe endpoint (Sarvam STT) and replace the Web Speech text
|
| 141 |
+
// with Sarvam's authoritative result. Web Speech remains the fallback if
|
| 142 |
+
// Sarvam times out, errors, or the audio path failed to initialise.
|
| 143 |
+
// ----------------------------------------------------------------------
|
| 144 |
+
const mediaStreamRef = useRef<MediaStream | null>(null);
|
| 145 |
+
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
| 146 |
+
const chunksRef = useRef<Blob[]>([]);
|
| 147 |
+
const recorderMimeRef = useRef<string>("audio/webm");
|
| 148 |
+
// True only when MediaRecorder.start() actually succeeded. If false we
|
| 149 |
+
// bypass the Sarvam path and use Web Speech transcripts directly.
|
| 150 |
+
const recorderActiveRef = useRef(false);
|
| 151 |
+
// Promise resolved on the recorder's next `stop` event so we can wait
|
| 152 |
+
// for the final ondataavailable chunk before building the blob.
|
| 153 |
+
const recorderStopWaiterRef = useRef<(() => void) | null>(null);
|
| 154 |
+
|
| 155 |
+
const [isSupported] = useState<boolean>(() => resolveCtor() !== null);
|
| 156 |
+
|
| 157 |
+
const clearRestartTimer = useCallback(() => {
|
| 158 |
+
if (restartTimerRef.current !== null) {
|
| 159 |
+
clearTimeout(restartTimerRef.current);
|
| 160 |
+
restartTimerRef.current = null;
|
| 161 |
+
}
|
| 162 |
+
}, []);
|
| 163 |
+
|
| 164 |
+
const safeStart = useCallback(() => {
|
| 165 |
+
const rec = recognitionRef.current;
|
| 166 |
+
if (!rec) return;
|
| 167 |
+
try {
|
| 168 |
+
rec.start();
|
| 169 |
+
} catch {
|
| 170 |
+
// start() throws InvalidStateError if recognition is already running.
|
| 171 |
+
// Safe to ignore β onstart/onend will keep state in sync.
|
| 172 |
+
}
|
| 173 |
+
}, []);
|
| 174 |
+
|
| 175 |
+
// Pick the best MediaRecorder mimeType. iOS Safari only supports
|
| 176 |
+
// audio/mp4; Chromium/Firefox prefer audio/webm. Mirrors page.tsx PTT
|
| 177 |
+
// recorder + the KI-134 fallback logic.
|
| 178 |
+
const pickRecorderMime = useCallback((): string => {
|
| 179 |
+
if (typeof window === "undefined" || typeof MediaRecorder === "undefined") {
|
| 180 |
+
return "";
|
| 181 |
+
}
|
| 182 |
+
const candidates = ["audio/webm;codecs=opus", "audio/webm", "audio/mp4", "audio/mpeg"];
|
| 183 |
+
for (const m of candidates) {
|
| 184 |
+
try {
|
| 185 |
+
if (MediaRecorder.isTypeSupported(m)) return m;
|
| 186 |
+
} catch {
|
| 187 |
+
// ignore
|
| 188 |
+
}
|
| 189 |
+
}
|
| 190 |
+
return "";
|
| 191 |
+
}, []);
|
| 192 |
+
|
| 193 |
+
const stopRecorder = useCallback((): Promise<void> => {
|
| 194 |
+
const recorder = mediaRecorderRef.current;
|
| 195 |
+
if (!recorder || recorder.state === "inactive") {
|
| 196 |
+
return Promise.resolve();
|
| 197 |
+
}
|
| 198 |
+
return new Promise<void>((resolve) => {
|
| 199 |
+
recorderStopWaiterRef.current = () => resolve();
|
| 200 |
+
try {
|
| 201 |
+
recorder.stop();
|
| 202 |
+
} catch {
|
| 203 |
+
// already stopped
|
| 204 |
+
recorderStopWaiterRef.current = null;
|
| 205 |
+
resolve();
|
| 206 |
+
}
|
| 207 |
+
});
|
| 208 |
+
}, []);
|
| 209 |
+
|
| 210 |
+
const teardownAudio = useCallback(() => {
|
| 211 |
+
const recorder = mediaRecorderRef.current;
|
| 212 |
+
if (recorder) {
|
| 213 |
+
try {
|
| 214 |
+
if (recorder.state !== "inactive") recorder.stop();
|
| 215 |
+
} catch {
|
| 216 |
+
// ignore
|
| 217 |
+
}
|
| 218 |
+
recorder.ondataavailable = null;
|
| 219 |
+
recorder.onstop = null;
|
| 220 |
+
recorder.onerror = null;
|
| 221 |
+
}
|
| 222 |
+
mediaRecorderRef.current = null;
|
| 223 |
+
const stream = mediaStreamRef.current;
|
| 224 |
+
if (stream) {
|
| 225 |
+
stream.getTracks().forEach((t) => {
|
| 226 |
+
try { t.stop(); } catch { /* ignore */ }
|
| 227 |
+
});
|
| 228 |
+
}
|
| 229 |
+
mediaStreamRef.current = null;
|
| 230 |
+
chunksRef.current = [];
|
| 231 |
+
recorderActiveRef.current = false;
|
| 232 |
+
recorderStopWaiterRef.current = null;
|
| 233 |
+
}, []);
|
| 234 |
+
|
| 235 |
+
const ensureAudioCapture = useCallback(async (): Promise<boolean> => {
|
| 236 |
+
if (mediaRecorderRef.current && recorderActiveRef.current) return true;
|
| 237 |
+
if (typeof navigator === "undefined" || !navigator.mediaDevices) return false;
|
| 238 |
+
if (typeof MediaRecorder === "undefined") return false;
|
| 239 |
+
try {
|
| 240 |
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
| 241 |
+
const mime = pickRecorderMime();
|
| 242 |
+
recorderMimeRef.current = mime || "audio/webm";
|
| 243 |
+
const recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
| 244 |
+
chunksRef.current = [];
|
| 245 |
+
recorder.ondataavailable = (ev: BlobEvent) => {
|
| 246 |
+
if (ev.data && ev.data.size > 0) chunksRef.current.push(ev.data);
|
| 247 |
+
};
|
| 248 |
+
recorder.onstop = () => {
|
| 249 |
+
const waiter = recorderStopWaiterRef.current;
|
| 250 |
+
recorderStopWaiterRef.current = null;
|
| 251 |
+
if (waiter) waiter();
|
| 252 |
+
};
|
| 253 |
+
recorder.onerror = (ev: Event) => {
|
| 254 |
+
console.debug("[useStreamingVoice] MediaRecorder error", ev);
|
| 255 |
+
};
|
| 256 |
+
mediaStreamRef.current = stream;
|
| 257 |
+
mediaRecorderRef.current = recorder;
|
| 258 |
+
// 1s timeslice so chunks land progressively β ondataavailable fires
|
| 259 |
+
// once per second instead of only on stop().
|
| 260 |
+
recorder.start(1000);
|
| 261 |
+
recorderActiveRef.current = true;
|
| 262 |
+
console.debug("[useStreamingVoice] MediaRecorder started", { mime: recorderMimeRef.current });
|
| 263 |
+
return true;
|
| 264 |
+
} catch (err) {
|
| 265 |
+
console.debug("[useStreamingVoice] MediaRecorder init failed β falling back to Web Speech only", err);
|
| 266 |
+
recorderActiveRef.current = false;
|
| 267 |
+
return false;
|
| 268 |
+
}
|
| 269 |
+
}, [pickRecorderMime]);
|
| 270 |
+
|
| 271 |
+
const buildRecognition = useCallback((): SpeechRecognitionInstance | null => {
|
| 272 |
+
const Ctor = resolveCtor();
|
| 273 |
+
if (!Ctor) return null;
|
| 274 |
+
const rec = new Ctor();
|
| 275 |
+
rec.lang = language;
|
| 276 |
+
rec.continuous = false;
|
| 277 |
+
rec.interimResults = true;
|
| 278 |
+
rec.maxAlternatives = 1;
|
| 279 |
+
|
| 280 |
+
rec.onstart = () => {
|
| 281 |
+
onListeningRef.current(true);
|
| 282 |
+
};
|
| 283 |
+
|
| 284 |
+
rec.onresult = (ev: SpeechRecognitionEventLike) => {
|
| 285 |
+
let interim = "";
|
| 286 |
+
// Walk every result; finals get pushed onto finalsRef, interims get
|
| 287 |
+
// concatenated into a running string that's displayed in the input.
|
| 288 |
+
for (let i = 0; i < ev.results.length; i++) {
|
| 289 |
+
const result = ev.results[i];
|
| 290 |
+
const alt = result[0];
|
| 291 |
+
if (!alt) continue;
|
| 292 |
+
if (result.isFinal) {
|
| 293 |
+
const t = alt.transcript.trim();
|
| 294 |
+
if (t) finalsRef.current.push(t);
|
| 295 |
+
} else {
|
| 296 |
+
interim += alt.transcript;
|
| 297 |
+
}
|
| 298 |
+
}
|
| 299 |
+
const running = (finalsRef.current.join(" ") + " " + interim).trim();
|
| 300 |
+
onInterimRef.current(running);
|
| 301 |
+
};
|
| 302 |
+
|
| 303 |
+
rec.onerror = (ev: SpeechRecognitionErrorEventLike) => {
|
| 304 |
+
const code = ev.error;
|
| 305 |
+
// `no-speech` and `aborted` are routine in continuous-restart mode β
|
| 306 |
+
// no audio detected in a window, or we deliberately stopped. Silent
|
| 307 |
+
// restart via onend.
|
| 308 |
+
if (code === "no-speech" || code === "aborted") return;
|
| 309 |
+
if (code === "not-allowed" || code === "service-not-allowed") {
|
| 310 |
+
wantRunningRef.current = false;
|
| 311 |
+
onErrorRef.current(
|
| 312 |
+
"Mic permission denied. Click the lock icon in your browser's URL bar to enable the microphone.",
|
| 313 |
+
);
|
| 314 |
+
return;
|
| 315 |
+
}
|
| 316 |
+
if (code === "audio-capture") {
|
| 317 |
+
wantRunningRef.current = false;
|
| 318 |
+
onErrorRef.current("No microphone detected. Check your audio device and try again.");
|
| 319 |
+
return;
|
| 320 |
+
}
|
| 321 |
+
if (code === "network") {
|
| 322 |
+
// Transient β let onend's restart loop pick it up with backoff.
|
| 323 |
+
errorBackoffRef.current = Math.min(errorBackoffRef.current + 500, 3000);
|
| 324 |
+
return;
|
| 325 |
+
}
|
| 326 |
+
onErrorRef.current(`Voice error: ${code}${ev.message ? ` (${ev.message})` : ""}`);
|
| 327 |
+
};
|
| 328 |
+
|
| 329 |
+
rec.onend = () => {
|
| 330 |
+
onListeningRef.current(false);
|
| 331 |
+
const webSpeechText = finalsRef.current.join(" ").trim();
|
| 332 |
+
finalsRef.current = [];
|
| 333 |
+
|
| 334 |
+
// KI-168 PHASE 2 β race guard: if a typed-text turn is in flight,
|
| 335 |
+
// drop both transcripts on the floor (text wins). Don't start a
|
| 336 |
+
// Sarvam fetch we'd be throwing away.
|
| 337 |
+
const textRacing = isTextRequestPendingRef.current;
|
| 338 |
+
|
| 339 |
+
const scheduleRestart = () => {
|
| 340 |
+
if (wantRunningRef.current && !isTextRequestPendingRef.current) {
|
| 341 |
+
const backoff = errorBackoffRef.current;
|
| 342 |
+
errorBackoffRef.current = 0;
|
| 343 |
+
clearRestartTimer();
|
| 344 |
+
restartTimerRef.current = setTimeout(() => {
|
| 345 |
+
restartTimerRef.current = null;
|
| 346 |
+
if (wantRunningRef.current) safeStart();
|
| 347 |
+
}, Math.max(50, backoff));
|
| 348 |
+
} else if (wantRunningRef.current && isTextRequestPendingRef.current) {
|
| 349 |
+
// Text turn in flight β retry shortly so mic resumes the moment
|
| 350 |
+
// the text turn lands.
|
| 351 |
+
clearRestartTimer();
|
| 352 |
+
restartTimerRef.current = setTimeout(() => {
|
| 353 |
+
restartTimerRef.current = null;
|
| 354 |
+
if (wantRunningRef.current && !isTextRequestPendingRef.current) safeStart();
|
| 355 |
+
}, 250);
|
| 356 |
+
}
|
| 357 |
+
};
|
| 358 |
+
|
| 359 |
+
// Pull the chunks we've accumulated so far so the recorder can keep
|
| 360 |
+
// capturing the next utterance without us re-running getUserMedia.
|
| 361 |
+
const drainChunks = (): Blob[] => {
|
| 362 |
+
const drained = chunksRef.current;
|
| 363 |
+
chunksRef.current = [];
|
| 364 |
+
return drained;
|
| 365 |
+
};
|
| 366 |
+
|
| 367 |
+
// If there's no audio recorder, or text is racing, fall back to the
|
| 368 |
+
// Phase 1 behaviour: submit the Web Speech transcript and bail.
|
| 369 |
+
if (!recorderActiveRef.current || textRacing) {
|
| 370 |
+
if (webSpeechText && !textRacing) {
|
| 371 |
+
onFinalRef.current(webSpeechText);
|
| 372 |
+
}
|
| 373 |
+
// Best-effort: clear any partial audio so the next utterance isn't
|
| 374 |
+
// contaminated with the previous one's tail.
|
| 375 |
+
if (recorderActiveRef.current) drainChunks();
|
| 376 |
+
scheduleRestart();
|
| 377 |
+
return;
|
| 378 |
+
}
|
| 379 |
+
|
| 380 |
+
// Sarvam path. Fire-and-forget so we don't block the recognition
|
| 381 |
+
// restart loop on the network round-trip.
|
| 382 |
+
void (async () => {
|
| 383 |
+
// Snapshot user-visible interim so the input area doesn't go blank
|
| 384 |
+
// while Sarvam is in flight. The page-side input still shows the
|
| 385 |
+
// Web Speech transcript; we'll overwrite it via onFinalTranscript
|
| 386 |
+
// once Sarvam returns.
|
| 387 |
+
if (webSpeechText) onInterimRef.current(webSpeechText);
|
| 388 |
+
|
| 389 |
+
// We need to stop the recorder to get the final dataavailable
|
| 390 |
+
// chunk; then we re-arm a new recorder for the next utterance.
|
| 391 |
+
await stopRecorder();
|
| 392 |
+
const drained = drainChunks();
|
| 393 |
+
const totalSize = drained.reduce((n, b) => n + b.size, 0);
|
| 394 |
+
console.debug("[useStreamingVoice] silence-detect", {
|
| 395 |
+
webSpeechLen: webSpeechText.length,
|
| 396 |
+
chunkCount: drained.length,
|
| 397 |
+
blobBytes: totalSize,
|
| 398 |
+
});
|
| 399 |
+
|
| 400 |
+
// Re-arm audio capture for the next utterance (don't block on it).
|
| 401 |
+
teardownAudio();
|
| 402 |
+
// Only restart the audio pipeline if the user still wants the mic
|
| 403 |
+
// live. Fire-and-forget; recognition restart is scheduled below.
|
| 404 |
+
if (wantRunningRef.current) {
|
| 405 |
+
void ensureAudioCapture();
|
| 406 |
+
}
|
| 407 |
+
|
| 408 |
+
// Skip submit when there's effectively no audio or no Web Speech
|
| 409 |
+
// text. ~3 KB is the empirical noise floor used by the PTT path's
|
| 410 |
+
// KI-134 silence guard (page.tsx uses 1 KB; we're stricter here
|
| 411 |
+
// because Live mode auto-fires on every silence pause).
|
| 412 |
+
const MIN_BLOB_BYTES = 3000;
|
| 413 |
+
if (!webSpeechText && totalSize < MIN_BLOB_BYTES) {
|
| 414 |
+
console.debug("[useStreamingVoice] skipping submit β no text and tiny blob");
|
| 415 |
+
scheduleRestart();
|
| 416 |
+
return;
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
// If Web Speech got nothing, but we have a real blob, still try
|
| 420 |
+
// Sarvam β Web Speech occasionally drops short utterances on noisy
|
| 421 |
+
// mics that Sarvam handles fine.
|
| 422 |
+
if (!webSpeechText && totalSize < MIN_BLOB_BYTES) {
|
| 423 |
+
scheduleRestart();
|
| 424 |
+
return;
|
| 425 |
+
}
|
| 426 |
+
|
| 427 |
+
// Race-check again after the await β text turn may have started
|
| 428 |
+
// while we were stopping the recorder.
|
| 429 |
+
if (isTextRequestPendingRef.current) {
|
| 430 |
+
console.debug("[useStreamingVoice] text turn started mid-await; dropping voice turn");
|
| 431 |
+
scheduleRestart();
|
| 432 |
+
return;
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
let authoritativeText = webSpeechText;
|
| 436 |
+
if (drained.length > 0 && totalSize >= MIN_BLOB_BYTES) {
|
| 437 |
+
const blob = new Blob(drained, { type: recorderMimeRef.current || "audio/webm" });
|
| 438 |
+
const controller = new AbortController();
|
| 439 |
+
const timeoutId = setTimeout(() => controller.abort(), 8000);
|
| 440 |
+
try {
|
| 441 |
+
console.debug("[useStreamingVoice] POST /api/transcribe", { bytes: blob.size, mime: blob.type, lang: language });
|
| 442 |
+
const sarvam = await postTranscribe(blob, language, controller.signal);
|
| 443 |
+
const sarvamText = (sarvam.text || "").trim();
|
| 444 |
+
if (sarvamText) {
|
| 445 |
+
authoritativeText = sarvamText;
|
| 446 |
+
console.debug("[useStreamingVoice] Sarvam OK", {
|
| 447 |
+
latency_ms: sarvam.latency_ms,
|
| 448 |
+
webSpeechLen: webSpeechText.length,
|
| 449 |
+
sarvamLen: sarvamText.length,
|
| 450 |
+
});
|
| 451 |
+
} else {
|
| 452 |
+
console.debug("[useStreamingVoice] Sarvam returned empty; using Web Speech fallback");
|
| 453 |
+
}
|
| 454 |
+
} catch (err) {
|
| 455 |
+
console.debug("[useStreamingVoice] Sarvam failed; using Web Speech fallback", err);
|
| 456 |
+
} finally {
|
| 457 |
+
clearTimeout(timeoutId);
|
| 458 |
+
}
|
| 459 |
+
}
|
| 460 |
+
|
| 461 |
+
if (authoritativeText && !isTextRequestPendingRef.current) {
|
| 462 |
+
onFinalRef.current(authoritativeText);
|
| 463 |
+
}
|
| 464 |
+
scheduleRestart();
|
| 465 |
+
})();
|
| 466 |
+
};
|
| 467 |
+
|
| 468 |
+
return rec;
|
| 469 |
+
}, [language, isTextRequestPendingRef, clearRestartTimer, safeStart, stopRecorder, teardownAudio, ensureAudioCapture]);
|
| 470 |
+
|
| 471 |
+
const start = useCallback(() => {
|
| 472 |
+
if (!isSupported) {
|
| 473 |
+
onErrorRef.current(
|
| 474 |
+
"Live voice not supported in this browser. Use push-to-talk or type instead.",
|
| 475 |
+
);
|
| 476 |
+
return;
|
| 477 |
+
}
|
| 478 |
+
wantRunningRef.current = true;
|
| 479 |
+
if (!recognitionRef.current) {
|
| 480 |
+
recognitionRef.current = buildRecognition();
|
| 481 |
+
}
|
| 482 |
+
finalsRef.current = [];
|
| 483 |
+
// Kick off audio capture in parallel with recognition. If it fails we
|
| 484 |
+
// degrade to Web Speech-only β onend handles the fallback path.
|
| 485 |
+
void ensureAudioCapture();
|
| 486 |
+
safeStart();
|
| 487 |
+
}, [isSupported, buildRecognition, safeStart, ensureAudioCapture]);
|
| 488 |
+
|
| 489 |
+
const stop = useCallback(() => {
|
| 490 |
+
wantRunningRef.current = false;
|
| 491 |
+
clearRestartTimer();
|
| 492 |
+
const rec = recognitionRef.current;
|
| 493 |
+
if (rec) {
|
| 494 |
+
try {
|
| 495 |
+
rec.abort();
|
| 496 |
+
} catch {
|
| 497 |
+
// ignore
|
| 498 |
+
}
|
| 499 |
+
}
|
| 500 |
+
teardownAudio();
|
| 501 |
+
finalsRef.current = [];
|
| 502 |
+
onListeningRef.current(false);
|
| 503 |
+
}, [clearRestartTimer, teardownAudio]);
|
| 504 |
+
|
| 505 |
+
// Drive start/stop from the `enabled` prop so the hook is fire-and-forget
|
| 506 |
+
// for the caller (mirrors useLiveConversation's `live` state semantics).
|
| 507 |
+
useEffect(() => {
|
| 508 |
+
if (enabled) {
|
| 509 |
+
start();
|
| 510 |
+
} else {
|
| 511 |
+
stop();
|
| 512 |
+
}
|
| 513 |
+
return () => {
|
| 514 |
+
stop();
|
| 515 |
+
};
|
| 516 |
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 517 |
+
}, [enabled]);
|
| 518 |
+
|
| 519 |
+
// Unmount cleanup.
|
| 520 |
+
useEffect(() => {
|
| 521 |
+
return () => {
|
| 522 |
+
wantRunningRef.current = false;
|
| 523 |
+
clearRestartTimer();
|
| 524 |
+
const rec = recognitionRef.current;
|
| 525 |
+
if (rec) {
|
| 526 |
+
try { rec.abort(); } catch {}
|
| 527 |
+
rec.onresult = null;
|
| 528 |
+
rec.onerror = null;
|
| 529 |
+
rec.onend = null;
|
| 530 |
+
rec.onstart = null;
|
| 531 |
+
}
|
| 532 |
+
recognitionRef.current = null;
|
| 533 |
+
teardownAudio();
|
| 534 |
+
};
|
| 535 |
+
}, [clearRestartTimer, teardownAudio]);
|
| 536 |
+
|
| 537 |
+
return { start, stop, isSupported };
|
| 538 |
+
}
|