Spaces:
Sleeping
fix(voice+recall): KI-214 + KI-217 + KI-218 bundle
Browse filesKI-214 β Block recognition for ENTIRE request-in-flight window.
Previously dropResultsRef only blocked during <audio> playback (KI-203),
leaving a 1-3s gap between send() firing and bot audio rendering+playing
where mic captured bot voice from speakers. Now onresult also early-
returns when isTextRequestPendingRef is true. Eliminates the entire
echo-leak window from user submit until bot finishes speaking.
KI-217 β Fix first-phrase cutoff in utterance batching.
User showed "My name is Rohit Sarma" reduced to "last name" β start
of utterance dropped. Root cause: rec.onend reset finalsRef.current=[]
on every restart cycle. Chrome occasionally delivers buffered
isFinal=true events for the early speech AFTER onend (same dirty-
window quirk that motivated KI-203). Eager wipe meant late finals
landed in a freshly emptied ref, then got wiped again before submit.
Fix: introduced finalsConsumedRef monotonic cursor. onend reads
finalsRef.current.slice(consumed), appends new finals to pending,
bumps cursor. finalsRef itself only resets at the actual submit
boundary inside submitPendingUtterance + on user-toggled lifecycle
(start/stop). Late-delivered finals from the dying recognition
instance now stick around to be picked up on the next onend.
KI-218 β Richer welcome-back profile summary.
KI-196's "first buy" was sparse because the prior profile genuinely
only had primary_goal captured. Generalized the formatter to render
every captured slot in natural English: "age 29, single, metro city,
income βΉ25L+, first-time buyer, no pre-existing conditions, no
existing cover". When the stored profile has fewer slots, the
formatter renders what's there. Module-level label dicts map enums
to readable phrases (dependents/income/location/goal/budget).
VERIFICATION:
py_compile clean (orchestrator).
npx tsc --noEmit clean (frontend).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- backend/orchestrator.py +108 -10
- frontend/src/lib/useStreamingVoice.ts +29 -4
|
@@ -383,6 +383,111 @@ def _format_known_profile_summary(profile) -> str:
|
|
| 383 |
return summary
|
| 384 |
|
| 385 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 386 |
_HELPFUL_GAP_LABELS = {
|
| 387 |
"age": "your age",
|
| 388 |
"dependents": "who you're covering",
|
|
@@ -820,16 +925,9 @@ async def handle_turn(
|
|
| 820 |
# the user should see right now. Subsequent turns
|
| 821 |
# WILL go through the brain with the recall directive
|
| 822 |
# in the system prompt (see _build_system_prompt).
|
| 823 |
-
|
| 824 |
-
|
| 825 |
-
|
| 826 |
-
if summary.get("location_tier"):
|
| 827 |
-
_bits.append(str(summary["location_tier"]))
|
| 828 |
-
if summary.get("dependents"):
|
| 829 |
-
_bits.append(str(summary["dependents"]))
|
| 830 |
-
if summary.get("primary_goal"):
|
| 831 |
-
_bits.append(str(summary["primary_goal"]).replace("_", " "))
|
| 832 |
-
_summary_phrase = ", ".join(_bits) if _bits else "your earlier captures"
|
| 833 |
sb_result.reply_text = (
|
| 834 |
f"Welcome back, {session.profile.name} β I have a profile under your name "
|
| 835 |
f"from before: {_summary_phrase}. Continue from there or start fresh?"
|
|
|
|
| 383 |
return summary
|
| 384 |
|
| 385 |
|
| 386 |
+
# KI-218 (2026-05-15) β readable welcome-back recall summary.
|
| 387 |
+
# The original KI-196 implementation joined raw enum values with commas
|
| 388 |
+
# ("first buy" alone). This formatter enumerates every captured slot in
|
| 389 |
+
# natural English so the user can recognise their stored profile at a glance.
|
| 390 |
+
_DEPENDENTS_LABEL = {
|
| 391 |
+
"self": "single",
|
| 392 |
+
"self+spouse": "with spouse",
|
| 393 |
+
"self+spouse+kids": "with spouse and kids",
|
| 394 |
+
"self+parents": "with parents",
|
| 395 |
+
"self+spouse+kids+parents": "with spouse, kids and parents",
|
| 396 |
+
}
|
| 397 |
+
_INCOME_LABEL = {
|
| 398 |
+
"under_5L": "income under βΉ5L",
|
| 399 |
+
"5L-10L": "income βΉ5-10L",
|
| 400 |
+
"10L-25L": "income βΉ10-25L",
|
| 401 |
+
"25L+": "income βΉ25L+",
|
| 402 |
+
}
|
| 403 |
+
_LOCATION_LABEL = {
|
| 404 |
+
"metro": "metro city",
|
| 405 |
+
"tier1": "tier-1 city",
|
| 406 |
+
"tier2": "tier-2 city",
|
| 407 |
+
"tier3": "tier-3 city",
|
| 408 |
+
}
|
| 409 |
+
_GOAL_LABEL = {
|
| 410 |
+
"first_buy": "first-time buyer",
|
| 411 |
+
"upgrade": "upgrading existing cover",
|
| 412 |
+
"compare_specific": "comparing specific policies",
|
| 413 |
+
"tax_planning": "tax-planning",
|
| 414 |
+
}
|
| 415 |
+
_BUDGET_LABEL = {
|
| 416 |
+
"under_15k": "budget under βΉ15k",
|
| 417 |
+
"15k_30k": "budget βΉ15-30k",
|
| 418 |
+
"30k_60k": "budget βΉ30-60k",
|
| 419 |
+
"60k+": "budget βΉ60k+",
|
| 420 |
+
}
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def _format_recall_summary_phrase(summary: dict) -> str:
|
| 424 |
+
"""Build a natural-English phrase enumerating captured profile slots.
|
| 425 |
+
|
| 426 |
+
Skips slots that are None / unset / empty. Maps enum values to
|
| 427 |
+
user-readable words (no JSON, no underscores). Returns a comma-joined
|
| 428 |
+
string suitable to slot into the welcome-back ask.
|
| 429 |
+
|
| 430 |
+
See KI-218 β the prior implementation showed only raw enum tokens for
|
| 431 |
+
the 4 of 8 slots it touched, leaving the user unable to recognise the
|
| 432 |
+
stored profile.
|
| 433 |
+
"""
|
| 434 |
+
bits: list[str] = []
|
| 435 |
+
|
| 436 |
+
age = summary.get("age")
|
| 437 |
+
if age is not None:
|
| 438 |
+
bits.append(f"age {age}")
|
| 439 |
+
|
| 440 |
+
dependents = summary.get("dependents")
|
| 441 |
+
if dependents:
|
| 442 |
+
bits.append(_DEPENDENTS_LABEL.get(dependents, str(dependents).replace("_", " ")))
|
| 443 |
+
|
| 444 |
+
location = summary.get("location_tier")
|
| 445 |
+
if location:
|
| 446 |
+
bits.append(_LOCATION_LABEL.get(location, str(location)))
|
| 447 |
+
|
| 448 |
+
income = summary.get("income_band")
|
| 449 |
+
if income:
|
| 450 |
+
bits.append(_INCOME_LABEL.get(income, f"income {income}"))
|
| 451 |
+
|
| 452 |
+
goal = summary.get("primary_goal")
|
| 453 |
+
if goal:
|
| 454 |
+
bits.append(_GOAL_LABEL.get(goal, str(goal).replace("_", " ")))
|
| 455 |
+
|
| 456 |
+
# health_conditions: [] is a meaningful capture ("no PEDs") but the
|
| 457 |
+
# orchestrator's pre-filter drops empty lists, so [] won't reach us.
|
| 458 |
+
# We still handle it defensively in case future call sites pass it through.
|
| 459 |
+
if "health_conditions" in summary:
|
| 460 |
+
hc = summary.get("health_conditions")
|
| 461 |
+
if isinstance(hc, list):
|
| 462 |
+
if len(hc) == 0:
|
| 463 |
+
bits.append("no pre-existing conditions")
|
| 464 |
+
else:
|
| 465 |
+
bits.append("conditions: " + ", ".join(str(c) for c in hc))
|
| 466 |
+
elif hc:
|
| 467 |
+
bits.append(f"conditions: {hc}")
|
| 468 |
+
|
| 469 |
+
existing = summary.get("existing_cover_inr")
|
| 470 |
+
if existing is not None:
|
| 471 |
+
try:
|
| 472 |
+
n = int(existing)
|
| 473 |
+
if n == 0:
|
| 474 |
+
bits.append("no existing cover")
|
| 475 |
+
elif n >= 100000:
|
| 476 |
+
lakhs = n / 100000
|
| 477 |
+
lakh_str = f"{lakhs:.0f}" if lakhs.is_integer() else f"{lakhs:.1f}"
|
| 478 |
+
bits.append(f"βΉ{lakh_str}L existing cover")
|
| 479 |
+
else:
|
| 480 |
+
bits.append(f"βΉ{n:,} existing cover")
|
| 481 |
+
except (TypeError, ValueError):
|
| 482 |
+
bits.append(f"existing cover {existing}")
|
| 483 |
+
|
| 484 |
+
budget = summary.get("budget_band")
|
| 485 |
+
if budget:
|
| 486 |
+
bits.append(_BUDGET_LABEL.get(budget, f"budget {budget}"))
|
| 487 |
+
|
| 488 |
+
return ", ".join(bits) if bits else "your earlier captures"
|
| 489 |
+
|
| 490 |
+
|
| 491 |
_HELPFUL_GAP_LABELS = {
|
| 492 |
"age": "your age",
|
| 493 |
"dependents": "who you're covering",
|
|
|
|
| 925 |
# the user should see right now. Subsequent turns
|
| 926 |
# WILL go through the brain with the recall directive
|
| 927 |
# in the system prompt (see _build_system_prompt).
|
| 928 |
+
# KI-218 β rich, all-8-slot natural-English summary
|
| 929 |
+
# so user can recognise their stored profile.
|
| 930 |
+
_summary_phrase = _format_recall_summary_phrase(summary)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 931 |
sb_result.reply_text = (
|
| 932 |
f"Welcome back, {session.profile.name} β I have a profile under your name "
|
| 933 |
f"from before: {_summary_phrase}. Continue from there or start fresh?"
|
|
@@ -193,6 +193,15 @@ export function useStreamingVoice(
|
|
| 193 |
|
| 194 |
const recognitionRef = useRef<SpeechRecognitionInstance | null>(null);
|
| 195 |
const finalsRef = useRef<string[]>([]);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
const wantRunningRef = useRef(false); // mirrors `enabled` for handler closures
|
| 197 |
const restartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
| 198 |
const errorBackoffRef = useRef(0);
|
|
@@ -410,8 +419,11 @@ export function useStreamingVoice(
|
|
| 410 |
// TTS audio ("perfect days to get started Rohit") was leaking into
|
| 411 |
// the user input field between `audio.play()` firing and our abort()
|
| 412 |
// actually taking effect.
|
| 413 |
-
if (dropResultsRef.current) {
|
| 414 |
-
console.debug("[useStreamingVoice] KI-203 dropping recognition result
|
|
|
|
|
|
|
|
|
|
| 415 |
return;
|
| 416 |
}
|
| 417 |
let interim = "";
|
|
@@ -460,8 +472,14 @@ export function useStreamingVoice(
|
|
| 460 |
|
| 461 |
rec.onend = () => {
|
| 462 |
onListeningRef.current(false);
|
| 463 |
-
|
| 464 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 465 |
|
| 466 |
// KI-168 PHASE 2 β race guard: if a typed-text turn is in flight,
|
| 467 |
// drop both transcripts on the floor (text wins). Don't start a
|
|
@@ -576,6 +594,11 @@ export function useStreamingVoice(
|
|
| 576 |
const accumulatedChunks = pendingChunksRef.current;
|
| 577 |
pendingUtteranceRef.current = "";
|
| 578 |
pendingChunksRef.current = [];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 579 |
console.debug("[useStreamingVoice] KI-202 grace window elapsed β submitting", {
|
| 580 |
textLen: accumulatedText.length,
|
| 581 |
chunkCount: accumulatedChunks.length,
|
|
@@ -694,6 +717,7 @@ export function useStreamingVoice(
|
|
| 694 |
recognitionRef.current = buildRecognition();
|
| 695 |
}
|
| 696 |
finalsRef.current = [];
|
|
|
|
| 697 |
// Kick off audio capture in parallel with recognition. If it fails we
|
| 698 |
// degrade to Web Speech-only β onend handles the fallback path.
|
| 699 |
void ensureAudioCapture();
|
|
@@ -713,6 +737,7 @@ export function useStreamingVoice(
|
|
| 713 |
}
|
| 714 |
teardownAudio();
|
| 715 |
finalsRef.current = [];
|
|
|
|
| 716 |
// KI-202 β drop any pending utterance so toggling voice off mid-grace
|
| 717 |
// doesn't auto-submit a stale half-sentence next time voice comes on.
|
| 718 |
if (pendingSubmitTimerRef.current !== null) {
|
|
|
|
| 193 |
|
| 194 |
const recognitionRef = useRef<SpeechRecognitionInstance | null>(null);
|
| 195 |
const finalsRef = useRef<string[]>([]);
|
| 196 |
+
// KI-217 (2026-05-15) β track how many entries of finalsRef have already
|
| 197 |
+
// been drained to pendingUtteranceRef. Each onend reads the slice from
|
| 198 |
+
// `finalsConsumedRef.current` to end, then bumps the cursor. finalsRef
|
| 199 |
+
// itself is NOT reset between restart cycles β only after the grace-timer
|
| 200 |
+
// submit (when onFinalRef fires) or on user-toggled start/stop. This
|
| 201 |
+
// prevents a Chrome quirk where late-delivered isFinal results arriving
|
| 202 |
+
// after onend on a mid-utterance restart cycle would land in a freshly
|
| 203 |
+
// wiped finalsRef and get dropped on the NEXT onend cycle's drain.
|
| 204 |
+
const finalsConsumedRef = useRef<number>(0);
|
| 205 |
const wantRunningRef = useRef(false); // mirrors `enabled` for handler closures
|
| 206 |
const restartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
| 207 |
const errorBackoffRef = useRef(0);
|
|
|
|
| 419 |
// TTS audio ("perfect days to get started Rohit") was leaking into
|
| 420 |
// the user input field between `audio.play()` firing and our abort()
|
| 421 |
// actually taking effect.
|
| 422 |
+
if (dropResultsRef.current || isTextRequestPendingRef.current) {
|
| 423 |
+
console.debug("[useStreamingVoice] KI-203/214 dropping recognition result", {
|
| 424 |
+
drop: dropResultsRef.current,
|
| 425 |
+
textPending: isTextRequestPendingRef.current,
|
| 426 |
+
});
|
| 427 |
return;
|
| 428 |
}
|
| 429 |
let interim = "";
|
|
|
|
| 472 |
|
| 473 |
rec.onend = () => {
|
| 474 |
onListeningRef.current(false);
|
| 475 |
+
// KI-217 β drain only the NEW finals (everything past the consumed
|
| 476 |
+
// cursor). DO NOT reset finalsRef here: a late-delivered isFinal
|
| 477 |
+
// chunk arriving after onend would otherwise be wiped before the
|
| 478 |
+
// next onend cycle can pick it up. finalsRef is reset on actual
|
| 479 |
+
// utterance submit (grace-timer flush) and on user start/stop.
|
| 480 |
+
const newFinals = finalsRef.current.slice(finalsConsumedRef.current);
|
| 481 |
+
const webSpeechText = newFinals.join(" ").trim();
|
| 482 |
+
finalsConsumedRef.current = finalsRef.current.length;
|
| 483 |
|
| 484 |
// KI-168 PHASE 2 β race guard: if a typed-text turn is in flight,
|
| 485 |
// drop both transcripts on the floor (text wins). Don't start a
|
|
|
|
| 594 |
const accumulatedChunks = pendingChunksRef.current;
|
| 595 |
pendingUtteranceRef.current = "";
|
| 596 |
pendingChunksRef.current = [];
|
| 597 |
+
// KI-217 β the utterance is now being submitted; safe to wipe
|
| 598 |
+
// finalsRef + reset the consumed cursor. Any late results that
|
| 599 |
+
// arrive after this point are for a NEW utterance.
|
| 600 |
+
finalsRef.current = [];
|
| 601 |
+
finalsConsumedRef.current = 0;
|
| 602 |
console.debug("[useStreamingVoice] KI-202 grace window elapsed β submitting", {
|
| 603 |
textLen: accumulatedText.length,
|
| 604 |
chunkCount: accumulatedChunks.length,
|
|
|
|
| 717 |
recognitionRef.current = buildRecognition();
|
| 718 |
}
|
| 719 |
finalsRef.current = [];
|
| 720 |
+
finalsConsumedRef.current = 0;
|
| 721 |
// Kick off audio capture in parallel with recognition. If it fails we
|
| 722 |
// degrade to Web Speech-only β onend handles the fallback path.
|
| 723 |
void ensureAudioCapture();
|
|
|
|
| 737 |
}
|
| 738 |
teardownAudio();
|
| 739 |
finalsRef.current = [];
|
| 740 |
+
finalsConsumedRef.current = 0;
|
| 741 |
// KI-202 β drop any pending utterance so toggling voice off mid-grace
|
| 742 |
// doesn't auto-submit a stale half-sentence next time voice comes on.
|
| 743 |
if (pendingSubmitTimerRef.current !== null) {
|