Spaces:
Sleeping
fix(closure+voice+latency): KI-253 — P-bundle from U1 fails (P2 watchdog + P3 mark_rec gate + P4 instrumentation)
Browse filesU1 Playwright re-test surfaced 4 remaining bugs after V-bundle landed. P-agents
diagnosed + fixed:
P1 — React #418 hydration: NO CODE CHANGE. V3 fix at page.tsx:229-232 + 357-363
(mounted flag gating micPermissionDenied) is correct + committed (52e5881).
P1 local rebuild + serve produces zero #418 errors. U1's failure was a stale
Next.js cache on HF Space — KI-252 redeploy will refresh.
P2 — Voice "green pill, zero audio" silent failure (V4 incomplete):
V4's catch worked for hard rejections (NotAllowedError), but TWO silent paths
bypassed it:
(a) Fake / silent stream — getUserMedia resolves but recorder.state never
reaches "recording" (Playwright fake-mic, Android WebView, codec
rejection). No error thrown, pill flipped to "Voice on" over dead stream.
(b) OS-stalled getUserMedia — locked-down Chromium / mic-busy / WebViews
leave getUserMedia pending forever instead of rejecting. V4's catch
never fires; user sits in limbo.
(c) PTT path bypassed the banner entirely — startRecording's catch called
pushAssistant but NOT setVoiceErrorBanner, so PTT failures were invisible.
Fix:
- useStreamingVoice.ensureAudioCapture: Promise.race against 2s StallTimeout
watchdog + assert recorder.state === "recording" post-acquire, else throw
RecorderNotRecording. Both fall into the existing mic_permission_denied catch.
- page.tsx startRecording: same watchdog + state validation; outer catch now
routes to setVoiceErrorBanner + setVoicePermDenied + setRecording(false) so
PTT failures show the same red banner.
Playwright headless 3-shim test confirmed:
REJECT → banner at 101ms, pill reverts to "Mic blocked"
STALL → banner at 2028ms (watchdog), pill reverts
FAKE → banner at 102ms (state check), pill reverts
P3 — mark_recommendation never invoked on soft-close:
U1 Test 9: "I'll go with that one" → bot offered next steps but didn't call
mark_recommendation(is_final=true). RULE 7 wasn't pulling the tool.
Fix:
- single_brain.py RULE 7 reframed as ordered STEP 1 (mandatory tool call) /
STEP 2 (prose), with explicit "NEVER SKIP" + chosen_id resolution table
(ordinal / insurer / "that one" → session.last_recommendation_ids[0]) +
verbatim worked example + explicit "RULE 7 violation" framing.
- main.py: server-side _CLOSER_KEYWORD_RE safety net catches "go with",
"I'll take", "let's do", "buy this", "purchase", "sign me up", "I'll go
with". After single_brain returns, if user_text matches AND brain_used
doesn't already contain mark_recommendation AND session.last_recommendation_ids
is non-empty, auto-calls brain_tools.mark_recommendation(session, [pid],
is_final=True). Guarantees the event is recorded even when Gemini forgets.
P4 — Latency 18.7s + profile_complete missing:
- single_brain.handle_turn now logs per-iter latency:
INFO single_brain iter=N gemini=Xs tools=Ys tool_calls=[...] per_tool=[...]
INFO single_brain retrieve_policies elapsed=Xs chunks=N query_len=N
Lets ops see whether Gemini sequential calls, retrieve, or cold-start
dominate before optimizing further. Hypothesized breakdown for the 18.7s:
3 Gemini round-trips @ ~5s each + 2.2s retrieve = sequential calls dominate.
- main.py: ChatResponse gained profile_complete: bool field; helper
_compute_profile_complete delegates to brain_tools._profile_complete
(single source of truth for the 7-slot list). Wired into all 4 response
build sites (success, timeout fallback, exception fallback, malformed body).
Frontend can now read profile_complete directly without a second /api/profile/completeness
roundtrip.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- backend/main.py +95 -0
- backend/single_brain.py +77 -5
- frontend/next-env.d.ts +1 -1
- frontend/src/app/page.tsx +42 -7
- frontend/src/lib/useStreamingVoice.ts +44 -8
|
@@ -12,6 +12,7 @@ import asyncio
|
|
| 12 |
import base64
|
| 13 |
import json
|
| 14 |
import logging
|
|
|
|
| 15 |
import time
|
| 16 |
import uuid
|
| 17 |
from pathlib import Path
|
|
@@ -39,6 +40,18 @@ USE_SINGLE_BRAIN = _os.environ.get("USE_SINGLE_BRAIN", "false").lower() in (
|
|
| 39 |
"1", "true", "yes", "on",
|
| 40 |
)
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
# Singleton provider instances (initialized on first call)
|
| 43 |
_stt: Optional[SarvamSTT] = None
|
| 44 |
_tts: Optional[SarvamTTS] = None
|
|
@@ -133,6 +146,21 @@ class ChatResponse(BaseModel):
|
|
| 133 |
"flash an acknowledgment + refresh the completeness panel."
|
| 134 |
),
|
| 135 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
|
| 137 |
|
| 138 |
class TTSRequest(BaseModel):
|
|
@@ -180,6 +208,27 @@ class UploadResponse(BaseModel):
|
|
| 180 |
elapsed_ms: int
|
| 181 |
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
# ---------- app ----------
|
| 184 |
|
| 185 |
app = FastAPI(
|
|
@@ -232,6 +281,7 @@ async def _validation_exception_handler(request: Request, exc: RequestValidation
|
|
| 232 |
"faithfulness_reasons": [],
|
| 233 |
"blocked": False,
|
| 234 |
"profile_updates": {},
|
|
|
|
| 235 |
},
|
| 236 |
)
|
| 237 |
# Default behaviour for every other endpoint.
|
|
@@ -620,6 +670,7 @@ async def chat(req: ChatRequest, request: Request):
|
|
| 620 |
faithfulness_reasons=[],
|
| 621 |
blocked=False,
|
| 622 |
profile_updates={},
|
|
|
|
| 623 |
)
|
| 624 |
except Exception as e:
|
| 625 |
logging.exception(
|
|
@@ -646,6 +697,48 @@ async def chat(req: ChatRequest, request: Request):
|
|
| 646 |
faithfulness_reasons=[],
|
| 647 |
blocked=False,
|
| 648 |
profile_updates={},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 649 |
)
|
| 650 |
|
| 651 |
audio_b64 = None
|
|
@@ -740,6 +833,7 @@ async def chat(req: ChatRequest, request: Request):
|
|
| 740 |
faithfulness_reasons=turn.faithfulness_reasons,
|
| 741 |
blocked=turn.blocked,
|
| 742 |
profile_updates=turn.profile_updates,
|
|
|
|
| 743 |
)
|
| 744 |
except Exception as _resp_err: # noqa: BLE001
|
| 745 |
# Anything else (TypeError/AttributeError/ValidationError) on the
|
|
@@ -765,6 +859,7 @@ async def chat(req: ChatRequest, request: Request):
|
|
| 765 |
faithfulness_reasons=[],
|
| 766 |
blocked=False,
|
| 767 |
profile_updates={},
|
|
|
|
| 768 |
)
|
| 769 |
|
| 770 |
|
|
|
|
| 12 |
import base64
|
| 13 |
import json
|
| 14 |
import logging
|
| 15 |
+
import re
|
| 16 |
import time
|
| 17 |
import uuid
|
| 18 |
from pathlib import Path
|
|
|
|
| 40 |
"1", "true", "yes", "on",
|
| 41 |
)
|
| 42 |
|
| 43 |
+
# U1 Test 9 — safety net for RULE 7. If Gemini forgets to call
|
| 44 |
+
# mark_recommendation when the user clearly commits to a policy ("I'll go
|
| 45 |
+
# with that one", "let's do #2", "buy this"), the post-turn detector below
|
| 46 |
+
# auto-calls mark_recommendation against session.last_recommendation_ids[:1]
|
| 47 |
+
# so the closure event is recorded for analytics even when the LLM whiffs.
|
| 48 |
+
# Word-boundary anchored; case-insensitive at match-time.
|
| 49 |
+
_CLOSER_KEYWORD_RE = re.compile(
|
| 50 |
+
r"\b(go with|i'?ll take|i will take|let'?s do|let me get|sign me up|"
|
| 51 |
+
r"purchase|buy this|i want to purchase|i'?ll go with|i want to buy)\b",
|
| 52 |
+
re.IGNORECASE,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
# Singleton provider instances (initialized on first call)
|
| 56 |
_stt: Optional[SarvamSTT] = None
|
| 57 |
_tts: Optional[SarvamTTS] = None
|
|
|
|
| 146 |
"flash an acknowledgment + refresh the completeness panel."
|
| 147 |
),
|
| 148 |
)
|
| 149 |
+
# Issue B (KI-Z6-PROFILECOMPLETE, 2026-05-15) — frontend was making a
|
| 150 |
+
# second roundtrip to /api/profile/completeness after every chat turn to
|
| 151 |
+
# learn whether the 7 required slots are now captured. Surface it in the
|
| 152 |
+
# primary chat response so the UI can flip to 100% in the same render
|
| 153 |
+
# cycle. Computed via brain_tools._REQUIRED_FOR_READY (same slot list
|
| 154 |
+
# used by retrieve_policies' profile-complete gate) so client + server
|
| 155 |
+
# never disagree.
|
| 156 |
+
profile_complete: bool = Field(
|
| 157 |
+
False,
|
| 158 |
+
description=(
|
| 159 |
+
"True when every required profile slot (name, age, dependents, "
|
| 160 |
+
"location_tier, income_band, primary_goal, health_conditions) is "
|
| 161 |
+
"non-empty on the live session.profile at end-of-turn."
|
| 162 |
+
),
|
| 163 |
+
)
|
| 164 |
|
| 165 |
|
| 166 |
class TTSRequest(BaseModel):
|
|
|
|
| 208 |
elapsed_ms: int
|
| 209 |
|
| 210 |
|
| 211 |
+
# Issue B (KI-Z6-PROFILECOMPLETE, 2026-05-15) — single source of truth for
|
| 212 |
+
# "is this profile ready to recommend against". brain_tools._profile_complete
|
| 213 |
+
# uses the same _REQUIRED_FOR_READY tuple; we mirror the call here instead of
|
| 214 |
+
# duplicating the slot list so a future addition (e.g. risk_appetite) only
|
| 215 |
+
# needs to be applied in brain_tools.
|
| 216 |
+
def _compute_profile_complete(session_id: str) -> bool:
|
| 217 |
+
"""Read the live session profile and return True iff every required slot
|
| 218 |
+
is populated. Tolerant of every failure mode (no session yet, session
|
| 219 |
+
state import explodes, profile missing attrs) — returns False on any
|
| 220 |
+
error so the frontend NEVER sees a stale `true` from a partial profile.
|
| 221 |
+
"""
|
| 222 |
+
try:
|
| 223 |
+
from backend.session_state import get_session
|
| 224 |
+
from backend.brain_tools import _profile_complete
|
| 225 |
+
|
| 226 |
+
sess = get_session(session_id)
|
| 227 |
+
return bool(_profile_complete(sess.profile))
|
| 228 |
+
except Exception: # noqa: BLE001 — never block a chat reply for this
|
| 229 |
+
return False
|
| 230 |
+
|
| 231 |
+
|
| 232 |
# ---------- app ----------
|
| 233 |
|
| 234 |
app = FastAPI(
|
|
|
|
| 281 |
"faithfulness_reasons": [],
|
| 282 |
"blocked": False,
|
| 283 |
"profile_updates": {},
|
| 284 |
+
"profile_complete": False,
|
| 285 |
},
|
| 286 |
)
|
| 287 |
# Default behaviour for every other endpoint.
|
|
|
|
| 670 |
faithfulness_reasons=[],
|
| 671 |
blocked=False,
|
| 672 |
profile_updates={},
|
| 673 |
+
profile_complete=_compute_profile_complete(session_id),
|
| 674 |
)
|
| 675 |
except Exception as e:
|
| 676 |
logging.exception(
|
|
|
|
| 697 |
faithfulness_reasons=[],
|
| 698 |
blocked=False,
|
| 699 |
profile_updates={},
|
| 700 |
+
profile_complete=_compute_profile_complete(session_id),
|
| 701 |
+
)
|
| 702 |
+
|
| 703 |
+
# U1 Test 9 — server-side closer-keyword safety net for RULE 7.
|
| 704 |
+
# If the user clearly committed to a policy this turn but Gemini did
|
| 705 |
+
# NOT call mark_recommendation (single_brain stamps "mark_recommendation"
|
| 706 |
+
# into turn.brain_used when the tool fires — see single_brain.py:1052),
|
| 707 |
+
# auto-call mark_recommendation against session.last_recommendation_ids[:1]
|
| 708 |
+
# so the closure event is recorded for analytics regardless of whether
|
| 709 |
+
# the LLM remembered to pull the tool. Best-effort; never blocks the
|
| 710 |
+
# reply if anything goes wrong.
|
| 711 |
+
try:
|
| 712 |
+
if (
|
| 713 |
+
USE_SINGLE_BRAIN
|
| 714 |
+
and turn is not None
|
| 715 |
+
and getattr(turn, "reply_text", None)
|
| 716 |
+
and _CLOSER_KEYWORD_RE.search(req.user_text or "")
|
| 717 |
+
and "mark_recommendation" not in (turn.brain_used or "")
|
| 718 |
+
):
|
| 719 |
+
from backend.session_state import get_session as _get_session
|
| 720 |
+
from backend import brain_tools as _brain_tools
|
| 721 |
+
|
| 722 |
+
_closer_session = _get_session(session_id)
|
| 723 |
+
_last_recs = list(
|
| 724 |
+
getattr(_closer_session, "last_recommendation_ids", []) or []
|
| 725 |
+
)
|
| 726 |
+
if _last_recs:
|
| 727 |
+
_result = _brain_tools.mark_recommendation(
|
| 728 |
+
session=_closer_session,
|
| 729 |
+
policy_ids=_last_recs[:1],
|
| 730 |
+
is_final=True,
|
| 731 |
+
)
|
| 732 |
+
logging.info(
|
| 733 |
+
"U1-T9 closer auto-mark (session=%s) user_text=%r "
|
| 734 |
+
"policy_ids=%s result=%s",
|
| 735 |
+
session_id, req.user_text, _last_recs[:1], _result,
|
| 736 |
+
)
|
| 737 |
+
except Exception as _closer_err: # noqa: BLE001
|
| 738 |
+
# Safety-net must never break the reply.
|
| 739 |
+
logging.warning(
|
| 740 |
+
"U1-T9 closer auto-mark failed (session=%s): %s: %s",
|
| 741 |
+
session_id, type(_closer_err).__name__, _closer_err,
|
| 742 |
)
|
| 743 |
|
| 744 |
audio_b64 = None
|
|
|
|
| 833 |
faithfulness_reasons=turn.faithfulness_reasons,
|
| 834 |
blocked=turn.blocked,
|
| 835 |
profile_updates=turn.profile_updates,
|
| 836 |
+
profile_complete=_compute_profile_complete(session_id),
|
| 837 |
)
|
| 838 |
except Exception as _resp_err: # noqa: BLE001
|
| 839 |
# Anything else (TypeError/AttributeError/ValidationError) on the
|
|
|
|
| 859 |
faithfulness_reasons=[],
|
| 860 |
blocked=False,
|
| 861 |
profile_updates={},
|
| 862 |
+
profile_complete=_compute_profile_complete(session_id),
|
| 863 |
)
|
| 864 |
|
| 865 |
|
|
@@ -227,11 +227,38 @@ Do NOT call retrieve_policies for out-of-scope queries.
|
|
| 227 |
RULE 7 — Soft close after the customer picks one
|
| 228 |
═══════════════════════════════════════════════════════════
|
| 229 |
Once you have recommended AND the user has chosen a single policy ("I'll
|
| 230 |
-
go with #2", "let's pick the HDFC one", "sounds good"
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
Do not re-pitch alternatives after the user has chosen — only act on
|
| 236 |
their next instruction.
|
| 237 |
|
|
@@ -936,6 +963,13 @@ async def handle_turn(
|
|
| 936 |
last_payload: dict = {}
|
| 937 |
|
| 938 |
for it in range(MAX_ITERATIONS):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
try:
|
| 940 |
payload = await _gemini_call(
|
| 941 |
api_key=api_key,
|
|
@@ -951,6 +985,7 @@ async def handle_turn(
|
|
| 951 |
raise SingleBrainError(
|
| 952 |
f"gemini_call unexpected error: {type(e).__name__}: {e}"
|
| 953 |
) from e
|
|
|
|
| 954 |
|
| 955 |
last_payload = payload
|
| 956 |
parts = _extract_parts(payload)
|
|
@@ -962,6 +997,11 @@ async def handle_turn(
|
|
| 962 |
# called out — completely valid, return immediately.
|
| 963 |
if not function_calls:
|
| 964 |
last_text = text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 965 |
break
|
| 966 |
|
| 967 |
# CASE B — one or more function calls. Append the model turn
|
|
@@ -975,12 +1015,35 @@ async def handle_turn(
|
|
| 975 |
}
|
| 976 |
)
|
| 977 |
|
|
|
|
|
|
|
| 978 |
response_parts: list[dict] = []
|
| 979 |
for fc in function_calls:
|
| 980 |
name = fc["name"]
|
| 981 |
args = fc.get("args") or {}
|
| 982 |
tool_calls_made.append(name)
|
|
|
|
| 983 |
result = await _execute_tool(session, name, args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 984 |
|
| 985 |
# Bookkeeping for the TurnResult fields.
|
| 986 |
if name == "save_profile_field" and result.get("saved"):
|
|
@@ -1001,6 +1064,15 @@ async def handle_turn(
|
|
| 1001 |
}
|
| 1002 |
}
|
| 1003 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
|
| 1005 |
contents.append({"role": "user", "parts": response_parts})
|
| 1006 |
# And loop — Gemini gets another shot to either call more
|
|
|
|
| 227 |
RULE 7 — Soft close after the customer picks one
|
| 228 |
═══════════════════════════════════════════════════════════
|
| 229 |
Once you have recommended AND the user has chosen a single policy ("I'll
|
| 230 |
+
go with #2", "let's pick the HDFC one", "sounds good", "I'll take that",
|
| 231 |
+
"let's do the first one", "sign me up", "buy this", "I want to purchase"):
|
| 232 |
+
|
| 233 |
+
STEP 1 (MANDATORY, NEVER SKIP) — Call the tool FIRST, before writing prose:
|
| 234 |
+
mark_recommendation(policy_ids=[chosen_id], is_final=true)
|
| 235 |
+
|
| 236 |
+
To resolve "chosen_id":
|
| 237 |
+
- "the first one" / "first" / "#1" → session.last_recommendation_ids[0]
|
| 238 |
+
- "the second" / "#2" → session.last_recommendation_ids[1]
|
| 239 |
+
- "the HDFC one" → match insurer slug in last rec list
|
| 240 |
+
- "that one" / "this one" / bare "I'll go with that"
|
| 241 |
+
→ most recent recommendation =
|
| 242 |
+
session.last_recommendation_ids[0]
|
| 243 |
+
|
| 244 |
+
STEP 2 — Only AFTER the tool call, write the prose reply:
|
| 245 |
+
"Great choice! [Policy Name] is a solid pick for your profile. Would
|
| 246 |
+
you like me to walk through the purchase steps, or summarise the key
|
| 247 |
+
benefits?"
|
| 248 |
+
|
| 249 |
+
WORKED EXAMPLE
|
| 250 |
+
User: "I'll go with that one"
|
| 251 |
+
Your flow:
|
| 252 |
+
i. IDENTIFY which policy "that one" refers to. With no ordinal cue,
|
| 253 |
+
default to the most recent recommendation =
|
| 254 |
+
session.last_recommendation_ids[0].
|
| 255 |
+
ii. Call mark_recommendation(policy_ids=[chosen_id], is_final=true)
|
| 256 |
+
FIRST. This is non-negotiable — the recommendation MUST be
|
| 257 |
+
recorded for analytics before any prose is written.
|
| 258 |
+
iii. THEN write the prose reply offering next steps.
|
| 259 |
+
DO NOT skip step (ii). Offering "would you like purchase steps?" without
|
| 260 |
+
the mark_recommendation tool call is a RULE 7 violation.
|
| 261 |
+
|
| 262 |
Do not re-pitch alternatives after the user has chosen — only act on
|
| 263 |
their next instruction.
|
| 264 |
|
|
|
|
| 963 |
last_payload: dict = {}
|
| 964 |
|
| 965 |
for it in range(MAX_ITERATIONS):
|
| 966 |
+
# Issue A instrumentation (KI-Z6-LATENCY, 2026-05-15) — Priya T3
|
| 967 |
+
# timed at 18.7s vs an 8s budget. We need per-iteration breakdown
|
| 968 |
+
# of (Gemini call time) vs (tool exec time) to identify whether
|
| 969 |
+
# cold-start, embedding/Chroma, or sequential LLM calls dominate.
|
| 970 |
+
# Wall-clock timers below feed `_log.info("iter %d: ...")` so HF
|
| 971 |
+
# Space logs surface the breakdown without any extra plumbing.
|
| 972 |
+
_t_iter0 = time.perf_counter()
|
| 973 |
try:
|
| 974 |
payload = await _gemini_call(
|
| 975 |
api_key=api_key,
|
|
|
|
| 985 |
raise SingleBrainError(
|
| 986 |
f"gemini_call unexpected error: {type(e).__name__}: {e}"
|
| 987 |
) from e
|
| 988 |
+
_t_gemini = time.perf_counter() - _t_iter0
|
| 989 |
|
| 990 |
last_payload = payload
|
| 991 |
parts = _extract_parts(payload)
|
|
|
|
| 997 |
# called out — completely valid, return immediately.
|
| 998 |
if not function_calls:
|
| 999 |
last_text = text
|
| 1000 |
+
_log.info(
|
| 1001 |
+
"single_brain iter=%d gemini=%.2fs tools=%.2fs "
|
| 1002 |
+
"tool_calls=[] final_text=True",
|
| 1003 |
+
it, _t_gemini, 0.0,
|
| 1004 |
+
)
|
| 1005 |
break
|
| 1006 |
|
| 1007 |
# CASE B — one or more function calls. Append the model turn
|
|
|
|
| 1015 |
}
|
| 1016 |
)
|
| 1017 |
|
| 1018 |
+
_t_tools0 = time.perf_counter()
|
| 1019 |
+
_per_tool_latency: list[str] = [] # logged tail for iter summary
|
| 1020 |
response_parts: list[dict] = []
|
| 1021 |
for fc in function_calls:
|
| 1022 |
name = fc["name"]
|
| 1023 |
args = fc.get("args") or {}
|
| 1024 |
tool_calls_made.append(name)
|
| 1025 |
+
_t_tool0 = time.perf_counter()
|
| 1026 |
result = await _execute_tool(session, name, args)
|
| 1027 |
+
_t_tool = time.perf_counter() - _t_tool0
|
| 1028 |
+
_per_tool_latency.append(f"{name}={_t_tool:.2f}s")
|
| 1029 |
+
|
| 1030 |
+
# Issue A — when retrieve_policies dominates iter latency we
|
| 1031 |
+
# need to know whether it's the embedding step or the Chroma
|
| 1032 |
+
# ANN query. brain_tools.retrieve_policies already returns
|
| 1033 |
+
# chunks + count; surface the elapsed wall-clock here so the
|
| 1034 |
+
# log line tags retrieve_policies separately. The deeper
|
| 1035 |
+
# embedding vs Chroma breakdown lives inside rag.retrieve and
|
| 1036 |
+
# is out of scope for this patch; this gives ops enough signal
|
| 1037 |
+
# to decide whether to drill further.
|
| 1038 |
+
if name == "retrieve_policies":
|
| 1039 |
+
_log.info(
|
| 1040 |
+
"single_brain retrieve_policies elapsed=%.2fs "
|
| 1041 |
+
"chunks=%d query_len=%d filter_ids=%s",
|
| 1042 |
+
_t_tool,
|
| 1043 |
+
len(result.get("chunks") or []),
|
| 1044 |
+
len(str(args.get("query") or "")),
|
| 1045 |
+
bool(args.get("policy_filter_ids")),
|
| 1046 |
+
)
|
| 1047 |
|
| 1048 |
# Bookkeeping for the TurnResult fields.
|
| 1049 |
if name == "save_profile_field" and result.get("saved"):
|
|
|
|
| 1064 |
}
|
| 1065 |
}
|
| 1066 |
)
|
| 1067 |
+
_t_tools = time.perf_counter() - _t_tools0
|
| 1068 |
+
|
| 1069 |
+
_log.info(
|
| 1070 |
+
"single_brain iter=%d gemini=%.2fs tools=%.2fs "
|
| 1071 |
+
"tool_calls=[%s] per_tool=[%s]",
|
| 1072 |
+
it, _t_gemini, _t_tools,
|
| 1073 |
+
",".join(fc["name"] for fc in function_calls),
|
| 1074 |
+
" ".join(_per_tool_latency),
|
| 1075 |
+
)
|
| 1076 |
|
| 1077 |
contents.append({"role": "user", "parts": response_parts})
|
| 1078 |
# And loop — Gemini gets another shot to either call more
|
|
@@ -1,6 +1,6 @@
|
|
| 1 |
/// <reference types="next" />
|
| 2 |
/// <reference types="next/image-types/global" />
|
| 3 |
-
import "./.next/types/routes.d.ts";
|
| 4 |
|
| 5 |
// NOTE: This file should not be edited
|
| 6 |
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
|
|
|
| 1 |
/// <reference types="next" />
|
| 2 |
/// <reference types="next/image-types/global" />
|
| 3 |
+
import "./.next/dev/types/routes.d.ts";
|
| 4 |
|
| 5 |
// NOTE: This file should not be edited
|
| 6 |
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
|
@@ -790,13 +790,26 @@ export default function Page() {
|
|
| 790 |
// KI-185 (2026-05-15) — match the useStreamingVoice AEC constraints
|
| 791 |
// on the PTT path too, so echo cancellation applies whether the user
|
| 792 |
// is in live-voice mode OR push-to-talk.
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
|
| 799 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 800 |
const mime = MediaRecorder.isTypeSupported("audio/webm") ? "audio/webm" : "";
|
| 801 |
const recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
| 802 |
mediaRecorderRef.current = recorder;
|
|
@@ -962,6 +975,20 @@ export default function Page() {
|
|
| 962 |
}
|
| 963 |
};
|
| 964 |
recorder.start();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 965 |
setRecording(true);
|
| 966 |
|
| 967 |
// KI-027 — VAD auto-cutoff is now ALWAYS on for the push-to-talk
|
|
@@ -1020,6 +1047,14 @@ export default function Page() {
|
|
| 1020 |
}
|
| 1021 |
} catch (e) {
|
| 1022 |
console.error(e);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1023 |
pushAssistant(`Sorry — mic permission denied or unavailable.`);
|
| 1024 |
}
|
| 1025 |
}
|
|
|
|
| 790 |
// KI-185 (2026-05-15) — match the useStreamingVoice AEC constraints
|
| 791 |
// on the PTT path too, so echo cancellation applies whether the user
|
| 792 |
// is in live-voice mode OR push-to-talk.
|
| 793 |
+
// W2 (2026-05-15) — 2s watchdog so a stalled getUserMedia
|
| 794 |
+
// (corporate-locked Chromium, OS-mic-busy, certain Android WebViews)
|
| 795 |
+
// doesn't leave the PTT button red-pulsing over a dead mic with no
|
| 796 |
+
// banner. Mirrors useStreamingVoice.ensureAudioCapture's W2 race.
|
| 797 |
+
const stream: MediaStream = await Promise.race([
|
| 798 |
+
navigator.mediaDevices.getUserMedia({
|
| 799 |
+
audio: {
|
| 800 |
+
echoCancellation: true,
|
| 801 |
+
noiseSuppression: true,
|
| 802 |
+
autoGainControl: true,
|
| 803 |
+
},
|
| 804 |
+
}),
|
| 805 |
+
new Promise<MediaStream>((_, reject) => {
|
| 806 |
+
setTimeout(() => {
|
| 807 |
+
const e = new Error("getUserMedia stalled >2s") as Error & { name: string };
|
| 808 |
+
e.name = "StallTimeout";
|
| 809 |
+
reject(e);
|
| 810 |
+
}, 2000);
|
| 811 |
+
}),
|
| 812 |
+
]);
|
| 813 |
const mime = MediaRecorder.isTypeSupported("audio/webm") ? "audio/webm" : "";
|
| 814 |
const recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
| 815 |
mediaRecorderRef.current = recorder;
|
|
|
|
| 975 |
}
|
| 976 |
};
|
| 977 |
recorder.start();
|
| 978 |
+
// W2 (2026-05-15) — affirmative post-acquire validation. recorder.start()
|
| 979 |
+
// returns void and does NOT throw on a fake-mic / dead-stream / codec
|
| 980 |
+
// rejection; the only reliable signal is recorder.state. Without this
|
| 981 |
+
// check, Playwright's fake stream let the button flip to red-pulsing
|
| 982 |
+
// "Stop" over a silent capture with no banner. Treat any non-"recording"
|
| 983 |
+
// state as a hard fail and surface mic_permission_denied.
|
| 984 |
+
if (recorder.state !== "recording") {
|
| 985 |
+
try { stream.getTracks().forEach((t) => t.stop()); } catch { /* ignore */ }
|
| 986 |
+
mediaRecorderRef.current = null;
|
| 987 |
+
throw Object.assign(
|
| 988 |
+
new Error(`MediaRecorder did not enter recording state (got ${recorder.state})`),
|
| 989 |
+
{ name: "RecorderNotRecording" },
|
| 990 |
+
);
|
| 991 |
+
}
|
| 992 |
setRecording(true);
|
| 993 |
|
| 994 |
// KI-027 — VAD auto-cutoff is now ALWAYS on for the push-to-talk
|
|
|
|
| 1047 |
}
|
| 1048 |
} catch (e) {
|
| 1049 |
console.error(e);
|
| 1050 |
+
// W2 (2026-05-15) — route to the structured banner the same way the
|
| 1051 |
+
// live-voice path does, so PTT denials / stalls / fake-stream failures
|
| 1052 |
+
// surface in the same red banner (not just an in-chat assistant
|
| 1053 |
+
// message). Also revert the button by clearing recording state and
|
| 1054 |
+
// setting voicePermDenied so the "🔇 Mic blocked" branch fires.
|
| 1055 |
+
setRecording(false);
|
| 1056 |
+
setVoicePermDenied(true);
|
| 1057 |
+
setVoiceErrorBanner({ type: "mic_permission_denied", ts: Date.now() });
|
| 1058 |
pushAssistant(`Sorry — mic permission denied or unavailable.`);
|
| 1059 |
}
|
| 1060 |
}
|
|
@@ -432,13 +432,30 @@ export function useStreamingVoice(
|
|
| 432 |
// For headphone users this gives near-perfect echo cancellation;
|
| 433 |
// for speaker users it's 70-90% reduction (some bleed unavoidable
|
| 434 |
// without server-side reference cancellation).
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
const mime = pickRecorderMime();
|
| 443 |
recorderMimeRef.current = mime || "audio/webm";
|
| 444 |
const recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
|
@@ -459,8 +476,27 @@ export function useStreamingVoice(
|
|
| 459 |
// 1s timeslice so chunks land progressively — ondataavailable fires
|
| 460 |
// once per second instead of only on stop().
|
| 461 |
recorder.start(1000);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
recorderActiveRef.current = true;
|
| 463 |
-
console.debug("[useStreamingVoice] MediaRecorder started", {
|
|
|
|
|
|
|
|
|
|
| 464 |
return true;
|
| 465 |
} catch (err) {
|
| 466 |
// W1 (2026-05-15) — DOMException name → VoiceError mapping.
|
|
|
|
| 432 |
// For headphone users this gives near-perfect echo cancellation;
|
| 433 |
// for speaker users it's 70-90% reduction (some bleed unavoidable
|
| 434 |
// without server-side reference cancellation).
|
| 435 |
+
// W2 (2026-05-15) — 2s watchdog around getUserMedia.
|
| 436 |
+
// Some devices (Chromium on locked-down corporate Windows, certain
|
| 437 |
+
// Android WebViews, OS-level mic-busy states) STALL getUserMedia
|
| 438 |
+
// indefinitely instead of rejecting. Without a watchdog the pill
|
| 439 |
+
// sits at "Voice on" forever, no banner, no recovery path.
|
| 440 |
+
// Race the permission prompt against a 2000ms timeout that
|
| 441 |
+
// rejects with name="StallTimeout" so the catch below treats it
|
| 442 |
+
// identically to a hard denial (mic_permission_denied banner).
|
| 443 |
+
const stream: MediaStream = await Promise.race([
|
| 444 |
+
navigator.mediaDevices.getUserMedia({
|
| 445 |
+
audio: {
|
| 446 |
+
echoCancellation: true,
|
| 447 |
+
noiseSuppression: true,
|
| 448 |
+
autoGainControl: true,
|
| 449 |
+
},
|
| 450 |
+
}),
|
| 451 |
+
new Promise<MediaStream>((_, reject) => {
|
| 452 |
+
setTimeout(() => {
|
| 453 |
+
const e = new Error("getUserMedia stalled >2s") as Error & { name: string };
|
| 454 |
+
e.name = "StallTimeout";
|
| 455 |
+
reject(e);
|
| 456 |
+
}, 2000);
|
| 457 |
+
}),
|
| 458 |
+
]);
|
| 459 |
const mime = pickRecorderMime();
|
| 460 |
recorderMimeRef.current = mime || "audio/webm";
|
| 461 |
const recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream);
|
|
|
|
| 476 |
// 1s timeslice so chunks land progressively — ondataavailable fires
|
| 477 |
// once per second instead of only on stop().
|
| 478 |
recorder.start(1000);
|
| 479 |
+
// W2 (2026-05-15) — affirmative post-acquire validation. A
|
| 480 |
+
// MediaRecorder that .start()s without throwing is NOT proof the
|
| 481 |
+
// capture is alive: Playwright's fake-mic stream, a stream from a
|
| 482 |
+
// device that was unplugged between getUserMedia and start(), or a
|
| 483 |
+
// codec rejection that fires `onerror` async — all leave recorder.state
|
| 484 |
+
// anything other than "recording". Without this check, the pill flipped
|
| 485 |
+
// to "Voice on" over a silent stream. Treat any non-"recording" state
|
| 486 |
+
// as a hard fail and route to the same mic_permission_denied banner.
|
| 487 |
+
if (recorder.state !== "recording") {
|
| 488 |
+
try { stream.getTracks().forEach((t) => t.stop()); } catch { /* ignore */ }
|
| 489 |
+
mediaStreamRef.current = null;
|
| 490 |
+
mediaRecorderRef.current = null;
|
| 491 |
+
throw Object.assign(new Error(`MediaRecorder did not enter recording state (got ${recorder.state})`), {
|
| 492 |
+
name: "RecorderNotRecording",
|
| 493 |
+
});
|
| 494 |
+
}
|
| 495 |
recorderActiveRef.current = true;
|
| 496 |
+
console.debug("[useStreamingVoice] MediaRecorder started", {
|
| 497 |
+
mime: recorderMimeRef.current,
|
| 498 |
+
state: recorder.state,
|
| 499 |
+
});
|
| 500 |
return true;
|
| 501 |
} catch (err) {
|
| 502 |
// W1 (2026-05-15) — DOMException name → VoiceError mapping.
|