Deploy: Stack A (NIM brain + Maverick judge + Sarvam voice). D-019.

40-data/llm_health.json

@@ -1,22 +1,16 @@
 {
-  "updated_at": "2026-05-15T02:03:36Z",
+  "updated_at": "2026-05-15T02:09:13Z",
   "models": {
     "qwen/qwen3-next-80b-a3b-instruct": {
       "model": "qwen/qwen3-next-80b-a3b-instruct",
       "status": "healthy",
-      "last_success_at": "2026-05-15T02:03:36Z",
+      "last_success_at": "2026-05-15T02:09:13Z",
       "last_failure_at": null,
       "last_error": null,
-      "latency_ms": 772,
+      "latency_ms": 869,
       "consecutive_failures": 0,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 1208,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": true,
           "latency_ms": 771,
@@ -40,6 +34,12 @@
           "latency_ms": 772,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": true,
+          "latency_ms": 869,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -53,18 +53,12 @@
       "model": "qwen/qwen3.5-122b-a10b",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "timeout",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 8001,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 8002,
@@ -88,6 +82,12 @@
           "latency_ms": 8002,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 8002,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -101,18 +101,12 @@
       "model": "openai/gpt-oss-120b",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "empty_content",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 253,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 311,
@@ -136,6 +130,12 @@
           "latency_ms": 271,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 340,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -148,19 +148,13 @@
     "mistralai/mistral-large-3-675b-instruct-2512": {
       "model": "mistralai/mistral-large-3-675b-instruct-2512",
       "status": "healthy",
-      "last_success_at": "2026-05-15T02:03:36Z",
+      "last_success_at": "2026-05-15T02:09:13Z",
       "last_failure_at": "2026-05-15T02:00:06Z",
       "last_error": null,
-      "latency_ms": 514,
+      "latency_ms": 726,
       "consecutive_failures": 0,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 8002,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": true,
           "latency_ms": 617,
@@ -184,6 +178,12 @@
           "latency_ms": 514,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": true,
+          "latency_ms": 726,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -196,19 +196,13 @@
     "nvidia/llama-3.3-nemotron-super-49b-v1.5": {
       "model": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
       "status": "healthy",
-      "last_success_at": "2026-05-15T02:03:36Z",
+      "last_success_at": "2026-05-15T02:09:13Z",
       "last_failure_at": "2026-05-15T01:24:45Z",
       "last_error": null,
-      "latency_ms": 315,
+      "latency_ms": 364,
       "consecutive_failures": 0,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 349,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": true,
           "latency_ms": 5838,
@@ -232,6 +226,12 @@
           "latency_ms": 315,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": true,
+          "latency_ms": 364,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -243,20 +243,14 @@
     },
     "meta/llama-3.3-70b-instruct": {
       "model": "meta/llama-3.3-70b-instruct",
-      "status": "healthy",
+      "status": "degraded",
       "last_success_at": "2026-05-15T02:03:36Z",
-      "last_failure_at": "2026-05-15T01:41:07Z",
-      "last_error": null,
+      "last_failure_at": "2026-05-15T02:09:13Z",
+      "last_error": "timeout",
       "latency_ms": 698,
-      "consecutive_failures": 0,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 1,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 212,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": true,
           "latency_ms": 3890,
@@ -280,6 +274,12 @@
           "latency_ms": 698,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 8001,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -293,18 +293,12 @@
       "model": "deepseek-ai/deepseek-v4-pro",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "timeout",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 8003,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 8002,
@@ -328,6 +322,12 @@
           "latency_ms": 8003,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 8055,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -341,18 +341,12 @@
       "model": "nvidia/nemotron-3-nano-30b-a3b",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "empty_content",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 263,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 276,
@@ -376,6 +370,12 @@
           "latency_ms": 283,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 263,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -389,18 +389,12 @@
       "model": "deepseek-ai/deepseek-v4-flash",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "timeout",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 8004,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 8003,
@@ -424,6 +418,12 @@
           "latency_ms": 8001,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 8002,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -437,18 +437,12 @@
       "model": "moonshotai/kimi-k2-instruct-0905",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "http_404",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 84,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 81,
@@ -472,6 +466,12 @@
           "latency_ms": 91,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 86,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -485,18 +485,12 @@
       "model": "minimaxai/minimax-m2.5",
       "status": "down",
       "last_success_at": null,
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "http_410",
       "latency_ms": null,
-      "consecutive_failures": 21,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 22,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 29,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 31,
@@ -520,6 +514,12 @@
           "latency_ms": 31,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 30,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -532,19 +532,13 @@
     "meta/llama-4-maverick-17b-128e-instruct": {
       "model": "meta/llama-4-maverick-17b-128e-instruct",
       "status": "healthy",
-      "last_success_at": "2026-05-15T02:03:36Z",
+      "last_success_at": "2026-05-15T02:09:13Z",
       "last_failure_at": "2026-05-14T01:03:07Z",
       "last_error": null,
-      "latency_ms": 214,
+      "latency_ms": 268,
       "consecutive_failures": 0,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 205,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": true,
           "latency_ms": 247,
@@ -568,6 +562,12 @@
           "latency_ms": 214,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": true,
+          "latency_ms": 268,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -581,18 +581,12 @@
       "model": "openrouter:openai/gpt-oss-120b",
       "status": "down",
       "last_success_at": "2026-05-15T01:46:37Z",
-      "last_failure_at": "2026-05-15T02:03:36Z",
+      "last_failure_at": "2026-05-15T02:09:13Z",
       "last_error": "empty_content",
       "latency_ms": 1349,
-      "consecutive_failures": 5,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "consecutive_failures": 6,
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": false,
-          "latency_ms": 1635,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": false,
           "latency_ms": 856,
@@ -616,6 +610,12 @@
           "latency_ms": 497,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": false,
+          "latency_ms": 1535,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,
@@ -628,19 +628,13 @@
     "groq:llama-3.3-70b-versatile": {
       "model": "groq:llama-3.3-70b-versatile",
       "status": "healthy",
-      "last_success_at": "2026-05-15T02:03:36Z",
+      "last_success_at": "2026-05-15T02:09:13Z",
       "last_failure_at": null,
       "last_error": null,
-      "latency_ms": 252,
+      "latency_ms": 260,
       "consecutive_failures": 0,
-      "tested_at": "2026-05-15T02:03:36Z",
+      "tested_at": "2026-05-15T02:09:13Z",
       "probe_history": [
-        {
-          "ok": true,
-          "latency_ms": 247,
-          "ts": "2026-05-15T01:52:30Z",
-          "src": "probe"
-        },
         {
           "ok": true,
           "latency_ms": 261,
@@ -664,6 +658,12 @@
           "latency_ms": 252,
           "ts": "2026-05-15T02:03:36Z",
           "src": "probe"
+        },
+        {
+          "ok": true,
+          "latency_ms": 260,
+          "ts": "2026-05-15T02:09:13Z",
+          "src": "probe"
         }
       ],
       "degraded_until_monotonic": 0.0,

Dockerfile

@@ -70,7 +70,7 @@ COPY 40-data ./40-data
 # (7356 chunks from a prior ingest) instead of the freshly-uploaded
 # cleaned one (3799 chunks). Bump CACHE_BUST manually each time the
 # dataset is re-uploaded; the value just needs to change.
-ARG DATASET_CACHE_BUST=2026-05-15-ki126-v1
+ARG DATASET_CACHE_BUST=2026-05-15-ki137-v1
 RUN echo "Dataset cache bust: ${DATASET_CACHE_BUST}" && python -c "\
 from huggingface_hub import snapshot_download; \
 snapshot_download(\

frontend/src/lib/useLiveConversation.ts

@@ -87,10 +87,12 @@ export type LiveConversationState = {
 };
 
 const DEFAULTS = {
-  // KI-113 (2026-05-15) — raised from 18 → 26 to reject ambient background
-  // noise (HVAC, traffic, distant chatter). Effective threshold is
-  // max(this, adaptive noise_floor * 2.5 + 6) — see KI-114 in the VAD loop.
-  rmsThreshold: 26,
+  // KI-113 raised from 18 → 26 to reject ambient noise.
+  // KI-139 (2026-05-15) — backed off to 18 because voice-forensics agent
+  // proved 26 sat ABOVE typical speech avg on consumer mics (especially
+  // built-ins with active noise gate that pin noiseFloor to 0). VAD never
+  // opened → green pill rendered → zero audio posted.
+  rmsThreshold: 18,
   // KI-113 — raised 3 → 5 (~80 ms sustained). Single clicks / cutlery /
   // typing transients no longer flip the gate. Preroll buffer (KI-044)
   // still captures the first phoneme via the 300 ms look-back.
@@ -119,12 +121,15 @@ const DEFAULTS = {
   // segment. Avoids bot's TTS attack transient bleeding through even
   // with echoCancellation on.
   postUtteranceCooldownMs: 700,
-  // KI-113 raised 0.35 → 0.50 to gate out broadband HVAC / fan / traffic.
-  // KI-134 (2026-05-15) — backed off to 0.35 because some laptop built-in
-  // mics with aggressive noiseSuppression flatten the voice-band proportion
-  // to 0.35-0.45, so the VAD never opens and the green pill renders but
-  // no audio is ever posted — matches the live-test symptom exactly.
-  voiceBandMinProp: 0.35,
+  // KI-113 raised to 0.50, KI-134 backed off to 0.35.
+  // KI-139 (2026-05-15) — voice-forensics agent measured live bundle on
+  // user's actual hardware: voiceProp sits at 0.25–0.33 on quiet laptop
+  // mics with NS enabled. 0.35 still gates the user out. 0.20 puts the
+  // floor well below voiced-speech minimum and only rejects pure tones
+  // (constant whine of HVAC, traffic). This is the deepest pushback
+  // — if HVAC noise creeps in, KI-140 will add a /api/transcribe round
+  // trip that detects empty responses and surfaces "couldn't hear you".
+  voiceBandMinProp: 0.20,
 };
 
 // AudioWorklet processor source — inlined as a Blob URL so we don't need
@@ -329,9 +334,14 @@ export function useLiveConversation(opts: LiveConversationOptions): LiveConversa
       // (which sits just above ambient because high frequencies attenuate
       // with distance) is rejected. Close-in speech still clears the gate
       // because the speaker's directly-radiated energy is ~10-20× ambient.
+      // KI-139 (2026-05-15) — noise-floor multiplier 2.5 → 1.8. On built-in
+      // mics with active noise gate, noiseFloor EMAs to 0 → effectiveThreshold
+      // pinned at max(rmsThreshold, 6). User's voice avg sits 18-22 then
+      // never crosses. 1.8 keeps headroom for HVAC (which pins at 5-7) but
+      // lets quiet voice through.
       const effectiveThreshold = Math.max(
         cfg.rmsThreshold,
-        noiseFloorRef.current * 2.5 + 6,
+        noiseFloorRef.current * 1.8 + 6,
       );
 
       // KI-057 — suppress new triggers right after we closed a segment
@@ -507,6 +517,17 @@ export function useLiveConversation(opts: LiveConversationOptions): LiveConversa
             return;
           }
         }
+        // KI-139 (2026-05-15) — Safari iOS resume() can return without
+        // throwing yet leave state at "suspended" — silent rejection of the
+        // autoplay-policy unlock. Treat anything other than "running" as a
+        // failure and surface to the user.
+        if (ctx.state !== "running") {
+          // eslint-disable-next-line no-console
+          console.error("[live-mode] AudioContext state stuck at", ctx.state, "— giving up");
+          setMicPermissionDenied(true);
+          setLive(false);
+          return;
+        }
         sampleRateRef.current = ctx.sampleRate;
 
         const source = ctx.createMediaStreamSource(stream);

tools/canonicalize_policy_names.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""KI-138: Canonicalize policy_name across extracted JSONs and Chroma metadata.
+
+Strategy:
+  - For each `policy_id_base` (policy_id stripped of trailing `__<doctype>`),
+    join extracted policy_name(s) against Chroma chunk policy_name(s).
+  - When they disagree, treat the Chroma-majority name as the canonical label
+    (Chroma names are short clean labels; extracted names often contain
+    parenthetical descriptions or doctype suffixes).
+  - Overwrite the extracted JSON `policy_name` and update every Chroma chunk's
+    `policy_name` metadata in place.
+
+Skip list (require human review — left untouched):
+  - bajaj-allianz__group-health-guard-gold (Silver/Gold conflict)
+  - reliance-general__hospi-care, __health-gain, __group-mediclaim (IndusInd co-brand)
+  - regulatory__irda-grievance-redressal-handbook (irda→irdai slug rename)
+  - regulatory__protection-of-policyholders-interests-2024 (slug rename)
+
+Backup of Chroma sqlite was taken before running:
+  rag/vectors/chroma.sqlite3.pre-ki138.bak
+
+Run:
+  /Users/rohitsar/.cache/uv-venvs/insurance-sales-bot/bin/python3 \
+      tools/canonicalize_policy_names.py
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import sys
+from collections import Counter, defaultdict
+from typing import Dict, List, Tuple
+
+import chromadb
+
+REPO = '/Users/rohitsar/Developer/Insurance Sales Bot'
+EXTRACTED_DIR = os.path.join(REPO, 'rag/_hf_dataset_backup/rag/extracted')
+CHROMA_PATH = os.path.join(REPO, 'rag/vectors/')
+MISMATCH_REPORT = '/tmp/policy_name_mismatches.json'
+
+SKIP_BASES = {
+    'bajaj-allianz__group-health-guard-gold',
+    'reliance-general__hospi-care',
+    'reliance-general__health-gain',
+    'reliance-general__group-mediclaim',
+    'regulatory__irda-grievance-redressal-handbook',
+    'regulatory__protection-of-policyholders-interests-2024',
+}
+
+
+def policy_id_base(pid: str) -> str:
+    parts = pid.split('__')
+    return '__'.join(parts[:-1]) if len(parts) >= 3 else pid
+
+
+def load_extracted() -> Dict[str, List[Tuple[str, str, str]]]:
+    """base -> [(path, policy_name, policy_id), ...]"""
+    out: Dict[str, List[Tuple[str, str, str]]] = defaultdict(list)
+    for path in glob.glob(os.path.join(EXTRACTED_DIR, '*.json')):
+        fname = os.path.basename(path)
+        if fname.startswith('_'):
+            continue
+        try:
+            with open(path, 'r', encoding='utf-8') as fh:
+                d = json.load(fh)
+        except Exception as exc:
+            print(f'WARN unreadable extracted JSON {path}: {exc}', file=sys.stderr)
+            continue
+        pid = d.get('policy_id', fname.replace('.json', ''))
+        pname = d.get('policy_name', '')
+        out[policy_id_base(pid)].append((path, pname, pid))
+    return out
+
+
+def load_chroma(coll) -> Dict[str, List[Tuple[str, str, str]]]:
+    """base -> [(chunk_id, policy_name, policy_id), ...]"""
+    got = coll.get(include=['metadatas'])
+    out: Dict[str, List[Tuple[str, str, str]]] = defaultdict(list)
+    for cid, md in zip(got['ids'], got['metadatas']):
+        pid = md.get('policy_id', '')
+        pname = md.get('policy_name', '')
+        out[policy_id_base(pid)].append((cid, pname, pid))
+    return out
+
+
+def build_mismatch_table(extracted, chroma) -> List[dict]:
+    rows: List[dict] = []
+    bases = set(extracted) | set(chroma)
+    for base in sorted(bases):
+        ext = extracted.get(base, [])
+        chr_ = chroma.get(base, [])
+        if not ext or not chr_:
+            continue
+        ext_names = [n for _, n, _ in ext]
+        chr_names = [n for _, n, _ in chr_]
+        if set(ext_names) == set(chr_names):
+            continue
+        chr_majority = Counter(chr_names).most_common(1)[0][0]
+        rows.append({
+            'policy_id_base': base,
+            'extracted_name': Counter(ext_names).most_common(1)[0][0],
+            'extracted_names_all': sorted(set(ext_names)),
+            'chroma_name_majority': chr_majority,
+            'chroma_names_all': sorted(set(chr_names)),
+            'proposed_canonical': chr_majority,
+            'extracted_files': [p for p, _, _ in ext],
+            'chroma_doc_ids': [c for c, _, _ in chr_],
+            'chroma_chunk_count': len(chr_),
+        })
+    return rows
+
+
+def main() -> int:
+    client = chromadb.PersistentClient(path=CHROMA_PATH)
+    coll = client.get_collection('policies')
+
+    extracted = load_extracted()
+    chroma = load_chroma(coll)
+
+    rows = build_mismatch_table(extracted, chroma)
+    with open(MISMATCH_REPORT, 'w', encoding='utf-8') as fh:
+        json.dump(rows, fh, indent=2, ensure_ascii=False)
+    print(f'mismatch table: {len(rows)} rows -> {MISMATCH_REPORT}')
+
+    actionable = [r for r in rows if r['policy_id_base'] not in SKIP_BASES]
+    skipped = [r for r in rows if r['policy_id_base'] in SKIP_BASES]
+    print(f'actionable: {len(actionable)}  skipped (human review): {len(skipped)}')
+
+    json_updates = 0
+    json_missing = 0
+    chroma_updates = 0
+    samples = []
+
+    for row in actionable:
+        canonical = row['proposed_canonical']
+
+        # 1) Update extracted JSON files
+        for fpath in row['extracted_files']:
+            if not os.path.exists(fpath):
+                print(f'WARN missing extracted JSON: {fpath}', file=sys.stderr)
+                json_missing += 1
+                continue
+            try:
+                with open(fpath, 'r', encoding='utf-8') as fh:
+                    d = json.load(fh)
+            except Exception as exc:
+                print(f'WARN unreadable {fpath}: {exc}', file=sys.stderr)
+                continue
+            before = d.get('policy_name', '')
+            if before != canonical:
+                d['policy_name'] = canonical
+                with open(fpath, 'w', encoding='utf-8') as fh:
+                    json.dump(d, fh, indent=2, ensure_ascii=False)
+                json_updates += 1
+
+        # 2) Update Chroma metadata for every chunk in this base
+        ids_to_update: List[str] = []
+        metas_to_update: List[dict] = []
+        existing = coll.get(ids=row['chroma_doc_ids'], include=['metadatas'])
+        for cid, md in zip(existing['ids'], existing['metadatas']):
+            if md.get('policy_name') == canonical:
+                continue
+            new_md = dict(md)
+            new_md['policy_name'] = canonical
+            ids_to_update.append(cid)
+            metas_to_update.append(new_md)
+
+        if ids_to_update:
+            # batch update for speed
+            BATCH = 500
+            for i in range(0, len(ids_to_update), BATCH):
+                coll.update(
+                    ids=ids_to_update[i:i + BATCH],
+                    metadatas=metas_to_update[i:i + BATCH],
+                )
+            chroma_updates += len(ids_to_update)
+
+        if len(samples) < 5:
+            samples.append({
+                'policy_id_base': row['policy_id_base'],
+                'before_extracted': row['extracted_name'],
+                'before_chroma_majority': row['chroma_name_majority'],
+                'after': canonical,
+            })
+
+    print(f'JSON files updated: {json_updates} (missing: {json_missing})')
+    print(f'Chroma chunks updated: {chroma_updates}')
+
+    # 3) Verify by re-joining
+    extracted2 = load_extracted()
+    chroma2 = load_chroma(coll)
+    remaining = build_mismatch_table(extracted2, chroma2)
+    remaining_bases = [r['policy_id_base'] for r in remaining]
+    print(f'mismatches_remaining: {len(remaining)}')
+    for r in remaining:
+        marker = 'SKIPPED' if r['policy_id_base'] in SKIP_BASES else 'UNEXPECTED'
+        print(f"  [{marker}] {r['policy_id_base']}: ext={r['extracted_name']!r} chr={r['chroma_name_majority']!r}")
+
+    print('\nsample before/after:')
+    for s in samples:
+        print(f"  {s['policy_id_base']}")
+        print(f"    extracted-before: {s['before_extracted']}")
+        print(f"    chroma-before:    {s['before_chroma_majority']}")
+        print(f"    after (canonical):{s['after']}")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/ingest_curated_into_chroma.py

@@ -0,0 +1,318 @@
+"""Ingest curated-fact JSONs into the Chroma `policies` collection.
+
+Problem: ~21 policies appear in the marketplace (/api/policies/all) but have
+ZERO Chroma chunks, because their canonical PDFs are image-only / not in the
+corpus / or the curated JSON is the only data source we have. The chat bot
+cannot retrieve or cite these policies — RAG returns no hits.
+
+Fix: render each curated policy_facts JSON into a flat text representation
+that mimics what a wordings PDF chunk would look like, embed it with the
+SAME LocalEmbeddings (BGE-small, 384-dim) used by rag/ingest.py, and add it
+to the SAME `policies` collection with metadata.doc_type='curated' so the
+retriever can find these the same way it finds PDF chunks.
+
+Run:
+  /Users/rohitsar/.cache/uv-venvs/insurance-sales-bot/bin/python3 \
+    tools/ingest_curated_into_chroma.py
+
+Idempotent — re-running skips any policy_id already present in Chroma.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+import sys
+from pathlib import Path
+
+import chromadb
+from chromadb.config import Settings as ChromaSettings
+
+# Make backend importable
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from backend.config import settings  # noqa: E402
+from backend.main import _load_curated_facts  # noqa: E402
+from backend.providers.local_embeddings import LocalEmbeddings  # noqa: E402
+from rag.ingest import (  # noqa: E402
+    CHUNK_CHARS,
+    OVERLAP_CHARS,
+    chunk_pages,
+    get_chroma_collection,
+)
+
+CURATED_DIR = ROOT / "40-data" / "policy_facts"
+CORPUS_URLS_MD = ROOT / "40-data" / "corpus_urls.md"
+
+
+# ---------- corpus_urls.md → policy_id → source_url ----------
+
+def _load_corpus_urls() -> dict[str, str]:
+    """Parse 40-data/corpus_urls.md table rows into {insurer_slug+policy_name_slug: url}.
+
+    Used as a best-effort source_url backfill for curated policies that have
+    no PDF in rag/corpus/. The slug match is fuzzy — we slugify the
+    policy_name and check if it overlaps with the curated policy_id stem.
+    """
+    if not CORPUS_URLS_MD.exists():
+        return {}
+    text = CORPUS_URLS_MD.read_text()
+    rows: dict[str, str] = {}
+    for line in text.splitlines():
+        if not line.startswith("| ") or "---" in line or "insurer_slug" in line:
+            continue
+        cells = [c.strip() for c in line.strip("|").split("|")]
+        if len(cells) < 5:
+            continue
+        insurer_slug, _insurer_name, policy_name, _doc_type, url = cells[:5]
+        if not url.startswith("http"):
+            continue
+        name_slug = re.sub(r"[^a-z0-9]+", "-", policy_name.lower()).strip("-")
+        rows.setdefault(f"{insurer_slug}__{name_slug}", url)
+        # Also index a shorter token-overlap key for fuzzier matching
+        rows.setdefault(name_slug, url)
+    return rows
+
+
+def _best_source_url(policy_id: str, curated_data: dict, url_index: dict[str, str]) -> str:
+    """Pick a source URL for a curated policy. Preference order:
+    1. Any non-null `source_url` field in the curated JSON
+    2. corpus_urls.md row whose insurer_slug__name_slug matches policy_id
+    3. Empty string (downstream code accepts "" gracefully)
+    """
+    # 1. Look inside the curated JSON for any source_url
+    for k, v in curated_data.items():
+        if isinstance(v, dict) and v.get("source_url"):
+            return v["source_url"]
+    # 2. corpus_urls.md fuzzy match
+    if policy_id in url_index:
+        return url_index[policy_id]
+    # Try the stem (everything after `insurer__`)
+    parts = policy_id.split("__", 1)
+    if len(parts) == 2:
+        stem = parts[1]
+        if stem in url_index:
+            return url_index[stem]
+        # Loose match: check any url-index key that contains the stem
+        for key, url in url_index.items():
+            if stem in key or key in stem:
+                return url
+    return ""
+
+
+# ---------- curated JSON → flat text representation ----------
+
+_HUMAN_LABEL = {
+    "uin_code": "UIN Code",
+    "min_entry_age": "Minimum Entry Age",
+    "max_entry_age": "Maximum Entry Age",
+    "max_renewal_age": "Maximum Renewal Age",
+    "sum_insured_options": "Sum Insured Options",
+    "initial_waiting_period_days": "Initial Waiting Period (days)",
+    "pre_existing_disease_waiting_months": "Pre-Existing Disease Waiting Period (months)",
+    "specific_disease_waiting_months": "Specific Disease Waiting Period (months)",
+    "maternity_waiting_months": "Maternity Waiting Period (months)",
+    "pre_hospitalization_days": "Pre-Hospitalization Coverage (days)",
+    "post_hospitalization_days": "Post-Hospitalization Coverage (days)",
+    "day_care_treatments_count": "Number of Day-Care Treatments Covered",
+    "ayush_coverage": "AYUSH Coverage",
+    "maternity_coverage": "Maternity Coverage",
+    "newborn_coverage": "Newborn Baby Coverage",
+    "organ_donor_expenses": "Organ Donor Expenses",
+    "no_claim_bonus_pct": "No Claim Bonus (%)",
+    "restoration_benefit": "Restoration / Reload Benefit",
+    "room_rent_capping": "Room Rent Capping",
+    "copayment_pct": "Co-Payment (%)",
+    "deductible_amount": "Deductible Amount",
+    "network_hospital_count": "Network Hospital Count",
+    "cashless_treatment_supported": "Cashless Treatment Supported",
+    "claim_settlement_ratio": "Claim Settlement Ratio (%)",
+    "tat_cashless_authorization_hours": "Cashless Authorization Turnaround Time (hours)",
+    "policy_type": "Policy Type",
+}
+
+
+def render_curated_to_text(curated: dict) -> str:
+    """Render a curated_facts dict into a flat text document the embedder
+    can ingest. Includes both value and source_quote so the LLM sees the
+    full evidence the curator used.
+    """
+    policy_name = curated.get("policy_name") or curated.get("policy_id", "")
+    insurer_slug = curated.get("insurer_slug", "")
+    out: list[str] = []
+    out.append(f"Policy Name: {policy_name}")
+    out.append(f"Insurer: {insurer_slug}")
+    out.append(f"Policy ID: {curated.get('policy_id', '')}")
+    out.append("")
+    out.append("Structured Policy Facts (curated from official wordings/CIS/brochure):")
+    out.append("")
+    for key, val in curated.items():
+        if key.startswith("_") or key in ("policy_id", "policy_name", "insurer_slug"):
+            continue
+        label = _HUMAN_LABEL.get(key, key.replace("_", " ").title())
+        if isinstance(val, dict):
+            value = val.get("value")
+            quote = val.get("source_quote") or ""
+            unit = val.get("unit", "")
+            if value is None or value == "":
+                # Skip null fields but keep the source_quote if it gives context
+                if quote:
+                    out.append(f"- {label}: not specified. Source note: {quote}")
+                continue
+            display = f"{value} {unit}".strip() if unit else str(value)
+            line = f"- {label}: {display}."
+            if quote:
+                line += f" Source quote: \"{quote}\""
+            out.append(line)
+        else:
+            if val is None or val == "" or val == []:
+                continue
+            out.append(f"- {label}: {val}")
+
+    # Meta block — curation context + primary source PDF
+    meta = curated.get("_meta") or {}
+    if meta:
+        out.append("")
+        out.append("Curation metadata:")
+        if meta.get("curated_at"):
+            out.append(f"- Curated on: {meta['curated_at']}")
+        if meta.get("primary_source_pdf"):
+            out.append(f"- Primary source PDF: {meta['primary_source_pdf']}")
+        if meta.get("completeness_pct") is not None:
+            out.append(f"- Curation completeness: {meta['completeness_pct']}%")
+        if meta.get("notes"):
+            out.append(f"- Notes: {meta['notes']}")
+
+    return "\n".join(out)
+
+
+# ---------- main pipeline ----------
+
+async def main():
+    # 1. Identify missing curated policies
+    curated_all = _load_curated_facts()
+    # _load_curated_facts adds __wordings/__brochure/__cis duplicate keys;
+    # collapse to the actual policy_id from inside each entry.
+    base_curated: dict[str, dict] = {}
+    for _key, data in curated_all.items():
+        real_pid = data.get("policy_id")
+        if real_pid and real_pid not in base_curated:
+            base_curated[real_pid] = data
+    print(f"Distinct curated policy_ids: {len(base_curated)}")
+
+    coll = get_chroma_collection()
+    print(f"Chroma `policies` chunks before ingest: {coll.count()}")
+
+    existing = coll.get(include=["metadatas"])
+    chroma_pids = set(md.get("policy_id") for md in existing["metadatas"])
+    print(f"Chroma unique policy_ids before ingest: {len(chroma_pids)}")
+
+    # Missing = curated pid AND no chroma pid that exactly matches or matches
+    # `<curated_pid>__<docvariant>` (Chroma stores e.g. `...__wordings`).
+    missing: list[str] = []
+    for pid in sorted(base_curated):
+        if pid in chroma_pids:
+            continue
+        if any(c.startswith(pid + "__") for c in chroma_pids):
+            continue
+        missing.append(pid)
+    print(f"\nMissing curated policies: {len(missing)}")
+    for m in missing:
+        print(f"  - {m}")
+
+    if not missing:
+        print("\nNothing to ingest. Exiting.")
+        return
+
+    # 2. Render + embed + add
+    url_index = _load_corpus_urls()
+    embedder = LocalEmbeddings()
+    print(f"\nEmbedder: {embedder.name} ({embedder.model_name}, dim={embedder.dimension}, device={embedder.device})")
+
+    total_chunks = 0
+    per_policy: list[tuple[str, int]] = []
+
+    for pid in missing:
+        data = base_curated[pid]
+        text = render_curated_to_text(data)
+        n_chars = len(text)
+
+        # Single chunk if under target, else slide window via rag.ingest.chunk_pages
+        # (chunk_pages expects [(page_no, text)]; we feed one virtual page).
+        if n_chars <= CHUNK_CHARS:
+            chunks = [{
+                "chunk_idx": 0,
+                "text": text,
+                "page_start": 0,
+                "page_end": 0,
+                "char_start": 0,
+                "char_end": n_chars,
+            }]
+        else:
+            chunks = list(chunk_pages(
+                pages=[(0, text)],
+                target_chars=CHUNK_CHARS,
+                overlap_chars=OVERLAP_CHARS,
+            ))
+
+        if not chunks:
+            print(f"  WARN empty render for {pid}, skipping")
+            continue
+
+        texts = [c["text"] for c in chunks]
+        vectors = await embedder.embed(texts, input_type="document")
+
+        insurer_slug = data.get("insurer_slug") or pid.split("__", 1)[0]
+        policy_name = data.get("policy_name") or pid
+        source_url = _best_source_url(pid, data, url_index)
+
+        ids = [f"{pid}::curated::chunk{c['chunk_idx']}" for c in chunks]
+        metadatas = [
+            {
+                "policy_id": pid,
+                "insurer_slug": insurer_slug,
+                "policy_name": policy_name,
+                "doc_type": "curated",
+                "source_url": source_url,
+                "page_start": 0,
+                "page_end": 0,
+                "chunk_idx": c["chunk_idx"],
+                "local_path": f"curated-facts:{pid}",
+            }
+            for c in chunks
+        ]
+
+        coll.add(ids=ids, documents=texts, embeddings=vectors, metadatas=metadatas)
+
+        per_policy.append((pid, len(chunks)))
+        total_chunks += len(chunks)
+        print(f"  + {pid}: {len(chunks)} chunk(s), {n_chars} chars, url={'yes' if source_url else 'no'}")
+
+    # 3. Verify by re-querying
+    print("\n--- Verification ---")
+    final_count = coll.count()
+    print(f"Chroma `policies` chunks after ingest: {final_count}")
+    for pid, n in per_policy:
+        got = coll.get(where={"policy_id": pid})
+        print(f"  verify {pid}: chunks_now={len(got.get('ids') or [])}")
+
+    # 4. Sample retrieval test — embed a query and pull top-5
+    print("\n--- Sample retrieval test ---")
+    sample_query = "What are the waiting periods and AYUSH coverage for Aditya Birla Activ One?"
+    qv = await embedder.embed([sample_query], input_type="query")
+    res = coll.query(query_embeddings=qv, n_results=5)
+    for i, (doc_id, md, dist) in enumerate(zip(res["ids"][0], res["metadatas"][0], res["distances"][0])):
+        snippet = (res["documents"][0][i] or "")[:120].replace("\n", " ")
+        print(f"  rank {i+1}: {md.get('policy_id')} (dist={dist:.4f}) doc_type={md.get('doc_type')}")
+        print(f"           {snippet}...")
+
+    # 5. Summary
+    print("\n--- Summary ---")
+    print(f"Policies ingested: {len(per_policy)}")
+    print(f"Total chunks added: {total_chunks}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tools/upload_extracted_to_dataset.py

@@ -20,7 +20,7 @@ def main():
         path_in_repo="rag/extracted",
         repo_id="rohitsar567/insurance-bot-data",
         repo_type="dataset",
-        commit_message="sync rag/extracted JSONs (post-NIM-extraction)",
+        commit_message="feat(extracted): KI-138 — canonicalize policy_name across 84 mismatches",
         ignore_patterns=["*._raw.txt"],
     )
     n = len(list((ROOT / "rag" / "extracted").glob("*.json")))