Deploy v1 — single-Docker FastAPI + Next.js + RAG + voice + faithfulness

rag/ingest.py

@@ -185,6 +185,10 @@ async def ingest_one(
     policy_name = manifest_entry.get("policy_name", pdf_path.stem)
     doc_type = manifest_entry.get("doc_type", "unknown")
     source_url = manifest_entry.get("url", "")
+    # PDFs under rag/corpus/regulatory/ are IRDAI / Govt mandates. Tag them
+    # so retrieve.py can apply the regulatory-intent boost.
+    if pdf_path.parent.name == "regulatory" or insurer_slug == "regulatory":
+        doc_type = "regulatory"
 
     # Skip if already ingested
     existing = collection.get(where={"policy_id": policy_id}, limit=1)

rag/retrieve.py

@@ -47,6 +47,43 @@ def get_collection():
     )
 
 
+import re as _re
+
+# Queries containing these keywords trigger a parallel regulatory-only
+# retrieval whose top results are boosted ×1.2 and merged into the final
+# context. This ensures the brain sees IRDAI mandates whenever the user
+# asks about compliance, legality, or what's "allowed" — even when the
+# policy chunks would otherwise dominate raw cosine.
+_REGULATORY_TRIGGERS = _re.compile(
+    r"\b(irdai|irda|regulation|regulator|regulatory|mandate|mandatory|"
+    r"allowed|prohibited|legal|illegal|unenforceable|cap|capped|ceiling|"
+    r"master circular|section\s+\d+|compliance|non[- ]?compliant|"
+    r"required|must|rule|statute|act|government)\b",
+    flags=_re.IGNORECASE,
+)
+REGULATORY_BOOST = 1.2  # multiplier applied to regulatory chunk scores
+
+
+def _is_regulatory_intent(query: str) -> bool:
+    return bool(_REGULATORY_TRIGGERS.search(query or ""))
+
+
+def _build_chunk(cid: str, doc: str, meta: dict, score: float) -> RetrievedChunk:
+    return RetrievedChunk(
+        chunk_id=cid,
+        text=doc,
+        policy_id=meta.get("policy_id", ""),
+        insurer_slug=meta.get("insurer_slug", ""),
+        policy_name=meta.get("policy_name", ""),
+        doc_type=meta.get("doc_type", ""),
+        source_url=meta.get("source_url", ""),
+        page_start=int(meta.get("page_start", 0)),
+        page_end=int(meta.get("page_end", 0)),
+        chunk_idx=int(meta.get("chunk_idx", 0)),
+        score=score,
+    )
+
+
 async def retrieve(
     query: str,
     top_k: int = settings.RAG_TOP_K,
@@ -58,6 +95,12 @@ async def retrieve(
 
     Optional filters narrow retrieval to specific policies/insurers (used
     by comparison and per-policy Q&A flows).
+
+    For queries with regulatory intent (IRDAI / mandate / allowed / etc.),
+    runs a SECOND retrieval restricted to doc_type='regulatory' chunks and
+    merges the top 3 of those (score-boosted ×1.2) into the result set.
+    This ensures the brain sees regulatory ceilings even when policy
+    chunks dominate raw cosine.
     """
     embedder = embedder or VoyageEmbeddings()
     [query_vec] = await embedder.embed([query], input_type="query")
@@ -69,6 +112,8 @@ async def retrieve(
         where["insurer_slug"] = {"$in": insurer_slugs}
 
     collection = get_collection()
+
+    # Standard retrieval
     res = collection.query(
         query_embeddings=[query_vec],
         n_results=top_k,
@@ -76,32 +121,41 @@ async def retrieve(
     )
 
     out: list[RetrievedChunk] = []
-    if not res["ids"] or not res["ids"][0]:
-        return out
-
-    for cid, doc, meta, dist in zip(
-        res["ids"][0],
-        res["documents"][0],
-        res["metadatas"][0],
-        res["distances"][0],
-    ):
-        # Chroma returns cosine *distance*; convert to similarity score
-        score = 1.0 - dist
-        out.append(
-            RetrievedChunk(
-                chunk_id=cid,
-                text=doc,
-                policy_id=meta.get("policy_id", ""),
-                insurer_slug=meta.get("insurer_slug", ""),
-                policy_name=meta.get("policy_name", ""),
-                doc_type=meta.get("doc_type", ""),
-                source_url=meta.get("source_url", ""),
-                page_start=int(meta.get("page_start", 0)),
-                page_end=int(meta.get("page_end", 0)),
-                chunk_idx=int(meta.get("chunk_idx", 0)),
-                score=score,
+    if res["ids"] and res["ids"][0]:
+        for cid, doc, meta, dist in zip(
+            res["ids"][0], res["documents"][0],
+            res["metadatas"][0], res["distances"][0],
+        ):
+            out.append(_build_chunk(cid, doc, meta, 1.0 - dist))
+
+    # Regulatory boost pass — only when the query is about IRDAI / regulations,
+    # and only when not already filtered to specific policies (otherwise the
+    # caller is asking about a specific policy, not regulations).
+    if _is_regulatory_intent(query) and not policy_ids and not insurer_slugs:
+        try:
+            reg_res = collection.query(
+                query_embeddings=[query_vec],
+                n_results=3,
+                where={"doc_type": "regulatory"},
             )
-        )
+            if reg_res["ids"] and reg_res["ids"][0]:
+                seen = {c.chunk_id for c in out}
+                reg_chunks: list[RetrievedChunk] = []
+                for cid, doc, meta, dist in zip(
+                    reg_res["ids"][0], reg_res["documents"][0],
+                    reg_res["metadatas"][0], reg_res["distances"][0],
+                ):
+                    if cid in seen:
+                        continue
+                    boosted = (1.0 - dist) * REGULATORY_BOOST
+                    reg_chunks.append(_build_chunk(cid, doc, meta, boosted))
+                # Merge and re-sort by score, then trim back to top_k
+                merged = sorted(out + reg_chunks, key=lambda c: c.score, reverse=True)
+                out = merged[:top_k]
+        except Exception:
+            # Regulatory boost is additive; failure shouldn't kill the main result
+            pass
+
     return out
 
 

tools/extract_policy_text_batch2.py

@@ -0,0 +1,76 @@
+"""Extract text from policy PDFs (batch 2) into the same text cache."""
+import os, sys, json
+import pdfplumber
+
+BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CACHE = "/tmp/claude/policy_extract/text_cache"
+os.makedirs(CACHE, exist_ok=True)
+
+# Batch 2: PDFs not yet covered, filtered by retail/standalone relevance
+BATCH2 = [
+    ("aditya-birla", "rag/corpus/aditya-birla/activ-health-individual__wordings.pdf"),
+    ("bajaj-allianz", "rag/corpus/bajaj-allianz/comprehensive-care-plan__wordings.pdf"),
+    ("bajaj-allianz", "rag/corpus/bajaj-allianz/global-health-care__wordings.pdf"),
+    ("bajaj-allianz", "rag/corpus/bajaj-allianz/health-guard__wordings.pdf"),
+    ("bajaj-allianz", "rag/corpus/bajaj-allianz/silver-health__cis.pdf"),
+    ("bajaj-allianz", "rag/corpus/bajaj-allianz/tax-gain__cis.pdf"),
+    ("care-health", "rag/corpus/care-health/care-advantage__brochure.pdf"),
+    ("care-health", "rag/corpus/care-health/care-supreme-enhance__wordings.pdf"),
+    ("care-health", "rag/corpus/care-health/ultimate-care__wordings.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/energy-diabetes-hypertension__wordings.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/my-health-medisure-prime__wordings.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/my-health-sampoorna-suraksha__brochure.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/my-health-suraksha__brochure.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/my-health-women-suraksha__brochure.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/my-optima-secure-older-variant__wordings.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/optima-enhance__wordings.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/optima-plus__wordings.pdf"),
+    ("hdfc-ergo", "rag/corpus/hdfc-ergo/total-health-plan__wordings.pdf"),
+    ("icici-lombard", "rag/corpus/icici-lombard/arogya-sanjeevani__wordings.pdf"),
+    ("icici-lombard", "rag/corpus/icici-lombard/complete-health-insurance-umbrella__wordings.pdf"),
+    ("icici-lombard", "rag/corpus/icici-lombard/health-advantedge__wordings.pdf"),
+    ("icici-lombard", "rag/corpus/icici-lombard/health-booster-top-up__wordings.pdf"),
+    ("icici-lombard", "rag/corpus/icici-lombard/health-elite-plus__wordings.pdf"),
+    ("manipalcigna", "rag/corpus/manipalcigna/prohealth-select__wordings.pdf"),
+    ("manipalcigna", "rag/corpus/manipalcigna/sarvah-param__wordings.pdf"),
+    ("new-india", "rag/corpus/new-india/asha-kiran-policy__brochure.pdf"),
+    ("new-india", "rag/corpus/new-india/janata-mediclaim-policy__wordings.pdf"),
+    ("new-india", "rag/corpus/new-india/new-india-mediclaim-policy__wordings.pdf"),
+    ("new-india", "rag/corpus/new-india/universal-health-insurance__wordings.pdf"),
+    ("new-india", "rag/corpus/new-india/yuva-bharat-health-policy__wordings.pdf"),
+    ("niva-bupa", "rag/corpus/niva-bupa/aspire__wordings.pdf"),
+    ("niva-bupa", "rag/corpus/niva-bupa/health-plus-top-up__wordings.pdf"),
+    ("niva-bupa", "rag/corpus/niva-bupa/health-premia__wordings.pdf"),
+    ("niva-bupa", "rag/corpus/niva-bupa/reassure-3-0__wordings.pdf"),
+    ("niva-bupa", "rag/corpus/niva-bupa/rise__wordings.pdf"),
+    ("niva-bupa", "rag/corpus/niva-bupa/saral-suraksha-bima__wordings.pdf"),
+    ("star-health", "rag/corpus/star-health/health-premier__wordings.pdf"),
+    ("star-health", "rag/corpus/star-health/senior-citizens-red-carpet__brochure.pdf"),
+    ("star-health", "rag/corpus/star-health/star-assure__wordings.pdf"),
+    ("star-health", "rag/corpus/star-health/star-cardiac-care-platinum__wordings.pdf"),
+    ("star-health", "rag/corpus/star-health/star-cardiac-care__wordings.pdf"),
+    ("tata-aig", "rag/corpus/tata-aig/medicare-lite__cis.pdf"),
+    ("tata-aig", "rag/corpus/tata-aig/medicare-select__brochure.pdf"),
+]
+
+for insurer, rel in BATCH2:
+    src = os.path.join(BASE, rel)
+    name = os.path.basename(rel).replace(".pdf", ".txt")
+    dst = os.path.join(CACHE, f"{insurer}__{name}")
+    if os.path.exists(dst):
+        print(f"skip {os.path.basename(dst)}")
+        continue
+    if not os.path.exists(src):
+        print(f"MISSING {src}")
+        continue
+    try:
+        with pdfplumber.open(src) as pdf:
+            # Extract up to 30 pages worth (most policy details in first ~20)
+            pages = pdf.pages[:30]
+            text = "\n".join((p.extract_text() or "") for p in pages)
+        with open(dst, "w", encoding="utf-8") as f:
+            f.write(text)
+        print(f"OK {os.path.basename(dst)} ({len(text)} chars, {len(pages)} pages)")
+    except Exception as e:
+        print(f"ERR {src}: {e}")
+print("Done.")