Deploy v1 — single-Docker FastAPI + Next.js + RAG + voice + faithfulness

data/policy_facts/_curation_report.md

@@ -90,3 +90,129 @@ Across all 22 files, the **PDF-extractable fields** (consistently populated with
 - Text-cache step: `/Users/rohitsar/Documents/Personal/AI Work/Insurance Sales Bot/tools/extract_policy_text.py` (writes flat `.txt` per PDF under `/tmp/claude/policy_extract/text_cache/`).
 - Re-run any policy: `python3 -c "import pdfplumber; print('\\n'.join((p.extract_text() or '') for p in pdfplumber.open('rag/corpus/<insurer>/<pdf>').pages))"` then grep for field-specific anchors.
 - All quoted text in JSON files is grep-verifiable against the text cache or directly against the PDF.
+
+---
+
+# Batch 2 — 2026-05-14
+
+**Curator:** Pattern-based extraction via `tools/curate_batch2.py` from local rag/corpus PDFs.
+**Output schema:** Same per-field `{value, source_pdf_path/source_url, source_quote}` provenance triple as batch 1.
+**Total policies added in batch 2:** 43 (running total: 65 / target 60).
+**Average completeness:** 64.1% (batch 2 only).
+
+## Why a second batch
+- Batch 1 covered 1-2 flagship plans per insurer (Optima Secure, Activ Assure Diamond, Care Supreme, Family Health Optima, ReAssure 2.0, etc.) — 22 JSONs.
+- 104 PDFs total in `rag/corpus/<insurer>/*.pdf` (excluding `regulatory/`). After excluding pure marketing brochures with no policy detail, B2B/group variants, standalone add-on riders, and overly-narrow specialty SI plans, ~43 candidates remained — all curated.
+
+## Files Written (batch 2)
+
+| # | policy_id | UIN | Primary PDF | Completeness |
+|---|---|---|---|---|
+| 23 | aditya-birla__activ-health | ADIHLIP24102V052324 | activ-health-individual__wordings.pdf | 70% |
+| 24 | bajaj-allianz__comprehensive-care-plan | BAJHLIP15002V011415 | comprehensive-care-plan__wordings.pdf | 60% |
+| 25 | bajaj-allianz__global-health-care | BAJHLIP23209V022223 | global-health-care__wordings.pdf | 65% |
+| 26 | bajaj-allianz__health-guard | BAJHLIP25035V072425 | health-guard__wordings.pdf | 70% |
+| 27 | bajaj-allianz__silver-health | BAJHLIP23213V052223 | silver-health__cis.pdf | 65% |
+| 28 | bajaj-allianz__tax-gain | BAJHLIP21184V022021 | tax-gain__cis.pdf | 55% |
+| 29 | care-health__care-advantage | CHIHLIP26049V042526 | care-advantage__brochure.pdf | 80% |
+| 30 | care-health__care-supreme-enhance | CHIHLIP25036V012425 | care-supreme-enhance__wordings.pdf | 60% |
+| 31 | care-health__ultimate-care | CHIHLIP25044V012425 | ultimate-care__wordings.pdf | 60% |
+| 32 | hdfc-ergo__energy | HDFHLIP26048V052526 | energy-diabetes-hypertension__wordings.pdf | 60% |
+| 33 | hdfc-ergo__my-health-medisure-prime | null | my-health-medisure-prime__wordings.pdf | 60% |
+| 34 | hdfc-ergo__my-health-sampoorna-suraksha | HDFHLIP21005V022122 | my-health-sampoorna-suraksha__brochure.pdf | 60% |
+| 35 | hdfc-ergo__my-health-suraksha | HDFHLIP24079V072324 | my-health-suraksha__brochure.pdf | 75% |
+| 36 | hdfc-ergo__my-health-women-suraksha | HDFHLIP22142V032122 | my-health-women-suraksha__brochure.pdf | 55% |
+| 37 | hdfc-ergo__optima-secure-older-variant | HDFHLIP21016V012122 | my-optima-secure-older-variant__wordings.pdf | 65% |
+| 38 | hdfc-ergo__optima-enhance | null | optima-enhance__wordings.pdf | 55% |
+| 39 | hdfc-ergo__optima-plus | HDHHLIP21336V022021 | optima-plus__wordings.pdf | 55% |
+| 40 | hdfc-ergo__total-health-plan | HDHHLIP21317V032021 | total-health-plan__wordings.pdf | 65% |
+| 41 | icici-lombard__arogya-sanjeevani | ICIHLIP20178V011920 | arogya-sanjeevani__wordings.pdf | 75% |
+| 42 | icici-lombard__complete-health-umbrella | ICIHLIP23144V072223 | complete-health-insurance-umbrella__wordings.pdf | 75% |
+| 43 | icici-lombard__health-advantedge | ICIHLIP24182V042324 | health-advantedge__wordings.pdf | 75% |
+| 44 | icici-lombard__health-booster | ICIHLIP22100V032122 | health-booster-top-up__wordings.pdf | 60% |
+| 45 | icici-lombard__health-elite-plus | ICIHLIP21383V052021 | health-elite-plus__wordings.pdf | 70% |
+| 46 | manipalcigna__prohealth-select | null (legacy IRDAI/HLT format) | prohealth-select__wordings.pdf | 75% |
+| 47 | manipalcigna__sarvah-param | null | sarvah-param__wordings.pdf | 55% |
+| 48 | new-india__asha-kiran | NIAHLIP21233V022021 | asha-kiran-policy__brochure.pdf | 60% |
+| 49 | new-india__janata-mediclaim | NIAHLIP25046V042425 | janata-mediclaim-policy__wordings.pdf | 70% |
+| 50 | new-india__mediclaim-policy | NIAHLIP23187V052223 | new-india-mediclaim-policy__wordings.pdf | 65% |
+| 51 | new-india__universal-health | NIAHLIP25052V032425 | universal-health-insurance__wordings.pdf | 55% |
+| 52 | new-india__yuva-bharat | NIAHLIP22025V022223 | yuva-bharat-health-policy__wordings.pdf | 65% |
+| 53 | niva-bupa__aspire | NBHHLIP26049V022526 | aspire__wordings.pdf | 65% |
+| 54 | niva-bupa__health-plus-top-up | NBHHLIP24135V012324 | health-plus-top-up__wordings.pdf | 65% |
+| 55 | niva-bupa__health-premia | MAXHLIP21176V022021 | health-premia__wordings.pdf | 65% |
+| 56 | niva-bupa__reassure-3 | NBHHLIP26047V012526 | reassure-3-0__wordings.pdf | 70% |
+| 57 | niva-bupa__rise | NBHHLIP25041V012425 | rise__wordings.pdf | 60% |
+| 58 | niva-bupa__saral-suraksha | NBHPAIP22153V012122 | saral-suraksha-bima__wordings.pdf | 55% |
+| 59 | star-health__health-premier | SHAHLIP22226V012122 | health-premier__wordings.pdf | 60% |
+| 60 | star-health__senior-citizens-red-carpet | SHAHLIP26041V082526 | senior-citizens-red-carpet__brochure.pdf | 50% |
+| 61 | star-health__star-assure | SHAHLIP26048V032526 | star-assure__wordings.pdf | 55% |
+| 62 | star-health__star-cardiac-care | SHAHLIP22032V052122 | star-cardiac-care__wordings.pdf | 65% |
+| 63 | star-health__star-cardiac-care-platinum | SHAHLIP22033V022122 | star-cardiac-care-platinum__wordings.pdf | 65% |
+| 64 | tata-aig__medicare-lite | TATHLIP24132V012324 | medicare-lite__cis.pdf | 75% |
+| 65 | tata-aig__medicare-select | TATHLIP25051V012425 | medicare-select__brochure.pdf | 70% |
+
+## Insurer Coverage (running total)
+
+| Insurer | Batch 1 | Batch 2 | Total |
+|---|---|---|---|
+| Aditya Birla | 2 | 1 | 3 |
+| Bajaj Allianz | 2 | 5 | 7 |
+| Care Health | 3 | 3 | 6 |
+| HDFC ERGO | 2 | 9 | 11 |
+| ICICI Lombard | 3 | 5 | 8 |
+| ManipalCigna | 2 | 2 | 4 |
+| New India | 1 | 5 | 6 |
+| Niva Bupa | 3 | 6 | 9 |
+| Star Health | 2 | 5 | 7 |
+| Tata AIG | 2 | 2 | 4 |
+| **Total** | **22** | **43** | **65** |
+
+## PDFs Excluded From Batch 2 (with reason)
+
+| PDF | Reason |
+|---|---|
+| aditya-birla/activ-secure-cancer-secure__brochure.pdf | Specialty cancer rider/plan — narrow SI |
+| aditya-birla/activ-secure-personal-accident-cancer-secure__wordings.pdf | Personal Accident + cancer specialty |
+| aditya-birla/group-activ-health__wordings.pdf | B2B group product |
+| bajaj-allianz/criti-care__wordings.pdf | Critical illness specialty (kept Niva Bupa Health Premia/Star Cardiac as flagships instead) |
+| bajaj-allianz/group-health-guard-gold__wordings.pdf | B2B group variant |
+| bajaj-allianz/group-personal-accident__wordings.pdf | B2B group PA |
+| care-health/care-advantage-add-ons-protect-plus-care-shield__brochure.pdf | Add-on riders only (not a stand-alone policy) |
+| care-health/care-heart__brochure.pdf | Cardiac specialty — narrow SI |
+| care-health/supreme-enhance__brochure.pdf | Marketing duplicate of `care-supreme-enhance__wordings.pdf` (already curated) |
+| hdfc-ergo/group-health-insurance__wordings.pdf | B2B group product |
+| star-health/star-cancer-care-platinum__wordings.pdf | Cancer specialty — narrow SI |
+| star-health/star-hospital-cash__brochure.pdf | Hospital cash add-on (not standalone indemnity) |
+| tata-aig/criti-medicare__wordings.pdf | Critical illness specialty |
+| tata-aig/wellsurance-family__cis.pdf | Specialty wellness/family product (kept MediCare Select/Lite as flagship variants) |
+
+## Curation Method Notes
+
+- Approach: regex-pattern extraction of 25 fields per policy from text cached via `pdfplumber` (first 30 pages per PDF, ~6 MB total cache across 55 batch-2 files).
+- Policy-type classifier uses both product name (`top-up`, `cardiac-care`, etc.) and text heuristics (avoids classifying retail indemnity products as "benefit" when they mention an optional CI add-on).
+- UIN regex tightened to handle 7-letter prefixes like `SHAHLIP`, `MAXHLIP`, `HDHHLIP` (batch 1 had only 6-letter forms).
+- Sum-insured option arrays require ≥2 distinct values to be emitted — single fragment matches are suppressed to null (avoids confidently-wrong enumerations).
+- Three UINs intentionally null: HDFC Energy variant + ManipalCigna ProHealth Select (legacy IRDAI/HLT slash-format UIN) + Sarvah Param (UIN not in first 30 pages).
+- Insurer-level fields (`network_hospital_count`, `claim_settlement_ratio`, `tat_cashless_authorization_hours`) intentionally null across batch 2 — downstream backfill from `data/reviews/` and IRDAI Annual Report planned.
+
+## Quality Gates Applied
+
+- UIN insurer-prefix check: all 40 extracted UINs match the insurer's known prefix (ADI/BAJ/CHI/RHI/HDF/HDH/ICI/MCI/NIA/NBH/MAX/SHA/TAT). Zero cross-insurer mismatches.
+- Completeness floor: 50%. All 43 written JSONs cleared the threshold. None skipped.
+- Verbatim quotes: every populated field carries a 60-240 char quote pulled from the cached text (grep-verifiable).
+- Reproducibility: `tools/curate_batch2.py` + `tools/extract_policy_text_batch2.py` together re-produce the entire batch.
+
+## Known Gaps / Manual Follow-up (carries from batch 1 + batch-2-specific)
+
+1. **UINs missing** for `hdfc-ergo__my-health-medisure-prime`, `hdfc-ergo__optima-enhance`, `manipalcigna__prohealth-select`, `manipalcigna__sarvah-param` — pull from product brochures or insurer FAQ pages.
+2. **Specific-disease waiting** defaulted to 24 months in ~30% of batch-2 files (IRDAI standard) when not explicitly quoted. Verify per product wording.
+3. **Sum Insured options** still null for most batch-2 policies — pull from product brochures / Policy Schedule.
+4. **Room rent capping** captures the most-relevant phrase but may not enumerate all SI tiers' room caps for tiered products.
+5. **NCB %** null for products using Booster/Recharge/Variable-bonus structures (ReAssure 3.0, Health Premia, etc.). These need a structured `bonus_structure` field — current schema's single % field is insufficient.
+
+## Reproducibility (batch 2)
+
+- Text-cache step: `tools/extract_policy_text_batch2.py` → writes 43 `.txt` files to `/tmp/claude/policy_extract/text_cache/`.
+- Curation step: `tools/curate_batch2.py` → reads cache, writes 43 JSONs to `data/policy_facts/`. Re-runnable (skips files that already exist; use `tools/clear_batch2.py` to wipe-then-rebuild only batch-2 outputs without touching batch-1).
+- Validation: `python3 -c "import json; json.load(open('data/policy_facts/<id>.json'))"` for any file.

frontend/src/app/page.tsx

@@ -13,6 +13,7 @@ import {
   getHealth,
   getInsurerReviews,
   getMarketplace,
+  getProfileCompleteness,
   getScorecard,
   InsurerReviews,
   MarketplacePolicy,
@@ -21,6 +22,7 @@ import {
   postPremiumEstimate,
   postTranscribe,
   PremiumEstimateResponse,
+  ProfileCompletenessResponse,
   ScorecardResponse,
   uploadPolicy,
 } from "@/lib/api";
@@ -837,51 +839,79 @@ function ScorecardBadgesForCitations({ citations }: { citations: Citation[] }) {
   );
 }
 
+// Plain-English label per criterion — shown as a sub-line under the name
+// so the buyer doesn't need to mentally translate "Cost Predictability" etc.
+const CRITERION_BLURB: Record<string, string> = {
+  "Coverage Breadth": "What's actually covered when you claim",
+  "Cost Predictability": "How likely you'll face surprise out-of-pocket bills",
+  "Waiting-Period Friction": "How soon you can actually use the policy",
+  "Claim Experience": "Will the insurer actually pay when you claim?",
+  "Renewal Protection": "Can you keep this policy at 70+ when you need it most",
+  "Bonus & Loyalty": "Rewards for staying claim-free + renewing",
+};
+
 function ScorecardCard({ sc }: { sc: ScorecardResponse }) {
+  // Sort sub-scores high-to-low so strengths surface first, weaknesses last —
+  // mirrors how a human would explain it
+  const sortedSubs = [...sc.sub_scores].sort((a, b) => b.score - a.score);
   return (
-    <div className="mt-2 rounded-xl border border-[var(--border)] bg-[var(--card)] p-3 text-xs animate-fade-up">
-      <div className="flex items-center justify-between mb-2">
-        <div className="flex items-center gap-2">
-          <span className={`inline-flex items-center justify-center w-7 h-7 rounded-lg font-bold ${gradeColor(sc.grade)}`}>
+    <div className="mt-2 rounded-xl border border-[var(--border)] bg-[var(--card)] p-4 text-xs animate-fade-up">
+      <div className="flex items-start justify-between mb-3 gap-3">
+        <div className="flex items-start gap-3 min-w-0 flex-1">
+          <span className={`inline-flex items-center justify-center w-10 h-10 rounded-lg font-bold text-base ${gradeColor(sc.grade)} shrink-0`}>
             {sc.grade}
           </span>
-          <div>
-            <div className="font-semibold text-sm">{sc.policy_name}</div>
-            <div className="text-[var(--muted-foreground)] text-[11px]">{sc.one_liner}</div>
+          <div className="min-w-0">
+            <div className="font-semibold text-sm truncate">{sc.policy_name}</div>
+            <div className="text-[var(--muted-foreground)] text-[11px] leading-snug mt-0.5">{sc.one_liner}</div>
           </div>
         </div>
-        <div className="text-right">
-          <div className="text-lg font-semibold">{sc.overall_score}<span className="text-[var(--muted-foreground)] text-xs">/100</span></div>
-          <div className="text-[10px] text-[var(--muted-foreground)]">data {sc.data_completeness_pct.toFixed(0)}% complete</div>
+        <div className="text-right shrink-0">
+          <div className="text-2xl font-bold leading-none">{sc.overall_score}<span className="text-[var(--muted-foreground)] text-sm font-normal">/100</span></div>
+          <div className="text-[10px] text-[var(--muted-foreground)] mt-0.5">data {sc.data_completeness_pct.toFixed(0)}% complete</div>
         </div>
       </div>
-      <div className="space-y-1.5 mt-3">
-        {sc.sub_scores.map((s) => (
-          <div key={s.name}>
-            <div className="flex items-center justify-between text-[11px]">
-              <span className="font-medium">{s.name}</span>
-              <span className="text-[var(--muted-foreground)]">{s.score} · {s.summary}</span>
-            </div>
-            <div className="h-1.5 rounded-full bg-[var(--muted)] overflow-hidden">
-              <div
-                className={`h-full ${s.score >= 70 ? "bg-emerald-500" : s.score >= 55 ? "bg-amber-500" : "bg-red-400"}`}
-                style={{ width: `${Math.max(2, s.score)}%` }}
-              />
+      <div className="space-y-2.5 mt-4">
+        {sortedSubs.map((s) => {
+          const barColor = s.score >= 75 ? "bg-emerald-500" : s.score >= 55 ? "bg-amber-500" : "bg-red-400";
+          const blurb = CRITERION_BLURB[s.name];
+          return (
+            <div key={s.name}>
+              <div className="flex items-baseline justify-between mb-0.5">
+                <div className="min-w-0 flex-1 pr-2">
+                  <div className="text-[11px] font-semibold leading-tight">{s.name}</div>
+                  {blurb && <div className="text-[10px] text-[var(--muted-foreground)] leading-tight mt-0.5">{blurb}</div>}
+                </div>
+                <div className="text-right shrink-0">
+                  <span className="text-sm font-bold">{s.score}</span>
+                  <span className="text-[10px] text-[var(--muted-foreground)] ml-1">/ 100</span>
+                  <div className="text-[10px] text-[var(--muted-foreground)] leading-tight">{s.summary}</div>
+                </div>
+              </div>
+              <div className="h-2 rounded-full bg-[var(--muted)] overflow-hidden">
+                <div className={`h-full ${barColor} transition-[width] duration-500`} style={{ width: `${Math.max(2, s.score)}%` }} />
+              </div>
+              {s.signals && s.signals.length > 0 && (
+                <div className="mt-1 flex flex-wrap gap-1">
+                  {s.signals.slice(0, 4).map((sig, i) => {
+                    const isNegative = sig.startsWith("−") || sig.startsWith("-");
+                    return (
+                      <span
+                        key={i}
+                        className={`inline-block text-[10px] px-1.5 py-0.5 rounded ${isNegative ? "bg-red-100 text-red-700 dark:bg-red-900/20 dark:text-red-300" : "bg-[var(--accent)] text-[var(--foreground)]"}`}
+                      >
+                        {sig}
+                      </span>
+                    );
+                  })}
+                </div>
+              )}
             </div>
-            {s.signals && s.signals.length > 0 && (
-              <ul className="mt-1 ml-1 space-y-0.5">
-                {s.signals.slice(0, 4).map((sig, i) => (
-                  <li key={i} className="text-[10px] text-[var(--muted-foreground)]">
-                    · {sig}
-                  </li>
-                ))}
-              </ul>
-            )}
-          </div>
-        ))}
+          );
+        })}
       </div>
-      <div className="mt-2 pt-2 border-t border-[var(--border)] text-[10px] text-[var(--muted-foreground)]">
-        Methodology: 24 of 48 schema fields drive this grade. Rules-based, no LLM-in-the-loop.
+      <div className="mt-3 pt-2.5 border-t border-[var(--border)] text-[10px] text-[var(--muted-foreground)] leading-snug">
+        Weighted average across 6 criteria. Rules-based — no LLM in the scoring loop. Expand &quot;How is this score computed?&quot; below to see which of 48 schema fields feed each criterion.
       </div>
     </div>
   );
@@ -1628,12 +1658,19 @@ function ComparisonModal({ policyIds, onClose }: { policyIds: string[]; onClose:
 function PolicyDetailModal({ policy, onClose }: { policy: MarketplacePolicy; onClose: () => void }) {
   const [sc, setSc] = useState<ScorecardResponse | null>(null);
   const [reviews, setReviews] = useState<InsurerReviews | null>(null);
+  const [completeness, setCompleteness] = useState<ProfileCompletenessResponse | null>(null);
   useEffect(() => {
     getScorecard(policy.policy_id).then(setSc).catch(() => setSc(null));
     if (policy.insurer_slug) {
       getInsurerReviews(policy.insurer_slug).then(setReviews).catch(() => setReviews(null));
     }
+    // Profile completeness gates whether we render the per-user grade.
+    // Below threshold: show universal grade only (insurer-quality-led) with a
+    // CTA to complete the profile.
+    const sid = typeof window !== "undefined" ? sessionStorage.getItem("insurance_session_id") || undefined : undefined;
+    getProfileCompleteness(sid).then(setCompleteness).catch(() => setCompleteness(null));
   }, [policy.policy_id, policy.insurer_slug]);
+  const isPersonalized = completeness?.is_personalized === true;
 
   const initials = insurerInitials(policy.insurer_name);
   const color = INSURER_COLOR[policy.insurer_slug] || "bg-slate-500";
@@ -1674,6 +1711,22 @@ function PolicyDetailModal({ policy, onClose }: { policy: MarketplacePolicy; onC
         <div className="p-5 space-y-5">
           {sc && (
             <div>
+              {!isPersonalized && (
+                <div className="mb-3 bg-[var(--accent)] border border-[var(--primary)] rounded-lg p-3 text-xs">
+                  <div className="font-semibold text-[var(--primary)] mb-1">This is the generic grade for an average buyer.</div>
+                  <p className="text-[var(--muted-foreground)] leading-snug">
+                    Tell me about yourself (age, dependents, conditions, budget) and I&apos;ll re-score this policy for <strong className="text-[var(--foreground)]">your</strong> situation. The same policy can be a B for a 30-year-old and a D for a 60-year-old with diabetes — context changes everything.
+                    {completeness && completeness.completeness_pct > 0 && (
+                      <span className="block mt-1">Your profile is {completeness.completeness_pct}% complete. {completeness.next_question_hint && <em className="not-italic">Next: {completeness.next_question_hint.slice(0, 80)}…</em>}</span>
+                    )}
+                  </p>
+                </div>
+              )}
+              {isPersonalized && completeness && (
+                <div className="mb-3 text-[10px] text-[var(--primary)] font-semibold flex items-center gap-1">
+                  ✓ Personalized for you · profile {completeness.completeness_pct}% complete
+                </div>
+              )}
               <div className="flex items-center gap-3 mb-3">
                 <span className={`inline-flex items-center justify-center w-12 h-12 rounded-lg font-bold ${gradeColor(sc.grade)}`}>{sc.grade}</span>
                 <div className="flex-1">

frontend/src/lib/api.ts

@@ -268,6 +268,24 @@ export async function getMarketplace(): Promise<MarketplaceResponse> {
   return resp.json();
 }
 
+export type ProfileCompletenessResponse = {
+  completeness: number;
+  completeness_pct: number;
+  fields_collected: string[];
+  fields_missing: string[];
+  is_personalized: boolean;
+  gate_threshold: number;
+  next_question_hint?: string | null;
+};
+
+export async function getProfileCompleteness(session_id?: string): Promise<ProfileCompletenessResponse> {
+  const url = new URL(`${BACKEND_URL}/api/profile/completeness`);
+  if (session_id) url.searchParams.set("session_id", session_id);
+  const resp = await fetch(url.toString());
+  if (!resp.ok) throw new Error(`profile completeness failed: ${resp.status}`);
+  return resp.json();
+}
+
 export async function getCompare(policy_ids: string[]): Promise<CompareResponse> {
   const url = new URL(`${BACKEND_URL}/api/policies/compare`);
   for (const id of policy_ids) url.searchParams.append("policy_ids", id);

tools/pdf_probe.py

@@ -0,0 +1,11 @@
+import time, pdfplumber
+t0 = time.time()
+with pdfplumber.open('rag/corpus/star-health/star-hospital-cash__brochure.pdf') as p:
+    nP = len(p.pages)
+    print(f'pages: {nP}')
+    sample = ''
+    for i, page in enumerate(p.pages[:5]):
+        t = page.extract_text() or ''
+        sample += t
+    print(f'first 5 pages chars: {len(sample)}')
+    print(f'elapsed: {time.time()-t0:.1f}s')