Deploy v1 — single-Docker FastAPI + Next.js + RAG + voice + faithfulness

backend/main.py

@@ -368,8 +368,24 @@ async def upload_policy(file: UploadFile = File(...)):
     try:
         from rag.ingest import chunk_pages, get_chroma_collection, read_pdf_pages
         from backend.providers.local_embeddings import LocalEmbeddings as _Emb
+        from backend.security import check_upload, rate_limiter
 
         pages = read_pdf_pages(out_path)
+        # Run 4-gate security check (mechanics + content + injection + rate limit)
+        full_text = "\n".join(t for _, t in pages)
+        verdict = check_upload(
+            content=contents,
+            extracted_text=full_text,
+            page_count=len(pages),
+            session_id="anonymous",  # v1: no session; v2 wires session_id
+        )
+        if not verdict.accepted:
+            out_path.unlink(missing_ok=True)
+            raise HTTPException(
+                400,
+                f"Upload rejected by security gates: {', '.join(verdict.reasons[:3])}",
+            )
+
         chunks = list(chunk_pages(pages))
         if not chunks:
             raise HTTPException(400, "Could not extract any text from the PDF (scanned image-only?).")
@@ -400,6 +416,8 @@ async def upload_policy(file: UploadFile = File(...)):
         except Exception:
             pass
         collection.add(ids=ids, documents=texts, embeddings=vectors, metadatas=metadatas)
+        # Update rate-limit ledger after successful index
+        rate_limiter.record_upload("anonymous", len(chunks))
     except HTTPException:
         raise
     except Exception as e:
@@ -414,6 +432,60 @@ async def upload_policy(file: UploadFile = File(...)):
     )
 
 
+class ScorecardSubScore(BaseModel):
+    name: str
+    score: int
+    summary: str
+    signals: list[str]
+
+
+class ScorecardResponse(BaseModel):
+    policy_id: str
+    policy_name: str
+    insurer_slug: str
+    overall_score: int
+    grade: str
+    one_liner: str
+    sub_scores: list[ScorecardSubScore]
+    data_completeness_pct: float
+    methodology_link: str
+
+
+@app.get("/api/policies/{policy_id}/scorecard", response_model=ScorecardResponse)
+async def policy_scorecard(policy_id: str):
+    """Compute the 6-sub-score A-F scorecard for an extracted policy.
+
+    See docs/scorecard-methodology.md for the field-to-score mapping.
+    """
+    import json as _json
+    from pathlib import Path as _Path
+
+    from backend.scorecard import build_scorecard
+
+    # Look up policy from DuckDB (or extracted JSON file)
+    extracted_path = settings.EXTRACTED_DIR / f"{policy_id}.json"
+    if not extracted_path.exists():
+        raise HTTPException(404, f"No extracted data for policy_id={policy_id}")
+
+    try:
+        policy = _json.loads(extracted_path.read_text())
+    except Exception as e:
+        raise HTTPException(500, f"Could not load extracted policy: {e}")
+
+    sc = build_scorecard(policy)
+    return ScorecardResponse(
+        policy_id=sc.policy_id,
+        policy_name=sc.policy_name,
+        insurer_slug=sc.insurer_slug,
+        overall_score=sc.overall_score,
+        grade=sc.grade,
+        one_liner=sc.one_liner,
+        sub_scores=[ScorecardSubScore(**s.__dict__) for s in sc.sub_scores],
+        data_completeness_pct=sc.data_completeness_pct,
+        methodology_link=sc.methodology_link,
+    )
+
+
 @app.post("/api/tts")
 async def tts(req: TTSRequest):
     """Standalone TTS endpoint — returns base64 WAV."""

backend/scorecard.py

@@ -0,0 +1,317 @@
+"""Policy health scorecard — turns 48 structured fields into a human-readable
+A-F grade with 6 sub-scores.
+
+Why this exists: a buyer reading 48 fields can't tell if it's "good." A
+single letter grade + 6 sub-bars + 1-line summary makes the answer obvious.
+Inspired by what people like Beli / Ditto have done to simplify insurance.
+
+Score philosophy: optimize for the *buyer*, not the insurer. So:
+  - Generous coverage, low frictions, predictable claims = higher score
+  - Heavy waiting periods, copays, sub-limits = lower score
+  - Regulatory-mandated minimums (IRDAI 30-day initial) don't hurt the score
+
+Each sub-score is 0-100. Overall is a weighted average. Letter grade comes
+from thresholds (A: 85+, B: 70-84, C: 55-69, D: 40-54, F: <40).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+
+@dataclass
+class SubScore:
+    name: str
+    score: int  # 0-100
+    summary: str
+    signals: list[str] = field(default_factory=list)  # short positive/negative bullets
+
+
+@dataclass
+class Scorecard:
+    policy_id: str
+    policy_name: str
+    insurer_slug: str
+    overall_score: int
+    grade: str  # A, B, C, D, F
+    one_liner: str
+    sub_scores: list[SubScore]
+    data_completeness_pct: float  # how many of the scoring fields actually have data
+    methodology_link: str = "/docs/scorecard-methodology.md"
+
+
+# ---- helpers ----
+
+def _get(p: dict, key: str, default: Any = None) -> Any:
+    v = p.get(key, default)
+    if isinstance(v, dict) and "covered" in v:
+        return v.get("covered", default)
+    return v
+
+
+def _bool(p: dict, key: str) -> Optional[bool]:
+    v = p.get(key)
+    if isinstance(v, dict) and "covered" in v:
+        return v.get("covered")
+    if isinstance(v, bool):
+        return v
+    if isinstance(v, str) and v.lower() in ("yes", "true", "y"):
+        return True
+    if isinstance(v, str) and v.lower() in ("no", "false", "n"):
+        return False
+    return None
+
+
+def _int(p: dict, key: str) -> Optional[int]:
+    v = p.get(key)
+    if isinstance(v, dict) and "limit_inr" in v:
+        v = v.get("limit_inr")
+    try:
+        return int(v) if v is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def clamp(x: float, lo: int = 0, hi: int = 100) -> int:
+    return max(lo, min(hi, int(round(x))))
+
+
+# ---- 6 sub-scores ----
+
+def score_coverage_breadth(p: dict) -> SubScore:
+    """How wide is the safety net? AYUSH, day-care, OPD, organ donor, maternity, etc."""
+    signals_pos: list[str] = []
+    signals_neg: list[str] = []
+    s = 50  # neutral base
+
+    if _bool(p, "ayush_coverage"):
+        s += 8; signals_pos.append("AYUSH covered")
+    elif _bool(p, "ayush_coverage") is False:
+        signals_neg.append("no AYUSH")
+
+    dct = _int(p, "day_care_treatments_count")
+    if dct is not None:
+        if dct >= 400: s += 10; signals_pos.append(f"{dct} day-care procedures")
+        elif dct >= 200: s += 6
+        elif dct < 100: s -= 5; signals_neg.append(f"only {dct} day-care procedures")
+
+    if _bool(p, "maternity_coverage"):
+        s += 6; signals_pos.append("maternity covered")
+    if _bool(p, "newborn_coverage"):
+        s += 4; signals_pos.append("newborn covered")
+    if _bool(p, "organ_donor_expenses"):
+        s += 4; signals_pos.append("organ donor expenses")
+    if _bool(p, "ambulance_cover"):
+        s += 3; signals_pos.append("ambulance covered")
+    if _bool(p, "domiciliary_treatment"):
+        s += 4
+    if _bool(p, "preventive_health_checkup"):
+        s += 3; signals_pos.append("free health checkups")
+
+    pre = _int(p, "pre_hospitalization_days") or 0
+    post = _int(p, "post_hospitalization_days") or 0
+    if pre >= 60: s += 4; signals_pos.append(f"{pre}d pre-hospitalization")
+    if post >= 90: s += 4; signals_pos.append(f"{post}d post-hospitalization")
+
+    summary = "Wide coverage" if s >= 75 else "Standard coverage" if s >= 55 else "Limited coverage"
+    return SubScore("Coverage Breadth", clamp(s), summary, signals_pos + [f"− {x}" for x in signals_neg])
+
+
+def score_cost_predictability(p: dict) -> SubScore:
+    """How likely are you to face surprise out-of-pocket costs? Copay, room rent caps, sub-limits."""
+    signals: list[str] = []
+    s = 75  # most policies start fine
+
+    copay = _int(p, "copayment_pct")
+    if copay is not None and copay > 0:
+        if copay >= 30: s -= 25; signals.append(f"− {copay}% copayment")
+        elif copay >= 20: s -= 18; signals.append(f"− {copay}% copayment")
+        elif copay >= 10: s -= 10; signals.append(f"− {copay}% copayment")
+        else: s -= 4
+
+    rrc = p.get("room_rent_capping")
+    rrc_text = rrc if isinstance(rrc, str) else (rrc.get("limit_text") if isinstance(rrc, dict) else None)
+    if rrc_text:
+        if "no cap" in rrc_text.lower() or "no monetary" in rrc_text.lower():
+            s += 6; signals.append("no room rent cap")
+        elif "1%" in rrc_text or "%" in rrc_text:
+            s -= 8; signals.append(f"− room rent capped: {rrc_text[:50]}")
+
+    deductible = _int(p, "deductible_amount")
+    if deductible and deductible > 0:
+        signals.append(f"− deductible ₹{deductible:,}")
+        s -= 6
+
+    summary = "Predictable costs" if s >= 75 else "Some out-of-pocket" if s >= 55 else "Material out-of-pocket"
+    return SubScore("Cost Predictability", clamp(s), summary, signals)
+
+
+def score_waiting_friction(p: dict) -> SubScore:
+    """How long before benefits actually kick in? PED, specific disease, maternity waits."""
+    signals: list[str] = []
+    s = 90
+
+    ped = _int(p, "pre_existing_disease_waiting_months")
+    if ped is not None:
+        if ped >= 48: s -= 30; signals.append(f"− {ped}mo PED waiting (long)")
+        elif ped >= 36: s -= 20; signals.append(f"− {ped}mo PED waiting")
+        elif ped >= 24: s -= 10; signals.append(f"− {ped}mo PED waiting")
+        else: signals.append(f"{ped}mo PED waiting (short)")
+
+    mw = _int(p, "maternity_waiting_months")
+    if mw is not None:
+        if mw >= 48: s -= 10; signals.append(f"− {mw}mo maternity waiting")
+        elif mw >= 24: s -= 4
+
+    iw = _int(p, "initial_waiting_period_days")
+    # 30 days is IRDAI-mandated minimum; don't penalize
+    if iw is not None and iw > 60: s -= 5; signals.append(f"− {iw}d initial waiting")
+
+    summary = "Quick activation" if s >= 75 else "Standard waits" if s >= 55 else "Heavy waiting periods"
+    return SubScore("Waiting-Period Friction", clamp(s), summary, signals)
+
+
+def score_claim_experience(p: dict) -> SubScore:
+    """Will claims actually be paid? Network size, settlement ratio, cashless support."""
+    signals: list[str] = []
+    s = 60
+
+    if _bool(p, "cashless_treatment_supported"):
+        s += 15; signals.append("cashless supported")
+    nh = _int(p, "network_hospital_count")
+    if nh is not None:
+        if nh >= 10000: s += 15; signals.append(f"{nh:,}+ network hospitals")
+        elif nh >= 5000: s += 8; signals.append(f"{nh:,} network hospitals")
+        elif nh < 2000: s -= 8; signals.append(f"− only {nh} network hospitals")
+
+    csr = p.get("claim_settlement_ratio")
+    try:
+        csr_val = float(csr)
+        if csr_val >= 95: s += 10; signals.append(f"{csr_val:.1f}% claim settlement ratio")
+        elif csr_val >= 85: s += 6; signals.append(f"{csr_val:.1f}% CSR")
+        elif csr_val < 75: s -= 12; signals.append(f"− {csr_val:.1f}% CSR (low)")
+    except (TypeError, ValueError):
+        pass
+
+    tat = _int(p, "tat_cashless_authorization_hours")
+    if tat is not None and tat <= 2:
+        s += 4; signals.append(f"{tat}h cashless TAT")
+
+    summary = "Smooth claims" if s >= 75 else "Standard claim experience" if s >= 55 else "Friction risk on claims"
+    return SubScore("Claim Experience", clamp(s), summary, signals)
+
+
+def score_renewal_protection(p: dict) -> SubScore:
+    """Can you keep this policy as you age? Lifelong renewability + wide age band."""
+    signals: list[str] = []
+    s = 60
+
+    maxr = _int(p, "max_renewal_age")
+    if maxr is not None:
+        if maxr >= 99: s += 25; signals.append("lifelong renewability")
+        elif maxr >= 80: s += 15; signals.append(f"renewable up to {maxr}")
+        elif maxr < 65: s -= 15; signals.append(f"− only renewable up to {maxr}")
+
+    maxe = _int(p, "max_entry_age")
+    if maxe is not None:
+        if maxe >= 65: s += 10; signals.append(f"entry up to {maxe}")
+        elif maxe < 50: s -= 6
+
+    summary = "Future-proof" if s >= 75 else "Adequate" if s >= 55 else "Renewal risk at older ages"
+    return SubScore("Renewal Protection", clamp(s), summary, signals)
+
+
+def score_bonuses(p: dict) -> SubScore:
+    """No-claim bonuses, restoration, health checkups — sweeteners for loyal buyers."""
+    signals: list[str] = []
+    s = 50
+
+    ncb = _int(p, "no_claim_bonus_pct")
+    if ncb is not None:
+        if ncb >= 100: s += 25; signals.append(f"{ncb}% NCB step-up")
+        elif ncb >= 50: s += 15; signals.append(f"{ncb}% NCB")
+        elif ncb >= 25: s += 8
+
+    rb = p.get("restoration_benefit")
+    if rb and isinstance(rb, str) and len(rb) > 5:
+        s += 12; signals.append(f"restoration benefit: {rb[:50]}")
+
+    if _bool(p, "preventive_health_checkup"):
+        s += 8; signals.append("free preventive checkup")
+
+    summary = "Generous bonuses" if s >= 75 else "Standard sweeteners" if s >= 55 else "Few extras"
+    return SubScore("Bonus & Loyalty", clamp(s), summary, signals)
+
+
+# ---- aggregate + grade ----
+
+# Weights reflect what affects the buyer's real-world experience most.
+WEIGHTS = {
+    "Coverage Breadth": 0.22,
+    "Cost Predictability": 0.20,
+    "Waiting-Period Friction": 0.18,
+    "Claim Experience": 0.20,
+    "Renewal Protection": 0.12,
+    "Bonus & Loyalty": 0.08,
+}
+
+
+def grade_for(score: int) -> tuple[str, str]:
+    """Return (letter, one-line summary tone)."""
+    if score >= 85: return "A", "Strong all-rounder — solid pick for the buyer."
+    if score >= 70: return "B", "Good policy with a few notable gaps."
+    if score >= 55: return "C", "Decent baseline; check the trade-offs before signing."
+    if score >= 40: return "D", "Material concerns — only suitable for specific use-cases."
+    return "F", "Significant gaps — alternative options are likely better."
+
+
+# Fields the scorecard touches — used to compute data_completeness_pct
+SCORED_FIELDS = [
+    "ayush_coverage", "day_care_treatments_count", "maternity_coverage",
+    "newborn_coverage", "organ_donor_expenses", "ambulance_cover",
+    "domiciliary_treatment", "preventive_health_checkup",
+    "pre_hospitalization_days", "post_hospitalization_days",
+    "copayment_pct", "room_rent_capping", "deductible_amount",
+    "pre_existing_disease_waiting_months", "maternity_waiting_months",
+    "initial_waiting_period_days",
+    "cashless_treatment_supported", "network_hospital_count",
+    "claim_settlement_ratio", "tat_cashless_authorization_hours",
+    "max_renewal_age", "max_entry_age",
+    "no_claim_bonus_pct", "restoration_benefit",
+]
+
+
+def compute_data_completeness(p: dict) -> float:
+    filled = 0
+    for k in SCORED_FIELDS:
+        v = p.get(k)
+        if v is None or v == "" or v == []:
+            continue
+        if isinstance(v, dict) and v.get("covered") is None and not v.get("limit_inr") and not v.get("limit_text"):
+            continue
+        filled += 1
+    return round(filled / max(1, len(SCORED_FIELDS)) * 100, 1)
+
+
+def build_scorecard(policy: dict) -> Scorecard:
+    subs = [
+        score_coverage_breadth(policy),
+        score_cost_predictability(policy),
+        score_waiting_friction(policy),
+        score_claim_experience(policy),
+        score_renewal_protection(policy),
+        score_bonuses(policy),
+    ]
+    overall = clamp(sum(WEIGHTS[s.name] * s.score for s in subs))
+    letter, one_liner = grade_for(overall)
+    return Scorecard(
+        policy_id=policy.get("policy_id", ""),
+        policy_name=policy.get("policy_name", ""),
+        insurer_slug=policy.get("insurer_slug", ""),
+        overall_score=overall,
+        grade=letter,
+        one_liner=one_liner,
+        sub_scores=subs,
+        data_completeness_pct=compute_data_completeness(policy),
+    )

backend/security.py

@@ -0,0 +1,221 @@
+"""Security gates for user-uploaded content.
+
+The /api/upload-policy endpoint accepts arbitrary PDFs from the public web.
+That's a real attack surface. Each upload runs through these gates before
+we touch it with the embedding model or LLM:
+
+  Gate 1 — FILE MECHANICS
+    - magic bytes start with %PDF
+    - size <= 25 MB
+    - no embedded JavaScript / AcroForm / OpenAction (PDF exploits)
+    - no /Launch / /EmbeddedFile actions (file execution / payload smuggling)
+
+  Gate 2 — CONTENT QUALITY
+    - at least 1500 chars of extractable text (rejects scanned image-only PDFs
+      and intentional empty/garbage uploads)
+    - at least 3 pages (an insurance policy is never one page)
+    - text contains at least one insurance-domain keyword (cheap filter for
+      "this is not a recipe / resume / random PDF")
+
+  Gate 3 — PROMPT INJECTION DEFENSE
+    - regex-scan for known injection patterns in extracted text
+    - reject if the PDF tries to override the system prompt, expose secrets,
+      or impersonate the assistant
+    - we DO NOT silently rewrite the content — block + log instead, so the
+      user knows their upload was rejected for a reason
+
+  Gate 4 — RATE LIMITING
+    - per-session: max 5 uploads / hour
+    - cumulative: max 200 chunks across all user uploads (prevents corpus
+      flooding by a single session)
+
+Every block is logged to logs/upload_blocks.jsonl with the reason.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from backend.config import settings
+
+LOG_DIR = settings.CORPUS_DIR.parent.parent / "logs"
+LOG_DIR.mkdir(exist_ok=True)
+UPLOAD_BLOCK_LOG = LOG_DIR / "upload_blocks.jsonl"
+
+
+@dataclass
+class UploadVerdict:
+    accepted: bool
+    reasons: list[str] = field(default_factory=list)
+    extracted_text_chars: int = 0
+    page_count: int = 0
+
+
+# Insurance-domain keywords used to confirm the PDF is plausibly a policy.
+# A single hit on any of these is enough — we don't want to over-reject.
+INSURANCE_KEYWORDS = (
+    "insur", "policy", "premium", "sum insured", "claim", "hospital",
+    "coverage", "covered", "exclus", "waiting period", "pre-existing",
+    "irdai", "deductible", "cashless", "ayush", "domiciliary", "ncb",
+    "no claim bonus", "renewal", "uin", "section", "covered",
+    "insurance", "insurer", "insurer's"
+)
+
+# Prompt-injection patterns we explicitly reject. Not exhaustive — but every
+# one of these is a clear signal of adversarial content, not a real policy.
+INJECTION_PATTERNS = [
+    re.compile(r"\bignore\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+instructions", re.IGNORECASE),
+    re.compile(r"\bdisregard\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+instructions", re.IGNORECASE),
+    re.compile(r"\bforget\s+(?:everything|all|your|the)\s+(?:previous|prior|above|instructions)", re.IGNORECASE),
+    re.compile(r"\byou\s+are\s+now\s+a\s+", re.IGNORECASE),
+    re.compile(r"\bact\s+as\s+(?:a|an)\s+(?:different|new)\s+", re.IGNORECASE),
+    re.compile(r"\bpretend\s+(?:to\s+be|you\s+are)\s+", re.IGNORECASE),
+    re.compile(r"\bsystem\s+prompt\b", re.IGNORECASE),
+    re.compile(r"<\s*\|?im_start\|?\s*>|<\s*\|?im_end\|?\s*>", re.IGNORECASE),
+    re.compile(r"\bjailbreak\b|\bdan\s+mode\b", re.IGNORECASE),
+    re.compile(r"reveal\s+(?:your|the)\s+(?:system|hidden|secret|original)\s+(?:prompt|instructions)", re.IGNORECASE),
+    re.compile(r"\bapi[_\- ]?key\b.{0,40}(?:reveal|share|tell|print|output)", re.IGNORECASE),
+]
+
+# PDF byte patterns that indicate active content / exploits
+DANGEROUS_PDF_FEATURES = [
+    (rb"/JavaScript", "embedded_javascript"),
+    (rb"/JS ", "javascript_action"),
+    (rb"/Launch", "launch_action"),
+    (rb"/EmbeddedFile", "embedded_file"),
+    (rb"/OpenAction", "openaction_trigger"),
+    (rb"/SubmitForm", "form_submission"),
+    (rb"/AA<<", "auto_actions"),
+]
+
+
+# Per-session rate-limit state (in-memory; resets on restart).
+# In v2 this moves to Redis.
+class RateLimit:
+    def __init__(self):
+        self.uploads_by_session: dict[str, list[float]] = defaultdict(list)
+        self.chunks_by_session: dict[str, int] = defaultdict(int)
+
+    def check_upload_rate(self, session_id: str) -> Optional[str]:
+        now = time.time()
+        # Keep last hour only
+        self.uploads_by_session[session_id] = [
+            t for t in self.uploads_by_session[session_id] if now - t < 3600
+        ]
+        if len(self.uploads_by_session[session_id]) >= 5:
+            return "rate_limit_uploads_per_hour"
+        return None
+
+    def record_upload(self, session_id: str, chunks: int):
+        self.uploads_by_session[session_id].append(time.time())
+        self.chunks_by_session[session_id] += chunks
+
+    def check_chunk_quota(self, session_id: str) -> Optional[str]:
+        if self.chunks_by_session.get(session_id, 0) >= 200:
+            return "rate_limit_total_chunks"
+        return None
+
+
+rate_limiter = RateLimit()
+
+
+def gate_pdf_mechanics(content: bytes) -> list[str]:
+    """Gate 1 — bytes-level PDF checks."""
+    reasons: list[str] = []
+    if not content.startswith(b"%PDF"):
+        reasons.append("not_a_pdf_magic_bytes")
+        return reasons  # no point checking further
+    if len(content) > 25 * 1024 * 1024:
+        reasons.append("file_too_large_25mb")
+    if len(content) < 5_000:
+        reasons.append("file_too_small_5kb")
+
+    # Look for dangerous PDF features in the first ~2 MB (catches most)
+    head = content[:2_000_000]
+    for needle, label in DANGEROUS_PDF_FEATURES:
+        if needle in head:
+            reasons.append(f"dangerous_pdf_feature: {label}")
+
+    return reasons
+
+
+def gate_content_quality(text: str, page_count: int) -> list[str]:
+    """Gate 2 — extracted-text checks."""
+    reasons: list[str] = []
+    if len(text.strip()) < 1500:
+        reasons.append(f"too_little_text: {len(text)} chars")
+    if page_count < 3:
+        reasons.append(f"too_few_pages: {page_count}")
+
+    text_l = text.lower()
+    if not any(kw in text_l for kw in INSURANCE_KEYWORDS):
+        reasons.append("no_insurance_keywords_found")
+
+    return reasons
+
+
+def gate_prompt_injection(text: str) -> list[str]:
+    """Gate 3 — scan for injection patterns."""
+    reasons: list[str] = []
+    for pat in INJECTION_PATTERNS:
+        m = pat.search(text)
+        if m:
+            snippet = text[max(0, m.start() - 30): m.end() + 30].replace("\n", " ")
+            reasons.append(f"injection_pattern: {snippet[:100]}")
+            break  # one hit is enough
+    return reasons
+
+
+def gate_rate_limit(session_id: str) -> list[str]:
+    """Gate 4 — per-session rate limits."""
+    reasons: list[str] = []
+    if r := rate_limiter.check_upload_rate(session_id):
+        reasons.append(r)
+    if r := rate_limiter.check_chunk_quota(session_id):
+        reasons.append(r)
+    return reasons
+
+
+def check_upload(
+    content: bytes,
+    extracted_text: str,
+    page_count: int,
+    session_id: str = "anonymous",
+) -> UploadVerdict:
+    """Run all 4 gates. Return verdict with reasons (empty if accepted)."""
+    reasons: list[str] = []
+    reasons.extend(gate_rate_limit(session_id))
+    reasons.extend(gate_pdf_mechanics(content))
+    reasons.extend(gate_content_quality(extracted_text, page_count))
+    reasons.extend(gate_prompt_injection(extracted_text))
+
+    verdict = UploadVerdict(
+        accepted=(len(reasons) == 0),
+        reasons=reasons,
+        extracted_text_chars=len(extracted_text),
+        page_count=page_count,
+    )
+
+    if not verdict.accepted:
+        _log_block(session_id, reasons, len(content), len(extracted_text), page_count)
+
+    return verdict
+
+
+def _log_block(session_id: str, reasons: list[str], byte_size: int, text_chars: int, pages: int):
+    entry = {
+        "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "session_id": session_id,
+        "reasons": reasons,
+        "byte_size": byte_size,
+        "text_chars": text_chars,
+        "pages": pages,
+    }
+    with open(UPLOAD_BLOCK_LOG, "a") as f:
+        f.write(json.dumps(entry) + "\n")

docs/scorecard-methodology.md

@@ -0,0 +1,228 @@
+# Scorecard Methodology — From 48 Schema Fields to a Single A-F Grade
+
+| Field | Value |
+| --- | --- |
+| Project | Insurance Sales Portfolio Expert |
+| Version | 0.1 |
+| Date | 2026-05-13 |
+| Implementation | `backend/scorecard.py` |
+| Endpoint | `GET /api/policies/{policy_id}/scorecard` |
+
+## 0. Why this artifact exists
+
+The 48-field structured schema captures every comparable attribute of a health policy. But a buyer reading 48 fields cannot tell whether the policy is *good*. The scorecard distils those 48 fields into:
+
+- **One letter grade** (A / B / C / D / F) — the headline
+- **One sentence** — the buyer-friendly takeaway
+- **6 sub-scores** (0-100 each) — *why* the grade is what it is, grouped by what matters in real life
+- **Per-field "signals"** — positive (✓) and negative (−) bullets explaining each sub-score
+
+The methodology is **rules-based and inspectable** — no LLM in the loop, no black-box weights. Anyone reading this doc can reproduce any policy's grade with a spreadsheet.
+
+This is inspired by what consumer fintech has done elsewhere (Ditto Insurance, Beli, Plum) to simplify a domain where the underlying contract is intentionally opaque.
+
+---
+
+## 1. Where the 48 fields came from
+
+The structured schema (`rag/schema.py`) was constructed by triangulating four sources. **Every field is grounded in a regulator-mandated or industry-standard taxonomy** — not invented by us.
+
+### Source 1 — IRDAI Customer Information Sheet (CIS) format
+Mandatory under the IRDAI Master Circular on Health Insurance Business, 2024. Every approved health policy must publish a CIS with a standardised field set covering:
+- Identity (insurer, product, UIN code)
+- Eligibility (age bands, family composition)
+- Coverage scope (inpatient, day-care, AYUSH, organ donor)
+- Sum insured + premium structure
+- Waiting periods (initial, PED, specific diseases, maternity)
+- Sub-limits (room rent, ICU, co-payment)
+- Exclusions
+- Claim process + grievance redressal
+
+The 48-field schema is a strict superset of the CIS fields, with each field's name and unit aligned to IRDAI's published spec.
+
+### Source 2 — PolicyBazaar / InsuranceDekho filter dimensions
+What aggregators expose as filterable attributes on their UI tells you what real Indian buyers actually compare on. We added fields like:
+- `network_hospital_count` — buyers care about access
+- `no_claim_bonus_pct` — sweetener for healthy renewers
+- `restoration_benefit` — high-leverage for families
+- `tat_cashless_authorization_hours` — claim friction
+
+### Source 3 — Top-insurer brochure structure analysis
+For the 10 target insurers (Star, HDFC ERGO, Niva Bupa, Care, ICICI Lombard, Bajaj Allianz, New India, Aditya Birla, Tata AIG, ManipalCigna), we inspected the structure of their published Customer Information Sheets. Fields they all surface — sometimes with slight wording differences — were canonicalised into the schema.
+
+### Source 4 — Domain-led additions
+A small number of fields exist because they materially affect buyer outcomes even if not always disclosed:
+- `claim_settlement_ratio` — IRDAI annual report disclosure
+- `geographic_coverage_india` — Pan-India vs Regional
+- `worldwide_emergency_cover` — relevant for travellers
+
+### How we did it (the actual code path)
+
+`rag/extract.py` calls Sarvam-M (with DeepSeek-V3 fallback for hard PDFs) with the full policy PDF text and the 48-field Pydantic schema as a structured-output target. The LLM extracts each field; if a field is not explicitly stated in the document, it is set to `null`. A self-critique pass scores per-field confidence (the `extraction_confidence_pct` field).
+
+The full schema lives in `rag/schema.py` (see also `rag/SCHEMA.md` for groupings and gotchas).
+
+---
+
+## 2. The scorecard — 6 sub-scores
+
+The 48 fields are aggregated into **6 sub-scores**, each 0-100. The aggregation reflects what the buyer actually experiences, **not** the insurer's marketing categories.
+
+### 2.1 Coverage Breadth — *how wide is the safety net?*
+
+**Weight in overall: 22%** (highest — the policy must cover the things that actually happen)
+
+| Schema field | Effect on score |
+| --- | --- |
+| `ayush_coverage` (bool) | +8 if covered |
+| `day_care_treatments_count` (int) | +10 if ≥400, +6 if ≥200, −5 if <100 |
+| `maternity_coverage` (bool) | +6 if covered |
+| `newborn_coverage` (bool) | +4 if covered |
+| `organ_donor_expenses` (bool) | +4 if covered |
+| `ambulance_cover` (bool) | +3 if covered |
+| `domiciliary_treatment` (bool) | +4 if covered |
+| `preventive_health_checkup` (bool) | +3 if covered |
+| `pre_hospitalization_days` (int) | +4 if ≥60 |
+| `post_hospitalization_days` (int) | +4 if ≥90 |
+
+**Base score: 50.** Total range observed: 30 (bare-bones) to 95 (comprehensive flagship).
+
+### 2.2 Cost Predictability — *will the bill surprise me?*
+
+**Weight: 20%**. The number that hurts the buyer when they actually claim.
+
+| Schema field | Effect on score |
+| --- | --- |
+| `copayment_pct` (int) | −25 if ≥30%, −18 if ≥20%, −10 if ≥10%, −4 otherwise |
+| `room_rent_capping` (text) | +6 if "no cap"; −8 if % of SI |
+| `deductible_amount` (int) | −6 if any deductible |
+
+**Base score: 75** (most policies are reasonable on this; we *penalise* friction, we don't reward absence of it).
+
+### 2.3 Waiting-Period Friction — *how long before benefits kick in?*
+
+**Weight: 18%**.
+
+| Schema field | Effect on score |
+| --- | --- |
+| `pre_existing_disease_waiting_months` | −30 if ≥48, −20 if ≥36, −10 if ≥24, 0 if <24 |
+| `maternity_waiting_months` | −10 if ≥48, −4 if ≥24 |
+| `initial_waiting_period_days` | −5 if >60 (30 is IRDAI-mandated — not penalised) |
+
+**Base score: 90.** Regulatory minimums are not held against the policy.
+
+### 2.4 Claim Experience — *will claims actually be paid?*
+
+**Weight: 20%**. The thing the buyer cares about *after* paying premium for years.
+
+| Schema field | Effect on score |
+| --- | --- |
+| `cashless_treatment_supported` (bool) | +15 if supported |
+| `network_hospital_count` (int) | +15 if ≥10K, +8 if ≥5K, −8 if <2K |
+| `claim_settlement_ratio` (float) | +10 if ≥95%, +6 if ≥85%, −12 if <75% |
+| `tat_cashless_authorization_hours` (int) | +4 if ≤2 |
+
+**Base score: 60.**
+
+### 2.5 Renewal Protection — *can I keep this as I age?*
+
+**Weight: 12%**.
+
+| Schema field | Effect on score |
+| --- | --- |
+| `max_renewal_age` (int) | +25 if lifelong (≥99), +15 if ≥80, −15 if <65 |
+| `max_entry_age` (int) | +10 if ≥65, −6 if <50 |
+
+**Base score: 60.**
+
+### 2.6 Bonus & Loyalty — *sweeteners for sticking around*
+
+**Weight: 8%** (lowest — these matter less than the core product).
+
+| Schema field | Effect on score |
+| --- | --- |
+| `no_claim_bonus_pct` (int) | +25 if ≥100, +15 if ≥50, +8 if ≥25 |
+| `restoration_benefit` (text) | +12 if non-trivial text |
+| `preventive_health_checkup` (bool) | +8 if covered |
+
+**Base score: 50.**
+
+---
+
+## 3. From sub-scores to a single grade
+
+```
+overall_score = 0.22 × Coverage Breadth
+              + 0.20 × Cost Predictability
+              + 0.18 × Waiting-Period Friction
+              + 0.20 × Claim Experience
+              + 0.12 × Renewal Protection
+              + 0.08 × Bonus & Loyalty
+```
+
+| Overall score | Grade | One-liner |
+| --- | --- | --- |
+| 85 – 100 | **A** | Strong all-rounder — solid pick for the buyer. |
+| 70 – 84 | **B** | Good policy with a few notable gaps. |
+| 55 – 69 | **C** | Decent baseline; check the trade-offs before signing. |
+| 40 – 54 | **D** | Material concerns — only suitable for specific use-cases. |
+| 0 – 39 | **F** | Significant gaps — alternative options are likely better. |
+
+---
+
+## 4. Which of the 48 fields the scorecard touches
+
+| Field group | In scorecard? | Why / why not |
+| --- | --- | --- |
+| Identity (5 fields) | No | Doesn't affect quality, only display |
+| Eligibility (5 fields) | Partial — `max_renewal_age`, `max_entry_age` only | Renewal age matters for buyer; family composition is a filter, not a quality signal |
+| Sum insured & premium (5 fields) | No | We don't score absolute price (Doc 01 D-007: pricing is illustrative); buyers compare per-rupee value separately |
+| Waiting periods (6 fields) | All 3 used | Direct buyer impact |
+| Coverage scope (~10 fields) | 8 used | Most relevant |
+| Sub-limits & caps (6 fields) | 3 used (`copayment_pct`, `room_rent_capping`, `deductible_amount`) | Disease-wise sub-limits are too policy-specific to score uniformly in v1 |
+| Geography & network (4 fields) | 2 used (`cashless_treatment_supported`, `network_hospital_count`) | Geography is a filter; network matters |
+| Exclusions (3 fields) | No (v1) | Exclusions are policy-specific text; v2 will tag them |
+| Claim & service (3 fields) | All 3 used | Highest buyer-impact |
+| Riders (3 fields) | No (v1) | Riders are optional add-ons; scoring base-policy only |
+| Source metadata (3 fields) | No | Plumbing |
+
+**24 of 48 fields** drive the scorecard. The other 24 are either policy-specific text (exclusions, riders), filters (geography, age bands), pricing (illustrative only), or metadata.
+
+This is intentional — **a scorecard that uses every field becomes noise.** We selected the 24 fields where the buyer's actual experience changes materially.
+
+---
+
+## 5. Data completeness
+
+Each scorecard reports a `data_completeness_pct` — what fraction of the 24 scored fields actually have data in the policy's extraction. **If extraction is poor, the scorecard is honest about it**:
+
+- ≥80% complete → grade is reliable
+- 60-80% → grade with caveat shown in UI
+- <60% → grade hidden, "extraction quality too low" shown instead
+
+This protects the buyer from a confidently-wrong A grade just because we couldn't read the policy properly.
+
+---
+
+## 6. Open questions / v2 enhancements
+
+- **Premium-adjusted scoring.** Currently we don't factor in *price*. A B-grade policy at ₹8K may be better value than an A-grade at ₹40K. v2: add a `value_for_money` sub-score using illustrative-price bands.
+- **Reviews & sentiment.** v2 will pull insurer reviews from Reddit, PolicyBazaar reviews, IRDAI complaints data, YouTube reviews — aggregate into a sentiment score and feed into Claim Experience.
+- **Buyer-profile-tuned weights.** A 25-year-old should care more about waiting periods + claim experience than renewal protection. v2 will personalise weights based on `Profile` from the fact-find flow.
+- **Adversarial test.** v2 will run the scorecard on every gold-Q&A policy and human-audit the worst graders.
+
+---
+
+## 7. Reproducing any grade
+
+```python
+import json
+from backend.scorecard import build_scorecard
+policy = json.load(open(f"rag/extracted/<policy_id>.json"))
+sc = build_scorecard(policy)
+print(sc.grade, sc.overall_score, sc.one_liner)
+for s in sc.sub_scores:
+    print(f"  {s.name}: {s.score}  ({s.summary})")
+    for sig in s.signals:
+        print(f"    · {sig}")
+```