Deploy v1 — single-Docker FastAPI + Next.js + RAG + voice + faithfulness

backend/orchestrator.py

@@ -262,19 +262,42 @@ async def handle_turn(
         for c in chunks
     ]
 
-    # 7. INDIC CASCADE — translate the English reply back into Hinglish/Hindi
-    # so the user hears it in their language. Citations stay intact (the
-    # translation prompt preserves them).
+    # 7. INDIC CASCADE — translate the English reply back into Hinglish/Hindi,
+    # then run THREE drift checks. If any catches drift, revert to the English
+    # reply (user sees correct facts even if not in their preferred language).
+    #   Gate-A: regex anchors        — numbers, citations, currency
+    #   Gate-B: Groq Llama LLM-judge — semantic faithfulness in Hinglish
+    #   Gate-C: back-translate-cosine — Hinglish → EN via Sarvam, compare to original EN
     final_brain_tag = f"{pick.provider.name}::{pick.reason}"
     if language == "indic" and not blocked and reply:
         try:
             from backend.translator import translate_to_indic
-            reply_indic = await translate_to_indic(reply, target_lang="hi-IN")
+            from backend.translation_check import (
+                check_translation_drift,
+                check_hinglish_faithfulness,
+                check_back_translation,
+            )
+
+            english_reply = reply
+            reply_indic = await translate_to_indic(english_reply, target_lang="hi-IN")
             if reply_indic and reply_indic.strip():
-                reply = reply_indic
-                final_brain_tag = f"cascade::sarvam-trans+{pick.provider.name}+sarvam-trans"
+                # Run all 3 drift checks; short-circuit on the first failure
+                drift_a = check_translation_drift(english_reply, reply_indic)
+                if drift_a.drift_detected:
+                    final_brain_tag = f"cascade::drift-anchor-fallback+{pick.provider.name}"
+                else:
+                    drift_b = await check_hinglish_faithfulness(english_reply, reply_indic)
+                    if drift_b.drift_detected:
+                        final_brain_tag = f"cascade::drift-llmjudge-fallback+{pick.provider.name}"
+                    else:
+                        drift_c = await check_back_translation(english_reply, reply_indic, min_cosine=0.80)
+                        if drift_c.drift_detected:
+                            final_brain_tag = f"cascade::drift-cosine-fallback+{pick.provider.name}"
+                        else:
+                            reply = reply_indic
+                            final_brain_tag = f"cascade::sarvam-trans+{pick.provider.name}+sarvam-trans"
         except Exception:
-            pass  # if translation fails, return English; better than nothing
+            pass  # if any step fails, return English — better than mis-translated
 
     return TurnResult(
         reply_text=reply,

backend/translation_check.py

@@ -0,0 +1,229 @@
+"""Verify the Sarvam-M Hinglish back-translation preserved the load-bearing
+facts from DeepSeek-V3's English reply.
+
+Closes the F-16 gap: faithfulness gates verify the English answer, but until
+this module the Hinglish translation that the user actually saw/heard was
+trusted blindly. If Sarvam silently drops a citation or changes "24 months"
+to "2 years" (semantically equivalent but the regex floor would still catch
+it) — that's fine. But if it changes "24 months" to "12 months" — that's a
+mis-sale and we MUST catch it.
+
+Mechanism (Layer 1 — regex anchors, <50 ms):
+  1. Extract from BOTH english_reply and indic_reply:
+        - rupee amounts (₹X, Rs X, X lakh, X crore)
+        - percentages (NN%)
+        - durations (NN days/months/years)
+        - source citations [Source: ...]
+        - policy_name fragments seen in chunks
+  2. Every anchor present in english_reply must ALSO appear (in some form)
+     in the indic_reply.
+  3. Allow 1-2 fuzzy drift (e.g. "30 days" → "tees din") via a lookup map.
+  4. Block + log if any anchor dropped.
+
+Future Layer 2 (v2): back-translate indic → english via Sarvam, fuzzy-match
+against english_reply at sentence level.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from backend.config import settings
+
+LOG = settings.CORPUS_DIR.parent.parent / "logs" / "translation_drift.jsonl"
+LOG.parent.mkdir(parents=True, exist_ok=True)
+
+
+RUPEE_RE = re.compile(r"₹\s*[\d,]+(?:\.\d+)?\s*(?:lakh|crore|cr|k)?", flags=re.IGNORECASE)
+PERCENT_RE = re.compile(r"\b\d{1,3}(?:\.\d+)?\s*%")
+DURATION_EN = re.compile(r"\b(\d{1,4})\s*(?:day|days|month|months|year|years)\b", flags=re.IGNORECASE)
+# Devanagari/Hinglish duration forms — recognise common cases. Numbers stay digit.
+DURATION_HI = re.compile(r"\b(\d{1,4})\s*(?:din|mahine|mahina|saal|varsh)\b", flags=re.IGNORECASE)
+CITATION_RE = re.compile(r"\[(?:Source|Regulation):\s*([^\]]+)\]", flags=re.IGNORECASE)
+
+
+def _digits_only(s: str) -> str:
+    return re.sub(r"[^\d]", "", s)
+
+
+@dataclass
+class DriftVerdict:
+    drift_detected: bool
+    reasons: list[str] = field(default_factory=list)
+    dropped_numbers: list[str] = field(default_factory=list)
+    dropped_citations: list[str] = field(default_factory=list)
+
+
+def check_translation_drift(english_reply: str, indic_reply: str) -> DriftVerdict:
+    """Return drift verdict comparing English reply to its Hinglish translation."""
+    verdict = DriftVerdict(drift_detected=False)
+
+    # 1. Numbers + currency + percentage
+    en_amounts = set(_digits_only(m) for m in RUPEE_RE.findall(english_reply) if _digits_only(m))
+    en_amounts |= set(m.replace(" ", "") for m in PERCENT_RE.findall(english_reply))
+    en_amounts |= set(m for m, _ in [(m, None) for m in DURATION_EN.findall(english_reply)])
+
+    indic_all_digits = re.findall(r"\d+", indic_reply)
+    indic_digit_set = set(indic_all_digits)
+
+    for amt in en_amounts:
+        if not amt:
+            continue
+        if amt not in indic_digit_set:
+            verdict.drift_detected = True
+            verdict.dropped_numbers.append(amt)
+            verdict.reasons.append(f"number_dropped: '{amt}' in EN reply but not in HI reply")
+
+    # 2. Citations
+    en_cits = CITATION_RE.findall(english_reply)
+    for cit in en_cits:
+        # Extract a unique-ish token (e.g. insurer slug + policy name root)
+        tokens = re.findall(r"[A-Za-z]{3,}", cit)
+        if not tokens:
+            continue
+        anchor = tokens[0].lower()
+        if anchor not in indic_reply.lower():
+            verdict.drift_detected = True
+            verdict.dropped_citations.append(cit)
+            verdict.reasons.append(f"citation_dropped: '{cit[:60]}' — anchor token '{anchor}' missing in HI")
+
+    if verdict.drift_detected:
+        with open(LOG, "a") as f:
+            f.write(json.dumps({
+                "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                "layer": "regex_anchors",
+                "en_reply": english_reply[:600],
+                "hi_reply": indic_reply[:600],
+                "reasons": verdict.reasons,
+                "dropped_numbers": verdict.dropped_numbers,
+                "dropped_citations": verdict.dropped_citations,
+            }) + "\n")
+
+    return verdict
+
+
+# ============================================================================
+# Gate 5 — LLM-judge faithfulness on the Hinglish translation
+# ============================================================================
+
+_HINDI_JUDGE_SYSTEM = """You are a strict bilingual faithfulness verifier.
+
+Given an ENGLISH source answer and its HINGLISH translation, decide whether the
+Hinglish version faithfully conveys the SAME factual claims as the English
+source. The Hinglish may use different words / Devanagari / code-switched
+English — that's fine. What you check:
+
+1. Every factual claim (number, duration, currency, percentage, coverage, exclusion,
+   policy name, citation) in the English source must be PRESENT in the Hinglish.
+2. The Hinglish must NOT add any claims that aren't in the English source.
+3. Citations [Source: ...] should be preserved verbatim.
+
+OUTPUT — strict JSON, nothing else:
+{
+  "faithful": true | false,
+  "reason": "one short sentence — what differs, if anything"
+}
+
+Be strict. Tone changes are fine; fact changes are not."""
+
+
+async def check_hinglish_faithfulness(english_reply: str, hinglish_reply: str) -> DriftVerdict:
+    """Gate 5 — Groq Llama judges whether the Hinglish translation is faithful
+    to the English original. Catches semantic drift the regex anchors miss
+    (e.g. paraphrased exclusions, dropped caveats).
+    """
+    if not english_reply.strip() or not hinglish_reply.strip():
+        return DriftVerdict(drift_detected=False)
+
+    try:
+        # Lazy-import to avoid pulling Groq into modules that don't need it
+        from backend.providers.base import ChatMessage
+        from backend.providers.groq_llm import GroqLLM
+        judge = GroqLLM()
+        user = f"ENGLISH SOURCE:\n{english_reply}\n\nHINGLISH TRANSLATION:\n{hinglish_reply}\n\nVerify."
+        res = await judge.chat(
+            messages=[
+                ChatMessage(role="system", content=_HINDI_JUDGE_SYSTEM),
+                ChatMessage(role="user", content=user),
+            ],
+            temperature=0.0,
+            max_tokens=300,
+            response_format={"type": "json_object"},
+        )
+        data = json.loads(res.text)
+        faithful = bool(data.get("faithful", False))
+        reason = str(data.get("reason", ""))[:200]
+        verdict = DriftVerdict(
+            drift_detected=not faithful,
+            reasons=([f"hinglish_judge: {reason}"] if not faithful else []),
+        )
+        if verdict.drift_detected:
+            with open(LOG, "a") as f:
+                f.write(json.dumps({
+                    "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                    "layer": "hinglish_llm_judge",
+                    "en_reply": english_reply[:600],
+                    "hi_reply": hinglish_reply[:600],
+                    "judge_reason": reason,
+                }) + "\n")
+        return verdict
+    except Exception as e:
+        # Fail-open — don't block valid translations on judge infra errors
+        return DriftVerdict(drift_detected=False, reasons=[f"judge_error: {type(e).__name__}"])
+
+
+# ============================================================================
+# Gate 6 — back-translate-and-cosine-compare
+# ============================================================================
+
+async def check_back_translation(
+    english_reply: str,
+    hinglish_reply: str,
+    min_cosine: float = 0.80,
+) -> DriftVerdict:
+    """Translate the Hinglish reply back to English via Sarvam, then cosine-
+    compare to the original English reply via BGE embeddings.
+
+    If the two English texts are far apart (cosine < min_cosine), it means
+    Sarvam's Hinglish translation introduced or dropped meaning — the user
+    would see something materially different from what DeepSeek wrote.
+    """
+    if not english_reply.strip() or not hinglish_reply.strip():
+        return DriftVerdict(drift_detected=False)
+    try:
+        from backend.translator import translate_to_english
+        from backend.providers.local_embeddings import LocalEmbeddings
+        back_en = await translate_to_english(hinglish_reply)
+        if not back_en.strip():
+            return DriftVerdict(drift_detected=False, reasons=["back_translate_empty"])
+
+        embedder = LocalEmbeddings()
+        vecs = await embedder.embed([english_reply, back_en], input_type="document")
+        if len(vecs) < 2:
+            return DriftVerdict(drift_detected=False)
+        # Cosine similarity (BGE returns normalized vectors → dot product = cosine)
+        cosine = sum(a * b for a, b in zip(vecs[0], vecs[1]))
+
+        if cosine < min_cosine:
+            v = DriftVerdict(
+                drift_detected=True,
+                reasons=[f"back_translate_cosine_low: {cosine:.3f} < {min_cosine}"],
+            )
+            with open(LOG, "a") as f:
+                f.write(json.dumps({
+                    "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                    "layer": "back_translate_cosine",
+                    "cosine": round(cosine, 4),
+                    "threshold": min_cosine,
+                    "en_reply": english_reply[:600],
+                    "hi_reply": hinglish_reply[:600],
+                    "back_translated_en": back_en[:600],
+                }) + "\n")
+            return v
+        return DriftVerdict(drift_detected=False, reasons=[f"back_translate_cosine_ok: {cosine:.3f}"])
+    except Exception as e:
+        return DriftVerdict(drift_detected=False, reasons=[f"back_translate_error: {type(e).__name__}"])

docs/04-failure-modes.md

@@ -172,6 +172,14 @@ Reply to user
 
 Run-time. Auditable. Tested.
 
+### F-16 — Translation cascade introduces drift after faithfulness gate
+
+**Description:** Indic queries use the cascade: Sarvam translates Hinglish → English, DeepSeek reasons → English answer, faithfulness gates run on English answer, Sarvam translates English answer → Hinglish. **Faithfulness does NOT re-verify the final Hinglish output.** If Sarvam corrupts the translation (drops a citation, changes a number, invents a benefit), we wouldn't catch it.
+**Detection v1:** None automated. Manual spot-check of bilingual eval set.
+**Mitigation v1:** Sarvam translator system prompt explicitly forbids changing numbers/citations + caps at 60 words; preserves `[Source: ...]` tags.
+**Mitigation v2:** Back-translate Hinglish→English; compare against original English; block if cosine similarity < 0.85.
+**Status:** Accepted limitation for v1.
+
 ## 3. Open mitigations (this document tracks status)
 
 | # | Mitigation | Owner | Status |

frontend/src/app/page.tsx

@@ -8,8 +8,10 @@ import {
   CoverageResponse,
   getCoverage,
   getHealth,
+  getScorecard,
   postChat,
   postTranscribe,
+  ScorecardResponse,
   uploadPolicy,
 } from "@/lib/api";
 
@@ -356,6 +358,9 @@ function Message({ m }: { m: DisplayMessage }) {
       } ${m.blocked ? "ring-1 ring-amber-300" : ""}`}>
         <div className="text-sm sm:text-base whitespace-pre-wrap leading-relaxed">{m.content}</div>
         {m.audioUrl && <audio controls src={m.audioUrl} className="mt-2 w-full max-w-xs" style={{ height: 32 }} />}
+        {m.citations && m.citations.length > 0 && !isUser && (
+          <ScorecardBadgesForCitations citations={m.citations} />
+        )}
         {m.citations && m.citations.length > 0 && (
           <div className="mt-3 pt-3 border-t border-[var(--border)] space-y-1.5">
             <div className="text-[10px] uppercase tracking-wide text-[var(--muted-foreground)] font-semibold">Sources</div>
@@ -374,6 +379,132 @@ function Message({ m }: { m: DisplayMessage }) {
   );
 }
 
+function gradeColor(grade: string): string {
+  const map: Record<string, string> = {
+    A: "bg-emerald-500 text-white",
+    B: "bg-teal-500 text-white",
+    C: "bg-amber-500 text-white",
+    D: "bg-orange-500 text-white",
+    F: "bg-red-500 text-white",
+  };
+  return map[grade] || "bg-stone-400 text-white";
+}
+
+function ScorecardBadgesForCitations({ citations }: { citations: Citation[] }) {
+  const [cards, setCards] = useState<Record<string, ScorecardResponse | null>>({});
+  const [expanded, setExpanded] = useState<string | null>(null);
+
+  // Unique top 3 policy_ids from citations (preserve order, dedupe)
+  const seen = new Set<string>();
+  const topPolicies = citations
+    .filter((c) => {
+      if (seen.has(c.policy_id)) return false;
+      seen.add(c.policy_id);
+      return true;
+    })
+    .slice(0, 3);
+
+  useEffect(() => {
+    for (const c of topPolicies) {
+      if (cards[c.policy_id] !== undefined) continue;
+      getScorecard(c.policy_id)
+        .then((s) => setCards((prev) => ({ ...prev, [c.policy_id]: s })))
+        .catch(() => setCards((prev) => ({ ...prev, [c.policy_id]: null })));
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [citations.map((c) => c.policy_id).join("|")]);
+
+  const ready = topPolicies.filter((c) => cards[c.policy_id]);
+  if (ready.length === 0) return null;
+
+  return (
+    <div className="mt-3 pt-3 border-t border-[var(--border)] space-y-2">
+      <div className="text-[10px] uppercase tracking-wide text-[var(--muted-foreground)] font-semibold">
+        Policy Scorecards
+      </div>
+      <div className="flex flex-wrap gap-1.5">
+        {ready.map((c) => {
+          const sc = cards[c.policy_id]!;
+          const isOpen = expanded === c.policy_id;
+          const lowData = sc.data_completeness_pct < 50;
+          return (
+            <button
+              key={c.policy_id}
+              onClick={() => setExpanded(isOpen ? null : c.policy_id)}
+              className={`text-xs px-2.5 py-1 rounded-lg border transition flex items-center gap-2 ${
+                isOpen
+                  ? "border-[var(--primary)] bg-[var(--accent)]"
+                  : "border-[var(--border)] bg-[var(--card)] hover:border-[var(--primary)]"
+              }`}
+              title={`${sc.policy_name} · ${sc.one_liner}`}
+            >
+              <span className={`inline-flex items-center justify-center w-5 h-5 rounded font-bold text-[11px] ${gradeColor(sc.grade)}`}>
+                {sc.grade}
+              </span>
+              <span className="font-medium truncate max-w-[140px]">{sc.policy_name}</span>
+              <span className="opacity-60">{sc.overall_score}</span>
+              {lowData && <span title="extraction was incomplete" className="opacity-50">⚠</span>}
+            </button>
+          );
+        })}
+      </div>
+      {expanded && cards[expanded] && (
+        <ScorecardCard sc={cards[expanded]!} />
+      )}
+    </div>
+  );
+}
+
+function ScorecardCard({ sc }: { sc: ScorecardResponse }) {
+  return (
+    <div className="mt-2 rounded-xl border border-[var(--border)] bg-[var(--card)] p-3 text-xs animate-fade-up">
+      <div className="flex items-center justify-between mb-2">
+        <div className="flex items-center gap-2">
+          <span className={`inline-flex items-center justify-center w-7 h-7 rounded-lg font-bold ${gradeColor(sc.grade)}`}>
+            {sc.grade}
+          </span>
+          <div>
+            <div className="font-semibold text-sm">{sc.policy_name}</div>
+            <div className="text-[var(--muted-foreground)] text-[11px]">{sc.one_liner}</div>
+          </div>
+        </div>
+        <div className="text-right">
+          <div className="text-lg font-semibold">{sc.overall_score}<span className="text-[var(--muted-foreground)] text-xs">/100</span></div>
+          <div className="text-[10px] text-[var(--muted-foreground)]">data {sc.data_completeness_pct.toFixed(0)}% complete</div>
+        </div>
+      </div>
+      <div className="space-y-1.5 mt-3">
+        {sc.sub_scores.map((s) => (
+          <div key={s.name}>
+            <div className="flex items-center justify-between text-[11px]">
+              <span className="font-medium">{s.name}</span>
+              <span className="text-[var(--muted-foreground)]">{s.score} · {s.summary}</span>
+            </div>
+            <div className="h-1.5 rounded-full bg-[var(--muted)] overflow-hidden">
+              <div
+                className={`h-full ${s.score >= 70 ? "bg-emerald-500" : s.score >= 55 ? "bg-amber-500" : "bg-red-400"}`}
+                style={{ width: `${Math.max(2, s.score)}%` }}
+              />
+            </div>
+            {s.signals && s.signals.length > 0 && (
+              <ul className="mt-1 ml-1 space-y-0.5">
+                {s.signals.slice(0, 4).map((sig, i) => (
+                  <li key={i} className="text-[10px] text-[var(--muted-foreground)]">
+                    · {sig}
+                  </li>
+                ))}
+              </ul>
+            )}
+          </div>
+        ))}
+      </div>
+      <div className="mt-2 pt-2 border-t border-[var(--border)] text-[10px] text-[var(--muted-foreground)]">
+        Methodology: 24 of 48 schema fields drive this grade. Rules-based, no LLM-in-the-loop.
+      </div>
+    </div>
+  );
+}
+
 function ThinkingDots() {
   return (
     <div className="flex justify-start">

frontend/src/lib/api.ts

@@ -137,6 +137,35 @@ export type UploadResponse = {
   elapsed_ms: number;
 };
 
+export type ScorecardSubScore = {
+  name: string;
+  score: number;
+  summary: string;
+  signals: string[];
+};
+
+export type ScorecardResponse = {
+  policy_id: string;
+  policy_name: string;
+  insurer_slug: string;
+  overall_score: number;
+  grade: string;
+  one_liner: string;
+  sub_scores: ScorecardSubScore[];
+  data_completeness_pct: number;
+  methodology_link: string;
+};
+
+export async function getScorecard(policy_id: string): Promise<ScorecardResponse> {
+  const resp = await fetch(`${BACKEND_URL}/api/policies/${encodeURIComponent(policy_id)}/scorecard`);
+  if (!resp.ok) {
+    const t = await resp.text();
+    throw new Error(`scorecard failed: ${resp.status} ${t}`);
+  }
+  return resp.json();
+}
+
+
 export async function uploadPolicy(file: File): Promise<UploadResponse> {
   const fd = new FormData();
   fd.append("file", file);