# backend/verifier.py """ Post-generation numeric verification. The LLM prompt says "use only the context" — but a prompt is a request, not a guarantee. This module CHECKS: every number that appears in a generated answer must also exist in the source material (retrieved chunks + extracted metrics). Numbers that can't be traced are flagged as unverified — visibly, in the response — instead of being shipped as if they were facts. What counts as "the same number": financial texts express one value many ways (₹9,64,693 crore vs 9646.93 billion vs "9.6 lakh crore"), so each candidate is normalized to an absolute value across every plausible unit interpretation, and matching allows small rounding tolerance (an answer saying "$394.3B" against a source "394,328 million" should verify). What is deliberately NOT verified: years (2020-2035), small enumerators (section "1.", "top 3", risk score "7/10"), and page references — flagging those would bury real signal in noise. """ import re # rounding slack between an answer's number and a source number — # "$394.3B" vs 394,328M is a 0.007% gap; 1% keeps us safe on display # rounding without accepting genuinely different figures RELATIVE_TOLERANCE = 0.01 UNIT_MULTIPLIERS = { "trillion": 1_000_000_000_000, "lakh crore": 1_000_000_000_000, "lakh crores": 1_000_000_000_000, "lakh cr": 1_000_000_000_000, "billion": 1_000_000_000, "million": 1_000_000, "thousand": 1_000, "crore": 10_000_000, "crores": 10_000_000, "cr": 10_000_000, "lakh": 100_000, "lakhs": 100_000, "lac": 100_000, "b": 1_000_000_000, "m": 1_000_000, "k": 1_000, } # number, optionally preceded by a currency marker and followed by a unit # word/suffix or % — the shapes financial narration actually produces # (includes format_money's own output: "₹60,812.0 Cr", "₹1.23 Lakh Cr") _NUMBER_RE = re.compile( r"(?P[₹$€£]|Rs\.?\s?|INR\s?|USD\s?)?" r"(?P\d[\d,]*(?:\.\d+)?)" r"\s*(?Plakh\s+crores?|lakh\s+cr\b|trillion|billion|million|thousand" r"|crores?\b|cr\b|lakhs?\b|lac\b|%|[BMKbmk]\b)?", re.UNICODE | re.IGNORECASE, ) def _parse_number(num_str: str) -> float | None: try: return float(num_str.replace(",", "")) except ValueError: return None def _interpretations(value: float, unit: str | None) -> list: """All absolute values this (number, unit) pair could mean. With an explicit unit there's one reading; without, the bare value is used as-is (source-side bare numbers get the same treatment, so matching still works when both sides omit units).""" if unit and unit != "%": key = re.sub(r"\s+", " ", unit.lower().strip()) mult = UNIT_MULTIPLIERS.get(key) if mult: return [value * mult] return [value] def _is_ignorable(value: float, unit: str | None, currency: str | None) -> bool: """Years, tiny enumerators, and unit-less small numbers are noise, not claims worth verifying.""" if unit == "%": return False # percentages are real claims if unit or currency: return False # explicit money/unit context — always verify if 1900 <= value <= 2035 and value == int(value): return True # a year if abs(value) < 100: return True # list indices, "3 companies", risk score 7... return False def extract_numbers(text: str) -> list: """All verifiable numeric claims in a text. Returns [{"raw": str, "value": float, "unit": str|None, "currency": str|None, "absolutes": [float, ...]}]""" out = [] for m in _NUMBER_RE.finditer(text): value = _parse_number(m.group("num")) if value is None: continue unit = m.group("unit") currency = m.group("currency") if _is_ignorable(value, unit, currency): continue out.append({ "raw": m.group(0).strip(), "value": value, "unit": (unit or "").strip() or None, "currency": (currency or "").strip() or None, "absolutes": _interpretations(value, unit), }) return out def _source_values(source_text: str, metrics: dict | None) -> list: """Every absolute value present in the source material. Source numbers get ALL unit interpretations (a source table saying '9,64,693' next to a 'crore' header may surface as a bare number in chunk text) plus the bare reading.""" values = [] for m in _NUMBER_RE.finditer(source_text or ""): v = _parse_number(m.group("num")) if v is None: continue unit = m.group("unit") values.extend(_interpretations(v, unit)) values.append(v) # bare reading always allowed on the source side # bare source numbers might be in implicit crore/million (Indian # statement tables state the unit once in the header, far from the # number) — allow those readings too so real matches aren't missed if not unit: values.append(v * 10_000_000) # crore values.append(v * 100_000) # lakh values.append(v * 1_000_000) # million if metrics: for metric in metrics.values(): if isinstance(metric, dict): if metric.get("value") is not None: values.append(float(metric["value"])) for alt in metric.get("alternatives", []): if isinstance(alt, dict) and alt.get("value") is not None: values.append(float(alt["value"])) elif isinstance(metric, (int, float)): values.append(float(metric)) return values def _matches(candidates: list, source_values: list) -> bool: for c in candidates: for s in source_values: if s == 0: if c == 0: return True continue if abs(c - s) / abs(s) <= RELATIVE_TOLERANCE: return True return False def verify_answer(answer: str, source_text: str, metrics: dict | None = None) -> dict: """Check every numeric claim in `answer` against the source material. Returns: {"all_verified": bool, "checked": int, "unverified": [{"raw", "value", "unit"}, ...], "note": str} """ claims = extract_numbers(answer) if not claims: return { "all_verified": True, "checked": 0, "unverified": [], "note": "No numeric claims to verify." } sources = _source_values(source_text, metrics) unverified = [] for claim in claims: if not _matches(claim["absolutes"], sources): unverified.append({ "raw": claim["raw"], "value": claim["value"], "unit": claim["unit"], }) all_verified = not unverified return { "all_verified": all_verified, "checked": len(claims), "unverified": unverified, "note": ( "Every number in this answer was found in the source filing." if all_verified else f"{len(unverified)} of {len(claims)} numbers could NOT be traced " f"to the source filing — treat them as unreliable." ) } if __name__ == "__main__": source = ( "The Consolidated total revenue from operations for the year was " "₹ 9,64,693 crore, compared to ₹ 9,01,064 crore in the previous year. " "Profit after tax was ₹ 60,812 crore. Attrition was 12.5%." ) metrics = { "revenue": {"value": 9_64_693 * 10_000_000, "confidence": "high"}, "profit_after_tax": {"value": 60_812 * 10_000_000, "confidence": "high"}, } good = "Revenue was ₹9,64,693 crore and PAT was ₹60,812 crore. Attrition stood at 12.5%." bad = "Revenue was ₹9,64,693 crore. The company also spent ₹5,000 crore on R&D and grew 45%." def safe(obj): # Windows cp1252 consoles choke on the ₹ glyph return str(obj).encode("ascii", "replace").decode("ascii") print("GOOD:", safe(verify_answer(good, source, metrics))) print("BAD:", safe(verify_answer(bad, source, metrics)))