backend/verifier.py

# backend/verifier.py
"""
Post-generation numeric verification.

The LLM prompt says "use only the context" — but a prompt is a request,
not a guarantee. This module CHECKS: every number that appears in a
generated answer must also exist in the source material (retrieved
chunks + extracted metrics). Numbers that can't be traced are flagged as
unverified — visibly, in the response — instead of being shipped as if
they were facts.

What counts as "the same number": financial texts express one value many
ways (₹9,64,693 crore vs 9646.93 billion vs "9.6 lakh crore"), so each
candidate is normalized to an absolute value across every plausible unit
interpretation, and matching allows small rounding tolerance (an answer
saying "$394.3B" against a source "394,328 million" should verify).

What is deliberately NOT verified: years (2020-2035), small enumerators
(section "1.", "top 3", risk score "7/10"), and page references — flagging
those would bury real signal in noise.
"""

import re

# rounding slack between an answer's number and a source number —
# "$394.3B" vs 394,328M is a 0.007% gap; 1% keeps us safe on display
# rounding without accepting genuinely different figures
RELATIVE_TOLERANCE = 0.01

UNIT_MULTIPLIERS = {
    "trillion": 1_000_000_000_000,
    "lakh crore": 1_000_000_000_000,
    "lakh crores": 1_000_000_000_000,
    "lakh cr": 1_000_000_000_000,
    "billion": 1_000_000_000,
    "million": 1_000_000,
    "thousand": 1_000,
    "crore": 10_000_000,
    "crores": 10_000_000,
    "cr": 10_000_000,
    "lakh": 100_000,
    "lakhs": 100_000,
    "lac": 100_000,
    "b": 1_000_000_000,
    "m": 1_000_000,
    "k": 1_000,
}

# number, optionally preceded by a currency marker and followed by a unit
# word/suffix or % — the shapes financial narration actually produces
# (includes format_money's own output: "₹60,812.0 Cr", "₹1.23 Lakh Cr")
_NUMBER_RE = re.compile(
    r"(?P<currency>[₹$€£]|Rs\.?\s?|INR\s?|USD\s?)?"
    r"(?P<num>\d[\d,]*(?:\.\d+)?)"
    r"\s*(?P<unit>lakh\s+crores?|lakh\s+cr\b|trillion|billion|million|thousand"
    r"|crores?\b|cr\b|lakhs?\b|lac\b|%|[BMKbmk]\b)?",
    re.UNICODE | re.IGNORECASE,
)


def _parse_number(num_str: str) -> float | None:
    try:
        return float(num_str.replace(",", ""))
    except ValueError:
        return None


def _interpretations(value: float, unit: str | None) -> list:
    """All absolute values this (number, unit) pair could mean. With an
    explicit unit there's one reading; without, the bare value is used
    as-is (source-side bare numbers get the same treatment, so matching
    still works when both sides omit units)."""
    if unit and unit != "%":
        key = re.sub(r"\s+", " ", unit.lower().strip())
        mult = UNIT_MULTIPLIERS.get(key)
        if mult:
            return [value * mult]
    return [value]


def _is_ignorable(value: float, unit: str | None, currency: str | None) -> bool:
    """Years, tiny enumerators, and unit-less small numbers are noise,
    not claims worth verifying."""
    if unit == "%":
        return False  # percentages are real claims
    if unit or currency:
        return False  # explicit money/unit context — always verify
    if 1900 <= value <= 2035 and value == int(value):
        return True  # a year
    if abs(value) < 100:
        return True  # list indices, "3 companies", risk score 7...
    return False


def extract_numbers(text: str) -> list:
    """All verifiable numeric claims in a text. Returns
    [{"raw": str, "value": float, "unit": str|None, "currency": str|None,
      "absolutes": [float, ...]}]"""
    out = []
    for m in _NUMBER_RE.finditer(text):
        value = _parse_number(m.group("num"))
        if value is None:
            continue
        unit = m.group("unit")
        currency = m.group("currency")
        if _is_ignorable(value, unit, currency):
            continue
        out.append({
            "raw": m.group(0).strip(),
            "value": value,
            "unit": (unit or "").strip() or None,
            "currency": (currency or "").strip() or None,
            "absolutes": _interpretations(value, unit),
        })
    return out


def _source_values(source_text: str, metrics: dict | None) -> list:
    """Every absolute value present in the source material. Source numbers
    get ALL unit interpretations (a source table saying '9,64,693' next to
    a 'crore' header may surface as a bare number in chunk text) plus the
    bare reading."""
    values = []

    for m in _NUMBER_RE.finditer(source_text or ""):
        v = _parse_number(m.group("num"))
        if v is None:
            continue
        unit = m.group("unit")
        values.extend(_interpretations(v, unit))
        values.append(v)  # bare reading always allowed on the source side
        # bare source numbers might be in implicit crore/million (Indian
        # statement tables state the unit once in the header, far from the
        # number) — allow those readings too so real matches aren't missed
        if not unit:
            values.append(v * 10_000_000)   # crore
            values.append(v * 100_000)      # lakh
            values.append(v * 1_000_000)    # million
    if metrics:
        for metric in metrics.values():
            if isinstance(metric, dict):
                if metric.get("value") is not None:
                    values.append(float(metric["value"]))
                for alt in metric.get("alternatives", []):
                    if isinstance(alt, dict) and alt.get("value") is not None:
                        values.append(float(alt["value"]))
            elif isinstance(metric, (int, float)):
                values.append(float(metric))

    return values


def _matches(candidates: list, source_values: list) -> bool:
    for c in candidates:
        for s in source_values:
            if s == 0:
                if c == 0:
                    return True
                continue
            if abs(c - s) / abs(s) <= RELATIVE_TOLERANCE:
                return True
    return False


def verify_answer(answer: str, source_text: str, metrics: dict | None = None) -> dict:
    """Check every numeric claim in `answer` against the source material.

    Returns:
        {"all_verified": bool,
         "checked": int,
         "unverified": [{"raw", "value", "unit"}, ...],
         "note": str}
    """
    claims = extract_numbers(answer)
    if not claims:
        return {
            "all_verified": True,
            "checked": 0,
            "unverified": [],
            "note": "No numeric claims to verify."
        }

    sources = _source_values(source_text, metrics)

    unverified = []
    for claim in claims:
        if not _matches(claim["absolutes"], sources):
            unverified.append({
                "raw": claim["raw"],
                "value": claim["value"],
                "unit": claim["unit"],
            })

    all_verified = not unverified
    return {
        "all_verified": all_verified,
        "checked": len(claims),
        "unverified": unverified,
        "note": (
            "Every number in this answer was found in the source filing."
            if all_verified else
            f"{len(unverified)} of {len(claims)} numbers could NOT be traced "
            f"to the source filing — treat them as unreliable."
        )
    }


if __name__ == "__main__":
    source = (
        "The Consolidated total revenue from operations for the year was "
        "₹ 9,64,693 crore, compared to ₹ 9,01,064 crore in the previous year. "
        "Profit after tax was ₹ 60,812 crore. Attrition was 12.5%."
    )
    metrics = {
        "revenue": {"value": 9_64_693 * 10_000_000, "confidence": "high"},
        "profit_after_tax": {"value": 60_812 * 10_000_000, "confidence": "high"},
    }

    good = "Revenue was ₹9,64,693 crore and PAT was ₹60,812 crore. Attrition stood at 12.5%."
    bad = "Revenue was ₹9,64,693 crore. The company also spent ₹5,000 crore on R&D and grew 45%."

    def safe(obj):  # Windows cp1252 consoles choke on the ₹ glyph
        return str(obj).encode("ascii", "replace").decode("ascii")

    print("GOOD:", safe(verify_answer(good, source, metrics)))
    print("BAD:", safe(verify_answer(bad, source, metrics)))