verify.py

"""Numeric verification: every figure in the answer must be traceable to a
number that actually appeared in a tool result this turn.

We collect every numeric value from tool outputs, then parse numeric claims
out of the answer text and try to match each against the evidence set across
scale variants (raw, thousands, millions, billions, trillions, percent).
Derived figures (growth rates, margins, differences, ratios of evidence
numbers) are also accepted. Unmatched figures are flagged.
"""

import itertools
import re

TOLERANCE = 0.015  # 1.5% — covers rounding like "$416B" for 416.161B

SCALES = {
    "": 1.0,
    "k": 1e3, "thousand": 1e3,
    "m": 1e6, "million": 1e6, "mn": 1e6,
    "b": 1e9, "billion": 1e9, "bn": 1e9,
    "t": 1e12, "trillion": 1e12, "tn": 1e12,
    # Indian scales: answers about Indian companies use lakh (1e5) and crore (1e7)
    "lakh": 1e5, "lac": 1e5,
    "crore": 1e7, "cr": 1e7,
}

# The number body accepts both Western (1,234,567) and Indian (12,34,567)
# comma grouping — commas are stripped before parsing, so any digit-comma run
# works. The currency prefix covers $ and ₹.
CLAIM_RE = re.compile(
    r"(?<![\w.])([\$₹]?)(\d(?:[\d,]*\d)?(?:\.\d+)?)\s*"
    r"(trillion|billion|million|thousand|crore|lakh|lac|tn|bn|mn|cr|[tbmk])?(?![A-Za-z])\s*(%?)",
    re.IGNORECASE,
)

# Numbers that are almost never financial claims (years, small counts)
def _is_trivial(value: float, has_unit: bool) -> bool:
    if has_unit:
        return False
    if 1900 <= value <= 2100 and value == int(value):  # years
        return True
    return abs(value) < 20 and value == int(value)     # small ordinals/counts


def collect_evidence(obj, out: set[float] | None = None) -> set[float]:
    """Recursively pull every number out of a tool result."""
    if out is None:
        out = set()
    if isinstance(obj, bool):
        return out
    if isinstance(obj, (int, float)):
        out.add(float(obj))
    elif isinstance(obj, dict):
        for value in obj.values():
            collect_evidence(value, out)
    elif isinstance(obj, (list, tuple)):
        for value in obj:
            collect_evidence(value, out)
    elif isinstance(obj, str):
        for match in CLAIM_RE.finditer(obj):
            number = float(match.group(2).replace(",", ""))
            scale = SCALES.get((match.group(3) or "").lower(), 1.0)
            out.add(number * scale)
    return out


def _matches(claim: float, evidence: float) -> bool:
    if evidence == 0:
        return abs(claim) < 1e-9
    return abs(claim - evidence) / abs(evidence) <= TOLERANCE


def _match_any(value: float, evidence: set[float]) -> bool:
    # Scales in both directions: filings often state raw dollars while answers
    # say "$X million", and vice versa.
    scales = (1.0, 1e3, 1e6, 1e9, 1e12, 1e-3, 1e-6, 1e-9, 1e-12, 1e2, 1e-2)
    for ev in evidence:
        for scale in scales:
            if _matches(value * scale, ev):
                return True
    return False


def _match_derived(value: float, evidence: list[float]) -> bool:
    """Accept figures derivable from evidence pairs: growth %, margins, deltas."""
    sample = evidence[:120]  # bound the O(n^2) work
    for a, b in itertools.permutations(sample, 2):
        if b == 0:
            continue
        ratio = a / b
        for candidate in (
            ratio * 100.0,            # margin / share expressed in %
            (ratio - 1.0) * 100.0,    # growth rate in %
            a - b,                    # absolute delta
        ):
            if _matches(value, candidate):
                return True
    return False


def verify_answer(answer: str, tool_results: list[dict]) -> dict:
    evidence = set()
    for result in tool_results:
        collect_evidence(result, evidence)
    evidence_list = sorted(evidence, key=abs, reverse=True)

    checked = 0
    unverified: list[str] = []
    seen: set[str] = set()

    for match in CLAIM_RE.finditer(answer):
        raw = match.group(0).strip()
        number = float(match.group(2).replace(",", ""))
        unit = (match.group(3) or "").lower()
        is_pct = match.group(4) == "%"
        has_unit = bool(match.group(1) or unit or is_pct)

        if _is_trivial(number, has_unit) or raw in seen:
            continue
        seen.add(raw)
        checked += 1

        value = number * SCALES.get(unit, 1.0)
        ok = _match_any(value, evidence)
        if not ok and (is_pct or not has_unit):
            ok = _match_derived(number, evidence_list)
        if not ok:
            unverified.append(raw)

    return {
        "figures_checked": checked,
        "figures_verified": checked - len(unverified),
        "unverified": unverified,
        "evidence_numbers": len(evidence),
    }