"""Numeric verification: every figure in the answer must be traceable to a number that actually appeared in a tool result this turn. We collect every numeric value from tool outputs, then parse numeric claims out of the answer text and try to match each against the evidence set across scale variants (raw, thousands, millions, billions, trillions, percent). Derived figures (growth rates, margins, differences, ratios of evidence numbers) are also accepted. Unmatched figures are flagged. """ import itertools import re TOLERANCE = 0.015 # 1.5% — covers rounding like "$416B" for 416.161B SCALES = { "": 1.0, "k": 1e3, "thousand": 1e3, "m": 1e6, "million": 1e6, "mn": 1e6, "b": 1e9, "billion": 1e9, "bn": 1e9, "t": 1e12, "trillion": 1e12, "tn": 1e12, # Indian scales: answers about Indian companies use lakh (1e5) and crore (1e7) "lakh": 1e5, "lac": 1e5, "crore": 1e7, "cr": 1e7, } # The number body accepts both Western (1,234,567) and Indian (12,34,567) # comma grouping — commas are stripped before parsing, so any digit-comma run # works. The currency prefix covers $ and ₹. CLAIM_RE = re.compile( r"(? bool: if has_unit: return False if 1900 <= value <= 2100 and value == int(value): # years return True return abs(value) < 20 and value == int(value) # small ordinals/counts def collect_evidence(obj, out: set[float] | None = None) -> set[float]: """Recursively pull every number out of a tool result.""" if out is None: out = set() if isinstance(obj, bool): return out if isinstance(obj, (int, float)): out.add(float(obj)) elif isinstance(obj, dict): for value in obj.values(): collect_evidence(value, out) elif isinstance(obj, (list, tuple)): for value in obj: collect_evidence(value, out) elif isinstance(obj, str): for match in CLAIM_RE.finditer(obj): number = float(match.group(2).replace(",", "")) scale = SCALES.get((match.group(3) or "").lower(), 1.0) out.add(number * scale) return out def _matches(claim: float, evidence: float) -> bool: if evidence == 0: return abs(claim) < 1e-9 return abs(claim - evidence) / abs(evidence) <= TOLERANCE def _match_any(value: float, evidence: set[float]) -> bool: # Scales in both directions: filings often state raw dollars while answers # say "$X million", and vice versa. scales = (1.0, 1e3, 1e6, 1e9, 1e12, 1e-3, 1e-6, 1e-9, 1e-12, 1e2, 1e-2) for ev in evidence: for scale in scales: if _matches(value * scale, ev): return True return False def _match_derived(value: float, evidence: list[float]) -> bool: """Accept figures derivable from evidence pairs: growth %, margins, deltas.""" sample = evidence[:120] # bound the O(n^2) work for a, b in itertools.permutations(sample, 2): if b == 0: continue ratio = a / b for candidate in ( ratio * 100.0, # margin / share expressed in % (ratio - 1.0) * 100.0, # growth rate in % a - b, # absolute delta ): if _matches(value, candidate): return True return False def verify_answer(answer: str, tool_results: list[dict]) -> dict: evidence = set() for result in tool_results: collect_evidence(result, evidence) evidence_list = sorted(evidence, key=abs, reverse=True) checked = 0 unverified: list[str] = [] seen: set[str] = set() for match in CLAIM_RE.finditer(answer): raw = match.group(0).strip() number = float(match.group(2).replace(",", "")) unit = (match.group(3) or "").lower() is_pct = match.group(4) == "%" has_unit = bool(match.group(1) or unit or is_pct) if _is_trivial(number, has_unit) or raw in seen: continue seen.add(raw) checked += 1 value = number * SCALES.get(unit, 1.0) ok = _match_any(value, evidence) if not ok and (is_pct or not has_unit): ok = _match_derived(number, evidence_list) if not ok: unverified.append(raw) return { "figures_checked": checked, "figures_verified": checked - len(unverified), "unverified": unverified, "evidence_numbers": len(evidence), }