research/verify_loop.py

"""External verification loop for the FSI suit (harness doctrine #5).

Big-tech basis (docs/harness_research.md): LLMs cannot self-correct with
intrinsic critique (arXiv 2310.01798); small models need STRONG EXTERNAL
verifiers (arXiv 2404.09931); CRITIC makes tools the critic (2305.11738);
Chain-of-Verification = draft -> verify -> revise (2309.09308).

Flow (deterministic suit logic; the head never grades itself):
  1. DRAFT: constrained analyst verdict on the claim (or rule spine first).
  2. VERIFY: plan checkable value probes (numbers/years/times/quoted values)
     from the claim; for each, retrieve the record evidence and run the
     deterministic spine (research/verify.py).
  3. REVISE: if the spine resolves (supports/refutes), the spine verdict
     WINS (it cannot hallucinate); if unresolved (no-values/partial), keep the
     draft verdict but do not raise confidence; if any check contradicts the
     draft, downgrade to LOW and flag the discrepancy.
  4. TRACE: full chain-of-custody (checks, sources, decisions).

Usage:
  from research.verify_loop import verify_case, plan_checks, run_checks
"""
import re

from research.provenance import evaluate_source_policy
from research.verify import deterministic_verdict

_QUOTE = re.compile(r"[\"']([^\"']{4,60})[\"']")
_NUM = re.compile(r"\b\d+(?:,\d{3})*(?:\.\d+)?%?\b")
_YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
_TIME = re.compile(r"\d{1,2}:\d{2}")  # no trailing \b: "9:30am" has none


def plan_checks(claim):
    """Extract checkable value probes from a claim (deterministic)."""
    checks = []
    seen = set()
    for m in _QUOTE.finditer(claim):
        q = m.group(1).strip()
        if q.lower() not in seen:
            seen.add(q.lower())
            checks.append({"kind": "quote", "value": q})
    # times first so their digits are not double-counted as numbers
    for m in _TIME.finditer(claim):
        v = m.group(0)
        if v.lower() not in seen:
            seen.add(v.lower())
            checks.append({"kind": "time", "value": v})
    rest = _TIME.sub(" ", claim)
    for pat, kind in ((_NUM, "number"), (_YEAR, "year")):
        for m in pat.finditer(rest):
            v = m.group(0)
            if v.lower() not in seen:
                seen.add(v.lower())
                checks.append({"kind": kind, "value": v})
    return checks[:8]


def _retrieval_bundle(raw):
    """Normalize legacy text retrieval and traceable evidence bundles."""
    if isinstance(raw, dict):
        evidence = raw.get("evidence", raw.get("text", ""))
        sources = raw.get("sources", [])
        relation = str(raw.get("claim_relation", "")).strip().lower()
        return str(evidence or ""), list(sources) if isinstance(sources, list) else [], relation
    return str(raw or ""), [], ""


def run_checks(checks, retrieve, spine=deterministic_verdict,
               require_source_policy=False):
    """Run each probe: retrieve evidence for the value, deterministic compare.

    retrieve(value) -> text (legacy) or an evidence bundle:
      {"evidence": str, "sources": [...], "claim_relation": supports|refutes}.
    When ``require_source_policy`` is true, a bundle must pass SOP 09 and its
    claim relation must agree with the deterministic value check. Otherwise it
    is a lead, not verified evidence.
    Returns list of {kind, value, evidence, verdict, kind_of_spine, explain}.
    """
    results = []
    for c in checks:
        evidence, sources, relation = _retrieval_bundle(retrieve(c["value"]))
        if not evidence:
            results.append({**c, "evidence": "", "verdict": "not enough information",
                            "kind": "no-evidence", "explain": "no record retrieved"})
            continue
        doc = f"Claim: {c['value']}\nEvidence: {evidence}"
        r = spine(doc)
        result = {**c, "evidence": evidence[:200], "value_verdict": r["verdict"],
                  "verdict": r["verdict"], "kind": r["kind"],
                  "explain": r["explain"]}
        if require_source_policy:
            policy = evaluate_source_policy(sources)
            result["source_policy"] = policy
            result["source_ids"] = [card["source_id"] for card in policy["sources"]]
            if not policy["verified"]:
                result.update(verdict="not enough information", kind="source-policy-failed",
                              explain=policy["reason"])
            elif relation not in ("supports", "refutes"):
                result.update(verdict="not enough information", kind="source-relation-missing",
                              explain="verified source bundle lacks a checked claim relation")
            elif relation != r["verdict"]:
                result.update(verdict="not enough information", kind="source-relation-conflict",
                              explain="claim relation conflicts with deterministic value check")
            else:
                result.update(verdict=relation, kind="source-policy-verified",
                              explain="source policy and value check agree")
        results.append(result)
    return results


def _classify(checks):
    """Aggregate spine results: supports / refutes / mixed / unresolved."""
    resolved = [c for c in checks if c["verdict"] in ("supports", "refutes")]
    if not resolved:
        return "unresolved", None
    supports = sum(1 for c in resolved if c["verdict"] == "supports")
    if supports == len(resolved):
        return "supports", None
    if supports == 0:
        return "refutes", None
    return "mixed", [c for c in resolved if c["verdict"] == "refutes"]


def verify_case(claim, draft_verdict, draft_conf, retrieve,
                spine=deterministic_verdict, require_source_policy=True):
    """Draft -> verify -> revise. Returns a decision dict with trace.

    claim: the claim under investigation (text).
    draft_verdict/conf: the constrained analyst verdict + confidence.
    retrieve(value) -> record evidence text or a traceable evidence bundle.
    require_source_policy: fail closed unless the bundle passes SOP 09.
    """
    checks = plan_checks(claim)
    results = run_checks(checks, retrieve, spine=spine,
                         require_source_policy=require_source_policy)
    status, refuting = _classify(results)

    if status == "supports":
        verdict, conf = "true", "HIGH"
        basis = "source-policy-verified" if require_source_policy else "rule-verified"
    elif status == "refutes":
        verdict, conf, basis = "false", "HIGH", "rule-refuted"
    elif status == "mixed":
        verdict, conf = ("not enough information", "LOW") if require_source_policy else ("low confidence", "LOW")
        basis = "rule-mixed:" + ",".join(c["value"] for c in refuting[:3])
    else:
        if require_source_policy:
            verdict, conf, basis = "not enough information", "LOW", "source-policy-incomplete"
            return {
                "verdict": verdict,
                "confidence": conf,
                "basis": basis,
                "checks": results,
                "sources": [sid for c in results for sid in c.get("source_ids", [])][:8],
                "abstained": True,
            }
        # unresolved: the spine cannot confirm; keep the draft but never raise
        verdict = draft_verdict or "not enough information"
        conf = draft_conf if draft_conf in ("LOW", "MEDIUM", "HIGH") else "LOW"
        basis = "unresolved-by-spine"
        if draft_conf == "HIGH":
            conf, basis = "MEDIUM", "draft-high-downgraded-unverified"

    return {
        "verdict": verdict,
        "confidence": conf,
        "basis": basis,
        "checks": results,
        "sources": [sid for c in results for sid in c.get("source_ids", [])][:8]
                   if require_source_policy else [c["evidence"] for c in results if c["evidence"]][:6],
        "abstained": verdict == "not enough information",
    }