python/app.py

# -*- coding: utf-8 -*-
"""
ClaimCheck - an LLM answer verification gate.

DESIGN PHILOSOPHY (read this before trusting any number this tool prints)
========================================================================
We cannot detect every hallucination. Nobody can, and a tool that claims to
is lying to you.

But we do not have to. The observation this whole program is built on is:

    *Dangerous hallucinations are specific.*

A model that invents "the contract terminates on 2023-11-04 under Article 12,
with a penalty of 3,400,000 JPY" is far more harmful than one that says
"the contract may terminate at some point". And the specific one is *checkable
as a string*: numbers, dates, quotes, named entities and URLs are literal
tokens that either do or do not appear in (or follow arithmetically from) the
source context you gave the model.

So ClaimCheck deliberately verifies only what it can verify *deterministically*
and *locally*. Vague prose is declared out of scope, loudly.

That is why every result carries TWO independent numbers:

    grounding_score = (supported + derived) / verifiable_claims
    coverage        = sentences_that_produced_a_claim / all_sentences

grounding_score alone is a trap. An answer of pure vague hedging produces zero
claims and would score 1.0 on any naive metric. coverage is what stops you from
reading that as "fully verified". NEVER display one without the other.

OPERATING CONSTRAINTS (Hugging Face Spaces free tier: 2 vCPU / 16GB / ephemeral disk)
  * No torch / transformers / sklearn / spacy. Only gradio, huggingface_hub, pandas.
  * All verification is local, deterministic and pure-Python.
  * Remote enrichment (section L) is strictly optional and best-effort: if the
    token is missing, the endpoint is down, or the monthly inference credit
    (~$0.10) is exhausted, it returns None and the core gate is unaffected.
  * No exception may escape a public function. Ever. A crashed Space is a
    worse outcome than a wrong verdict.
  * Import time does no work beyond compiling regexes.
  * Verification is measured on every call and the latency is shown in the UI.
"""

from __future__ import annotations

import json
import math
import os
import re
import sys
import tempfile
import threading
import time
import traceback
import unicodedata
from bisect import bisect_left, bisect_right
from collections import Counter, deque
from decimal import Decimal, InvalidOperation, localcontext
from difflib import SequenceMatcher

import pandas as pd

try:  # gradio is required for the UI but the verification core must import without it
    import gradio as gr
except Exception:  # pragma: no cover - only hit in headless unit-test use
    gr = None


# =============================================================================
# (A) CONFIGURATION
# =============================================================================

def _env_str(name: str, default: str = "") -> str:
    """Guarantees: returns a stripped env value or the default, never raises."""
    try:
        v = os.environ.get(name)
        return default if v is None else str(v).strip()
    except Exception:
        return default


def _env_int(name: str, default: int, lo: int = 0, hi: int = 10_000_000) -> int:
    """Guarantees: returns an int clamped into [lo, hi]; bad input falls back to default."""
    try:
        v = int(float(_env_str(name, "") or default))
    except Exception:
        v = default
    return max(lo, min(hi, v))


def _env_float(name: str, default: float, lo: float = -1e12, hi: float = 1e12) -> float:
    """Guarantees: returns a float clamped into [lo, hi]; bad input falls back to default."""
    try:
        v = float(_env_str(name, "") or default)
    except Exception:
        v = default
    if v != v:  # NaN
        v = default
    return max(lo, min(hi, v))


def _env_bool(name: str, default: bool) -> bool:
    """Guarantees: returns a bool; accepts 1/true/yes/on (case-insensitive)."""
    raw = _env_str(name, "")
    if not raw:
        return default
    return raw.lower() in ("1", "true", "yes", "on", "y")


APP_NAME = "ClaimCheck"
APP_VERSION = "1.1.0"
START_TS = time.time()

HF_TOKEN = _env_str("HF_TOKEN") or _env_str("HUGGING_FACE_HUB_TOKEN")
LOG_CAPACITY = _env_int("LOG_CAPACITY", 500, 10, 200_000)
MAX_TEXT_CHARS = _env_int("MAX_TEXT_CHARS", 20_000, 200, 400_000)
NGRAM_LEAK_N = _env_int("NGRAM_LEAK_N", 8, 3, 64)
LEAK_THRESHOLD = _env_float("LEAK_THRESHOLD", 0.15, 0.0, 1.0)
PRICE_IN_PER_1K = _env_float("PRICE_IN_PER_1K", 0.0, 0.0, 1000.0)
PRICE_OUT_PER_1K = _env_float("PRICE_OUT_PER_1K", 0.0, 0.0, 1000.0)

# Observability privacy: answer bodies are NOT logged by default. Operators who
# need excerpts for debugging opt in explicitly with a positive prefix length.
STORE_ANSWER_PREFIX = _env_int("STORE_ANSWER_PREFIX", 0, 0, 2000)

# Output-safety tuning.
ENTROPY_THRESHOLD = _env_float("ENTROPY_THRESHOLD", 3.6, 1.0, 6.0)
ENTROPY_MIN_LEN = _env_int("ENTROPY_MIN_LEN", 24, 8, 256)

# Verifier tuning (all overridable per-request through policy_json / options).
DEFAULT_CONTRADICTION_REL = _env_float("CONTRADICTION_REL", 0.25, 0.0, 10.0)
DEFAULT_APPROX_RATIO = _env_float("APPROX_RATIO", 0.82, 0.30, 1.0)
DERIVE_MAX_TERMS = _env_int("DERIVE_MAX_TERMS", 3, 2, 3)
DERIVE_BUDGET_MS = _env_int("DERIVE_BUDGET_MS", 120, 5, 5000)
DERIVE_MAX_NUMBERS = _env_int("DERIVE_MAX_NUMBERS", 40, 4, 200)
# Hard ceiling on claims per answer. A pathological input must degrade
# gracefully (and say so) rather than pin a 2-vCPU box.
MAX_CLAIMS = _env_int("MAX_CLAIMS", 1500, 10, 50_000)
# Shared per-request budget for fuzzy (approximate) matching.
FUZZY_BUDGET_MS = _env_int("FUZZY_BUDGET_MS", 250, 5, 20_000)
# Similarity at or above this, but below approx_ratio, means "a passage like this
# exists but says something materially different" -> contradicted. A safe floor
# only because anchored multi-width matching scores real paraphrases accurately
# (0.87-0.93); with the old diluted windows this band was full of false positives.
QUOTE_CONTRADICTION_FLOOR = _env_float("QUOTE_CONTRADICTION_FLOOR", 0.65, 0.3, 1.0)

# Optional remote enrichment (section L).
ENRICH_ENABLED = _env_bool("ENRICH_ENABLED", bool(HF_TOKEN))
ENRICH_MODEL = _env_str("ENRICH_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
ENRICH_TIMEOUT_S = _env_float("ENRICH_TIMEOUT_S", 3.0, 0.2, 30.0)
ENRICH_MAX_CHARS = _env_int("ENRICH_MAX_CHARS", 900, 100, 4000)

UI_CONCURRENCY = _env_int("UI_CONCURRENCY", 4, 1, 32)
UI_QUEUE_SIZE = _env_int("UI_QUEUE_SIZE", 32, 1, 512)

STATUSES = ("supported", "derived", "approximate", "unsupported", "contradicted")
CLAIM_TYPES = ("NUMERIC", "DATE", "QUOTE", "ENTITY", "URL")
VERDICTS = ("pass", "annotate", "retry", "block")


# =============================================================================
# SHARED UTILITIES
# =============================================================================

def _err(exc: BaseException, where: str) -> dict:
    """Guarantees: converts any exception into a serialisable error envelope."""
    return {
        "ok": False,
        "error": {
            "type": type(exc).__name__,
            "message": str(exc)[:800],
            "where": where,
            "trace": traceback.format_exc(limit=4)[-1400:],
        },
    }


def guarded(fn):
    """Guarantees: the wrapped function returns a dict and never raises (rule 4)."""
    def _wrapped(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except Exception as exc:  # noqa: BLE001 - deliberate catch-all
            return _err(exc, fn.__name__)
    _wrapped.__name__ = getattr(fn, "__name__", "guarded")
    _wrapped.__doc__ = getattr(fn, "__doc__", "")
    _wrapped.__wrapped__ = fn
    return _wrapped


def clamp_text(s, limit: int = None) -> tuple:
    """Guarantees: returns (safe_str, was_truncated) with len(safe_str) <= limit."""
    limit = MAX_TEXT_CHARS if limit is None else limit
    if s is None:
        return "", False
    if not isinstance(s, str):
        try:
            s = str(s)
        except Exception:
            return "", False
    if len(s) > limit:
        return s[:limit], True
    return s, False


def estimate_tokens(s) -> int:
    """Guarantees: dependency-free token estimate (ASCII~4 chars/tok, CJK~1 char/tok)."""
    try:
        if not s:
            return 0
        if not isinstance(s, str):
            s = str(s)
        ascii_n = 0
        cjk_n = 0
        other_n = 0
        for ch in s:
            o = ord(ch)
            if o < 128:
                ascii_n += 1
            elif (
                0x3040 <= o <= 0x30FF          # kana
                or 0x4E00 <= o <= 0x9FFF       # CJK unified
                or 0x3400 <= o <= 0x4DBF       # CJK ext A
                or 0xAC00 <= o <= 0xD7AF       # hangul
                or 0xFF00 <= o <= 0xFFEF       # fullwidth forms
            ):
                cjk_n += 1
            else:
                other_n += 1
        return int(math.ceil(ascii_n / 4.0 + cjk_n * 1.0 + other_n / 2.0))
    except Exception:
        return 0


def shannon_entropy(s: str) -> float:
    """Guarantees: returns Shannon entropy in bits/char (0.0 for empty input), no deps."""
    try:
        if not s:
            return 0.0
        n = float(len(s))
        counts = Counter(s)
        acc = 0.0
        for c in counts.values():
            p = c / n
            acc -= p * math.log(p, 2)
        return acc
    except Exception:
        return 0.0


def luhn_ok(digits: str) -> bool:
    """Guarantees: True only if the digit string passes the Luhn mod-10 checksum."""
    try:
        ds = [int(c) for c in digits if c.isdigit()]
        if len(ds) < 13 or len(ds) > 19:
            return False
        total = 0
        parity = len(ds) % 2
        for i, d in enumerate(ds):
            if i % 2 == parity:
                d *= 2
                if d > 9:
                    d -= 9
            total += d
        return total % 10 == 0
    except Exception:
        return False


def _fkey(value) -> str:
    """Guarantees: a stable 9-significant-digit string key, so 0.1+0.2 and 0.3 collide."""
    try:
        f = float(value)
    except Exception:
        return ""
    if f != f or f in (float("inf"), float("-inf")):
        return ""
    if f == 0.0:
        return "0"
    s = "%.9g" % f
    if s in ("-0", "-0.0"):
        return "0"
    return s


def _safe_float(d) -> float:
    """Guarantees: a float, or NaN when the value cannot be converted; never raises."""
    try:
        return float(d)
    except Exception:
        return float("nan")


def _now_iso() -> str:
    """Guarantees: the current local time as a sortable ISO-8601 second-precision string."""
    return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime())


def _spans_overlap(a, b) -> bool:
    """Guarantees: True iff the two half-open character ranges share at least one position."""
    return not (a[1] <= b[0] or b[1] <= a[0])


class _SpanMask:
    """Guarantees: O(span length) overlap/containment tests instead of O(number of spans).

    Extraction used to ask "does this span hit any of the N spans I already
    took?" with a linear scan, which is quadratic once an answer contains
    hundreds of claims. A byte-per-character occupancy map makes both questions
    constant-ish time and keeps latency linear in the input size.
    """

    __slots__ = ("m",)

    def __init__(self, n: int):
        """Guarantees: an all-clear occupancy map of n characters."""
        self.m = bytearray(max(0, int(n)))

    def add(self, s: int, e: int) -> None:
        """Guarantees: marks [s, e) as occupied; out-of-order or negative ranges are ignored."""
        if e > s >= 0:
            self.m[s:e] = b"\x01" * (e - s)

    def add_all(self, spans) -> None:
        """Guarantees: marks every well-formed span in the iterable and skips malformed ones."""
        for sp in spans:
            try:
                self.add(int(sp[0]), int(sp[1]))
            except Exception:
                continue

    def hits(self, s: int, e: int) -> bool:
        """Guarantees: True iff the span overlaps anything already marked."""
        if e <= s:
            return False
        return self.m.find(1, max(0, s), max(0, e)) != -1

    def contains(self, s: int, e: int) -> bool:
        """Guarantees: True iff every character of the span is already marked."""
        if e <= s:
            return False
        chunk = self.m[max(0, s):max(0, e)]
        return len(chunk) == (e - s) and chunk.count(1) == len(chunk)


_LCS_MOD = (1 << 61) - 1
_LCS_BASE = 257


def _rk_positions(s: str, k: int):
    """Guarantees: {rolling_hash: first_position} for every k-gram of s, in O(len(s))."""
    n = len(s)
    out = {}
    if k <= 0 or k > n:
        return out
    h = 0
    for i in range(k):
        h = (h * _LCS_BASE + ord(s[i])) % _LCS_MOD
    out[h] = 0
    power = pow(_LCS_BASE, k, _LCS_MOD)
    for i in range(k, n):
        h = (h * _LCS_BASE + ord(s[i]) - ord(s[i - k]) * power) % _LCS_MOD
        if h not in out:
            out[h] = i - k + 1
    return out


def _common_substring_of_length(a: str, b: str, k: int):
    """Guarantees: an actual shared substring of length k (hash-verified), or None."""
    ha = _rk_positions(a, k)
    if not ha:
        return None
    hb = _rk_positions(b, k)
    if not hb:
        return None
    if len(ha) > len(hb):
        ha, hb = hb, ha
        a, b = b, a
    for h, pos in ha.items():
        j = hb.get(h)
        if j is None:
            continue
        cand = a[pos:pos + k]
        if cand == b[j:j + k]:   # guard against hash collisions
            return cand
    return None


def _longest_common_fragment(a: str, b: str, cap: int = 3000) -> str:
    """Guarantees: the longest shared substring, in O(n log n) and bounded by `cap` chars.

    difflib.SequenceMatcher.find_longest_match was the original implementation
    and it degenerates to O(n^2) on repetitive text - a 20k answer of one
    repeated character took >12s, which is a denial-of-service on a 2-vCPU box.
    Binary search over a Rabin-Karp hash is exact and predictable instead.
    """
    try:
        a = (a or "")[:cap]
        b = (b or "")[:cap]
        if not a or not b:
            return ""
        lo, hi, best = 1, min(len(a), len(b)), ""
        while lo <= hi:
            mid = (lo + hi) // 2
            found = _common_substring_of_length(a, b, mid)
            if found:
                best = found
                lo = mid + 1
            else:
                hi = mid - 1
        return best
    except Exception:
        return ""


# =============================================================================
# (C) NORMALIZER
# =============================================================================

_PUNCT_MAP = {
    "，": ",", "、": ",", "､": ",",
    "．": ".", "。": ".", "｡": ".",
    "：": ":", "；": ";",
    "（": "(", "）": ")", "［": "[", "］": "]", "｛": "{", "｝": "}",
    "「": '"', "」": '"', "『": '"', "』": '"',
    "“": '"', "”": '"', "„": '"', "‟": '"',
    "‘": "'", "’": "'", "‚": "'",
    # NOTE: "ー" (U+30FC, KATAKANA-HIRAGANA PROLONGED SOUND MARK) is deliberately
    # NOT mapped to "-". It is a letter, not punctuation: folding it turned
    # データセンター into デ-タセンタ- and broke Japanese quote/entity matching.
    "－": "-", "−": "-", "—": "-", "–": "-", "―": "-", "‐": "-", "‑": "-",
    "％": "%", "＄": "$", "￥": "¥", "／": "/", "＼": "\\", "＃": "#", "＆": "&",
    "！": "!", "？": "?", "〜": "~", "～": "~", "・": " ",
}
_PUNCT_TABLE = {ord(k): v for k, v in _PUNCT_MAP.items()}
_WS_RE = re.compile(r"\s+")


def norm_text(s) -> str:
    """Guarantees: NFKC + lowercase + collapsed whitespace + unified punctuation; never raises."""
    try:
        if s is None:
            return ""
        if not isinstance(s, str):
            s = str(s)
        s = s.translate(_PUNCT_TABLE)
        s = unicodedata.normalize("NFKC", s)
        s = s.lower()
        s = _WS_RE.sub(" ", s)
        return s.strip()
    except Exception:
        return ""


_MAG_FACTORS = {
    "千": Decimal(10) ** 3, "万": Decimal(10) ** 4, "億": Decimal(10) ** 8, "兆": Decimal(10) ** 12,
    "k": Decimal(10) ** 3, "K": Decimal(10) ** 3,
    "m": Decimal(10) ** 6, "M": Decimal(10) ** 6,
    "b": Decimal(10) ** 9, "B": Decimal(10) ** 9,
}

_NUM_CLEAN_RE = re.compile(r"[,\s_]")
_CUR_STRIP_RE = re.compile(r"^(?:us\$|usd|jpy|eur|gbp|krw|cny|[$¥€£₩])|(?:usd|jpy|eur|gbp|円|ドル|ユーロ)$")


def norm_number(s):
    """Guarantees: returns a Decimal for any recognised numeric literal, else None.

    Handles thousands separators, whitespace, currency symbols, fullwidth digits,
    percent signs, exponent notation and magnitude suffixes (千/万/億/兆/k/M/B).
    """
    try:
        if s is None:
            return None
        if isinstance(s, (int, float, Decimal)):
            return Decimal(str(s))
        s = str(s).strip()
        if not s:
            return None
        s = unicodedata.normalize("NFKC", s)
        s = s.replace("−", "-").replace("－", "-").replace("，", ",")
        sign = Decimal(1)
        # Japanese accounting notation for negatives: ▲1,200 / △1,200 / (1,200)
        if s[:1] in ("▲", "△"):
            sign = Decimal(-1)
            s = s[1:].strip()
        if s.startswith("(") and s.endswith(")"):
            sign = Decimal(-1)
            s = s[1:-1].strip()
        if s[:1] in ("+", "-"):
            if s[0] == "-":
                sign = sign * Decimal(-1)
            s = s[1:].strip()

        low = s.lower()
        low = _CUR_STRIP_RE.sub("", low).strip()
        s = low

        percent = False
        for suf in ("%", "パーセント", "percent", "pct"):
            if s.endswith(suf):
                s = s[: -len(suf)].strip()
                percent = True
                break
        if s.endswith("割"):  # 3割 == 30%
            s = s[:-1].strip()
            base = _decimal_of(_NUM_CLEAN_RE.sub("", s))
            return None if base is None else sign * base * Decimal(10)

        factor = Decimal(1)
        changed = True
        while changed and s:
            changed = False
            for suf, fac in _MAG_FACTORS.items():
                if len(suf) == 1 and s.endswith(suf):
                    # Only treat a trailing latin letter as a magnitude when the
                    # remainder is numeric ("3.5k" yes, "ok" no).
                    rest = s[:-1].strip()
                    if rest and any(c.isdigit() for c in rest):
                        factor = factor * fac
                        s = rest
                        changed = True
                        break
        # Strip any trailing non-numeric unit ("件", "人", "kg", ...).
        s = re.sub(r"[^0-9.eE+\-]+$", "", s).strip()
        s = _NUM_CLEAN_RE.sub("", s)
        base = _decimal_of(s)
        if base is None:
            return None
        val = sign * base * factor
        if percent:
            # We keep the *displayed* magnitude (15% -> 15) and let the verifier
            # separately try the 15 <-> 0.15 equivalence. Silently dividing here
            # would make "15%" fail to match a literal "15" in the context.
            return val
        return val
    except Exception:
        return None


def _decimal_of(s):
    """Guarantees: a finite Decimal parsed at 28 digits of precision, or None; never raises."""
    try:
        if not s:
            return None
        with localcontext() as ctx:
            ctx.prec = 28
            d = Decimal(s)
        if d.is_nan() or d.is_infinite():
            return None
        return d
    except (InvalidOperation, ValueError, ArithmeticError):
        return None
    except Exception:
        return None


_MONTHS = {
    "jan": 1, "january": 1, "feb": 2, "february": 2, "mar": 3, "march": 3,
    "apr": 4, "april": 4, "may": 5, "jun": 6, "june": 6, "jul": 7, "july": 7,
    "aug": 8, "august": 8, "sep": 9, "sept": 9, "september": 9, "oct": 10,
    "october": 10, "nov": 11, "november": 11, "dec": 12, "december": 12,
}
# Japanese era -> (offset such that year = offset + era_year). 令和1 == 2019.
_ERAS = {"令和": 2018, "reiwa": 2018, "平成": 1988, "heisei": 1988,
         "昭和": 1925, "showa": 1925, "大正": 1911, "taisho": 1911,
         "明治": 1867, "meiji": 1867}

_D_ISO = re.compile(r"^(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})$")
_D_ISO_YM = re.compile(r"^(\d{4})[-/.](\d{1,2})$")
_D_JP = re.compile(r"^(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日?$")
_D_JP_YM = re.compile(r"^(\d{4})\s*年\s*(\d{1,2})\s*月$")
_D_JP_Y = re.compile(r"^(\d{4})\s*年(?:度)?$")
_D_ERA = re.compile(r"^(令和|平成|昭和|大正|明治)\s*(\d{1,2}|元)\s*年(?:\s*(\d{1,2})\s*月(?:\s*(\d{1,2})\s*日?)?)?$")
_D_EN_MDY = re.compile(r"^([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})$")
_D_EN_DMY = re.compile(r"^(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(\d{4})$")
_D_EN_MY = re.compile(r"^([A-Za-z]{3,9})\.?,?\s+(\d{4})$")
_D_Y = re.compile(r"^(?:fy)?(\d{4})$", re.I)


def _mk_date(y, m=None, d=None):
    """Guarantees: an ISO date string only when every supplied part is in range, else None."""
    try:
        y = int(y)
        if y < 1000 or y > 3000:
            return None
        if m is None:
            return "%04d" % y
        m = int(m)
        if m < 1 or m > 12:
            return None
        if d is None:
            return "%04d-%02d" % (y, m)
        d = int(d)
        if d < 1 or d > 31:
            return None
        return "%04d-%02d-%02d" % (y, m, d)
    except Exception:
        return None


def norm_date(s):
    """Guarantees: returns an ISO string (YYYY / YYYY-MM / YYYY-MM-DD) or None.

    Absorbs separator differences, Japanese era years (令和/平成/昭和/大正/明治)
    and English month names. Ambiguous or out-of-range values return None rather
    than a guess - a wrong normalisation is worse than an unverified claim.
    """
    try:
        if not s:
            return None
        t = unicodedata.normalize("NFKC", str(s)).strip()
        t = t.replace("−", "-").replace("．", ".")
        t = re.sub(r"\s+", " ", t).strip().rstrip(".,")
        if not t:
            return None

        m = _D_JP.match(t)
        if m:
            return _mk_date(m.group(1), m.group(2), m.group(3))
        m = _D_JP_YM.match(t)
        if m:
            return _mk_date(m.group(1), m.group(2))
        m = _D_ERA.match(t)
        if m:
            era, ey = m.group(1), m.group(2)
            ey = 1 if ey == "元" else int(ey)
            year = _ERAS.get(era, 0) + ey
            return _mk_date(year, m.group(3), m.group(4))
        m = _D_ISO.match(t)
        if m:
            return _mk_date(m.group(1), m.group(2), m.group(3))
        m = _D_ISO_YM.match(t)
        if m:
            return _mk_date(m.group(1), m.group(2))
        m = _D_EN_MDY.match(t)
        if m and m.group(1).lower().rstrip(".") in _MONTHS:
            return _mk_date(m.group(3), _MONTHS[m.group(1).lower().rstrip(".")], m.group(2))
        m = _D_EN_DMY.match(t)
        if m and m.group(2).lower().rstrip(".") in _MONTHS:
            return _mk_date(m.group(3), _MONTHS[m.group(2).lower().rstrip(".")], m.group(1))
        m = _D_EN_MY.match(t)
        if m and m.group(1).lower().rstrip(".") in _MONTHS:
            return _mk_date(m.group(2), _MONTHS[m.group(1).lower().rstrip(".")])
        m = _D_JP_Y.match(t)
        if m:
            return _mk_date(m.group(1))
        m = _D_Y.match(t)
        if m:
            return _mk_date(m.group(1))
        return None
    except Exception:
        return None


# =============================================================================
# (B) CLAIM EXTRACTOR
# =============================================================================
# Everything below is compiled once at import. Nothing else happens at import
# time (constraint 5: no heavy startup work on a free CPU Basic Space).

_URL_RE = re.compile(
    r"(?:https?://|ftp://|www\.)[^\s<>\"'()\[\]{}、。，！？]+|"
    r"\b[a-z0-9][a-z0-9\-]{0,62}\.(?:com|org|net|io|ai|co|jp|dev|app|gov|edu|info|me|cloud|xyz)"
    r"(?:/[^\s<>\"'()\[\]{}、。，！？]*)?",
    re.I,
)

_NUM_CORE = (
    r"(?:[0-9]{1,3}(?:,[0-9]{3})+(?:\.[0-9]+)?"
    r"|[0-9]+(?:\.[0-9]+)?"
    r"|\.[0-9]+)"
)
_CUR_PAT = r"(?:US\$|USD|JPY|EUR|GBP|KRW|CNY|[$¥€£₩])"
_MAG_PAT = r"(?:千|万|億|兆|[kKmMbB](?![A-Za-z0-9]))"
_PCT_PAT = r"(?:%|パーセント|ポイント|割)"
_UNIT_PAT = (
    r"(?:円|ドル|ユーロ|元|ウォン|件|人|名|社|店|台|個|本|枚|冊|回|倍|点|位|票|室|席|品|語|"
    r"文字|字|行|列|頁|ページ|年度|年間|年|ヶ月|か月|カ月|ヵ月|箇月|週間|日間|時間|分間|"
    r"秒|分|時|日|歳|才|km|cm|mm|kg|mg|GB|MB|TB|KB|PB|Mbps|Gbps|bps|kHz|MHz|GHz|Hz|"
    r"kW|MW|W|mL|L|m2|㎡|平方メートル|立方メートル|℃|°C|°F|m|g|t|V|A)"
)
NUM_RE = re.compile(
    r"(?<![0-9A-Za-z._\-])"
    r"(?P<sign>[-+▲△])?"
    # The whitespace is allowed only AFTER a currency symbol. Leaving it outside
    # the optional group let the match begin on a newline ("\n2"), which both
    # broke list-marker suppression and pushed leading whitespace into the span.
    r"(?:(?P<cur>" + _CUR_PAT + r")\s{0,2})?"
    r"(?P<num>" + _NUM_CORE + r")"
    r"(?P<exp>[eE][-+]?[0-9]{1,3})?"
    r"\s?(?P<mag>" + _MAG_PAT + r")?"
    r"\s?(?P<pct>" + _PCT_PAT + r")?"
    # The letter-lookahead belongs INSIDE the optional group. Outside it, the
    # whole match failed whenever a number was followed by letters, which
    # silently dropped every ordinal ("25th anniversary") from extraction.
    r"(?:\s?(?P<unit>" + _UNIT_PAT + r")(?![A-Za-z]))?"
)

_DATE_PATTERNS = [
    re.compile(r"(?:令和|平成|昭和|大正|明治)\s*(?:\d{1,2}|元)\s*年(?:\s*\d{1,2}\s*月(?:\s*\d{1,2}\s*日)?)?"),
    re.compile(r"\d{4}\s*年\s*\d{1,2}\s*月\s*\d{1,2}\s*日"),
    re.compile(r"\d{4}\s*年\s*\d{1,2}\s*月"),
    # NOT \b: in "契約は2024-01-05に" both the CJK char and the digit are word
    # characters, so \b never fires and the date was silently invisible.
    re.compile(r"(?<![0-9A-Za-z])\d{4}[-/.]\d{1,2}[-/.]\d{1,2}(?![0-9A-Za-z])"),
    re.compile(r"(?<![0-9A-Za-z])\d{4}[-/]\d{1,2}(?![-/.\d])"),
    re.compile(
        r"(?<![0-9A-Za-z])(?:Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August"
        r"|Sep|Sept|September|Oct|October|Nov|November|Dec|December)\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}(?![0-9A-Za-z])",
        re.I,
    ),
    re.compile(
        r"(?<![0-9A-Za-z])\d{1,2}(?:st|nd|rd|th)?\s+(?:Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June"
        r"|Jul|July|Aug|August|Sep|Sept|September|Oct|October|Nov|November|Dec|December)\.?,?\s+\d{4}(?![0-9A-Za-z])",
        re.I,
    ),
    re.compile(
        r"(?<![0-9A-Za-z])(?:Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August"
        r"|Sep|Sept|September|Oct|October|Nov|November|Dec|December)\.?,?\s+\d{4}(?![0-9A-Za-z])",
        re.I,
    ),
    re.compile(r"(?<![0-9A-Za-z])(?:FY)\s?\d{4}(?![0-9A-Za-z])", re.I),
    re.compile(r"\d{4}\s*年度"),
    re.compile(r"(?<![0-9])\d{4}\s*年(?![0-9度])"),
]

# 相対表現は「別扱い」。検証できないので claim にせず、件数だけ数えて
# coverage を過大評価しないようにする。
_RELATIVE_DATE_RE = re.compile(
    r"(?:昨年|一昨年|来年|再来年|今年|本年|先月|今月|来月|先週|今週|来週|昨日|本日|今日|明日|明後日|"
    r"現在|最近|近年|直近|先般|過去\d+年|今後\d+年|"
    r"\byesterday\b|\btoday\b|\btomorrow\b|\blast\s+(?:year|month|week|quarter)\b|"
    r"\bnext\s+(?:year|month|week|quarter)\b|\bthis\s+(?:year|month|week|quarter)\b|"
    r"\brecently\b|\bcurrently\b|\bnowadays\b)",
    re.I,
)

# QUOTE: bracketed spans, and text introduced by an attribution marker.
_QUOTE_BRACKET_RE = re.compile(r"[「『“\"]([^「」『』“”\"\n]{4,180})[」』”\"]")
_QUOTE_MARKER_RE = re.compile(
    r"(?:によると|によれば|には|と記載(?:されて)?(?:い)?(?:ます|る)?|と明記|と述べ|と書かれ|と報告|と発表|"
    r"states?\s+that|according\s+to|reported\s+that|said\s+that|notes?\s+that|writes?\s+that|"
    r"claims?\s+that|indicates?\s+that)",
    re.I,
)

_ENT_CAPWORDS_RE = re.compile(r"\b[A-Z][A-Za-z0-9&'’.\-]{1,30}(?:\s+(?:of|the|and|for|de|von|van)\s+|\s+)(?:[A-Z][A-Za-z0-9&'’.\-]{1,30})(?:\s+[A-Z][A-Za-z0-9&'’.\-]{1,30}){0,3}")
_ENT_MODEL_RE = re.compile(r"\b(?=[A-Za-z0-9\-]{4,24}\b)(?=[^\s]*\d)[A-Z]{1,6}[A-Za-z]{0,4}[-_]?\d{2,6}[A-Za-z0-9\-]{0,8}\b")
_ENT_ARTICLE_RE = re.compile(r"(?:第\s*\d+\s*条(?:\s*の\s*\d+)?(?:\s*第\s*\d+\s*項)?|Article\s+\d+(?:\.\d+)?|Sections?\s+\d+(?:\.\d+)?|§\s?\d+(?:\.\d+)?|ISO\s?\d{3,5}(?::\d{4})?|RFC\s?\d{3,5})", re.I)
_ENT_ACRONYM_RE = re.compile(r"\b[A-Z]{3,8}\b")

# ENTITY 検証は最も偽陽性を生みやすい。一般語・機能語・よくある固有名詞風の
# 語をここで抑制する。運用側は deploy.md の指示どおり、まず ENTITY を切って
# NUMERIC / DATE から始めるとよい。
# ISO currency codes are generic tokens, not factual entities. Left in, the
# acronym extractor claimed "USD" and its span blocked the NUMERIC match that
# begins at the currency code, so "USD 45,000" lost its number entirely.
_CURRENCY_CODES = set("""
usd jpy eur gbp krw cny chf aud cad hkd sgd inr brl rub sek nok dkk nzd mxn zar
thb twd php idr vnd pln czk huf try ils aed sar qar myr clp cop ars
""".split())

_ENTITY_STOPWORDS = set("""
the a an and or but if then than that this these those there here it its it's is are was were be been
i you we they he she him her his our your their us them my me not no yes of in on at to for from by with
as into over under about after before between during without within across per via such same other another
however therefore moreover furthermore additionally meanwhile although because since while when where which who whom
note notes please thank thanks hello hi ok okay yes no true false null none
january february march april may june july august september october november december
monday tuesday wednesday thursday friday saturday sunday
summary conclusion introduction overview background result results method methods discussion reference references
answer question context example examples figure table appendix section chapter page pages
company service product system data user users client customer report document file page site page
ai llm api url json html http https pdf csv xml sql cpu gpu ram ssd usb faq ceo cfo cto url uri id ids
note caution warning important warning info tip
""".split()) | _CURRENCY_CODES

_ABBREVIATIONS = {
    "mr", "mrs", "ms", "dr", "prof", "st", "vs", "etc", "e.g", "i.e", "fig", "no",
    "inc", "ltd", "co", "corp", "jr", "sr", "approx", "al", "u.s", "u.k", "cf",
    "vol", "pp", "est", "dept", "univ", "jan", "feb", "mar", "apr", "jun", "jul",
    "aug", "sep", "sept", "oct", "nov", "dec",
}

_SENT_TERMINATORS = "。．！？!?\n"


def split_sentences(text: str, protected=()) -> list:
    """Guarantees: returns [(start, end, text)] covering every non-blank sentence, offsets exact.

    Terminators inside `protected` regions (URLs) are ignored, and a '.' between
    digits or after a known abbreviation does not split (so "3.14" and "e.g."
    stay whole).
    """
    out = []
    try:
        if not text:
            return out
        n = len(text)
        start = 0
        i = 0
        while i < n:
            ch = text[i]
            if ch in _SENT_TERMINATORS:
                inside = False
                for r in protected:
                    if r[0] <= i < r[1]:
                        inside = True
                        break
                if inside:
                    i += 1
                    continue
                if ch == "．" or ch == ".":
                    prev = text[i - 1] if i > 0 else ""
                    nxt = text[i + 1] if i + 1 < n else ""
                    if prev.isdigit() and nxt.isdigit():
                        i += 1
                        continue
                    tail = text[max(0, i - 12):i].lower()
                    word = re.split(r"[^a-z.]", tail)[-1] if tail else ""
                    if word and word in _ABBREVIATIONS:
                        i += 1
                        continue
                # consume a run of terminators / trailing quotes
                j = i + 1
                while j < n and (text[j] in _SENT_TERMINATORS or text[j] in "」』”\")"):
                    j += 1
                seg = text[start:j]
                if seg.strip():
                    out.append((start, j, seg))
                start = j
                i = j
                continue
            i += 1
        if start < n:
            seg = text[start:n]
            if seg.strip():
                out.append((start, n, seg))
        return out
    except Exception:
        return [(0, len(text or ""), text or "")] if text else []


def _iter_urls(text: str):
    """Guarantees: yields (start, end, url) for each URL, with trailing punctuation excluded."""
    for m in _URL_RE.finditer(text or ""):
        raw = m.group(0)
        # Trailing punctuation is almost never part of the URL.
        trimmed = raw.rstrip(".,;:!?)»”\"'")
        yield m.start(), m.start() + len(trimmed), trimmed


def norm_url(u: str) -> str:
    """Guarantees: a comparable URL form (lowercased, scheme/www/trailing-slash stripped)."""
    try:
        s = norm_text(u)
        s = re.sub(r"^(?:https?://|ftp://)", "", s)
        s = re.sub(r"^www\.", "", s)
        s = s.rstrip("/")
        s = s.split("#", 1)[0]
        return s
    except Exception:
        return ""


def url_host(u: str) -> str:
    """Guarantees: the host portion of a normalised URL, or ''."""
    try:
        return norm_url(u).split("/", 1)[0].split("?", 1)[0]
    except Exception:
        return ""


def _numeric_claim_from_match(m, offset=0):
    """Guarantees: builds a NUMERIC claim dict from a NUM_RE match, or None if unparseable."""
    raw = m.group(0).strip()
    if not raw:
        return None
    sign = m.group("sign") or ""
    num = m.group("num") or ""
    exp = m.group("exp") or ""
    mag = m.group("mag") or ""
    pct = m.group("pct") or ""
    unit = m.group("unit") or ""
    cur = m.group("cur") or ""

    base = _decimal_of(num.replace(",", "") + exp)
    if base is None:
        return None
    if sign in ("-", "▲", "△"):
        base = -base
    if mag:
        f = _MAG_FACTORS.get(mag)
        if f is not None:
            base = base * f
    kind = "number"
    if pct in ("%", "パーセント"):
        kind = "percent"
    elif pct == "割":
        base = base * Decimal(10)
        kind = "percent"
    elif pct == "ポイント":
        kind = "point"
    elif cur:
        kind = "currency"
    elif unit:
        kind = "unit"

    start = offset + m.start()
    end = offset + m.start() + len(m.group(0).rstrip())
    # Parenthesised accounting negatives: "(1,200)" means -1,200 in financial
    # tables. norm_number already handled this; the extractor did not, so a
    # loss in the context silently read as a profit.
    whole = m.string
    if (not sign and start > 0 and whole[start - 1] == "("
            and end < len(whole) and whole[end] == ")"):
        base = -base
        start -= 1
        end += 1
    return {
        "type": "NUMERIC",
        "value": raw,
        "normalized": str(base),
        "kind": kind,
        "unit": (pct or unit or cur or mag or "").strip(),
        "span": [start, end],
        "_dec": base,
    }


def _is_list_marker(text: str, m) -> bool:
    """Guarantees: True for enumeration markers like '1.' or '2)' at a line start.

    誤判定例: 箇条書きの "1. まず..." の "1" を数値主張として扱うと、文脈に 1 が
    無いだけで unsupported が量産される。これは典型的な偽陽性なので除外する。
    """
    try:
        s = m.start()
        line_start = text.rfind("\n", 0, s) + 1
        if text[line_start:s].strip() not in ("", "-", "*", "・", "(", "（"):
            return False
        after = text[m.end():m.end() + 2]
        return after[:1] in (".", ")", "、", "．", "）") or after[:1] == ""
    except Exception:
        return False


def extract_claims(answer: str, options: dict = None) -> list:
    """Guarantees: returns a list of typed, span-accurate claim dicts; never raises.

    Each claim: {"type","value","normalized","span":[s,e],"sentence_index", ...}.
    Sentences that yield no claim are what makes `coverage` < 1.0 - they are the
    part of the answer this tool explicitly did NOT check.
    """
    try:
        options = options or {}
        if not answer:
            return []
        want_quote = bool(options.get("enable_quote", True))
        want_entity = bool(options.get("enable_entity", True))
        want_url = bool(options.get("enable_url", True))
        want_numeric = bool(options.get("enable_numeric", True))
        want_date = bool(options.get("enable_date", True))

        n_chars = len(answer)
        max_claims = int(options.get("max_claims", MAX_CLAIMS) or MAX_CLAIMS)

        url_spans = []
        urls = []
        for s, e, u in _iter_urls(answer):
            url_spans.append((s, e))
            urls.append((s, e, u))

        sentences = split_sentences(answer, protected=url_spans)
        if not sentences:
            sentences = [(0, n_chars, answer)]
        sent_starts = [sp[0] for sp in sentences]

        def sent_index(pos):
            # bisect, not a linear scan: this is called once per claim and an
            # answer can hold thousands of both.
            i = bisect_right(sent_starts, pos) - 1
            if i < 0:
                return 0
            return min(i, len(sentences) - 1)

        claims = []
        taken = _SpanMask(n_chars)   # regions that block lower-priority extractors

        # --- URL (highest priority: never split a URL into numbers/entities) ---
        if want_url:
            for s, e, u in urls:
                claims.append({
                    "type": "URL", "value": u, "normalized": norm_url(u),
                    "span": [s, e], "sentence_index": sent_index(s),
                })
        taken.add_all(url_spans)

        # --- QUOTE ---
        quote_mask = _SpanMask(n_chars)
        quote_spans = []
        if want_quote:
            for m in _QUOTE_BRACKET_RE.finditer(answer):
                s, e = m.start(1), m.end(1)
                if taken.contains(s, e):
                    continue
                body = m.group(1).strip()
                if len(body) < 4:
                    continue
                quote_spans.append((s, e))
                quote_mask.add(s, e)
                claims.append({
                    "type": "QUOTE", "value": body, "normalized": norm_text(body),
                    "span": [s, e], "sentence_index": sent_index(s), "source": "bracket",
                })
            for m in _QUOTE_MARKER_RE.finditer(answer):
                s = m.end()
                si = sent_index(s)
                sent_end = sentences[si][1] if si < len(sentences) else len(answer)
                body = answer[s:sent_end]
                body = body.strip(" 　:：,、。.\n\"'「」『』“”")
                if len(body) < 8:
                    continue
                e = s + len(answer[s:sent_end]) - len(answer[s:sent_end].lstrip(" 　:：,、\n"))
                bs = s + (len(answer[s:sent_end]) - len(answer[s:sent_end].lstrip(" 　:：,、\n")))
                be = bs + len(body)
                if quote_mask.hits(bs, be):
                    continue
                if taken.contains(bs, be):
                    continue
                quote_spans.append((bs, be))
                quote_mask.add(bs, be)
                claims.append({
                    "type": "QUOTE", "value": body[:200], "normalized": norm_text(body[:200]),
                    "span": [bs, min(be, bs + 200)], "sentence_index": si, "source": "marker",
                })

        # --- DATE ---
        date_spans = []
        date_mask = _SpanMask(n_chars)
        url_mask = _SpanMask(n_chars)
        url_mask.add_all(url_spans)
        if want_date:
            for pat in _DATE_PATTERNS:
                for m in pat.finditer(answer):
                    s, e = m.start(), m.end()
                    if date_mask.hits(s, e):
                        continue
                    if url_mask.contains(s, e):
                        continue
                    raw = m.group(0)
                    iso = norm_date(raw)
                    if iso is None:
                        continue
                    date_spans.append((s, e))
                    date_mask.add(s, e)
                    claims.append({
                        "type": "DATE", "value": raw, "normalized": iso,
                        "span": [s, e], "sentence_index": sent_index(s),
                    })
        taken.add_all(date_spans)

        # --- ENTITY (article numbers / model numbers / capitalised names / acronyms) ---
        ent_spans = []
        ent_mask = _SpanMask(n_chars)
        if want_entity:
            for pat, sub in ((_ENT_ARTICLE_RE, "article"), (_ENT_MODEL_RE, "model")):
                for m in pat.finditer(answer):
                    s, e = m.start(), m.end()
                    if taken.contains(s, e) or ent_mask.hits(s, e):
                        continue
                    v = m.group(0).strip()
                    # "USD45,000" is a price, not a part number.
                    lead = re.match(r"^[A-Za-z]+", v)
                    if sub == "model" and lead and lead.group(0).lower() in _CURRENCY_CODES:
                        continue
                    ent_spans.append((s, e))
                    ent_mask.add(s, e)
                    claims.append({
                        "type": "ENTITY", "value": v, "normalized": norm_text(v),
                        "span": [s, e], "sentence_index": sent_index(s), "subtype": sub,
                    })
            for m in _ENT_CAPWORDS_RE.finditer(answer):
                s, e = m.start(), m.end()
                if taken.contains(s, e) or ent_mask.hits(s, e):
                    continue
                v = m.group(0).strip()
                toks = [t for t in re.split(r"[\s]+", norm_text(v)) if t]
                if not toks:
                    continue
                if all(t.strip(".,'’-") in _ENTITY_STOPWORDS for t in toks):
                    continue
                # Trim leading/trailing function words and shrink the span to
                # match, so "The NASA report" is compared as "NASA report".
                words = v.split()
                while len(words) > 1 and norm_text(words[0]).strip(".,'’-") in _ENTITY_STOPWORDS:
                    s += len(words[0]) + 1
                    words = words[1:]
                while len(words) > 1 and norm_text(words[-1]).strip(".,'’-") in _ENTITY_STOPWORDS:
                    e -= len(words[-1]) + 1
                    words = words[:-1]
                v = " ".join(words)
                if len(v) < 4 or e <= s:
                    continue
                ent_spans.append((s, e))
                ent_mask.add(s, e)
                claims.append({
                    "type": "ENTITY", "value": v, "normalized": norm_text(v),
                    "span": [s, e], "sentence_index": sent_index(s), "subtype": "name",
                })
            for m in _ENT_ACRONYM_RE.finditer(answer):
                s, e = m.start(), m.end()
                if taken.contains(s, e) or ent_mask.hits(s, e):
                    continue
                v = m.group(0)
                if norm_text(v) in _ENTITY_STOPWORDS:
                    continue
                ent_spans.append((s, e))
                ent_mask.add(s, e)
                claims.append({
                    "type": "ENTITY", "value": v, "normalized": norm_text(v),
                    "span": [s, e], "sentence_index": sent_index(s), "subtype": "acronym",
                })

        # --- NUMERIC (last: dates, URLs and article numbers already claimed theirs) ---
        if want_numeric:
            block = _SpanMask(n_chars)
            block.add_all(url_spans)
            block.add_all(date_spans)
            block.add_all(ent_spans)
            for m in NUM_RE.finditer(answer):
                s, e = m.start(), m.end()
                if block.hits(s, e):
                    continue
                if _is_list_marker(answer, m):
                    continue
                c = _numeric_claim_from_match(m)
                if c is None:
                    continue
                c["sentence_index"] = sent_index(c["span"][0])
                claims.append(c)

        claims.sort(key=lambda c: (c["span"][0], c["span"][1]))
        if len(claims) > max_claims:
            # No silent truncation: the dropped count is reported by verify() so
            # nobody reads a partial check as a complete one.
            dropped = len(claims) - max_claims
            claims = claims[:max_claims]
            claims.append({"type": "_TRUNCATION_NOTICE", "dropped": dropped,
                           "span": [0, 0], "sentence_index": 0, "value": "", "normalized": ""})
        for i, c in enumerate(claims):
            c["id"] = "c%04d" % i
        return claims
    except Exception:
        return []


# =============================================================================
# (D) VERIFIER - the core
# =============================================================================

class DerivationIndex:
    """Guarantees: a bounded, time-budgeted map from a derived value to its formula.

    Built ONCE per verify() call over the context's numbers, so every numeric
    claim is an O(1) dict lookup instead of a fresh combinatorial search.

    誤判定が起きうる具体例:
      - 文脈に 2 と 3 と 6 があるとき、応答の「6」は 2*3 としても導出できる。
        偶然の一致で derived になることがあるため、evidence(式)を必ず併記して
        人が見て棄却できるようにしている。
      - 小さい整数(0,1,2,...)は組み合わせ爆発で何にでも当たる。よって
        `min_abs` 未満の値は索引に入れない。
    """

    _OPS2 = ("+", "-", "*", "/", "%of", "%chg")

    def __init__(self, numbers, max_terms=3, budget_ms=DERIVE_BUDGET_MS,
                 max_numbers=DERIVE_MAX_NUMBERS, max_entries=250_000, min_abs=1e-9):
        """Guarantees: builds within the time and size budget, or sets .truncated; never raises."""
        self.map = {}
        self.truncated = False
        self.n_inputs = 0
        try:
            t0 = time.perf_counter()
            budget = max(0.005, budget_ms / 1000.0)
            uniq = []
            seen = set()
            for dec, raw in numbers:
                f = _safe_float(dec)
                if f != f:
                    continue
                k = _fkey(f)
                if not k or k in seen:
                    continue
                seen.add(k)
                uniq.append((f, raw))
                if len(uniq) >= max_numbers:
                    self.truncated = True
                    break
            self.n_inputs = len(uniq)
            if len(uniq) < 2:
                return

            pair_results = []
            for i in range(len(uniq)):
                ai, araw = uniq[i]
                for j in range(len(uniq)):
                    if i == j:
                        continue
                    bj, braw = uniq[j]
                    for val, expr in self._combine(ai, araw, bj, braw):
                        self._put(val, expr, min_abs)
                        if len(pair_results) < 800:
                            pair_results.append((val, expr))
                if time.perf_counter() - t0 > budget:
                    self.truncated = True
                    return
                if len(self.map) > max_entries:
                    self.truncated = True
                    return

            if max_terms >= 3 and len(uniq) <= 20:
                for val, expr in pair_results:
                    if time.perf_counter() - t0 > budget:
                        self.truncated = True
                        return
                    if len(self.map) > max_entries:
                        self.truncated = True
                        return
                    for c, craw in uniq:
                        for v2, e2 in self._combine(val, "(" + expr + ")", c, craw, third=True):
                            self._put(v2, e2, min_abs)
        except Exception:
            # A failed index must never fail verification; it just means fewer
            # "derived" verdicts.
            self.truncated = True

    @staticmethod
    def _combine(a, araw, b, braw, third=False):
        """Guarantees: every finite (value, formula) pair reachable from two operands."""
        out = []
        try:
            out.append((a + b, "%s + %s" % (araw, braw)))
            out.append((a - b, "%s - %s" % (araw, braw)))
            out.append((a * b, "%s * %s" % (araw, braw)))
            if b != 0:
                out.append((a / b, "%s / %s" % (araw, braw)))
                if not third:
                    out.append((a / b * 100.0, "%s / %s * 100 (percentage)" % (araw, braw)))
                    out.append(((a - b) / b * 100.0, "(%s - %s) / %s * 100 (change rate)" % (araw, braw, braw)))
            if not third:
                out.append((a * b / 100.0, "%s * %s%% (percent of)" % (araw, braw)))
        except (ZeroDivisionError, OverflowError, ValueError):
            pass
        except Exception:
            pass
        return [(v, e) for v, e in out if v == v and abs(v) != float("inf")]

    def _put(self, val, expr, min_abs):
        """Guarantees: records the FIRST formula found for a value, so evidence stays stable."""
        if abs(val) < min_abs:
            return
        k = _fkey(val)
        if not k:
            return
        if k not in self.map:
            self.map[k] = expr

    def lookup(self, value):
        """Guarantees: returns a formula string if the value is derivable, else None."""
        try:
            return self.map.get(_fkey(value))
        except Exception:
            return None


def _fuzzy_anchors(needle: str, haystack: str, max_anchors: int = 96):
    """Guarantees: candidate window starts where needle and haystack share a character k-gram.

    Whole-token anchoring does not work for Japanese: a sentence with no spaces
    is a single token, so nothing is ever found and the search degenerates into
    a full sliding-window scan. Character k-grams anchor CJK and Latin alike,
    and each is located with str.find, which runs at C speed.
    """
    nl = len(needle)
    # k must stay strictly below the needle length, or the only k-gram IS the
    # needle and nothing anchors. Short identifiers need k=2: "第13条" against a
    # context holding "第12条" shares no 3-gram, and that one-digit-off article
    # number is exactly the fabrication this tool exists to catch.
    k = 2 if nl <= 6 else min(8, max(4, nl // 3))
    k = min(k, max(2, nl - 1))
    if nl < 2:
        return [0] if needle in haystack else []
    offsets = sorted({0, nl // 4, nl // 2, (3 * nl) // 4, max(0, nl - k)})
    anchors = []
    for off in offsets:
        gram = needle[off:off + k]
        if len(gram) < k:
            continue
        idx = haystack.find(gram)
        hits = 0
        while idx != -1 and hits < 20 and len(anchors) < max_anchors:
            anchors.append(max(0, idx - off))
            hits += 1
            idx = haystack.find(gram, idx + 1)
        if len(anchors) >= max_anchors:
            break
    return anchors


def _best_fuzzy(needle: str, haystack: str, max_evals: int = 240):
    """Guarantees: returns (best_ratio, best_window_text) for an order-preserving match.

    Two things decide accuracy here, and an earlier version got both wrong:

    1. ANCHOR ALIGNMENT. A candidate window is positioned so the anchoring
       k-gram sits at the same offset it occupies inside the needle, not at the
       window's midpoint.
    2. WINDOW WIDTH. A window much longer than the needle drags the ratio down
       with characters that were never supposed to match. Scoring several widths
       and keeping the best turned a paraphrase that truly matches at 0.93 from
       0.59 into 0.87 - the difference between "approximate" and a false
       "contradicted".

    Work is hard-bounded: no shared k-gram means no candidate windows and an
    immediate 0.0, because an order-preserving match above ~0.6 cannot exist
    without one. That bound is what keeps a 20k-character context from turning
    this into a full sliding-window scan.
    """
    try:
        if not needle or not haystack:
            return 0.0, ""
        nl = len(needle)
        anchors = _fuzzy_anchors(needle, haystack)
        if not anchors:
            return 0.0, ""
        widths = sorted({max(3, nl - 2), nl, int(nl * 1.25) + 3})
        best = 0.0
        best_txt = ""
        evals = 0
        sm = SequenceMatcher(autojunk=False)
        sm.set_seq2(needle)
        seen = set()
        for st in anchors:
            st = max(0, min(st, max(0, len(haystack) - 1)))
            for w in widths:
                if (st, w) in seen:
                    continue
                seen.add((st, w))
                window = haystack[st:st + w]
                if not window:
                    continue
                sm.set_seq1(window)
                if sm.real_quick_ratio() <= best or sm.quick_ratio() <= best:
                    continue
                evals += 1
                r = sm.ratio()
                if r > best:
                    best = r
                    best_txt = window
                if best >= 0.995 or evals >= max_evals:
                    return best, best_txt
        return best, best_txt
    except Exception:
        return 0.0, ""


def _context_numbers(context: str):
    """Guarantees: returns [(Decimal, raw_text)] for every numeric literal in the context."""
    out = []
    try:
        context = context or ""
        mask = _SpanMask(len(context))
        mask.add_all((s, e) for s, e, _ in _iter_urls(context))
        for pat in _DATE_PATTERNS:
            for m in pat.finditer(context):
                mask.add(m.start(), m.end())
        for m in NUM_RE.finditer(context):
            if mask.hits(m.start(), m.end()):
                continue
            c = _numeric_claim_from_match(m)
            if c is not None:
                out.append((c["_dec"], c["value"]))
    except Exception:
        pass
    return out


def _context_dates(context: str):
    """Guarantees: returns a set of ISO date strings present in the context."""
    out = set()
    try:
        for pat in _DATE_PATTERNS:
            for m in pat.finditer(context):
                iso = norm_date(m.group(0))
                if iso:
                    out.add(iso)
    except Exception:
        pass
    return out


class _ContextIndex:
    """Guarantees: one pass over the context, reused by every claim (keeps latency flat)."""

    def __init__(self, context: str, options: dict):
        """Guarantees: one pass over the context produces every lookup table the verifiers need."""
        self.raw = context or ""
        self.norm = norm_text(self.raw)
        self.numbers = _context_numbers(self.raw)
        self.num_map = {}
        for dec, raw in self.numbers:
            k = _fkey(dec)
            if k and k not in self.num_map:
                self.num_map[k] = raw
        self.num_floats = []
        for dec, raw in self.numbers:
            f = _safe_float(dec)
            if f == f:
                self.num_floats.append((f, raw))
        _ordered = sorted(self.num_floats, key=lambda t: t[0])
        self.sorted_values = [t[0] for t in _ordered]
        self.sorted_raws = [t[1] for t in _ordered]
        self.dates = _context_dates(self.raw)
        # Years mentioned only as part of a context date are still legitimate
        # support for a bare year in the answer. Without this, context "FY2024"
        # against answer "2024" came out unsupported.
        self.date_years = set()
        for d in self.dates:
            try:
                self.date_years.add(int(d.split("-")[0]))
            except Exception:
                continue
        self.urls = {}
        for _s, _e, u in _iter_urls(self.raw):
            self.urls[norm_url(u)] = u
        self.hosts = {url_host(u) for u in self.urls.values()}
        self.tokens = set(t for t in re.split(r"[^0-9a-z぀-ヿ一-鿿]+", self.norm) if t)
        self._deriv = None
        self._opts = options or {}
        # Fuzzy matching is the only super-linear step left. Give the whole
        # request one shared budget; once it is gone we fall back to exact
        # matching and SAY SO, rather than quietly taking seconds of CPU.
        self.fuzzy_deadline = time.perf_counter() + max(
            0.005, float((options or {}).get("fuzzy_budget_ms", FUZZY_BUDGET_MS)) / 1000.0)
        self.fuzzy_exhausted = False

    def fuzzy(self, needle):
        """Guarantees: a bounded fuzzy match, or (0.0, "") once the request's budget is spent."""
        if time.perf_counter() > self.fuzzy_deadline:
            self.fuzzy_exhausted = True
            return 0.0, ""
        return _best_fuzzy(needle, self.norm)

    @property
    def deriv(self):
        """Guarantees: the derivation index is built lazily, at most once per verify() call."""
        if self._deriv is None:
            self._deriv = DerivationIndex(
                self.numbers,
                max_terms=int(self._opts.get("derive_max_terms", DERIVE_MAX_TERMS)),
                budget_ms=int(self._opts.get("derive_budget_ms", DERIVE_BUDGET_MS)),
                max_numbers=int(self._opts.get("derive_max_numbers", DERIVE_MAX_NUMBERS)),
            )
        return self._deriv


_NEAREST_EXACT_LIMIT = 400


def _nearest_number(target: float, candidates, sorted_values=None, sorted_raws=None):
    """Guarantees: returns (value, raw, relative_diff) for the closest context number, or None.

    Up to _NEAREST_EXACT_LIMIT candidates this is an exact scan. Above that it
    falls back to a bisect window around the absolute-nearest values, which is
    an approximation of "nearest by RELATIVE difference" - acceptable because
    the result only feeds a human-readable contradiction message and a
    threshold test, and because the exact scan would be O(claims x context).
    """
    try:
        if sorted_values is not None and len(sorted_values) > _NEAREST_EXACT_LIMIT:
            i = bisect_left(sorted_values, target)
            lo = max(0, i - 12)
            hi = min(len(sorted_values), i + 12)
            pool = zip(sorted_values[lo:hi], sorted_raws[lo:hi])
        else:
            pool = candidates
        best = None
        for f, raw in pool:
            try:
                denom = max(abs(target), abs(f), 1e-12)
                rel = abs(target - f) / denom
                if best is None or rel < best[2]:
                    best = (f, raw, rel)
            except Exception:
                continue
        return best
    except Exception:
        return None


def _order_of_magnitude_off(a: float, b: float):
    """Guarantees: returns the integer power-of-ten offset if a/b is ~10^k (k!=0), else None."""
    try:
        if a == 0 or b == 0:
            return None
        r = abs(a) / abs(b)
        if r <= 0:
            return None
        lg = math.log10(r)
        k = round(lg)
        if k != 0 and abs(lg - k) < 0.02 and abs(k) <= 9:
            return int(k)
        return None
    except Exception:
        return None


def _verify_numeric(claim, ctx, opts):
    """Guarantees: assigns exactly one status to a NUMERIC claim, with evidence when matched.

    誤判定が起きうる具体例:
      1) 文脈「約1,200件」に対し応答「1200件」-> 桁区切りと「約」を除去して
         正規化するので supported。逆に文脈「1,200」応答「1,250」は
         相対差 4% なので contradicted(近接不一致)になる。丸めた記述を
         許したい運用では contradiction_rel を下げるのではなく
         numeric_tolerance を上げること。
      2) 文脈「売上100、費用40」に対し応答「利益は60」-> derived (100 - 40)。
         ただし文脈に 60 が別文脈で存在すれば supported が優先される。
      3) 単位の取り違え: 文脈「3.5%」応答「3.5ポイント」は値が一致するので
         supported になる。単位の意味的な誤りはこのツールでは検出できない。
    """
    dec = claim.get("_dec")
    if dec is None:
        dec = norm_number(claim.get("value"))
    if dec is None:
        claim["status"] = "unsupported"
        claim["evidence"] = None
        claim["reason"] = "unparseable_number"
        return claim
    f = _safe_float(dec)
    tol = float(opts.get("numeric_tolerance", 0.0))
    contr_rel = float(opts.get("contradiction_rel", DEFAULT_CONTRADICTION_REL))

    k = _fkey(f)
    if k and k in ctx.num_map:
        claim["status"] = "supported"
        claim["evidence"] = "context literal: %s" % ctx.num_map[k]
        return claim

    if tol > 0:
        near = _nearest_number(f, ctx.num_floats, ctx.sorted_values, ctx.sorted_raws)
        if near and near[2] <= tol:
            claim["status"] = "supported"
            claim["evidence"] = "context literal within tolerance %.3g: %s" % (tol, near[1])
            return claim

    # percent <-> fraction equivalence (15% vs 0.15)
    if claim.get("kind") == "percent":
        for alt, note in ((f / 100.0, "%.6g == %.6g%% (fraction form in context)"),):
            ak = _fkey(alt)
            if ak and ak in ctx.num_map:
                claim["status"] = "derived"
                claim["evidence"] = (note % (alt, f)) + " -> %s" % ctx.num_map[ak]
                return claim
    else:
        alt = f * 100.0
        ak = _fkey(alt)
        if ak and ak in ctx.num_map and abs(f) < 1:
            claim["status"] = "derived"
            claim["evidence"] = "%.6g == %.6g%% (percent form in context: %s)" % (f, alt, ctx.num_map[ak])
            return claim

    # An exact sign flip is checked BEFORE derivation. Otherwise a loss reported
    # as a profit gets excused by a coincidental formula: context "▲1,200" and
    # answer "1,200" came back derived via "(2 - ▲1,200) - 2", and the whole
    # answer passed. Three-term arithmetic can reach almost any value, so the
    # dangerous, unambiguous cases must be settled first.
    if f != 0:
        flip = _fkey(-f)
        if flip and flip in ctx.num_map:
            claim["status"] = "contradicted"
            claim["evidence"] = ("context states %s, the exact negation of this value "
                                 "(sign error)" % ctx.num_map[flip])
            claim["nearest_context_value"] = ctx.num_map[flip]
            return claim

    if opts.get("enable_derivation", True):
        expr = ctx.deriv.lookup(f)
        if expr:
            claim["status"] = "derived"
            claim["evidence"] = "derivable from context: %s = %s" % (expr, _fkey(f))
            return claim

    # A bare 4-digit integer in a plausible year range, matching a year that the
    # context states as a date. Deliberately narrow: restricted to 1900-2100 and
    # to numbers carrying no unit or currency.
    if (claim.get("kind") == "number" and not claim.get("unit")
            and float(f).is_integer() and 1900 <= f <= 2100
            and int(f) in ctx.date_years):
        claim["status"] = "supported"
        claim["evidence"] = "matches a year stated in the context (%d)" % int(f)
        return claim

    near = _nearest_number(f, ctx.num_floats, ctx.sorted_values, ctx.sorted_raws)
    if near is not None:
        nf, nraw, rel = near
        oom = _order_of_magnitude_off(f, nf)
        # 小さい裸の整数 (単位も通貨も無い 0..10) は何にでも「近い」ので
        # contradicted を出さない。偽陽性の主要因だった。
        bare_small = (claim.get("kind") == "number" and abs(f) <= 10 and float(f).is_integer())
        if oom is not None and not bare_small:
            claim["status"] = "contradicted"
            claim["evidence"] = "closest context value %s differs by 10^%d (order-of-magnitude error)" % (nraw, oom)
            claim["nearest_context_value"] = nraw
            return claim
        if not bare_small and nf != 0 and f == -nf:
            claim["status"] = "contradicted"
            claim["evidence"] = "closest context value %s has the opposite sign" % nraw
            claim["nearest_context_value"] = nraw
            return claim
        if not bare_small and rel <= contr_rel:
            claim["status"] = "contradicted"
            claim["evidence"] = "closest context value is %s (relative difference %.1f%%)" % (nraw, rel * 100.0)
            claim["nearest_context_value"] = nraw
            return claim
        claim["nearest_context_value"] = nraw

    claim["status"] = "unsupported"
    claim["evidence"] = None
    return claim


def _verify_date(claim, ctx, opts):
    """Guarantees: assigns exactly one status to a DATE claim.

    誤判定が起きうる具体例:
      - 文脈「2024年1月」に対し応答「2024-01-15」: 文脈より詳細なので
        approximate(日付の粒度が増えている=モデルが補完した可能性)。
      - 文脈「令和6年1月5日」応答「2024-01-05」: 元号を正規化するので supported。
      - 会計年度「FY2024」と暦年「2024年」を同一視する。国や企業により
        FY の開始月は異なるため、ここは偽陽性になりうる。
    """
    iso = claim.get("normalized") or norm_date(claim.get("value"))
    if not iso:
        claim["status"] = "unsupported"
        claim["evidence"] = None
        claim["reason"] = "unparseable_date"
        return claim
    if iso in ctx.dates:
        claim["status"] = "supported"
        claim["evidence"] = "context date: %s" % iso
        return claim
    # claim is coarser than context ("2024-01" vs context "2024-01-05")
    for d in ctx.dates:
        if d.startswith(iso):
            claim["status"] = "supported"
            claim["evidence"] = "context date %s falls inside %s" % (d, iso)
            return claim
    # claim is finer than context -> the model added precision the source lacks
    for d in ctx.dates:
        if iso.startswith(d):
            claim["status"] = "approximate"
            claim["evidence"] = "context only states %s; the answer adds precision (%s)" % (d, iso)
            return claim
    best = None
    for d in ctx.dates:
        common = 0
        for a, b in zip(iso.split("-"), d.split("-")):
            if a == b:
                common += 1
            else:
                break
        if best is None or common > best[1]:
            best = (d, common)
    if best and best[1] >= 1:
        claim["status"] = "contradicted"
        claim["evidence"] = "context has a near but different date: %s" % best[0]
        claim["nearest_context_value"] = best[0]
        return claim
    claim["status"] = "unsupported"
    claim["evidence"] = None
    return claim


def _verify_quote(claim, ctx, opts):
    """Guarantees: assigns exactly one status to a QUOTE claim using normalised matching.

    誤判定が起きうる具体例:
      - 文脈が「当社は2024年に新製品を投入する」、応答の引用が
        「2024年に新製品を投入する」-> 正規化後の部分文字列一致で supported。
      - 助詞や敬体を変えた言い換え(「投入します」)は approximate になる。
        意味は同じでも「引用」としては不正確なので、これは仕様どおり。
      - 閾値 approx_ratio 付近では判定が揺れる。低くしすぎると別の文を
        引用元と誤認する。
    """
    q = claim.get("normalized") or norm_text(claim.get("value"))
    if not q or len(q) < 3:
        claim["status"] = "unsupported"
        claim["evidence"] = None
        claim["reason"] = "quote_too_short"
        return claim
    if q in ctx.norm:
        claim["status"] = "supported"
        claim["evidence"] = "exact (normalised) substring of the context"
        return claim
    ratio, window = ctx.fuzzy(q)
    thr = float(opts.get("approx_ratio", DEFAULT_APPROX_RATIO))
    if ratio >= thr:
        claim["status"] = "approximate"
        claim["evidence"] = "closest context passage (similarity %.2f): %s" % (ratio, window[:180])
        claim["similarity"] = round(ratio, 4)
        return claim
    claim["similarity"] = round(ratio, 4)
    # Narrow band only. Below it the passage is simply not in the context
    # (unsupported); a quote whose NUMBERS were altered is caught by the NUMERIC
    # claims inside the same sentence, so this does not need a wide net.
    if ratio >= QUOTE_CONTRADICTION_FLOOR:
        claim["status"] = "contradicted"
        claim["evidence"] = "a similar but materially different passage exists (similarity %.2f): %s" % (
            ratio, window[:180])
        claim["nearest_context_value"] = window[:180]
        return claim
    claim["status"] = "unsupported"
    claim["evidence"] = None
    return claim


def _verify_entity(claim, ctx, opts):
    """Guarantees: assigns exactly one status to an ENTITY claim, with stopword suppression.

    誤判定が起きうる具体例:
      - 応答「Machine Learning の手法」で文脈に同語が無い場合 unsupported に
        なるが、これは一般名詞であり幻覚ではない。除外リストで抑えきれない
        ため ENTITY は最も偽陽性が多い。導入初期は enable_entity=false を推奨。
      - 略語の展開 (文脈 "World Health Organization" / 応答 "WHO") は
        トークン一致しないので unsupported。これも典型的な偽陽性。
    """
    v = claim.get("normalized") or norm_text(claim.get("value"))
    if not v:
        claim["status"] = "unsupported"
        claim["evidence"] = None
        return claim
    if v in _ENTITY_STOPWORDS:
        claim["status"] = "supported"
        claim["evidence"] = "common word, not treated as a factual entity"
        claim["suppressed"] = True
        return claim
    if v in ctx.norm:
        claim["status"] = "supported"
        claim["evidence"] = "appears verbatim in the context"
        return claim
    toks = [t for t in re.split(r"[^0-9a-z぀-ヿ一-鿿]+", v) if t]
    meaningful = [t for t in toks if t not in _ENTITY_STOPWORDS]
    if meaningful and all(t in ctx.tokens for t in meaningful):
        claim["status"] = "approximate"
        claim["evidence"] = "all tokens appear in the context but not as this phrase"
        return claim
    ratio, window = ctx.fuzzy(v)
    claim["similarity"] = round(ratio, 4)
    # Identifiers are exact-or-wrong: "Article 15" is not "approximately
    # Article 12". This branch must come BEFORE the generic near-match branch,
    # or a one-digit-off article number - the dangerous fabrication - gets
    # reported as a harmless approximation.
    if claim.get("subtype") in ("model", "article") and ratio >= 0.72:
        claim["status"] = "contradicted"
        claim["evidence"] = "a similar identifier exists in the context: %s" % window[:120]
        claim["nearest_context_value"] = window[:120]
        return claim
    if ratio >= 0.90 and len(v) >= 6:
        claim["status"] = "approximate"
        claim["evidence"] = "near match in context (similarity %.2f): %s" % (ratio, window[:120])
        return claim
    claim["status"] = "unsupported"
    claim["evidence"] = None
    return claim


def _verify_url(claim, ctx, opts):
    """Guarantees: assigns exactly one status to a URL claim (fabricated-link detection).

    誤判定が起きうる具体例:
      - 文脈が https://example.com/docs/v2 、応答が https://example.com/docs
        -> ホストは一致、パスが違うので contradicted。実際には正しい上位
        ページかもしれない。
      - トラッキングパラメータの有無で不一致になる場合がある。
    """
    u = claim.get("normalized") or norm_url(claim.get("value"))
    if not u:
        claim["status"] = "unsupported"
        claim["evidence"] = None
        return claim
    if u in ctx.urls:
        claim["status"] = "supported"
        claim["evidence"] = "context URL: %s" % ctx.urls[u]
        return claim
    for cu in ctx.urls:
        if cu.startswith(u) or u.startswith(cu):
            claim["status"] = "approximate"
            claim["evidence"] = "context has a related URL: %s" % ctx.urls[cu]
            return claim
    h = url_host(u)
    if h and h in ctx.hosts:
        claim["status"] = "contradicted"
        claim["evidence"] = "host %s appears in the context but this exact path does not (possible fabricated link)" % h
        claim["nearest_context_value"] = h
        return claim
    claim["status"] = "unsupported"
    claim["evidence"] = None
    return claim


_VERIFIERS = {
    "NUMERIC": _verify_numeric,
    "DATE": _verify_date,
    "QUOTE": _verify_quote,
    "ENTITY": _verify_entity,
    "URL": _verify_url,
}


@guarded
def verify(answer, context, options=None) -> dict:
    """Guarantees: returns grounding_score AND coverage separately, plus per-claim status.

    grounding_score answers "of what I checked, how much held up?".
    coverage answers "how much did I even check?".
    Reporting the first without the second is how users end up trusting an
    unverified answer, so both are always present.
    """
    t0 = time.perf_counter()
    opts = dict(options or {})
    answer, a_trunc = clamp_text(answer, int(opts.get("max_chars", MAX_TEXT_CHARS)))
    context, c_trunc = clamp_text(context, int(opts.get("max_chars", MAX_TEXT_CHARS)))

    claims = extract_claims(answer, opts)
    claims_dropped = 0
    real_claims = []
    for c in claims:
        if c.get("type") == "_TRUNCATION_NOTICE":
            claims_dropped = int(c.get("dropped", 0) or 0)
            continue
        real_claims.append(c)
    claims = real_claims
    ctx = _ContextIndex(context, opts)

    url_spans = [(s, e) for s, e, _ in _iter_urls(answer)]
    sentences = split_sentences(answer, protected=url_spans)
    n_sent = len(sentences)
    sent_with_claims = set()

    # Identical claim text always gets the identical verdict against the same
    # context, so a repetitive answer costs one verification per distinct value
    # rather than one per occurrence.
    memo = {}
    _CARRY = ("status", "evidence", "reason", "similarity", "nearest_context_value", "suppressed")
    for c in claims:
        ctype = c.get("type")
        key = (ctype, c.get("normalized"), c.get("value"), c.get("subtype"))
        cached = memo.get(key)
        if cached is not None:
            for k in _CARRY:
                if k in cached:
                    c[k] = cached[k]
        else:
            fn = _VERIFIERS.get(ctype)
            try:
                if fn is None:
                    c["status"] = "unsupported"
                    c["evidence"] = None
                else:
                    fn(c, ctx, opts)
            except Exception as exc:  # a single bad claim must not sink the batch
                c["status"] = "unsupported"
                c["evidence"] = None
                c["reason"] = "verifier_error:%s" % type(exc).__name__
            memo[key] = {k: c[k] for k in _CARRY if k in c}
        if not c.get("suppressed"):
            sent_with_claims.add(c.get("sentence_index", -1))

    counts = {s: 0 for s in STATUSES}
    by_type = {t: {s: 0 for s in STATUSES} for t in CLAIM_TYPES}
    effective = []
    for c in claims:
        st = c.get("status", "unsupported")
        if c.get("suppressed"):
            continue
        effective.append(c)
        counts[st] = counts.get(st, 0) + 1
        t = c.get("type")
        if t in by_type:
            by_type[t][st] = by_type[t].get(st, 0) + 1

    total = len(effective)
    good = counts.get("supported", 0) + counts.get("derived", 0)
    grounding = (good / total) if total else 0.0
    coverage = (len(sent_with_claims) / n_sent) if n_sent else 0.0

    relative_dates = len(_RELATIVE_DATE_RE.findall(answer)) if answer else 0
    unverified_sentences = [
        {"sentence_index": i, "text": sentences[i][2].strip()[:200]}
        for i in range(n_sent) if i not in sent_with_claims
    ]

    out_claims = []
    for c in effective:
        d = {k: v for k, v in c.items() if not k.startswith("_")}
        d.setdefault("evidence", None)
        out_claims.append(d)

    elapsed = (time.perf_counter() - t0) * 1000.0
    return {
        "ok": True,
        "grounding_score": round(grounding, 4),
        "coverage": round(coverage, 4),
        "claims": out_claims,
        "counts": {
            "claims_total": total,
            "sentences_total": n_sent,
            "sentences_verified": len(sent_with_claims),
            "sentences_unverified": max(0, n_sent - len(sent_with_claims)),
            "relative_date_mentions": relative_dates,
            "by_status": counts,
            "by_type": by_type,
            "context_numbers": len(ctx.numbers),
            "context_dates": len(ctx.dates),
            "derivation_entries": len(ctx.deriv.map) if ctx._deriv is not None else 0,
            "derivation_truncated": bool(ctx._deriv.truncated) if ctx._deriv is not None else False,
            "claims_dropped_by_cap": claims_dropped,
            "distinct_claims_verified": len(memo),
            "fuzzy_budget_exhausted": bool(ctx.fuzzy_exhausted),
        },
        "unverified_sentences": unverified_sentences[:50],
        "truncated": {"answer": a_trunc, "context": c_trunc},
        "latency_ms": round(elapsed, 2),
        "notes": [
            "grounding_score is computed over verifiable claims only.",
            "coverage is the share of sentences that produced at least one verifiable claim; "
            "the rest were NOT checked.",
        ] + ([
            "%d further claims were dropped by the max_claims cap (%d in effect) and were NOT checked."
            % (claims_dropped, int(opts.get("max_claims", MAX_CLAIMS) or MAX_CLAIMS))
        ] if claims_dropped else []) + ([
            "The fuzzy-matching budget (fuzzy_budget_ms) ran out; later QUOTE/ENTITY claims "
            "were matched exactly only, so some 'approximate' results may read as 'unsupported'."
        ] if ctx.fuzzy_exhausted else []),
    }


# =============================================================================
# (E) LEAK DETECTOR
# =============================================================================

def _leak_tokens(s: str):
    """Guarantees: a mixed-script token stream (CJK per character, latin per word)."""
    out = []
    try:
        for chunk in re.findall(r"[0-9a-z]+|[぀-ヿ一-鿿가-힯]", norm_text(s)):
            out.append(chunk)
    except Exception:
        pass
    return out


def _ngrams(tokens, n):
    """Guarantees: the set of all n-token windows, empty when the stream is shorter than n."""
    if n <= 0 or len(tokens) < n:
        return set()
    return {" ".join(tokens[i:i + n]) for i in range(len(tokens) - n + 1)}


@guarded
def detect_system_prompt_leak(answer, system_prompt, n=None, threshold=None) -> dict:
    """Guarantees: reports the share of answer n-grams that echo the system prompt, plus the longest fragment.

    誤判定が起きうる具体例: system prompt に「日本語で簡潔に回答してください」
    のような定型句があり、応答が同じ語を自然に使うと重なり率が上がる。
    閾値 LEAK_THRESHOLD は運用データを見て調整すること。
    """
    n = NGRAM_LEAK_N if n is None else max(3, int(n))
    threshold = LEAK_THRESHOLD if threshold is None else float(threshold)
    answer, _ = clamp_text(answer)
    system_prompt, _ = clamp_text(system_prompt)
    if not answer or not system_prompt:
        return {"ok": True, "leak": False, "overlap": 0.0, "n": n, "threshold": threshold,
                "matched_ngrams": 0, "total_ngrams": 0, "longest_fragment": "", "fragment_len": 0,
                "reason": "empty_input"}
    a_tok = _leak_tokens(answer)
    s_tok = _leak_tokens(system_prompt)
    a_ng = _ngrams(a_tok, n)
    s_ng = _ngrams(s_tok, n)
    inter = a_ng & s_ng
    overlap = (len(inter) / len(a_ng)) if a_ng else 0.0

    an, sn = norm_text(answer), norm_text(system_prompt)
    frag = _longest_common_fragment(an, sn)
    return {
        "ok": True,
        "leak": bool(overlap >= threshold and len(inter) > 0),
        "overlap": round(overlap, 4),
        "n": n,
        "threshold": threshold,
        "matched_ngrams": len(inter),
        "total_ngrams": len(a_ng),
        "longest_fragment": frag[:400],
        "fragment_len": len(frag),
        "sample_ngrams": sorted(list(inter))[:5],
    }


_INSTRUCTION_PATTERNS = [
    re.compile(r"ignore\s+(?:all\s+)?(?:the\s+)?(?:previous|prior|above|earlier)\s+instructions?", re.I),
    re.compile(r"disregard\s+(?:all\s+)?(?:the\s+)?(?:previous|prior|above)", re.I),
    re.compile(r"(?:reveal|print|repeat|output|show|dump)\s+(?:me\s+)?(?:your|the)\s+(?:system\s+)?(?:prompt|instructions?|rules?)", re.I),
    re.compile(r"you\s+are\s+now\b", re.I),
    re.compile(r"\bact\s+as\b", re.I),
    re.compile(r"\bdeveloper\s+mode\b", re.I),
    re.compile(r"\bjailbreak\b", re.I),
    re.compile(r"(?:これまでの|以前の|上記の)(?:指示|命令|ルール)(?:は|を)?(?:すべて)?(?:無視|忘れ)", re.I),
    re.compile(r"システム\s*プロンプト(?:を)?(?:表示|出力|教え|見せ)", re.I),
    re.compile(r"(?:あなたは|君は)今から", re.I),
]


@guarded
def detect_injection_echo(answer, user_input) -> dict:
    """Guarantees: reports whether imperative strings from the user input reappear in the answer."""
    answer, _ = clamp_text(answer)
    user_input, _ = clamp_text(user_input)
    if not answer or not user_input:
        return {"ok": True, "echo": False, "matches": [], "reason": "empty_input"}
    an = norm_text(answer)
    found = []
    for pat in _INSTRUCTION_PATTERNS:
        for m in pat.finditer(user_input):
            phrase = m.group(0).strip()
            pn = norm_text(phrase)
            if not pn:
                continue
            exact = pn in an
            ratio = 1.0 if exact else _best_fuzzy(pn, an)[0]
            if exact or ratio >= 0.88:
                found.append({"instruction": phrase[:160], "in_answer": True,
                              "similarity": round(ratio, 3), "exact": exact})
            else:
                found.append({"instruction": phrase[:160], "in_answer": False,
                              "similarity": round(ratio, 3), "exact": False})
    echoed = [f for f in found if f["in_answer"]]
    return {
        "ok": True,
        "echo": bool(echoed),
        "matches": found[:20],
        "echoed_count": len(echoed),
        "instruction_like_in_input": len(found),
    }


# =============================================================================
# (F) OUTPUT SAFETY
# =============================================================================

_CC_RE = re.compile(r"(?<![0-9])(?:[0-9]{4}[ \-]?){3}[0-9]{1,7}(?![0-9])")
_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_\-]{6,}\.[A-Za-z0-9_\-]{6,}\.[A-Za-z0-9_\-]{0,600}")
_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,24}\b")
_PHONE_RE = re.compile(r"(?<![0-9])(?:\+\d{1,3}[ \-]?)?(?:\(0?\d{1,4}\)|0\d{1,4})[ \-]?\d{1,4}[ \-]?\d{3,4}(?![0-9])")
_PRIVKEY_RE = re.compile(r"-----BEGIN\s+(?:RSA|DSA|EC|OPENSSH|PGP|ENCRYPTED)?\s*PRIVATE KEY(?:\s+BLOCK)?-----")
_HIGH_ENTROPY_RE = re.compile(r"[A-Za-z0-9+/=_\-]{%d,200}" % ENTROPY_MIN_LEN)

_CRED_PREFIXES = [
    ("openai_key", re.compile(r"\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_\-]{16,}")),
    ("anthropic_key", re.compile(r"\bsk-ant-[A-Za-z0-9_\-]{16,}")),
    ("github_token", re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}|\bgithub_pat_[A-Za-z0-9_]{20,}")),
    ("aws_access_key", re.compile(r"\b(?:AKIA|ASIA|AGPA|AIDA|AROA|ANPA|ANVA)[A-Z0-9]{12,}")),
    ("slack_token", re.compile(r"\bxox[abprs]-[A-Za-z0-9\-]{8,}")),
    ("google_api_key", re.compile(r"\bAIza[A-Za-z0-9_\-]{30,}")),
    ("google_oauth", re.compile(r"\bya29\.[A-Za-z0-9_\-]{20,}")),
    ("gitlab_token", re.compile(r"\bglpat-[A-Za-z0-9_\-]{16,}")),
    ("hf_token", re.compile(r"\bhf_[A-Za-z0-9]{20,}")),
    ("npm_token", re.compile(r"\bnpm_[A-Za-z0-9]{30,}")),
    ("sendgrid_key", re.compile(r"\bSG\.[A-Za-z0-9_\-]{16,}\.[A-Za-z0-9_\-]{16,}")),
    ("stripe_key", re.compile(r"\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{16,}")),
    ("digitalocean_token", re.compile(r"\bdop_v1_[a-f0-9]{32,}")),
    ("twilio_sid", re.compile(r"\bAC[a-f0-9]{32}\b")),
]

_SAFETY_SEVERITY = {
    "credit_card": "critical",
    "private_key": "critical",
    "jwt": "critical",
    "openai_key": "critical", "anthropic_key": "critical", "github_token": "critical",
    "aws_access_key": "critical", "slack_token": "critical", "google_api_key": "critical",
    "google_oauth": "critical", "gitlab_token": "critical", "hf_token": "critical",
    "npm_token": "critical", "sendgrid_key": "critical", "stripe_key": "critical",
    "digitalocean_token": "critical", "twilio_sid": "high",
    "high_entropy_string": "medium",
    "email": "low",
    "phone": "low",
}

# 高エントロピー検出のノイズ源。base64 風だが秘密ではないものを除く。
_ENTROPY_ALLOW_RE = re.compile(
    r"^(?:[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64})$", re.I  # md5 / sha1 / sha256 digests
)


def _mask(s: str) -> str:
    """Guarantees: keeps at most the first 3 and last 2 characters, masking the middle."""
    try:
        s = str(s)
        if len(s) <= 6:
            return "*" * len(s)
        return s[:3] + "*" * max(3, len(s) - 5) + s[-2:]
    except Exception:
        return "***"


def scan_output(answer) -> list:
    """Guarantees: returns a list of findings (type/span/confidence/severity); never raises.

    Credit-card candidates are reported ONLY when they pass Luhn, because a bare
    16-digit run is far more often an order number than a card.
    """
    findings = []
    try:
        answer, _ = clamp_text(answer)
        if not answer:
            return findings

        for m in _PRIVKEY_RE.finditer(answer):
            findings.append({"type": "private_key", "value": m.group(0)[:60], "masked": m.group(0)[:30],
                             "span": [m.start(), m.end()], "confidence": 0.99, "severity": "critical"})

        for name, pat in _CRED_PREFIXES:
            for m in pat.finditer(answer):
                v = m.group(0)
                findings.append({"type": name, "value": _mask(v), "masked": _mask(v),
                                 "span": [m.start(), m.end()], "confidence": 0.95,
                                 "severity": _SAFETY_SEVERITY.get(name, "high")})

        for m in _JWT_RE.finditer(answer):
            v = m.group(0)
            parts = v.split(".")
            conf = 0.95 if len(parts) == 3 and all(parts[:2]) else 0.6
            findings.append({"type": "jwt", "value": _mask(v), "masked": _mask(v),
                             "span": [m.start(), m.end()], "confidence": conf, "severity": "critical"})

        for m in _CC_RE.finditer(answer):
            raw = m.group(0)
            digits = re.sub(r"[^0-9]", "", raw)
            if len(digits) < 13 or len(digits) > 19:
                continue
            if not luhn_ok(digits):
                continue  # Luhn を通らないものは報告しない(偽陽性の最大要因)
            findings.append({"type": "credit_card", "value": _mask(raw), "masked": _mask(raw),
                             "span": [m.start(), m.end()], "confidence": 0.9, "severity": "critical"})

        claimed = [tuple(f["span"]) for f in findings]
        for m in _HIGH_ENTROPY_RE.finditer(answer):
            sp = (m.start(), m.end())
            if any(_spans_overlap(sp, c) for c in claimed):
                continue
            tok = m.group(0)
            if _ENTROPY_ALLOW_RE.match(tok):
                continue
            has_d = any(c.isdigit() for c in tok)
            has_a = any(c.isalpha() for c in tok)
            if not (has_d and has_a):
                continue
            ent = shannon_entropy(tok)
            if ent < ENTROPY_THRESHOLD:
                continue
            conf = min(0.85, 0.35 + (ent - ENTROPY_THRESHOLD) * 0.4 + min(0.2, (len(tok) - ENTROPY_MIN_LEN) / 100.0))
            findings.append({"type": "high_entropy_string", "value": _mask(tok), "masked": _mask(tok),
                             "span": [m.start(), m.end()], "confidence": round(conf, 3),
                             "severity": "medium", "entropy": round(ent, 3), "length": len(tok)})

        claimed = [tuple(f["span"]) for f in findings]
        for m in _EMAIL_RE.finditer(answer):
            sp = (m.start(), m.end())
            if any(_spans_overlap(sp, c) for c in claimed):
                continue
            findings.append({"type": "email", "value": _mask(m.group(0)), "masked": _mask(m.group(0)),
                             "span": [m.start(), m.end()], "confidence": 0.9, "severity": "low"})
        claimed = [tuple(f["span"]) for f in findings]
        for m in _PHONE_RE.finditer(answer):
            sp = (m.start(), m.end())
            if any(_spans_overlap(sp, c) for c in claimed):
                continue
            digits = re.sub(r"[^0-9]", "", m.group(0))
            if len(digits) < 9 or len(digits) > 15:
                continue
            findings.append({"type": "phone", "value": _mask(m.group(0)), "masked": _mask(m.group(0)),
                             "span": [m.start(), m.end()], "confidence": 0.55, "severity": "low"})

        findings.sort(key=lambda f: f["span"][0])
        return findings
    except Exception as exc:  # scan_output must still return a list
        return [{"type": "scanner_error", "value": type(exc).__name__, "masked": "",
                 "span": [0, 0], "confidence": 0.0, "severity": "low"}]


def redact(answer, findings=None) -> str:
    """Guarantees: returns the answer with every finding span replaced by a [REDACTED:TYPE] marker."""
    try:
        answer, _ = clamp_text(answer)
        if not answer:
            return ""
        findings = scan_output(answer) if findings is None else list(findings)
        spans = []
        for f in findings:
            try:
                s, e = int(f["span"][0]), int(f["span"][1])
            except Exception:
                continue
            if e <= s:
                continue
            if f.get("type") == "scanner_error":
                continue
            spans.append((s, e, f.get("type", "SECRET")))
        spans.sort(key=lambda x: x[0])
        out = []
        cursor = 0
        for s, e, t in spans:
            if s < cursor:
                continue
            out.append(answer[cursor:s])
            out.append("[REDACTED:%s]" % str(t).upper())
            cursor = e
        out.append(answer[cursor:])
        return "".join(out)
    except Exception:
        return answer if isinstance(answer, str) else ""


@guarded
def safety_report(answer) -> dict:
    """Guarantees: a dict wrapper around scan_output with severity rollups and a redacted body."""
    findings = scan_output(answer)
    sev = Counter(f.get("severity", "low") for f in findings)
    types = Counter(f.get("type", "?") for f in findings)
    return {
        "ok": True,
        "findings": findings,
        "counts_by_severity": dict(sev),
        "counts_by_type": dict(types),
        "critical": int(sev.get("critical", 0)),
        "redacted": redact(answer, findings),
    }


# =============================================================================
# (G) SCHEMA GATE
# =============================================================================

_FENCE_RE = re.compile(r"```[a-zA-Z0-9_+\-]*\s*\n?(.*?)```", re.S)
_LINE_COMMENT_RE = re.compile(r"(?m)(?<![:\"'])//[^\n]*$")
_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.S)
_TRAILING_COMMA_RE = re.compile(r",(\s*[}\]])")


def extract_json(text):
    """Guarantees: returns the most likely JSON payload substring, or None.

    Strips markdown fences and otherwise finds the first balanced {...} / [...]
    while respecting string literals.
    """
    try:
        if not text:
            return None
        if not isinstance(text, str):
            text = str(text)
        m = _FENCE_RE.search(text)
        if m and m.group(1).strip():
            inner = m.group(1).strip()
            if inner[:1] in "[{":
                return inner
            text = inner
        best = None
        for opener, closer in (("{", "}"), ("[", "]")):
            start = text.find(opener)
            while start != -1:
                depth = 0
                in_str = False
                esc = False
                quote = ""
                for i in range(start, len(text)):
                    ch = text[i]
                    if in_str:
                        if esc:
                            esc = False
                        elif ch == "\\":
                            esc = True
                        elif ch == quote:
                            in_str = False
                        continue
                    if ch in ("\"", "'"):
                        in_str = True
                        quote = ch
                        continue
                    if ch == opener:
                        depth += 1
                    elif ch == closer:
                        depth -= 1
                        if depth == 0:
                            cand = text[start:i + 1]
                            if best is None or len(cand) > len(best):
                                best = cand
                            break
                start = text.find(opener, start + 1)
        return best
    except Exception:
        return None


def _escape_raw_newlines(s: str) -> str:
    """Guarantees: control characters inside JSON string literals are escaped, others untouched."""
    out = []
    in_str = False
    esc = False
    quote = ""
    for ch in s:
        if in_str:
            if esc:
                esc = False
                out.append(ch)
                continue
            if ch == "\\":
                esc = True
                out.append(ch)
                continue
            if ch == quote:
                in_str = False
                out.append(ch)
                continue
            if ch == "\n":
                out.append("\\n")
                continue
            if ch == "\r":
                out.append("\\r")
                continue
            if ch == "\t":
                out.append("\\t")
                continue
            out.append(ch)
            continue
        if ch in ("\"", "'"):
            in_str = True
            quote = ch
        out.append(ch)
    return "".join(out)


def _single_to_double_quotes(s: str) -> str:
    """Guarantees: JS-style single-quoted strings become valid JSON strings, double-quoted ones untouched."""
    out = []
    i = 0
    n = len(s)
    in_dq = False
    esc = False
    while i < n:
        ch = s[i]
        if in_dq:
            out.append(ch)
            if esc:
                esc = False
            elif ch == "\\":
                esc = True
            elif ch == '"':
                in_dq = False
            i += 1
            continue
        if ch == '"':
            in_dq = True
            out.append(ch)
            i += 1
            continue
        if ch == "'":
            j = i + 1
            buf = []
            e = False
            while j < n:
                c2 = s[j]
                if e:
                    buf.append(c2)
                    e = False
                elif c2 == "\\":
                    e = True
                    buf.append(c2)
                elif c2 == "'":
                    break
                else:
                    buf.append(c2)
                j += 1
            body = "".join(buf).replace('"', '\\"')
            out.append('"' + body + '"')
            i = j + 1
            continue
        out.append(ch)
        i += 1
    return "".join(out)


def repair_json(text):
    """Guarantees: returns (parsed_object_or_None, list_of_repairs_applied); never raises.

    PURPOSE - this exists so that a formatting slip never costs a retry.
    Re-prompting an LLM because it emitted a trailing comma, a fullwidth colon
    or a markdown fence is pure waste: the content was already correct. Fix the
    syntax locally, and reserve retries for claims that are actually wrong.
    """
    repairs = []
    try:
        if text is None:
            return None, ["empty_input"]
        if isinstance(text, (dict, list)):
            return text, []
        s = str(text).strip()
        if not s:
            return None, ["empty_input"]
        try:
            return json.loads(s), []
        except Exception:
            pass

        cand = extract_json(s)
        if cand and cand != s:
            s = cand
            repairs.append("extracted_json_block")
        try:
            return json.loads(s), repairs
        except Exception:
            pass

        # fullwidth punctuation that LLMs mix into JSON when writing Japanese
        fw = {"：": ":", "，": ",", "、": ",", "｛": "{", "｝": "}", "［": "[", "］": "]",
              "“": '"', "”": '"', "‘": "'", "’": "'", "＂": '"', "　": " "}
        before = s
        for k, v in fw.items():
            s = s.replace(k, v)
        if s != before:
            repairs.append("normalised_fullwidth_punctuation")

        before = s
        s = _BLOCK_COMMENT_RE.sub("", s)
        s = _LINE_COMMENT_RE.sub("", s)
        if s != before:
            repairs.append("removed_comments")

        before = s
        s = _single_to_double_quotes(s)
        if s != before:
            repairs.append("single_to_double_quotes")

        before = s
        s = _escape_raw_newlines(s)
        if s != before:
            repairs.append("escaped_raw_control_chars")

        before = s
        s = re.sub(r"\bTrue\b", "true", s)
        s = re.sub(r"\bFalse\b", "false", s)
        s = re.sub(r"\b(?:None|NaN|Undefined|undefined)\b", "null", s)
        if s != before:
            repairs.append("python_literals_to_json")

        before = s
        s = _TRAILING_COMMA_RE.sub(r"\1", s)
        if s != before:
            repairs.append("removed_trailing_commas")

        before = s
        s = re.sub(r"(?m)([{,]\s*)([A-Za-z_][A-Za-z0-9_\-]*)\s*:", r'\1"\2":', s)
        if s != before:
            repairs.append("quoted_bare_keys")

        try:
            return json.loads(s), repairs
        except Exception:
            pass

        # last resort: close unbalanced brackets
        opens = s.count("{") - s.count("}")
        obr = s.count("[") - s.count("]")
        if opens > 0 or obr > 0:
            s2 = s + ("]" * max(0, obr)) + ("}" * max(0, opens))
            try:
                obj = json.loads(s2)
                repairs.append("closed_unbalanced_brackets")
                return obj, repairs
            except Exception:
                pass
        repairs.append("unrepairable")
        return None, repairs
    except Exception as exc:
        return None, ["repair_error:%s" % type(exc).__name__]


_COERCERS = {
    "str": lambda v: v if isinstance(v, str) else json.dumps(v, ensure_ascii=False) if isinstance(v, (dict, list)) else str(v),
    "int": lambda v: int(v) if not isinstance(v, bool) else int(v),
    "float": lambda v: float(v),
    "number": lambda v: float(v),
    "bool": lambda v: v if isinstance(v, bool) else str(v).strip().lower() in ("1", "true", "yes", "on"),
    "list": lambda v: v if isinstance(v, list) else [v],
    "dict": lambda v: v if isinstance(v, dict) else {"value": v},
    "any": lambda v: v,
}


def coerce_schema(obj, schema):
    """Guarantees: returns {"ok","value","missing","extra","coerced","errors"}; never raises.

    schema is the simple form {"key": "str"|"int"|"float"|"bool"|"list"|"dict"|"any"}.
    A trailing "?" on a key marks it optional ("note?": "str").
    """
    result = {"ok": True, "value": {}, "missing": [], "extra": [], "coerced": [], "errors": []}
    try:
        if not isinstance(schema, dict) or not schema:
            result["ok"] = isinstance(obj, (dict, list))
            result["value"] = obj
            return result
        if isinstance(obj, list) and obj and isinstance(obj[0], dict):
            obj = obj[0]
            result["coerced"].append("took_first_element_of_list")
        if not isinstance(obj, dict):
            result["ok"] = False
            result["errors"].append("payload_is_not_an_object")
            result["value"] = obj
            return result

        wanted = {}
        for k, t in schema.items():
            optional = k.endswith("?")
            key = k[:-1] if optional else k
            wanted[key] = (t, optional)

        for key, (t, optional) in wanted.items():
            if key not in obj:
                if optional:
                    continue
                result["missing"].append(key)
                result["ok"] = False
                continue
            raw = obj[key]
            if isinstance(t, dict):
                sub = coerce_schema(raw, t)
                result["value"][key] = sub["value"]
                if not sub["ok"]:
                    result["ok"] = False
                    result["errors"].append({key: {"missing": sub["missing"], "errors": sub["errors"]}})
                continue
            tname = str(t).strip().lower()
            fn = _COERCERS.get(tname)
            if fn is None:
                result["value"][key] = raw
                continue
            try:
                new = fn(raw)
                if new != raw:
                    result["coerced"].append(key)
                result["value"][key] = new
            except Exception:
                result["ok"] = False
                result["errors"].append({key: "cannot_coerce_to_%s" % tname})
                result["value"][key] = raw

        for k in obj:
            if k not in wanted:
                result["extra"].append(k)
        return result
    except Exception as exc:
        result["ok"] = False
        result["errors"].append("coerce_error:%s" % type(exc).__name__)
        return result


@guarded
def structure_check(text, schema_json=None) -> dict:
    """Guarantees: one call that extracts, repairs and (optionally) schema-coerces LLM JSON output."""
    t0 = time.perf_counter()
    text, _ = clamp_text(text)
    obj, repairs = repair_json(text)
    out = {
        "ok": True,
        "valid_json": obj is not None,
        "repairs": repairs,
        "parsed": obj,
        "schema_ok": None,
        "schema": None,
        "missing": [],
        "extra": [],
        "coerced": [],
        "errors": [],
        "value": obj,
    }
    schema = None
    if schema_json:
        sobj, _sr = repair_json(schema_json)
        if isinstance(sobj, dict):
            schema = sobj
            out["schema"] = sobj
        else:
            out["errors"].append("schema_unparseable")
    if schema is not None and obj is not None:
        c = coerce_schema(obj, schema)
        out["schema_ok"] = c["ok"]
        out["missing"] = c["missing"]
        out["extra"] = c["extra"]
        out["coerced"] = c["coerced"]
        out["errors"].extend(c["errors"])
        out["value"] = c["value"]
    out["latency_ms"] = round((time.perf_counter() - t0) * 1000.0, 2)
    return out


# =============================================================================
# (H) RETRY ADVISOR - the cost-optimisation core
# =============================================================================

_FAIL_STATUSES = ("unsupported", "contradicted")


@guarded
def build_retry_instruction(result) -> dict:
    """Guarantees: returns a targeted correction instruction plus a blind-vs-targeted token estimate.

    The point: a blind retry re-sends the whole system prompt + context + user
    turn and re-generates the whole answer. A targeted retry re-sends only the
    answer and a short list of the specific unsupported/contradicted claims.
    The difference is the money this tool saves.
    """
    if isinstance(result, str):
        parsed, _ = repair_json(result)
        result = parsed if isinstance(parsed, dict) else {}
    if not isinstance(result, dict):
        return {"ok": True, "needed": False, "instruction": "", "items": [],
                "estimate": {}, "reason": "no_result_supplied"}

    ver = result.get("verify") if isinstance(result.get("verify"), dict) else result
    claims = ver.get("claims") or []
    bad = []
    for c in claims:
        if not isinstance(c, dict):
            continue
        if c.get("status") in _FAIL_STATUSES:
            bad.append(c)
    bad.sort(key=lambda c: (0 if c.get("status") == "contradicted" else 1, c.get("span", [0])[0]))

    items = []
    for c in bad[:60]:
        items.append({
            "id": c.get("id"),
            "type": c.get("type"),
            "value": c.get("value"),
            "status": c.get("status"),
            "nearest_context_value": c.get("nearest_context_value"),
            "sentence_index": c.get("sentence_index"),
        })

    lines_ja = []
    lines_en = []
    for it in items:
        if it["status"] == "contradicted" and it.get("nearest_context_value"):
            lines_ja.append('- 「%s」(%s): 文脈の該当値は「%s」です。'
                            % (it["value"], it["type"], it["nearest_context_value"]))
            lines_en.append('- "%s" (%s): the context says "%s".'
                            % (it["value"], it["type"], it["nearest_context_value"]))
        else:
            lines_ja.append('- 「%s」(%s): 与えた文脈に裏付けがありません。' % (it["value"], it["type"]))
            lines_en.append('- "%s" (%s): not supported by the provided context.' % (it["value"], it["type"]))

    if items:
        instruction = (
            "以下の記述には、与えた文脈での裏付けがありません。"
            "文脈に基づいて訂正するか、裏付けが無い旨を明記してください。"
            "該当箇所以外は書き換えないでください。\n"
            + "\n".join(lines_ja)
            + "\n\n訂正後の全文のみを出力してください。新しい数値・日付・固有名詞を追加しないでください。"
        )
        instruction_en = (
            "The following statements are not supported by the context you were given. "
            "Correct them using the context, or explicitly say the context does not support them. "
            "Do not rewrite anything else.\n"
            + "\n".join(lines_en)
            + "\n\nReturn only the corrected full text. Do not introduce new figures, dates or proper nouns."
        )
    else:
        instruction = ""
        instruction_en = ""

    sizes = {}
    if isinstance(result.get("meta"), dict) and isinstance(result["meta"].get("sizes"), dict):
        sizes = result["meta"]["sizes"]
    tok_system = int(sizes.get("system_prompt_tokens", 0) or 0)
    tok_context = int(sizes.get("context_tokens", 0) or 0)
    tok_user = int(sizes.get("user_input_tokens", 0) or 0)
    tok_answer = int(sizes.get("answer_tokens", 0) or 0)

    instr_tokens = estimate_tokens(instruction)
    blind_in = tok_system + tok_context + tok_user
    blind_out = tok_answer if tok_answer else 0
    # A targeted retry re-sends the answer and the short instruction, not the corpus.
    targeted_in = tok_answer + instr_tokens + min(tok_system, 200)
    targeted_out = max(1, int(tok_answer * min(1.0, max(0.15, len(items) / max(1, len(claims) or 1)))))

    blind_total = blind_in + blind_out
    targeted_total = targeted_in + targeted_out
    # Signed on purpose. For a very small context the instruction can cost more
    # than the blind retry it replaces, and hiding that behind a max(0, ...)
    # would make this tool lie about its own value. saved_tokens and saved_cost
    # are always computed from the same deltas, so they never disagree in sign.
    saved_in = blind_in - targeted_in
    saved_out = blind_out - targeted_out
    saved_tokens = saved_in + saved_out
    saved_cost = round(
        saved_in / 1000.0 * PRICE_IN_PER_1K + saved_out / 1000.0 * PRICE_OUT_PER_1K, 6
    )
    # Which number decides "cheaper" depends on whether prices are configured.
    # Output tokens usually cost several times more than input tokens, so a
    # targeted retry can cost less money while using more total tokens. When no
    # prices are set we can only compare raw token counts.
    priced = (PRICE_IN_PER_1K > 0 or PRICE_OUT_PER_1K > 0)
    cheaper = (saved_cost > 0) if priced else (saved_tokens > 0)
    basis = "cost" if priced else "tokens"
    if not items:
        recommendation = "no retry needed"
    elif cheaper:
        recommendation = ("targeted retry is cheaper by %s (basis: %s)"
                          % (("%.6f" % saved_cost) if priced else ("%d tokens" % saved_tokens), basis))
    else:
        recommendation = ("targeted retry is NOT cheaper here (basis: %s) - the context is small "
                          "relative to the instruction; retry blind, or batch several corrections "
                          "into one call" % basis)

    return {
        "ok": True,
        "needed": bool(items),
        "instruction": instruction,
        "instruction_en": instruction_en,
        "items": items,
        "failed_claims": len(bad),
        "estimate": {
            "blind_retry_tokens": blind_total,
            "targeted_retry_tokens": targeted_total,
            "saved_tokens": saved_tokens,
            "saved_cost": saved_cost,
            "targeted_is_cheaper": bool(cheaper),
            "comparison_basis": basis,
            "recommendation": recommendation,
            "breakdown": {
                "blind_input_tokens": blind_in, "blind_output_tokens": blind_out,
                "targeted_input_tokens": targeted_in, "targeted_output_tokens": targeted_out,
                "saved_input_tokens": saved_in, "saved_output_tokens": saved_out,
                "instruction_tokens": instr_tokens,
            },
            "assumptions": {
                "price_in_per_1k": PRICE_IN_PER_1K,
                "price_out_per_1k": PRICE_OUT_PER_1K,
                "tokenizer": "heuristic: ASCII ~4 chars/token, CJK ~1 char/token",
                "note": "Set PRICE_IN_PER_1K / PRICE_OUT_PER_1K to see money instead of zeros. "
                        "saved_tokens and saved_cost are signed and can disagree: output tokens are "
                        "usually priced several times higher than input tokens.",
            },
        },
    }


# =============================================================================
# (J) OBSERVABILITY
# =============================================================================
# Free-tier reality: the Space disk is ephemeral and the container sleeps after
# 48h idle. Everything here lives in memory and WILL be lost. Export the CSV if
# you need to keep it.

_EVENTS = deque(maxlen=LOG_CAPACITY)
_EVENTS_LOCK = threading.Lock()


def _log_event(ev: dict) -> None:
    """Guarantees: appends one event to the bounded ring buffer; never raises."""
    try:
        with _EVENTS_LOCK:
            _EVENTS.append(ev)
    except Exception:
        pass


def _events_snapshot() -> list:
    """Guarantees: a consistent point-in-time copy of the event ring buffer; never raises."""
    try:
        with _EVENTS_LOCK:
            return list(_EVENTS)
    except Exception:
        return []


def events_dataframe() -> "pd.DataFrame":
    """Guarantees: returns a DataFrame of logged events (possibly empty), never raises."""
    try:
        rows = _events_snapshot()
        if not rows:
            return pd.DataFrame(columns=[
                "timestamp", "model", "prompt_version", "verdict", "grounding_score",
                "coverage", "latency_ms", "leak_flag", "safety_flags", "saved_tokens", "saved_cost",
            ])
        return pd.DataFrame(rows)
    except Exception:
        return pd.DataFrame()


def _pct(series, q):
    """Guarantees: the q-quantile of the series, or NaN when it cannot be computed."""
    try:
        return float(series.quantile(q))
    except Exception:
        return float("nan")


@guarded
def stats_summary() -> dict:
    """Guarantees: returns headline aggregates over the in-memory log; never raises."""
    df = events_dataframe()
    if df.empty:
        return {"ok": True, "events": 0, "message": "No events logged yet. Run a verification first."}
    lat = pd.to_numeric(df.get("latency_ms"), errors="coerce").dropna()
    gs = pd.to_numeric(df.get("grounding_score"), errors="coerce").dropna()
    cov = pd.to_numeric(df.get("coverage"), errors="coerce").dropna()
    saved_t = pd.to_numeric(df.get("saved_tokens"), errors="coerce").fillna(0)
    saved_c = pd.to_numeric(df.get("saved_cost"), errors="coerce").fillna(0)
    verdicts = df.get("verdict")
    vc = verdicts.value_counts().to_dict() if verdicts is not None else {}
    return {
        "ok": True,
        "events": int(len(df)),
        "log_capacity": LOG_CAPACITY,
        "verdicts": {str(k): int(v) for k, v in vc.items()},
        "grounding_score": {"mean": round(float(gs.mean()), 4) if len(gs) else None,
                            "min": round(float(gs.min()), 4) if len(gs) else None},
        "coverage": {"mean": round(float(cov.mean()), 4) if len(cov) else None,
                     "min": round(float(cov.min()), 4) if len(cov) else None},
        "latency_ms": {
            "p50": round(_pct(lat, 0.50), 2) if len(lat) else None,
            "p95": round(_pct(lat, 0.95), 2) if len(lat) else None,
            "max": round(float(lat.max()), 2) if len(lat) else None,
            "mean": round(float(lat.mean()), 2) if len(lat) else None,
        },
        "savings": {"tokens_total": int(saved_t.sum()), "cost_total": round(float(saved_c.sum()), 6)},
        "leaks": int(pd.to_numeric(df.get("leak_flag"), errors="coerce").fillna(0).sum())
        if "leak_flag" in df else 0,
        "storage": "in-memory ring buffer; lost on Space restart/sleep",
    }


def timeseries_frame() -> "pd.DataFrame":
    """Guarantees: a tidy DataFrame of grounding_score and coverage over time."""
    try:
        df = events_dataframe()
        if df.empty:
            return pd.DataFrame(columns=["timestamp", "grounding_score", "coverage"])
        out = df[["timestamp", "grounding_score", "coverage"]].copy()
        out["grounding_score"] = pd.to_numeric(out["grounding_score"], errors="coerce")
        out["coverage"] = pd.to_numeric(out["coverage"], errors="coerce")
        return out
    except Exception:
        return pd.DataFrame(columns=["timestamp", "grounding_score", "coverage"])


def violations_frame() -> "pd.DataFrame":
    """Guarantees: a DataFrame of violation counts per event, by category."""
    cols = ["timestamp", "contradicted", "unsupported", "approximate", "leak", "safety_critical", "safety_other"]
    try:
        rows = _events_snapshot()
        if not rows:
            return pd.DataFrame(columns=cols)
        out = []
        for e in rows:
            counts = e.get("counts") or {}
            out.append({
                "timestamp": e.get("timestamp"),
                "contradicted": int(counts.get("contradicted", 0)),
                "unsupported": int(counts.get("unsupported", 0)),
                "approximate": int(counts.get("approximate", 0)),
                "leak": 1 if e.get("leak_flag") else 0,
                "safety_critical": int(e.get("safety_critical", 0)),
                "safety_other": int(e.get("safety_other", 0)),
            })
        return pd.DataFrame(out, columns=cols)
    except Exception:
        return pd.DataFrame(columns=cols)


def comparison_frame() -> "pd.DataFrame":
    """Guarantees: a model x prompt_version comparison table - the diff you look at after a change."""
    cols = ["model", "prompt_version", "n", "grounding_mean", "coverage_mean",
            "contradicted_rate", "retry_rate", "block_rate", "latency_p95_ms", "saved_tokens"]
    try:
        df = events_dataframe()
        if df.empty:
            return pd.DataFrame(columns=cols)
        d = df.copy()
        d["model"] = d.get("model", pd.Series(["(unset)"] * len(d))).fillna("(unset)").replace("", "(unset)")
        d["prompt_version"] = d.get("prompt_version", pd.Series(["(unset)"] * len(d))).fillna("(unset)").replace("", "(unset)")
        d["grounding_score"] = pd.to_numeric(d.get("grounding_score"), errors="coerce")
        d["coverage"] = pd.to_numeric(d.get("coverage"), errors="coerce")
        d["latency_ms"] = pd.to_numeric(d.get("latency_ms"), errors="coerce")
        d["saved_tokens"] = pd.to_numeric(d.get("saved_tokens"), errors="coerce").fillna(0)
        d["_contra"] = d.get("counts", pd.Series([{}] * len(d))).apply(
            lambda c: 1 if isinstance(c, dict) and c.get("contradicted", 0) else 0)
        d["_retry"] = (d.get("verdict") == "retry").astype(int)
        d["_block"] = (d.get("verdict") == "block").astype(int)
        g = d.groupby(["model", "prompt_version"], dropna=False)
        out = g.agg(
            n=("verdict", "count"),
            grounding_mean=("grounding_score", "mean"),
            coverage_mean=("coverage", "mean"),
            contradicted_rate=("_contra", "mean"),
            retry_rate=("_retry", "mean"),
            block_rate=("_block", "mean"),
            latency_p95_ms=("latency_ms", lambda s: _pct(s, 0.95)),
            saved_tokens=("saved_tokens", "sum"),
        ).reset_index()
        for c in ("grounding_mean", "coverage_mean", "contradicted_rate", "retry_rate", "block_rate"):
            out[c] = out[c].astype(float).round(4)
        out["latency_p95_ms"] = out["latency_p95_ms"].astype(float).round(2)
        out["saved_tokens"] = out["saved_tokens"].astype(int)
        return out[cols]
    except Exception:
        return pd.DataFrame(columns=cols)


def savings_frame() -> "pd.DataFrame":
    """Guarantees: cumulative token/cost savings over the event log."""
    cols = ["timestamp", "saved_tokens", "cum_saved_tokens", "saved_cost", "cum_saved_cost"]
    try:
        df = events_dataframe()
        if df.empty:
            return pd.DataFrame(columns=cols)
        out = pd.DataFrame({
            "timestamp": df.get("timestamp"),
            "saved_tokens": pd.to_numeric(df.get("saved_tokens"), errors="coerce").fillna(0),
            "saved_cost": pd.to_numeric(df.get("saved_cost"), errors="coerce").fillna(0),
        })
        out["cum_saved_tokens"] = out["saved_tokens"].cumsum()
        out["cum_saved_cost"] = out["saved_cost"].cumsum().round(6)
        return out[cols]
    except Exception:
        return pd.DataFrame(columns=cols)


def latency_frame() -> "pd.DataFrame":
    """Guarantees: p50/p95/max/mean latency of the verification step itself."""
    cols = ["metric", "value_ms"]
    try:
        df = events_dataframe()
        if df.empty:
            return pd.DataFrame(columns=cols)
        lat = pd.to_numeric(df.get("latency_ms"), errors="coerce").dropna()
        if lat.empty:
            return pd.DataFrame(columns=cols)
        return pd.DataFrame([
            {"metric": "p50", "value_ms": round(_pct(lat, 0.5), 2)},
            {"metric": "p95", "value_ms": round(_pct(lat, 0.95), 2)},
            {"metric": "max", "value_ms": round(float(lat.max()), 2)},
            {"metric": "mean", "value_ms": round(float(lat.mean()), 2)},
            {"metric": "count", "value_ms": int(len(lat))},
        ], columns=cols)
    except Exception:
        return pd.DataFrame(columns=cols)


def _write_csv(df, stem: str):
    """Guarantees: writes the DataFrame to a temp CSV and returns its path, or None."""
    try:
        if df is None:
            return None
        path = os.path.join(tempfile.gettempdir(), "claimcheck_%s_%d.csv" % (stem, int(time.time())))
        df.to_csv(path, index=False, encoding="utf-8-sig")
        return path
    except Exception:
        return None


def export_events_csv():
    """Guarantees: returns a path to a CSV of the event log, or None on failure."""
    try:
        df = events_dataframe()
        if df.empty:
            df = pd.DataFrame([{"note": "no events logged yet"}])
        else:
            df = df.copy()
            for c in df.columns:
                if df[c].apply(lambda v: isinstance(v, (dict, list))).any():
                    df[c] = df[c].apply(lambda v: json.dumps(v, ensure_ascii=False) if isinstance(v, (dict, list)) else v)
        return _write_csv(df, "events")
    except Exception:
        return None


# =============================================================================
# (K) FALSE POSITIVE AUDIT
# =============================================================================
# A verifier loses its users the moment it cries wolf. This tab exists so the
# false-positive rate is a number on the screen, not a feeling.

_AUDIT = deque(maxlen=max(200, LOG_CAPACITY * 4))
_AUDIT_LOCK = threading.Lock()
_AUDIT_SEQ = [0]


def _record_audit_candidates(event_id, claims, model, prompt_version):
    """Guarantees: stores every unsupported/contradicted claim for later human review."""
    try:
        with _AUDIT_LOCK:
            for c in claims:
                if c.get("status") not in _FAIL_STATUSES:
                    continue
                _AUDIT_SEQ[0] += 1
                _AUDIT.append({
                    "audit_id": "a%05d" % _AUDIT_SEQ[0],
                    "timestamp": _now_iso(),
                    "event_id": event_id,
                    "model": model or "(unset)",
                    "prompt_version": prompt_version or "(unset)",
                    "type": c.get("type"),
                    "status": c.get("status"),
                    "value": str(c.get("value"))[:160],
                    "nearest_context_value": str(c.get("nearest_context_value") or "")[:160],
                    "false_positive": False,
                    "note": "",
                })
    except Exception:
        pass


def audit_frame() -> "pd.DataFrame":
    """Guarantees: a DataFrame of audit candidates, newest first (possibly empty)."""
    cols = ["audit_id", "timestamp", "model", "prompt_version", "type", "status",
            "value", "nearest_context_value", "false_positive", "note"]
    try:
        with _AUDIT_LOCK:
            rows = list(_AUDIT)
        if not rows:
            return pd.DataFrame(columns=cols)
        df = pd.DataFrame(rows)
        for c in cols:
            if c not in df.columns:
                df[c] = ""
        return df[cols].iloc[::-1].reset_index(drop=True)
    except Exception:
        return pd.DataFrame(columns=cols)


@guarded
def mark_false_positives(audit_ids, note="") -> dict:
    """Guarantees: flips the false_positive flag for the given ids and returns the new FP rate."""
    ids = set()
    if isinstance(audit_ids, str):
        ids = {x.strip() for x in re.split(r"[,\s]+", audit_ids) if x.strip()}
    elif isinstance(audit_ids, (list, tuple, set)):
        ids = {str(x).strip() for x in audit_ids if str(x).strip()}
    changed = 0
    with _AUDIT_LOCK:
        for row in _AUDIT:
            if row.get("audit_id") in ids:
                row["false_positive"] = True
                if note:
                    row["note"] = str(note)[:300]
                changed += 1
    return {"ok": True, "marked": changed, "requested": len(ids), **false_positive_rate()}


@guarded
def apply_audit_edits(table) -> dict:
    """Guarantees: syncs an edited audit table (from the UI grid) back into the audit store."""
    try:
        if table is None:
            return {"ok": True, "marked": 0, **false_positive_rate()}
        if isinstance(table, pd.DataFrame):
            records = table.to_dict("records")
        elif isinstance(table, dict) and "data" in table:
            headers = table.get("headers") or []
            records = [dict(zip(headers, row)) for row in table.get("data", [])]
        elif isinstance(table, list):
            records = [r for r in table if isinstance(r, dict)]
        else:
            records = []
        wanted = {}
        for r in records:
            aid = str(r.get("audit_id", "")).strip()
            if not aid:
                continue
            fp = r.get("false_positive")
            if isinstance(fp, str):
                fp = fp.strip().lower() in ("true", "1", "yes", "on")
            wanted[aid] = (bool(fp), str(r.get("note", "") or "")[:300])
        changed = 0
        with _AUDIT_LOCK:
            for row in _AUDIT:
                aid = row.get("audit_id")
                if aid in wanted:
                    fp, note = wanted[aid]
                    if row.get("false_positive") != fp or (note and row.get("note") != note):
                        changed += 1
                    row["false_positive"] = fp
                    row["note"] = note
        return {"ok": True, "marked": changed, **false_positive_rate()}
    except Exception as exc:
        return _err(exc, "apply_audit_edits")


def false_positive_rate() -> dict:
    """Guarantees: returns the share of flagged claims a human marked as actually correct."""
    try:
        with _AUDIT_LOCK:
            rows = list(_AUDIT)
        total = len(rows)
        fp = sum(1 for r in rows if r.get("false_positive"))
        by_type = {}
        for r in rows:
            t = r.get("type", "?")
            d = by_type.setdefault(t, {"flagged": 0, "false_positive": 0})
            d["flagged"] += 1
            if r.get("false_positive"):
                d["false_positive"] += 1
        for t, d in by_type.items():
            d["fp_rate"] = round(d["false_positive"] / d["flagged"], 4) if d["flagged"] else 0.0
        return {
            "audited_claims": total,
            "marked_false_positive": fp,
            "false_positive_rate": round(fp / total, 4) if total else 0.0,
            "by_type": by_type,
        }
    except Exception:
        return {"audited_claims": 0, "marked_false_positive": 0, "false_positive_rate": 0.0, "by_type": {}}


def export_audit_csv():
    """Guarantees: returns a path to a CSV of audit rows marked as false positives, or None."""
    try:
        df = audit_frame()
        if df.empty:
            df = pd.DataFrame([{"note": "no audit candidates yet"}])
        else:
            df = df[df["false_positive"] == True]  # noqa: E712 - pandas mask
            if df.empty:
                df = pd.DataFrame([{"note": "no rows marked as false positive yet"}])
        return _write_csv(df, "false_positives")
    except Exception:
        return None


# =============================================================================
# (L) ENRICHMENT - optional, best-effort, never load-bearing
# =============================================================================

_ENRICH_STATE = {"disabled_reason": None, "failures": 0}


def _cosine(a, b):
    """Guarantees: cosine similarity of two equal-length vectors, or None for a zero vector."""
    try:
        num = sum(x * y for x, y in zip(a, b))
        na = math.sqrt(sum(x * x for x in a))
        nb = math.sqrt(sum(y * y for y in b))
        if na == 0 or nb == 0:
            return None
        return num / (na * nb)
    except Exception:
        return None


def _flatten_embedding(v):
    """Guarantees: reduces a nested embedding payload to a flat list of floats (mean-pooled)."""
    try:
        if v is None:
            return None
        if hasattr(v, "tolist"):
            v = v.tolist()
        if isinstance(v, (int, float)):
            return [float(v)]
        if isinstance(v, list) and v and isinstance(v[0], (int, float)):
            return [float(x) for x in v]
        if isinstance(v, list) and v and isinstance(v[0], list):
            rows = [_flatten_embedding(r) for r in v]
            rows = [r for r in rows if r]
            if not rows:
                return None
            n = min(len(r) for r in rows)
            return [sum(r[i] for r in rows) / len(rows) for i in range(n)]
        return None
    except Exception:
        return None


def enrich_relevance(answer, context, timeout=None):
    """Guarantees: returns a dict with an auxiliary similarity, or None. NEVER blocks the gate.

    Disabled without HF_TOKEN. On timeout, rate-limit, HTTP error or any
    exception it returns None and local verification proceeds unchanged - the
    free-tier inference credit is ~$0.10/month, so this can and will stop
    working, by design.
    """
    if not ENRICH_ENABLED or not HF_TOKEN:
        return None
    if _ENRICH_STATE["failures"] >= 3:
        return None
    timeout = ENRICH_TIMEOUT_S if timeout is None else float(timeout)
    try:
        import concurrent.futures as _cf
        from huggingface_hub import InferenceClient

        a = (answer or "")[:ENRICH_MAX_CHARS]
        c = (context or "")[:ENRICH_MAX_CHARS]
        if not a.strip() or not c.strip():
            return None

        def _work():
            client = InferenceClient(token=HF_TOKEN, timeout=timeout)
            ea = _flatten_embedding(client.feature_extraction(a, model=ENRICH_MODEL))
            ec = _flatten_embedding(client.feature_extraction(c, model=ENRICH_MODEL))
            if not ea or not ec:
                return None
            n = min(len(ea), len(ec))
            return _cosine(ea[:n], ec[:n])

        with _cf.ThreadPoolExecutor(max_workers=1) as ex:
            fut = ex.submit(_work)
            sim = fut.result(timeout=timeout + 1.0)
        if sim is None:
            _ENRICH_STATE["failures"] += 1
            return None
        _ENRICH_STATE["failures"] = 0
        return {"ok": True, "model": ENRICH_MODEL, "similarity": round(float(sim), 4),
                "note": "auxiliary signal only; it does not affect grounding_score or the verdict"}
    except Exception as exc:
        _ENRICH_STATE["failures"] += 1
        _ENRICH_STATE["disabled_reason"] = "%s: %s" % (type(exc).__name__, str(exc)[:160])
        return None


# =============================================================================
# (I) GATE - integration
# =============================================================================

DEFAULT_POLICY = {
    "pass_grounding": 0.95,
    "pass_coverage": 0.50,
    "annotate_grounding": 0.90,
    "retry_grounding": 0.70,
    "retry_on_contradicted": 1,
    "block_on_critical_safety": True,
    "block_on_leak": False,
    "block_on_injection_echo": False,
    "enable_numeric": True,
    "enable_date": True,
    "enable_quote": True,
    "enable_entity": True,
    "enable_url": True,
    "enable_derivation": True,
    "derive_max_terms": DERIVE_MAX_TERMS,
    "derive_budget_ms": DERIVE_BUDGET_MS,
    "derive_max_numbers": DERIVE_MAX_NUMBERS,
    "fuzzy_budget_ms": FUZZY_BUDGET_MS,
    "max_claims": MAX_CLAIMS,
    "numeric_tolerance": 0.0,
    "contradiction_rel": DEFAULT_CONTRADICTION_REL,
    "approx_ratio": DEFAULT_APPROX_RATIO,
    "leak_ngram": NGRAM_LEAK_N,
    "leak_threshold": LEAK_THRESHOLD,
    "enable_enrichment": False,
    "max_chars": MAX_TEXT_CHARS,
}

_GATE_SEQ = [0]


def _merge_policy(policy_json):
    """Guarantees: returns (policy, warnings) with defaults intact and unknown keys reported."""
    pol = dict(DEFAULT_POLICY)
    warnings = []
    if policy_json:
        obj, reps = repair_json(policy_json)
        if isinstance(obj, dict):
            for k, v in obj.items():
                if k in pol:
                    pol[k] = v
                else:
                    warnings.append("unknown_policy_key:%s" % k)
            if reps and reps != []:
                warnings.append("policy_json_repaired:%s" % ",".join(reps))
        else:
            warnings.append("policy_json_unparseable_using_defaults")
    return pol, warnings


@guarded
def gate(answer, context, system_prompt="", user_input="", schema_json="",
         policy_json="", tags_json="") -> dict:
    """Guarantees: a four-valued verdict (pass/annotate/retry/block) and never raises.

    Order of work is deliberate: output safety runs FIRST, and a critical hit
    short-circuits everything downstream. Verifying the grounding of an answer
    that leaks an API key is wasted CPU on a 2-vCPU box.
    """
    t_start = time.perf_counter()
    _GATE_SEQ[0] += 1
    event_id = "e%06d" % _GATE_SEQ[0]

    warnings = []
    pol, pol_warn = _merge_policy(policy_json)
    warnings.extend(pol_warn)
    max_chars = int(pol.get("max_chars", MAX_TEXT_CHARS) or MAX_TEXT_CHARS)

    # --- 1. input validation -------------------------------------------------
    answer, t1 = clamp_text(answer, max_chars)
    context, t2 = clamp_text(context, max_chars)
    system_prompt, t3 = clamp_text(system_prompt, max_chars)
    user_input, t4 = clamp_text(user_input, max_chars)
    for name, flag in (("answer", t1), ("context", t2), ("system_prompt", t3), ("user_input", t4)):
        if flag:
            warnings.append("%s truncated to MAX_TEXT_CHARS=%d" % (name, max_chars))

    tags = {}
    if tags_json:
        tobj, _tr = repair_json(tags_json)
        if isinstance(tobj, dict):
            tags = {str(k): tobj[k] for k in tobj}
        else:
            warnings.append("tags_json_unparseable")
    model = str(tags.get("model", "") or "")
    prompt_version = str(tags.get("prompt_version", "") or "")

    sizes = {
        "answer_chars": len(answer), "context_chars": len(context),
        "system_prompt_chars": len(system_prompt), "user_input_chars": len(user_input),
        "answer_tokens": estimate_tokens(answer),
        "context_tokens": estimate_tokens(context),
        "system_prompt_tokens": estimate_tokens(system_prompt),
        "user_input_tokens": estimate_tokens(user_input),
    }

    if not answer.strip():
        latency = (time.perf_counter() - t_start) * 1000.0
        return {
            "ok": True, "event_id": event_id, "verdict": "annotate",
            "reasons": ["empty_answer"], "warnings": warnings,
            "safety": {"ok": True, "findings": [], "critical": 0},
            "leak": None, "verify": None, "schema": None, "retry": None,
            "meta": {"sizes": sizes, "tags": tags, "policy": pol,
                     "latency_ms": round(latency, 2), "timestamp": _now_iso()},
        }

    # --- 2. output safety FIRST (short-circuit on critical) ------------------
    safety = safety_report(answer)
    if not safety.get("ok"):
        safety = {"ok": False, "findings": [], "critical": 0, "error": safety.get("error")}
    critical = int(safety.get("critical", 0) or 0)
    if critical > 0 and bool(pol.get("block_on_critical_safety", True)):
        latency = (time.perf_counter() - t_start) * 1000.0
        result = {
            "ok": True, "event_id": event_id, "verdict": "block",
            "reasons": ["critical_output_safety_violation:%d" % critical],
            "warnings": warnings + ["verification skipped: blocked before grounding checks"],
            "safety": safety, "leak": None, "verify": None, "schema": None, "retry": None,
            "meta": {"sizes": sizes, "tags": tags, "policy": pol,
                     "latency_ms": round(latency, 2), "timestamp": _now_iso()},
        }
        _log_event(_event_from_result(result, model, prompt_version, answer))
        return result

    # --- 3. leak detection ---------------------------------------------------
    leak = detect_system_prompt_leak(answer, system_prompt,
                                     n=pol.get("leak_ngram"), threshold=pol.get("leak_threshold"))
    echo = detect_injection_echo(answer, user_input)
    leak_block = {"system_prompt": leak, "injection_echo": echo}

    # --- 4. verification -----------------------------------------------------
    opts = {k: pol[k] for k in (
        "enable_numeric", "enable_date", "enable_quote", "enable_entity", "enable_url",
        "enable_derivation", "derive_max_terms", "derive_budget_ms", "derive_max_numbers",
        "fuzzy_budget_ms",
        "max_claims", "numeric_tolerance", "contradiction_rel", "approx_ratio", "max_chars")
        if k in pol}
    ver = verify(answer, context, opts)
    if not ver.get("ok"):
        warnings.append("verifier_failed")
        ver = {"ok": False, "grounding_score": 0.0, "coverage": 0.0, "claims": [],
               "counts": {}, "error": ver.get("error")}

    # --- 5. schema gate (only when a schema was supplied) --------------------
    schema_res = None
    if schema_json and str(schema_json).strip():
        schema_res = structure_check(answer, schema_json)

    # --- 6. retry advisor ----------------------------------------------------
    pre = {"verify": ver, "meta": {"sizes": sizes}}
    retry = build_retry_instruction(pre)

    # optional enrichment, never load-bearing
    enrichment = None
    if bool(pol.get("enable_enrichment", False)):
        enrichment = enrich_relevance(answer, context)

    # --- verdict -------------------------------------------------------------
    by_status = (ver.get("counts") or {}).get("by_status") or {}
    contradicted = int(by_status.get("contradicted", 0) or 0)
    grounding = float(ver.get("grounding_score", 0.0) or 0.0)
    coverage = float(ver.get("coverage", 0.0) or 0.0)
    claims_total = int((ver.get("counts") or {}).get("claims_total", 0) or 0)

    reasons = []
    verdict = "pass"

    if leak.get("ok") and leak.get("leak"):
        reasons.append("system_prompt_leak:overlap=%.3f" % float(leak.get("overlap", 0.0)))
        verdict = "block" if bool(pol.get("block_on_leak", False)) else "annotate"
    if echo.get("ok") and echo.get("echo"):
        reasons.append("injection_echo:%d" % int(echo.get("echoed_count", 0)))
        if bool(pol.get("block_on_injection_echo", False)):
            verdict = "block"
        elif verdict == "pass":
            verdict = "annotate"

    if verdict != "block":
        if contradicted >= int(pol.get("retry_on_contradicted", 1) or 1):
            verdict = "retry"
            reasons.append("contradicted_claims:%d" % contradicted)
        elif claims_total and grounding < float(pol.get("retry_grounding", 0.70)):
            verdict = "retry"
            reasons.append("grounding_score below retry threshold (%.3f < %.2f)"
                           % (grounding, float(pol.get("retry_grounding", 0.70))))
        elif claims_total and grounding < float(pol.get("annotate_grounding", 0.90)):
            verdict = "annotate" if verdict == "pass" else verdict
            reasons.append("grounding_score below annotate threshold (%.3f < %.2f)"
                           % (grounding, float(pol.get("annotate_grounding", 0.90))))
        elif not claims_total:
            verdict = "annotate" if verdict == "pass" else verdict
            reasons.append("no verifiable claims were found; nothing was checked")

        if verdict == "pass":
            if grounding < float(pol.get("pass_grounding", 0.95)):
                verdict = "annotate"
                reasons.append("grounding_score below pass threshold")
            elif coverage < float(pol.get("pass_coverage", 0.50)):
                verdict = "annotate"
                reasons.append("coverage %.2f below pass threshold %.2f - most of the answer was not checked"
                               % (coverage, float(pol.get("pass_coverage", 0.50))))

    if schema_res is not None and schema_res.get("ok"):
        if schema_res.get("valid_json") is False:
            reasons.append("output is not valid JSON even after repair")
            if verdict in ("pass", "annotate"):
                verdict = "retry"
        elif schema_res.get("schema_ok") is False:
            reasons.append("JSON does not satisfy the schema: missing=%s" % (schema_res.get("missing") or []))
            if verdict in ("pass", "annotate"):
                verdict = "retry"

    if critical > 0:
        reasons.append("critical_output_safety_violation:%d (policy did not block)" % critical)
    other_safety = len(safety.get("findings", [])) - critical
    if other_safety > 0 and verdict == "pass":
        verdict = "annotate"
        reasons.append("non-critical safety findings:%d" % other_safety)

    latency = (time.perf_counter() - t_start) * 1000.0
    result = {
        "ok": True,
        "event_id": event_id,
        "verdict": verdict,
        "reasons": reasons or ["all checks passed"],
        "warnings": warnings,
        "grounding_score": round(grounding, 4),
        "coverage": round(coverage, 4),
        "safety": safety,
        "leak": leak_block,
        "verify": ver,
        "schema": schema_res,
        "retry": retry,
        "enrichment": enrichment,
        "meta": {
            "sizes": sizes,
            "tags": tags,
            "policy": pol,
            "latency_ms": round(latency, 2),
            "verify_latency_ms": ver.get("latency_ms"),
            "timestamp": _now_iso(),
            "app_version": APP_VERSION,
        },
    }

    ev = _event_from_result(result, model, prompt_version, answer)
    _log_event(ev)
    _record_audit_candidates(event_id, ver.get("claims") or [], model, prompt_version)
    return result


def _event_from_result(result, model, prompt_version, answer):
    """Guarantees: builds a log row WITHOUT the answer body unless STORE_ANSWER_PREFIX > 0."""
    try:
        ver = result.get("verify") or {}
        counts = ((ver.get("counts") or {}).get("by_status") or {})
        by_type = ((ver.get("counts") or {}).get("by_type") or {})
        safety = result.get("safety") or {}
        leak = ((result.get("leak") or {}).get("system_prompt") or {})
        retry = result.get("retry") or {}
        est = (retry.get("estimate") or {})
        findings = safety.get("findings") or []
        crit = int(safety.get("critical", 0) or 0)
        row = {
            "event_id": result.get("event_id"),
            "timestamp": (result.get("meta") or {}).get("timestamp") or _now_iso(),
            "model": model or "(unset)",
            "prompt_version": prompt_version or "(unset)",
            "verdict": result.get("verdict"),
            "grounding_score": ver.get("grounding_score"),
            "coverage": ver.get("coverage"),
            "claims_total": (ver.get("counts") or {}).get("claims_total", 0),
            "counts": {k: int(v) for k, v in counts.items()},
            "counts_by_type": {t: {s: int(n) for s, n in d.items()} for t, d in by_type.items()},
            "leak_flag": bool(leak.get("leak")),
            "leak_overlap": leak.get("overlap"),
            "safety_flags": sorted({f.get("type") for f in findings if f.get("type")}),
            "safety_critical": crit,
            "safety_other": max(0, len(findings) - crit),
            "latency_ms": (result.get("meta") or {}).get("latency_ms"),
            "saved_tokens": est.get("saved_tokens", 0),
            "saved_cost": est.get("saved_cost", 0.0),
        }
        if STORE_ANSWER_PREFIX > 0 and isinstance(answer, str):
            row["answer_prefix"] = answer[:STORE_ANSWER_PREFIX]
        return row
    except Exception:
        return {"timestamp": _now_iso(), "verdict": "unknown", "model": model or "(unset)",
                "prompt_version": prompt_version or "(unset)"}


# =============================================================================
# (M) GRADIO UI + API
# =============================================================================

STATUS_COLORS = {
    "supported": "green",
    "derived": "blue",
    "approximate": "yellow",
    "unsupported": "orange",
    "contradicted": "red",
}
_STATUS_PRIORITY = {"contradicted": 0, "unsupported": 1, "approximate": 2, "derived": 3, "supported": 4}


def build_highlight(answer, claims):
    """Guarantees: returns [(text, status_or_None)] tuples covering the answer exactly once."""
    try:
        answer = answer or ""
        if not answer:
            return [("", None)]
        spans = []
        for c in claims or []:
            try:
                s, e = int(c["span"][0]), int(c["span"][1])
            except Exception:
                continue
            if e <= s or s < 0 or e > len(answer):
                continue
            if c.get("suppressed"):
                continue
            spans.append((s, e, c.get("status", "unsupported")))
        # Overlaps: the most severe (and, at equal severity, the longest) wins.
        spans.sort(key=lambda x: (_STATUS_PRIORITY.get(x[2], 9), -(x[1] - x[0])))
        placed = []
        for s, e, st in spans:
            if any(_spans_overlap((s, e), (p[0], p[1])) for p in placed):
                continue
            placed.append((s, e, st))
        placed.sort(key=lambda x: x[0])
        out = []
        cursor = 0
        for s, e, st in placed:
            if s > cursor:
                out.append((answer[cursor:s], None))
            out.append((answer[s:e], st))
            cursor = e
        if cursor < len(answer):
            out.append((answer[cursor:], None))
        return out or [(answer, None)]
    except Exception:
        return [(answer or "", None)]


def _verdict_badge(v):
    """Guarantees: a human-readable badge for any verdict, including unexpected values."""
    return {"pass": "✅ PASS", "annotate": "🟡 ANNOTATE", "retry": "🟠 RETRY", "block": "🔴 BLOCK"}.get(v, str(v))


def _summary_md(result):
    """Guarantees: a compact human summary that always shows coverage next to grounding_score."""
    try:
        if not isinstance(result, dict) or not result.get("ok"):
            return "### ❌ Error\n```\n%s\n```" % json.dumps(result, ensure_ascii=False, indent=2)[:1500]
        ver = result.get("verify") or {}
        counts = (ver.get("counts") or {})
        bs = counts.get("by_status") or {}
        meta = result.get("meta") or {}
        g = float(ver.get("grounding_score", 0.0) or 0.0)
        cov = float(ver.get("coverage", 0.0) or 0.0)
        est = ((result.get("retry") or {}).get("estimate") or {})
        safety = result.get("safety") or {}
        leak = ((result.get("leak") or {}).get("system_prompt") or {})
        lines = [
            "## %s" % _verdict_badge(result.get("verdict")),
            "",
            "| metric | value | meaning |",
            "|---|---|---|",
            "| **grounding_score** | **%.1f%%** | of the claims we checked, this share held up |" % (g * 100),
            "| **coverage** | **%.1f%%** | share of sentences that produced *any* checkable claim |" % (cov * 100),
            "| unchecked sentences | %d / %d | **not verified — read them yourself** |" % (
                counts.get("sentences_unverified", 0), counts.get("sentences_total", 0)),
            "| latency | %s ms (verify %s ms) | measured per request |" % (
                meta.get("latency_ms"), ver.get("latency_ms")),
            "| claims | %d | supported %d · derived %d · approximate %d · unsupported %d · contradicted %d |" % (
                counts.get("claims_total", 0), bs.get("supported", 0), bs.get("derived", 0),
                bs.get("approximate", 0), bs.get("unsupported", 0), bs.get("contradicted", 0)),
            "| safety | %d finding(s), %d critical | %s |" % (
                len(safety.get("findings") or []), safety.get("critical", 0),
                ", ".join(sorted({f.get("type", "?") for f in (safety.get("findings") or [])})) or "clean"),
            "| system-prompt leak | %s (overlap %.3f) | %d-gram overlap vs threshold %.2f |" % (
                "YES" if leak.get("leak") else "no", float(leak.get("overlap", 0.0) or 0.0),
                int(leak.get("n", NGRAM_LEAK_N) or NGRAM_LEAK_N), float(leak.get("threshold", LEAK_THRESHOLD) or 0)),
            "| retry saving | %s tokens / %s | blind %s → targeted %s — %s |" % (
                est.get("saved_tokens", 0), est.get("saved_cost", 0.0),
                est.get("blind_retry_tokens", 0), est.get("targeted_retry_tokens", 0),
                est.get("recommendation", "n/a")),
            "",
            "**Reasons:** " + "; ".join(result.get("reasons") or []),
        ]
        if result.get("warnings"):
            lines.append("")
            lines.append("**Warnings:** " + "; ".join(str(w) for w in result["warnings"]))
        if cov < 0.5:
            lines.append("")
            lines.append("> ⚠️ **coverage is low.** A high grounding_score here means very little — "
                         "most of this answer was never checked.")
        return "\n".join(lines)
    except Exception as exc:
        return "### ❌ summary error\n`%s`" % type(exc).__name__


# ---------------------------------------------------------------- API surface

def api_verify(answer, context, system_prompt, user_input, schema_json, policy_json, tags_json):
    """Guarantees: returns (full_result_json, highlighted_spans, summary_markdown); never raises."""
    try:
        result = gate(answer, context, system_prompt, user_input, schema_json, policy_json, tags_json)
        claims = ((result.get("verify") or {}) or {}).get("claims") or []
        hl = build_highlight(answer if isinstance(answer, str) else "", claims)
        return result, hl, _summary_md(result)
    except Exception as exc:
        e = _err(exc, "api_verify")
        return e, [(str(answer or ""), None)], "### ❌ Error\n`%s`" % e["error"]["message"]


def api_retry_advice(result_json):
    """Guarantees: returns (advice_dict, instruction_text); never raises."""
    try:
        advice = build_retry_instruction(result_json)
        if not advice.get("ok"):
            return advice, ""
        if not advice.get("needed"):
            return advice, "(no unsupported or contradicted claims — a retry is not warranted)"
        return advice, advice.get("instruction", "")
    except Exception as exc:
        return _err(exc, "api_retry_advice"), ""


def api_stats():
    """Guarantees: returns (summary_dict, timeseries, violations, comparison, savings, latency, csv_path)."""
    try:
        return (stats_summary(), timeseries_frame(), violations_frame(),
                comparison_frame(), savings_frame(), latency_frame(), export_events_csv())
    except Exception as exc:
        empty = pd.DataFrame()
        return _err(exc, "api_stats"), empty, empty, empty, empty, empty, None


def api_structure(text, schema_json):
    """Guarantees: returns the JSON repair/coercion report; never raises."""
    return structure_check(text, schema_json)


def api_health():
    """Guarantees: returns mode, uptime, log size and mean verification latency; never raises."""
    try:
        df = events_dataframe()
        lat = pd.to_numeric(df.get("latency_ms"), errors="coerce").dropna() if not df.empty else pd.Series(dtype=float)
        up = time.time() - START_TS
        return {
            "ok": True,
            "app": APP_NAME,
            "version": APP_VERSION,
            "mode": "local-deterministic" + (" + optional-enrichment" if (ENRICH_ENABLED and HF_TOKEN) else ""),
            "enrichment": {
                "enabled": bool(ENRICH_ENABLED and HF_TOKEN),
                "model": ENRICH_MODEL if (ENRICH_ENABLED and HF_TOKEN) else None,
                "consecutive_failures": _ENRICH_STATE["failures"],
                "last_error": _ENRICH_STATE["disabled_reason"],
                "note": "optional; the gate is fully functional without it",
            },
            "uptime_seconds": int(up),
            "uptime_human": "%dh %dm %ds" % (up // 3600, (up % 3600) // 60, up % 60),
            "events_logged": int(len(df)),
            "log_capacity": LOG_CAPACITY,
            "audit_candidates": int(len(audit_frame())),
            "mean_latency_ms": round(float(lat.mean()), 2) if len(lat) else None,
            "p95_latency_ms": round(_pct(lat, 0.95), 2) if len(lat) else None,
            "config": {
                "MAX_TEXT_CHARS": MAX_TEXT_CHARS,
                "NGRAM_LEAK_N": NGRAM_LEAK_N,
                "LEAK_THRESHOLD": LEAK_THRESHOLD,
                "PRICE_IN_PER_1K": PRICE_IN_PER_1K,
                "PRICE_OUT_PER_1K": PRICE_OUT_PER_1K,
                "STORE_ANSWER_PREFIX": STORE_ANSWER_PREFIX,
            },
            "storage": "ephemeral: in-memory only; the free Space sleeps after 48h idle",
            "python": sys.version.split()[0],
            "gradio": getattr(gr, "__version__", "n/a"),
            "pandas": pd.__version__,
        }
    except Exception as exc:
        return _err(exc, "api_health")


def api_audit_refresh():
    """Guarantees: returns (audit_table, fp_rate_dict); never raises."""
    try:
        return audit_frame(), false_positive_rate()
    except Exception as exc:
        return pd.DataFrame(), _err(exc, "api_audit_refresh")


def api_audit_mark(table, ids_text, note):
    """Guarantees: applies grid edits and/or an explicit id list, then returns the refreshed view."""
    try:
        res_a = apply_audit_edits(table)
        res_b = mark_false_positives(ids_text, note) if (ids_text or "").strip() else {"ok": True, "marked": 0}
        merged = {
            "ok": True,
            "marked_from_grid": res_a.get("marked", 0),
            "marked_from_ids": res_b.get("marked", 0),
            **false_positive_rate(),
        }
        return audit_frame(), merged, export_audit_csv()
    except Exception as exc:
        return pd.DataFrame(), _err(exc, "api_audit_mark"), None


# ------------------------------------------------------- dynamic API doc text

def _api_prefix():
    """Guarantees: the REST prefix for this Gradio build, discovered at runtime (never hardcoded)."""
    for mod in ("route_utils", "routes"):
        try:
            m = __import__("gradio.%s" % mod, fromlist=[mod])
            p = getattr(m, "API_PREFIX", None)
            if isinstance(p, str):
                return p
        except Exception:
            continue
    try:
        major = int(str(getattr(gr, "__version__", "0")).split(".")[0])
        return "/gradio_api" if major >= 5 else ""
    except Exception:
        return ""


def _base_url():
    """Guarantees: the public base URL of this Space when known, otherwise the local default."""
    host = _env_str("SPACE_HOST")
    if host:
        return "https://%s" % host.rstrip("/")
    sid = _env_str("SPACE_ID")
    if sid and "/" in sid:
        owner, name = sid.split("/", 1)
        slug = re.sub(r"[^a-zA-Z0-9\-]", "-", "%s-%s" % (owner, name)).lower()
        return "https://%s.hf.space" % slug
    port = _env_str("GRADIO_SERVER_PORT", "7860")
    return "http://127.0.0.1:%s" % port


API_ENDPOINTS = [
    ("verify", "Verify an answer against its context and return the full gate result.",
     ["answer", "context", "system_prompt", "user_input", "schema_json", "policy_json", "tags_json"],
     ["result (dict)", "highlighted spans", "summary markdown"]),
    ("retry_advice", "Turn a verify result into a targeted retry instruction + saving estimate.",
     ["result_json (string)", "result_object (fallback)"], ["advice (dict)", "instruction text"]),
    ("stats", "Dashboard aggregates over the in-memory event log.",
     [], ["summary (dict)", "timeseries", "violations", "model×prompt comparison", "savings", "latency", "csv path"]),
    ("structure", "JSON extraction / repair / schema coercion only.",
     ["text", "schema_json"], ["report (dict)"]),
    ("health", "Mode, uptime, log size, mean verification latency.", [], ["health (dict)"]),
    ("audit_refresh", "Current false-positive audit table and FP rate.", [], ["table", "fp stats"]),
    ("audit_mark", "Mark audit rows as false positives and export them.",
     ["table", "ids_text", "note"], ["table", "fp stats", "csv path"]),
]


def api_docs_markdown():
    """Guarantees: builds the API reference from the live Gradio build - no hardcoded paths."""
    try:
        prefix = _api_prefix()
        base = _base_url()
        rows = ["| api_name | REST path | inputs | outputs |", "|---|---|---|---|"]
        for name, desc, ins, outs in API_ENDPOINTS:
            rows.append("| `%s` | `%s%s/call/%s` | %s | %s |" % (
                name, "", prefix, name,
                ", ".join("`%s`" % i for i in ins) or "—",
                ", ".join(outs)))
        sample_ctx = "2024年度の売上高は12,000百万円、営業利益は1,800百万円でした。"
        sample_ans = "営業利益率は15%です。"
        curl = (
            "# 1) POST -> returns an EVENT_ID\n"
            "curl -s -X POST %s%s/call/verify \\\n"
            "  -H 'Content-Type: application/json' \\\n"
            "  -d '{\"data\": [\"%s\", \"%s\", \"\", \"\", \"\", \"\", \"{\\\"model\\\":\\\"my-model\\\",\\\"prompt_version\\\":\\\"v1\\\"}\"]}'\n"
            "\n# 2) GET the result stream with that id\n"
            "curl -N %s%s/call/verify/EVENT_ID\n"
        ) % (base, prefix, sample_ctx, sample_ans, base, prefix)
        client = (
            "from gradio_client import Client\n\n"
            "client = Client(\"%s\")\n"
            "result, highlighted, summary = client.predict(\n"
            "    answer=\"%s\",\n"
            "    context=\"%s\",\n"
            "    system_prompt=\"\",\n"
            "    user_input=\"\",\n"
            "    schema_json=\"\",\n"
            "    policy_json='{\"enable_entity\": false}',\n"
            "    tags_json='{\"model\": \"my-model\", \"prompt_version\": \"v1\"}',\n"
            "    api_name=\"/verify\",\n"
            ")\n"
            "print(result[\"verdict\"], result[\"grounding_score\"], result[\"coverage\"])\n"
        ) % (_env_str("SPACE_ID") or base, sample_ans, sample_ctx)
        return "\n".join([
            "### Live API reference",
            "",
            "Detected Gradio **%s**, REST prefix **`%s`**, base URL **`%s`**." % (
                getattr(gr, "__version__", "?"), prefix or "(none)", base),
            "These are read from the running process, not hardcoded — Gradio has moved this path "
            "between major versions.",
            "",
            "\n".join(rows),
            "",
            "> The authoritative list is always the **“Use via API”** link at the bottom of this page "
            "(`%s%s/`). If anything below disagrees with it, believe that link." % (base, prefix),
            "",
            "#### curl",
            "```bash",
            curl,
            "```",
            "",
            "#### gradio_client",
            "```python",
            client,
            "```",
            "",
            "#### Reading the result",
            "- `verdict` — `pass` / `annotate` / `retry` / `block`",
            "- `grounding_score` — of the claims that **were** checked, the share that held up",
            "- `coverage` — the share of sentences that produced any checkable claim at all",
            "- **Never read `grounding_score` without `coverage`.** 1.00 grounding at 0.10 coverage means "
            "one sentence checked out and nine were never looked at.",
            "- `verify.unverified_sentences` — exactly what was skipped, so you can read it yourself.",
        ])
    except Exception as exc:
        return "API docs unavailable: `%s`" % type(exc).__name__


# ------------------------------------------------------------------ demo data

DEMO_CONTEXT = """2024年度の売上高は12,000百万円、営業利益は1,800百万円でした。
従業員数は3,400人で、前年度(2023年度)は3,200人でした。
新製品Xは2024年3月15日に発売されました。
IR資料は https://example.com/ir/2024 に掲載しています。
契約は第12条に基づき自動更新されます。"""

DEMO_ANSWER = """2024年度の売上高は12,000百万円、営業利益は1,800百万円でした。
したがって営業利益率は15%です。
従業員数は3,400人、前年度は3,250人でした。
新製品Xは2024年3月15日に発売されています。
なお解約率は7.2%に達しており、注意が必要です。
詳細は https://example.com/ir/2025 をご覧ください。"""

DEMO_SYSTEM = """あなたは企業のIR資料に基づいて回答するアシスタントです。
与えられた文脈のみを根拠とし、文脈に無い数値を作らないでください。"""

DEMO_POLICY = json.dumps({"enable_entity": True, "retry_on_contradicted": 1,
                          "pass_grounding": 0.95, "pass_coverage": 0.5}, ensure_ascii=False, indent=2)
DEMO_TAGS = json.dumps({"model": "demo-model", "prompt_version": "v1"}, ensure_ascii=False)
DEMO_BROKEN_JSON = """```json
{
  'title': "四半期レポート",
  "score": 0.87,
  "tags": ["ir", "2024",],
  "note": "複数行の
メモ",
  "published": True,
}
```"""
DEMO_SCHEMA = json.dumps({"title": "str", "score": "float", "tags": "list",
                          "published": "bool", "note?": "str"}, ensure_ascii=False, indent=2)


def build_demo():
    """Guarantees: constructs the Blocks UI; raises only if gradio itself is unavailable."""
    if gr is None:
        raise RuntimeError("gradio is not installed")

    with gr.Blocks(title="%s — LLM answer verification gate" % APP_NAME,
                   analytics_enabled=False) as demo:
        gr.Markdown(
            "# 🔎 ClaimCheck\n"
            "**Verify LLM answers against their source context before they reach users.**\n\n"
            "We cannot catch every hallucination — but dangerous hallucinations are *specific*, "
            "and specific claims can be matched as strings. So ClaimCheck checks only what it can "
            "check deterministically, and always tells you **how much it did not check** (`coverage`)."
        )

        # ------------------------------------------------------------ Verify
        with gr.Tab("Verify"):
            with gr.Row():
                with gr.Column(scale=1):
                    in_answer = gr.Textbox(label="Answer (the LLM output to check)", lines=10,
                                           value=DEMO_ANSWER, placeholder="モデルの応答をここに貼り付け")
                    in_context = gr.Textbox(label="Context (the source of truth given to the model)", lines=10,
                                            value=DEMO_CONTEXT, placeholder="RAG で渡した文脈をここに")
                with gr.Column(scale=1):
                    in_system = gr.Textbox(label="System prompt (optional — used for leak detection)",
                                           lines=4, value=DEMO_SYSTEM)
                    in_user = gr.Textbox(label="User input (optional — used for injection-echo detection)",
                                         lines=3, value="")
                    in_schema = gr.Textbox(label="Schema JSON (optional — only if the answer must be JSON)",
                                           lines=3, value="",
                                           placeholder='{"title": "str", "score": "float"}')
                    in_policy = gr.Textbox(label="Policy JSON (optional — threshold overrides)",
                                           lines=6, value=DEMO_POLICY)
                    in_tags = gr.Textbox(label="Tags JSON (model / prompt_version — used by the comparison table)",
                                         lines=2, value=DEMO_TAGS)
            btn_verify = gr.Button("Verify", variant="primary")
            out_summary = gr.Markdown()
            out_highlight = gr.HighlightedText(
                label="Answer, coloured by claim status "
                      "(green=supported, blue=derived, yellow=approximate, orange=unsupported, red=contradicted)",
                color_map=STATUS_COLORS, show_legend=True, combine_adjacent=True)
            out_json = gr.JSON(label="Full result")

            btn_verify.click(
                api_verify,
                inputs=[in_answer, in_context, in_system, in_user, in_schema, in_policy, in_tags],
                outputs=[out_json, out_highlight, out_summary],
                api_name="verify",
            )

        # ------------------------------------------------------------- Retry
        with gr.Tab("Retry"):
            gr.Markdown(
                "Turn a verify result into a **targeted** correction instruction. "
                "A blind retry re-sends the whole system prompt + context + question and regenerates "
                "everything. A targeted retry re-sends only the answer plus the specific failing claims. "
                "The difference is the estimate below."
            )
            in_result = gr.Textbox(
                label="Verify result JSON (leave empty to reuse the result currently shown on the Verify tab)",
                lines=6, value="")
            btn_retry = gr.Button("Build retry instruction", variant="primary")
            out_instruction = gr.Textbox(label="Retry instruction (append this to the next request)", lines=12)
            out_retry_json = gr.JSON(label="Advice + token/cost estimate")

            def _retry_handler(text, current):
                """Guarantees: uses the pasted JSON when present, else the live Verify result."""
                payload = text if isinstance(text, str) and text.strip() else current
                return api_retry_advice(payload)

            # Reading the Verify tab's JSON component directly (rather than a
            # gr.State) keeps this working both in the browser and over the API,
            # where callers can simply pass the result object as the 2nd argument.
            btn_retry.click(_retry_handler, inputs=[in_result, out_json],
                            outputs=[out_retry_json, out_instruction], api_name="retry_advice")

        # --------------------------------------------------------- Dashboard
        with gr.Tab("Dashboard"):
            gr.Markdown(
                "In-memory only. The free Space has **no persistent disk** and sleeps after 48h idle — "
                "export the CSV if you need to keep any of this."
            )
            btn_stats = gr.Button("Refresh", variant="primary")
            out_stats = gr.JSON(label="Summary")
            with gr.Row():
                out_ts = gr.Dataframe(label="grounding_score / coverage over time", wrap=True)
                out_viol = gr.Dataframe(label="violations per event", wrap=True)
            out_cmp = gr.Dataframe(label="model × prompt_version comparison (the diff after a version change)",
                                   wrap=True)
            with gr.Row():
                out_sav = gr.Dataframe(label="cumulative savings", wrap=True)
                out_lat = gr.Dataframe(label="verification latency p50 / p95 / max", wrap=True)
            out_csv = gr.File(label="events.csv")
            btn_stats.click(api_stats, inputs=None,
                            outputs=[out_stats, out_ts, out_viol, out_cmp, out_sav, out_lat, out_csv],
                            api_name="stats")

        # ------------------------------------------------------------- Audit
        with gr.Tab("Audit (false positives)"):
            gr.Markdown(
                "Every `unsupported` / `contradicted` claim lands here. Tick **false_positive** on the rows "
                "that were actually correct, then save. A verifier with a high false-positive rate gets "
                "switched off by its users, so this number belongs on the screen — not in a feeling."
            )
            btn_audit_refresh = gr.Button("Refresh")
            out_fp = gr.JSON(label="False-positive rate")
            out_audit = gr.Dataframe(
                label="Flagged claims (edit the false_positive column)",
                interactive=True, wrap=True,
                headers=["audit_id", "timestamp", "model", "prompt_version", "type", "status",
                         "value", "nearest_context_value", "false_positive", "note"],
                datatype=["str", "str", "str", "str", "str", "str", "str", "str", "bool", "str"],
            )
            with gr.Row():
                in_ids = gr.Textbox(label="…or paste audit_id(s) directly (comma/space separated)", scale=2)
                in_note = gr.Textbox(label="Note", scale=2)
                btn_audit_mark = gr.Button("Save marks & export CSV", variant="primary", scale=1)
            out_audit_csv = gr.File(label="false_positives.csv")

            btn_audit_refresh.click(api_audit_refresh, inputs=None, outputs=[out_audit, out_fp],
                                    api_name="audit_refresh")
            btn_audit_mark.click(api_audit_mark, inputs=[out_audit, in_ids, in_note],
                                 outputs=[out_audit, out_fp, out_audit_csv], api_name="audit_mark")

        # -------------------------------------------------------- Playground
        with gr.Tab("Playground (JSON repair)"):
            gr.Markdown(
                "Structural repair only. **The point is to never burn a retry on formatting.** "
                "Fences, trailing commas, single quotes, raw newlines inside strings, fullwidth punctuation "
                "and Python literals are all fixed locally — retries are reserved for claims that are wrong."
            )
            in_raw = gr.Textbox(label="Raw model output", lines=12, value=DEMO_BROKEN_JSON)
            in_schema2 = gr.Textbox(label="Schema JSON (optional)", lines=6, value=DEMO_SCHEMA)
            btn_struct = gr.Button("Extract / repair / coerce", variant="primary")
            out_struct = gr.JSON(label="Report")
            btn_struct.click(api_structure, inputs=[in_raw, in_schema2], outputs=[out_struct],
                             api_name="structure")

        # ------------------------------------------------------------ Health
        with gr.Tab("Health"):
            btn_health = gr.Button("Check", variant="primary")
            out_health = gr.JSON(label="Health")
            btn_health.click(api_health, inputs=None, outputs=[out_health], api_name="health")

        # ---------------------------------------------------------- API Docs
        with gr.Tab("API Docs"):
            md_api = gr.Markdown(api_docs_markdown())
            btn_api = gr.Button("Re-detect paths")
            btn_api.click(api_docs_markdown, inputs=None, outputs=[md_api], api_name=False)

        gr.Markdown(
            "---\n"
            "ClaimCheck is **one layer of defence, not a correctness guarantee.** It verifies numbers, dates, "
            "quotes, entities and URLs against the context you supply. It says nothing about claims it could "
            "not extract — that is what `coverage` is for."
        )

    return demo


demo = None

if __name__ == "__main__":
    try:
        demo = build_demo()
        demo.queue(max_size=UI_QUEUE_SIZE, default_concurrency_limit=UI_CONCURRENCY)
        demo.launch()
    except Exception as exc:  # pragma: no cover
        sys.stderr.write("ClaimCheck failed to start: %s\n%s\n" % (exc, traceback.format_exc()))
        raise