"""
app.py — Closed-World Grounding: live fabrication viewer
=========================================================================
An interactive Hugging Face Space that lets a judge SEE hallucination get
caught. It reads the benchmark's saved outputs (results/tier1_benchmark_detail.json)
and, for any variant, shows the ungrounded output next to the grounded one, with
every detected fabrication highlighted inline.
No API key, no live generation: the Space displays the exact model outputs that
produced the published benchmark numbers, so it loads instantly and never breaks.
Run locally:
pip install gradio
python app.py
On Hugging Face Spaces: set SDK = gradio and this file as the entrypoint.
"""
from __future__ import annotations
import json
import html
import re
from pathlib import Path
import gradio as gr
# -------------------------------------------------------------------------
# Data loading
# -------------------------------------------------------------------------
DETAIL_PATH = Path("results/tier1_benchmark_detail.json")
# Load the benchmark detail once at startup. Each entry:
# {accession, arm, output, fabrication:{hgvs_c_fabrications, ungrounded_evidence_terms,
# citation_misattributions, clinical_classification_leak}, calibration:{...}}
with open(DETAIL_PATH, encoding="utf-8") as f:
_RAW = json.load(f)
# Reshape into {accession: {"grounded": entry, "ungrounded": entry}}
VARIANTS: dict[str, dict[str, dict]] = {}
for entry in _RAW:
acc = entry["accession"]
VARIANTS.setdefault(acc, {})[entry["arm"]] = entry
# Sorted list of accessions that have both arms (for the dropdown).
ACCESSIONS = sorted(a for a, arms in VARIANTS.items()
if "grounded" in arms and "ungrounded" in arms)
# Fabrication categories -> (label, css class, adjudication note)
FAB_FIELDS = {
"hgvs_c_fabrications": ("Invented HGVS c. notation", "fab-hgvs"),
"ungrounded_evidence_terms": ("Wrong / invented evidence value", "fab-evidence"),
"citation_misattributions": ("Misattributed / invented citation", "fab-citation"),
"clinical_classification_leak": ("Clinical P/LP over-assertion", "fab-clinical"),
}
# -------------------------------------------------------------------------
# Highlighting
# -------------------------------------------------------------------------
def _collect_spans(fab: dict) -> list[tuple[str, str, str]]:
"""Return (needle, css_class, label) for each fabrication string to mark."""
spans = []
for field, (label, css) in FAB_FIELDS.items():
for item in fab.get(field, []):
# Detector items can be phrases or "asserted X != source Y" notes.
# Pull a searchable needle: the leading token(s) before " != " or ":".
needle = re.split(r"\s+!=\s+|:\s", str(item))[0].strip()
# For quoted claims like "...'functional stud'...", extract the quote.
m = re.search(r"'([^']+)'", str(item))
if m:
needle = m.group(1)
if needle:
spans.append((needle, css, label))
return spans
def highlight(text: str, fab: dict) -> str:
"""Return HTML with fabrication substrings wrapped in colored marks."""
if not text:
return "(no output)"
escaped = html.escape(text)
spans = _collect_spans(fab)
# Longest needles first so nested matches don't get clobbered.
for needle, css, label in sorted(spans, key=lambda s: -len(s[0])):
esc_needle = html.escape(needle)
if esc_needle and esc_needle in escaped:
mark = (f''
f'{esc_needle}')
escaped = escaped.replace(esc_needle, mark)
return escaped.replace("\n", "
")
def _count_fab(fab: dict, adjudicated: bool, arm: str) -> int:
"""Total fabrications; if adjudicated, drop the grounded clinical false positives."""
total = sum(len(fab.get(k, [])) for k in FAB_FIELDS)
if adjudicated and arm == "grounded":
total -= len(fab.get("clinical_classification_leak", []))
return max(total, 0)
def _badge(n: int) -> str:
if n == 0:
return '✓ 0 fabrications'
unit = "fabrication" if n == 1 else "fabrications"
return f'⚠ {n} {unit}'
# -------------------------------------------------------------------------
# View builder
# -------------------------------------------------------------------------
def render(accession: str, adjudicated: bool):
pair = VARIANTS[accession]
g, u = pair["grounded"], pair["ungrounded"]
g_fab, u_fab = g["fabrication"], u["fabrication"]
g_n = _count_fab(g_fab, adjudicated, "grounded")
u_n = _count_fab(u_fab, adjudicated, "ungrounded")
def panel(title, arm_class, badge_html, body_html, cal):
cal_note = ""
if cal:
ok = cal.get("appropriate")
lvl = cal.get("confidence_level", "?")
chip = "cal-ok" if ok else "cal-bad"
txt = "calibrated" if ok else "miscalibrated"
cal_note = (f'confidence: {txt} '
f'(evidence: {lvl})')
return (f'
Adjudication on: ' f'{len(g_fab["clinical_classification_leak"])} grounded ' 'clinical flag(s) here are detector false positives ' '(hedged/negated language) and are not counted.
') return f'Same variant, same model, two prompts. The left arm runs unconstrained; the right is held to a closed world of verified facts. Highlights mark every fabrication the detectors caught — pick a variant and compare.