phase1/scripts/32_score_foundational.py

"""Stage-1 scorer: foundational-authority recall + control no-regression.
Runs each gold query through the live deep/fast API and checks whether the tagged seminal
authority surfaces in the returned results. Baseline before the citation-graph + agent work.

Run on Thor:  python3 32_score_foundational.py            (hits localhost:8000)
"""
import json, re, urllib.request, urllib.parse, os

GOLD = os.path.join(os.path.dirname(__file__), "..", "eval", "gold_foundational.json")
GOLD = os.path.normpath(GOLD) if os.path.exists(GOLD) else "gold_foundational.json"
BASE = "http://127.0.0.1:8000"

EP = os.environ.get("THEMIS_EP", "search_stream")
def search(q):
    url = f"{BASE}/api/{EP}?q=" + urllib.parse.quote(q)
    results = []
    with urllib.request.urlopen(url, timeout=120) as r:
        for raw in r:
            line = raw.decode("utf-8", "ignore")
            if line.startswith("data: "):
                ev = json.loads(line[6:])
                if ev.get("t") == "results":
                    results = ev["results"]
    return results

def matches(case_name, alts):
    nm = (case_name or "").lower()
    return any(all(re.search(r"\b" + re.escape(tok) + r"\b", nm) for tok in alt) for alt in alts)  # word-boundary: 'neeta' won't match 'aneeta'

gold = json.load(open(GOLD))
found_q = [g for g in gold if g["foundational"] and not g["control"]]
ctrl_q = [g for g in gold if g["control"]]

h5 = h8 = 0   # @5 = what the grounded answer can actually use (synthesis sees top-5); @8 = full result list
print("=== FOUNDATIONAL-AUTHORITY RECALL ===")
for g in found_q:
    res = search(g["query"])
    names = [r.get("case_name") for r in res]
    rank = next((i + 1 for i, n in enumerate(names) if matches(n, g["foundational"])), None)
    in5 = rank is not None and rank <= 5
    in8 = rank is not None and rank <= 8
    h5 += in5; h8 += in8
    tag = "/".join("+".join(a) for a in g["foundational"])
    mark = "HIT@5" if in5 else ("HIT@8" if in8 else "MISS ")
    print(f"  {mark} {g['id']:10} foundational[{tag}]" + (f" at #{rank}" if rank else f"  (top: {names[0][:38] if names else '-'})"))
print(f"\nFoundational Recall@5 (groundable): {h5}/{len(found_q)} = {h5/len(found_q):.2f}   [CP-A baseline]")
print(f"Foundational Recall@8 (shown):      {h8}/{len(found_q)} = {h8/len(found_q):.2f}")

print("\n=== CONTROL (no-regression: lookup must still return the exact case at #1) ===")
creg = 0
for g in ctrl_q:
    res = search(g["query"])
    names = [r.get("case_name") for r in res]
    top1 = matches(names[0], g["foundational"]) if names else False
    creg += top1
    print(f"  {'OK  ' if top1 else 'REGRESSED'} {g['id']:10} top1={names[0][:42] if names else '-'}")
print(f"\nControl held: {creg}/{len(ctrl_q)}")