File size: 2,737 Bytes
1d9bd9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """Stage-1 scorer: foundational-authority recall + control no-regression.
Runs each gold query through the live deep/fast API and checks whether the tagged seminal
authority surfaces in the returned results. Baseline before the citation-graph + agent work.
Run on Thor: python3 32_score_foundational.py (hits localhost:8000)
"""
import json, re, urllib.request, urllib.parse, os
GOLD = os.path.join(os.path.dirname(__file__), "..", "eval", "gold_foundational.json")
GOLD = os.path.normpath(GOLD) if os.path.exists(GOLD) else "gold_foundational.json"
BASE = "http://127.0.0.1:8000"
EP = os.environ.get("THEMIS_EP", "search_stream")
def search(q):
url = f"{BASE}/api/{EP}?q=" + urllib.parse.quote(q)
results = []
with urllib.request.urlopen(url, timeout=120) as r:
for raw in r:
line = raw.decode("utf-8", "ignore")
if line.startswith("data: "):
ev = json.loads(line[6:])
if ev.get("t") == "results":
results = ev["results"]
return results
def matches(case_name, alts):
nm = (case_name or "").lower()
return any(all(re.search(r"\b" + re.escape(tok) + r"\b", nm) for tok in alt) for alt in alts) # word-boundary: 'neeta' won't match 'aneeta'
gold = json.load(open(GOLD))
found_q = [g for g in gold if g["foundational"] and not g["control"]]
ctrl_q = [g for g in gold if g["control"]]
h5 = h8 = 0 # @5 = what the grounded answer can actually use (synthesis sees top-5); @8 = full result list
print("=== FOUNDATIONAL-AUTHORITY RECALL ===")
for g in found_q:
res = search(g["query"])
names = [r.get("case_name") for r in res]
rank = next((i + 1 for i, n in enumerate(names) if matches(n, g["foundational"])), None)
in5 = rank is not None and rank <= 5
in8 = rank is not None and rank <= 8
h5 += in5; h8 += in8
tag = "/".join("+".join(a) for a in g["foundational"])
mark = "HIT@5" if in5 else ("HIT@8" if in8 else "MISS ")
print(f" {mark} {g['id']:10} foundational[{tag}]" + (f" at #{rank}" if rank else f" (top: {names[0][:38] if names else '-'})"))
print(f"\nFoundational Recall@5 (groundable): {h5}/{len(found_q)} = {h5/len(found_q):.2f} [CP-A baseline]")
print(f"Foundational Recall@8 (shown): {h8}/{len(found_q)} = {h8/len(found_q):.2f}")
print("\n=== CONTROL (no-regression: lookup must still return the exact case at #1) ===")
creg = 0
for g in ctrl_q:
res = search(g["query"])
names = [r.get("case_name") for r in res]
top1 = matches(names[0], g["foundational"]) if names else False
creg += top1
print(f" {'OK ' if top1 else 'REGRESSED'} {g['id']:10} top1={names[0][:42] if names else '-'}")
print(f"\nControl held: {creg}/{len(ctrl_q)}")
|