phase1/scripts/20_score_benchmark.py

"""Benchmark harness — scoring. Score Themis vs CaseMine top-K with the isolated
relevance reviewer (relevant=1, partial=0.5, not=0) -> relevance@K per query,
per intent, and overall. Reports the head-to-head + per-query win/loss.

Inputs (phase1/eval/): themis_bench_results.json, casemine_bench_results.json
  each: [{id,intent,query,results:[{case_name, passage|snippet, neutral_citation?, court?, year?}]}]
Run:  set -a; . ./.env; set +a; .venv/bin/python phase1/scripts/20_score_benchmark.py
"""
import os, sys, json
from collections import defaultdict
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from reviewer import review
from concurrent.futures import ThreadPoolExecutor

HERE = os.path.dirname(__file__); EVAL = os.path.normpath(os.path.join(HERE, "..", "eval"))
THEMIS = json.load(open(os.path.join(EVAL, "themis_bench_results.json")))
CM_PATH = os.path.join(EVAL, "casemine_bench_results.json")
CASEMINE = json.load(open(CM_PATH)) if os.path.exists(CM_PATH) else []
TOPK = int(os.getenv("TOPK", "10"))
SCORE = {"relevant": 1.0, "partial": 0.5, "not": 0.0}

def case_text(r):
    return (r.get("case_name") or "") + ". " + (r.get("passage") or r.get("snippet") or "")

def score_system(entries):
    """entries: list of {id,intent,query,results}. Returns {id: relevance@K} + verdict detail."""
    by_id = {}
    tasks = []
    for e in entries:
        for r in e["results"][:TOPK]:
            tasks.append((e["id"], e["query"], r))
    def judge(t):
        qid, q, r = t
        v = review(q, case_text(r))
        return (qid, SCORE.get(v["verdict"], 0.0))
    agg = defaultdict(list)
    with ThreadPoolExecutor(8) as ex:
        for qid, s in ex.map(judge, tasks):
            agg[qid].append(s)
    for e in entries:
        sc = agg.get(e["id"], [])
        by_id[e["id"]] = sum(sc) / TOPK if sc else 0.0  # graded relevance@K (out of K slots)
    return by_id

print(f"scoring Themis ({len(THEMIS)} queries) ...", flush=True)
themis = score_system(THEMIS)
cm = score_system(CASEMINE) if CASEMINE else {}
intent_of = {e["id"]: e["intent"] for e in THEMIS}
qtext = {e["id"]: e["query"] for e in THEMIS}

print("\n=== per-query relevance@%d ===" % TOPK)
print(f"{'id':12s} {'intent':9s} | Themis | CaseMine | winner")
ti = defaultdict(list); ci = defaultdict(list)
for qid in [e["id"] for e in THEMIS]:
    t = themis.get(qid, 0.0); c = cm.get(qid)
    ti[intent_of[qid]].append(t)
    win = ""
    if c is not None:
        ci[intent_of[qid]].append(c)
        win = "THEMIS" if t > c + 0.05 else ("CaseMine" if c > t + 0.05 else "tie")
    print(f"{qid:12s} {intent_of[qid]:9s} |  {t:4.0%}  |  {('%4.0f%%'%(c*100)) if c is not None else '  -- '}  | {win}")

print("\n=== per-intent average ===")
for it in ["fact", "issue", "vague", "citation", "casename"]:
    t = sum(ti[it])/len(ti[it]) if ti[it] else None
    c = sum(ci[it])/len(ci[it]) if ci[it] else None
    print(f"{it:10s} Themis {('%3.0f%%'%(t*100)) if t is not None else ' -- '} | CaseMine {('%3.0f%%'%(c*100)) if c is not None else ' -- '}")
allt = sum(themis.values())/len(themis)
allc = sum(cm.values())/len(cm) if cm else None
print(f"\nOVERALL  Themis {allt:.0%}" + (f"  |  CaseMine {allc:.0%}  |  delta {(allt-allc)*100:+.0f} pts" if allc is not None else "  (CaseMine not collected yet)"))