File size: 3,303 Bytes
1d9bd9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Benchmark harness — scoring. Score Themis vs CaseMine top-K with the isolated
relevance reviewer (relevant=1, partial=0.5, not=0) -> relevance@K per query,
per intent, and overall. Reports the head-to-head + per-query win/loss.
Inputs (phase1/eval/): themis_bench_results.json, casemine_bench_results.json
each: [{id,intent,query,results:[{case_name, passage|snippet, neutral_citation?, court?, year?}]}]
Run: set -a; . ./.env; set +a; .venv/bin/python phase1/scripts/20_score_benchmark.py
"""
import os, sys, json
from collections import defaultdict
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from reviewer import review
from concurrent.futures import ThreadPoolExecutor
HERE = os.path.dirname(__file__); EVAL = os.path.normpath(os.path.join(HERE, "..", "eval"))
THEMIS = json.load(open(os.path.join(EVAL, "themis_bench_results.json")))
CM_PATH = os.path.join(EVAL, "casemine_bench_results.json")
CASEMINE = json.load(open(CM_PATH)) if os.path.exists(CM_PATH) else []
TOPK = int(os.getenv("TOPK", "10"))
SCORE = {"relevant": 1.0, "partial": 0.5, "not": 0.0}
def case_text(r):
return (r.get("case_name") or "") + ". " + (r.get("passage") or r.get("snippet") or "")
def score_system(entries):
"""entries: list of {id,intent,query,results}. Returns {id: relevance@K} + verdict detail."""
by_id = {}
tasks = []
for e in entries:
for r in e["results"][:TOPK]:
tasks.append((e["id"], e["query"], r))
def judge(t):
qid, q, r = t
v = review(q, case_text(r))
return (qid, SCORE.get(v["verdict"], 0.0))
agg = defaultdict(list)
with ThreadPoolExecutor(8) as ex:
for qid, s in ex.map(judge, tasks):
agg[qid].append(s)
for e in entries:
sc = agg.get(e["id"], [])
by_id[e["id"]] = sum(sc) / TOPK if sc else 0.0 # graded relevance@K (out of K slots)
return by_id
print(f"scoring Themis ({len(THEMIS)} queries) ...", flush=True)
themis = score_system(THEMIS)
cm = score_system(CASEMINE) if CASEMINE else {}
intent_of = {e["id"]: e["intent"] for e in THEMIS}
qtext = {e["id"]: e["query"] for e in THEMIS}
print("\n=== per-query relevance@%d ===" % TOPK)
print(f"{'id':12s} {'intent':9s} | Themis | CaseMine | winner")
ti = defaultdict(list); ci = defaultdict(list)
for qid in [e["id"] for e in THEMIS]:
t = themis.get(qid, 0.0); c = cm.get(qid)
ti[intent_of[qid]].append(t)
win = ""
if c is not None:
ci[intent_of[qid]].append(c)
win = "THEMIS" if t > c + 0.05 else ("CaseMine" if c > t + 0.05 else "tie")
print(f"{qid:12s} {intent_of[qid]:9s} | {t:4.0%} | {('%4.0f%%'%(c*100)) if c is not None else ' -- '} | {win}")
print("\n=== per-intent average ===")
for it in ["fact", "issue", "vague", "citation", "casename"]:
t = sum(ti[it])/len(ti[it]) if ti[it] else None
c = sum(ci[it])/len(ci[it]) if ci[it] else None
print(f"{it:10s} Themis {('%3.0f%%'%(t*100)) if t is not None else ' -- '} | CaseMine {('%3.0f%%'%(c*100)) if c is not None else ' -- '}")
allt = sum(themis.values())/len(themis)
allc = sum(cm.values())/len(cm) if cm else None
print(f"\nOVERALL Themis {allt:.0%}" + (f" | CaseMine {allc:.0%} | delta {(allt-allc)*100:+.0f} pts" if allc is not None else " (CaseMine not collected yet)"))
|