File size: 2,001 Bytes
b2ebc95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | #!/usr/bin/env python3
"""Summarise a verifiers eval output dir: mean reward + Wilson 95% CI."""
import json, sys, math, os, collections
def wilson(k, n, z=1.96):
if n == 0: return (0.0, 0.0, 0.0)
p = k / n
d = 1 + z*z/n
c = (p + z*z/(2*n)) / d
h = z*math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / d
return (p, max(0.0, c-h), min(1.0, c+h))
def summarize(path):
f = os.path.join(path, "traces.jsonl")
rows = [json.loads(l) for l in open(f)] if os.path.exists(f) else []
scores, stops, errs, ntok, turns, dur = [], collections.Counter(), 0, [], [], []
for r in rows:
for t in r.get("traces", []):
rw = {k: v for k, v in (t.get("rewards") or {}).items()
if isinstance(v, dict) and v.get("score") is not None}
if not rw:
errs += 1; continue
tot = sum(v["score"]*v.get("weight",1.0) for v in rw.values())
wsum = sum(v.get("weight",1.0) for v in rw.values()) or 1.0
scores.append(tot/wsum)
stops[t.get("stop_condition")] += 1
calls = t.get("calls") or []
turns.append(len(calls))
ntok.append(sum((c.get("usage") or {}).get("completion_tokens",0) for c in calls))
ti = t.get("timing") or {}
a = ti.get("agent") or {}
if a.get("start") and a.get("end"): dur.append(a["end"]-a["start"])
if not r.get("ok", True): errs += 1
n = len(scores); k = sum(1 for s in scores if s >= 0.999)
p, lo, hi = wilson(k, n)
print(f"{path}")
print(f" n={n} solved={k} score={p:.4f} ci95=[{lo:.4f},{hi:.4f}] errors={errs}")
if n:
print(f" mean_reward={sum(scores)/n:.4f} mean_turns={sum(turns)/len(turns):.1f} "
f"mean_completion_tok={sum(ntok)/len(ntok):.0f} mean_agent_sec={(sum(dur)/len(dur) if dur else 0):.0f}")
print(f" stops={dict(stops)}")
return p, lo, hi, n, k
if __name__ == "__main__":
for p in sys.argv[1:]: summarize(p)
|