Buckets:
| """V2 comparison: any readouts and reasoners side by side, reasoning scored on the items every reasoner answered. | |
| python metrics_v2.py --readout base=PATH v1_joint=PATH v2_joint=PATH ... --reason base=PATH v1_joint=PATH v2_joint=PATH | |
| --partner v1_joint=v1_joint v2_joint=v2_joint --out results/metrics_v2.json | |
| Temperatures (readout and post-chain, per question type) are fitted on the calibration items only and written to | |
| calibration_<name>.json next to --out, ready to ship with each checkpoint. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import numpy as np | |
| from common import auroc, brier, ece, nll, read_jsonl, softmax | |
| from metrics import EVAL_SETS, FRACTIONS, boot, calibrated, fit_temps, gap_recovered, margin, od_capability, routing_curve | |
| TARGETS = {"readout_acc_all": 0.725, "od_capability": 70.0, "agree_all": 0.78, "post_ece": 0.05} | |
| def kv(pairs): | |
| return dict(p.split("=", 1) for p in pairs or []) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--readout", nargs="+", required=True) | |
| ap.add_argument("--reason", nargs="*", default=[]) | |
| ap.add_argument("--partner", nargs="*", default=[], help="readout=reasoner pairs for agreement and routing") | |
| ap.add_argument("--eval", default="data/eval.jsonl") | |
| ap.add_argument("--calib", default="data/calib.jsonl") | |
| ap.add_argument("--out", default="results/metrics_v2.json") | |
| args = ap.parse_args() | |
| ev, cal = read_jsonl(args.eval), read_jsonl(args.calib) | |
| os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) | |
| out = {"readout": {}, "reasoning": {}, "pairs": {}, "targets": {}} | |
| probs_of = {} | |
| for name, path in kv(args.readout).items(): | |
| ro = {r["id"]: r["lp"] for r in read_jsonl(path)} | |
| temps = fit_temps(cal, [ro[it["id"]] for it in cal]) | |
| probs = calibrated(ev, [ro[it["id"]] for it in ev], temps) | |
| raw = calibrated(ev, [ro[it["id"]] for it in ev], {k: 1.0 for k in temps}) | |
| probs_of[name] = probs | |
| ok = np.array([int(np.argmax(p)) == it["gold"] for it, p in zip(ev, probs)]) | |
| conf = np.array([p.max() for p in probs]) | |
| od = [i for i, it in enumerate(ev) if it["set"] == "opendecisions"] | |
| out["readout"][name] = {"temps": temps, | |
| "acc": {s: float(ok[[i for i, it in enumerate(ev) if it["set"] == s]].mean()) for s in EVAL_SETS} | {"all": float(ok.mean())}, | |
| "ece15": ece(conf, ok), "ece15_raw": ece(np.array([p.max() for p in raw]), ok), | |
| "nll": nll(probs, [it["gold"] for it in ev]), "brier": brier(probs, [it["gold"] for it in ev]), | |
| "od": od_capability([ev[i] for i in od], [probs[i] for i in od])} | |
| json.dump({"readout": temps}, open(os.path.join(os.path.dirname(args.out) or ".", f"calibration_{name}.json"), "w"), indent=1) | |
| reas = {} | |
| for name, path in kv(args.reason).items(): | |
| reas[name] = {r["id"]: r for r in read_jsonl(path)} | |
| common = set.intersection(*[set(r) for r in reas.values()]) if reas else set() | |
| ev_r = [it for it in ev if it["id"] in common] | |
| out["reasoning_items"] = len(ev_r) | |
| for name, rr in reas.items(): | |
| rows = [rr[it["id"]] for it in ev_r] | |
| ok = np.array([r["answer"] == it["gold"] for r, it in zip(rows, ev_r)]) | |
| toks = np.array([r["tokens"] for r in rows]) | |
| loops = np.array([r["looped"] for r in rows]) | |
| cal_rows = [(it, rr[it["id"]]) for it in cal if it["id"] in rr and rr[it["id"]]["post_lp"] is not None] | |
| pt = fit_temps([it for it, _ in cal_rows], [r["post_lp"] for _, r in cal_rows]) | |
| has = [i for i, r in enumerate(rows) if r["post_lp"] is not None] | |
| pp = [softmax(np.asarray(rows[i]["post_lp"]), pt[ev_r[i]["qtype"]]) for i in has] | |
| pok = np.array([int(np.argmax(p)) == ev_r[i]["gold"] for p, i in zip(pp, has)]) | |
| praw = [softmax(np.asarray(rows[i]["post_lp"]), 1.0) for i in has] | |
| out["reasoning"][name] = {"acc": {s: float(ok[[i for i, it in enumerate(ev_r) if it["set"] == s]].mean()) for s in EVAL_SETS} | {"all": float(ok.mean())}, | |
| "tokens_median": float(np.median(toks)), "tokens_p90": float(np.percentile(toks, 90)), | |
| "loops": int(loops.sum()), "loops_per_1k": float(1000 * loops.mean()), | |
| "loops_by_set": {s: int(loops[[i for i, it in enumerate(ev_r) if it["set"] == s]].sum()) for s in EVAL_SETS}, | |
| "post_temps": pt, "post_ece15": ece(np.array([p.max() for p in pp]), pok), | |
| "post_ece15_raw": ece(np.array([p.max() for p in praw]), pok), | |
| "post_top_above_0.99": float(np.mean([p.max() > 0.99 for p in praw]))} | |
| calf = os.path.join(os.path.dirname(args.out) or ".", f"calibration_{name}.json") | |
| c = json.load(open(calf)) if os.path.exists(calf) else {} | |
| c["thinking"] = pt | |
| json.dump(c, open(calf, "w"), indent=1) | |
| # agreement, disagreement AUROC and routing for each readout with its reasoner, on the common items | |
| idx = {it["id"]: i for i, it in enumerate(ev)} | |
| for ro_name, rs_name in kv(args.partner).items(): | |
| if ro_name not in probs_of or rs_name not in reas: | |
| continue | |
| rr = reas[rs_name] | |
| P = [probs_of[ro_name][idx[it["id"]]] for it in ev_r] | |
| rows = [rr[it["id"]] for it in ev_r] | |
| has = np.array([r["answer"] is not None for r in rows]) | |
| agree = np.array([int(np.argmax(p)) == r["answer"] for p, r in zip(P, rows)]) & has | |
| r_ok = np.array([int(np.argmax(p)) == it["gold"] for p, it in zip(P, ev_r)]) | |
| t_ok = np.array([r["answer"] == it["gold"] for r, it in zip(rows, ev_r)]) | |
| toks = np.array([r["tokens"] for r in rows]) | |
| curve = routing_curve(r_ok, np.where(has, t_ok, r_ok), toks, np.array([margin(p) for p in P])) | |
| out["pairs"][ro_name] = {"reasoner": rs_name, "agree_all": float(agree[has].mean()), | |
| "agree_by_set": {s: float(agree[[i for i, it in enumerate(ev_r) if it["set"] == s and has[i]]].mean()) for s in EVAL_SETS}, | |
| "dis_auroc": auroc(np.array([1 - p.max() for p in P])[has], ~agree[has]), | |
| "routing": curve, "gap_recovered_f0.3": gap_recovered(curve)} | |
| # the V2 targets, for every readout that has them | |
| for name, b in out["readout"].items(): | |
| t = {"readout_acc_all": b["acc"]["all"] >= TARGETS["readout_acc_all"], "od_capability": (b["od"]["capability"] or 0) >= TARGETS["od_capability"]} | |
| if name in out["pairs"]: | |
| t["agree_all"] = out["pairs"][name]["agree_all"] >= TARGETS["agree_all"] | |
| rs = out["pairs"][name]["reasoner"] | |
| t["post_ece"] = out["reasoning"][rs]["post_ece15"] <= TARGETS["post_ece"] | |
| if "base" in out["reasoning"]: | |
| t["reasoning_vs_base"] = out["reasoning"][rs]["acc"]["all"] >= out["reasoning"]["base"]["acc"]["all"] - 0.01 | |
| t["loops_vs_base"] = out["reasoning"][rs]["loops"] <= out["reasoning"]["base"]["loops"] | |
| out["targets"][name] = t | |
| os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) | |
| json.dump(out, open(args.out, "w"), indent=1, default=float) | |
| for name, b in out["readout"].items(): | |
| print(f"readout {name:12s} all {b['acc']['all']:.3f} OD Cap {b['od']['capability']:.1f} ECE {b['ece15']:.3f} NLL {b['nll']:.3f} targets {out['targets'].get(name)}") | |
| for name, b in out["reasoning"].items(): | |
| print(f"reason {name:12s} all {b['acc']['all']:.3f} tokens {b['tokens_median']:.0f} loops {b['loops']} post-ECE {b['post_ece15']:.3f} (raw {b['post_ece15_raw']:.3f})") | |
| for name, b in out["pairs"].items(): | |
| print(f"pair {name:12s} agree {b['agree_all']:.3f} dis-AUROC {b['dis_auroc']:.3f} gap@0.3 {b['gap_recovered_f0.3']:.2f}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 8.04 kB
- Xet hash:
- d76860f3d35c7f4eb25ba9683f71f3b2adcc8d086bce5dbdb8c6498b4bc4cf87
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.