Buckets:
| """Per-item table for the qualitative error analysis of rl60 (V2 + 60 GRPO steps) on the 1,498-question thinking subset. | |
| Mirrors metrics_v2.py: readout probabilities are per-qtype temperature-calibrated (temps fitted on runs/v2/data/calib.jsonl | |
| with metrics.fit_temps), readout correctness is argmax == gold, routing sends the lowest-margin share (margin on the | |
| calibrated readout, stable mergesort over eval-file order) to thinking, and thinking correctness is the chain's parsed | |
| letter == gold (metrics_v2 scores `answer`; post_lp is used there only for calibration). The post-chain readout argmax is | |
| kept alongside as a sensitivity column. | |
| CPU only. Writes one JSON line per subset item to --out (default: the scratchpad table used by the other examples_ scripts). | |
| CUDA_VISIBLE_DEVICES="" python analysis/examples_build.py --out /path/items.jsonl | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import re | |
| import sys | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, os.path.join(ROOT, "runs", "v2")) | |
| import numpy as np # noqa: E402 | |
| from common import read_jsonl, softmax, split_chain # noqa: E402 | |
| from metrics import calibrated, fit_temps, margin # noqa: E402 | |
| V2 = os.path.join(ROOT, "runs", "v2") | |
| P2 = os.path.join(V2, "results", "predictions") | |
| P0 = os.path.join(ROOT, "results", "predictions") | |
| LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
| FRAC = 0.3 | |
| def load(path): | |
| return {r["id"]: r for r in read_jsonl(path)} | |
| def readout_probs(ev, cal, path): | |
| ro = load(path) | |
| temps = fit_temps(cal, [ro[it["id"]]["lp"] for it in cal]) | |
| return calibrated(ev, [ro[it["id"]]["lp"] for it in ev], temps), temps | |
| def post_temps(cal, rr): | |
| rows = [(it, rr[it["id"]]) for it in cal if it["id"] in rr and rr[it["id"]]["post_lp"] is not None] | |
| return fit_temps([it for it, _ in rows], [r["post_lp"] for _, r in rows]) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--out", required=True) | |
| args = ap.parse_args() | |
| ev = read_jsonl(os.path.join(V2, "data", "eval.jsonl")) | |
| cal = read_jsonl(os.path.join(V2, "data", "calib.jsonl")) | |
| keep = {l.strip() for l in open(os.path.join(V2, "data", "eval_subset_ids.txt")) if l.strip()} | |
| ro_rl, t_rl = readout_probs(ev, cal, os.path.join(P2, "readout_rl60.jsonl")) | |
| ro_v2, _ = readout_probs(ev, cal, os.path.join(P2, "readout_v2_joint.jsonl")) | |
| ro_b, _ = readout_probs(ev, cal, os.path.join(P0, "readout_base.jsonl")) | |
| rs = {"rl60": load(os.path.join(P2, "reason_rl60.jsonl")), "v2": load(os.path.join(P2, "reason_v2_joint.jsonl")), | |
| "base": load(os.path.join(P0, "reason_base.jsonl"))} | |
| pt = {k: post_temps(cal, v) for k, v in rs.items()} | |
| print("readout temps rl60", t_rl, "post temps rl60", pt["rl60"]) | |
| common = keep & set.intersection(*[set(v) for v in rs.values()]) | |
| idx = [i for i, it in enumerate(ev) if it["id"] in common] | |
| sub = [ev[i] for i in idx] | |
| P = [ro_rl[i] for i in idx] | |
| marg = np.array([margin(p) for p in P]) | |
| order = np.argsort(marg, kind="mergesort") | |
| m = int(round(FRAC * len(sub))) | |
| routed = np.zeros(len(sub), bool) | |
| routed[order[:m]] = True | |
| rank = np.empty(len(sub), int) | |
| rank[order] = np.arange(len(sub)) | |
| rows = [] | |
| for j, (i, it) in enumerate(zip(idx, sub)): | |
| p = P[j] | |
| g = it["gold"] | |
| row = {"id": it["id"], "set": it["set"], "family": it.get("family"), "qtype": it["qtype"], "k": len(it["options"]), | |
| "gold": g, "target": it["target"], | |
| "ro_pick": int(np.argmax(p)), "ro_p": float(p.max()), "ro_p_gold": float(p[g]), "ro_ok": int(np.argmax(p)) == g, | |
| "margin": float(marg[j]), "margin_rank": int(rank[j]), "routed": bool(routed[j]), | |
| "ro_v2_ok": int(np.argmax(ro_v2[i])) == g, "ro_base_ok": int(np.argmax(ro_b[i])) == g} | |
| for name, rr in rs.items(): | |
| r = rr[it["id"]] | |
| ans = r["answer"] | |
| post = None if r["post_lp"] is None else softmax(np.asarray(r["post_lp"]), pt[name][it["qtype"]]) | |
| chain, tail = split_chain(r["text"]) | |
| row[name] = {"answer": ans, "ok": ans == g, "post_pick": None if post is None else int(np.argmax(post)), | |
| "post_p": None if post is None else float(post.max()), | |
| "post_p_gold": None if post is None else float(post[g]), | |
| "post_ok": post is not None and int(np.argmax(post)) == g, | |
| "tokens": r["tokens"], "looped": r["looped"], "closed": r["closed"], "finish": r["finish"], | |
| "waits": len(re.findall(r"\bWait\b", chain or r["text"])), "chars": len(r["text"])} | |
| rows.append(row) | |
| with open(args.out, "w") as f: | |
| for r in rows: | |
| f.write(json.dumps(r) + "\n") | |
| # sanity: reproduce metrics_rl.json | |
| ro_ok = np.array([r["ro_ok"] for r in rows]) | |
| t_ok = np.array([r["rl60"]["ok"] for r in rows]) | |
| has = np.array([r["rl60"]["answer"] is not None for r in rows]) | |
| print(f"n={len(rows)} readout {ro_ok.mean():.4f} thinking {t_ok.mean():.4f} post-readout " | |
| f"{np.mean([r['rl60']['post_ok'] for r in rows]):.4f} routed@0.3 {np.where(routed, np.where(has, t_ok, ro_ok), ro_ok).mean():.4f}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.31 kB
- Xet hash:
- 09688268aa093dd6293051db5d0fe9308376e39b2042ebc917efe7b4fe697374
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.