Buckets:
| """Routed S1MB (V2 + RL fast readout, least certain 30% answered by thinking) vs fast readout alone. | |
| Writes results/report_figs/s1mb_routed.png, results/s1mb_routed.json and a manifest entry. Run with the report venv.""" | |
| import json | |
| import os | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| R = "/workspace/decision-reasoner" | |
| OUT = f"{R}/results/report_figs" | |
| plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 9, "axes.titlesize": 10.5, "axes.spines.top": False, | |
| "axes.spines.right": False}) | |
| BLUE, G1, G2, G3 = "#2563eb", "#9ca3af", "#6b7280", "#374151" | |
| d = json.load(open(f"{R}/results/s1mb.json")) | |
| tasks = {} | |
| for b in json.load(open(f"{R}/data/s1mb/english-v1.json"))["benchmarks"]: | |
| bid = b if isinstance(b, str) else (b.get("id") or b.get("name")) | |
| tasks[bid] = "noul" if bid.endswith("-noul-test-v1") or "-noul-" in bid else ("score" if "-score-" in bid or bid.endswith("score-test-v1") else "choice") | |
| fast = d["ours"]["rl60"]["calibrated"]["adjusted"] | |
| routed = d["ours"]["rl60_routed30"]["raw"]["adjusted"] | |
| board = {r["model"]: r for r in d["leaderboard"]} | |
| pub = [r for r in d["leaderboard"] if not r["model"].startswith("OURS:")] | |
| above = lambda r: sum(p["borda"] > r["borda"] for p in pub) | |
| above_ta = lambda r: sum(p["task_avg"] > r["task_avg"] for p in pub) | |
| rf, rr = board["OURS:rl60"], board["OURS:rl60_routed30"] | |
| summ = json.load(open(f"{R}/data/s1mb/think/rl60_routed30.summary.json")) | |
| diff = {b: routed[b] - fast[b] for b in fast if fast[b] is not None and routed.get(b) is not None} | |
| by_task = {t: [diff[b] for b in diff if tasks.get(b) == t] for t in ("noul", "choice", "score")} | |
| res = {"fast": {k: rf[k] for k in ("task_avg", "noul", "choice", "score", "borda")}, "routed": {k: rr[k] for k in ("task_avg", "noul", "choice", "score", "borda")}, | |
| "published_above_borda": {"fast": above(rf), "routed": above(rr)}, "published_above_task_avg": {"fast": above_ta(rf), "routed": above_ta(rr)}, | |
| "benchmarks_up": sum(v > 0.01 for v in diff.values()), "benchmarks_down": sum(v < -0.01 for v in diff.values()), "n": len(diff), | |
| "by_task_up_down": {t: [sum(v > 0.01 for v in vs), sum(v < -0.01 for v in vs), len(vs)] for t, vs in by_task.items()}, | |
| "top_gains": sorted(diff.items(), key=lambda x: -x[1])[:8], "top_losses": sorted(diff.items(), key=lambda x: x[1])[:8], "think": summ} | |
| json.dump(res, open(f"{R}/results/s1mb_routed.json", "w"), indent=1) | |
| fig, (a1, a2) = plt.subplots(1, 2, figsize=(6.5, 3.3), gridspec_kw={"width_ratios": [1.05, 1]}) | |
| labs = [("task_avg", "Task Avg"), ("noul", "Yes/no"), ("choice", "Choice"), ("score", "Score")] | |
| x = np.arange(len(labs)); w = 0.36 | |
| fv = [rf[k] for k, _ in labs]; rv = [rr[k] for k, _ in labs] | |
| a1.bar(x - w / 2, fv, w, color=G1, label="fast readout") | |
| a1.bar(x + w / 2, rv, w, color=BLUE, label="30% routed to thinking") | |
| for i in range(len(labs)): | |
| a1.text(x[i] - w / 2, fv[i] + 1, f"{fv[i]:.1f}", ha="center", fontsize=7.4, color=G3) | |
| a1.text(x[i] + w / 2, rv[i] + 1, f"{rv[i]:.1f}", ha="center", fontsize=7.4, color=BLUE, weight="bold") | |
| a1.set_xticks(x); a1.set_xticklabels([l for _, l in labs]); a1.set_ylim(0, 95) | |
| a1.set_ylabel("adjusted skill x100"); a1.grid(axis="y", color="#e5e7eb"); a1.set_axisbelow(True) | |
| a1.legend(frameon=False, fontsize=7.4, loc="upper left", ncol=2, handlelength=1) | |
| a1.set_title("V2 + RL on S1MB", loc="left", fontsize=9.5) | |
| vals = np.array(sorted(diff.values())) | |
| cols = [BLUE if v > 0.01 else ("#dc2626" if v < -0.01 else G1) for v in vals] | |
| a2.bar(np.arange(len(vals)), vals * 100, color=cols, width=1.0) | |
| a2.axhline(0, color=G2, lw=0.8) | |
| a2.set_xlabel(f"{len(vals)} benchmarks, sorted"); a2.set_ylabel("routed minus fast, points") | |
| a2.set_xticks([]); a2.grid(axis="y", color="#e5e7eb"); a2.set_axisbelow(True) | |
| a2.set_title(f"Per benchmark: {res['benchmarks_up']} up, {res['benchmarks_down']} down", loc="left", fontsize=9.5) | |
| d_ta = rr["task_avg"] - rf["task_avg"] | |
| fig.suptitle(f"Routing 30% to thinking moves S1MB Task Avg {rf['task_avg']:.1f} → {rr['task_avg']:.1f} ({d_ta:+.1f})", x=0.01, ha="left", fontsize=10.5) | |
| fig.tight_layout() | |
| fig.savefig(f"{OUT}/s1mb_routed.png", dpi=200, bbox_inches="tight"); plt.close(fig) | |
| cap = (f"S1MB with the deployed routing: V2 + RL's fast readout on every judgment, its thinking mode (post-chain readout, " | |
| f"calibration-fitted temperatures) on the {summ['routed']:,} judgments with the smallest margin (30%); " | |
| f"{summ['unfinished']} chains without an answer kept the fast answer. Mean {summ['thinking_tokens_per_judgment']:.0f} thinking tokens per judgment. " | |
| f"Source: results/s1mb.json (OURS:rl60_routed30), data/s1mb/think/.") | |
| man = f"{OUT}/manifest_mine.json" | |
| m = [x for x in json.load(open(man)) if x["slug"] != "s1mb_routed"] + [{"slug": "s1mb_routed", "title": "S1MB routed", "caption": cap}] | |
| json.dump(m, open(man, "w"), indent=1) | |
| print(json.dumps({k: v for k, v in res.items() if k not in ("top_gains", "top_losses")}, indent=1)) | |
| print("gains", [(b[:40], round(v, 2)) for b, v in res["top_gains"]]) | |
| print("losses", [(b[:40], round(v, 2)) for b, v in res["top_losses"]]) | |
Xet Storage Details
- Size:
- 5.14 kB
- Xet hash:
- 62fd02b146ab3db99133250b722a4c45800c51c854f0d80bdf4af983c6fdf478
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.