Buckets:
| """Eval and benchmark figures for the PDF report. | |
| CUDA_VISIBLE_DEVICES="" <reportvenv>/bin/python analysis/fig_eval_plots.py [slug ...] | |
| Reads results/report_figs/fig_eval_data.json (from analysis/fig_eval_compute.py, project venv), results/final_stats.json, | |
| results/jevals.json, results/s1mb.json, results/yesno_offset.json and results/examples_analysis.json; writes | |
| results/report_figs/<slug>.png (200 dpi) and results/report_figs/manifest_eval.json. Every number printed in a figure is | |
| checked against results/FINAL_REPORT.md's value (REPORT below) before drawing. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import sys | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| from matplotlib.lines import Line2D # noqa: E402 | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| OUT = f"{ROOT}/results/report_figs" | |
| plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 9, "axes.titlesize": 10.5, "axes.spines.top": False, | |
| "axes.spines.right": False, "axes.edgecolor": "#6b7280", "axes.linewidth": 0.8, | |
| "xtick.color": "#374151", "ytick.color": "#374151", "axes.labelcolor": "#111827", | |
| "xtick.major.size": 3, "ytick.major.size": 3, "figure.facecolor": "white", "savefig.facecolor": "white"}) | |
| BLUE, G1, G2, G3, AMBER, TEAL = "#2563eb", "#9ca3af", "#6b7280", "#374151", "#d97706", "#059669" | |
| GRID, INK, INK2, FIELD = "#e5e7eb", "#111827", "#4b5563", "#d1d5db" | |
| GOOD, GOOD_L, BAD, BAD_L = "#16a34a", "#bbf7d0", "#dc2626", "#fecaca" | |
| MC = {"base": G1, "v1": G2, "v2": G3, "rl60": BLUE} | |
| ML = {"base": "Base Qwen3.5-4B", "v1": "v1", "v2": "V2", "rl60": "V2 + RL"} | |
| SETS = ["eval_proc", "eval_ts", "opendecisions", "localllama"] | |
| SL = {"eval_proc": "Procedural", "eval_ts": "tasksource", "opendecisions": "OpenDecisions", "localllama": "LocalLLaMA"} | |
| D = json.load(open(f"{OUT}/fig_eval_data.json")) | |
| FS = json.load(open(f"{ROOT}/results/final_stats.json")) | |
| JE = json.load(open(f"{ROOT}/results/jevals.json")) | |
| S1 = json.load(open(f"{ROOT}/results/s1mb.json")) | |
| YN = json.load(open(f"{ROOT}/results/yesno_offset.json")) | |
| EX = json.load(open(f"{ROOT}/results/examples_analysis.json")) | |
| manifest = [] | |
| # Values as printed in results/FINAL_REPORT.md; check() fails loudly when a figure number would round differently. | |
| REPORT = { | |
| "fast": {"base": 62.7, "v1": 69.5, "v2": 72.7, "rl60": 72.5}, "fast_sub": {"base": 61.8, "v1": 70.3, "v2": 76.0, "rl60": 75.9}, | |
| "think": {"base": 84.0, "v1": 84.4, "v2": 81.2, "rl60": 83.1}, "loop": {"base": 0.9, "v1": 1.4, "v2": 5.1, "rl60": 0.2}, | |
| "routed": {"base": 73.0, "v1": 78.0, "v2": 81.8, "rl60": 81.8}, "tok30": {"base": 1069, "v1": 1166, "v2": 1749, "rl60": 729}, | |
| "per_set_base": {"eval_proc": 46.8, "eval_ts": 60.4, "opendecisions": 66.5, "localllama": 63.0}, | |
| "per_set_rl60": {"eval_proc": 89.5, "eval_ts": 72.3, "opendecisions": 73.9, "localllama": 66.3}, | |
| "ece_fast_rl60": 0.030, "ece_post_rl60": 0.057, | |
| "2b_fast": {"base2b": 50.2, "v2b": 65.1, "v2b_ro": 66.0}, "2b_think": {"base2b": 79.4, "v2b": 71.7, "v2b_ro": 70.8}, | |
| "2b_loop": {"base2b": 0.1, "v2b": 7.4, "v2b_ro": 2.1}, "2b_r30": {"v2b": 71.9, "v2b_ro": 71.0, "hybrid2b": 75.6, "rl60": 81.9}, | |
| "2b_r50": {"v2b": 72.0, "v2b_ro": 71.0, "hybrid2b": 78.0}, "2b_tok30": {"hybrid2b": 253, "rl60": 742}, "2b_tok50": {"hybrid2b": 411}, | |
| "jev": {("pubmedqa", "rl60"): 58.6, ("pubmedqa", "rl60_think"): 63.5, ("pubmedqa", "v2_2b_ro"): 44.6, | |
| ("banking77", "rl60"): 61.3, ("banking77", "rl60_think"): 59.3, ("banking77", "v2_2b_ro"): 53.9, | |
| ("helpsteer2", "rl60"): 2.4, ("helpsteer2", "rl60_think"): -20.1, ("helpsteer2", "v2_2b_ro"): -0.4}, | |
| "jev_board": {("pubmedqa", "Gemini 3.8 Flash"): 73.0, ("pubmedqa", "Jev"): 69.0, ("banking77", "Gemini 3.8 Flash"): 74.1, | |
| ("banking77", "Jev"): 67.8, ("helpsteer2", "Gemini 3.8 Flash"): 4.6, ("helpsteer2", "Jev"): 9.2}, | |
| "s1_task": {"rl60": 47.9, "v1_joint": 43.5, "v2_2b_ro": 37.3, "base": 32.8, "base_2b": 18.1}, | |
| "s1_noul": {"rl60": 50.7, "v1_joint": 50.1, "v2_2b_ro": 42.3, "base": 33.8, "base_2b": 22.9}, | |
| "s1_choice": {"rl60": 57.2, "v1_joint": 52.3, "v2_2b_ro": 45.7, "base": 45.7, "base_2b": 29.5}, | |
| "s1_score": {"rl60": 35.8, "v1_joint": 28.0, "v2_2b_ro": 24.0, "base": 18.9, "base_2b": 2.0}, | |
| "s1_above": {"rl60": 74, "v1_joint": 100, "v2_2b_ro": 123, "base": 134, "base_2b": 162}, | |
| "s1_best4b": (67.0, 65.7, 76.5, 58.8), "s1_median": 41.2, "s1_best": 74.4, | |
| } | |
| _checked = [] | |
| def check(label, value, ref, nd=1): | |
| ok = round(value, nd) == round(ref, nd) if isinstance(ref, float) else int(value) == ref | |
| _checked.append((label, value, ref, ok)) | |
| if not ok: | |
| raise AssertionError(f"{label}: figure {value} vs FINAL_REPORT {ref}") | |
| return value | |
| def style(ax, axis="y"): | |
| ax.grid(axis=axis, color=GRID, lw=0.6) | |
| ax.grid(axis="x" if axis == "y" else "y", visible=False) | |
| ax.set_axisbelow(True) | |
| ax.tick_params(labelsize=8) | |
| def suptitle(fig, text): | |
| fig.suptitle(text, x=0.01, ha="left", fontsize=10.5, color=INK) | |
| # Report captions (1-2 sentences, ASCII for the PDF fonts); computed numbers come from each figure's key_numbers. | |
| CAPTIONS = { | |
| "headline": lambda k: "Fast readout on all 5,442 eval questions; thinking and routed 30% (lowest-margin 30% answered by thinking; " | |
| "labels: mean thinking tokens per question) on the 1,498-question subset, where the fast readout scores 61.8 / 70.3 / 76.0 / " | |
| "75.9%. Source: saved predictions, loaded and scored as in analysis/final_stats.py.", | |
| "routing_curve": lambda k: "Accuracy vs mean thinking tokens per question as the lowest-margin share sent to thinking goes from 0% " | |
| "(left) to 100% (right) on the 1,498-question subset; a chain with no answer falls back to the readout (metrics_v2.py rule, " | |
| f"hence V2's {k['v2']['acc100']:.1f}% at 100% vs 81.2% thinking-only), large dots mark 30%. Under this rule the 2B hybrid " | |
| f"(readout-only 2B fast + untrained 2B thinking) gives {k['hybrid2b']['acc30']:.1f}% at {k['hybrid2b']['tok30']:.0f} tokens at " | |
| "30% vs 75.6% / 253 in the report's 2B table (probability-gap rule, no fallback); source: saved predictions, CI from " | |
| "results/final_stats.json.", | |
| "per_set": lambda k: "Fast-readout accuracy on each eval set (5,442 questions in all), base -> v1 -> V2 -> V2 + RL; the two sets " | |
| "least like the training data (OpenDecisions, LocalLLaMA) gained least. Source: saved predictions (equal to " | |
| "runs/v2/results/metrics_rl.json).", | |
| "calibration": lambda k: "Reliability diagrams (15 equal-width confidence bins; bottom: share of items per bin) for V2 + RL's fast " | |
| "readout on all 5,442 questions and its letter readout after the chain on the 1,498-question subset, both with per-type " | |
| "temperatures fitted on the calibration items. Post-chain ECE V2 + RL - V2: +0.014 (95% CI -0.006 to +0.028); source: saved " | |
| "predictions, results/final_stats.json.", | |
| "thinking_tokens": lambda k: "Thinking tokens per chain on the 1,498-question subset (log-spaced bins, share of all chains); red: " | |
| "chains that hit the length cap without an answer. Looped-chain rate V2 + RL - V2: -4.9 points (95% CI -6.0 to -3.8); source: " | |
| "saved predictions, results/final_stats.json.", | |
| "fix_break": lambda k: "V2 + RL on the 1,498-question subset: questions where thinking fixed a wrong fast answer or broke a right " | |
| "one, split by whether they fall in the lowest-margin 30% that routing sends to thinking. In the routed 30%, " | |
| f"{k['all']['fix_routed']} fixes - {k['all']['break_routed']} breaks = +{k['all']['fix_routed'] - k['all']['break_routed']} " | |
| f"questions (+{100 * (k['all']['fix_routed'] - k['all']['break_routed']) / k['all']['n']:.1f} points, 75.9% -> 81.8%); source: " | |
| "results/examples_analysis.json, recomputed from saved predictions.", | |
| "twob": lambda k: "MiniCPM5-2B variants next to the 4B V2 + RL (labels in bars: mean thinking tokens per question); the routed " | |
| "column uses the report's 2B-table rule (lowest probability gap to thinking, the chain's parsed letter as answer), which gives " | |
| "the 4B 81.9% / 742 tokens vs 81.8% / 729 under the headline rule. Hybrid at 50% routed: " | |
| f"{k['hybrid_routed50'][0]:.1f}% at {k['hybrid_routed50'][1]} tokens; source: saved predictions in runs/v2_2b and runs/v2.", | |
| "jevals": lambda k: "Decision Score (Brier / RPS skill vs the label prior, x100, 95% CI) on the three Jevals 0.1.0 boards (300 " | |
| "items each): published entries in grey (ordered by mean score) and ours, scored by our reimplementation of the Jevals rules " | |
| "with calibration-fitted temperatures. Source: results/jevals.json.", | |
| "s1mb_field": lambda k: "Left: S1MB English v1 Task Avg of the 211 complete published models (grey, ranked) and our five " | |
| "(fast readout); 74 published models rank above V2 + RL in the leaderboard's Borda order. Right: per-task scores of our 4B " | |
| "models vs the best published 4B (KnowLine-4B-Gen5, outlined) and the published median (dashed); source: results/s1mb.json " | |
| "(routed rows excluded).", | |
| "s1mb_seen_unseen": lambda k: "Base -> V2 + RL on S1MB by whether a benchmark's source dataset contributed train-split items to " | |
| "our training pool (seen 41, unseen 96), with 95% CIs from 10,000 benchmark-level bootstrap resamples. Seen holds 1 score " | |
| "benchmark vs 20 unseen, so the task-matched contrast is the composition-fair one; source: results/final_stats.json.", | |
| "forest": lambda k: "Differences in accuracy (and looped-chain rate), in points, with 95% percentile intervals from 10,000 paired bootstrap resamples of " | |
| "questions (same resample for every model, temperatures fixed); blue rows involve V2 + RL. Single training seed, so the " | |
| "intervals cover question sampling only; source: results/final_stats.json.", | |
| "yesno": lambda k: "Share of yes/no items the fast readout answers yes vs the gold yes share (our eval) or the target yes mass " | |
| "(S1MB); filled: per-type temperatures, open: plus one yes/no logit offset fitted on the 109 yes/no calibration items, notes: " | |
| "balanced accuracy (eval) or yes/no adjusted skill x100 (S1MB), without -> with the offset. Source: results/yesno_offset.json; " | |
| "S1MB targets from data/s1mb/items.jsonl.", | |
| } | |
| def save(fig, slug, title, caption, key_numbers): | |
| fig.savefig(f"{OUT}/{slug}.png", dpi=200, bbox_inches="tight", pad_inches=0.06) | |
| plt.close(fig) | |
| if slug in CAPTIONS: | |
| caption = CAPTIONS[slug](key_numbers) | |
| manifest.append({"slug": slug, "title": title, "caption": caption, "key_numbers": key_numbers}) | |
| def fmt_tok(x): | |
| return f"{x:,.0f}" | |
| # ====================================================================================================== 1 headline | |
| def headline(): | |
| models = ["base", "v1", "v2", "rl60"] | |
| fast = [check(f"fast {m}", D["readout"][m]["acc_all"], REPORT["fast"][m]) for m in models] | |
| think = [check(f"think {m}", D["thinking"][m]["acc"], REPORT["think"][m]) for m in models] | |
| routed = [check(f"routed30 {m}", 100 * D["routing"][m][30]["acc"], REPORT["routed"][m]) for m in models] | |
| toks = [D["routing"][m][30]["tokens"] for m in models] | |
| for m, t in zip(models, toks): | |
| check(f"tok30 {m}", int(t), REPORT["tok30"][m]) # report truncates base's 1,069.7 | |
| for m in models: | |
| check(f"fast subset {m}", D["readout"][m]["acc_subset"], REPORT["fast_sub"][m]) | |
| fig, axes = plt.subplots(1, 3, figsize=(6.5, 3.2), sharey=True, layout="constrained") | |
| y = np.arange(len(models))[::-1] | |
| panels = [("Fast readout", "all 5,442 questions", fast, None), ("Thinking", "1,498-question subset", think, None), | |
| ("Routed 30% to thinking", "1,498-question subset", routed, toks)] | |
| for ax, (t1, t2, vals, tk) in zip(axes, panels): | |
| cols = [MC[m] for m in models] | |
| ax.barh(y, vals, height=0.64, color=cols, edgecolor="white", linewidth=1) | |
| for yi, v, m, i in zip(y, vals, models, range(4)): | |
| ax.text(v + 1.5, yi, f"{v:.1f}", va="center", ha="left", fontsize=8.2, color=INK, | |
| weight="bold" if m == "rl60" else "normal") | |
| if tk is not None: | |
| ax.text(v - 2, yi, f"{fmt_tok(int(tk[i]))} tok/q", va="center", ha="right", fontsize=7, | |
| color=INK if m == "base" else "white") | |
| ax.set_xlim(0, 106) | |
| ax.set_xticks([0, 25, 50, 75, 100]) | |
| ax.set_title(f"{t1}\n", loc="left", fontsize=9, color=INK, pad=2) | |
| ax.text(0, 1.015, t2, transform=ax.transAxes, fontsize=7.5, color=INK2, va="bottom") | |
| ax.set_xlabel("Accuracy (%)", fontsize=8) | |
| style(ax, "x") | |
| ax.tick_params(axis="y", length=0) | |
| axes[0].set_yticks(y, [ML[m] for m in models], fontsize=8.5) | |
| suptitle(fig, "Training adds 10 points to the fast readout; routing 30% to thinking adds 6 more") | |
| save(fig, "headline", "Headline accuracy by mode", | |
| "Fast readout on all 5,442 eval questions; thinking and the routed system (lowest-margin 30% answered by thinking " | |
| "mode) on the 1,498-question thinking subset, where the fast readout scores 61.8 / 70.3 / 76.0 / 75.9%. Labels inside " | |
| "the routed bars are mean thinking tokens per question. Source: saved predictions, as in analysis/final_stats.py.", | |
| {"fast": dict(zip(models, [round(v, 1) for v in fast])), "thinking": dict(zip(models, [round(v, 1) for v in think])), | |
| "routed30": dict(zip(models, [round(v, 1) for v in routed])), "tokens30": dict(zip(models, [round(t, 1) for t in toks]))}) | |
| # ====================================================================================================== 2 routing curve | |
| def routing_curve(): | |
| fig, ax = plt.subplots(figsize=(6.5, 3.4), layout="constrained") | |
| series = [("base", G1, 1.5), ("v1", G2, 1.5), ("v2", G3, 1.5), ("hybrid2b", TEAL, 1.6), ("rl60", BLUE, 2.3)] | |
| series_c = [(m, c) for m, c, _ in series] | |
| pts = {} | |
| for m, c, lw in series: | |
| cur = D["routing"][m] | |
| x = np.array([r["tokens"] for r in cur]); yv = np.array([100 * r["acc"] for r in cur]) | |
| ax.plot(x, yv, color=c, lw=lw, zorder=3 if m == "rl60" else 2, solid_capstyle="round") | |
| ax.plot(x[30], yv[30], "o", ms=7 if m == "rl60" else 6, color=c, mec="white", mew=1.2, zorder=4) | |
| ax.plot(x[-1], yv[-1], "o", ms=3.2, color=c, zorder=4) | |
| pts[m] = (x, yv) | |
| for m in ("base", "v1", "v2", "rl60"): | |
| check(f"curve30 {m}", pts[m][1][30], REPORT["routed"][m]) | |
| check(f"curve30 tok {m}", int(pts[m][0][30]), REPORT["tok30"][m]) | |
| # curve-end names (4B); the 2B hybrid is named next to its 30% point | |
| ends = {"base": ("Base", 55, -0.25, "left", "center"), "v1": ("v1", 55, 0.25, "left", "center"), | |
| "v2": ("V2", 55, 0.0, "left", "center"), "rl60": ("V2 + RL", -10, 0.45, "right", "bottom")} | |
| for m, (txt, dx, dy, ha, va) in ends.items(): | |
| x, yv = pts[m] | |
| ax.text(x[-1] + dx, yv[-1] + dy, txt, color=INK, fontsize=8.2, va=va, ha=ha, weight="bold" if m == "rl60" else "normal") | |
| x, yv = pts["hybrid2b"] | |
| ax.text(x[30] + 45, yv[30] - 0.55, "2B hybrid", fontsize=8.2, color=INK, ha="left", va="center") | |
| # same accuracy, fewer tokens: connector between the two 30% points | |
| xa, ya = pts["v2"][0][30], pts["v2"][1][30] | |
| xb, yb = pts["rl60"][0][30], pts["rl60"][1][30] | |
| d = FS["thinking_subset"]["tokens30_rl60-v2"] | |
| ax.annotate("", xy=(xb + 55, yb), xytext=(xa - 55, ya), | |
| arrowprops=dict(arrowstyle="-|>", color=INK, lw=0.9, ls=(0, (2, 1.5)), mutation_scale=9)) | |
| # table of the 30% points (doubles as the colour key) | |
| tx, ty, dy = 2330, 72.6, 1.45 | |
| ax.text(tx, ty, "Routed 30%", fontsize=7.6, color=INK, weight="bold", va="center") | |
| ax.text(tx + 1180, ty, "accuracy", fontsize=7.6, color=INK2, va="center", ha="right") | |
| ax.text(tx + 1850, ty, "tokens/q", fontsize=7.6, color=INK2, va="center", ha="right") | |
| for i, m in enumerate(["rl60", "v2", "v1", "base"]): | |
| x, yv = pts[m] | |
| yy = ty - (i + 1) * dy | |
| ax.plot(tx + 40, yy, "o", ms=5.5, color=dict(series_c)[m], mec="white", mew=0.8) | |
| ax.text(tx + 130, yy, {"base": "Base"}.get(m, ML[m]), fontsize=7.6, color=INK, va="center", weight="bold" if m == "rl60" else "normal") | |
| ax.text(tx + 1180, yy, f"{yv[30]:.1f}%", fontsize=7.6, color=INK, va="center", ha="right") | |
| ax.text(tx + 1850, yy, fmt_tok(int(x[30])), fontsize=7.6, color=INK, va="center", ha="right") | |
| yy = ty - 5 * dy - 0.35 | |
| ax.text(tx, yy, f"V2 + RL − V2: {d['point']:,.0f} tokens per question\n(95% CI {d['ci95'][0]:,.0f} to {d['ci95'][1]:,.0f}) at equal accuracy".replace("-", "−"), | |
| fontsize=7.3, color=INK2, va="center") | |
| ax.set_xlim(-60, 4300) | |
| ax.set_ylim(60, 87) | |
| ax.set_xlabel("Mean thinking tokens per question (0 for questions not routed)", fontsize=8.5) | |
| ax.set_ylabel("Accuracy (%)", fontsize=8.5) | |
| style(ax, "y") | |
| ax.set_title("V2 + RL, 30% routed: V2's accuracy at 42% of its thinking tokens", loc="left", fontsize=10.5, color=INK) | |
| h = D["routing"]["hybrid2b"] | |
| save(fig, "routing_curve", "Accuracy vs thinking tokens under margin routing", | |
| "Routed fraction swept from 0% (left end: fast readout only) to 100% (right end) on the 1,498-question subset; " | |
| "questions with the smallest calibrated log-margin go to thinking first and a chain with no answer falls back to " | |
| "the readout (metrics_v2.py rule), which is why V2's 100% point (84.3%) exceeds its thinking-only accuracy (81.2%). " | |
| "Large dots: 30% routed. 2B hybrid = readout-only 2B fast + untrained MiniCPM5-2B thinking; under this rule it gives " | |
| f"{100 * h[30]['acc']:.1f}% at {h[30]['tokens']:.0f} tokens at 30% (the report's 2B table, probability-gap rule " | |
| "without fallback: 75.6% at 253).", | |
| {m: {"acc30": round(100 * D["routing"][m][30]["acc"], 2), "tok30": round(D["routing"][m][30]["tokens"], 1), | |
| "acc100": round(100 * D["routing"][m][-1]["acc"], 2), "tok100": round(D["routing"][m][-1]["tokens"], 1)} | |
| for m in ("base", "v1", "v2", "rl60", "hybrid2b")} | {"tokens30_rl60_minus_v2": [round(d["point"]), [round(v) for v in d["ci95"]]]}) | |
| # ====================================================================================================== 3 per set | |
| def per_set(): | |
| models = ["base", "v1", "v2", "rl60"] | |
| fig, axes = plt.subplots(1, 4, figsize=(6.5, 3.2), sharey=True, layout="constrained") | |
| kn = {} | |
| for ax, s in zip(axes, SETS): | |
| v = [D["readout"][m]["acc_by_set"][s] for m in models] | |
| check(f"per_set base {s}", v[0], REPORT["per_set_base"][s]); check(f"per_set rl60 {s}", v[-1], REPORT["per_set_rl60"][s]) | |
| x = np.arange(4) | |
| ax.plot(x, v, color=G1, lw=1.2, zorder=2) | |
| ax.scatter(x, v, s=[30, 30, 30, 46], c=[MC[m] for m in models], zorder=3, edgecolors="white", linewidths=0.8) | |
| ax.text(0, v[0] - 3.2, f"{v[0]:.1f}", ha="center", va="top", fontsize=7.8, color=INK) | |
| ax.text(3, v[-1] + 3.0, f"{v[-1]:.1f}", ha="center", va="bottom", fontsize=7.8, color=INK, weight="bold") | |
| g = v[-1] - v[0] | |
| ax.text(0.5, 0.04, f"+{g:.0f} points", transform=ax.transAxes, ha="center", fontsize=8.5, color=INK, weight="bold") | |
| ax.set_xticks(x, ["Base", "v1", "V2", "V2+RL"], fontsize=7.6) | |
| ax.set_xlim(-0.45, 3.45) | |
| ax.set_title(f"{SL[s]}\n{D['set_n_eval'][s]:,} questions", loc="left", fontsize=8.5, color=INK) | |
| style(ax, "y") | |
| kn[s] = {m: round(val, 1) for m, val in zip(models, v)} | |
| axes[0].set_ylim(35, 100) | |
| axes[0].set_ylabel("Fast-readout accuracy (%)", fontsize=8.5) | |
| suptitle(fig, "Gains are largest on procedural questions and smallest on LocalLLaMA") | |
| save(fig, "per_set", "Fast-readout accuracy by eval set", | |
| "Fast-readout accuracy on each of the four eval sets (5,442 questions in all), base -> v1 -> V2 -> V2 + RL; " | |
| "per-type temperatures fitted on the calibration items. The two sets least like the training data " | |
| "(OpenDecisions, LocalLLaMA) gained least. Source: saved predictions; equals runs/v2/results/metrics_rl.json.", kn) | |
| # ====================================================================================================== 4 calibration | |
| def calibration(): | |
| fig = plt.figure(figsize=(6.5, 3.5), layout="constrained") | |
| gs = fig.add_gridspec(2, 2, height_ratios=[3.2, 1]) | |
| ef = check("ece fast", D["readout"]["rl60"]["ece15"], REPORT["ece_fast_rl60"], 3) | |
| ep = check("ece post", D["thinking"]["rl60"]["post_ece15"], REPORT["ece_post_rl60"], 3) | |
| cfg = [(0, D["reliability_fast_rl60"], BLUE, "V2 + RL, fast readout", f"ECE {ef:.3f} (5,442 questions, 15 bins)"), | |
| (1, D["reliability_post_rl60"], AMBER, "V2 + RL, readout after the chain", | |
| f"ECE {ep:.3f} ({D['thinking']['rl60']['post_n']:,} chains, 15 bins)")] | |
| for col, rows, c, t, lab in cfg: | |
| ax = fig.add_subplot(gs[0, col]); axh = fig.add_subplot(gs[1, col], sharex=ax) | |
| ax.plot([0, 1], [0, 1], ls=(0, (3, 3)), color=G1, lw=0.9) | |
| conf = np.array([r["conf"] for r in rows]); acc = np.array([r["acc"] for r in rows]); n = np.array([r["n"] for r in rows]) | |
| big = n >= 10 | |
| ax.plot(conf[big], acc[big], color=c, lw=1.6, zorder=3) | |
| ax.scatter(conf[big], acc[big], s=22, color=c, zorder=4, edgecolors="white", linewidths=0.7) | |
| if (~big).any(): | |
| ax.scatter(conf[~big], acc[~big], s=16, facecolors="white", edgecolors=c, linewidths=0.9, zorder=4) | |
| ax.set_xlim(0, 1.0); ax.set_ylim(0, 1.0) | |
| ax.set_ylabel("Accuracy in bin", fontsize=8) | |
| ax.set_title(t, loc="left", fontsize=8.8, color=INK) | |
| ax.text(0.03, 0.95, lab, transform=ax.transAxes, fontsize=7.6, color=INK, va="top") | |
| if (~big).any(): | |
| ax.text(0.03, 0.86, "open dots: bins with < 10 items", transform=ax.transAxes, fontsize=7, color=INK2, va="top") | |
| ax.text(0.42, 0.1, "dashed: perfect calibration", fontsize=7, color=G2, ha="left") | |
| style(ax, "y"); ax.tick_params(labelbottom=False) | |
| lo = np.array([r["lo"] for r in rows]) | |
| axh.bar(lo + 1 / 30, n / n.sum() * 100, width=1 / 15 * 0.86, color=c, alpha=0.35, edgecolor="none") | |
| axh.set_ylabel("% of items", fontsize=7.5) | |
| axh.set_xlabel("Confidence (top option probability)", fontsize=8) | |
| style(axh, "y") | |
| axh.set_ylim(0, max(50, (n / n.sum() * 100).max() * 1.1)) | |
| suptitle(fig, "The fast readout is calibrated; after the chain, mid-confidence answers are overconfident") | |
| save(fig, "calibration", "Reliability diagrams, V2 + RL", | |
| "Accuracy vs mean confidence in 15 equal-width confidence bins for V2 + RL. Left: fast readout on all 5,442 questions. " | |
| "Right: the letter readout after the chain on the 1,498-question subset (chains with a post-chain readout). Per-type " | |
| "temperatures fitted on the calibration items for both. Bottom: share of items per bin. ECE as in final_stats.json " | |
| "(post-chain V2 + RL - V2: +0.014, 95% CI -0.006 to +0.028).", | |
| {"ece_fast": round(ef, 4), "ece_post": round(ep, 4), "post_n": D["thinking"]["rl60"]["post_n"]}) | |
| # ====================================================================================================== 5 thinking tokens | |
| def thinking_tokens(): | |
| fig, axes = plt.subplots(2, 1, figsize=(6.5, 3.4), sharex=True, layout="constrained") | |
| bins = np.logspace(np.log10(180), np.log10(33500), 46) | |
| kn = {} | |
| for ax, m in zip(axes, ["v2", "rl60"]): | |
| t = D["thinking"][m] | |
| tok = np.array(t["tokens"]); lp = np.array(t["looped_flags"]).astype(bool) | |
| n = len(tok) | |
| h_ok, _ = np.histogram(tok[~lp], bins); h_lp, _ = np.histogram(tok[lp], bins) | |
| w = np.diff(bins) | |
| ax.bar(bins[:-1], h_ok / n * 100, width=w * 0.92, align="edge", color=MC[m], edgecolor="none") | |
| ax.bar(bins[:-1], h_lp / n * 100, width=w * 0.92, align="edge", bottom=h_ok / n * 100, color=BAD, edgecolor="none") | |
| med, mean = t["tokens_median"], t["tokens_mean"] | |
| ax.axvline(med, color=INK, lw=0.9, ls=(0, (3, 2))) | |
| ax.text(med * 1.06, 6.6, f"median {med:,.0f}", fontsize=7.6, color=INK, va="top") | |
| lr = check(f"loop {m}", t["loop_rate"], REPORT["loop"][m]) | |
| ax.text(0.94, 0.95, f"mean {mean:,.0f} tokens per chain", transform=ax.transAxes, ha="right", va="top", fontsize=7.6, color=INK) | |
| ax.text(0.94, 0.82, f"looped, no answer: {lr:.1f}% ({t['n_looped']} chains)", transform=ax.transAxes, | |
| ha="right", va="top", fontsize=7.6, color=INK) | |
| if m == "rl60": | |
| ax.annotate("", xy=(31000, (h_ok[-1] + h_lp[-1]) / n * 100 + 0.2), xytext=(31000, 5.3), | |
| arrowprops=dict(arrowstyle="-|>", color=BAD, lw=0.9, mutation_scale=8)) | |
| ax.text(0.005, 0.93, f"{ML[m]}", transform=ax.transAxes, fontsize=9, color=INK, weight="bold", va="top") | |
| ax.set_ylim(0, 7.0) | |
| ax.set_ylabel("% of chains", fontsize=8) | |
| style(ax, "y") | |
| kn[m] = {"median": med, "mean": round(mean), "loop_rate": round(lr, 2), "n_looped": t["n_looped"], "p90": round(t["tokens_p90"])} | |
| ax = axes[1] | |
| ax.set_xscale("log") | |
| ax.set_xticks([200, 500, 1000, 2000, 5000, 10000, 32768], ["200", "500", "1K", "2K", "5K", "10K", "32K"]) | |
| ax.minorticks_off() | |
| ax.set_xlim(180, 34000) | |
| ax.set_xlabel("Thinking tokens per chain (log scale)", fontsize=8.5) | |
| suptitle(fig, "RL halved chain length (mean 3,748 → 1,802 tokens) and removed the loops") | |
| save(fig, "thinking_tokens", "Thinking tokens per chain, V2 vs V2 + RL", | |
| "Distribution of generated thinking tokens per chain on the 1,498-question subset (log-spaced bins; share of all chains). " | |
| "Red: chains that ran to the length cap without an answer. Looped-chain rate V2 + RL - V2: -4.9 points " | |
| "(95% CI -6.0 to -3.8, final_stats.json).", kn) | |
| # ====================================================================================================== 6 fix / break | |
| def fix_break(): | |
| fb = D["fix_break_rl60"] | |
| for s in SETS + ["all"]: # cross-check against results/examples_analysis.json | |
| c = EX["counts"][s] | |
| assert fb[s]["fix"] == c["cells"]["WR"] and fb[s]["break"] == c["cells"]["RW"], s | |
| assert fb[s]["fix_routed"] == c["cells_routed"]["WR"] and fb[s]["break_routed"] == c["cells_routed"]["RW"], s | |
| rows = ["all"] + SETS | |
| fig, ax = plt.subplots(figsize=(6.5, 3.2), layout="constrained") | |
| y = np.array([5.3, 3.6, 2.4, 1.2, 0.0]) | |
| h = 0.62 | |
| for yi, s in zip(y, rows): | |
| r = fb[s] | |
| ax.barh(yi, r["fix_routed"], height=h, color=GOOD, edgecolor="white", lw=0.8) | |
| ax.barh(yi, r["fix"] - r["fix_routed"], left=r["fix_routed"], height=h, color=GOOD_L, edgecolor="white", lw=0.8) | |
| ax.barh(yi, -r["break_routed"], height=h, color=BAD, edgecolor="white", lw=0.8) | |
| ax.barh(yi, -(r["break"] - r["break_routed"]), left=-r["break_routed"], height=h, color=BAD_L, edgecolor="white", lw=0.8) | |
| ax.text(r["fix"] + 4, yi, f"{r['fix']} fixed ({r['fix_routed']} routed)", va="center", fontsize=7.8, color=INK) | |
| ax.text(-r["break"] - 4, yi, f"{r['break']} broken ({r['break_routed']} routed)", va="center", ha="right", fontsize=7.8, color=INK) | |
| ax.axvline(0, color=G3, lw=0.9) | |
| names = [f"All ({fb['all']['n']:,})\n{fb['all']['routed']} routed"] + [f"{SL[s]} ({fb[s]['n']})\n{fb[s]['routed']} routed" for s in SETS] | |
| ax.set_yticks(y, names, fontsize=8) | |
| ax.tick_params(axis="y", length=0) | |
| ax.set_xlim(-285, 330) | |
| ax.set_ylim(-0.55, 6.25) | |
| ticks = [-200, -100, 0, 100, 200, 300] | |
| ax.set_xticks(ticks, [str(abs(t)) for t in ticks]) | |
| ax.set_xlabel("Questions (left: thinking broke a correct fast answer; right: thinking fixed a wrong one)", fontsize=8) | |
| style(ax, "x") | |
| a = fb["all"] | |
| ax.text(a["fix_routed"] / 2, y[0] + 0.52, "in routed 30%", ha="center", fontsize=7, color=INK2) | |
| ax.text(a["fix_routed"] + (a["fix"] - a["fix_routed"]) / 2, y[0] + 0.52, "not routed", ha="center", fontsize=7, color=INK2) | |
| ax.axhline(4.45, color=GRID, lw=0.8) | |
| net = a["fix_routed"] - a["break_routed"] | |
| share = a["fix_routed"] / a["fix"] | |
| suptitle(fig, f"Thinking fixes twice what it breaks; the routed 30% holds {100 * share:.0f}% of the fixes") | |
| save(fig, "fix_break", "Where thinking fixes and breaks the fast answer", | |
| f"V2 + RL on the 1,498-question subset: questions where thinking turned a wrong fast-readout answer right (fixed) or a " | |
| f"right one wrong (broken), split by whether the question falls in the lowest-margin 30% that routing sends to " | |
| f"thinking. In the routed 30%, {a['fix_routed']} fixes minus {a['break_routed']} breaks = +{net} questions " | |
| f"(+{100 * net / a['n']:.1f} points: 75.9% -> 81.8%). Source: results/examples_analysis.json (recomputed).", | |
| {s: {k: fb[s][k] for k in ("n", "routed", "fix", "fix_routed", "break", "break_routed")} for s in rows}) | |
| # ====================================================================================================== 7 2B | |
| def twob(): | |
| F, T, RT = D["readout"], D["thinking"], D["routing_table2b"] | |
| rows = [("base2b", "base2b", None, "MiniCPM5-2B, untrained", G1), ("v2b", "v2b", "v2b", "2B, V2 recipe", G2), | |
| ("v2b_ro", "v2b_ro", "v2b_ro", "2B, readout-only", G3), | |
| ("v2b_ro", "base2b", "hybrid2b", "2B hybrid: readout-only\nfast + untrained thinking", TEAL), | |
| ("rl60", "rl60", "rl60", "4B V2 + RL (reference)", BLUE)] | |
| for k in ("base2b", "v2b", "v2b_ro"): | |
| check(f"2b fast {k}", F[k]["acc_all"], REPORT["2b_fast"][k]); check(f"2b think {k}", T[k]["acc"], REPORT["2b_think"][k]) | |
| check(f"2b loop {k}", T[k]["loop_rate"], REPORT["2b_loop"][k]) | |
| for k in ("v2b", "v2b_ro", "hybrid2b", "rl60"): | |
| check(f"2b r30 {k}", 100 * RT[k][1]["acc"], REPORT["2b_r30"][k]) | |
| for k in ("v2b", "v2b_ro", "hybrid2b"): | |
| check(f"2b r50 {k}", 100 * RT[k][2]["acc"], REPORT["2b_r50"][k]) | |
| check("2b tok30 hybrid", round(RT["hybrid2b"][1]["tokens"]), REPORT["2b_tok30"]["hybrid2b"]) | |
| check("2b tok30 rl60", round(RT["rl60"][1]["tokens"]), REPORT["2b_tok30"]["rl60"]) | |
| check("2b tok50 hybrid", round(RT["hybrid2b"][2]["tokens"]), REPORT["2b_tok50"]["hybrid2b"]) | |
| fig, axes = plt.subplots(1, 3, figsize=(6.5, 3.3), sharey=True, layout="constrained") | |
| y = np.array([4.4, 3.3, 2.2, 1.1, -0.25]) | |
| kn = {} | |
| for pi, ax in enumerate(axes): | |
| for yi, (ro, th, pair, lab, c) in zip(y, rows): | |
| if pi == 0: | |
| v, tk = F[ro]["acc_all"], None | |
| elif pi == 1: | |
| v, tk = T[th]["acc"], None | |
| else: | |
| if pair is None: | |
| ax.text(1, yi, "not reported", va="center", fontsize=7.4, color=INK2) | |
| continue | |
| v, tk = 100 * RT[pair][1]["acc"], RT[pair][1]["tokens"] | |
| ax.barh(yi, v, height=0.72, color=c, edgecolor="white") | |
| ax.text(v + 1.5, yi, f"{v:.1f}", va="center", fontsize=8, color=INK, weight="bold" if c in (TEAL, BLUE) else "normal") | |
| if tk is not None: | |
| ax.text(v - 2, yi, f"{tk:,.0f} tok/q", va="center", ha="right", fontsize=6.9, color="white") | |
| kn.setdefault(lab.replace("\n", " "), {})[["fast", "thinking", "routed30"][pi]] = round(v, 1) | |
| ax.set_xlim(0, 106); ax.set_xticks([0, 25, 50, 75, 100]) | |
| ax.set_xlabel("Accuracy (%)", fontsize=8) | |
| style(ax, "x"); ax.tick_params(axis="y", length=0) | |
| axes[0].set_title("Fast readout\n", loc="left", fontsize=9, color=INK, pad=2) | |
| axes[1].set_title("Thinking\n", loc="left", fontsize=9, color=INK, pad=2) | |
| axes[2].set_title("Routed 30% to thinking\n", loc="left", fontsize=9, color=INK, pad=2) | |
| for ax, t2 in zip(axes, ["all 5,442 questions", "1,498-question subset", "1,498-question subset"]): | |
| ax.text(0, 1.015, t2, transform=ax.transAxes, fontsize=7.5, color=INK2, va="bottom") | |
| axes[0].set_yticks(y, [r[3] for r in rows], fontsize=8) | |
| for ax in axes: | |
| ax.axhline(0.45, color=GRID, lw=0.8) | |
| suptitle(fig, "2B: fine-tuning hurt thinking, so trained readout + untouched thinking works best") | |
| h = RT["hybrid2b"] | |
| save(fig, "twob", "The 2B variants", | |
| "MiniCPM5-2B variants next to the 4B V2 + RL. Routed column as in the report's 2B table: lowest probability gap " | |
| "between the top two calibrated readout options goes to thinking, the chain's parsed letter is the answer (gives " | |
| f"81.9% / 742 tokens for the 4B, vs 81.8% / 729 under the headline rule). Hybrid at 50% routed: " | |
| f"{100 * h[2]['acc']:.1f}% at {h[2]['tokens']:.0f} tokens. Labels inside bars: mean thinking tokens per question.", | |
| kn | {"hybrid_routed50": [round(100 * h[2]["acc"], 1), round(h[2]["tokens"])], "loop_rates": {k: round(T[k]["loop_rate"], 1) for k in ("base2b", "v2b", "v2b_ro")}}) | |
| # ====================================================================================================== 8 Jevals | |
| def jevals(): | |
| boards = [("pubmedqa", "PubMedQA (yes/no)"), ("banking77", "Banking77 (77 intents)"), ("helpsteer2", "HelpSteer2 (5-level score)")] | |
| board = [e for e in JE["board"] if e["interface"] != "baseline"] | |
| names = sorted({e["name"] for e in board}, key=lambda n: -np.mean([e["ds"] for e in board if e["name"] == n])) | |
| ours = [("rl60", "Ours: V2 + RL, fast", BLUE), ("rl60_think", "Ours: V2 + RL, thinking", AMBER), | |
| ("v2_2b_ro", "Ours: 2B readout-only, fast", TEAL)] | |
| for (b, k), v in REPORT["jev"].items(): | |
| check(f"jev {b} {k}", JE["ours"][k]["calibrated"][b]["ds"], v) | |
| for (b, nm), v in REPORT["jev_board"].items(): | |
| check(f"jev board {b} {nm}", next(e["ds"] for e in board if e["board"] == b and e["name"] == nm), v) | |
| nrow = len(names) + len(ours) | |
| ys = {n: nrow - i + 0.6 for i, n in enumerate(names)} | |
| ys |= {k: len(ours) - i - 0.2 for i, (k, _, _) in enumerate(ours)} | |
| fig, axes = plt.subplots(1, 3, figsize=(6.5, 3.5), sharey=True, layout="constrained") | |
| kn = {} | |
| for ax, (b, bt) in zip(axes, boards): | |
| for e in board: | |
| if e["board"] != b: | |
| continue | |
| yy = ys[e["name"]] | |
| ax.plot(e["ci"], [yy, yy], color=FIELD, lw=1.6, solid_capstyle="butt") | |
| ax.plot(e["ds"], yy, "o", ms=4.6, color=G2) | |
| for k, lab, c in ours: | |
| r = JE["ours"][k]["calibrated"][b] | |
| yy = ys[k] | |
| ax.plot(r["ci"], [yy, yy], color=c, lw=1.6, alpha=0.45, solid_capstyle="butt") | |
| ax.plot(r["ds"], yy, "o", ms=5.6, color=c, mec="white", mew=0.8) | |
| ax.text(r["ci"][1] + 1.5, yy, f"{r['ds']:.1f}", va="center", fontsize=7.4, color=INK, | |
| bbox=dict(fc="white", ec="none", pad=0.4), zorder=5) | |
| kn.setdefault(b, {})[k] = [round(r["ds"], 1), [round(v, 1) for v in r["ci"]]] | |
| ax.axvline(0, color=G1, lw=0.8, ls=(0, (2, 2))) | |
| lo = min(min(e["ci"][0] for e in board if e["board"] == b), min(JE["ours"][k]["calibrated"][b]["ci"][0] for k, _, _ in ours)) | |
| hi = max(max(e["ci"][1] for e in board if e["board"] == b), max(JE["ours"][k]["calibrated"][b]["ci"][1] for k, _, _ in ours)) | |
| pad = (hi - lo) * 0.06 | |
| ax.set_xlim(min(lo - pad, -2 if b == "helpsteer2" else lo - pad), hi + pad + (hi - lo) * 0.12) | |
| ax.set_title(bt, loc="left", fontsize=8.8, color=INK) | |
| ax.axhline(len(ours) + 0.4, color=GRID, lw=0.8) | |
| style(ax, "x"); ax.tick_params(axis="y", length=0) | |
| ax.set_xlabel("Decision Score", fontsize=8) | |
| axes[2].text(0, -0.55, "label prior", fontsize=6.8, color=G2, va="center", ha="center", | |
| bbox=dict(fc="white", ec="none", pad=0.5), zorder=5) | |
| ticks = [ys[n] for n in names] + [ys[k] for k, _, _ in ours] | |
| axes[0].set_yticks(ticks, names + [lab for _, lab, _ in ours], fontsize=7.8) | |
| for tl in axes[0].get_yticklabels()[len(names):]: | |
| tl.set_fontweight("bold") | |
| axes[0].set_ylim(-0.9, nrow + 1.3) | |
| suptitle(fig, "Jevals: the fast readout sits mid-pack; thinking helps only on PubMedQA") | |
| save(fig, "jevals", "Jevals Decision Scores", | |
| "Decision Score (Brier / RPS skill vs the label prior, x100) with 95% CIs on the three Jevals 0.1.0 boards (300 items each): " | |
| "published board entries (grey, ordered by mean score) and ours, scored with our reimplementation of the Jevals rules " | |
| "and per-type temperatures from our calibration items. Source: results/jevals.json.", kn) | |
| # ====================================================================================================== 9 S1MB field | |
| def s1mb_field(): | |
| L = S1["leaderboard"] | |
| pub = [e for e in L if not e["model"].startswith("OURS:")] | |
| ours_keys = ["base_2b", "base", "v2_2b_ro", "v1_joint", "rl60"] | |
| ours = {e["model"][5:]: e for e in L if e["model"].startswith("OURS:") and e["model"][5:] in ours_keys} | |
| assert len(pub) == 211 and len(ours) == 5 | |
| for k in ours_keys: | |
| check(f"s1 task {k}", ours[k]["task_avg"], REPORT["s1_task"][k]); check(f"s1 noul {k}", ours[k]["noul"], REPORT["s1_noul"][k]) | |
| check(f"s1 choice {k}", ours[k]["choice"], REPORT["s1_choice"][k]); check(f"s1 score {k}", ours[k]["score"], REPORT["s1_score"][k]) | |
| i = next(j for j, e in enumerate(L) if e["model"] == f"OURS:{k}") | |
| check(f"s1 above {k}", sum(1 for e in L[:i] if not e["model"].startswith("OURS:")), REPORT["s1_above"][k]) | |
| best4 = next(e for e in pub if e["model"] == "PelaAI__KnowLine-4B-Gen5") | |
| for v, r in zip((best4["task_avg"], best4["noul"], best4["choice"], best4["score"]), REPORT["s1_best4b"]): | |
| check("s1 best4b", v, r) | |
| med = {k: float(np.median([e[k] for e in pub])) for k in ("task_avg", "noul", "choice", "score")} | |
| check("s1 median", med["task_avg"], REPORT["s1_median"]) | |
| check("s1 best", max(e["task_avg"] for e in pub), REPORT["s1_best"]) | |
| fig = plt.figure(figsize=(6.5, 3.4), layout="constrained") | |
| gs = fig.add_gridspec(1, 2, width_ratios=[1.35, 1]) | |
| ax = fig.add_subplot(gs[0, 0]) | |
| allr = sorted(pub + list(ours.values()), key=lambda e: -e["task_avg"]) | |
| rank = {e["model"]: i + 1 for i, e in enumerate(allr)} | |
| ax.scatter([rank[e["model"]] for e in pub], [e["task_avg"] for e in pub], s=9, color="#c4c9d1", edgecolors="none", zorder=2) | |
| ax.axhline(med["task_avg"], color=G2, lw=0.8, ls=(0, (3, 2)), zorder=1) | |
| ax.text(216, med["task_avg"] + 0.8, f"published median {med['task_avg']:.1f}", fontsize=6.9, color=INK2, ha="right", va="bottom") | |
| lab = {"rl60": ("V2 + RL", BLUE), "v1_joint": ("v1", G2), "v2_2b_ro": ("2B readout-only", G3), | |
| "base": ("Qwen3.5-4B", G1), "base_2b": ("MiniCPM5-2B", G1)} | |
| for k, (nm, c) in lab.items(): | |
| e = ours[k] | |
| r, v = rank[e["model"]], e["task_avg"] | |
| ax.scatter([r], [v], s=46 if k == "rl60" else 34, color=c, edgecolors="white", linewidths=0.9, zorder=4) | |
| ax.text(r - 4, v - 1.6, f"{nm} {v:.1f}", ha="right", va="top", fontsize=7.2, color=INK, weight="bold" if k == "rl60" else "normal", | |
| bbox=dict(fc="white", ec="none", pad=0.3), zorder=5) | |
| r4 = rank[best4["model"]] | |
| ax.scatter([r4], [best4["task_avg"]], s=34, facecolors="white", edgecolors=G3, linewidths=1.2, zorder=4) | |
| ax.text(r4 + 6, best4["task_avg"] + 0.6, f"best published 4B {best4['task_avg']:.1f}", fontsize=6.9, color=INK2, va="bottom") | |
| top = allr[0] | |
| ax.text(rank[top["model"]] + 6, top["task_avg"] + 0.6, f"best published {top['task_avg']:.1f}", fontsize=6.9, color=INK2, va="bottom") | |
| ax.set_ylim(0, 82) | |
| ax.set_xlim(-4, 222) | |
| ax.set_xticks([1, 50, 100, 150, 200]) | |
| ax.set_xlabel("Rank by Task Avg (211 published models + ours)", fontsize=8) | |
| ax.set_ylabel("S1MB Task Avg (adjusted skill x100)", fontsize=8) | |
| ax.set_title("The published field (grey) and our five models", loc="left", fontsize=8.8, color=INK) | |
| style(ax, "y") | |
| # per-task bars | |
| bx = fig.add_subplot(gs[0, 1]) | |
| tasks = [("noul", "Yes/no"), ("choice", "Choice"), ("score", "Score")] | |
| series = [("base", ours["base"], G1, "Qwen3.5-4B"), ("v1", ours["v1_joint"], G2, "v1"), ("rl60", ours["rl60"], BLUE, "V2 + RL"), | |
| ("best4", best4, "white", "Best 4B")] | |
| w = 0.2 | |
| for ti, (tk, tl) in enumerate(tasks): | |
| for si, (k, e, c, nm) in enumerate(series): | |
| x = ti + (si - 1.5) * w | |
| bx.bar(x, e[tk], width=w * 0.9, color=c, edgecolor=G3 if k == "best4" else "white", lw=0.8 if k == "best4" else 0.6) | |
| if k in ("rl60", "best4"): | |
| bx.text(x, e[tk] + 1.2, f"{e[tk]:.0f}", ha="center", va="bottom", fontsize=6.6, color=INK) | |
| if ti == 0: | |
| bx.text(x, 1.5, nm, rotation=90, ha="center", va="bottom", fontsize=6.4, | |
| color="white" if k in ("rl60", "v1") else INK) | |
| bx.plot([ti - 2.1 * w, ti + 2.1 * w], [med[tk]] * 2, color=INK, lw=1.0, ls=(0, (2, 1.5)), zorder=4) | |
| bx.text(2 + 2.15 * w, med["score"], "median", fontsize=6.6, color=INK2, va="center", ha="left") | |
| bx.set_xticks(range(3), [t for _, t in tasks], fontsize=8) | |
| bx.set_xlim(-0.5, 2.75) | |
| bx.set_ylim(0, 85) | |
| bx.set_ylabel("Adjusted skill x100", fontsize=8) | |
| bx.set_title("Per task: 4B models", loc="left", fontsize=8.8, color=INK) | |
| style(bx, "y"); bx.tick_params(axis="x", length=0) | |
| suptitle(fig, "S1MB: training lifted the 4B from 32.8 to 47.9, above the median published model") | |
| save(fig, "s1mb_field", "S1MB against the published field", | |
| "Left: S1MB English v1 Task Avg (137 benchmarks) of the 211 complete published models (grey dots, ranked by Task Avg) " | |
| "and our five models (fast readout); 74 published models rank above V2 + RL in the leaderboard's Borda order. Right: per-task " | |
| "scores of our 4B models vs the best published 4B (KnowLine-4B-Gen5, outlined) and the published median (dashed). " | |
| "Source: results/s1mb.json (our rows exclude the routed runs).", | |
| {"ours_task_avg": {k: round(ours[k]["task_avg"], 1) for k in ours_keys}, | |
| "ours_per_task": {k: [round(ours[k][t], 1) for t in ("noul", "choice", "score")] for k in ours_keys}, | |
| "published_above_borda": {k: REPORT["s1_above"][k] for k in ours_keys}, "median": {k: round(v, 1) for k, v in med.items()}, | |
| "best_4b": [round(best4[t], 1) for t in ("task_avg", "noul", "choice", "score")]}) | |
| # ====================================================================================================== 10 S1MB seen / unseen | |
| def s1mb_seen_unseen(): | |
| S = FS["s1mb"] | |
| for nm, ref in (("seen_mean_base", 44.5), ("seen_mean_rl60", 52.8), ("unseen_mean_base", 33.0), ("unseen_mean_rl60", 50.4)): | |
| check(nm, S[nm]["point"], ref) | |
| cnt = S["split_counts"] | |
| nse, nun = sum(cnt["seen"].values()), sum(cnt["unseen"].values()) | |
| assert (nse, nun) == (41, 96) | |
| fig = plt.figure(figsize=(6.5, 3.2), layout="constrained") | |
| gs = fig.add_gridspec(1, 2, width_ratios=[0.85, 1.15]) | |
| ax = fig.add_subplot(gs[0, 0]) | |
| for gi, sp in enumerate(["seen", "unseen"]): | |
| for si, (m, c) in enumerate((("base", G1), ("rl60", BLUE))): | |
| r = S[f"{sp}_mean_{m}"] | |
| x = gi + (si - 0.5) * 0.36 | |
| ax.bar(x, r["point"], width=0.33, color=c, edgecolor="white") | |
| ax.plot([x, x], r["ci95"], color=G3, lw=0.9) | |
| ax.text(x, r["ci95"][1] + 1.2, f"{r['point']:.1f}", ha="center", va="bottom", fontsize=7.8, color=INK, | |
| weight="bold" if m == "rl60" else "normal") | |
| if gi == 0: | |
| ax.text(x, 1.5, "Qwen3.5-4B" if m == "base" else "V2 + RL", rotation=90, ha="center", va="bottom", fontsize=7, | |
| color=INK if m == "base" else "white") | |
| ax.set_xticks([0, 1], [f"Seen source\n({nse})", f"Unseen source\n({nun})"], fontsize=8) | |
| ax.set_ylim(0, 70) | |
| ax.set_ylabel("Mean adjusted skill x100", fontsize=8) | |
| ax.set_title("Benchmark means, 95% CI", loc="left", fontsize=8.8, color=INK) | |
| style(ax, "y"); ax.tick_params(axis="x", length=0) | |
| bx = fig.add_subplot(gs[0, 1]) | |
| rows = [("seen_gain", "Gain, seen source", BLUE), ("unseen_gain", "Gain, unseen source", BLUE), | |
| ("unseen_minus_seen_gain", "Unseen − seen gain, all", G3), | |
| ("unseen_minus_seen_gain_task_matched", "Unseen − seen, task-matched", G3), | |
| ("unseen_minus_seen_gain_noul_only", f" yes/no only ({cnt['seen']['noul']} vs {cnt['unseen']['noul']})", G3), | |
| ("unseen_minus_seen_gain_choice_only", f" choice only ({cnt['seen']['choice']} vs {cnt['unseen']['choice']})", G3)] | |
| yy = np.array([5.4, 4.4, 3.0, 2.0, 1.0, 0.0]) | |
| kn = {} | |
| for yi, (k, lab, c) in zip(yy, rows): | |
| r = S[k] | |
| bx.plot(r["ci95"], [yi, yi], color=c, lw=1.6, alpha=0.55, solid_capstyle="butt") | |
| filled = r["excludes_0"] | |
| bx.plot(r["point"], yi, "o", ms=6, color=c if filled else "white", mec=c, mew=1.3) | |
| bx.text(27, yi, f"{r['point']:+.1f} [{r['ci95'][0]:+.1f}, {r['ci95'][1]:+.1f}]".replace("-", "−"), va="center", fontsize=7.6, color=INK) | |
| kn[k] = [round(r["point"], 1), [round(v, 1) for v in r["ci95"]]] | |
| check("seen gain", S["seen_gain"]["point"], 8.4); check("unseen gain", S["unseen_gain"]["point"], 17.4) | |
| check("unseen-seen", S["unseen_minus_seen_gain"]["point"], 9.0) | |
| check("task matched", S["unseen_minus_seen_gain_task_matched"]["point"], 8.2) | |
| bx.axvline(0, color=G2, lw=0.8) | |
| bx.axhline(3.7, color=GRID, lw=0.8) | |
| bx.set_yticks(yy, [r[1] for r in rows], fontsize=7.8) | |
| bx.tick_params(axis="y", length=0) | |
| bx.set_xlim(-3, 47) | |
| bx.set_xticks([0, 10, 20]) | |
| bx.set_xlabel("Base → V2 + RL gain, points (95% CI)", fontsize=8) | |
| bx.set_title("Gains and their difference (open dot: CI includes 0)", loc="left", fontsize=8.8, color=INK) | |
| style(bx, "x") | |
| suptitle(fig, "S1MB gains were larger on benchmarks from datasets we never trained on") | |
| save(fig, "s1mb_seen_unseen", "S1MB gains by seen vs unseen source", | |
| "Base Qwen3.5-4B -> V2 + RL on S1MB, split by whether the benchmark's source dataset contributed train-split items to our " | |
| f"training pool (seen: {nse}; unseen: {nun}). Seen holds 1 score benchmark against 20 unseen, so the task-matched contrast " | |
| "(mean of the yes/no and choice contrasts) is the composition-fair one. 10,000 benchmark-level bootstrap resamples; " | |
| "source: results/final_stats.json.", kn) | |
| # ====================================================================================================== 11 forest | |
| def forest(): | |
| F, TS, IN, OOD = FS["fast_readout"], FS["thinking_subset"], FS["in_distribution"], FS["out_of_distribution"] | |
| groups = [ | |
| ("Fast readout, 5,442 questions", [ | |
| (F["acc_rl60-base"], "V2 + RL − base", True), (F["acc_rl60-v1"], "V2 + RL − v1", True), | |
| (F["acc_v2-v1"], "V2 − v1", False), (F["acc_rl60-v2"], "V2 + RL − V2", True), | |
| (F["acc_rl40-rl60"], "RL step 40 − step 60", False)]), | |
| ("Thinking, 1,498 questions", [ | |
| (TS["think_rl60-v2"], "V2 + RL − V2", True), (TS["think_rl60-base"], "V2 + RL − base", True), | |
| (TS["think_v2-base"], "V2 − base", False), (TS["think_rl40-rl60"], "RL step 40 − step 60", False), | |
| (TS["loop_rl60-v2"], "looped chains, V2 + RL − V2", True)]), | |
| ("Routed 30%, 1,498 questions", [ | |
| (TS["routed30_rl60-base"], "V2 + RL − base", True), (TS["routed30_rl60-v1"], "V2 + RL − v1", True), | |
| (TS["routed30_rl60-v2"], "V2 + RL − V2", True), (TS["routed30_rl40-rl60"], "RL step 40 − step 60", False)]), | |
| ("Votes mixed in − gold only (V2 ablations)", [ | |
| (F["acc_abl_mix-abl_gold"], "all 5,442", False), (IN["acc_abl_mix-abl_gold"], "in-distribution (1,125)", False), | |
| (OOD["acc_abl_mix-abl_gold"], "out-of-distribution (4,317)", False), | |
| (IN["in_minus_ood_acc_abl_mix-abl_gold"], "in-distribution − OOD effect", False)]), | |
| ] | |
| refs = {("Fast readout, 5,442 questions", 0): 9.79, ("Fast readout, 5,442 questions", 1): 3.00, | |
| ("Fast readout, 5,442 questions", 2): 3.23, ("Fast readout, 5,442 questions", 3): -0.24, | |
| ("Fast readout, 5,442 questions", 4): 0.29, ("Thinking, 1,498 questions", 0): 1.9, ("Thinking, 1,498 questions", 1): -0.9, | |
| ("Thinking, 1,498 questions", 2): -2.9, ("Thinking, 1,498 questions", 3): 0.5, ("Thinking, 1,498 questions", 4): -4.9, | |
| ("Routed 30%, 1,498 questions", 0): 8.8, ("Routed 30%, 1,498 questions", 1): 3.9, ("Routed 30%, 1,498 questions", 2): 0.1, | |
| ("Routed 30%, 1,498 questions", 3): 0.2, | |
| ("Votes mixed in − gold only (V2 ablations)", 0): -1.19, ("Votes mixed in − gold only (V2 ablations)", 1): 1.51, | |
| ("Votes mixed in − gold only (V2 ablations)", 2): -1.90, ("Votes mixed in − gold only (V2 ablations)", 3): 3.41} | |
| fig, ax = plt.subplots(figsize=(6.5, 5.0), layout="constrained") | |
| y, ticks, labels, kn = 0.0, [], [], {} | |
| heads = [] | |
| for gname, rows in groups: | |
| heads.append((y, gname)) | |
| y -= 1.0 | |
| for i, (r, lab, feat) in enumerate(rows): | |
| nd = 2 if gname.startswith("Fast") or gname.startswith("Votes") else 1 | |
| check(f"forest {gname} {lab}", r["point"], refs[(gname, i)], nd) | |
| c = BLUE if feat else G3 | |
| ax.plot(r["ci95"], [y, y], color=c, lw=1.7, alpha=0.55, solid_capstyle="butt") | |
| ax.plot(r["point"], y, "o", ms=5.6, color=c if r["excludes_0"] else "white", mec=c, mew=1.3, zorder=3) | |
| ax.text(13.3, y, f"{r['point']:+.{nd}f} [{r['ci95'][0]:+.{nd}f}, {r['ci95'][1]:+.{nd}f}]".replace("-", "−"), va="center", | |
| fontsize=7.6, color=INK) | |
| ticks.append(y); labels.append(lab) | |
| kn[f"{gname}: {lab}"] = [round(r["point"], 2), [round(v, 2) for v in r["ci95"]]] | |
| y -= 1.0 | |
| y -= 0.5 | |
| for yh, g in heads: | |
| ax.text(-7.5, yh, g, fontsize=8.4, color=INK, weight="bold", va="center", ha="left", transform=ax.transData, | |
| bbox=dict(fc="white", ec="none", pad=0.6), zorder=5) | |
| ax.axvline(0, color=G2, lw=0.8) | |
| ax.set_yticks(ticks, labels, fontsize=7.9) | |
| ax.tick_params(axis="y", length=0) | |
| ax.set_xlim(-7.6, 19.5) | |
| ax.set_xticks([-6, -4, -2, 0, 2, 4, 6, 8, 10, 12]) | |
| ax.set_ylim(y + 0.9, 0.7) | |
| ax.set_xlabel("Difference, percentage points (paired bootstrap, 95% CI)", fontsize=8.5) | |
| style(ax, "x") | |
| ax.text(13.3, 0.0, "estimate [95% CI]", fontsize=7.6, color=INK2, va="center") | |
| ax.legend(handles=[Line2D([], [], marker="o", ls="", color=G3, ms=5.5), Line2D([], [], marker="o", ls="", mfc="white", mec=G3, mew=1.3, ms=5.5)], | |
| labels=["CI excludes 0", "CI includes 0"], loc="center left", bbox_to_anchor=(0.46, 0.555), frameon=False, fontsize=7.4, | |
| handletextpad=0.3) | |
| suptitle(fig, "Which differences hold up: RL changed thinking and its cost, not the fast readout") | |
| save(fig, "forest", "Paired-bootstrap differences", | |
| "Differences in accuracy (points) with 95% percentile intervals from 10,000 paired bootstrap resamples of questions; every " | |
| "model scored on the same resample, temperatures fixed. Blue rows involve V2 + RL. Single training seed: the intervals " | |
| "cover question sampling only. Source: results/final_stats.json.", kn) | |
| # ====================================================================================================== 12 yes / no | |
| def yesno(): | |
| s1_gold = D["s1mb_target_yes_mass"] | |
| assert D["s1mb_noul_judgments"] == 12357 | |
| check("s1 target yes", 100 * s1_gold, 27.5) | |
| check("s1 yes base", 100 * YN["base"]["s1mb"]["calibrated"]["s1mb_yes_rate"], 18.4) | |
| check("s1 yes rl60", 100 * YN["rl60"]["s1mb"]["calibrated"]["s1mb_yes_rate"], 31.6) | |
| check("s1 noul rl60", YN["rl60"]["s1mb"]["calibrated"]["noul"], 50.7) | |
| models = [("base", "Qwen3.5-4B", G1), ("v1_joint", "v1", G2), ("rl60", "V2 + RL", BLUE)] | |
| fig, axes = plt.subplots(1, 2, figsize=(6.5, 3.2), sharey=True, layout="constrained") | |
| yy = {"base": 2, "v1_joint": 1, "rl60": 0} | |
| gold_eval = YN["rl60"]["eval_gold_yes"] | |
| kn = {} | |
| for ax, which in zip(axes, ("eval", "s1mb")): | |
| for k, nm, c in models: | |
| r = YN[k] | |
| if which == "eval": | |
| a, b = r["eval_yes_rate"], r["eval_yes_rate_offset"] | |
| note = f"bal. acc. {r['eval_ba']:.3f} → {r['eval_ba_offset']:.3f}" | |
| else: | |
| a, b = r["s1mb"]["calibrated"]["s1mb_yes_rate"], r["s1mb"]["calibrated+offset"]["s1mb_yes_rate"] | |
| note = f"yes/no skill {r['s1mb']['calibrated']['noul']:.1f} → {r['s1mb']['calibrated+offset']['noul']:.1f}" | |
| a, b = 100 * a, 100 * b | |
| ax.annotate("", xy=(b, yy[k]), xytext=(a, yy[k]), | |
| arrowprops=dict(arrowstyle="-|>", color=c, lw=1.2, mutation_scale=9, shrinkA=4, shrinkB=4)) | |
| ax.plot(a, yy[k], "o", ms=7, color=c, mec="white", mew=0.8, zorder=3) | |
| ax.plot(b, yy[k], "o", ms=6.5, color="white", mec=c, mew=1.4, zorder=3) | |
| ax.text((a + b) / 2, yy[k] - 0.3, note, ha="center", va="top", fontsize=7.1, color=INK2, | |
| bbox=dict(fc="white", ec="none", pad=0.3), zorder=5) | |
| kn.setdefault(which, {})[k] = {"yes_rate": round(a, 1), "yes_rate_offset": round(b, 1), "note": note} | |
| g = 100 * (gold_eval if which == "eval" else s1_gold) | |
| ax.axvline(g, color=INK, lw=1.0, ls=(0, (3, 2))) | |
| ax.text(g + 0.4, 2.62, f"gold {g:.1f}%" if which == "eval" else f"target {g:.1f}%", ha="left", va="bottom", fontsize=7.4, color=INK, | |
| bbox=dict(fc="white", ec="none", pad=0.5), zorder=5) | |
| style(ax, "x"); ax.tick_params(axis="y", length=0) | |
| ax.set_xlabel("Share of yes/no questions answered yes (%)", fontsize=8) | |
| kn[which]["gold_yes"] = round(g, 1) | |
| axes[0].set_xlim(34, 70); axes[1].set_xlim(13, 37) | |
| axes[0].set_title(f"Our eval: {YN['rl60']['eval_noul_n']:,} yes/no questions", loc="left", fontsize=8.8, color=INK) | |
| axes[1].set_title(f"S1MB: {D['s1mb_noul_judgments']:,} yes/no judgments", loc="left", fontsize=8.8, color=INK) | |
| axes[0].set_yticks([2, 1, 0], [m[1] for m in models], fontsize=8.5) | |
| axes[0].set_ylim(-0.75, 3.0) | |
| axes[0].legend(handles=[Line2D([], [], marker="o", ls="", color=G3, ms=6), | |
| Line2D([], [], marker="o", ls="", mfc="white", mec=G3, mew=1.3, ms=6)], | |
| labels=["temperature only", "+ yes/no offset"], loc="upper left", | |
| bbox_to_anchor=(0.0, 1.0), frameon=False, fontsize=6.9, handletextpad=0.2) | |
| suptitle(fig, "Training turned a 'no' bias into a 'yes' bias; a fitted offset helps our eval but not S1MB") | |
| save(fig, "yesno", "Yes-rate vs gold yes-rate", | |
| "Share of yes/no items the fast readout answers yes, against the gold yes share (our eval) or the target yes mass " | |
| "(S1MB). Filled: per-type temperatures from our calibration items; open: plus one yes/no logit offset fitted on the " | |
| "109 yes/no calibration items (never eval or S1MB labels). Notes under each arrow: balanced accuracy on our eval, " | |
| "S1MB yes/no adjusted skill (x100), without -> with the offset. Source: results/yesno_offset.json; S1MB target from data/s1mb/items.jsonl.", kn) | |
| FIGS = {"headline": headline, "routing_curve": routing_curve, "per_set": per_set, "calibration": calibration, | |
| "thinking_tokens": thinking_tokens, "fix_break": fix_break, "twob": twob, "jevals": jevals, "s1mb_field": s1mb_field, | |
| "s1mb_seen_unseen": s1mb_seen_unseen, "forest": forest, "yesno": yesno} | |
| if __name__ == "__main__": | |
| want = sys.argv[1:] or list(FIGS) | |
| for s in want: | |
| FIGS[s]() | |
| mp = f"{OUT}/manifest_eval.json" | |
| old = json.load(open(mp)) if os.path.exists(mp) and sys.argv[1:] else [] | |
| keep = [m for m in old if m["slug"] not in {x["slug"] for x in manifest}] | |
| order = list(FIGS) | |
| allm = sorted(keep + manifest, key=lambda m: order.index(m["slug"])) | |
| json.dump(allm, open(mp, "w"), indent=1) | |
| print(f"{len(manifest)} figures drawn; {len(_checked)} numbers checked against FINAL_REPORT.md, all match") | |
Xet Storage Details
- Size:
- 56.4 kB
- Xet hash:
- 4353478b3a618b49be048105a2bafdfda8f22a2b566987800fb4f4d0490fee4c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.