File size: 7,606 Bytes
8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 231820f 8c85090 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | """Every figure for the Sudoku result, generated from the raw run files.
Nothing here is hand-entered: each panel reads result.json from the runs it
plots, so a figure cannot drift from the numbers it claims to show.
"""
import glob, json, os
from collections import defaultdict
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
RUNS = os.environ.get("RUNS", "runs")
OUT = os.environ.get("OUT", "figures")
os.makedirs(OUT, exist_ok=True)
# classical baselines, measured on this exact test set by sudoku_compare.py
CLASSICAL = {"constraint propagation": 0.0300, "greedy MRV": 0.0590}
def cfg_label(a):
"""Identify a configuration.
The training budget is part of the identity. An earlier version of this
function left it out, so a 32-layer run at batch 256 (94.50%) and one at
batch 64 (83.40%) were averaged into a single "88.95%" that belonged to
neither. That artefact is what produced the claim that depth was worth
nothing. Runs at different budgets are different configurations and are
never pooled.
"""
if a.get("objective") == "ar":
base = "autoregressive prefix-LM"
elif a.get("objective") == "direct":
base = "direct recurrent classifier"
elif not a.get("recurrent"):
base = f"feed-forward {a['layers']}L ({a['mode']})"
else:
tags = []
if a.get("no_inject"):
tags.append("no-inject")
if a.get("no_deep_sup"):
tags.append("no-deepsup")
base = "recurrent " + a["mode"] + (" [" + ",".join(tags) + "]" if tags else "")
return f"{base} @bs{a.get('bs')}"
runs = defaultdict(list)
for f in sorted(glob.glob(f"{RUNS}/*/result.json")):
n = os.path.basename(os.path.dirname(f))
if not n.startswith(("rec1-", "abl-", "base-", "fix-", "inj-", "sudh-")):
continue
try:
r = json.load(open(f))
except Exception:
continue
if r.get("decode"):
runs[cfg_label(r["args"])].append(r)
# ---- Figure 1: board accuracy against decoding passes -----------------------
fig, ax = plt.subplots(figsize=(7.2, 4.6))
GRID = [1, 2, 4, 8, 16, 32, 61]
style = {"recurrent full @bs256": ("#1a5276", "o", 2.4),
"recurrent mdlm @bs256": ("#2980b9", "s", 1.6),
"feed-forward 12L (full) @bs256": ("#c0392b", "^", 1.6),
"feed-forward 12L (mdlm) @bs256": ("#e67e22", "v", 1.4),
"feed-forward 32L (full) @bs256": ("#8e44ad", "D", 1.4),
"recurrent full [no-inject] @bs64": ("#16a085", "P", 1.4),
"feed-forward 32L (full) @bs64": ("#7f8c8d", "X", 1.2)}
for label, rs in sorted(runs.items()):
if label not in style:
continue
col, mk, lw = style[label]
xs, ys = [], []
for k in GRID:
v = [r["decode"][f"fixedK_{k}"]["board"] for r in rs
if f"fixedK_{k}" in r["decode"]]
if v:
xs.append(k); ys.append(np.mean(v) * 100)
if xs:
ax.plot(xs, ys, marker=mk, color=col, lw=lw, ms=5, label=label)
for name, v in CLASSICAL.items():
ax.axhline(v * 100, ls=":", lw=1.2, color="#555")
ax.text(1.05, v * 100 + 1.5, name, fontsize=7, color="#555")
ax.set_xscale("log", base=2)
ax.set_xlabel("denoising passes (NFE)")
ax.set_ylabel("board accuracy (%) — all 81 cells correct")
ax.set_title("Hard Sudoku: recurrence beats capacity\n(batch size is part of each label - runs at different budgets are not pooled)",
fontsize=10)
ax.grid(alpha=.3); ax.legend(fontsize=7.5, loc="lower right"); ax.set_ylim(-3, 103)
fig.tight_layout(); fig.savefig(f"{OUT}/sud_fig1_passes.png", dpi=170)
# ---- Figure 2: parameters against accuracy ---------------------------------
fig, ax = plt.subplots(figsize=(6.6, 4.4))
pts = []
for label, rs in runs.items():
a = rs[0]["args"]
npar = 212e3 if a.get("recurrent") else (
6.34e6 if a.get("layers") == 32 else (2.4e6 if a.get("objective") == "ar"
else 37.9e6))
best = np.mean([max(v["board"] for v in r["decode"].values()) for r in rs])
pts.append((npar, best * 100, label))
for npar, acc, label in pts:
hero = label == "recurrent full @bs256"
ax.scatter(npar, acc, s=170 if hero else 70,
color="#1a5276" if hero else "#888",
marker="*" if hero else "o", zorder=3 if hero else 2)
ax.annotate(label, (npar, acc), fontsize=6.6,
xytext=(6, 5 if acc < 95 else -11), textcoords="offset points")
ax.axhline(99.5, ls="--", color="#117a65", lw=1.2)
ax.text(2.3e5, 99.5 + 0.9, "Recurrent Transformer 99.5% @211k params "
"(different clue distribution)", fontsize=6.6, color="#117a65")
ax.set_xscale("log")
ax.set_xlabel("parameters"); ax.set_ylabel("board accuracy (%)")
ax.set_title("At batch 256: fewer parameters, better accuracy, monotonically\n"
"37.9M → 88.90% | 6.34M → 94.50% | 212k → 99.70%", fontsize=10)
ax.grid(alpha=.3); ax.set_ylim(-5, 108)
fig.tight_layout(); fig.savefig(f"{OUT}/sud_fig2_params.png", dpi=170)
# ---- Figure 3: recurrence extrapolates past its training depth --------------
sw = os.path.join(OUT, "sudoku_rsweep.json")
if os.path.exists(sw):
data = json.load(open(sw))
fig, ax = plt.subplots(figsize=(6.6, 4.2))
for name, d in data.items():
Rs = sorted(int(k) for k in d)
ys = [d[str(R)] * 100 for R in Rs]
ax.plot(Rs, ys, marker="o", ms=4,
color="#1a5276" if "full" in name else "#c0392b",
alpha=.85, label=name)
ax.axvline(32, ls="--", color="k", lw=1)
ax.text(33, 12, "trained at R = 32", fontsize=7.5, rotation=90)
ax.set_xscale("log", base=2)
ax.set_xlabel("recurrences at inference"); ax.set_ylabel("board accuracy (%)")
ax.set_title("Deep supervision makes the recurrence iterable past training depth")
ax.grid(alpha=.3); ax.legend(fontsize=7); ax.set_ylim(-3, 103)
fig.tight_layout(); fig.savefig(f"{OUT}/sud_fig3_rsweep.png", dpi=170)
# ---- Figure 4: ablation ladder ---------------------------------------------
order = ["feed-forward 12L (mdlm) @bs256", "feed-forward 12L (full) @bs256",
"feed-forward 32L (full) @bs256", "recurrent mdlm @bs256",
"recurrent full [no-deepsup] @bs256", "recurrent full @bs256",
"feed-forward 32L (full) @bs64", "recurrent full [no-inject] @bs64"]
have = [(o, np.mean([max(v["board"] for v in r["decode"].values())
for r in runs[o]]) * 100) for o in order if o in runs]
if have:
fig, ax = plt.subplots(figsize=(7.4, 4.0))
names = [h[0] for h in have]; vals = [h[1] for h in have]
cols = ["#1a5276" if n == "recurrent full @bs256"
else ("#d5dbdb" if "@bs64" in n else "#9fb6c8") for n in names]
ax.barh(range(len(vals)), vals, color=cols)
for i, v in enumerate(vals):
ax.text(v + 0.8, i, f"{v:.2f}%", va="center", fontsize=8)
ax.set_yticks(range(len(names))); ax.set_yticklabels(names, fontsize=8)
ax.set_xlabel("board accuracy (%)"); ax.set_xlim(0, 108)
ax.set_title("Ablation at matched budget\n"
"pale = batch 64, a separate budget: compare within a batch size",
fontsize=10)
ax.grid(alpha=.3, axis="x")
fig.tight_layout(); fig.savefig(f"{OUT}/sud_fig4_ablation.png", dpi=170)
print("configurations plotted:")
for k in sorted(runs):
v = [max(x["board"] for x in r["decode"].values()) for r in runs[k]]
print(f" {k:<42} n={len(v)} {np.mean(v)*100:6.2f}% {[round(x*100,1) for x in v]}")
print(f"\nwrote figures to {OUT}")
|