File size: 4,190 Bytes
1c9c214 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Sudoku results: the decoding comparison, and the benchmark comparison.
Two separate questions, kept separate because they have different standards of
evidence.
1. THE INTERNAL CLAIM. Does committing by certainty beat committing a fixed
share per pass? This is a controlled comparison - same trained model, same
test split, only the decoder changed - so it is clean regardless of how the
absolute numbers land, and it is the claim the framework actually makes.
2. THE BENCHMARK COMPARISON. SATNet (Wang et al., ICML 2019) reports 98.3%
board-level accuracy on this split. Our number is comparable only for the
un-augmented arm, since SATNet did not use digit-relabelling augmentation.
The augmented arm is reported separately and must not be quoted against them.
Board accuracy means all 81 cells correct. Cell accuracy over blanks and the
fraction of outputs that are valid Sudoku are reported alongside, because a
solver can score well per-cell while almost never completing a board.
"""
import glob, json, os, sys
from collections import defaultdict
import numpy as np
RUNS = os.environ.get("RUNS", "runs")
SATNET = 0.983
runs = defaultdict(list)
for f in sorted(glob.glob(f"{RUNS}/sud1-*/result.json")):
r = json.load(open(f))
a = r["args"]
runs[(a["mode"], "aug" if a.get("augment") else "plain")].append(r)
if not runs:
sys.exit("no sudoku results yet")
def rows(rs, prefix):
out = {}
for r in rs:
for k, v in r["decode"].items():
if k.startswith(prefix):
out.setdefault(k, []).append(v)
return out
print("=" * 88)
print("SUDOKU (SATNet benchmark, 1000-puzzle test split) — BOARD accuracy = all 81 cells")
print("=" * 88)
best_overall = {}
for key in sorted(runs):
rs = runs[key]
print(f"\n--- {key[0]} / {key[1]} ({len(rs)} seeds)")
print(f" {'rule':<24}{'NFE':>7}{'BOARD':>10}{'cell':>9}{'valid':>9}")
for prefix, label in (("fixedK_", "fixed-K"),
("entbudget_", "entropy-budget"),
("conf_", "confidence")):
got = rows(rs, prefix)
for k in sorted(got, key=lambda s: float(s.split("_")[1])):
v = got[k]
b = np.mean([x["board"] for x in v])
print(f" {k:<24}{np.mean([x['nfe'] for x in v]):>7.1f}"
f"{b*100:>9.2f}%{np.mean([x['cell'] for x in v])*100:>8.2f}%"
f"{np.mean([x['valid_sudoku'] for x in v])*100:>8.2f}%")
fam = best_overall.setdefault((key, label), {"board": -1})
if b > fam["board"]:
best_overall[(key, label)] = {
"board": b, "rule": k,
"nfe": float(np.mean([x["nfe"] for x in v]))}
print()
print("=" * 88)
print("1. THE CONTROLLED CLAIM — same model, same data, only the decoder changes")
print("=" * 88)
print(f"{'arm':<22}{'fixed-K best':>16}{'entropy-budget':>17}{'gap':>10}{'conf best':>12}")
for key in sorted(runs):
fk = best_overall.get((key, "fixed-K"))
eb = best_overall.get((key, "entropy-budget"))
cf = best_overall.get((key, "confidence"))
if not (fk and eb):
continue
print(f"{key[0] + '/' + key[1]:<22}{fk['board']*100:>15.2f}%{eb['board']*100:>16.2f}%"
f"{(eb['board']-fk['board'])*100:>+9.2f}"
f"{(cf['board']*100 if cf else float('nan')):>11.2f}%")
print()
print("=" * 88)
print("2. THE BENCHMARK COMPARISON — like-for-like is the UN-AUGMENTED arm only")
print("=" * 88)
print(f" SATNet (Wang et al., ICML 2019): {SATNET*100:.1f}% board accuracy\n")
for key in sorted(runs):
eb = best_overall.get((key, "entropy-budget"))
if not eb:
continue
tag = "comparable" if key[1] == "plain" else "NOT comparable (augmented)"
delta = (eb["board"] - SATNET) * 100
verdict = f"{delta:+.2f} pts vs SATNet" if key[1] == "plain" else "—"
print(f" {key[0] + '/' + key[1]:<20}{eb['board']*100:>7.2f}% at {eb['nfe']:>5.1f} passes"
f" [{tag}] {verdict}")
print()
print("Note: a decoder is only as good as the model it decodes. The controlled")
print("claim in section 1 holds regardless of whether section 2 clears 98.3%.")
|