"""Terminal summary of every experiment, for quick status checks.""" import glob, json, os, sys import numpy as np RUNS = os.environ.get("RUNS", "runs") def load(pattern): out = [] for f in glob.glob(f"{RUNS}/{pattern}/result.json"): try: r = json.load(open(f)) except Exception: continue r["name"] = os.path.basename(os.path.dirname(f)) out.append(r) return out print("=" * 68) print("ADDITION — exact match at ONE forward pass (grid 11)") print("=" * 68) rows = {} for r in load("g11-*"): a = r["args"] rows.setdefault((a["digits"], a["mode"]), []).append(r["accuracy_by_passes"]) print(f"{'digits':>7} {'method':<8}{'1 pass':>9}{'16 pass':>9}{'seeds>90%':>11}") for k in sorted(rows): v = rows[k] one = [x["1"] for x in v] six = [x["16"] for x in v] print(f"{k[0]:>7} {k[1]:<8}{np.mean(one)*100:8.1f} {np.mean(six)*100:8.1f} " f"{sum(o > 0.9 for o in one):>6}/{len(one)}") print() print("=" * 68) print("INFLECTION — shallow dependencies, where no method is needed (grid 12)") print("=" * 68) rows = {} for r in load("g12-*"): a = r["args"] rows[(a["lang"], a["mode"])] = r["accuracy_by_passes"] print(f"{'language':<11}{'mode':<7}{'1 pass':>9}{'32 pass':>9}{'gap':>8}") for k in sorted(rows): v = rows[k] one, many = v["1"] * 100, v["32"] * 100 print(f"{k[0]:<11}{k[1]:<7}{one:8.1f} {many:8.1f} {many - one:+7.1f}") print() print("=" * 68) print("HEAD-TO-HEAD vs INFERENCE-TIME METHODS") print("=" * 68) for f in sorted(glob.glob("figures/beat_d*.json")) + sorted(glob.glob("../figures/beat_d*.json")): b = json.load(open(f)) s = b.get("summary", {}) print(f"{b['digits']}-digit:") for k, v in b["inference_time_on_baseline"].items(): print(f" {k:<16} 1pass={v[0]*100:6.1f} best-any-budget={max(v)*100:6.1f}") print(f" {'OURS':<16} 1pass={b['ours']['confidence'][0]*100:6.1f}")