Buckets:
| """Final tables, hypothesis tests (paired bootstrap), and figures from whatever arms exist under runs/. | |
| Best K per arm is chosen on dev accuracy (hard items), never on Core; ties go to the smaller K. | |
| """ | |
| import json | |
| import os | |
| import sys | |
| import numpy as np | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from bootstrap import compare, fmt, load # noqa: E402 | |
| from common import GROUPS # noqa: E402 | |
| R = "/workspace/think-decide/runs" | |
| OUT = "/workspace/think-decide/results" | |
| KS = [1, 2, 4, 8] | |
| ARMS = [("S1", "s1", [1]), ("E1", "e1", [1]), ("E2", "e2", KS), ("E3", "e3", KS)] | |
| def dev_best_k(name, ks): | |
| path = "%s/%s/dev/predictions.jsonl" % (R, name) | |
| if not os.path.exists(path) or len(ks) == 1: | |
| return ks[0], {} | |
| rows = [json.loads(l) for l in open(path)] | |
| acc = {} | |
| for k in range(1, max(ks) + 1): | |
| hard = [r for r in rows if r.get("kind") == "hard"] | |
| acc[k] = float(np.mean([np.argmax(r["probs"][str(k)]) == r["gold_idx"] for r in hard])) | |
| best = max(ks, key=lambda k: (round(acc[k], 6), -k)) | |
| return best, acc | |
| def ece_bins(rows, k, nb=15): | |
| conf = np.array([max(r["probs"][str(k)]) for r in rows]) | |
| corr = np.array([np.argmax(r["probs"][str(k)]) == r["gold_idx"] for r in rows], float) | |
| edges = np.linspace(0, 1, nb + 1) | |
| out = [] | |
| for lo, hi in zip(edges[:-1], edges[1:]): | |
| m = (conf > lo) & (conf <= hi) | |
| if m.any(): | |
| out.append(((lo + hi) / 2, conf[m].mean(), corr[m].mean(), int(m.sum()))) | |
| return out | |
| def adaptive(rows, kmax=8, tol=0.01): | |
| """Stop at the first pass k>=2 whose head distribution moved < tol (total variation) from pass k-1.""" | |
| used, correct = [], [] | |
| for r in rows: | |
| k_stop = kmax | |
| for k in range(2, kmax + 1): | |
| tv = 0.5 * np.abs(np.array(r["probs"][str(k)]) - np.array(r["probs"][str(k - 1)])).sum() | |
| if tv < tol: | |
| k_stop = k | |
| break | |
| used.append(k_stop) | |
| correct.append(np.argmax(r["probs"][str(k_stop)]) == r["gold_idx"]) | |
| return float(np.mean(used)), float(np.mean(correct)) | |
| def main(): | |
| os.makedirs(OUT, exist_ok=True) | |
| preds, best, devacc, reports = {}, {}, {}, {} | |
| for label, name, ks in ARMS: | |
| p = "%s/%s/core/predictions.jsonl" % (R, name) | |
| if os.path.exists(p): | |
| preds[label] = load(p) | |
| reports[label] = json.load(open("%s/%s/core/report.json" % (R, name))) | |
| best[label], devacc[label] = dev_best_k(name, ks) | |
| lines = ["# Results", "", "Core, all 6,744 items (no length cut). Group = equal-weight mean of suite accuracies (%).", | |
| "Best K chosen on dev. ECE = mean of suite ECE15 (%).", "", | |
| "| Model | K | Reasoning | Lookup | Other | Core17 | Core14 | ECE reasoning | ECE lookup |", | |
| "|---|---|---|---|---|---|---|---|---|"] | |
| table = {} | |
| for label, name, ks in ARMS: | |
| if label not in reports: | |
| continue | |
| for k in ks: | |
| g = reports[label]["by_K"][str(k)]["groups"] | |
| row = dict(reasoning=g["reasoning"]["accuracy"], lookup=g["lookup"]["accuracy"], | |
| other=g["other"]["accuracy"], core17=g["core17_macro"]["accuracy"], | |
| core14=g["core14_macro"]["accuracy"], ece_r=g["reasoning"]["ece"], ece_l=g["lookup"]["ece"]) | |
| table["%s@%d" % (label, k)] = row | |
| mark = " (dev-best)" if len(ks) > 1 and k == best[label] else "" | |
| lines.append("| %s | %d%s | %.2f | %.2f | %.2f | %.2f | %.2f | %.2f | %.2f |" % ( | |
| label, k, mark, *[100 * row[c] for c in ("reasoning", "lookup", "other", "core17", "core14", | |
| "ece_r", "ece_l")])) | |
| lines += ["", "Dev accuracy by K (hard items): " + "; ".join( | |
| "%s: %s" % (a, " ".join("K%d %.2f" % (k, 100 * v) for k, v in d.items())) for a, d in devacc.items() if d)] | |
| tests = [] | |
| def test(name, A, KA, B, KB, group, rule=None): | |
| if A in preds and B in preds: | |
| r = compare(preds[A], KA, preds[B], KB, group) | |
| r.update(name=name, group=group, arm_a="%s@%d" % (A, KA), arm_b="%s@%d" % (B, KB)) | |
| if rule: | |
| r["verdict"] = rule(r) | |
| tests.append(r) | |
| h1 = lambda r: "meets +2.0 with CI>0" if r["diff"] >= 2.0 and r["lo"] > 0 else "not met" | |
| for arm in ("E2", "E3"): | |
| for k in (2, 4, 8): | |
| test("H1 %s K=%d vs K=1" % (arm, k), arm, k, arm, 1, "reasoning", h1 if k in (4, 8) else None) | |
| test("H1 %s K=%d vs K=1" % (arm, k), arm, k, arm, 1, "lookup", | |
| (lambda r: "ok (>= -1.0)" if r["diff"] >= -1.0 else "hurts lookup") if k in (4, 8) else None) | |
| if "E2" in best: | |
| test("H2 E2(best K) vs E1", "E2", best["E2"], "E1", 1, "reasoning", | |
| lambda r: "supported" if r["diff"] > 0 and r["lo"] > 0 else "not supported") | |
| if "E2" in best and "E3" in best: | |
| test("H3 E3(best K) vs E2(best K)", "E3", best["E3"], "E2", best["E2"], "reasoning", | |
| lambda r: "supported" if r["diff"] >= 1.5 and r["lo"] > 0 else "not supported") | |
| for g in ("reasoning", "lookup", "core17"): | |
| test("E1 vs S1", "E1", 1, "S1", 1, g) | |
| test("slot cost: E2@1 vs E1", "E2", 1, "E1", 1, g) | |
| if "E3" in best: | |
| test("E3(best) vs E1", "E3", best["E3"], "E1", 1, g) | |
| for g in ("reasoning_no_mnli", "core14"): | |
| if "E2" in best: | |
| test("sensitivity E2 K=8 vs K=1", "E2", 8, "E2", 1, g) | |
| test("sensitivity E2(best) vs E1", "E2", best["E2"], "E1", 1, g) | |
| if "E3" in best: | |
| test("sensitivity E3 K=8 vs K=1", "E3", 8, "E3", 1, g) | |
| lines += ["", "## Paired bootstrap (10,000 resamples, stratified by suite; difference in points [95% CI])", "", | |
| "| Comparison | Group | Difference | Verdict |", "|---|---|---|---|"] | |
| for t in tests: | |
| lines.append("| %s | %s | %s | %s |" % (t["name"], t["group"], fmt(t), t.get("verdict", ""))) | |
| # per-suite appendix | |
| lines += ["", "## Per-suite accuracy (%)", ""] | |
| cols = [(l, k) for l, n, ks in ARMS if l in reports for k in ks] | |
| lines.append("| Suite | n | " + " | ".join("%s@%d" % c for c in cols) + " |") | |
| lines.append("|---|---|" + "---|" * len(cols)) | |
| for s in sum(GROUPS.values(), []): | |
| n = reports["S1"]["by_K"]["1"]["suites"].get(s, {}).get("n") if "S1" in reports else "" | |
| vals = [100 * reports[l]["by_K"][str(k)]["suites"][s]["accuracy"] for l, k in cols] | |
| lines.append("| %s | %s | " % (s, n) + " | ".join("%.2f" % v for v in vals) + " |") | |
| # E4 adaptive stopping | |
| ad = {} | |
| for arm in ("E2", "E3"): | |
| if arm in preds: | |
| rows = list(preds[arm].values()) | |
| ad[arm] = {str(tol): adaptive(rows, 8, tol) for tol in (0.001, 0.003, 0.01, 0.03)} | |
| ad[arm]["fixed"] = {str(k): float(np.mean([np.argmax(r["probs"][str(k)]) == r["gold_idx"] for r in rows])) | |
| for k in range(1, 9)} | |
| if ad: | |
| lines += ["", "## E4: adaptive stopping (micro accuracy over all Core items)", ""] | |
| for arm, d in ad.items(): | |
| lines.append("- %s fixed K: " % arm + ", ".join("K%s %.2f" % (k, 100 * v) for k, v in d["fixed"].items())) | |
| for tol in ("0.001", "0.003", "0.01", "0.03"): | |
| lines.append(" - stop when TV < %s: mean passes %.2f, accuracy %.2f" % (tol, d[tol][0], 100 * d[tol][1])) | |
| open(OUT + "/results.md", "w").write("\n".join(lines) + "\n") | |
| json.dump(dict(table=table, tests=tests, best_k=best, dev_acc=devacc, adaptive=ad), | |
| open(OUT + "/stats.json", "w"), indent=1) | |
| print("\n".join(lines)) | |
| figures(preds, reports, best) | |
| def figures(preds, reports, best): | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) | |
| for ax, g in zip(axes, ("reasoning", "lookup")): | |
| for arm, c in (("E2", "#2a6fdb"), ("E3", "#d9480f")): | |
| if arm in reports: | |
| ks = list(range(1, 9)) | |
| ax.plot(ks, [100 * reports[arm]["by_K"][str(k)]["groups"][g]["accuracy"] for k in ks], "o-", | |
| color=c, label=arm, ms=4) | |
| for arm, ls in (("E1", "--"), ("S1", ":")): | |
| if arm in reports: | |
| ax.axhline(100 * reports[arm]["by_K"]["1"]["groups"][g]["accuracy"], ls=ls, color="gray", label=arm) | |
| ax.set_title("%s suites" % g) | |
| ax.set_xlabel("denoising passes K") | |
| ax.set_ylabel("accuracy (%)") | |
| ax.grid(alpha=.3) | |
| ax.legend(fontsize=8) | |
| plt.tight_layout() | |
| plt.savefig(OUT + "/acc_vs_k.png", dpi=150) | |
| plt.close() | |
| fig, ax = plt.subplots(figsize=(4.5, 4.2)) | |
| ax.plot([0, 1], [0, 1], color="gray", lw=1) | |
| for arm, k, c in (("E1", 1, "gray"), ("E2", best.get("E2", 1), "#2a6fdb"), ("E3", best.get("E3", 1), "#d9480f")): | |
| if arm in preds: | |
| b = ece_bins(list(preds[arm].values()), k) | |
| ax.plot([x[1] for x in b], [x[2] for x in b], "o-", ms=3, color=c, label="%s@%d" % (arm, k)) | |
| ax.set_xlabel("confidence") | |
| ax.set_ylabel("accuracy") | |
| ax.set_title("Reliability (all Core items)") | |
| ax.legend(fontsize=8) | |
| plt.tight_layout() | |
| plt.savefig(OUT + "/reliability.png", dpi=150) | |
| plt.close() | |
| fig, ax = plt.subplots(figsize=(5, 3.4)) | |
| for arm, c in (("e2", "#2a6fdb"), ("e3", "#d9480f")): | |
| p = "%s/%s/train_log.jsonl" % (R, arm) | |
| if os.path.exists(p): | |
| rows = [json.loads(l) for l in open(p)] | |
| rows = [r for r in rows if "gate" in r and r.get("event") is None] | |
| ax.plot([r["step"] for r in rows], [r["gate"] for r in rows], color=c, label=arm.upper()) | |
| ax.set_xlabel("step") | |
| ax.set_ylabel("gate g") | |
| ax.grid(alpha=.3) | |
| ax.legend() | |
| plt.tight_layout() | |
| plt.savefig(OUT + "/gate.png", dpi=150) | |
| plt.close() | |
| if __name__ == "__main__" and not (len(sys.argv) > 1 and sys.argv[1] == "proc"): | |
| main() | |
| # ---------------------------------------------------------------- procedural testbed (E6) | |
| PROC_ARMS = [("P0 S1 zero-shot", "p0", [1]), ("P1 control", "p1", [1]), ("P2o open gate", "p2o", list(range(1, 9))), | |
| ("P2s soft tokens", "p2s", list(range(1, 9)))] | |
| def proc_best_k(name, ks): | |
| path = "%s/%s/proc_dev/predictions.jsonl" % (R, name) | |
| if len(ks) == 1 or not os.path.exists(path): | |
| return ks[0], {} | |
| rows = [json.loads(l) for l in open(path)] | |
| hard = [r for r in rows if r.get("gold_idx") is not None] | |
| acc = {k: float(np.mean([np.argmax(r["probs"][str(k)]) == r["gold_idx"] for r in hard])) for k in ks} | |
| return max([1, 2, 4, 8], key=lambda k: (round(acc[k], 6), -k)), acc | |
| def analyze_proc(): | |
| preds, rep, best, dacc = {}, {}, {}, {} | |
| for label, name, ks in PROC_ARMS: | |
| p = "%s/%s/proc_test/predictions.jsonl" % (R, name) | |
| if os.path.exists(p): | |
| preds[name] = load(p) | |
| rep[name] = json.load(open("%s/%s/proc_test/report.json" % (R, name))) | |
| best[name], dacc[name] = proc_best_k(name, ks) | |
| L = ["", "## E6 procedural testbed (3,484 held-out items, 18 task families x 5 levels; accuracy %)", "", | |
| "| Arm | K | All | L0 | L1 | L2 | L3 | L4 |", "|---|---|---|---|---|---|---|---|"] | |
| for label, name, ks in PROC_ARMS: | |
| if name not in rep: | |
| continue | |
| for k in ([1, 2, 4, 8] if len(ks) > 1 else ks): | |
| pr = rep[name]["proc"][str(k)] | |
| mark = " (dev-best)" if len(ks) > 1 and k == best[name] else "" | |
| L.append("| %s | %d%s | %.2f | " % (label, k, mark, 100 * pr["all"]) + | |
| " | ".join("%.2f" % (100 * pr["by_level"][str(l)]) for l in range(5)) + " |") | |
| tests = [] | |
| for name in ("p2o", "p2s"): | |
| if name not in preds: | |
| continue | |
| for k in (2, 4, 8): | |
| for sub_name, sub in (("all", None), ("L3-4", lambda p: p["level"] in (3, 4)), ("L0-1", lambda p: p["level"] in (0, 1))): | |
| r = compare(preds[name], k, preds[name], 1, "core17", subset=sub) | |
| r.update(name="%s K=%d vs K=1" % (name, k), group=sub_name) | |
| tests.append(r) | |
| if "p1" in preds: | |
| for sub_name, sub in (("all", None), ("L3-4", lambda p: p["level"] in (3, 4))): | |
| r = compare(preds[name], best[name], preds["p1"], 1, "core17", subset=sub) | |
| r.update(name="%s(best K=%d) vs P1" % (name, best[name]), group=sub_name) | |
| tests.append(r) | |
| if tests: | |
| L += ["", "Paired bootstrap, stratified by task family (macro over families; points [95% CI]):", "", | |
| "| Comparison | Items | Difference |", "|---|---|---|"] | |
| L += ["| %s | %s | %s |" % (t["name"], t["group"], fmt(t)) for t in tests] | |
| if dacc: | |
| L += ["", "Dev accuracy by K: " + "; ".join("%s: %s" % (n, " ".join("K%d %.2f" % (k, 100 * v) for k, v in d.items())) | |
| for n, d in dacc.items() if d)] | |
| open(OUT + "/results_proc.md", "w").write("\n".join(L) + "\n") | |
| json.dump(dict(tests=tests, best_k=best, dev_acc=dacc), open(OUT + "/stats_proc.json", "w"), indent=1) | |
| print("\n".join(L)) | |
| if rep: | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| fig, axes = plt.subplots(1, 5, figsize=(16, 3.2), sharey=False) | |
| for l, ax in enumerate(axes): | |
| for name, c in (("p2o", "#2a6fdb"), ("p2s", "#d9480f")): | |
| if name in rep: | |
| ax.plot(range(1, 9), [100 * rep[name]["proc"][str(k)]["by_level"][str(l)] for k in range(1, 9)], "o-", | |
| ms=3, color=c, label=name.upper()) | |
| for name, ls in (("p1", "--"), ("p0", ":")): | |
| if name in rep: | |
| ax.axhline(100 * rep[name]["proc"]["1"]["by_level"][str(l)], ls=ls, color="gray", label=name.upper()) | |
| ax.set_title("level %d" % l) | |
| ax.set_xlabel("passes K") | |
| ax.grid(alpha=.3) | |
| axes[0].set_ylabel("accuracy (%)") | |
| axes[0].legend(fontsize=7) | |
| plt.tight_layout() | |
| plt.savefig(OUT + "/proc_acc_vs_k.png", dpi=150) | |
| plt.close() | |
| if __name__ == "__main__" and len(sys.argv) > 1 and sys.argv[1] == "proc": | |
| analyze_proc() | |
Xet Storage Details
- Size:
- 14.3 kB
- Xet hash:
- 619092ea20ace2d9027d9b83bd8d423b44e82078f359c2b42d190ce8ce488061
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.