Buckets:
| """MQ analysis: sequential / soft / oracle vs parallel on the same questions (paired bootstrap stratified by task | |
| family), by level, by family, by commit rank. Usage: analyze_mq.py runs/mq/test [runs/mqsoft/test ...]""" | |
| import json | |
| import sys | |
| from collections import defaultdict | |
| import numpy as np | |
| def boot(rows, ma, mb, R=10000, seed=0, sel=None): | |
| rows = [r for r in rows if sel is None or sel(r)] | |
| by = defaultdict(list) | |
| for r in rows: | |
| by[r["task"]].append(r) | |
| rng = np.random.default_rng(seed) | |
| diffs, pa, pb = np.zeros(R), [], [] | |
| for t, rs in by.items(): | |
| ca = np.array([np.argmax(r["pred"][ma]) == r["gold"] for r in rs], float) | |
| cb = np.array([np.argmax(r["pred"][mb]) == r["gold"] for r in rs], float) | |
| pa.append(ca.mean()), pb.append(cb.mean()) | |
| idx = rng.integers(0, len(rs), size=(R, len(rs))) | |
| diffs += ca[idx].mean(1) - cb[idx].mean(1) | |
| diffs /= len(by) | |
| lo, hi = np.percentile(diffs, [2.5, 97.5]) | |
| return 100 * (np.mean(pa) - np.mean(pb)), 100 * lo, 100 * hi, 100 * np.mean(pa), 100 * np.mean(pb), len(rows) | |
| def acc(rs, m): | |
| return 100 * float(np.mean([np.argmax(r["pred"][m]) == r["gold"] for r in rs])) if rs else float("nan") | |
| def main(path): | |
| rows = [json.loads(l) for l in open(path + "/predictions.jsonl")] | |
| modes = sorted(rows[0]["pred"]) | |
| out = ["", "## %s (%d questions, %d states)" % (path, len(rows), len({r["state"] for r in rows})), "", | |
| "| Mode | All | Multi-q states | Target q | L0 | L1 | L2 | L3 | L4 |", "|---|---|---|---|---|---|---|---|---|"] | |
| for m in modes: | |
| out.append("| %s | %.2f | %.2f | %.2f | " % (m, acc(rows, m), acc([r for r in rows if r["n_q"] > 1], m), | |
| acc([r for r in rows if r["is_target"]], m)) + | |
| " | ".join("%.2f" % acc([r for r in rows if r["level"] == l], m) for l in range(5)) + " |") | |
| out += ["", "Paired bootstrap vs parallel (macro over task families; points [95% CI]; n questions):", "", | |
| "| Comparison | Subset | Difference |", "|---|---|---|"] | |
| subsets = [("all", None), ("multi-q states", lambda r: r["n_q"] > 1), ("L3-4", lambda r: r["level"] >= 3), | |
| ("target q", lambda r: r["is_target"])] | |
| for m in modes: | |
| if m == "parallel": | |
| continue | |
| for name, sel in subsets: | |
| d, lo, hi, a, b, n = boot(rows, m, "parallel", sel=sel) | |
| out.append("| %s - parallel | %s | %+.2f [%+.2f, %+.2f] (%.2f vs %.2f; n=%d) |" % (m, name, d, lo, hi, a, b, n)) | |
| if "sequential" in modes: | |
| out += ["", "Sequential vs parallel by commit rank (0 = committed first):", ""] | |
| for k in range(7): | |
| rs = [r for r in rows if r.get("commit_rank") == k and r["n_q"] > 1] | |
| if len(rs) >= 50: | |
| out.append("- rank %d (n=%d): parallel %.2f -> sequential %.2f (oracle %.2f)" % ( | |
| k, len(rs), acc(rs, "parallel"), acc(rs, "sequential"), acc(rs, "oracle") if "oracle" in modes else float("nan"))) | |
| out += ["", "By family (parallel / sequential / oracle):", ""] | |
| for t in sorted({r["task"] for r in rows}): | |
| rs = [r for r in rows if r["task"] == t] | |
| out.append("- %-32s n=%4d %.1f / %.1f / %.1f" % (t, len(rs), acc(rs, "parallel"), acc(rs, "sequential"), | |
| acc(rs, "oracle") if "oracle" in modes else float("nan"))) | |
| print("\n".join(out)) | |
| return "\n".join(out) | |
| if __name__ == "__main__": | |
| text = "\n".join(main(p) for p in sys.argv[1:]) | |
| open("/workspace/think-decide/results/results_mq.md", "w").write("# MQ answer diffusion\n" + text + "\n") | |
Xet Storage Details
- Size:
- 3.71 kB
- Xet hash:
- 0c8eeb1123a9b23bfa3feac577229c2b708118d864731a05f6ca9d839aaf2c2c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.