Download paper/make_data.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/paper/make_data.py
- Command line
-
hf download hf://Nabidnur/PISCES/paper/make_data.py
-
curl -L -o make_data.py https://huggingface.co/Nabidnur/PISCES/resolve/main/paper/make_data.py
21.5 kB
| """Turn results/*.jsonl into the .dat tables the paper's PGFPlots figures read, plus macros.tex | |
| with every number quoted in the text, so the paper never hand-copies a result. | |
| Usage: python paper/make_data.py (from the repository root) | |
| """ | |
| import collections, json, os | |
| import numpy as np | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| OUT = os.path.join(ROOT, "paper", "data"); os.makedirs(OUT, exist_ok=True) | |
| R = lambda f: [json.loads(l) for l in open(os.path.join(ROOT, "results", f))] if os.path.exists(os.path.join(ROOT, "results", f)) else [] | |
| def table(rows, key="test_mse"): | |
| g = collections.defaultdict(dict) | |
| for r in rows: | |
| g[(r["method"], float(r["epochs"]))][r["seed"]] = r | |
| return g | |
| def paired(g, m, T, base=("official", None), Tb=None): | |
| o = g.get((base[0], Tb if Tb is not None else T), {}); p = g.get((m, T), {}) | |
| s = sorted(set(o) & set(p)) | |
| d = np.array([100 * (p[x]["test_mse"] - o[x]["test_mse"]) / o[x]["test_mse"] for x in s]) | |
| t = np.array([p[x]["seconds"] / o[x]["seconds"] for x in s]) | |
| return s, d, t | |
| macros = [] | |
| MAC = {} # raw values, for data files derived from macros | |
| def M(name, val, fmt="{:.2f}"): | |
| MAC[name] = val | |
| macros.append("\\newcommand{\\%s}{%s}" % (name, fmt.format(val))) | |
| # 1. quality vs wall-clock on DEEP* 1M test rows | |
| deep = table(R("final/deep1m_test.jsonl")) | |
| hist = table(R("deep1m_cpu.jsonl")) # development history (earlier session) | |
| for m in ["official", "pisces", "pisces_v10"]: | |
| with open(os.path.join(OUT, f"qvt_{m}.dat"), "w") as f: | |
| f.write("epochs seconds mse sd n\n") | |
| for T in sorted({T for (mm, T) in deep if mm == m}): | |
| v = list(deep[(m, T)].values()) | |
| f.write(f"{T:g} {np.mean([r['seconds'] for r in v]):.2f} {np.mean([r['test_mse'] for r in v]):.6f} " | |
| f"{np.std([r['test_mse'] for r in v]):.6f} {len(v)}\n") | |
| # 2. main table macros (PISCES vs official on test rows) | |
| for T, tag in [(1.0, "One"), (3.0, "Three"), (5.0, "Five")]: | |
| for m, mt in [("pisces", "P"), ("pemeans_v3", "V")]: | |
| s, d, t = paired(deep, m, T) | |
| if len(s): | |
| M(f"d{mt}{tag}", d.mean(), "{:+.2f}"); M(f"w{mt}{tag}", int((d < 0).sum()), "{}"); M(f"n{mt}{tag}", len(s), "{}") | |
| M(f"t{mt}{tag}", t.mean()); M(f"lo{mt}{tag}", np.percentile(np.random.default_rng(0).choice(d, (10000, len(d))).mean(1), 2.5)) | |
| M(f"hi{mt}{tag}", np.percentile(np.random.default_rng(0).choice(d, (10000, len(d))).mean(1), 97.5)) | |
| for m, mt in [("official", "O"), ("pisces", "P")]: | |
| v = list(deep.get((m, T), {}).values()) | |
| if v: | |
| M(f"mse{mt}{tag}", np.mean([r["test_mse"] for r in v]), "{:.5f}") | |
| M(f"sec{mt}{tag}", np.mean([r["seconds"] for r in v]), "{:.0f}") | |
| s, d, t = paired(deep, "pisces", 3.0, Tb=5.0) | |
| if len(s): | |
| M("dPThreeVsFive", d.mean(), "{:+.2f}"); M("tPThreeVsFive", t.mean()); M("wPThreeVsFive", int((d < 0).sum()), "{}"); M("nPThreeVsFive", len(s), "{}") | |
| s, d, t = paired(deep, "emeans_approx", 3.0) | |
| if len(s): | |
| M("dEAThree", d.mean(), "{:+.2f}"); M("tEAThree", t.mean()); M("wEAThree", int((d < 0).sum()), "{}"); M("nEAThree", len(s), "{}") | |
| s2, d2, _ = paired(deep, "pisces", 3.0, base=("emeans_approx", None)) | |
| if len(s2): | |
| M("dPvsEAThree", d2.mean(), "{:+.2f}"); M("wPvsEAThree", int((d2 < 0).sum()), "{}"); M("nPvsEAThree", len(s2), "{}") | |
| # 3. development history at 3 epochs (each version paired with official e-means of its own session) | |
| for m, tag in [("pemeans", "One"), ("pemeans_v2", "Two"), ("pemeans_v3", "Three"), ("pisces", "Ten")]: | |
| s_, d, t = paired(hist, m, 3.0) | |
| if len(s_): | |
| M(f"hist{tag}D", d.mean(), "{:+.2f}"); M(f"hist{tag}T", t.mean()) | |
| for m, tag in [("pisces_v10", "TenNow"), ("pisces", "Final")]: | |
| s_, d, t = paired(deep, m, 3.0) | |
| if len(s_): | |
| M(f"hist{tag}D", d.mean(), "{:+.2f}"); M(f"hist{tag}T", t.mean()); M(f"hist{tag}W", int((d < 0).sum()), "{}") | |
| s_, d, t = paired(deep, "pisces", 3.0, base=("pisces_v10", None)) | |
| if len(s_): | |
| M("engD", d.mean(), "{:+.3f}"); M("engT", t.mean()); M("engSame", int((np.abs(d) < 1e-6).sum()), "{}"); M("engN", len(s_), "{}") | |
| # 4. time to quality on validation rows | |
| ttq = table(R("final/deep1m_ttq_val.jsonl")) | |
| with open(os.path.join(OUT, "ttq.dat"), "w") as f: | |
| f.write("method epochs seconds mse\n") | |
| for (m, T), v in sorted(ttq.items()): | |
| f.write(f"{m} {T:g} {np.mean([r['seconds'] for r in v.values()]):.2f} {np.mean([r['test_mse'] for r in v.values()]):.6f}\n") | |
| for T, tag in [(1.0, "One"), (2.0, "Two"), (3.0, "Three"), (4.0, "Four")]: | |
| s, d, t = paired(ttq, "pisces", T, Tb=5.0) | |
| if len(s): | |
| M(f"ttqD{tag}", d.mean(), "{:+.2f}"); M(f"ttqT{tag}", t.mean()); M(f"ttqW{tag}", int((d < 0).sum()), "{}") | |
| # 5. held-out datasets (per-seed paired differences) | |
| for f_, tag in [("final/ssnpp1m_test.jsonl", "SS"), ("final/spacev1m_test.jsonl", "SV")]: | |
| g = table(R(f_)) | |
| for m, mt in [("pisces", "P"), ("emeans_approx", "E")]: | |
| s, d, t = paired(g, m, 3.0) | |
| if len(s): | |
| M(f"d{mt}{tag}", d.mean(), "{:+.3f}"); M(f"t{mt}{tag}", t.mean()); M(f"w{mt}{tag}", int((d < 0).sum()), "{}"); M(f"n{mt}{tag}", len(s), "{}") | |
| with open(os.path.join(OUT, f"heldout_{tag}_{m}.dat"), "w") as fo: | |
| fo.write("seed delta\n" + "".join(f"{x} {y:.5f}\n" for x, y in zip(s, d))) | |
| # 6. per-method time-to-quality curves for the figure | |
| for m in ["official", "pisces", "emeans_approx"]: | |
| with open(os.path.join(OUT, f"ttq_{m}.dat"), "w") as f: | |
| f.write("epochs seconds mse\n") | |
| for (mm, T), v in sorted(ttq.items()): | |
| if mm == m: | |
| f.write(f"{T:g} {np.mean([r['seconds'] for r in v.values()]):.2f} {np.mean([r['test_mse'] for r in v.values()]):.6f}\n") | |
| # 7. per-seed paired differences, all evaluations, for a strip plot (y = row index) | |
| with open(os.path.join(OUT, "strip.dat"), "w") as f: | |
| f.write("row delta\n") | |
| rows_ = [(1, deep, "pisces", 1.0, None), (2, deep, "pisces", 3.0, None), (3, deep, "pisces", 5.0, None), | |
| (4, table(R("final/spacev1m_test.jsonl")), "pisces", 3.0, None), (5, table(R("final/ssnpp1m_test.jsonl")), "pisces", 3.0, None), | |
| (6, deep, "emeans_approx", 3.0, None), (7, table(R("final/spacev1m_test.jsonl")), "emeans_approx", 3.0, None), | |
| (8, table(R("final/ssnpp1m_test.jsonl")), "emeans_approx", 3.0, None)] | |
| sw = {"p": [0, 0], "e": [0, 0]} | |
| for row, g, m, T, _ in rows_: | |
| _, d, _ = paired(g, m, T) | |
| f.write("".join(f"{row} {x:.5f}\n" for x in d)) | |
| c = sw["p" if m == "pisces" else "e"]; c[0] += int(np.sum(np.asarray(d) < 0)); c[1] += len(d) | |
| M("stripWinsP", sw["p"][0], "{}"); M("stripN", sw["p"][1], "{}") | |
| M("stripLossE", sw["e"][1] - sw["e"][0], "{}"); M("stripNE", sw["e"][1], "{}") | |
| # 8. leave-one-out ablation at full scale (validation rows, seeds 1-3, 3 epochs) | |
| abl = table(R("final/ablation_full_val.jsonl")) | |
| ABL = [("pisces", "Full"), ("no_principal_split", "NoSplit"), ("no_gain_ranking", "NoGain"), | |
| ("no_consistent_closing", "NoClose"), ("tau_0.1", "TauLow"), ("step_x1.0", "StepOne"), | |
| ("no_early_revival", "NoEarly"), ("exact_last_epoch", "ExactLast"), ("exact_assignment", "Exact"), ("no_engineering", "NoEng")] | |
| for _f in ["ablation.dat", "ablation_seeds.dat"]: | |
| if os.path.exists(os.path.join(OUT, _f)): | |
| os.remove(os.path.join(OUT, _f)) | |
| with open(os.path.join(OUT, "ablation.dat") if abl else os.devnull, "w") as f: | |
| f.write("idx name delta lo hi cost ratio\n") | |
| for i, (m, tag) in enumerate(ABL): | |
| s_, d, t = paired(abl, m, 3.0) | |
| if not len(s_): | |
| continue | |
| M(f"abl{tag}D", d.mean(), "{:+.3f}"); M(f"abl{tag}T", t.mean()); M(f"abl{tag}W", int((d < 0).sum()), "{}") | |
| M(f"abl{tag}N", len(s_), "{}") | |
| s2, d2, _ = paired(abl, m, 3.0, base=("pisces", None)) | |
| cost = d2.mean() if m != "pisces" else 0.0 | |
| M(f"abl{tag}C", cost, "{:+.3f}") | |
| f.write(f"{i} {tag} {d.mean():.4f} {d.min():.4f} {d.max():.4f} {cost:.4f} {t.mean():.4f}\n") | |
| # 9. scaling in k (validation rows, 3 epochs); k = 16,384 from the time-to-quality runs | |
| SC = [(1024, "OneK", "final/scaling_k1024_val.jsonl"), (4096, "FourK", "final/scaling_k4096_val.jsonl"), | |
| (16384, "SixteenK", "final/deep1m_ttq_val.jsonl"), (65536, "SixtyFourK", "final/scaling_k65536_val.jsonl")] | |
| with open(os.path.join(OUT, "scaling.dat"), "w") as f, open(os.path.join(OUT, "scaling_seeds.dat"), "w") as fs: | |
| f.write("k delta ratio\n"); fs.write("k delta speedup\n") | |
| for k_, tag, fn in SC: | |
| g = table(R(fn)) | |
| s_, d, t = paired(g, "pisces", 3.0) | |
| if not len(s_): | |
| continue | |
| M(f"sc{tag}D", d.mean(), "{:+.2f}"); M(f"sc{tag}T", t.mean()); M(f"sc{tag}W", int((d < 0).sum()), "{}") | |
| M(f"sc{tag}N", len(s_), "{}") | |
| M(f"sc{tag}MO", np.mean([r["test_mse"] for r in g[("official", 3.0)].values()]), "{:.5f}") | |
| M(f"sc{tag}SO", np.mean([r["seconds"] for r in g[("official", 3.0)].values()]), "{:.0f}") | |
| M(f"sc{tag}SP", np.mean([r["splits"] for r in g[("pisces", 3.0)].values()]), "{:,.0f}") | |
| M(f"sc{tag}SpO", np.mean([r["splits"] for r in g[("official", 3.0)].values()]), "{:,.0f}") | |
| f.write(f"{k_} {d.mean():.4f} {t.mean():.4f}\n") | |
| fs.write("".join(f"{k_} {di:.4f} {1 / ti:.4f}\n" for di, ti in zip(d, t))) | |
| # 9b. the small-k rule (iteration 23b): exact assignment below k = 4,096, same session as its official runs | |
| g23 = table(R("dev/iter23_small_k_val.jsonl")) | |
| if g23: | |
| s_, d, t = paired(g23, "exact_k1024", 3.0) | |
| if len(s_): | |
| M("scOneKRuleD", d.mean(), "{:+.2f}"); M("scOneKRuleT", t.mean()); M("scOneKRuleW", int((d < 0).sum()), "{}") | |
| M("scOneKRuleN", len(s_), "{}") | |
| # 9c. the early-revival rule (iteration 24): k = 65,536, PISCES without every-batch revival vs the official scaling runs | |
| g24 = table(R("final/scaling_k65536_val.jsonl") + R("dev/iter24_k65536_regime_val.jsonl")) | |
| if ("no_early_revival", 3.0) in g24: | |
| s_, d, t = paired(g24, "no_early_revival", 3.0) | |
| # time: the diagnostic ran in a later session than the official runs, so chain through PISCES timed in | |
| # the diagnostic's own session: (rule / PISCES, same session) x (PISCES / official, scaling session) | |
| g24own = table(R("dev/iter24_k65536_regime_val.jsonl")); gsc = table(R("final/scaling_k65536_val.jsonl")) | |
| t = np.array([g24own[("no_early_revival", 3.0)][q]["seconds"] / g24own[("pisces", 3.0)][q]["seconds"] | |
| * gsc[("pisces", 3.0)][q]["seconds"] / gsc[("official", 3.0)][q]["seconds"] for q in s_]) | |
| M("scSixtyFourKRuleD", d.mean(), "{:+.2f}"); M("scSixtyFourKRuleT", t.mean()); M("scSixtyFourKRuleW", int((d < 0).sum()), "{}") | |
| M("scSixtyFourKRuleN", len(s_), "{}") | |
| M("scSixtyFourKRuleSP", np.mean([r["splits"] for r in g24[("no_early_revival", 3.0)].values()]), "{:,.0f}") | |
| s_, d, t = paired(g24, "tau_0.1", 3.0); M("scSixtyFourKTauD", d.mean(), "{:+.2f}") | |
| s_, d, t = paired(g24, "tau_0.1_no_early", 3.0); M("scSixtyFourKBothD", d.mean(), "{:+.2f}") | |
| M("scSixtyFourKBothSP", np.mean([r["splits"] for r in g24[("tau_0.1_no_early", 3.0)].values()]), "{:,.0f}") | |
| # 9d. scaling figure overlay: the released rules at the two ends of the k range | |
| with open(os.path.join(OUT, "scaling_rules.dat"), "w") as f: | |
| f.write("k delta speedup\n") | |
| for k_, tag in [(1024, "OneK"), (65536, "SixtyFourK")]: | |
| if f"sc{tag}RuleD" in MAC: | |
| f.write(f"{k_} {MAC[f'sc{tag}RuleD']:.4f} {1 / MAC[f'sc{tag}RuleT']:.4f}\n") | |
| # 10. downstream IVF index | |
| ivf = R("final/ivf_downstream.jsonl") | |
| if ivf: | |
| def interp_ndis(curve, target): | |
| r = [c_["recall1"] for c_ in curve]; n_ = [c_["ndis"] for c_ in curve] | |
| for i in range(1, len(r)): | |
| if r[i] >= target > r[i - 1]: | |
| w = (target - r[i - 1]) / (r[i] - r[i - 1]); return np.exp(np.log(n_[i - 1]) + w * (np.log(n_[i]) - np.log(n_[i - 1]))) | |
| return float("nan") | |
| by = collections.defaultdict(list) | |
| for r in ivf: | |
| by[r["method"]].append(r) | |
| for m, tag in [("official", "O"), ("pisces", "P")]: | |
| rs = by.get(m, []) | |
| if not rs: | |
| continue | |
| nps = [c_["nprobe"] for c_ in rs[0]["curve"]] | |
| with open(os.path.join(OUT, f"ivf_{m}.dat"), "w") as f: | |
| f.write("nprobe ndis recall1 recall10\n") | |
| for i, npb in enumerate(nps): | |
| f.write(f"{npb} {np.mean([r['curve'][i]['ndis'] for r in rs]):.1f} {np.mean([r['curve'][i]['recall1'] for r in rs]):.5f} " | |
| f"{np.mean([r['curve'][i]['recall10'] for r in rs]):.5f}\n") | |
| M(f"ivfImb{tag}", np.mean([r["imbalance"] for r in rs]), "{:.3f}") | |
| M(f"ivfNdisNine{tag}", np.mean([interp_ndis(r["curve"], 0.9) for r in rs]), "{:.0f}") | |
| i16 = nps.index(16) if 16 in nps else 0 | |
| M(f"ivfRecSixteen{tag}", np.mean([r["curve"][i16]["recall1"] for r in rs]), "{:.4f}") | |
| M(f"ivfNdisSixteen{tag}", np.mean([r["curve"][i16]["ndis"] for r in rs]), "{:.0f}") | |
| seeds = sorted({r["seed"] for r in by["official"]} & {r["seed"] for r in by.get("pisces", [])}) | |
| if seeds: | |
| o = {r["seed"]: r for r in by["official"]}; p_ = {r["seed"]: r for r in by["pisces"]} | |
| rd = [100 * (p_[s_]["curve"][i]["recall1"] - o[s_]["curve"][i]["recall1"]) for s_ in seeds for i in range(len(o[s_]["curve"]))] | |
| nine = [interp_ndis(p_[s_]["curve"], 0.9) / interp_ndis(o[s_]["curve"], 0.9) for s_ in seeds] | |
| M("ivfNdisNineRatio", np.nanmean(nine)); M("ivfSeeds", len(seeds), "{}") | |
| M("ivfRecGainMax", max(rd), "{:+.2f}") | |
| # recall@1 of the PISCES index at the e-means index' search cost (log-linear interpolation in ndis) | |
| md = [] | |
| with open(os.path.join(OUT, "ivf_matched.dat"), "w") as f: | |
| f.write("ndis diff seed\n") | |
| for s_ in seeds: | |
| pn = np.log([c_["ndis"] for c_ in p_[s_]["curve"]]); pr = [c_["recall1"] for c_ in p_[s_]["curve"]] | |
| for c_ in o[s_]["curve"]: | |
| if pn[0] <= np.log(c_["ndis"]) <= pn[-1]: | |
| dd = 100 * (np.interp(np.log(c_["ndis"]), pn, pr) - c_["recall1"]); md.append(dd) | |
| f.write(f"{c_['ndis']:.1f} {dd:.4f} {s_}\n") | |
| M("ivfMatchedMean", np.mean(md), "{:+.2f}"); M("ivfMatchedMin", min(md), "{:+.2f}"); M("ivfMatchedMax", max(md), "{:+.2f}") | |
| # 11. mechanism (one seed): revival timeline, cluster sizes, Lemma 2 on real clusters, swap audit | |
| mf = os.path.join(ROOT, "results", "final", "mechanism_seed10.json") | |
| if os.path.exists(mf): | |
| mech = json.load(open(mf)) | |
| for m, tag in [("official", "O"), ("pisces", "P")]: | |
| r = mech[m]; tot = r["total_steps"] or max([s_ for s_, _ in r["split_log"]] + [1]) | |
| with open(os.path.join(OUT, f"splits_{m}.dat"), "w") as f: | |
| f.write("frac cum\n0 0\n"); c_ = 0 | |
| for st, nn in r["split_log"]: | |
| c_ += nn; f.write(f"{(st + 1) / tot:.4f} {c_}\n") | |
| sizes = np.array(r["sizes"]) | |
| with open(os.path.join(OUT, f"sizes_{m}.dat"), "w") as f: | |
| f.write("size cdf\n") | |
| for v in np.unique(np.concatenate([[0], np.unique(np.round(np.geomspace(1, sizes.max(), 60)))])): | |
| f.write(f"{max(v, 0.5):g} {(sizes <= v).mean():.5f}\n") | |
| M(f"mechSplits{tag}", sum(nn for _, nn in r["split_log"]), "{:,}") | |
| M(f"mechEmpty{tag}", int((sizes == 0).sum()), "{}") | |
| M(f"mechTiny{tag}", 100 * float((sizes < 10).mean()), "{:.2f}") | |
| M(f"mechCV{tag}", float(sizes.std() / sizes.mean()), "{:.3f}") | |
| M(f"mechSwaps{tag}", r["swaps"]["profitable_moves"], "{:,}") | |
| M(f"mechSwapGain{tag}", 100 * r["swaps"]["predicted_gain_per_row"] / r["train_mse"], "{:.3f}") | |
| lem = r["lemma"] | |
| pred = np.array([q["predicted"] for q in lem]); ax_ = np.array([q["axis_split"] for q in lem]); fr = np.array([q["free_2means"] for q in lem]) | |
| M(f"mechLemRatio{tag}", float(np.median(ax_ / pred)), "{:.2f}") | |
| M(f"mechLemFree{tag}", float(np.median(ax_ / fr)), "{:.2f}") | |
| M(f"mechLemCorr{tag}", float(np.corrcoef(np.log(pred), np.log(ax_))[0, 1]), "{:.3f}") | |
| with open(os.path.join(OUT, f"lemma_{m}.dat"), "w") as f: | |
| f.write("pred axis free\n" + "".join(f"{a_:.4f} {b_:.4f} {c__:.4f}\n" for a_, b_, c__ in zip(pred, ax_, fr))) | |
| M("mechSwapRatio", mech["official"]["swaps"]["profitable_moves"] / max(1, mech["pisces"]["swaps"]["profitable_moves"]), "{:.1f}") | |
| # 12. every negative result, recomputed from the development logs (proxy: DEEP 250k, k = 4,096, | |
| # validation rows, 3 epochs, 5 seeds), as a LaTeX table body | |
| def dev(fn): | |
| g = collections.defaultdict(dict) | |
| for line in open(os.path.join(ROOT, "results", "dev", fn)): | |
| r = json.loads(line) | |
| if "epochs" in r: | |
| g[(r.get("variant", r.get("method")), float(r["epochs"]))][r["seed"]] = r["val_mse"] | |
| return g | |
| NEG = [("iter2_swap.jsonl", "pe", "swap0.01", "streaming split--merge (swap) moves, 1\\% of $k$", 2), | |
| ("iter4_combine.jsonl", "sp_tau0.2_a1.4", "sp_tau0.2_a1.4_close0.5", "fresher closing window (last 50\\% of the final epoch)", 4), | |
| ("iter6_v3_candidates.jsonl", "v2", "v2_cm0.35", "children inherit 35\\% of the donor's mass each", 6), | |
| ("iter6_v3_candidates.jsonl", "v2", "v2_tau.3to.1", "revival threshold schedule $0.3\\to0.1$", 6), | |
| ("iter9b_graph_verified.jsonl", "v2", "v2_gv1pct", "graph-pruned assignment with label memory, self-verified", 9), | |
| ("iter13_growth.jsonl", "pisces_grow0", "pisces_grow3", "coarse-to-fine growth (3 doublings)", 13), | |
| ("iter15_candidates.jsonl", "pisces", "mom0.3", "heavy-ball momentum on centroid moves ($\\beta=0.3$)", 15), | |
| ("iter15_candidates.jsonl", "pisces", "probes8to16", "fewer probes early ($8\\to16$)", 15), | |
| ("iter17_hnsw.jsonl", "pisces", "h16_e16_32_r8", "HNSW over the centroids instead of IVF", 17), | |
| ("iter18_fp16_polish.jsonl", "pisces", "fp16", "fp16 IVF lists (same recall, faster scans)", 18), | |
| ("iter19_anneal.jsonl", "pol8", "an01", "annealed assignment to the runner-up ($\\varepsilon_0=0.01$)", 19), | |
| ("iter21_halfmean_batch.jsonl", "pisces", "half_half", "distribution-free half-mean split and score, $c\\pm\\E[\\mathrm{sign}(p)\\,e]$", 21), | |
| ("iter21_halfmean_batch.jsonl", "pisces", "abs_abs", "empirical axis half-mean $c\\pm\\E|p|\\,u_1$, score $N(\\E|p|)^2$", 21), | |
| ("iter21_halfmean_batch.jsonl", "pisces", "bs0.25to1.75", "batch-size schedule $0.25B\\to1.75B$", 21)] | |
| with open(os.path.join(OUT, "negatives.tex"), "w") as f: | |
| f.write("\\def\\negrows{%\n") | |
| for fn, ref, cand, label, it in NEG: | |
| if not os.path.exists(os.path.join(ROOT, "results", "dev", fn)): | |
| continue | |
| g = dev(fn); T = 3.0 | |
| o, r_, c_ = g[("official", T)], g[(ref, T)], g[(cand, T)] | |
| s_ = sorted(set(o) & set(r_) & set(c_)) | |
| dr = np.mean([100 * (r_[i] / o[i] - 1) for i in s_]); dc = np.mean([100 * (c_[i] / o[i] - 1) for i in s_]) | |
| worse = sum(c_[i] > r_[i] for i in s_) | |
| f.write(f"{label} & {it} & ${dr:+.2f}\\%$ & ${dc:+.2f}\\%$ & {worse}/{len(s_)}\\\\\n") | |
| f.write("}\n") | |
| # 13. optional closing polish (full scale, validation rows, seeds 1-5) | |
| pol = table(R("deep1m_polish_time_to_quality_val.jsonl")) | |
| for T, tag in [(1.0, "One"), (2.0, "Two"), (3.0, "Three"), (4.0, "Four")]: | |
| s_, d, _ = paired(pol, "pisces_polish", T, base=("pisces", None)) | |
| if len(s_): | |
| M(f"pol{tag}", d.mean(), "{:+.2f}"); M(f"polW{tag}", int((d < 0).sum()), "{}") | |
| # 14. data for the analytic figures | |
| # 14a. per-seed removal costs for the leave-one-out bar chart | |
| with open(os.path.join(OUT, "ablation_seeds.dat") if abl else os.devnull, "w") as f: | |
| f.write("idx cost\n") | |
| for i, (m, tag) in enumerate(ABL): | |
| if m == "pisces": | |
| continue | |
| s_, d2, _ = paired(abl, m, 3.0, base=("pisces", None)) | |
| f.write("".join(f"{i} {v:.4f}\n" for v in d2)) | |
| # 14b. real clusters for the split-geometry figure | |
| if os.path.exists(mf): | |
| for i, ex in enumerate(mech["pisces"].get("split_examples", [])[:3]): | |
| with open(os.path.join(OUT, f"splitex{i}_pts.dat"), "w") as f: | |
| f.write("a b\n" + "".join(f"{p_[0]:.4f} {p_[1]:.4f}\n" for p_ in ex["points"])) | |
| for kind, key in [("pisces", "pisces"), ("kmeans", "kmeans2")]: | |
| with open(os.path.join(OUT, f"splitex{i}_{kind}.dat"), "w") as f: | |
| f.write("a b\n" + "".join(f"{p_[0]:.4f} {p_[1]:.4f}\n" for p_ in ex[key])) | |
| M(f"splitexN{'ABC'[i]}", ex["N"], "{:,}") | |
| M(f"splitexR{'ABC'[i]}", ex["lam2"] / ex["lam1"], "{:.2f}") | |
| # 14c. time breakdown | |
| tb = os.path.join(ROOT, "results", "final", "time_breakdown.json") | |
| if os.path.exists(tb): | |
| T_ = json.load(open(tb)) | |
| with open(os.path.join(OUT, "time_breakdown.dat"), "w") as f: | |
| f.write("idx name search build stats other wall\n") | |
| for i, (m, nm) in enumerate([("official", "e-means"), ("pisces_v10", "PISCES-v1.0"), ("pisces", "PISCES")]): | |
| r = T_[m] | |
| f.write(f"{i} {nm} {r['assignment search']:.2f} {r['index build']:.2f} {r['statistics']:.2f} {r['other']:.2f} {r['wall']:.2f}\n") | |
| M(f"tb{'OVP'[i]}Wall", r["wall"], "{:.0f}"); M(f"tb{'OVP'[i]}Search", 100 * r["assignment search"] / r["wall"], "{:.0f}") | |
| open(os.path.join(ROOT, "paper", "macros.tex"), "w").write("\n".join(macros) + "\n") | |
| print(f"wrote {len(macros)} macros and data files to paper/data") | |