File size: 6,371 Bytes
2188a91 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | """Claim 6: summarise the k x mask-ratio sweep on PTB-XL and compare with Table 3."""
import json
import os
import re
import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots
RES = "results/sweep"
KS, RS = [6, 8, 10], [0.5, 0.6, 0.7, 0.8]
# Table 3 of the paper (F1 column), as percentages
PAPER_F1 = {(6, 0.5): 54.81, (6, 0.6): 56.68, (6, 0.7): 55.41, (6, 0.8): 54.32,
(8, 0.5): 62.60, (8, 0.6): 63.51, (8, 0.7): 62.10, (8, 0.8): 59.19,
(10, 0.5): 51.76, (10, 0.6): 50.70, (10, 0.7): 56.33, (10, 0.8): 55.70}
PAPER_ACC = {(6, 0.5): 70.19, (6, 0.6): 70.71, (6, 0.7): 70.63, (6, 0.8): 69.77,
(8, 0.5): 74.07, (8, 0.6): 73.98, (8, 0.7): 73.86, (8, 0.8): 72.35,
(10, 0.5): 67.36, (10, 0.6): 66.41, (10, 0.7): 71.07, (10, 0.8): 70.42}
def main():
rows = []
for k in KS:
for r in RS:
p = os.path.join(RES, f"ptbxl_k{k}_r{r}.json")
if not (os.path.exists(p) and os.path.getsize(p) > 0):
continue
d = json.load(open(p))
a = d["aggregate"]
rows.append({"k": k, "r": r,
"f1": a["f1"]["mean"], "f1_sd": a["f1"]["std"],
"acc": a["accuracy"]["mean"], "acc_sd": a["accuracy"]["std"],
"auroc": a["auroc"]["mean"],
"n_seeds": len(d["runs"]),
"paper_f1": PAPER_F1[(k, r)], "paper_acc": PAPER_ACC[(k, r)],
"sec": float(np.mean([x["seconds"] for x in d["runs"]]))})
if not rows:
print("no sweep results yet")
return
print(f"{'k':>3}{'r':>6}{'F1 ours':>12}{'F1 paper':>11}{'Acc ours':>11}"
f"{'Acc paper':>11}{'seeds':>7}")
for x in rows:
print(f"{x['k']:>3}{x['r']:>6}{x['f1']:>9.2f}±{x['f1_sd']:<3.1f}"
f"{x['paper_f1']:>11.2f}{x['acc']:>8.2f}±{x['acc_sd']:<3.1f}"
f"{x['paper_acc']:>11.2f}{x['n_seeds']:>7}")
out = {"rows": rows}
if len(rows) == 12:
f1 = {(x["k"], x["r"]): x["f1"] for x in rows}
best = max(f1, key=f1.get)
out["best_cell_ours"] = {"k": best[0], "r": best[1], "f1": f1[best]}
out["best_cell_paper"] = {"k": 8, "r": 0.6, "f1": 63.51}
# marginal effect of k at each r, and of r at each k
out["best_k_per_r"] = {str(r): max(KS, key=lambda k: f1[(k, r)]) for r in RS}
out["best_r_per_k"] = {str(k): max(RS, key=lambda r: f1[(k, r)]) for k in KS}
out["mean_f1_by_k"] = {str(k): float(np.mean([f1[(k, r)] for r in RS]))
for k in KS}
out["mean_f1_by_r"] = {str(r): float(np.mean([f1[(k, r)] for k in KS]))
for r in RS}
out["paper_mean_f1_by_k"] = {
str(k): float(np.mean([PAPER_F1[(k, r)] for r in RS])) for k in KS}
out["paper_mean_f1_by_r"] = {
str(r): float(np.mean([PAPER_F1[(k, r)] for k in KS])) for r in RS}
# does r>=0.8 degrade? compare r=0.8 against the mean of r in {0.5,0.6,0.7}
deg = {}
for k in KS:
lo = np.mean([f1[(k, r)] for r in (0.5, 0.6, 0.7)])
deg[str(k)] = {"mean_r_le_0.7": float(lo), "r_0.8": f1[(k, 0.8)],
"delta": float(f1[(k, 0.8)] - lo)}
out["r08_degradation"] = deg
# rank correlation between our grid and the paper's grid
ours_v = np.array([f1[(k, r)] for k in KS for r in RS])
pap_v = np.array([PAPER_F1[(k, r)] for k in KS for r in RS])
out["pearson_ours_vs_paper_grid"] = float(np.corrcoef(ours_v, pap_v)[0, 1])
out["spearman_ours_vs_paper_grid"] = float(np.corrcoef(
np.argsort(np.argsort(ours_v)), np.argsort(np.argsort(pap_v)))[0, 1])
print("\nbest cell (ours):", out["best_cell_ours"])
print("mean F1 by k (ours):", {k: round(v, 2)
for k, v in out["mean_f1_by_k"].items()})
print("mean F1 by k (paper):", {k: round(v, 2)
for k, v in out["paper_mean_f1_by_k"].items()})
print("mean F1 by r (ours):", {k: round(v, 2)
for k, v in out["mean_f1_by_r"].items()})
print("mean F1 by r (paper):", {k: round(v, 2)
for k, v in out["paper_mean_f1_by_r"].items()})
print("r=0.8 vs r<=0.7:", {k: round(v["delta"], 2)
for k, v in deg.items()})
print(f"grid correlation with Table 3: Pearson "
f"{out['pearson_ours_vs_paper_grid']:.3f}, "
f"Spearman {out['spearman_ours_vs_paper_grid']:.3f}")
json.dump(out, open("results/sweep_summary.json", "w"), indent=2)
# interactive figure for the logbook
ours = np.full((3, 4), np.nan)
for x in rows:
ours[KS.index(x["k"]), RS.index(x["r"])] = x["f1"]
paper = np.array([[PAPER_F1[(k, r)] for r in RS] for k in KS])
fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.13,
subplot_titles=("this reproduction (PTB-XL, thinned)",
"paper (Table 3)"))
for j, m in enumerate((ours, paper)):
fig.add_heatmap(z=m, x=[f"r={r}" for r in RS], y=[f"k={k}" for k in KS],
colorscale=[[0, "#F2F7F8"], [1, "#17697B"]], showscale=False,
text=[[("" if np.isnan(v) else f"{v:.2f}") for v in row]
for row in m],
texttemplate="%{text}", row=1, col=j + 1)
fig.update_layout(template="plotly_white", height=420,
title="Claim 6 — macro F1 (%) over bottleneck size k and mask ratio r")
fig.write_html("results/claim6_sweep.html", include_plotlyjs="cdn")
with open("results/sweep_results.csv", "w") as f:
f.write("k,mask_ratio,n_seeds,f1,f1_sd,acc,acc_sd,auroc,paper_f1,paper_acc,sec\n")
for x in rows:
f.write(f"{x['k']},{x['r']},{x['n_seeds']},{x['f1']:.3f},{x['f1_sd']:.3f},"
f"{x['acc']:.3f},{x['acc_sd']:.3f},{x['auroc']:.3f},"
f"{x['paper_f1']},{x['paper_acc']},{x['sec']:.0f}\n")
print("\nwrote results/sweep_summary.json, sweep_results.csv, claim6_sweep.html")
if __name__ == "__main__":
main()
|