repro-bundle-v2/repro/dump_numbers.py

"""Dump every number the logbook pages quote, so no value is ever typed by hand."""
import json
import os

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
V2 = os.path.join(ROOT, "results", "v2")
a = json.load(open(os.path.join(V2, "analysis_v2.json")))
DS = ["moons-hard", "rings", "gauss-xor", "digits-3v8"]

print("### INTEGRITY")
tot = 0.0
for ds in DS:
    i = a["integrity"][ds]
    tot += i["wall_time_sec"]
    print(f"{ds}: n_tr={i['n_train']} n_te={i['n_test']} d={i['d']} "
          f"seeds={i['n_seeds']} uniq_hash={i['unique_data_hashes']} "
          f"uniq_c1hinge={i['unique_cycle1_hinge']} wall={i['wall_time_sec']:.1f}s")
    print("   acc_std:", {k: round(v, 5) for k, v in i["acc_std_by_method"].items()})
print(f"TOTAL WALL = {tot:.1f}s = {tot/60:.1f} min")

print("\n### CLAIM 1")
c1 = a["claims"]["claim1"]
print("verdict", c1["verdict"], c1["n_pass"], "/", c1["n_checks"])
for ds in DS:
    d = c1["per_dataset"][ds]
    A, B, C = d["P1a"], d["P1b"], d["P1c"]
    print(f"\n{ds}")
    print(f"  P1a pass={A['pass']} max_head_drift_A={A['max_head_drift_in_phaseA']:.3e} "
          f"max_feat_drift_B={A['max_feat_drift_in_phaseB']:.3e} "
          f"min_feat_move_A={A['min_feat_move_in_phaseA']:.4f} "
          f"min_head_move_B={A['min_head_move_in_phaseB']:.4f} "
          f"min_disp_C={A['min_mean_disp_phaseC']:.4f} n_obs={A['n_cycle_observations']}")
    print(f"  P1b pass={B['pass']} ratio={B['ratio_mean']:.4f} "
          f"[{B['ratio_lo']:.4f},{B['ratio_hi']:.4f}] "
          f"final_hinge={B['final_hinge_mean']:.6f}+/-{B['final_hinge_ci_hw']:.6f}")
    print(f"  P1c pass={C['pass']} slope={C['slope']:.4f} R2={C['r2']:.4f} "
          f"degenerate={C['degenerate_exact_zero_hinge']} "
          f"zeros={C['n_exact_zero_observations']}/{C['n_observations']} "
          f"first_zero_cycle={C['first_zero_cycle']}")
    print(f"     curve={[round(v,5) for v in C['mean_curve']]}")

print("\n### CLAIM 2")
c2 = a["claims"]["claim2"]
print("verdict", c2["verdict"], c2["n_pass"], "/", c2["n_checks"])
for ds in DS:
    d = c2["per_dataset"][ds]
    A, B, C, D = d["P2a"], d["P2b"], d["P2c"], d["P2d"]
    ph = A["posthoc_cos_vs_random_paired"]
    print(f"\n{ds}")
    print(f"  P2a pass={A['pass']} cos={A['mean']:.4f} [{A['lo']:.4f},{A['hi']:.4f}] "
          f"| random-ctrl cos={A['control_random_push_cos_mean']:.4f}"
          f"+/-{A['control_random_push_cos_ci_hw']:.4f}")
    print(f"      posthoc paired cos(normal)-cos(random)={ph['mean']:+.4f} "
          f"[{ph['lo']:+.4f},{ph['hi']:+.4f}] p_t={ph['p_ttest']:.2e} "
          f"p_w={ph['p_wilcoxon']:.2e}")
    print(f"  P2b pass={B['pass']} d_active_margin={B['mean']:+.4f} "
          f"[{B['lo']:+.4f},{B['hi']:+.4f}]")
    print(f"  P2c pass={C['pass']} growth={C['growth_mean']:+.4f} "
          f"[{C['growth_lo']:+.4f},{C['growth_hi']:+.4f}] "
          f"viol_down_frac={C['frac_seeds_violators_down']:.2f}")
    print(f"  P2d pass={D['pass']} dNormMargin(ddsvm-rand)={D['mean']:+.5f} "
          f"[{D['lo']:+.5f},{D['hi']:+.5f}] p_t={D['p_ttest']:.4f}")
    print(f"     active_frac c1={d['active_frac_curve'][0]*100:.2f}% "
          f"c15={d['active_frac_curve'][-1]*100:.2f}% | "
          f"violators c1={d['violators_curve'][0]:.1f} c15={d['violators_curve'][-1]:.1f}")
    print(f"     margin pre c1={d['margin_pre_curve'][0]:.4f} -> "
          f"post c15={d['margin_post_curve'][-1]:.4f}")

print("\n### CLAIM 3")
c3 = a["claims"]["claim3"]
print("verdict", c3["verdict"], "both:", c3["n_datasets_both_baselines_beaten"],
      "any:", c3["n_datasets_any_baseline_beaten"],
      "regress:", c3["n_datasets_regression"])
for ds in DS:
    acc = a["datasets"][ds]["acc"]
    print(f"\n{ds}")
    for m, v in acc.items():
        print(f"   {m:12s} {v['mean']*100:6.2f} +/- {v['ci_hw']*100:.2f}")
    for k, v in a["datasets"][ds]["paired"].items():
        print(f"   {k:22s} {v['mean']*100:+7.3f}pp "
              f"[{v['lo']*100:+.3f},{v['hi']*100:+.3f}] "
              f"p_t={v['p_ttest']:.4f} p_w={v['p_wilcoxon']:.4f} "
              f"CI_excl0={v['ci_excludes_zero']}")
    mp = a["datasets"][ds]["margin_paired"]
    nm = a["datasets"][ds]["test_norm_margin"]
    print("   test norm margin:", {k: round(v["mean"], 5) for k, v in nm.items()})
    for k, v in mp.items():
        print(f"   margin {k:22s} {v['mean']:+.5f} [{v['lo']:+.5f},{v['hi']:+.5f}] "
              f"p_t={v['p_ttest']:.4f}")