File size: 2,323 Bytes
c2ede1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Compare a retraining (reproduce.sh) with the shipped results.
python -I compare_reproduction.py [TAG=repro]
fold 0 : models/<TAG>_fold0/test.csv vs models/ej_ad_mv_fold0/test.csv
5-fold : results/final_cv_long_<TAG>.csv vs results/final_cv_long.csv (rule ens-es and stage 1, mean over folds)
Stage 1 (mIF / RNA / text heads) is seeded and deterministic on the same GPU / library stack, so the emergent pairs
should agree to ~1e-3; stage 2 (H&E adapter, 3 early-stopped seeds) is not bit-deterministic on GPU, so the trained
H&E pairs agree within the run-to-run spread (5-fold seed std of a single adapter: mIF 0.009, text 0.011).
Tolerances: |diff| <= 0.01 (emergent / stage 1), <= 0.03 (fold-0 H&E pairs), <= 0.015 (5-fold means of H&E pairs).
"""
import os
import sys
import pandas as pd
ROOT = os.path.dirname(os.path.abspath(__file__))
TAG = sys.argv[1] if len(sys.argv) > 1 else "repro"
pd.set_option("display.width", 200)
ok = True
def show(D, tol_trained, title):
global ok
D["abs_diff"] = (D.reproduced - D.shipped).abs()
D["tol"] = [tol_trained if p.startswith("H&E") and r != "stage1" else 0.01 for p, r in zip(D.pair, D.get("rule", [""] * len(D)))]
D["ok"] = D.abs_diff <= D.tol
ok &= bool(D.ok.all())
print(f"\n{title}\n{D.round(4).to_string(index=False)}")
f0 = os.path.join(ROOT, "models", f"{TAG}_fold0", "test.csv")
if os.path.exists(f0):
a = pd.read_csv(f0).set_index("pair")["R@1"]
b = pd.read_csv(os.path.join(ROOT, "models", "ej_ad_mv_fold0", "test.csv")).set_index("pair")["R@1"]
show(pd.DataFrame({"pair": a.index, "reproduced": a.values, "shipped": b.loc[a.index].values}), 0.03,
"fold 0, held-out test R@1 (g100)")
cv = os.path.join(ROOT, "results", f"final_cv_long_{TAG}.csv")
if os.path.exists(cv):
m = lambda f: pd.read_csv(f).query("rule in ['stage1', 'ens-es']").groupby(["rule", "pair"], sort=False)["R@1"].mean()
a, b = m(cv), m(os.path.join(ROOT, "results", "final_cv_long.csv"))
show(pd.DataFrame({"rule": [r for r, _ in a.index], "pair": [p for _, p in a.index], "reproduced": a.values,
"shipped": b.loc[a.index].values}), 0.015, "5-fold mean test R@1 (g100)")
print("\nREPRODUCED within tolerance" if ok else "\nNOT reproduced within tolerance (see ok=False rows)")
sys.exit(0 if ok else 1)
|