scripts/final_summary.py

"""Readout-matched comparison and the campaign's final result record.

Everything above used the frozen probe on the MAE side, which is what EAT's own
proxy ranking used and so is the like-for-like comparison against +0.630. This
adds the arm the user asked for: the MAE side scored with the same AS-20K LoRA
readout at 3 pinned seeds that produced the EAT base ground truth, so proxy and
target differ in objective and scale but not in how they are read out.
"""
import csv
import glob
import json
import statistics as st
from pathlib import Path

import numpy as np
from scipy import stats

spec = json.loads(Path("/workspace/analysis/canonical64.json").read_text())
curve = json.loads(Path("/workspace/analysis/transfer_curve.json").read_text())
lora = json.loads(Path("/workspace/runs/mae-64/lora_summary.json").read_text())
final = max(r["step"] for r in curve)

eat_proxy = {}
for r in csv.DictReader(open("/workspace/analysis/eat-map-regmix/tables/runs.csv")):
    if r.get("campaign") != "regmix256":
        continue
    try:
        if float(r["pretrain_seed"]) == 0.0:
            eat_proxy[int(float(r["dist_id"]))] = float(r["as20k_map"])
    except (ValueError, KeyError):
        pass

ART = "/workspace/artifacts/audio-mixture-scaling/artifacts/audioset-dasheng-0.6b-k20-r31d6389"
train_prop = np.array(json.loads(
    Path("/workspace/data/asmel_flat/train.cluster_index.json").read_text())["proportions"])
ev = np.load(f"{ART}/test_cluster_index.npy")
delta = np.bincount(ev, minlength=20) / len(ev) - train_prop

dids = sorted(d for d in (int(v["dist_id"]) for v in spec.values())
              if d in eat_proxy and f"d{d}-s0" in lora)
base = np.array([spec[str(d)]["lora_mean"] for d in dids])
eatp = np.array([eat_proxy[d] for d in dids])
mae_probe = np.array([json.loads(Path(glob.glob(
    f"/workspace/runs/mae-64/d{d}-s0/exports/step_{final:08d}/probe.json")[0]
).read_text())["probe/map"] for d in dids])
mae_lora = np.array([lora[f"d{d}-s0"]["mean"] for d in dids])
lora_sd = np.array([lora[f"d{d}-s0"]["sd"] for d in dids])
em = np.array([float(np.array(spec[str(d)]["weights"]) @ delta) for d in dids])


def rho(a, b):
    return stats.spearmanr(a, b).statistic


def z(x):
    return (x - x.mean()) / x.std(ddof=1)


sigma_ft = float(np.sqrt(np.mean(lora_sd**2)))
between = float(mae_lora.std(ddof=1))
rel_lora = (between**2 - (sigma_ft / np.sqrt(3))**2) / between**2

print(f"n = {len(dids)} arms\n")
print("MAE-side LoRA readout (4 epochs, lr 3e-3, seeds 0/1/2 pinned):")
print(f"  mean {mae_lora.mean():.5f}   between-arm sd {between:.5f}")
print(f"  sigma_ft (pooled within arm)  {sigma_ft:.5f}   "
      f"SE of 3-seed mean {sigma_ft/np.sqrt(3):.5f}")
print(f"  reliability of the 3-seed mean {rel_lora:.3f}")
print(f"  max within-arm sd {lora_sd.max():.5f}  (collapse screen threshold 0.02)")

print(f"\n{'predictor -> EAT base (LoRA x3)':<38}{'rho':>9}")
rows = {
    "EAT proxy, frozen probe": eatp,
    "MAE proxy, frozen probe": mae_probe,
    "MAE proxy, LoRA x3 (readout-matched)": mae_lora,
    "eval-match (no training)": em,
    "EAT probe + MAE LoRA": z(eatp) + z(mae_lora),
    "EAT probe + MAE LoRA + eval-match": z(eatp) + z(mae_lora) + z(em),
}
for name, v in rows.items():
    print(f"{name:<38}{rho(v, base):>+9.3f}")

print(f"\nMAE probe vs MAE LoRA agreement: {rho(mae_probe, mae_lora):+.3f}")
print(f"EAT proxy vs MAE LoRA agreement : {rho(eatp, mae_lora):+.3f}")

rng = np.random.default_rng(0)
d = np.array([rho(mae_lora[i], base[i]) - rho(eatp[i], base[i])
              for i in (rng.integers(0, len(dids), len(dids)) for _ in range(4000))])
print(f"\nDelta rho (MAE LoRA - EAT probe): {rho(mae_lora, base)-rho(eatp, base):+.3f}   "
      f"95% CI [{np.percentile(d,2.5):+.3f}, {np.percentile(d,97.5):+.3f}]   "
      f"P(>0) = {(d>0).mean():.3f}")

out = {
    "n_arms": len(dids),
    "rho": {k: float(rho(v, base)) for k, v in rows.items()},
    "mae_lora": {"mean": float(mae_lora.mean()), "between_sd": between,
                 "sigma_ft": sigma_ft, "reliability_3seed": float(rel_lora)},
    "agreement": {"mae_probe_vs_mae_lora": float(rho(mae_probe, mae_lora)),
                  "eat_vs_mae_lora": float(rho(eatp, mae_lora))},
    "recipe": {"objective": "mae", "scale": "15m", "patch": "16x16", "lr": 5e-4,
               "budget_clips": 1912024, "mask_prob": 0.75, "clone_batch": 1,
               "compile": False, "readout_lora_epochs": 4, "readout_lora_lr": 3e-3,
               "node": "4x RTX 3090", "floor_p16": 0.03861},
}
Path("/workspace/analysis/FINAL_RESULT.json").write_text(json.dumps(out, indent=1))
print("\nwrote /workspace/analysis/FINAL_RESULT.json")