| """Summarise the encoder benchmark (runs/bench/*/test_embeddings.pt). |
| |
| - per-modality encoder: he<->m retrieval (trained pair) |
| - cross-modal (never trained together): if<->markers, if<->text, markers<->text for every encoder combination |
| - zero-shot baselines (no training): PLIP image<->PLIP text, ImageBind image<->ImageBind text |
| Metrics are tie-aware (duplicate template texts): expected rank under random tie-breaking. |
| Outputs results/bench_*.csv |
| """ |
| import glob |
| import itertools |
| import json |
| import os |
|
|
| import numpy as np |
| import pandas as pd |
| import torch |
| import torch.nn.functional as F |
|
|
| from common import CACHE, ROOT, RUNS, metrics |
|
|
| OUT = os.path.join(ROOT, "results") |
|
|
|
|
| runs = {} |
| for f in glob.glob(os.path.join(RUNS, "bench", "*", "test_embeddings.pt")): |
| a = json.load(open(os.path.join(os.path.dirname(f), "train.json")))["args"] |
| runs[(a["split"], a["fold"], a["if_enc"], a["mk_enc"], a["txt_enc"])] = torch.load(f, weights_only=False) |
|
|
| rows = [] |
| folds = sorted({k[:2] for k in runs}) |
| for split, fold in folds: |
| rs = {k[2:]: v for k, v in runs.items() if k[:2] == (split, fold)} |
| he = next(iter(rs.values()))["z"]["he"] |
| te = next(iter(rs.values()))["tile_id"] |
| by = {"if": {}, "markers": {}, "text": {}} |
| for (ie, me, txe), v in rs.items(): |
| by["if"][ie], by["markers"][me], by["text"][txe] = v["z"]["if"], v["z"]["markers"], v["z"]["text"] |
| |
| for m, encs in by.items(): |
| for e, z in encs.items(): |
| for q, g, name in [(he, z, f"he->{m}"), (z, he, f"{m}->he")]: |
| rows.append({"split": split, "fold": fold, "kind": "trained", "pair": name, "enc": e, **metrics(q, g)}) |
| |
| for m1, m2 in [("if", "markers"), ("if", "text"), ("markers", "text")]: |
| for e1, e2 in itertools.product(by[m1], by[m2]): |
| for q, g, name in [(by[m1][e1], by[m2][e2], f"{m1}->{m2}"), (by[m2][e2], by[m1][e1], f"{m2}->{m1}")]: |
| rows.append({"split": split, "fold": fold, "kind": "cross", "pair": name, "enc": f"{e1}+{e2}", **metrics(q, g)}) |
| |
| for fm in ["plip", "imagebind"]: |
| hz = F.normalize(torch.from_numpy(np.load(os.path.join(CACHE, f"he_{fm}.npy"))[te, 0].astype(np.float32)), dim=-1) |
| tz = F.normalize(torch.from_numpy(np.load(os.path.join(CACHE, f"text_{fm}.npy"))[te, 0].astype(np.float32)), dim=-1) |
| for q, g, name in [(hz, tz, "he->text"), (tz, hz, "text->he")]: |
| rows.append({"split": split, "fold": fold, "kind": "zeroshot", "pair": name, "enc": f"{fm} (zero-shot)", **metrics(q, g)}) |
|
|
| df = pd.DataFrame(rows) |
| df.to_csv(os.path.join(OUT, "bench_long.csv"), index=False) |
| met = ["R@1", "R@5", "MRR", "pair_acc", "R@1_g100"] |
| grp = df.assign(split=df.split.replace({"patient_5fold": "patient_5fold"})) |
| agg = grp.groupby(["kind", "pair", "enc", "split"])[met].agg(["mean", "std"]).round(3) |
| agg.to_csv(os.path.join(OUT, "bench_summary.csv")) |
| pd.set_option("display.width", 250); pd.set_option("display.max_rows", 500) |
| show = grp.groupby(["kind", "pair", "enc", "split"])[["R@1_g100", "pair_acc"]].mean().unstack("split").round(3) |
| print(show.to_string()) |
|
|
