File size: 3,235 Bytes
e8fa066 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Summarise the encoder benchmark (runs/bench/*/test_embeddings.pt).
- per-modality encoder: he<->m retrieval (trained pair)
- cross-modal (never trained together): if<->markers, if<->text, markers<->text for every encoder combination
- zero-shot baselines (no training): PLIP image<->PLIP text, ImageBind image<->ImageBind text
Metrics are tie-aware (duplicate template texts): expected rank under random tie-breaking.
Outputs results/bench_*.csv
"""
import glob
import itertools
import json
import os
import numpy as np
import pandas as pd
import torch
import torch.nn.functional as F
from common import CACHE, ROOT, RUNS, metrics
OUT = os.path.join(ROOT, "results")
runs = {}
for f in glob.glob(os.path.join(RUNS, "bench", "*", "test_embeddings.pt")):
a = json.load(open(os.path.join(os.path.dirname(f), "train.json")))["args"]
runs[(a["split"], a["fold"], a["if_enc"], a["mk_enc"], a["txt_enc"])] = torch.load(f, weights_only=False) # our own files (contain numpy tile ids)
rows = []
folds = sorted({k[:2] for k in runs})
for split, fold in folds:
rs = {k[2:]: v for k, v in runs.items() if k[:2] == (split, fold)}
he = next(iter(rs.values()))["z"]["he"]
te = next(iter(rs.values()))["tile_id"]
by = {"if": {}, "markers": {}, "text": {}}
for (ie, me, txe), v in rs.items():
by["if"][ie], by["markers"][me], by["text"][txe] = v["z"]["if"], v["z"]["markers"], v["z"]["text"]
# trained pairs
for m, encs in by.items():
for e, z in encs.items():
for q, g, name in [(he, z, f"he->{m}"), (z, he, f"{m}->he")]:
rows.append({"split": split, "fold": fold, "kind": "trained", "pair": name, "enc": e, **metrics(q, g)})
# cross pairs: all combinations
for m1, m2 in [("if", "markers"), ("if", "text"), ("markers", "text")]:
for e1, e2 in itertools.product(by[m1], by[m2]):
for q, g, name in [(by[m1][e1], by[m2][e2], f"{m1}->{m2}"), (by[m2][e2], by[m1][e1], f"{m2}->{m1}")]:
rows.append({"split": split, "fold": fold, "kind": "cross", "pair": name, "enc": f"{e1}+{e2}", **metrics(q, g)})
# zero-shot baselines (variant-0 text, untransformed H&E view)
for fm in ["plip", "imagebind"]:
hz = F.normalize(torch.from_numpy(np.load(os.path.join(CACHE, f"he_{fm}.npy"))[te, 0].astype(np.float32)), dim=-1)
tz = F.normalize(torch.from_numpy(np.load(os.path.join(CACHE, f"text_{fm}.npy"))[te, 0].astype(np.float32)), dim=-1)
for q, g, name in [(hz, tz, "he->text"), (tz, hz, "text->he")]:
rows.append({"split": split, "fold": fold, "kind": "zeroshot", "pair": name, "enc": f"{fm} (zero-shot)", **metrics(q, g)})
df = pd.DataFrame(rows)
df.to_csv(os.path.join(OUT, "bench_long.csv"), index=False)
met = ["R@1", "R@5", "MRR", "pair_acc", "R@1_g100"]
grp = df.assign(split=df.split.replace({"patient_5fold": "patient_5fold"}))
agg = grp.groupby(["kind", "pair", "enc", "split"])[met].agg(["mean", "std"]).round(3)
agg.to_csv(os.path.join(OUT, "bench_summary.csv"))
pd.set_option("display.width", 250); pd.set_option("display.max_rows", 500)
show = grp.groupby(["kind", "pair", "enc", "split"])[["R@1_g100", "pair_acc"]].mean().unstack("split").round(3)
print(show.to_string())
|