Migrate release files 181-210

experiments/26_stpath_infer.py

@@ -0,0 +1,427 @@
+"""STPath (Huang et al. 2025) zero-shot spatial gene-expression prediction from H&E GigaPath features.
+
+Source   code    https://github.com/Graph-and-Geometric-Learning/STPath  (commit 3b10e6d, no LICENSE file)
+         weights https://huggingface.co/tlhuang/STPath  stfm.pth  (no licence declared; not gated)
+         local copy: ext/stpath/{repo,weights}   -- treated as untrusted: state dict loaded with weights_only=True,
+         and only the pure-torch model modules (model.py, spatial_transformer.py, fa.py, nn_utils) are imported.
+Model    STFM spatial transformer: per-spot input = Linear(GigaPath tile feature 1536) + masked gene token
+         + organ token + tech token; attention over all spots of a slide with a 2D frame-averaged distance bias.
+         Coordinates are shifted to 0 and rescaled to [0,100] per slide (official STPathInference._normalize_coords).
+         Output: 38,984 Ensembl-gene vector per spot on the training scale log1p(raw UMI count) (NOT library-normalised;
+         training used normalize_method='log1pv2' = sc.pp.log1p on raw counts).
+Encoder  The released checkpoint was trained on prov-gigapath tile features ONLY (official pipeline: 224-px HEST patches
+         @ 0.5 um/px, CenterCrop 224, ImageNet norm).  prov-gigapath is gated: --extract needs HF_TOKEN in the env
+         (pass it as an env var only; this script never reads, prints or stores it).
+         Our features = mean of GigaPath(fp16) over the centre + 4 corner 224-px crops at ~0.5 um/px
+         (centre-only saved too as <set>_gigapath_center.npy):
+           ORION  333-px tiles (512-px @ 0.325 um/px resized -> ~0.49 um/px), cropped at native size.
+           TCGA   672-px PathGen patches = read_region level 0; visually 40x (RBC ~28-30 px -> ~0.25 um/px),
+                  so resized 672 -> 336 px (~0.5 um/px) before cropping.  (20x-scanned slides, if any, would be off 2x.)
+
+Sets
+  int2_check  repo example (HEST INT2, CCRCC, full slide 2541 spots) -> reproduces README mean Pearson 0.1562;
+              also INT2 in random 26-spot groups (sparse context, like our 1% HEST subsample) and single spots.
+  hest_val    cache/hest (9484 Visium spots, 1% subsample of 421 HEST samples, precomputed GigaPath features),
+              one STPath pass per sample (all its spots as context), organ = HEST organ, tech = Visium.
+              Samples in HEST-bench were excluded from STPath pretraining -> reported separately as held-out.
+  orion       ORION CRC tiles (common.load_all_tiles, 3187 tiles / 41 slides), one pass per slide with coords from
+              the filename x_y (level-0 px), organ = Bowel, tech = Visium.  + protein sanity check
+              (orion_protein_check.csv: predicted RNA vs <marker>_count/n_nuclei, tiles with >= 20 nuclei).
+  tcga_text   cache/allimg_meta.csv (6144 patches, 4088 WSIs -> mostly 1 patch = no spatial context), one pass per
+              WSI, coords from key x_y; organ from the TCGA project (public GDC files API, cached in
+              tcga_text_projects.csv) via PROJECT2ORGAN, unmapped -> 'Others'; tech = Visium.
+
+Outputs cache/stpath/<set>_pred.npy (N,G) float16, native STPath scale log1p(raw count); <set>_genes.csv; <set>_meta.csv
+        hest_val additionally: hest_val_metrics.csv (per-gene Pearson/Spearman), hest_val_within_sample.csv.
+Run     .venv/bin/python 26_stpath_infer.py --set int2_check|hest_val
+        HF_TOKEN=... .venv/bin/python 26_stpath_infer.py --set orion|tcga_text --extract   # GigaPath features
+        .venv/bin/python 26_stpath_infer.py --set orion|tcga_text [--feats F.npy]          # STPath prediction
+"""
+import argparse
+import json
+import os
+import pickle
+import re
+import sys
+import weakref
+
+import numpy as np
+import pandas as pd
+import torch
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+EXT = os.path.join(ROOT, "ext", "stpath")
+REPO = os.path.join(EXT, "repo")
+WEIGHTS = os.path.join(EXT, "weights", "stfm.pth")
+VOC = os.path.join(REPO, "utils_data", "symbol2ensembl.json")
+CACHE = os.path.join(ROOT, "cache")
+OUT = os.path.join(CACHE, "stpath")
+HEST = os.path.join(ROOT, "..", "HEST")
+MARKERS = ["PTPRC", "EPCAM", "KRT8", "ACTA2", "MKI67", "PECAM1", "CD68", "CD3E", "CDH1", "COL1A1"]
+HEST_BENCH = {  # MahmoodLab/hest-bench adata ids; STPath excluded these from pretraining (train.py)
+    "INT%d" % i for i in range(1, 25)} | {"TENX111", "TENX147", "TENX148", "TENX149", "NCBI642", "NCBI643",
+    "NCBI783", "NCBI785", "TENX95", "TENX99", "TENX118", "TENX141", "NCBI681", "NCBI682", "NCBI683", "NCBI684",
+    "TENX116", "TENX126", "TENX140", "ZEN36", "ZEN40", "ZEN48", "ZEN49", "TENX115", "TENX117"} | {
+    "MEND%d" % i for i in list(range(139, 155)) + list(range(156, 163))}
+
+
+# ----------------------------------------------------------------------------------------------- model
+class STPath:
+    def __init__(self, device):
+        sys.path.insert(0, REPO)
+        from stpath.model.model import STFM
+        from stpath.model.nn_utils.config import ModelConfig
+        import stpath.utils.constants as C
+
+        sym2ens = json.load(open(VOC))
+        ens = sorted(set(sym2ens.values()))
+        self.gene2id = {g: i + 2 for i, g in enumerate(ens)}  # 0 pad, 1 mask (GeneExpTokenizer)
+        self.sym2ens = sym2ens
+        self.n_genes = max(self.gene2id.values()) + 1
+        self.C = C
+        cfg = ModelConfig.get_default_config()
+        cfg.feature_dim, cfg.activation, cfg.backbone = 1536, "gelu", "spatial_transformer"
+        cfg.n_genes, cfg.n_tech, cfg.n_species, cfg.n_organs = self.n_genes, len(C.tech_voc), len(C.specie_voc), len(C.organ_voc)
+        self.model = STFM(cfg)
+        self.model.load_state_dict(torch.load(WEIGHTS, map_location="cpu", weights_only=True))
+        self.model.eval().to(device)
+        self.device = device
+
+    def organ_id(self, organ):  # IDTokenizer.encode(align_first=True); None -> "Others" as in STPathInference
+        organ = "Others" if organ is None or (isinstance(organ, float) and np.isnan(organ)) else organ
+        return self.C.organ_voc.index(self.C.organ_align_mapping.get(organ, "<pad>"))
+
+    def tech_id(self, tech):
+        return 0 if tech is None else self.C.tech_voc.index(self.C.tech_align_mapping.get(tech, "<pad>"))
+
+    def gene_cols(self, symbols):
+        """token column for each symbol (-1 if not in the STPath vocabulary)"""
+        return np.array([self.gene2id.get(self.sym2ens.get(s), -1) for s in symbols])
+
+    @torch.no_grad()
+    def predict(self, feats, coords, organ, tech, cols):
+        """feats (n,1536), coords (n,2) one slide -> (n, len(cols)) log1p(count)-scale, NaN for cols==-1"""
+        n = len(feats)
+        c = torch.as_tensor(np.asarray(coords, np.float64), dtype=torch.float32)
+        c = c - c.min(0).values
+        rng = c.max(0).values
+        c = torch.where(rng > 0, c / rng.clamp(min=1e-8) * 100, torch.zeros_like(c))  # rescale_coords; guard 1-spot/collinear
+        x = torch.as_tensor(np.asarray(feats, np.float32), device=self.device)
+        ge = torch.zeros(n, self.n_genes, device=self.device)
+        ge[:, 1] = 1  # all spots masked (pure image -> expression)
+        p = self.model.prediction_head(
+            img_tokens=x, coords=c.to(self.device), ge_tokens=ge,
+            batch_idx=torch.zeros(n, dtype=torch.long, device=self.device),
+            tech_tokens=torch.full((n,), self.tech_id(tech), dtype=torch.long, device=self.device),
+            organ_tokens=torch.full((n,), self.organ_id(organ), dtype=torch.long, device=self.device))
+        out = p[:, torch.as_tensor(np.clip(cols, 0, None), device=self.device)].float().cpu().numpy()
+        out[:, cols < 0] = np.nan
+        # library size of the predicted profile (all vocab genes) for optional CP10k re-normalisation
+        lib = torch.expm1(p[:, 2:].clamp(min=0)).sum(1).cpu().numpy()
+        return out, lib
+
+
+# ----------------------------------------------------------------------------------------------- metrics
+def corr_cols(a, b):
+    a = a - a.mean(0); b = b - b.mean(0)
+    return (a * b).sum(0) / np.sqrt((a * a).sum(0) * (b * b).sum(0) + 1e-20)
+
+
+def rank_cols(a):
+    from scipy.stats import rankdata
+    return np.apply_along_axis(rankdata, 0, a)
+
+
+def gene_metrics(pred, truth):
+    return corr_cols(pred, truth), corr_cols(rank_cols(pred), rank_cols(truth))
+
+
+# ----------------------------------------------------------------------------------------------- sets
+def run_int2(m):
+    import anndata as ad
+    import h5py
+    ex = os.path.join(REPO, "example_data")
+    hvg = json.load(open(os.path.join(ex, "var_50genes.json")))["genes"]
+    with h5py.File(os.path.join(ex, "INT2.h5")) as f:
+        E, Cc, bc = f["embeddings"][:], f["coords"][:], f["barcodes"][:].flatten().astype(str)
+    a = ad.read_h5ad(os.path.join(ex, "INT2.h5ad"))[bc, :]
+    gt = np.log1p(a[:, hvg].X.toarray())
+    cols = m.gene_cols(hvg)
+    p, _ = m.predict(E, Cc, "Kidney", "Visium", cols)
+    r = corr_cols(p, gt)
+    print(f"INT2 full slide ({len(E)} spots): mean Pearson 50 HVG = {np.nanmean(r):.4f} (README: 0.1562)")
+    rs = []
+    for seed in range(5):  # sparse context like our 1% HEST subsample: 26 random spots per pass
+        idx = np.random.default_rng(seed).permutation(len(E))
+        pp = np.zeros_like(p)
+        for ch in np.array_split(idx, len(E) // 26):
+            pp[ch] = m.predict(E[ch], Cc[ch], "Kidney", "Visium", cols)[0]
+        rs.append(np.nanmean(corr_cols(pp, gt)))
+    print(f"INT2 sparse context (random 26-spot groups, 5 seeds): mean Pearson = {np.mean(rs):.4f} +- {np.std(rs):.4f}")
+    p1 = np.concatenate([m.predict(E[i:i + 1], Cc[i:i + 1], "Kidney", "Visium", cols)[0] for i in range(len(E))])
+    print(f"INT2 no context (each spot alone): mean Pearson = {np.nanmean(corr_cols(p1, gt)):.4f}")
+
+
+def hest_coords():
+    from anndata._core import file_backing
+
+    def _ss(self, state):  # same unpickle patch as 21_hest_prep.py
+        self.__dict__ = state.copy()
+        ref = state.get("_adata_ref", state.get("_adata"))
+        self.__dict__["_adata_ref"] = weakref.ref(ref) if ref is not None else (lambda: None)
+
+    file_backing.AnnDataFileManager.__setstate__ = _ss
+    A = pickle.load(open(os.path.join(HEST, "visium_humandisease_adata.pickle"), "rb"))
+    H = pd.read_csv(os.path.join(HEST, "HEST_v1_1_0.csv"))
+    H = H[(H.st_technology == "Visium") & (H.species == "Homo sapiens")].reset_index(drop=True)
+    assert len(H) == len(A)
+    xy = {}
+    for a, sid in zip(A, H.id):
+        for b, c in zip(a.obs_names, np.asarray(a.obsm["spatial"], np.float64)):
+            xy[(sid, b)] = c
+    return xy
+
+
+def run_hest(m):
+    import scipy.sparse as sp
+    M = pd.read_csv(os.path.join(CACHE, "hest", "meta.csv"))
+    genes = pd.read_csv(os.path.join(CACHE, "hest", "genes.csv")).gene.tolist()
+    X = sp.load_npz(os.path.join(CACHE, "hest", "counts.npz")).tocsr()
+    F = np.load(os.path.join(CACHE, "hest", "he_gigapath.npy"))[:, 0].astype(np.float32)
+    xy = hest_coords()
+    coords = np.stack([xy[(s, b)] for s, b in zip(M["sample"], M.spot)])
+    cols = m.gene_cols(genes)
+    keep = cols >= 0
+    print(f"HEST genes {len(genes)}, in STPath vocab {keep.sum()}")
+    genes_k, cols_k = [g for g, k in zip(genes, keep) if k], cols[keep]
+    pred = np.zeros((len(M), len(cols_k)), np.float32)
+    lib = np.zeros(len(M), np.float32)
+    for sid, idx in M.groupby("sample").indices.items():
+        pred[idx], lib[idx] = m.predict(F[idx], coords[idx], M.organ.iloc[idx[0]], "Visium", cols_k)
+    os.makedirs(OUT, exist_ok=True)
+    np.save(os.path.join(OUT, "hest_val_pred.npy"), pred.astype(np.float16))
+    pd.Series(genes_k).to_csv(os.path.join(OUT, "hest_val_genes.csv"), index=False, header=["gene"])
+    mo = M[["idx", "sample", "spot", "organ", "oncotree"]].copy()
+    mo["x"], mo["y"] = coords[:, 0], coords[:, 1]
+    mo["hest_bench_heldout"] = mo["sample"].isin(HEST_BENCH)
+    mo["pred_lib_expm1"] = lib
+    mo["status"] = "ok"
+    mo.to_csv(os.path.join(OUT, "hest_val_meta.csv"), index=False)
+
+    # ---- evaluation: truth = log1p(CP10k) of measured counts (and log1p raw = STPath's training target)
+    Xk = X[:, np.where(keep)[0]]
+    tot = np.asarray(X.sum(1)).ravel()
+    L = sp.diags(1e4 / tot) @ Xk
+    L.data = np.log1p(L.data)
+    frac = np.asarray((Xk > 0).mean(0)).ravel()
+    mu = np.asarray(L.mean(0)).ravel()
+    var = np.asarray(L.multiply(L).mean(0)).ravel() - mu ** 2
+    var[frac < 0.01] = -1
+    top50 = list(np.argsort(-var)[:50])
+    gi = {g: i for i, g in enumerate(genes_k)}
+    mk = [gi[g] for g in MARKERS if g in gi]
+    sel = top50 + [i for i in mk if i not in top50]
+    T_cp = L[:, sel].toarray()
+    T_raw = np.log1p(Xk[:, sel].toarray())
+    P = pred[:, sel]
+    P_cp = np.log1p(np.expm1(np.clip(P, 0, None)) / lib[:, None] * 1e4)  # predicted profile re-normalised to CP10k
+    held = mo.hest_bench_heldout.values
+    rows = []
+    for subset, msk in [("all", np.ones(len(M), bool)), ("heldout_hestbench", held), ("seen_pretrain", ~held)]:
+        for tname, T, PP in [("log1p_cp10k", T_cp, P), ("log1p_raw", T_raw, P), ("log1p_cp10k_vs_predcp10k", T_cp, P_cp)]:
+            pr, sr = gene_metrics(PP[msk], T[msk])
+            for j, g in enumerate(sel):
+                rows.append({"subset": subset, "truth": tname, "gene": genes_k[g], "is_top50_hvg": g in top50,
+                             "is_marker": genes_k[g] in MARKERS, "pearson": pr[j], "spearman": sr[j],
+                             "n_spots": int(msk.sum())})
+    R = pd.DataFrame(rows)
+    R.to_csv(os.path.join(OUT, "hest_val_metrics.csv"), index=False)
+    # within-sample correlations (spatial-variation within a slide, the HEST-bench style), samples with >=30 spots
+    ws = []
+    for sid, idx in M.groupby("sample").indices.items():
+        if len(idx) >= 30:
+            pr, _ = gene_metrics(P[idx][:, :50], T_cp[idx][:, :50])
+            ws.append({"sample": sid, "heldout": sid in HEST_BENCH, "n": len(idx), "mean_pearson_top50": np.nanmean(pr)})
+    W = pd.DataFrame(ws)
+    W.to_csv(os.path.join(OUT, "hest_val_within_sample.csv"), index=False)
+    print(f"\nspots {len(M)}, samples {M['sample'].nunique()}; held-out (HEST-bench) spots {held.sum()} "
+          f"in {M[held]['sample'].nunique()} samples")
+    s = R.groupby(["subset", "truth", "is_top50_hvg"])[["pearson", "spearman"]].agg(["mean", "median"])
+    print(s.round(3).to_string())
+    print("\nmarkers, truth log1p CP10k (pooled across spots):")
+    print(R[R.is_marker & (R.truth == "log1p_cp10k")].pivot(index="gene", columns="subset", values=["pearson", "spearman"]).round(3).to_string())
+    print("\nwithin-sample mean Pearson top-50 HVG (samples >=30 spots):")
+    print(W.groupby("heldout").mean_pearson_top50.describe().round(3).to_string())
+
+
+def parse_xy(s):
+    mm = re.search(r"_(\d+)_(\d+)(?:_0_512_512)?(?:\.\w+)?$", s)
+    return (float(mm.group(1)), float(mm.group(2))) if mm else (np.nan, np.nan)
+
+
+def target_paths(name):
+    if name == "orion":
+        sys.path.insert(0, ROOT)
+        from common import load_all_tiles, ORION
+        return [os.path.join(ORION, p) for p in load_all_tiles().image_path], 333   # ~0.49 um/px already
+    d = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+    return d.image_path.tolist(), 336  # PathGen read_region level 0 = 40x (~0.25 um/px): 672 -> 336 = ~0.5 um/px
+
+
+def five_crops(img, size=224):
+    W, H = img.size
+    offs = [((W - size) // 2, (H - size) // 2), (0, 0), (W - size, 0), (0, H - size), (W - size, H - size)]
+    return [img.crop((x, y, x + size, y + size)) for x, y in offs]  # centre first
+
+
+def extract_gigapath(name, device, bs=32):
+    """GigaPath tile features (fp16) = mean over centre + 4 corner 224-px crops at ~0.5 um/px; centre-only also saved.
+    Needs HF_TOKEN in the environment (prov-gigapath is gated); the token is never read/stored by this script."""
+    import timm
+    from PIL import Image, PngImagePlugin
+    from torchvision import transforms
+    PngImagePlugin.MAX_TEXT_CHUNK = 64 * 1024 * 1024
+    enc = timm.create_model("hf_hub:prov-gigapath/prov-gigapath", pretrained=True).half().eval().to(device)
+    tf = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])
+    paths, side = target_paths(name)
+    feats = np.full((len(paths), 5, 1536), np.nan, np.float32)
+    status = ["ok"] * len(paths)
+    buf, rows = [], []
+
+    def flush():
+        if not buf:
+            return
+        with torch.inference_mode():
+            f = enc(torch.stack(buf).half().to(device)).float().cpu().numpy()
+        feats[np.array(rows)] = f.reshape(-1, 5, 1536)
+        buf.clear(); rows.clear()
+
+    for i, p in enumerate(paths):
+        try:
+            im = Image.open(p).convert("RGB")
+            if im.size != (side, side):
+                im = im.resize((side, side), Image.BICUBIC)
+            buf.extend(tf(c) for c in five_crops(im)); rows.append(i)
+        except Exception as e:
+            status[i] = f"unreadable:{type(e).__name__}"
+            print(f"[{name}] skip {p}: {e}", flush=True)
+        if len(rows) * 5 >= bs:
+            flush()
+        if i % 500 == 0:
+            print(f"[{name}] {i}/{len(paths)}", flush=True)
+    flush()
+    os.makedirs(OUT, exist_ok=True)
+    np.save(os.path.join(OUT, f"{name}_gigapath.npy"), feats.mean(1))
+    np.save(os.path.join(OUT, f"{name}_gigapath_center.npy"), feats[:, 0])
+    pd.DataFrame({"path": paths, "feat_status": status}).to_csv(os.path.join(OUT, f"{name}_gigapath_status.csv"), index_label="row")
+    print(f"[{name}] features {feats.shape[0]} rows, failed {sum(s != 'ok' for s in status)}; "
+          f"non-finite rows {int((~np.isfinite(feats.mean(1)).all(1)).sum())}")
+
+
+def tcga_projects(meta):
+    """TCGA project per WSI from the public GDC files API (file_name = <wsi>.svs); cached in cache/stpath."""
+    fp = os.path.join(OUT, "tcga_text_projects.csv")
+    if not os.path.exists(fp):
+        import requests
+        names = sorted(set(meta.slide + ".svs"))
+        rows = []
+        for i in range(0, len(names), 500):
+            q = {"filters": {"op": "in", "content": {"field": "file_name", "value": names[i:i + 500]}},
+                 "fields": "file_name,cases.project.project_id", "format": "JSON", "size": 1000}
+            r = requests.post("https://api.gdc.cancer.gov/files", json=q, timeout=120).json()
+            for h in r["data"]["hits"]:
+                rows.append({"slide": h["file_name"][:-4], "project": h["cases"][0]["project"]["project_id"]})
+        pd.DataFrame(rows).drop_duplicates("slide").to_csv(fp, index=False)
+    return dict(pd.read_csv(fp).values)
+
+
+# TCGA project -> STPath organ name (keys of constants.organ_align_mapping; lower-case ones are the STimage-1K4M
+# spellings that align to their own tokens, e.g. 'ovary'->'Ovary'); unlisted projects -> 'Others'
+PROJECT2ORGAN = {"BRCA": "Breast", "COAD": "Bowel", "READ": "Bowel", "LUAD": "Lung", "LUSC": "Lung",
+                 "KIRC": "Kidney", "KIRP": "Kidney", "KICH": "Kidney", "LIHC": "Liver", "PAAD": "Pancreas",
+                 "PRAD": "Prostate", "SKCM": "Skin", "UCEC": "Uterus", "UCS": "Uterus", "OV": "ovary",
+                 "GBM": "glioblastoma", "LGG": "glioma", "STAD": "stomach", "HNSC": "mouth", "UVM": "Eye"}
+
+
+def run_target(m, name, feats_path):
+    if name == "orion":
+        sys.path.insert(0, ROOT)
+        from common import load_all_tiles
+        d = load_all_tiles()
+        meta = pd.DataFrame({"key": d.image_path, "path": d.image_path, "slide": d.in_slide_name})
+        meta["x"], meta["y"] = zip(*[parse_xy(os.path.basename(p)) for p in d.image_path])
+        meta["organ"] = "Bowel"
+    else:
+        d = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+        meta = pd.DataFrame({"key": d.key, "path": d.image_path, "slide": d.wsi_id})
+        meta["x"], meta["y"] = zip(*[parse_xy(k) for k in d.key])
+        proj = tcga_projects(meta)  # TCGA project from GDC; organ token mapped, unknown -> 'Others'
+        meta["project"] = meta.slide.map(proj)
+        meta["organ"] = [PROJECT2ORGAN.get(str(p).replace("TCGA-", ""), "Others") for p in meta.project]
+    feats_path = feats_path or os.path.join(OUT, f"{name}_gigapath.npy")
+    if not os.path.exists(feats_path):
+        sys.exit(f"[{name}] BLOCKED: STPath's released checkpoint needs prov-gigapath tile features (1536-d) and "
+                 f"prov-gigapath is gated for this HF account. Provide --feats <GigaPath features (N={len(meta)},1536) "
+                 f"aligned to {name} row order>.")
+    F = np.load(feats_path).astype(np.float32)
+    F = F[:, 0] if F.ndim == 3 else F
+    assert F.shape == (len(meta), 1536), F.shape
+    ok = np.isfinite(F).all(1)
+    meta["status"] = np.where(ok, "ok", "no_feature")
+    meta["n_context"] = meta.groupby("slide").slide.transform("size").where(ok, 0)
+    vocab_genes = pd.read_csv(os.path.join(OUT, "hest_val_genes.csv")).gene.tolist()
+    cols = m.gene_cols(vocab_genes)
+    pred = np.full((len(meta), len(cols)), np.nan, np.float32)
+    for sl, idx in meta[ok].groupby("slide").indices.items():
+        idx = np.where(ok)[0][idx]
+        pred[idx] = m.predict(F[idx], meta[["x", "y"]].values[idx], meta.organ.values[idx[0]], "Visium", cols)[0]
+    np.save(os.path.join(OUT, f"{name}_pred.npy"), pred.astype(np.float16))
+    pd.Series(vocab_genes).to_csv(os.path.join(OUT, f"{name}_genes.csv"), index=False, header=["gene"])
+    meta.to_csv(os.path.join(OUT, f"{name}_meta.csv"), index_label="row")
+    print(f"[{name}] predicted {ok.sum()} / {len(meta)}; {meta.slide.nunique()} slides")
+    if name == "orion":
+        orion_protein_check(pred, vocab_genes)
+
+
+ORION_PAIRS = [("PTPRC", "CD45"), ("EPCAM", "Pan-CK"), ("KRT8", "Pan-CK"), ("CDH1", "E-cadherin"), ("ACTA2", "SMA"),
+               ("MKI67", "Ki67"), ("PECAM1", "CD31"), ("CD68", "CD68"), ("CD3E", "CD3e"), ("CD8A", "CD8a"),
+               ("MS4A1", "CD20"), ("CD163", "CD163")]
+
+
+def orion_protein_check(pred, genes):
+    """predicted RNA vs measured ORION protein (= <marker>_count / n_nuclei) per tile, tiles with >= 20 nuclei."""
+    from scipy.stats import pearsonr, spearmanr
+    sys.path.insert(0, ROOT)
+    from common import load_all_tiles
+    d = load_all_tiles()
+    ts = pd.read_csv(os.path.join(ROOT, "results", "tile_stats.csv"))
+    assert (ts.tile_id.values == d.tile_id.values).all()
+    keep = (ts.n_nuclei.values >= 20) & np.isfinite(pred).all(1)
+    gi = {g: i for i, g in enumerate(genes)}
+    rows = []
+    for g, p in ORION_PAIRS:
+        x = pred[keep, gi[g]].astype(np.float64)
+        y = (d[f"{p}_count"].values / np.maximum(ts.n_nuclei.values, 1))[keep]
+        ws = [spearmanr(x[i], y[i])[0] for i in d[keep].groupby("in_slide_name").indices.values() if len(i) >= 20]
+        rows.append({"gene": g, "protein": p, "n_tiles": int(keep.sum()), "pearson": pearsonr(x, y)[0],
+                     "spearman": spearmanr(x, y)[0], "within_slide_spearman_median": np.nanmedian(ws),
+                     "n_slides_within": len(ws)})
+    R = pd.DataFrame(rows)
+    R.to_csv(os.path.join(OUT, "orion_protein_check.csv"), index=False)
+    print(R.round(3).to_string(index=False))
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--set", required=True, choices=["int2_check", "hest_val", "orion", "tcga_text"])
+    ap.add_argument("--feats", default=None, help="GigaPath (N,1536) features for orion / tcga_text")
+    ap.add_argument("--extract", action="store_true", help="compute GigaPath features for --set orion|tcga_text (needs HF_TOKEN env)")
+    ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    a = ap.parse_args()
+    torch.manual_seed(0)
+    if a.extract:
+        extract_gigapath(a.set, a.device)
+        sys.exit(0)
+    model = STPath(a.device)
+    {"int2_check": lambda: run_int2(model), "hest_val": lambda: run_hest(model)}.get(
+        a.set, lambda: run_target(model, a.set, a.feats))()

experiments/27_azure_caption.py

@@ -0,0 +1,217 @@
+"""Generate H&E descriptions with GPT-6.1 Sol on Azure OpenAI for images that have no real text (plan A:
+one FIXED 3-shot prefix of real PathGen-1.6M captions -> prompt-cached; short caption; low reasoning effort).
+
+Targets (images on disk without paired text):
+  ge_patches : PathGen-1.6M gene-expression patches (brain*, gliomas*, idc*, saromas*, skin*)   1640
+  slide_rep  : PathGen-1.6M/slide_report/image_set/*.png                                           1
+  orion      : ORION H&E tiles (he/*.jpeg)                                                         6335
+  hemit      : HEMIT H&E inputs (he/*/input/*)                                                     55
+Config (env): AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_API_KEY, AZURE_OPENAI_DEPLOYMENT,
+              optional AZURE_OPENAI_API_VERSION (if unset the v1 endpoint <endpoint>/openai/v1/ is used)
+Usage:  python 27_azure_caption.py --dry                # list targets + 3-shot prompt, no API call
+        python 27_azure_caption.py --pilot 20           # 20 images, prints measured cost / image
+        python 27_azure_caption.py --budget 40          # everything; stops when the USD budget is reached
+Output: cache/gen_text/captions.jsonl (resumable; one line per image: set, key, path, caption, usage)
+Generated text is SYNTHETIC (seen only the H&E) -> never use it as test text; tag ttype='synthetic'.
+"""
+import argparse
+import base64
+import concurrent.futures as cf
+import glob
+import io
+import json
+import os
+import random
+import threading
+import time
+
+import numpy as np
+import pandas as pd
+from PIL import Image, PngImagePlugin
+
+from common import CACHE, ROOT
+
+PngImagePlugin.MAX_TEXT_CHUNK = 64 * 1024 * 1024
+P = os.path.join(ROOT, "..")
+OUT = os.path.join(CACHE, "gen_text")
+os.makedirs(OUT, exist_ok=True)
+PRICE = {"in": 2.0, "cached": 0.10, "out": 10.0}  # USD / 1M tokens, GPT-6.1 Sol standard
+MAXPX = 512  # images are downscaled to <=512 px (one vision tile) to keep the token count low
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--sets", nargs="+", default=["ge_patches", "slide_rep", "orion", "hemit"],
+                help="tcga_notext = TCGA images left without real text after 40_clean_text.py; dlpfc / idc = external test sets (all spots); tcga_calib = 1000 TCGA patches that HAVE real text (held-out patients of 31_), for calibrating synthetic vs real text")
+ap.add_argument("--pilot", type=int, default=0)
+ap.add_argument("--budget", type=float, default=40.0)
+ap.add_argument("--workers", type=int, default=8)
+ap.add_argument("--effort", default="low")
+ap.add_argument("--dry", action="store_true")
+args = ap.parse_args()
+
+
+def targets():
+    rows = []
+    ge = ["brain_image", "brain_train_image", "brain_trainnew_image", "gliomas_test_image_new", "gliomas_train_image",
+          "idc_image", "idc_train_image", "saromas_test_image_new", "saromas_train_image_new", "skin_image", "skin_train_image"]
+    src = {"ge_patches": [f for d in ge for f in glob.glob(os.path.join(P, "PathGen-1.6M", d, "*"))],
+           "slide_rep": glob.glob(os.path.join(P, "PathGen-1.6M", "slide_report", "image_set", "*")),
+           "orion": glob.glob(os.path.join(P, "ORION", "ORIONCRC_dataset_tile_20x", "he", "*.jpeg")),
+           "hemit": glob.glob(os.path.join(P, "HEMIT", "he", "*", "input", "*"))}
+    if "tcga_calib" in args.sets:  # same held-out image set as 31_virtual_validation.py (TCGA), random 1000
+        I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+        T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+        seen = set(pd.read_csv(os.path.join(CACHE, "pathgen_meta.csv")).query("split in ['train', 'val']").patient)
+        ev = sorted(set(T.img_idx) & set(I.index[~I.patient.isin(seen)]))
+        src["tcga_calib"] = list(I.image_path.values[np.random.default_rng(0).choice(ev, 1000, replace=False)])
+    if "tcga_notext" in args.sets:  # TCGA images whose real texts are all junk (40_clean_text.py)
+        src["tcga_notext"] = list(pd.read_csv(os.path.join(CACHE, "notext_meta.csv")).image_path)
+    for s in ("dlpfc", "idc"):  # external test sets (36_external_prep.py): the same 224-px patches as the anchor
+        if s in args.sets:      # (0.5 um/px; = UNI2 / MIPHEI input) -> path = "npy:<set>:<row>"
+            n = len(np.load(os.path.join(CACHE, "external", EXT[s]), mmap_mode="r"))
+            rows += [{"set": s, "key": f"external/{s}/{i}", "path": f"npy:{s}:{i}"} for i in range(n)]
+    for s in args.sets:
+        if s in EXT:
+            continue
+        for f in sorted(src[s]):
+            if f.lower().endswith((".png", ".jpg", ".jpeg", ".tif", ".tiff")):
+                rows.append({"set": s, "key": os.path.relpath(f, P), "path": f})
+    return pd.DataFrame(rows)
+
+
+EXT = {"dlpfc": "dlpfc_he_05um.npy", "idc": "idc_he.npy"}
+
+
+def b64(path):
+    if path.startswith("npy:"):
+        _, st, i = path.split(":")
+        im = Image.fromarray(np.asarray(np.load(os.path.join(CACHE, "external", EXT[st]), mmap_mode="r")[int(i)]))
+    else:
+        im = Image.open(path).convert("RGB")
+    im.thumbnail((MAXPX, MAXPX))
+    buf = io.BytesIO()
+    im.save(buf, format="JPEG", quality=90)
+    return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
+
+
+def few_shot():
+    """3 fixed real PathGen-1.6M captions (40-60 words), different slides; same for every request -> cacheable."""
+    T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+    I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+    T = T[(T.ttype == "caption") & T.source.str.contains("pathgen1.6m")]
+    w = T.caption.str.split().str.len()
+    T = T[(w >= 40) & (w <= 60)]
+    rng = random.Random(0)
+    picks, seen = [], set()
+    for i in rng.sample(list(T.index), len(T)):
+        wsi = I.wsi_id[T.img_idx[i]]
+        if wsi[:12] in seen:
+            continue
+        try:
+            img = b64(I.image_path[T.img_idx[i]])
+        except Exception:
+            continue
+        seen.add(wsi[:12]); picks.append((img, T.caption[i], T.key[i]))
+        if len(picks) == 3:
+            return picks
+
+
+SYSTEM = ("You are a pathologist writing a short description of one H&E-stained histopathology image patch. "
+          "Describe only what is visible: tissue type and architecture, cell types and their density, nuclear "
+          "features, stroma, inflammation, vessels, necrosis or mitoses when present. Follow the style and length "
+          "of the examples (one paragraph, about 50 words). Do not mention the examples, staining quality, "
+          "magnification or patient information, and do not invent a diagnosis that the image does not support.")
+
+
+def messages(shots, target_img):
+    m = [{"role": "system", "content": SYSTEM}]
+    for img, cap, _ in shots:
+        m.append({"role": "user", "content": [{"type": "text", "text": "Describe this H&E patch."},
+                                              {"type": "image_url", "image_url": {"url": img, "detail": "auto"}}]})
+        m.append({"role": "assistant", "content": cap})
+    m.append({"role": "user", "content": [{"type": "text", "text": "Describe this H&E patch."},
+                                          {"type": "image_url", "image_url": {"url": target_img, "detail": "auto"}}]})
+    return m
+
+
+def client():
+    from openai import AzureOpenAI, OpenAI
+    ep, key = os.environ["AZURE_OPENAI_ENDPOINT"].rstrip("/"), os.environ["AZURE_OPENAI_API_KEY"]
+    if os.environ.get("AZURE_OPENAI_API_VERSION"):
+        return AzureOpenAI(azure_endpoint=ep, api_key=key, api_version=os.environ["AZURE_OPENAI_API_VERSION"])
+    return OpenAI(base_url=ep + "/openai/v1/", api_key=key)
+
+
+def cost(u):
+    cached = (u.get("prompt_tokens_details") or {}).get("cached_tokens") or 0
+    return ((u["prompt_tokens"] - cached) * PRICE["in"] + cached * PRICE["cached"] + u["completion_tokens"] * PRICE["out"]) / 1e6
+
+
+D = targets()
+shots = few_shot()
+print(f"targets: {len(D)}  " + json.dumps(D.set.value_counts().to_dict()))
+print("3-shot examples (PathGen-1.6M keys):", [k for _, _, k in shots])
+fout = os.path.join(OUT, "captions.jsonl")
+done = set()
+spent = 0.0
+if os.path.exists(fout):
+    for line in open(fout):
+        r = json.loads(line)
+        done.add(r["key"]); spent += r.get("usd", 0)
+todo = D[~D.key.isin(done)]
+if args.pilot:
+    todo = todo.groupby("set", group_keys=False).apply(lambda g: g.sample(min(len(g), max(1, args.pilot // D.set.nunique())), random_state=0))
+print(f"already done {len(done)} (USD {spent:.2f}); to do now {len(todo)}")
+json.dump({"system": SYSTEM, "shots": [{"key": k, "caption": c} for _, c, k in shots], "maxpx": MAXPX,
+           "effort": args.effort, "deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT")},
+          open(os.path.join(OUT, "prompt.json"), "w"), indent=1)
+if args.dry:
+    for _, c, k in shots:
+        print(f"- {k}: {c}")
+    raise SystemExit
+
+cl, dep = client(), os.environ["AZURE_OPENAI_DEPLOYMENT"]
+lock = threading.Lock()
+stop = threading.Event()
+
+
+def run(row):
+    if stop.is_set():
+        return
+    for attempt in range(6):
+        try:
+            r = cl.chat.completions.create(model=dep, messages=messages(shots, b64(row.path)),
+                                           reasoning_effort=args.effort, max_completion_tokens=2000,
+                                           prompt_cache_key="he-caption-3shot-v1")
+            u = r.usage.model_dump()
+            rec = {"set": row.set, "key": row.key, "path": row.path, "caption": r.choices[0].message.content.strip(),
+                   "finish": r.choices[0].finish_reason, "usage": u, "usd": cost(u), "ttype": "synthetic"}
+            break
+        except Exception as e:  # rate limits / transient errors -> exponential backoff
+            err = repr(e)[:300]
+            if "content_filter" in err or attempt == 5:
+                rec = {"set": row.set, "key": row.key, "path": row.path, "caption": None, "error": err, "usd": 0}
+                break
+            time.sleep(2 ** attempt * 5)
+    global spent
+    with lock:
+        with open(fout, "a") as f:
+            f.write(json.dumps(rec) + "\n")
+        spent += rec["usd"]
+        if spent >= args.budget:
+            stop.set()
+
+
+t0 = time.time()
+with cf.ThreadPoolExecutor(args.workers) as ex:
+    for i, _ in enumerate(ex.map(run, [r for r in todo.itertuples()])):
+        if i % 200 == 0:
+            print(f"{i}/{len(todo)}  USD {spent:.2f}  {time.time() - t0:.0f}s", flush=True)
+R = pd.read_json(fout, lines=True)
+ok = R[R.caption.notna()]
+print(f"done {len(ok)} ok / {len(R)} total; failures {R.caption.isna().sum()}; USD spent {R.usd.sum():.2f}; "
+      f"USD/image {ok.usd.mean():.4f} -> projected for all {len(D)}: {ok.usd.mean() * len(D):.0f}")
+if len(ok):
+    u = pd.json_normalize(ok.usage)
+    print(u.filter(regex="prompt_tokens$|completion_tokens$|cached_tokens|reasoning_tokens").mean().round(0).to_string())
+if stop.is_set():
+    print(f"STOPPED: budget {args.budget} USD reached")

experiments/28_eval_anchor.py

@@ -0,0 +1,81 @@
+"""H&E foundation-model ablation (UNI2 vs GigaPath vs CONCH as the frozen ImageBind anchor).
+
+  --prep : cache/pathgen_he_<anchor>.npy from cache/allimg_he_<anchor>.npy (PathGen-1.6M captions are a subset of the
+           all-pairs pool, same image files -> identical features; verified for uni2, cosine 1.0)
+  default: evaluate
+    (A) ORION H&E<->mIF, trained pair: runs/bench_fm/*_<anchor>_<if_enc>_fm-geneformer_pathgenclip (17_bench_fm.py);
+        tie-aware R@1 on 100-tile galleries + Hungarian pairing acc on the full test fold, both directions
+    (B) emergent text<->mIF: text head trained ONLY on PathGen H&E-caption pairs with the same anchor
+        (runs/text_pathgen/<anchor>_<txt>, 12_train_text_pathgen.py) scores the concept prompts against the ORION
+        mIF embeddings of that anchor's run (text and mIF never co-occur in training); AUROC + Spearman (18_eval_fm.probe)
+Outputs results/anchor_ifret.csv, results/anchor_text_probes.csv
+"""
+import glob
+import json
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn.functional as F
+
+from common import CACHE, ROOT, RUNS, metrics
+
+ANCHORS = ["uni2", "gigapath", "conch"]
+OUT = os.path.join(ROOT, "results")
+
+if "--prep" in sys.argv:
+    P = pd.read_csv(os.path.join(CACHE, "pathgen_meta.csv"))
+    I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv")).set_index("key")
+    ix = I.loc[P.image_path.map(lambda p: os.path.basename(p).rsplit(".", 1)[0]), "idx"].values
+    assert (I.image_path.values[ix] == P.image_path.values).all()
+    for a in ANCHORS:
+        f = os.path.join(CACHE, f"pathgen_he_{a}.npy")
+        if not os.path.exists(f):
+            np.save(f, np.load(os.path.join(CACHE, f"allimg_he_{a}.npy"))[ix])
+            print("saved", f)
+    sys.exit()
+
+# reuse the probe of 18_eval_fm.py without running its main block
+src = open(os.path.join(ROOT, "18_eval_fm.py")).read().split("# collect test embeddings")[0]
+ns = {}
+exec(compile(src, "18_eval_fm_head", "exec"), ns)
+probe, prompts = ns["probe"], ns["prompts"]
+
+
+def load_head(anchor, enc, din, dout):
+    h = torch.nn.Sequential(torch.nn.Linear(din, 512), torch.nn.GELU(), torch.nn.Dropout(0.1), torch.nn.Linear(512, 512),
+                            torch.nn.GELU(), torch.nn.LayerNorm(512), torch.nn.Linear(512, dout))
+    h.load_state_dict(torch.load(os.path.join(RUNS, "text_pathgen", f"{anchor}_{enc}", "head.pt"), map_location="cpu"))
+    return h.eval()
+
+
+ret, prb = [], []
+for f in sorted(glob.glob(os.path.join(RUNS, "bench_fm", "*", "test_embeddings.pt"))):
+    a = json.load(open(os.path.join(os.path.dirname(f), "train.json")))["args"]
+    if a["anchor"] not in ANCHORS or a["mk_enc"] != "fm:geneformer" or a["txt_enc"] != "pathgenclip":
+        continue
+    E = torch.load(f, weights_only=False)  # our own files (contain numpy tile ids)
+    he, zi, tiles = E["z"]["he"], E["z"]["if"], np.asarray(E["tile_id"])
+    key = {"anchor": a["anchor"], "if_enc": a["if_enc"], "split": a["split"], "fold": a["fold"]}
+    for q, g, d in [(he, zi, "he->if"), (zi, he, "if->he")]:
+        ret.append({**key, "dir": d, **metrics(q, g)})
+    for t in ["biomedclip", "plip", "pathgenclip"]:
+        if not os.path.exists(os.path.join(RUNS, "text_pathgen", f"{a['anchor']}_{t}", "head.pt")):
+            continue
+        prm = torch.from_numpy(np.load(os.path.join(CACHE, f"prompts_text_{t}.npy"))[:, 0].astype(np.float32))
+        with torch.no_grad():
+            zp = F.normalize(load_head(a["anchor"], t, prm.shape[1], he.shape[1])(prm), dim=-1)
+        for m, z in [("he", he), ("if", zi)]:
+            for i, (c, pr) in enumerate(prompts.items()):
+                prb.append({**key, "text_enc": t, "modality": m, "concept": c, **probe((z @ zp[i]).numpy(), tiles, c, pr["label"])})
+
+R, P = pd.DataFrame(ret), pd.DataFrame(prb)
+R.to_csv(os.path.join(OUT, "anchor_ifret.csv"), index=False)
+P.to_csv(os.path.join(OUT, "anchor_text_probes.csv"), index=False)
+pd.set_option("display.width", 250)
+g = R.groupby(["if_enc", "anchor", "split", "fold"])[["R@1_g100", "pair_acc"]].mean().reset_index()
+print(g.groupby(["if_enc", "anchor", "split"])[["R@1_g100", "pair_acc"]].agg(["mean", "std"]).round(3).to_string())
+if len(P):
+    print(P.groupby(["text_enc", "modality", "if_enc", "anchor"])[["auroc", "spearman"]].mean().round(3).to_string())

experiments/29_virtual_completion.py

@@ -0,0 +1,230 @@
+"""Virtual modality completion: does training text <-> RNA / text <-> protein DIRECTLY on virtual pairs beat the
+emergent (H&E-anchor) route?
+
+Training pairs (TCGA all-pairs pool, 6,144 patches, every real text of a patch is a positive):
+  rna     : real text <-> STPath virtual RNA of the same patch (cache/stpath/tcga_text_pred.npy -> log1p CP10k, 2000 HEST HVGs)
+  protein : real text <-> GigaTIME virtual mIF of the same patch (cache/gigatime/tcga_text_means.npy, 11 channels shared with ORION)
+Model: frozen BiomedCLIP text tower + MLP head and an MLP head on the modality vector -> shared 512-d space
+(CLIP-style, multi-positive InfoNCE), patient GroupKFold(5) over TCGA, 10% of training patients for early stopping.
+Each feature is rank-gaussianised within its own dataset (virtual TCGA / real HEST / real ORION), because virtual
+values (STPath expected counts, GigaTIME probabilities) and real measurements live on different scales.
+
+Evaluation only on REAL measurements (never virtual):
+  rna     : HEST real RNA, concept prompts vs RNA of the concept gene (protocol of 24_eval_gene_emergent.py (B):
+            AUROC top quartile, per HEST study+patient fold, all spots / bowel spots)
+  protein : ORION real mIF tile means, concept prompts vs marker+ cell fraction (probe of 18_eval_fm.py, per
+            patient_5fold test fold); also ORION GigaTIME virtual means (same tiles) to separate the virtual->real gap
+Baselines (emergent, read from results/): text head trained on PathGen H&E-caption pairs, scored against the RNA
+(23_) / VirTues mIF (17_) embeddings bound to UNI2.
+Outputs results/virtual_completion.csv
+"""
+import glob
+import json
+import os
+
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.special import ndtri
+from scipy.stats import rankdata, spearmanr
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import GroupKFold
+
+from common import CACHE, ROOT, RUNS, SPLITS, load_all_tiles
+
+dev = "cuda"
+OUT = os.path.join(ROOT, "results")
+TXT_ENC = "biomedclip"
+prompts = json.load(open(os.path.join(ROOT, "prompts.json")))
+PKEYS = list(prompts)
+
+
+def rankgauss(x):
+    r = np.apply_along_axis(rankdata, 0, x)
+    return ndtri((r - 0.5) / len(x)).astype(np.float32)
+
+
+# ---------------- TCGA training pool (texts, images, virtual modalities)
+I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+TX = torch.from_numpy(np.load(os.path.join(CACHE, f"alltxt_text_{TXT_ENC}.npy"))[:, 0].astype(np.float32)).to(dev)
+PR = torch.from_numpy(np.load(os.path.join(CACHE, f"prompts_text_{TXT_ENC}.npy"))[:, 0].astype(np.float32)).to(dev)
+t_img = torch.as_tensor(T.img_idx.values, device=dev)
+assert (pd.read_csv(os.path.join(CACHE, "stpath", "tcga_text_meta.csv")).key.values == I.key.values).all()
+assert (pd.read_csv(os.path.join(CACHE, "gigatime", "tcga_text_meta.csv")).key.values == I.key.values).all()
+
+# RNA: STPath log1p(counts) -> CP10k -> log1p, HEST HVGs
+H = os.path.join(CACHE, "hest")
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+sg = pd.read_csv(os.path.join(CACHE, "stpath", "tcga_text_genes.csv")).gene.values
+c = np.expm1(np.load(os.path.join(CACHE, "stpath", "tcga_text_pred.npy")).astype(np.float32)).clip(0)
+c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+gix = pd.Series(range(len(sg)), index=sg).loc[hvg].values
+V_RNA = rankgauss(c[:, gix])
+del c
+
+# protein: GigaTIME channels shared with ORION (Actin-D + Transgelin -> SMA)
+ch = json.load(open(os.path.join(CACHE, "gigatime", "channels.json")))
+G_CH, SHARED = ch["channels"], ch["orion_shared"]
+ORION_CH = ["Hoechst", "CD31", "CD45", "CD68", "CD4", "FOXP3", "CD8a", "CD45RO", "CD20", "PD-L1", "CD3e",
+            "CD163", "E-cadherin", "PD-1", "Ki67", "Pan-CK", "SMA"]  # order of orion_real_if_means.npy (25_)
+P_ORION = sorted(set(SHARED.values()), key=ORION_CH.index)
+
+
+def giga_vec(means):
+    return np.stack([np.mean([means[:, G_CH.index(g)] for g, o in SHARED.items() if o == oc], 0) for oc in P_ORION], 1)
+
+
+V_PROT = rankgauss(giga_vec(np.load(os.path.join(CACHE, "gigatime", "tcga_text_means.npy"))))
+
+# ---------------- real evaluation sets
+M = pd.read_csv(os.path.join(H, "meta.csv"))
+hest_x = rankgauss(np.load(os.path.join(H, "hvg.npy")).astype(np.float32))
+genes = pd.read_csv(os.path.join(H, "genes.csv")).gene.values
+X = sp.load_npz(os.path.join(H, "counts.npz")).tocsr()
+L = sp.diags(1e4 / np.asarray(X.sum(1)).ravel()) @ X
+L.data = np.log1p(L.data)
+gpos = {g: i for i, g in enumerate(genes)}
+rna = lambda gs: np.asarray(L[:, [gpos[g] for g in gs]].mean(1)).ravel()
+CONCEPT_RNA = {"immune": ["PTPRC"], "tumor_epith": ["KRT8", "EPCAM"], "stroma": ["ACTA2"], "proliferation": ["MKI67"],
+               "vessels": ["PECAM1"], "macrophage": ["CD68"]}  # as 24_
+hest_fold = np.full(len(M), -1)
+for f in glob.glob(os.path.join(RUNS, "gene", "uni2_mlp_fold*", "test.pt")):
+    hest_fold[torch.load(f, weights_only=False)["idx"]] = int(f.split("_fold")[1].split(os.sep)[0])
+assert (hest_fold >= 0).all()
+
+df = load_all_tiles()
+nn_ = pd.read_csv(os.path.join(OUT, "tile_stats.csv")).set_index("tile_id").loc[df.tile_id, "n_nuclei"].values
+df["n_nuclei"] = nn_
+orion_real = rankgauss(np.load(os.path.join(CACHE, "gigatime", "orion_real_if_means.npy"))[:, [ORION_CH.index(o) for o in P_ORION]])
+orion_virt = rankgauss(giga_vec(np.load(os.path.join(CACHE, "gigatime", "orion_means.npy"))))
+assert (pd.read_csv(os.path.join(CACHE, "gigatime", "orion_meta.csv")).tile_id.values == df.tile_id.values).all()
+orion_fold = np.full(len(df), -1)
+for k in range(5):
+    s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+    orion_fold[s.tile_id[s.split == "test"].values] = k
+
+
+def orion_probe(scores, sel, key, label_col):
+    """18_eval_fm.probe restricted to tile mask `sel`"""
+    sub = sel if key == "cellularity" else sel & (nn_ >= 20)
+    y = df[label_col].values[sub].astype(float)
+    if label_col != "n_nuclei":
+        y = y / np.maximum(nn_[sub], 1)
+    lab = y >= np.quantile(y, 0.75)
+    return {"spearman": spearmanr(scores[sub], y).correlation,
+            "auroc": roc_auc_score(lab, scores[sub]) if 0 < lab.sum() < lab.size else np.nan}
+
+
+# ---------------- model
+def mlp(din, dout=512, hid=512):  # same head as 20_pairing.py
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float(((s > s.diag()[:, None]).sum(1) == 0).float().mean())
+
+
+def train(V, tr_t, va_t, epochs=60, bs=256, lr=3e-4, seed=0):
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    th, mh = mlp(TX.shape[1]).to(dev), mlp(V.shape[1]).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(th.parameters()) + list(mh.parameters()) + [logit], lr=lr, weight_decay=0.05)
+    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=epochs * max(1, len(tr_t) // bs), pct_start=0.05)
+    va = T.loc[va_t].groupby("img_idx").head(1).index.values  # one text per val image
+    va_i, va_t1 = torch.as_tensor(T.img_idx.values[va], device=dev), torch.as_tensor(va, device=dev)
+    best, state = -1, None
+    for ep in range(epochs):
+        th.train(); mh.train()
+        perm = rng.permutation(tr_t)
+        for s in range(0, len(perm) - bs + 1, bs):
+            b = torch.as_tensor(perm[s:s + bs], device=dev)
+            ii = t_img[b]
+            lg = logit.exp().clamp(max=100) * F.normalize(mh(V[ii]), dim=-1) @ F.normalize(th(TX[b]), dim=-1).T
+            pos = (ii[:, None] == ii[None, :]).float()
+            tgt = pos / pos.sum(1, keepdim=True)
+            loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sched.step()
+        th.eval(); mh.eval()
+        with torch.no_grad():
+            zv, zt = F.normalize(mh(V[va_i]), dim=-1), F.normalize(th(TX[va_t1]), dim=-1)
+            v = (r1(zv, zt) + r1(zt, zv)) / 2
+        if v > best:
+            best, state = v, ({k: x.clone() for k, x in th.state_dict().items()}, {k: x.clone() for k, x in mh.state_dict().items()})
+    th.load_state_dict(state[0]); mh.load_state_dict(state[1]); th.eval(); mh.eval()
+    return th, mh, best
+
+
+@torch.no_grad()
+def score(th, mh, x):
+    zp = F.normalize(th(PR), dim=-1)
+    z = F.normalize(mh(torch.from_numpy(x).to(dev)), dim=-1)
+    return (z @ zp.T).cpu().numpy()  # (n, n_prompts)
+
+
+rows = []
+folds = list(GroupKFold(5).split(I, groups=I.patient))
+for mod, Vnp in [("rna", V_RNA), ("protein", V_PROT)]:
+    V = torch.from_numpy(Vnp).to(dev)
+    for k, (trv, te) in enumerate(folds):
+        pats = np.random.default_rng(k).permutation(sorted(I.patient.iloc[trv].unique()))
+        va_p = set(pats[: max(1, len(pats) // 10)])
+        isplit = pd.Series("test", index=I.index)
+        isplit.iloc[trv] = np.where(I.patient.iloc[trv].isin(va_p), "val", "train")
+        ts = isplit.loc[T.img_idx].values
+        tr_t, va_t, te_t = (T.index[ts == s].values for s in ["train", "val", "test"])
+        th, mh, vbest = train(V, tr_t, va_t)
+        # in-domain sanity: held-out TCGA patients, text <-> virtual modality, 1 text per image, R@1 on 100-galleries
+        te1 = T.loc[te_t].groupby("img_idx").head(1).index.values
+        with torch.no_grad():
+            zv = F.normalize(mh(V[T.img_idx.values[te1]]), dim=-1).cpu()
+            zt = F.normalize(th(TX[te1]), dim=-1).cpu()
+        g = np.random.default_rng(0)
+        r = np.mean([(r1(zv[s], zt[s]) + r1(zt[s], zv[s])) / 2 for s in (g.choice(len(te1), 100, replace=False) for _ in range(50))])
+        rows.append({"modality": mod, "model_fold": k, "eval": "tcga_virtual_R@1_g100", "concept": "-", "auroc": np.nan, "spearman": r, "val_r1": vbest})
+        if mod == "rna":
+            S = score(th, mh, hest_x)
+            for c, gs in CONCEPT_RNA.items():
+                y_all = rna(gs)
+                for gal in ["all", "bowel"]:
+                    gm = np.ones(len(M), bool) if gal == "all" else (M.organ == "Bowel").values
+                    for hk in range(5):
+                        sel = gm & (hest_fold == hk)
+                        y = y_all[sel]; lab = y >= np.quantile(y, 0.75)
+                        s = S[sel, PKEYS.index(c)]
+                        rows.append({"modality": mod, "model_fold": k, "eval": f"hest_real_{gal}", "concept": c, "eval_fold": hk,
+                                     "auroc": roc_auc_score(lab, s) if 0 < lab.sum() < lab.size else np.nan, "spearman": spearmanr(s, y).correlation})
+        else:
+            for src, x in [("orion_real", orion_real), ("orion_virtual", orion_virt)]:
+                S = score(th, mh, x)
+                for i, (c, pr) in enumerate(prompts.items()):
+                    for ok in range(5):
+                        rows.append({"modality": mod, "model_fold": k, "eval": src, "concept": c, "eval_fold": ok,
+                                     **orion_probe(S[:, i], orion_fold == ok, c, pr["label"])})
+        print(mod, "fold", k, "val", round(vbest, 3), "tcga R@1_g100", round(r, 3), flush=True)
+
+R = pd.DataFrame(rows)
+R.to_csv(os.path.join(OUT, "virtual_completion.csv"), index=False)
+
+# ---------------- comparison with the emergent route (same prompts, same text encoder, same real labels)
+pd.set_option("display.width", 250)
+print(R[R["eval"] == "tcga_virtual_R@1_g100"].groupby("modality").spearman.agg(["mean", "std"]).round(3))
+D = R[R["eval"] != "tcga_virtual_R@1_g100"]
+print(D.groupby(["modality", "eval", "concept"]).auroc.mean().unstack("concept").round(3).to_string())
+print(D.groupby(["modality", "eval"]).auroc.mean().round(3).to_string())
+E = pd.read_csv(os.path.join(OUT, "gene_emergent_text.csv"))
+E = E[E.text_enc == TXT_ENC]
+print("emergent text<->RNA (HEST real):\n", E.groupby(["gallery", "modality", "concept"]).auroc.mean().unstack("concept").round(3).to_string())
+A = pd.read_csv(os.path.join(OUT, "anchor_text_probes.csv"))
+A = A[(A.text_enc == TXT_ENC) & (A.split == "patient_5fold") & (A.anchor == "uni2")]
+print("emergent text<->protein (ORION real, VirTues):\n", A.groupby(["modality", "concept"]).auroc.mean().unstack("concept").round(3).to_string())
+# single-marker reference on ORION real: the matched channel itself as the score
+REF = {"immune": "CD3e", "tumor_epith": "Pan-CK", "stroma": "SMA", "proliferation": "Ki67", "macrophage": "CD68", "cellularity": "Hoechst"}
+ref = {c: np.mean([orion_probe(orion_real[:, P_ORION.index(ch_)], orion_fold == ok, c, prompts[c]["label"])["auroc"] for ok in range(5)])
+       for c, ch_ in REF.items()}
+print("single real channel as score (reference):", {c: round(v, 3) for c, v in ref.items()})

experiments/30_virtues_shared.py

@@ -0,0 +1,189 @@
+"""VirTues embeddings on the protein panel shared by ORION (real mIF) and GigaTIME (virtual mIF from H&E), so that one
+mIF head trained on REAL ORION data can embed virtual mIF for validation (virtual data is never used for training).
+
+Shared panel (ORION name <- GigaTIME channel): Hoechst<-DAPI, CD68, CD4, CD8a<-CD8, CD20, PD-L1, CD3e<-CD3, Ki67,
+Pan-CK<-CK, SMA<-mean(Actin-D, Transgelin). PD-1 is left out (ORION PD-1 export is saturated / near constant).
+
+Usage: python 30_virtues_shared.py real                 -> cache/if_virtues_shared.npy            (3187, 8, 512)  8 dihedral views
+       python 30_virtues_shared.py virtual orion|tcga_text -> cache/if_virtues_shared_virtual_<set>.npy (N, 1, 512) view 0
+       python 30_virtues_shared.py virtual_qm orion,tcga_text -> cache/if_virtues_shared_virtual_qm_<set>_fold<k>.npy
+              (virtual pixels quantile-mapped per channel onto the real fold-k TRAIN pixel distribution)
+       --m16: 16-channel ORION panel shared with MIPHEI-ViT (all ORION markers but PD-1):
+       python 30_virtues_shared.py real --m16          -> cache/if_virtues_m16.npy (3187, 8, 512)
+       python 30_virtues_shared.py miphei <set> --m16  -> cache/if_virtues_m16_virtual_miphei_<set>.npy (N, 1, 512)
+Preprocessing = 14_extract_if_fm.py (per-slide 0.99-quantile clip, log1p, dataset mean/std, 3x3 Gaussian blur, ~1 um/px):
+  real    : ORION 333 px tiles (166 um) -> 168 px
+  virtual : GigaTIME maps (128 px; ORION tile 166 um, TCGA 672-px patch at ~0.25 um/px = 168 um) -> 168 px; values are
+            the uint8 sigmoid probabilities (0-255), standardised within their own set (slide = ORION slide / TCGA WSI)
+"""
+import importlib.util
+import json
+import os
+import sys
+import time
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn.functional as F
+
+from common import CACHE, ROOT, dihedral, load_all_tiles
+
+MODE = sys.argv[1]
+_argv, sys.argv = sys.argv, [sys.argv[0], "virtues", "--no_probe"]  # 14_ parses its CLI at import time
+spec = importlib.util.spec_from_file_location("virt14", os.path.join(ROOT, "14_extract_if_fm.py"))
+V = importlib.util.module_from_spec(spec); spec.loader.exec_module(V)
+sys.argv = _argv
+dev, SIZE = V.dev, V.SIZE
+
+SHARED = [("Hoechst", ["DAPI"]), ("CD68", ["CD68"]), ("CD4", ["CD4"]), ("CD8a", ["CD8"]), ("CD20", ["CD20"]),
+          ("PD-L1", ["PD-L1"]), ("CD3e", ["CD3"]), ("Ki67", ["Ki67"]), ("Pan-CK", ["CK"]), ("SMA", ["Actin-D", "Transgelin"])]
+ORION_NAMES = [m for m, _ in V.MARKERS]
+UNIPROT = dict(V.MARKERS)
+M16 = "--m16" in sys.argv  # 16-channel panel shared by ORION and MIPHEI-ViT (all ORION markers except PD-1)
+if M16:
+    sys.argv.remove("--m16")
+    SHARED = [(m, [m]) for m in ORION_NAMES if m != "PD-1"]
+SUF = "m16" if M16 else "shared"
+ids = [UNIPROT[m] for m, _ in SHARED]
+
+t0 = time.time()
+
+
+def load_virtual(st):
+    gch = json.load(open(os.path.join(CACHE, "gigatime", "channels.json")))["channels"]
+    maps = np.load(os.path.join(CACHE, "gigatime", f"{st}_maps.npy"), mmap_mode="r")  # N,21,128,128 uint8
+    meta = pd.read_csv(os.path.join(CACHE, "gigatime", f"{st}_meta.csv"))
+    N = len(maps)
+    X = np.empty((N, len(SHARED), SIZE, SIZE), np.float16)
+    for s in range(0, N, 256):
+        m = torch.from_numpy(np.asarray(maps[s:s + 256])).to(dev).float()
+        x = torch.stack([m[:, [gch.index(g) for g in gs]].mean(1) for _, gs in SHARED], 1)
+        X[s:s + 256] = F.interpolate(x, size=(SIZE, SIZE), mode="bilinear", align_corners=False).clamp(min=0).half().cpu().numpy()
+    if st == "orion":
+        df = load_all_tiles()
+        assert (meta.tile_id.values == df.tile_id.values).all()
+        slide = df.in_slide_name.values
+    else:
+        I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+        assert (meta.key.values == I.key.values).all()
+        slide = I.wsi_id.values
+    return X, slide
+
+
+def encode(X, slide, views, out_f):
+    X, uq, mean, std, valid, slide_k, n_zero = V.standardize(X, slide)
+    print("  log1p mean", np.round(mean, 2).tolist(), "\n  std", np.round(std, 2).tolist(), "| zero-uq (slide,ch):", int(n_zero.sum()))
+    model, midx = V.load_virtues(ids)
+    midx = midx.to(dev)
+    N = len(X)
+    out = np.zeros((N, views, 512), np.float16)
+    BS = 4 if views == 8 else 32  # 32 crops per forward (~4.7 GiB peak, shared GPU)
+    with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16):
+        for s in range(0, N, BS):
+            x = torch.from_numpy(X[s:s + BS]).to(dev).float()
+            crops = [dihedral(x, d)[i] for d in range(views) for i in range(len(x))]  # d-major
+            ps = model.encoder.forward_list(crops, [midx] * len(crops)).patch_summary_tokens
+            e = torch.stack([p.float().mean((0, 1)) for p in ps]).view(views, len(x), -1).transpose(0, 1)
+            out[s:s + BS] = e.cpu().numpy().astype(np.float16)
+    np.save(out_f, out)
+    print(f"saved {out_f} {out.shape} finite {np.isfinite(out).all()} [{time.time() - t0:.0f}s]", flush=True)
+
+
+if MODE == "miphei":
+    # MIPHEI virtual mIF (35_miphei_infer.py): already on the 8-bit ORION scale and resampled to SIZE like the real tiles
+    assert M16
+    st = sys.argv[2]
+    ch = json.load(open(os.path.join(CACHE, "miphei", "channels.json")))["channels"]
+    X = np.load(os.path.join(CACHE, "miphei", f"{st}_maps168.npy"))[:, [ch.index(m) for m, _ in SHARED]].astype(np.float16)
+    if st == "orion":
+        slide = load_all_tiles().in_slide_name.values
+    else:
+        slide = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv")).wsi_id.values if st == "tcga_text" else \
+            pd.read_csv(os.path.join(CACHE, "external", f"{st}_meta.csv"))["sample"].astype(str).values
+    assert len(slide) == len(X)
+    encode(X, slide, 1, os.path.join(CACHE, f"if_virtues_m16_virtual_miphei_{st}.npy"))
+    sys.exit(0)
+
+if MODE == "virtual_qm":
+    # channel-UNIFIED virtual mIF: per channel, quantile-map the GigaTIME probability pixels onto the REAL ORION pixel
+    # intensity distribution of the fold-k TRAINING tiles (monotone, keeps the spatial pattern), then the real
+    # preprocessing + VirTues. Only real training-split pixels define the target distribution; nothing is trained.
+    sets = sys.argv[2].split(",")
+    df = load_all_tiles()
+    X17, _ = V.load_resampled(df)
+    XR = X17[:, [ORION_NAMES.index(m) for m, _ in SHARED]]; del X17
+    P = np.linspace(0, 1, 1001)
+    rng = np.random.default_rng(0)
+    ref = {}
+    for k in range(5):
+        sp = pd.read_csv(os.path.join(ROOT, "splits", "patient_5fold", f"fold{k}.csv"))
+        tr = sp.tile_id[sp.split == "train"].values
+        sub = rng.choice(tr, min(400, len(tr)), replace=False)
+        ref[k] = np.stack([np.quantile(XR[sub, c].astype(np.float32).ravel(), P) for c in range(len(SHARED))])
+    del XR
+    np.savez(os.path.join(CACHE, "gigatime", "qm_ref_real_train.npz"), P=P, **{f"fold{k}": v for k, v in ref.items()})
+    for st in sets:
+        X0, slide = load_virtual(st)
+        sub = rng.choice(len(X0), min(400, len(X0)), replace=False)
+        vq = np.stack([np.quantile(X0[sub, c].astype(np.float32).ravel(), P) for c in range(len(SHARED))])
+        vq = vq + np.arange(len(P))[None] * 1e-6  # strictly increasing despite the many zero-probability ties
+        for k in range(5):
+            X = np.empty_like(X0)
+            for s in range(0, len(X0), 128):
+                x = X0[s:s + 128].astype(np.float32)
+                for c in range(len(SHARED)):
+                    x[:, c] = np.interp(x[:, c], vq[c], ref[k][c])
+                X[s:s + 128] = x.astype(np.float16)
+            print(f"[{time.time() - t0:.0f}s] {st} fold {k}: quantile-mapped", flush=True)
+            encode(X, slide, 1, os.path.join(CACHE, f"if_virtues_shared_virtual_qm_{st}_fold{k}.npy"))
+            del X
+    sys.exit(0)
+
+if MODE == "real":
+    df = load_all_tiles()
+    X17, _ = V.load_resampled(df)
+    X = np.ascontiguousarray(X17[:, [ORION_NAMES.index(m) for m, _ in SHARED]]); del X17
+    slide, views, out_f = df.in_slide_name.values, 8, os.path.join(CACHE, f"if_virtues_{SUF}.npy")
+elif MODE == "virtual":
+    st = sys.argv[2]
+    gch = json.load(open(os.path.join(CACHE, "gigatime", "channels.json")))["channels"]
+    maps = np.load(os.path.join(CACHE, "gigatime", f"{st}_maps.npy"), mmap_mode="r")  # N,21,128,128 uint8
+    meta = pd.read_csv(os.path.join(CACHE, "gigatime", f"{st}_meta.csv"))
+    N = len(maps)
+    X = np.empty((N, len(SHARED), SIZE, SIZE), np.float16)
+    for s in range(0, N, 256):
+        m = torch.from_numpy(np.asarray(maps[s:s + 256])).to(dev).float()
+        x = torch.stack([m[:, [gch.index(g) for g in gs]].mean(1) for _, gs in SHARED], 1)
+        X[s:s + 256] = F.interpolate(x, size=(SIZE, SIZE), mode="bilinear", align_corners=False).clamp(min=0).half().cpu().numpy()
+    if st == "orion":
+        df = load_all_tiles()
+        assert (meta.tile_id.values == df.tile_id.values).all()
+        slide = df.in_slide_name.values
+    else:
+        I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+        assert (meta.key.values == I.key.values).all()
+        slide = I.wsi_id.values
+    views, out_f = 1, os.path.join(CACHE, f"if_virtues_shared_virtual_{st}.npy")
+else:
+    raise ValueError(MODE)
+
+print(f"[{time.time() - t0:.0f}s] loaded {X.shape}; standardising", flush=True)
+X, uq, mean, std, valid, slide_k, n_zero = V.standardize(X, slide)
+print("  log1p mean", np.round(mean, 2).tolist(), "\n  std", np.round(std, 2).tolist(), "| zero-uq (slide,ch):", int(n_zero.sum()))
+model, midx = V.load_virtues(ids)
+midx = midx.to(dev)
+N = len(X)
+out = np.zeros((N, views, 512), np.float16)
+BS = 4 if views == 8 else 32  # 32 crops per forward (~4.7 GiB peak, shared GPU)
+with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16):
+    for s in range(0, N, BS):
+        x = torch.from_numpy(X[s:s + BS]).to(dev).float()
+        crops = [dihedral(x, d)[i] for d in range(views) for i in range(len(x))]  # d-major
+        ps = model.encoder.forward_list(crops, [midx] * len(crops)).patch_summary_tokens
+        e = torch.stack([p.float().mean((0, 1)) for p in ps]).view(views, len(x), -1).transpose(0, 1)
+        out[s:s + BS] = e.cpu().numpy().astype(np.float16)
+        if s % (BS * 50) == 0:
+            print(f"  encoded {s}/{N} [{time.time() - t0:.0f}s]", flush=True)
+np.save(out_f, out)
+print(f"saved {out_f} {out.shape} finite {np.isfinite(out).all()} [{time.time() - t0:.0f}s]")

experiments/31_virtual_validation.py

@@ -0,0 +1,288 @@
+"""Validate the 4-modality ImageBind model with VIRTUAL modalities (validation only; every encoder/head was trained
+on real pairs: H&E-mIF ORION (17_), H&E-text PathGen (12_), H&E-RNA HEST (23_); H&E = frozen UNI2).
+
+Virtual data fill the modality pairs that no dataset measures together, giving quadruplets:
+  ORION : H&E (real) | mIF (real)        | RNA (STPath, virtual) | text (GPT synthetic caption of the H&E, 27_/32_)
+  TCGA  : H&E (real) | text (real)       | mIF (GigaTIME, virtual) | RNA (STPath, virtual)
+  HEST  : H&E (real) | RNA (real) + RNA (STPath, virtual)  -> calibration of the virtual RNA
+  ORION : mIF real vs mIF virtual (GigaTIME) on the same tiles -> calibration of the virtual mIF
+Every embedding lives in the UNI2 space, so any two modalities are compared by cosine similarity.
+
+Heads:  RNA  runs/gene/uni2_mlp_fold<k>/head.pt (MLP on log1p-CP10k of the 2000 HEST HVGs; HEST fold k)
+        mIF  runs/bench_fm/patient_5fold_fold<k>_uni2_fm-virtues[_shared]_fm-geneformer_pathgenclip/heads.pt
+             (17-channel VirTues for real ORION; 10-channel shared panel (30_) for real-vs-virtual / TCGA)
+        text runs/text_pathgen/uni2_biomedclip/head.pt (PathGen captions); TCGA evaluation images exclude every
+             patient of its train/val split
+Metrics on random 100-item galleries (50 draws): R@1 (tie-aware, both directions averaged) and one-to-one Hungarian
+pairing accuracy (chance 0.01). "circular" = both sides are derived from the same H&E image (virtual-from-H&E), so the
+score is an upper bound of what real data would give, not evidence of biological agreement.
+Outputs results/virtual_validation.csv
+"""
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, RUNS, SPLITS, load_all_tiles
+
+OUT = os.path.join(ROOT, "results")
+H = os.path.join(CACHE, "hest")
+G100, DRAWS = 100, 50
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def load_mlp(sd):
+    h = mlp(sd["0.weight"].shape[1]); h.load_state_dict(sd); return h.eval()
+
+
+def nz(x):
+    return F.normalize(torch.as_tensor(np.asarray(x), dtype=torch.float32), dim=-1)
+
+
+def g100(a, b, seed=0):
+    """mean over random 100-galleries of tie-aware R@1 (both directions) and Hungarian pairing accuracy"""
+    rng = np.random.default_rng(seed)
+    r, p = [], []
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy()
+        d = np.diag(sim)[:, None]
+        r.append(((sim > d).sum(1) == 0).mean() / 2 + ((sim.T > d).sum(1) == 0).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        p.append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return float(np.mean(r)), float(np.mean(p))
+
+
+rows = []
+
+
+def add(dataset, fold, pair, za, zb, kind, circular):
+    r, p = g100(za, zb)
+    rows.append({"dataset": dataset, "fold": fold, "pair": pair, "kind": kind, "circular": circular, "n": len(za),
+                 "R@1_g100": r, "pair_acc_g100": p})
+
+
+# ---------------- RNA: STPath log1p(counts) -> log1p CP10k on the HEST HVGs (same input as the HEST-trained head)
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def virtual_rna(st):
+    g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+    c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+    c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+    return torch.from_numpy(c[:, pd.Series(range(len(g)), index=g).loc[hvg].values])
+
+
+rna_heads = [load_mlp(torch.load(os.path.join(RUNS, "gene", f"uni2_mlp_fold{k}", "head.pt"), map_location="cpu")) for k in range(5)]
+text_head = load_mlp(torch.load(os.path.join(RUNS, "text_pathgen", "uni2_biomedclip", "head.pt"), map_location="cpu"))
+
+
+def mif_heads(enc, k, split="patient_5fold"):
+    f = os.path.join(RUNS, "bench_fm", f"{split}_fold{k}_uni2_fm-{enc}_fm-geneformer_pathgenclip", "heads.pt")
+    if not os.path.exists(f):
+        return None
+    d = torch.load(f, map_location="cpu")
+    h = load_mlp(d["state"]["if"])
+    return lambda x: h((torch.as_tensor(x, dtype=torch.float32) - d["if_mean"]) / d["if_std"])
+
+
+# ---------------- (A) HEST: H&E <-> real RNA vs H&E <-> virtual RNA, same spots, HEST test folds
+M = pd.read_csv(os.path.join(H, "meta.csv"))
+hv = pd.read_csv(os.path.join(CACHE, "stpath", "hest_val_meta.csv"))
+assert (hv["sample"].values == M["sample"].values).all() and (hv.spot.values == M.spot.values).all()
+he_h = np.load(os.path.join(H, "he_uni2.npy"))[:, 0].astype(np.float32)
+real_h = torch.from_numpy(np.load(os.path.join(H, "hvg.npy")).astype(np.float32))
+virt_h = virtual_rna("hest_val")
+with torch.no_grad():
+    for k in range(5):
+        te = torch.load(os.path.join(RUNS, "gene", f"uni2_mlp_fold{k}", "test.pt"), weights_only=False)["idx"]
+        zr, zv, zh = nz(rna_heads[k](real_h[te])), nz(rna_heads[k](virt_h[te])), nz(he_h[te])
+        for sub, m in [("all", np.ones(len(te), bool)), ("STPath-heldout", hv.hest_bench_heldout.values[te])]:
+            if m.sum() < G100:
+                continue
+            add(f"HEST ({sub})", k, "H&E<->RNA real", zh[m], zr[m], "trained (real)", False)
+            add(f"HEST ({sub})", k, "H&E<->RNA virtual", zh[m], zv[m], "virtual calibration", True)
+            add(f"HEST ({sub})", k, "RNA real<->RNA virtual", zr[m], zv[m], "virtual calibration", False)
+print("HEST done", flush=True)
+
+# ---------------- (B) ORION: patient 5-fold test tiles
+df = load_all_tiles()
+assert (pd.read_csv(os.path.join(CACHE, "stpath", "orion_meta.csv")).key.values == df.image_path.values).all()
+he_o = np.load(os.path.join(CACHE, "he_uni2.npy"))[:, 0].astype(np.float32)
+rna_o = virtual_rna("orion")
+if17 = np.load(os.path.join(CACHE, "if_virtues.npy"))[:, 0].astype(np.float32)
+f_sh = os.path.join(CACHE, "if_virtues_shared.npy")
+f_vo = os.path.join(CACHE, "if_virtues_shared_virtual_orion.npy")
+ifs_r = np.load(f_sh)[:, 0].astype(np.float32) if os.path.exists(f_sh) else None
+ifs_v = np.load(f_vo)[:, 0].astype(np.float32) if os.path.exists(f_vo) else None
+f_gt = os.path.join(CACHE, "orion_gentext_text_biomedclip.npy")  # synthetic GPT captions of the ORION H&E (32_ + 08_)
+gt_o = torch.from_numpy(np.load(f_gt)[:, 0].astype(np.float32)) if os.path.exists(f_gt) else None
+with torch.no_grad():
+    for k in range(5):
+        s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+        te = s.tile_id[s.split == "test"].values
+        zh, zr = nz(he_o[te]), nz(rna_heads[k](rna_o[te]))
+        add("ORION", k, "H&E<->RNA virtual", zh, zr, "virtual (cross-dataset head)", True)
+        h17 = mif_heads("virtues", k)
+        if h17 is not None:
+            zp = nz(h17(if17[te]))
+            add("ORION", k, "H&E<->mIF real (17 ch)", zh, zp, "trained (real)", False)
+            add("ORION", k, "mIF real (17 ch)<->RNA virtual", zp, zr, "emergent, real x virtual", False)
+        hs = mif_heads("virtues_shared", k)
+        if hs is not None and ifs_r is not None:
+            zs = nz(hs(ifs_r[te]))
+            add("ORION", k, "H&E<->mIF real (10 ch)", zh, zs, "trained (real)", False)
+            add("ORION", k, "mIF real (10 ch)<->RNA virtual", zs, zr, "emergent, real x virtual", False)
+            if ifs_v is not None:
+                zv = nz(hs(ifs_v[te]))
+                add("ORION", k, "H&E<->mIF virtual (10 ch)", zh, zv, "virtual calibration", True)
+                add("ORION", k, "mIF real<->mIF virtual (10 ch)", zs, zv, "virtual calibration", False)
+                add("ORION", k, "mIF virtual<->RNA virtual", zv, zr, "emergent, virtual x virtual", True)
+        if gt_o is not None:  # synthetic text is written from the same H&E -> circular with H&E and H&E-derived data
+            zt = nz(text_head(gt_o[te]))
+            add("ORION", k, "H&E<->text synthetic", zh, zt, "virtual (text head from PathGen)", True)
+            add("ORION", k, "text synthetic<->RNA virtual", zt, zr, "emergent, virtual x virtual", True)
+            if h17 is not None:
+                add("ORION", k, "text synthetic<->mIF real (17 ch)", zt, zp, "emergent, real x virtual", False)
+            if hs is not None and ifs_r is not None:
+                add("ORION", k, "text synthetic<->mIF real (10 ch)", zt, zs, "emergent, real x virtual", False)
+print("ORION done", flush=True)
+
+# ---------------- (C) TCGA quadruplets: H&E, real text, virtual mIF, virtual RNA
+I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+P = pd.read_csv(os.path.join(CACHE, "pathgen_meta.csv"))
+seen = set(P.patient[P.split.isin(["train", "val"])])
+ev = I.index[~I.patient.isin(seen)].values
+rng = np.random.default_rng(0)
+t1 = T[T.img_idx.isin(ev)].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+ev = t1.index.values  # images with >= 1 text, patients unseen by the text head
+print(f"TCGA eval: {len(ev)} images, {I.patient.iloc[ev].nunique()} patients (excluded {len(seen)} text-head patients)", flush=True)
+he_t = np.load(os.path.join(CACHE, "allimg_he_uni2.npy"))[ev, 0].astype(np.float32)
+tx = torch.from_numpy(np.load(os.path.join(CACHE, "alltxt_text_biomedclip.npy"))[t1.values, 0].astype(np.float32))
+rna_t = virtual_rna("tcga_text")[ev]
+f_vt = os.path.join(CACHE, "if_virtues_shared_virtual_tcga_text.npy")
+ifv_t = np.load(f_vt)[ev, 0].astype(np.float32) if os.path.exists(f_vt) else None
+with torch.no_grad():
+    zh, zt = nz(he_t), nz(text_head(tx))
+    for k in range(5):
+        zr = nz(rna_heads[k](rna_t))
+        Z = {"H&E": zh, "text": zt, "RNA virtual": zr}
+        hs = mif_heads("virtues_shared", k)
+        if hs is not None and ifv_t is not None:
+            Z["mIF virtual"] = nz(hs(ifv_t))
+        names = list(Z)
+        for i in range(len(names)):
+            for j in range(i + 1, len(names)):
+                a, b = names[i], names[j]
+                if (a, b) == ("H&E", "text") and k > 0:
+                    continue  # does not depend on the fold heads
+                virt = [x for x in (a, b) if "virtual" in x]
+                circ = len(virt) == 2 or (len(virt) == 1 and "H&E" in (a, b))
+                kind = "trained (real)" if (a, b) == ("H&E", "text") else \
+                    "emergent, real x virtual" if len(virt) == 1 and "H&E" not in (a, b) else \
+                    "emergent, virtual x virtual" if len(virt) == 2 else "virtual (cross-dataset head)"
+                add("TCGA", k, f"{a}<->{b}", Z[a], Z[b], kind, circ)
+print("TCGA done", flush=True)
+
+# ---------------- (D) mIF as 10 channel MEANS (GigaTIME per-tile means are reliable, its pixel maps are not):
+# MLP head on rank-gaussianised shared-panel means, trained on REAL ORION (patient_5fold fold k train/val) vs UNI2 H&E;
+# applied unchanged to virtual means (rank-gaussianised within their own set)
+import json
+from scipy.special import ndtri
+from scipy.stats import rankdata
+
+rg = lambda x: torch.from_numpy(ndtri((np.apply_along_axis(rankdata, 0, x) - 0.5) / len(x)).astype(np.float32))
+ORION_CH = ["Hoechst", "CD31", "CD45", "CD68", "CD4", "FOXP3", "CD8a", "CD45RO", "CD20", "PD-L1", "CD3e",
+            "CD163", "E-cadherin", "PD-1", "Ki67", "Pan-CK", "SMA"]  # order of orion_real_if_means.npy (25_)
+PANEL = [("Hoechst", ["DAPI"]), ("CD68", ["CD68"]), ("CD4", ["CD4"]), ("CD8a", ["CD8"]), ("CD20", ["CD20"]),
+         ("PD-L1", ["PD-L1"]), ("CD3e", ["CD3"]), ("Ki67", ["Ki67"]), ("Pan-CK", ["CK"]), ("SMA", ["Actin-D", "Transgelin"])]  # = 30_
+gch = json.load(open(os.path.join(CACHE, "gigatime", "channels.json")))["channels"]
+gmean = lambda m: np.stack([m[:, [gch.index(g) for g in gs]].mean(1) for _, gs in PANEL], 1)
+mr_o = rg(np.load(os.path.join(CACHE, "gigatime", "orion_real_if_means.npy"))[:, [ORION_CH.index(o) for o, _ in PANEL]])
+mv_o = rg(gmean(np.load(os.path.join(CACHE, "gigatime", "orion_means.npy"))))
+mv_t = rg(gmean(np.load(os.path.join(CACHE, "gigatime", "tcga_text_means.npy"))))[ev]
+HE8 = torch.from_numpy(np.load(os.path.join(CACHE, "he_uni2.npy")).astype(np.float32))
+
+
+def train_means_head(tr, va, epochs=150, bs=256, lr=1e-3, seed=0):
+    torch.manual_seed(seed); rng_ = np.random.default_rng(seed)
+    h = mlp(mr_o.shape[1])
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32))
+    opt = torch.optim.AdamW(list(h.parameters()) + [logit], lr=lr, weight_decay=0.05)
+    best, state = -1, None
+    for ep in range(epochs):
+        h.train()
+        perm = rng_.permutation(tr)
+        for s0 in range(0, len(perm) - bs + 1, bs):
+            b = torch.as_tensor(perm[s0:s0 + bs])
+            za = F.normalize(HE8[b, torch.randint(0, 8, (len(b),))], dim=-1)
+            lg = logit.exp().clamp(max=100) * za @ F.normalize(h(mr_o[b]), dim=-1).T
+            t = torch.arange(len(b))
+            loss = (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+            opt.zero_grad(); loss.backward(); opt.step()
+        h.eval()
+        with torch.no_grad():
+            a, m = nz(HE8[va, 0]), nz(h(mr_o[va]))
+            sim = a @ m.T
+            v = float(((sim > sim.diag()[:, None]).sum(1) == 0).float().mean())
+        if v > best:
+            best, state = v, {k_: x.clone() for k_, x in h.state_dict().items()}
+    h.load_state_dict(state); h.eval()
+    return h
+
+
+for k in range(5):
+    s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+    tr, va, te = (s.tile_id[s.split == x].values for x in ["train", "val", "test"])
+    h = train_means_head(tr, va)
+    with torch.no_grad():
+        zh, zr = nz(he_o[te]), nz(rna_heads[k](rna_o[te]))
+        zmr, zmv = nz(h(mr_o[te])), nz(h(mv_o[te]))
+        add("ORION", k, "H&E<->mIF real (10 means)", zh, zmr, "trained (real)", False)
+        add("ORION", k, "H&E<->mIF virtual (10 means)", zh, zmv, "virtual calibration", True)
+        add("ORION", k, "mIF real<->mIF virtual (10 means)", zmr, zmv, "virtual calibration", False)
+        add("ORION", k, "mIF real (10 means)<->RNA virtual", zmr, zr, "emergent, real x virtual", False)
+        add("ORION", k, "mIF virtual (10 means)<->RNA virtual", zmv, zr, "emergent, virtual x virtual", True)
+        if gt_o is not None:
+            zt = nz(text_head(gt_o[te]))
+            add("ORION", k, "text synthetic<->mIF real (10 means)", zt, zmr, "emergent, real x virtual", False)
+            add("ORION", k, "text synthetic<->mIF virtual (10 means)", zt, zmv, "emergent, virtual x virtual", True)
+        zmt = nz(h(mv_t))
+        add("TCGA", k, "H&E<->mIF virtual (10 means)", nz(he_t), zmt, "virtual (cross-dataset head)", True)
+        add("TCGA", k, "text<->mIF virtual (10 means)", nz(text_head(tx)), zmt, "emergent, real x virtual", False)
+        add("TCGA", k, "RNA virtual<->mIF virtual (10 means)", nz(rna_heads[k](rna_t)), zmt, "emergent, virtual x virtual", True)
+    print("means head fold", k, flush=True)
+
+# ---------------- (E) calibration of the synthetic text: TCGA patches with REAL text AND a synthetic caption (27_ tcga_calib)
+f_cm, f_cf = os.path.join(CACHE, "tcga_calib_meta.csv"), os.path.join(CACHE, "tcga_calib_text_biomedclip.npy")
+if os.path.exists(f_cf):
+    CM = pd.read_csv(f_cm)
+    pos = pd.Series(range(len(ev)), index=ev)
+    assert CM.img_idx.isin(ev).all()
+    j = pos.loc[CM.img_idx].values  # rows of the TCGA eval arrays
+    with torch.no_grad():
+        zs = nz(text_head(torch.from_numpy(np.load(f_cf)[:, 0].astype(np.float32))))
+        zr_txt, zh_c = nz(text_head(tx))[j], nz(he_t)[j]
+        add("TCGA calib", 0, "H&E<->text real", zh_c, zr_txt, "trained (real)", False)
+        add("TCGA calib", 0, "H&E<->text synthetic", zh_c, zs, "virtual calibration", True)
+        add("TCGA calib", 0, "text real<->text synthetic", zr_txt, zs, "virtual calibration", False)
+        for k in range(5):
+            zrna = nz(rna_heads[k](rna_t))[j]
+            add("TCGA calib", k, "text real<->RNA virtual", zr_txt, zrna, "emergent, real x virtual", False)
+            add("TCGA calib", k, "text synthetic<->RNA virtual", zs, zrna, "emergent, virtual x virtual", True)
+    print("calibration done", len(CM), flush=True)
+
+R = pd.DataFrame(rows)
+R.to_csv(os.path.join(OUT, "virtual_validation.csv"), index=False)
+pd.set_option("display.width", 250); pd.set_option("display.max_colwidth", 60)
+S = R.groupby(["dataset", "pair", "kind", "circular"])[["R@1_g100", "pair_acc_g100", "n"]].agg(["mean", "std"]).round(3)
+print(S.drop(columns=[("n", "std")]).to_string())

experiments/32_gentext_prep.py

@@ -0,0 +1,55 @@
+"""Synthetic captions (27_azure_caption.py, GPT, 3-shot PathGen style) -> per-set meta for 08_extract_text.py.
+VALIDATION ONLY: these texts are never used to train any head.
+Output cache/orion_gentext_meta.csv (row i = ORION tile_id i of load_all_tiles(); `caption` column)
+"""
+import json
+import os
+
+import pandas as pd
+
+from common import CACHE, load_all_tiles
+
+C = pd.DataFrame([json.loads(l) for l in open(os.path.join(CACHE, "gen_text", "captions.jsonl"))])
+C = C[(C.set == "orion") & C.caption.notna() & (C.caption.str.len() > 0)].drop_duplicates("path", keep="last")
+cap = dict(zip(C.path.map(os.path.basename), C.caption))
+df = load_all_tiles()
+df["caption"] = df.image_path.map(lambda p: cap.get(os.path.basename(p)))
+print(f"ORION tiles {len(df)}, with synthetic caption {df.caption.notna().sum()}")
+assert df.caption.notna().all(), "missing captions -> rerun 27_azure_caption.py (resumable)"
+df[["tile_id", "image_path", "caption"]].to_csv(os.path.join(CACHE, "orion_gentext_meta.csv"), index=False)
+
+# TCGA calibration set (27_ --sets tcga_calib): synthetic captions of patches that also have REAL text
+A = pd.DataFrame([json.loads(l) for l in open(os.path.join(CACHE, "gen_text", "captions.jsonl"))])
+A = A[(A.set == "tcga_calib") & A.caption.notna()].drop_duplicates("path", keep="last")
+if len(A):
+    I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+    shots = [s["key"][:12] for s in json.load(open(os.path.join(CACHE, "gen_text", "prompt.json")))["shots"]]
+    A["img_idx"] = A.path.map(dict(zip(I.image_path, I.index)))
+    A = A[A.img_idx.notna() & ~A.img_idx.map(lambda i: I.patient[int(i)] in shots)]  # drop few-shot patients
+    A["img_idx"] = A.img_idx.astype(int)
+    A[["img_idx", "path", "caption"]].reset_index(drop=True).to_csv(os.path.join(CACHE, "tcga_calib_meta.csv"), index_label="idx")
+    print(f"TCGA calibration captions: {len(A)} (few-shot patients excluded: {shots})")
+
+# external test sets (27_ --sets dlpfc idc): row i = row i of cache/external/<set>_meta.csv
+for st in ("dlpfc", "idc"):
+    X = pd.DataFrame([json.loads(l) for l in open(os.path.join(CACHE, "gen_text", "captions.jsonl"))])
+    X = X[(X.set == st) & X.caption.notna() & (X.caption.str.len() > 0)].drop_duplicates("key", keep="last")
+    if not len(X):
+        continue
+    n = len(pd.read_csv(os.path.join(CACHE, "external", f"{st}_meta.csv")))
+    cap = dict(zip(X.path.str.split(":").str[2].astype(int), X.caption))
+    out = pd.DataFrame({"row": range(n), "caption": [cap.get(i) for i in range(n)]})
+    print(f"{st}: {n} spots, with synthetic caption {out.caption.notna().sum()}")
+    assert out.caption.notna().all(), "missing captions -> rerun 27_azure_caption.py --sets dlpfc idc (resumable)"
+    out.to_csv(os.path.join(CACHE, f"{st}_gentext_meta.csv"), index=False)
+
+# TCGA images without real text after junk removal (40_clean_text.py; 27_ --sets tcga_notext). A caption made for
+# the same image under another set (tcga_calib) is reused. Row i of the output = row i of notext_meta.csv.
+N = pd.read_csv(os.path.join(CACHE, "notext_meta.csv"))
+X = pd.DataFrame([json.loads(l) for l in open(os.path.join(CACHE, "gen_text", "captions.jsonl"))])
+X = X[X.caption.notna() & (X.caption.str.len() > 0)].drop_duplicates("path", keep="last")
+cap = dict(zip(X.path.map(os.path.abspath), X.caption))
+N["caption"] = N.image_path.map(lambda p: cap.get(os.path.abspath(p)))
+print(f"TCGA no-text images {len(N)}, with synthetic caption {N.caption.notna().sum()}")
+assert N.caption.notna().all(), "missing captions -> rerun 27_azure_caption.py --sets tcga_notext"
+N[["img_idx", "image_path", "patient", "caption"]].to_csv(os.path.join(CACHE, "tcga_notext_meta.csv"), index=False)

experiments/33_final_eval.py

@@ -0,0 +1,356 @@
+"""Final protocol: every head is trained on REAL pairs only, with an explicit patient-level train / val / test split per
+real dataset; virtual modalities (STPath RNA, GigaTIME mIF, GPT synthetic captions) are used ONLY at test time.
+
+Real datasets and splits (5 folds each; fold k = train / val (early stopping only) / test; patients never cross splits)
+  ORION  H&E <-> mIF  : splits/patient_5fold/fold<k>.csv (5-fold by patient, val = 3 training patients)
+                        + splits/orion_official/fold0.csv (the user's ORION train/val/test csvs)
+  HEST   H&E <-> RNA  : GroupKFold(5) over study|patient groups, val = 10% of training groups (= 23_bind_gene.py)
+  TCGA   H&E <-> text : GroupKFold(5) over patients on ALL image-text pairs, val = 10% of training patients (= 20_pairing.py)
+The HEST and TCGA split memberships are written to splits/final/{hest,tcga}_fold<k>.csv.
+
+Heads (frozen UNI2 H&E = anchor space):
+  mIF  runs/bench_fm/<split>_fold<k>_uni2_fm-virtues[_shared]_.../heads.pt       (17_bench_fm.py --save_heads)
+  RNA  runs/gene/uni2_mlp_fold<k>/head.pt                                         (23_bind_gene.py)
+  text runs/text_clean/uni2_biomedclip_fold<k>/head.pt trained here (text_head recipe of 20_pairing.py) on CLEAN real
+       texts only: junk texts (answer letters, "error", < 5 words; 40_clean_text.py) are removed from train/val/test.
+       The 256 images left without real text keep their split; their few-shot GPT captions (27_ --sets tcga_notext)
+       are evaluated only (pooled over the test folds, each image through its own test-fold head).
+       Previous heads trained with the junk texts: runs/text_all (results in results/before_text_clean/).
+Evaluation is on the TEST split of fold k with the fold-k heads (a head from another dataset never saw that dataset).
+  part 1 "real"    : both sides measured           (H&E<->mIF, H&E<->RNA, H&E<->text)
+  part 2 "virtual" : >= 1 side virtual, same test items; "circular" = both sides derived from the same H&E
+Metrics: random 100-item galleries (50 draws), both directions averaged: tie-aware R@1/R@5/R@10, MRR, and one-to-one
+Hungarian pairing accuracy (chance R@1 = pair acc = 0.01, R@5 = 0.05, R@10 = 0.10).
+mIF everywhere = the 10-channel panel shared by ORION and GigaTIME (30_virtues_shared.py); the full 17-channel ORION
+panel appears only as a reference row. "[qm]" = GigaTIME pixels quantile-mapped per channel onto the real fold-k
+training-tile intensity distribution before the identical preprocessing (30_ virtual_qm).
+Outputs results/final_long.csv, results/final_summary.csv; split sizes -> results/final_splits.csv
+"""
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+from sklearn.model_selection import GroupKFold
+
+from common import CACHE, ROOT, RUNS, SPLITS, load_all_tiles
+
+OUT = os.path.join(ROOT, "results")
+H = os.path.join(CACHE, "hest")
+G100, DRAWS, K = 100, 50, 5
+dev = "cuda" if torch.cuda.is_available() else "cpu"
+os.makedirs(os.path.join(SPLITS, "final"), exist_ok=True)
+
+
+def mlp(din, dout=1536, hid=512):  # same head as 12_/17_/20_/23_
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def load_mlp(sd):
+    h = mlp(sd["0.weight"].shape[1], sd[f"{len(mlp(1)) - 1}.weight"].shape[0]); h.load_state_dict(sd); return h.eval()
+
+
+def nz(x):
+    return F.normalize(torch.as_tensor(np.asarray(x), dtype=torch.float32), dim=-1)
+
+
+def g100(a, b, seed=0):
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy()
+        d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)  # tie-aware: 0 = correct item ranked first
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+rows, split_rows = [], []
+
+
+def add(dataset, split, fold, part, pair, za, zb, kind, circular):
+    rows.append({"dataset": dataset, "split": split, "fold": fold, "part": part, "pair": pair, "kind": kind,
+                 "circular": circular, "n_test": len(za), **g100(za, zb)})
+
+
+def log_split(dataset, split, fold, s, unit):
+    for x in ["train", "val", "test"]:
+        split_rows.append({"dataset": dataset, "split": split, "fold": fold, "role": x,
+                           "n_items": int((s.split == x).sum()), f"n_{unit}": s[unit][s.split == x].nunique(),
+                           "unit": unit})
+    g = {x: set(s[unit][s.split == x]) for x in ["train", "val", "test"]}
+    assert not (g["train"] & g["val"] or g["train"] & g["test"] or g["val"] & g["test"]), (dataset, fold, "leak")
+
+
+# ================= virtual RNA: STPath counts -> log1p CP10k on the 2000 HEST HVGs (input of the HEST-trained head)
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def virtual_rna(st):
+    g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+    c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+    c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+    return torch.from_numpy(c[:, pd.Series(range(len(g)), index=g).loc[hvg].values])
+
+
+def mif_head(enc, k, split):
+    d = torch.load(os.path.join(RUNS, "bench_fm", f"{split}_fold{k}_uni2_fm-{enc}_fm-geneformer_pathgenclip", "heads.pt"),
+                   map_location="cpu")
+    h = load_mlp(d["state"]["if"])
+    return lambda x: h((torch.as_tensor(x, dtype=torch.float32) - d["if_mean"]) / d["if_std"])
+
+
+rna_heads = [load_mlp(torch.load(os.path.join(RUNS, "gene", f"uni2_mlp_fold{k}", "head.pt"), map_location="cpu")) for k in range(K)]
+
+# ================= TCGA all image-text pairs: split + text heads (real text only)
+I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+JUNK = pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values
+assert len(JUNK) == len(T)
+IMG = torch.from_numpy(np.load(os.path.join(CACHE, "allimg_he_uni2.npy")).astype(np.float32))       # (n_img, 8, 1536)
+TXT = torch.from_numpy(np.load(os.path.join(CACHE, "alltxt_text_biomedclip.npy"))[:, 0].astype(np.float32))
+t_img = torch.as_tensor(T.img_idx.values)  # indexed by the ORIGINAL text row (kept after filtering)
+T = T[~JUNK]
+NOTXT = pd.read_csv(os.path.join(CACHE, "tcga_notext_meta.csv"))
+assert not NOTXT.img_idx.isin(T.img_idx).any()
+TX_N = torch.from_numpy(np.load(os.path.join(CACHE, "tcga_notext_text_biomedclip.npy"))[:, 0].astype(np.float32))
+
+
+def one_text_per_image(t_ids, rng):
+    sub = T.loc[t_ids]
+    pick = sub.groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values  # img ids, text ids
+
+
+def train_text_head(tr_t, va_t, epochs=60, bs=256, lr=3e-4, seed=0):
+    """20_pairing.py text_head: frozen UNI2 image = anchor, MLP on frozen BiomedCLIP text, multi-positive InfoNCE"""
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    X, Tx, ti = IMG.to(dev), TXT.to(dev), t_img.to(dev)
+    th = mlp(Tx.shape[1], X.shape[-1]).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(th.parameters()) + [logit], lr=lr, weight_decay=0.05)
+    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=epochs * max(1, len(tr_t) // bs), pct_start=0.05)
+    va_i, va_tt = one_text_per_image(va_t, np.random.default_rng(0))
+    za_va = F.normalize(X[torch.as_tensor(va_i, device=dev), 0], dim=-1)
+    best, state = -1, None
+    for ep in range(epochs):
+        th.train()
+        perm = rng.permutation(tr_t)
+        for s in range(0, len(perm) - bs + 1, bs):
+            b = torch.as_tensor(perm[s:s + bs], device=dev)
+            ii = ti[b]
+            zi = F.normalize(X[ii, torch.randint(0, 8, (len(b),), device=dev)], dim=-1)
+            lg = logit.exp().clamp(max=100) * zi @ F.normalize(th(Tx[b]), dim=-1).T
+            pos = (ii[:, None] == ii[None, :]).float()
+            tgt = pos / pos.sum(1, keepdim=True)
+            loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sched.step()
+        th.eval()
+        with torch.no_grad():
+            zt = F.normalize(th(Tx[torch.as_tensor(va_tt, device=dev)]), dim=-1)
+            sim = za_va @ zt.T
+            v = float((((sim > sim.diag()[:, None]).sum(1) == 0).float().mean() + ((sim.T > sim.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+        if v > best:
+            best, state = v, {k_: x.detach().cpu().clone() for k_, x in th.state_dict().items()}
+    return state, best
+
+
+tcga_folds = list(GroupKFold(K).split(I, groups=I.patient))
+text_heads, tcga_test, tcga_notext_test = [], [], []
+for k, (trv, te) in enumerate(tcga_folds):
+    pats = np.random.default_rng(k).permutation(sorted(I.patient.iloc[trv].unique()))
+    va_p = set(pats[: max(1, len(pats) // 10)])
+    sp = pd.Series("test", index=I.index)
+    sp.iloc[trv] = np.where(I.patient.iloc[trv].isin(va_p), "val", "train")
+    s = pd.DataFrame({"img_idx": I.index, "patient": I.patient, "split": sp.values})
+    s.to_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv"), index=False)
+    log_split("TCGA image-text", "patient 5-fold", k, s, "patient")
+    tsplit = sp.loc[T.img_idx].values
+    tr_t, va_t, te_t = (T.index[tsplit == x].values for x in ["train", "val", "test"])
+    f = os.path.join(RUNS, "text_clean", f"uni2_biomedclip_fold{k}", "head.pt")
+    if not os.path.exists(f):
+        state, best = train_text_head(tr_t, va_t)
+        os.makedirs(os.path.dirname(f), exist_ok=True); torch.save(state, f)
+        print(f"text head fold {k}: val R@1 {best:.3f} (train pairs {len(tr_t)}, val pairs {len(va_t)})", flush=True)
+    text_heads.append(load_mlp(torch.load(f, map_location="cpu")))
+    tcga_test.append(one_text_per_image(te_t, np.random.default_rng(k)))
+    tcga_notext_test.append(np.flatnonzero((sp.loc[NOTXT.img_idx] == "test").values))  # rows of NOTXT
+
+# ================= (1) HEST: H&E <-> RNA real; virtual RNA (STPath) on the same test spots
+M = pd.read_csv(os.path.join(H, "meta.csv"))
+hv = pd.read_csv(os.path.join(CACHE, "stpath", "hest_val_meta.csv"))
+assert (hv["sample"].values == M["sample"].values).all() and (hv.spot.values == M.spot.values).all()
+he_h = np.load(os.path.join(H, "he_uni2.npy"))[:, 0].astype(np.float32)
+real_h = torch.from_numpy(np.load(os.path.join(H, "hvg.npy")).astype(np.float32))
+virt_h = virtual_rna("hest_val")
+with torch.no_grad():
+    for k, (trv, te) in enumerate(GroupKFold(K).split(M, groups=M.group)):  # identical folds to 23_bind_gene.py
+        grp = np.random.default_rng(k).permutation(sorted(M.group.iloc[trv].unique()))
+        va_g = set(grp[: max(1, len(grp) // 10)])
+        sp = np.array(["test"] * len(M), dtype=object)
+        sp[trv] = np.where(M.group.iloc[trv].isin(va_g), "val", "train")
+        s = pd.DataFrame({"spot_idx": M.index, "sample": M["sample"], "group": M.group, "split": sp})
+        s.to_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"), index=False)
+        log_split("HEST spot-RNA", "group 5-fold", k, s, "group")
+        te_saved = torch.load(os.path.join(RUNS, "gene", f"uni2_mlp_fold{k}", "test.pt"), weights_only=False)["idx"]
+        assert np.array_equal(np.sort(te_saved), np.sort(te)), "HEST fold mismatch with 23_bind_gene.py"
+        zr, zv, zh = nz(rna_heads[k](real_h[te])), nz(rna_heads[k](virt_h[te])), nz(he_h[te])
+        add("HEST", "group 5-fold", k, "real", "H&E<->RNA", zh, zr, "trained pair", False)
+        for sub, m in [("all test spots", np.ones(len(te), bool)), ("STPath-unseen test spots", hv.hest_bench_heldout.values[te])]:
+            if m.sum() >= G100:
+                if sub != "all test spots":
+                    add(f"HEST ({sub})", "group 5-fold", k, "real", "H&E<->RNA", zh[m], zr[m], "trained pair", False)
+                add(f"HEST ({sub})" if sub != "all test spots" else "HEST", "group 5-fold", k, "virtual",
+                    "H&E<->RNA(virtual)", zh[m], zv[m], "calibration: virtual replaces real", True)
+                add(f"HEST ({sub})" if sub != "all test spots" else "HEST", "group 5-fold", k, "virtual",
+                    "RNA<->RNA(virtual)", zr[m], zv[m], "calibration: real vs virtual", False)
+print("HEST done", flush=True)
+
+# ================= (2) ORION: H&E <-> mIF real; virtual RNA / synthetic text / virtual mIF on the same test tiles
+df = load_all_tiles()
+assert (pd.read_csv(os.path.join(CACHE, "stpath", "orion_meta.csv")).key.values == df.image_path.values).all()
+he_o = np.load(os.path.join(CACHE, "he_uni2.npy"))[:, 0].astype(np.float32)
+rna_o = virtual_rna("orion")
+if17 = np.load(os.path.join(CACHE, "if_virtues.npy"))[:, 0].astype(np.float32)
+ifs_r = np.load(os.path.join(CACHE, "if_virtues_shared.npy"))[:, 0].astype(np.float32)
+ifs_v = np.load(os.path.join(CACHE, "if_virtues_shared_virtual_orion.npy"))[:, 0].astype(np.float32)
+gt_o = torch.from_numpy(np.load(os.path.join(CACHE, "orion_gentext_text_biomedclip.npy"))[:, 0].astype(np.float32))
+
+
+def qm_file(st, k):
+    f = os.path.join(CACHE, f"if_virtues_shared_virtual_qm_{st}_fold{k}.npy")
+    return np.load(f)[:, 0].astype(np.float32) if os.path.exists(f) else None
+
+
+qm_o = lambda k, _c={}: _c.setdefault(k, qm_file("orion", k))
+# MIPHEI-ViT virtual mIF on the 16-channel ORION panel (30_ --m16). MIPHEI was trained on THIS ORION tile set with the
+# official split -> clean only on the official TEST tiles; on patient 5-fold most test tiles were MIPHEI training tiles.
+f_m16, f_mo = os.path.join(CACHE, "if_virtues_m16.npy"), os.path.join(CACHE, "if_virtues_m16_virtual_miphei_orion.npy")
+m16_r = np.load(f_m16)[:, 0].astype(np.float32) if os.path.exists(f_m16) else None
+m16_v = np.load(f_mo)[:, 0].astype(np.float32) if os.path.exists(f_mo) else None
+with torch.no_grad():
+    for split, folds in [("patient 5-fold", range(K)), ("official", [0])]:
+        sdir = "patient_5fold" if split == "patient 5-fold" else "orion_official"
+        for k in folds:
+            s = pd.read_csv(os.path.join(SPLITS, sdir, f"fold{k}.csv"))
+            log_split("ORION H&E-mIF", split, k, s, "patient")
+            te = s.tile_id[s.split == "test"].values
+            h17, hsh = mif_head("virtues", k, sdir), mif_head("virtues_shared", k, sdir)
+            zh, zp, zs = nz(he_o[te]), nz(h17(if17[te])), nz(hsh(ifs_r[te]))
+            zr, zt = nz(rna_heads[k](rna_o[te])), nz(text_heads[k](gt_o[te]))  # heads of other datasets: never saw ORION
+            A = lambda *a: add("ORION", split, k, *a)
+            # mIF = the UNIFIED 10-channel panel shared with GigaTIME everywhere; 17-ch kept only as a reference row
+            A("real", "H&E<->mIF (17-ch full ORION panel, reference)", zh, zp, "trained pair", False)
+            A("real", "H&E<->mIF", zh, zs, "trained pair", False)
+            A("virtual", "mIF<->RNA(virtual)", zs, zr, "emergent: real x virtual", False)
+            A("virtual", "mIF<->text(synthetic)", zs, zt, "emergent: real x virtual", False)
+            A("virtual", "H&E<->RNA(virtual)", zh, zr, "virtual from this H&E", True)
+            A("virtual", "H&E<->text(synthetic)", zh, zt, "virtual from this H&E", True)
+            A("virtual", "RNA(virtual)<->text(synthetic)", zr, zt, "emergent: virtual x virtual", True)
+            if m16_r is not None and os.path.exists(os.path.join(RUNS, "bench_fm", f"{sdir}_fold{k}_uni2_fm-virtues_m16_fm-geneformer_pathgenclip", "heads.pt")):
+                h16 = mif_head("virtues_m16", k, sdir)
+                z16 = nz(h16(m16_r[te]))
+                A("real", "H&E<->mIF (16-ch MIPHEI panel)", zh, z16, "trained pair", False)
+                A("virtual", "mIF(16ch)<->RNA(virtual)", z16, zr, "emergent: real x virtual", False)
+                A("virtual", "mIF(16ch)<->text(synthetic)", z16, zt, "emergent: real x virtual", False)
+                if m16_v is not None:
+                    seen = split != "official"  # 5-fold test tiles were (mostly) MIPHEI training tiles
+                    tag = " [MIPHEI-seen tiles]" if seen else ""
+                    zm = nz(h16(m16_v[te]))
+                    A("virtual", f"H&E<->mIF(MIPHEI){tag}", zh, zm, "calibration: virtual replaces real", True)
+                    A("virtual", f"mIF<->mIF(MIPHEI){tag}", z16, zm, "calibration: real vs virtual", False)
+                    A("virtual", f"mIF(MIPHEI)<->RNA(virtual){tag}", zm, zr, "emergent: virtual x virtual", True)
+                    A("virtual", f"mIF(MIPHEI)<->text(synthetic){tag}", zm, zt, "emergent: virtual x virtual", True)
+            for tag, f in [("", ifs_v[te]), (" [qm]", qm_o(k)[te] if split == "patient 5-fold" and qm_o(k) is not None else None)]:
+                if f is None:
+                    continue  # [qm] = virtual pixels quantile-mapped onto the fold-k TRAIN real distribution (30_ virtual_qm)
+                zv = nz(hsh(f))
+                A("virtual", f"H&E<->mIF(virtual){tag}", zh, zv, "calibration: virtual replaces real", True)
+                A("virtual", f"mIF<->mIF(virtual){tag}", zs, zv, "calibration: real vs virtual", False)
+                A("virtual", f"mIF(virtual){tag}<->RNA(virtual)", zv, zr, "emergent: virtual x virtual", True)
+                A("virtual", f"mIF(virtual){tag}<->text(synthetic)", zv, zt, "emergent: virtual x virtual", True)
+print("ORION done", flush=True)
+
+# ================= (3) TCGA: H&E <-> text real; virtual RNA / virtual mIF on the same test images
+rna_t = virtual_rna("tcga_text")
+ifv_t = np.load(os.path.join(CACHE, "if_virtues_shared_virtual_tcga_text.npy"))[:, 0].astype(np.float32)
+CM = pd.read_csv(os.path.join(CACHE, "tcga_calib_meta.csv"))
+syn_t = torch.from_numpy(np.load(os.path.join(CACHE, "tcga_calib_text_biomedclip.npy"))[:, 0].astype(np.float32))
+with torch.no_grad():
+    for k, (ii, tt) in enumerate(tcga_test):
+        zh, zt = nz(IMG[ii, 0]), nz(text_heads[k](TXT[tt]))
+        zr = nz(rna_heads[k](rna_t[ii]))
+        A = lambda *a: add("TCGA", "patient 5-fold", k, *a)
+        A("real", "H&E<->text", zh, zt, "trained pair", False)
+        A("virtual", "text<->RNA(virtual)", zt, zr, "emergent: real x virtual", False)
+        A("virtual", "H&E<->RNA(virtual)", zh, zr, "virtual from this H&E", True)
+        f_mt = os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy")
+        if os.path.exists(f_mt) and os.path.exists(os.path.join(RUNS, "bench_fm", f"patient_5fold_fold{k}_uni2_fm-virtues_m16_fm-geneformer_pathgenclip", "heads.pt")):
+            zm = nz(mif_head("virtues_m16", k, "patient_5fold")(np.load(f_mt, mmap_mode="r")[ii, 0].astype(np.float32)))
+            A("virtual", "text<->mIF(MIPHEI)", zt, zm, "emergent: real x virtual", False)
+            A("virtual", "H&E<->mIF(MIPHEI)", zh, zm, "virtual from this H&E", True)
+            A("virtual", "RNA(virtual)<->mIF(MIPHEI)", zr, zm, "emergent: virtual x virtual", True)
+        hsh = mif_head("virtues_shared", k, "patient_5fold")  # unified 10-ch panel head, trained on real ORION fold k
+        qt = qm_file("tcga_text", k)
+        for tag, f in [("", ifv_t), (" [qm]", qt)]:
+            if f is None:
+                continue
+            zv = nz(hsh(f[ii]))
+            A("virtual", f"text<->mIF(virtual){tag}", zt, zv, "emergent: real x virtual", False)
+            A("virtual", f"H&E<->mIF(virtual){tag}", zh, zv, "virtual from this H&E", True)
+            A("virtual", f"RNA(virtual)<->mIF(virtual){tag}", zr, zv, "emergent: virtual x virtual", True)
+        # synthetic-caption calibration: test images that have BOTH a real text and a GPT caption
+        pos = pd.Series(range(len(ii)), index=ii)
+        m = CM.img_idx.isin(ii).values
+        if m.sum() >= G100:
+            j = pos.loc[CM.img_idx[m]].values
+            zs = nz(text_heads[k](syn_t[m]))
+            B = lambda *a: add("TCGA (test images with GPT caption)", "patient 5-fold", k, *a)
+            B("real", "H&E<->text", zh[j], zt[j], "trained pair", False)
+            B("virtual", "H&E<->text(synthetic)", zh[j], zs, "calibration: virtual replaces real", True)
+            B("virtual", "text<->text(synthetic)", zt[j], zs, "calibration: real vs virtual", False)
+            B("virtual", "text<->RNA(virtual)", zt[j], zr[j], "emergent: real x virtual", False)
+            B("virtual", "text(synthetic)<->RNA(virtual)", zs, zr[j], "emergent: virtual x virtual", True)
+# images without any real text (after junk removal): GPT few-shot caption, pooled over folds (each image in the test
+# split of exactly one fold -> embedded with that fold's heads; all heads map into the same frozen UNI2 space)
+f_mt = os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy")
+shot_p = [x["key"][:12] for x in __import__("json").load(open(os.path.join(CACHE, "gen_text", "prompt.json")))["shots"]]
+P = {x: [] for x in ["h", "s", "r", "m"]}
+with torch.no_grad():
+    for k, r in enumerate(tcga_notext_test):
+        r = r[~NOTXT.patient.iloc[r].isin(shot_p).values]  # drop few-shot patients
+        ii = NOTXT.img_idx.values[r]
+        P["h"].append(nz(IMG[ii, 0])); P["s"].append(nz(text_heads[k](TX_N[r]))); P["r"].append(nz(rna_heads[k](rna_t[ii])))
+        if os.path.exists(f_mt):
+            P["m"].append(nz(mif_head("virtues_m16", k, "patient_5fold")(np.load(f_mt, mmap_mode="r")[ii, 0].astype(np.float32))))
+P = {x: torch.cat(v) for x, v in P.items() if v}
+B = lambda *a: add("TCGA (test images WITHOUT real text, pooled folds)", "patient 5-fold", "pooled", *a)
+B("virtual", "H&E<->text(synthetic)", P["h"], P["s"], "virtual from this H&E", True)
+B("virtual", "text(synthetic)<->RNA(virtual)", P["s"], P["r"], "emergent: virtual x virtual", True)
+if "m" in P:
+    B("virtual", "text(synthetic)<->mIF(MIPHEI)", P["s"], P["m"], "emergent: virtual x virtual", True)
+print(f"TCGA no-text images evaluated: {len(P['h'])}")
+print("TCGA done", flush=True)
+
+R = pd.DataFrame(rows)
+R.to_csv(os.path.join(OUT, "final_long.csv"), index=False)
+SP = pd.DataFrame(split_rows)
+SP.to_csv(os.path.join(OUT, "final_splits.csv"), index=False)
+cols = ["R@1", "R@5", "R@10", "MRR", "pair_acc"]
+S = R.groupby(["part", "dataset", "split", "pair", "kind", "circular"], sort=False)[cols + ["n_test"]].agg(["mean", "std"])
+S.round(4).to_csv(os.path.join(OUT, "final_summary.csv"))
+pd.set_option("display.width", 250); pd.set_option("display.max_colwidth", 50); pd.set_option("display.max_rows", 200)
+print(SP.groupby(["dataset", "split", "role"])[["n_items", "n_patient", "n_group"]].mean().round(0).to_string())
+for part in ["real", "virtual"]:
+    print(f"\n===== {part} (test split, mean over folds) =====")
+    print(R[R.part == part].groupby(["dataset", "split", "pair", "kind", "circular"], sort=False)[cols + ["n_test"]].mean().round(3).to_string())

experiments/34_gigatime_diag.py

@@ -0,0 +1,112 @@
+"""Why GigaTIME virtual mIF fails at instance-level pairing (ORION, 10 shared channels, same tiles as the real mIF).
+
+  (a) pooled vs WITHIN-slide Spearman of tile means: pairing in a 100-tile gallery needs tile-to-tile differences
+      inside a slide, while pooled correlation is dominated by slide/tissue-level differences
+  (b) direct real-mean <-> virtual-mean retrieval (no learned head): rank-gaussianised channel means, cosine, g100
+  (c) in-domain reference: ridge UNI2 H&E -> real means, trained on ORION patient_5fold train, scored on test
+      (what H&E can predict about these 10 channels when the model has seen ORION)
+  (d) input distribution: real mIF intensities vs GigaTIME sigmoid probabilities (what the VirTues head receives)
+Output results/gigatime_diag.csv + printed tables.
+"""
+import json
+import os
+
+import numpy as np
+import pandas as pd
+from scipy.special import ndtri
+from scipy.stats import rankdata, spearmanr
+
+from common import CACHE, ROOT, SPLITS, load_all_tiles
+
+OUT = os.path.join(ROOT, "results")
+G = os.path.join(CACHE, "gigatime")
+ORION_CH = ["Hoechst", "CD31", "CD45", "CD68", "CD4", "FOXP3", "CD8a", "CD45RO", "CD20", "PD-L1", "CD3e",
+            "CD163", "E-cadherin", "PD-1", "Ki67", "Pan-CK", "SMA"]
+PANEL = [("Hoechst", ["DAPI"]), ("CD68", ["CD68"]), ("CD4", ["CD4"]), ("CD8a", ["CD8"]), ("CD20", ["CD20"]),
+         ("PD-L1", ["PD-L1"]), ("CD3e", ["CD3"]), ("Ki67", ["Ki67"]), ("Pan-CK", ["CK"]), ("SMA", ["Actin-D", "Transgelin"])]
+gch = json.load(open(os.path.join(G, "channels.json")))["channels"]
+df = load_all_tiles()
+slide = df.in_slide_name.values
+real = np.load(os.path.join(G, "orion_real_if_means.npy"))[:, [ORION_CH.index(o) for o, _ in PANEL]]
+vm = np.load(os.path.join(G, "orion_means.npy"))
+virt = np.stack([vm[:, [gch.index(g) for g in gs]].mean(1) for _, gs in PANEL], 1)
+he = np.load(os.path.join(CACHE, "he_uni2.npy"))[:, 0].astype(np.float64)
+names = [o for o, _ in PANEL]
+rows = []
+
+
+def within(a, b):
+    """mean within-slide Spearman (slides with >= 20 tiles), tiles-weighted"""
+    r, w = [], []
+    for s in np.unique(slide):
+        m = slide == s
+        if m.sum() >= 20 and a[m].std() > 0 and b[m].std() > 0:
+            r.append(spearmanr(a[m], b[m])[0]); w.append(m.sum())
+    return float(np.average(r, weights=w))
+
+
+rg = lambda x: ndtri((np.apply_along_axis(rankdata, 0, x) - 0.5) / len(x))
+
+
+def g100(a, b, draws=50, seed=0, groups=None):
+    a = a / np.linalg.norm(a, axis=1, keepdims=True); b = b / np.linalg.norm(b, axis=1, keepdims=True)
+    rng = np.random.default_rng(seed); r = []
+    for _ in range(draws):
+        if groups is None:
+            s = rng.choice(len(a), min(100, len(a)), replace=False)
+        else:  # gallery drawn from ONE slide: only within-slide differences can help
+            g = rng.choice(groups); s = np.flatnonzero(slide == g); s = rng.choice(s, min(100, len(s)), replace=False)
+        sim = a[s] @ b[s].T; d = np.diag(sim)[:, None]
+        r.append(((sim > d).sum(1) == 0).mean() / 2 + ((sim.T > d).sum(1) == 0).mean() / 2)
+    return float(np.mean(r))
+
+
+# ---- (c) ridge UNI2 -> real means (in-domain reference), patient 5-fold test predictions
+pred = np.zeros_like(real, dtype=np.float64)
+R_rg = rg(real)
+for k in range(5):
+    s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+    tr, va, te = (s.tile_id[s.split == x].values for x in ["train", "val", "test"])
+    mu, sd = he[tr].mean(0), he[tr].std(0) + 1e-6
+    X = lambda i: (he[i] - mu) / sd
+    best = None
+    for lam in [1e1, 1e2, 1e3, 1e4]:  # lambda chosen on val
+        W = np.linalg.solve(X(tr).T @ X(tr) + lam * np.eye(he.shape[1]), X(tr).T @ R_rg[tr])
+        v = np.mean([spearmanr(X(va) @ W[:, c], R_rg[va, c])[0] for c in range(len(names))])
+        if best is None or v > best[0]:
+            best = (v, W)
+    pred[te] = X(te) @ best[1]
+
+# ---- (a) correlations
+for c, n in enumerate(names):
+    rows.append({"channel": n,
+                 "GigaTIME pooled": spearmanr(virt[:, c], real[:, c])[0], "GigaTIME within-slide": within(virt[:, c], real[:, c]),
+                 "UNI2-ridge pooled (test)": spearmanr(pred[:, c], real[:, c])[0], "UNI2-ridge within-slide (test)": within(pred[:, c], real[:, c])})
+C = pd.DataFrame(rows).set_index("channel")
+C.loc["mean"] = C.mean()
+
+# ---- (b) direct retrieval, all tiles, galleries mixed across slides vs within one slide
+big = [s for s in np.unique(slide) if (slide == s).sum() >= 100]
+V_rg, P_rg = rg(virt), rg(pred)
+Rt = pd.DataFrame([
+    {"pair": "real means <-> GigaTIME means", "mixed-slide gallery": g100(R_rg, V_rg), "single-slide gallery": g100(R_rg, V_rg, groups=big)},
+    {"pair": "real means <-> UNI2-ridge prediction", "mixed-slide gallery": g100(R_rg, P_rg), "single-slide gallery": g100(R_rg, P_rg, groups=big)},
+    {"pair": "real means <-> shuffled real (chance)", "mixed-slide gallery": g100(R_rg, R_rg[np.random.default_rng(0).permutation(len(R_rg))]), "single-slide gallery": np.nan},
+]).set_index("pair")
+
+# ---- (d) input distribution of the pixel maps (200 tiles)
+maps = np.load(os.path.join(G, "orion_maps.npy"), mmap_mode="r")
+idx = np.random.default_rng(0).choice(len(maps), 200, replace=False)
+mv = np.asarray(maps[np.sort(idx)])[:, [gch.index(gs[0]) for _, gs in PANEL]].astype(np.float32) / 255
+D = pd.DataFrame({"channel": names,
+                  "virtual: frac px p<0.05": (mv < 0.05).mean((0, 2, 3)),
+                  "virtual: frac px p>0.95": (mv > 0.95).mean((0, 2, 3)),
+                  "virtual: frac px in (0.05,0.95)": ((mv >= 0.05) & (mv <= 0.95)).mean((0, 2, 3))}).set_index("channel")
+
+pd.set_option("display.width", 200)
+print("(a) Spearman of tile means vs real ORION mIF\n", C.round(3).to_string())
+print("\n(b) R@1 in 100-tile galleries (chance 0.01), no learned head\n", Rt.round(3).to_string())
+print("\n(d) GigaTIME pixel-value distribution (sigmoid probabilities)\n", D.round(3).to_string())
+C.round(4).to_csv(os.path.join(OUT, "gigatime_diag.csv"))
+Rt.round(4).to_csv(os.path.join(OUT, "gigatime_diag_retrieval.csv"))
+D.round(4).to_csv(os.path.join(OUT, "gigatime_diag_pixels.csv"))

experiments/35_miphei_infer.py

@@ -0,0 +1,112 @@
+"""MIPHEI-ViT (Sanofi, arXiv 2505.10294) virtual mIF: H&E -> 16 ORION channels (all ORION markers except PD-1).
+
+Model : ext/miphei/weights (GitHub release v1.0.0: decoder + LoRA, 26 MB; non-commercial academic licence) on top of
+        bioptimus/H-optimus-0 (HF, gated; HF token only via environment). model.py of the release (read in full) is
+        loaded from its path; nothing else from the downloaded repo is executed.
+Training data of MIPHEI = this very ORION tile set with the ORION official split (37 train / 2 val / 2 test slides =
+        our splits/orion_official) -> MIPHEI virtual mIF is held-out ONLY on the official test (and val) tiles.
+Input : RGB at 0.5 um/px (ORION tiles 333 px = 0.4997 um/px used as is; TCGA 672-px patches at ~0.25 um/px -> 336 px),
+        ImageNet mean/std (release config), reflect-padded to 512 px (input must be a power of 2), output cropped back.
+Output: tanh in [-0.9, 0.9] -> (y + 0.9) / 1.8 * 255 = the 8-bit scale of the real ORION mIF tiles (MIPHEI targets).
+  cache/miphei/<set>_maps168.npy uint8 (N, 16, 168, 168)  resampled exactly like the real tiles in 14_ (antialiased
+                                                           bilinear to ~1 um/px) -> VirTues input
+  cache/miphei/<set>_means.npy   float32 (N, 16)          tile means at full resolution (0-255 scale)
+  cache/miphei/<set>_meta.csv, channels.json
+Usage: HF_TOKEN=... .venv/bin/python 35_miphei_infer.py --set orion|tcga_text|dlpfc|idc [--bs 8]
+"""
+import argparse
+import importlib.util
+import json
+import os
+import time
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from PIL import Image, PngImagePlugin
+from safetensors.torch import load_file
+
+from common import CACHE, ORION, ROOT, load_all_tiles
+
+p = argparse.ArgumentParser()
+p.add_argument("--set", required=True)
+p.add_argument("--bs", type=int, default=8)
+args = p.parse_args()
+dev = "cuda"
+W = os.path.join(ROOT, "ext", "miphei", "weights")
+OUT = os.path.join(CACHE, "miphei")
+os.makedirs(OUT, exist_ok=True)
+CH = ["Hoechst", "CD31", "CD45", "CD68", "CD4", "FOXP3", "CD8a", "CD45RO", "CD20", "PD-L1", "CD3e", "CD163",
+      "E-cadherin", "Ki67", "Pan-CK", "SMA"]  # release config targ_channel_names (order of the output heads)
+MEAN = torch.tensor([123.675, 116.28, 103.53], device=dev).view(1, 3, 1, 1) / 255
+STD = torch.tensor([58.395, 57.12, 57.375], device=dev).view(1, 3, 1, 1) / 255
+PAD_TO, VSIZE = 512, 168
+PngImagePlugin.MAX_TEXT_CHUNK = 64 * 1024 ** 2  # some TCGA PNGs carry large text metadata chunks
+
+spec = importlib.util.spec_from_file_location("miphei_model", os.path.join(W, "model.py"))
+Mm = importlib.util.module_from_spec(spec); spec.loader.exec_module(Mm)
+vit = Mm.get_hoptimus0_hf("bioptimus/H-optimus-0")
+vit.set_input_size(img_size=(PAD_TO, PAD_TO))
+enc = Mm.Encoder(vit)
+model = Mm.MIPHEIViT(enc, Mm.Detail_Capture(emb_chans=enc.embed_dim, out_chans=len(CH), use_attention=True, activation=nn.Tanh()))
+sd = Mm.merge_lora_weights(model, load_file(os.path.join(W, "model.safetensors")))
+Mm.validate_load_info(model.load_state_dict(sd, strict=False))
+model = model.eval().to(dev)
+
+if args.set == "orion":
+    df = load_all_tiles()
+    paths, keys, scale = [os.path.join(ORION, x) for x in df.image_path], df.image_path.values, 1.0
+elif args.set == "tcga_text":
+    I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+    paths, keys, scale = list(I.image_path), I.key.values, 0.5
+elif args.set in ("dlpfc", "idc"):  # external sets (36_external_prep.py): uint8 RGB patches at 0.5 um/px, 224 px
+    E = os.path.join(CACHE, "external")
+    ARR = np.load(os.path.join(E, "dlpfc_he_05um.npy" if args.set == "dlpfc" else "idc_he.npy"), mmap_mode="r")
+    em = pd.read_csv(os.path.join(E, f"{args.set}_meta.csv"))
+    assert len(em) == len(ARR)
+    paths, keys, scale = list(range(len(ARR))), em.index.values, 1.0
+else:
+    raise ValueError(args.set)
+N = len(paths)
+maps = np.lib.format.open_memmap(os.path.join(OUT, f"{args.set}_maps168.npy"), "w+", np.uint8, (N, len(CH), VSIZE, VSIZE))
+means = np.zeros((N, len(CH)), np.float32)
+sizes = []
+t0 = time.time()
+
+
+def load(pth):
+    if isinstance(pth, int):
+        return np.asarray(ARR[pth])
+    im = Image.open(pth).convert("RGB")
+    if scale != 1.0:
+        im = im.resize((round(im.width * scale), round(im.height * scale)), Image.BICUBIC)
+    return np.asarray(im)
+
+
+with torch.inference_mode():
+    for s in range(0, N, args.bs):
+        ims = [load(x) for x in paths[s:s + args.bs]]
+        h, w = ims[0].shape[:2]
+        assert all(i.shape[:2] == (h, w) for i in ims) and h <= PAD_TO and w <= PAD_TO
+        x = torch.from_numpy(np.stack(ims)).to(dev).permute(0, 3, 1, 2).float() / 255
+        x = (x - MEAN) / STD
+        pt, pl = (PAD_TO - h) // 2, (PAD_TO - w) // 2
+        x = F.pad(x, (pl, PAD_TO - w - pl, pt, PAD_TO - h - pt), mode="reflect")
+        with torch.autocast("cuda", dtype=torch.float16):  # as in the release notebook
+            y = model(x)
+        y = y.float().clamp(-0.9, 0.9)[:, :, pt:pt + h, pl:pl + w]
+        y = (y + 0.9) / 1.8 * 255
+        means[s:s + len(ims)] = y.mean((2, 3)).cpu().numpy()
+        maps[s:s + len(ims)] = F.interpolate(y, size=(VSIZE, VSIZE), mode="bilinear", antialias=True,
+                                             align_corners=False).round().clamp(0, 255).byte().cpu().numpy()
+        sizes += [(h, w)] * len(ims)
+        if s % (args.bs * 100) == 0:
+            print(f"  {s}/{N} [{time.time() - t0:.0f}s]", flush=True)
+maps.flush()
+np.save(os.path.join(OUT, f"{args.set}_means.npy"), means)
+pd.DataFrame({"key": keys, "path": paths, "h": [a for a, _ in sizes], "w": [b for _, b in sizes]}).to_csv(
+    os.path.join(OUT, f"{args.set}_meta.csv"), index=False)
+json.dump({"channels": CH}, open(os.path.join(OUT, "channels.json"), "w"))
+print(f"done {args.set}: {N} tiles [{time.time() - t0:.0f}s]")

experiments/36_external_prep.py

@@ -0,0 +1,332 @@
+"""External H&E<->RNA test sets: DLPFC (spatialLIBD Visium) and breast IDC (HEST Xenium pseudo-spots).
+
+Run:  .venv/bin/python -I 36_external_prep.py [dlpfc] [idc]     (default: both)
+Needs the public spatialLIBD files in ext/external_raw/dlpfc_spatialLIBD/ (downloaded once, data only):
+  <sid>_filtered_feature_bc_matrix.h5   https://spatial-dlpfc.s3.us-east-2.amazonaws.com/h5/
+  <sid>_tissue_positions_list.txt       https://raw.githubusercontent.com/LieberInstitute/HumanPilot/master/10X/<sid>/
+  <sid>_scalefactors_json.json          (same)
+
+Findings this script relies on (verified, numbers printed again at run time):
+DLPFC  visium_gene_expression_list.pkl = log1p(count * 3034 / total_counts_all_33538_genes)   (3034 = median
+       total over all 47,681 spots, i.e. scanpy normalize_total default) restricted to the top-10000
+       scanpy 'seurat'-flavour HVGs computed on all 12 samples concatenated, columns in 10x feature (genome) order.
+       Rows = h5 barcode order of each sample, samples concatenated in sorted order (manifest spot_idx).
+       Images = ImageNet-normalised uint8 RGB, native 224x224 full-res crop centred on the spot.
+       Full-res scale: Visium centre-to-centre pitch 100 um = 138.0 px -> 0.7246 um/px (HEST: 0.725);
+       note the 'pxl_row/col' columns of the HumanPilot tissue_positions_list are (x, y), not (row, col).
+IDC    rebuilt from HEST-1k (MahmoodLab/hest): st/<id>.h5ad + the selected 224-px, 0.5 um/px patches of
+       patches/<id>.h5 for TENX95, TENX99, NCBI783, NCBI785 (= HEST-Bench IDC), paired by barcode and checked by
+       patch-centre vs spot-centre distance. Spots: in tissue, with a patch, >= 20 panel counts; > 1500 -> random
+       1500 (seed 0). The earlier IDC built from ../IDC/UNI was unusable: subsample.py drew its image indices
+       independently of its adata indices (key '/k/__list__' vs '/k/__arr__', hash()-randomised seed).
+       First run fetches patches with HTTP range reads: needs HF_TOKEN in the environment.
+Outputs cache/external/: see the OUT dict printed at the end.
+"""
+import json
+import os
+import pickle
+import sys
+
+import h5py
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+from PIL import Image
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+DATA = os.path.abspath(os.path.join(ROOT, ".."))
+RAW = os.path.join(ROOT, "ext", "external_raw", "dlpfc_spatialLIBD")
+OUT = os.path.join(ROOT, "cache", "external")
+os.makedirs(OUT, exist_ok=True)
+IMNET_MEAN = np.array([0.485, 0.456, 0.406], np.float32)
+IMNET_STD = np.array([0.229, 0.224, 0.225], np.float32)
+TARGET_UM = 0.5  # um/px of the resampled patches
+HVG = pd.read_csv(os.path.join(ROOT, "cache", "hest", "hvg_genes.csv"))["gene"].astype(str).tolist()
+HEST_META = pd.read_csv(os.path.join(DATA, "HEST", "HEST_v1_1_0.csv"))
+HEST_TRAIN = set(pd.read_csv(os.path.join(ROOT, "cache", "hest", "meta.csv"), usecols=["sample"])["sample"])
+
+
+def load_pickle(p):
+    with open(p, "rb") as f:
+        return pickle.load(f)
+
+
+def denorm(x):
+    """(N,3,H,W) ImageNet-normalised float -> (N,H,W,3) uint8; returns max dequantisation error (in 0..255 units)."""
+    v = (np.asarray(x, np.float32).transpose(0, 2, 3, 1) * IMNET_STD + IMNET_MEAN) * 255.0
+    u = np.clip(np.round(v), 0, 255).astype(np.uint8)
+    return u, float(np.abs(v - u).max())
+
+
+def resample(u8, um_native, fov_um=224 * TARGET_UM, out=224):
+    """Centre crop fov_um (if available) and resize to out x out => TARGET_UM um/px."""
+    res = []
+    for im in u8:
+        side = fov_um / um_native
+        h = im.shape[0]
+        if side > h:  # not enough context: keep full patch (FOV smaller than requested)
+            side = h
+        c = (h - side) / 2
+        res.append(np.asarray(Image.fromarray(im).resize((out, out), Image.BICUBIC, box=(c, c, c + side, c + side))))
+    return np.stack(res)
+
+
+def hvg_overlap(genes):
+    g = set(genes)
+    return int(sum(x in g for x in HVG))
+
+
+# ================================================================== DLPFC
+def read_h5(sid):
+    with h5py.File(os.path.join(RAW, f"{sid}_filtered_feature_bc_matrix.h5"), "r") as f:
+        g = f["matrix"]
+        M = sp.csc_matrix((g["data"][:], g["indices"][:], g["indptr"][:]), shape=g["shape"][:]).T.tocsr()
+        bcs = [b.decode() for b in g["barcodes"][:]]
+        names = [b.decode() for b in g["features"]["name"][:]]
+        ids = [b.decode() for b in g["features"]["id"][:]]
+    return M, bcs, names, ids
+
+
+def seurat_hvg(L, n_top):
+    """scanpy.pp.highly_variable_genes(flavor='seurat', n_top_genes=n_top) on log1p data L (csr)."""
+    E = L.copy()
+    E.data = np.expm1(E.data)
+    n = E.shape[0]
+    mean = np.asarray(E.mean(0)).ravel()
+    var = (np.asarray(E.multiply(E).mean(0)).ravel() - mean ** 2) * n / (n - 1)
+    with np.errstate(all="ignore"):
+        disp = var / mean
+        disp[disp == 0] = np.nan
+        disp = np.log(disp)
+    df = pd.DataFrame({"m": np.log1p(mean), "d": disp})
+    df["bin"] = pd.cut(df.m, bins=20)
+    g = df.groupby("bin", observed=False)["d"]
+    bm, bs, one = g.transform("mean"), g.transform("std"), g.transform("count") == 1
+    bs, bm = bs.where(~one, bm), bm.where(~one, 0)
+    dn = ((df.d - bm) / bs).to_numpy()
+    v = np.sort(dn[~np.isnan(dn)])[::-1]
+    cut = v[n_top - 1]
+    print(f"  HVG cut-off normalised dispersion {cut:.5f}; next {v[n_top]:.5f} (gap {cut - v[n_top]:.2e})")
+    return np.where(np.nan_to_num(dn) >= cut)[0]
+
+
+def dlpfc():
+    print("== DLPFC")
+    P = os.path.join(DATA, "DLPFC_visium", "GPFM")
+    man = json.load(open(os.path.join(DATA, "_manifest", "dlpfc.json")))
+    samples = man["samples"]
+    X = load_pickle(os.path.join(P, "visium_gene_expression_list.pkl"))
+    X = np.asarray(X.numpy() if hasattr(X, "numpy") else X, np.float64)
+    gidx = np.asarray(pd.read_csv(os.path.join(DATA, "DLPFC_visium", "global_spot_index.csv"))["global_spot_idx"])
+    assert np.array_equal(gidx, man["global_spot_idx"]) and X.shape == (len(gidx), 10000)
+
+    Ms, rows, names, ids = [], [], None, None
+    sf_by, pos_by, bcs_by = {}, {}, {}
+    for sid in samples:
+        M, bcs, names, ids = read_h5(sid)
+        assert M.shape[0] == man["n_spots"][sid], sid
+        Ms.append(M)
+        bcs_by[sid] = bcs
+        sf_by[sid] = json.load(open(os.path.join(RAW, f"{sid}_scalefactors_json.json")))
+        pos = pd.read_csv(os.path.join(RAW, f"{sid}_tissue_positions_list.txt"), header=None,
+                          names=["barcode", "in_tissue", "array_row", "array_col", "px_x", "px_y"]).set_index("barcode")
+        pos_by[sid] = pos
+        for i in man["spot_idx"][sid]:
+            rows.append((sid, i, bcs[i]))
+    M = sp.vstack(Ms).tocsr().astype(np.float64)
+    tot = np.asarray(M.sum(1)).ravel()
+    target = float(np.median(tot))
+    A = M[gidx]  # raw counts of the 482 spots, all 33538 genes
+    print(f"  all spots {M.shape}, median total counts {target}")
+
+    # 1) normalisation: x = log1p(count * target / total)
+    L = sp.diags(target / tot) @ M
+    L.data = np.log1p(L.data)
+    # 2) gene subset: reproduce scanpy seurat HVG (top 10000) on all 47,681 spots
+    sel = seurat_hvg(L, 10000)
+    assert len(sel) == 10000
+    Lsel = L[gidx][:, sel].toarray()
+    err = np.abs(Lsel - X).max()
+    # 3) independent check: exact per-column value match of recovered counts vs every h5 gene
+    Ad = A.toarray()
+    K = np.round(np.expm1(X) * (tot[gidx] / target)[:, None])
+    buckets = {}
+    for j in range(Ad.shape[1]):
+        buckets.setdefault(Ad[:, j].tobytes(), []).append(j)
+    cand = [buckets.get(K[:, c].tobytes(), []) for c in range(10000)]
+    n_unique = sum(len(c) == 1 for c in cand)
+    n_consistent = sum(sel[c] in cand[c] for c in range(10000))
+    n_unique_ok = sum(len(cand[c]) == 1 and cand[c][0] == sel[c] for c in range(10000))
+    print(f"  max |log-expr(reconstructed) - pickle| = {err:.2e}")
+    print(f"  exact-value column matches: {n_unique}/10000 columns match exactly one h5 gene "
+          f"({n_unique_ok} of them = HVG reconstruction); {n_consistent}/10000 HVG genes value-consistent; "
+          f"{int((K.sum(0) == 0).sum())} columns all-zero in the 482 spots")
+    assert err < 1e-5 and n_consistent == 10000
+
+    genes = pd.DataFrame({"col": np.arange(10000), "gene": np.array(names)[sel], "ensembl_id": np.array(ids)[sel],
+                          "h5_feature_idx": sel,
+                          "match": ["exact_unique" if len(c) == 1 else "exact_ambiguous_resolved_by_hvg" for c in cand]})
+    genes["duplicated_symbol"] = genes.gene.duplicated(keep=False)
+    genes.to_csv(os.path.join(OUT, "dlpfc_genes.csv"), index=False)
+    np.save(os.path.join(OUT, "dlpfc_counts.npy"), Ad[:, sel].astype(np.float32))
+    # full transcriptome version (same spots) from the public h5
+    np.save(os.path.join(OUT, "dlpfc_counts_full.npy"), Ad.astype(np.float32))
+    pd.DataFrame({"gene": names, "ensembl_id": ids, "in_10k": np.isin(np.arange(len(names)), sel)}).to_csv(
+        os.path.join(OUT, "dlpfc_genes_full.csv"), index=False)
+
+    # images
+    im = load_pickle(os.path.join(P, "visium_image_list.pkl"))
+    im = im.numpy() if hasattr(im, "numpy") else np.asarray(im)
+    u8, qerr = denorm(im)
+    print(f"  images {im.shape}: ImageNet-normalised, dequantisation error {qerr:.1e} (exact uint8)")
+    np.save(os.path.join(OUT, "dlpfc_he.npy"), u8)
+
+    hest_ids = dict(zip(HEST_META.subseries.astype(str), HEST_META.id))
+    meta = []
+    for k, (sid, i, bc) in enumerate(rows):
+        p = pos_by[sid]
+        q = p.reset_index()
+        nb = q.merge(q, on="array_row")
+        nb = nb[nb.array_col_y == nb.array_col_x + 2]
+        pitch = float(np.median(np.hypot(nb.px_x_x - nb.px_x_y, nb.px_y_x - nb.px_y_y)))  # 100 um
+        hid = hest_ids.get(sid)
+        meta.append(dict(sample=sid, spot_idx=i, global_spot_idx=int(gidx[k]), barcode=bc,
+                         array_row=int(p.loc[bc, "array_row"]), array_col=int(p.loc[bc, "array_col"]),
+                         x_fullres=float(p.loc[bc, "px_x"]), y_fullres=float(p.loc[bc, "px_y"]),
+                         organ="Brain", tissue="dorsolateral prefrontal cortex",
+                         um_per_px=100.0 / pitch, fov_um=224 * 100.0 / pitch,
+                         spot_diameter_fullres=sf_by[sid]["spot_diameter_fullres"],
+                         total_counts=float(tot[gidx[k]]), hest_id=hid, in_hest_train_cache=hid in HEST_TRAIN))
+    meta = pd.DataFrame(meta)
+    meta.to_csv(os.path.join(OUT, "dlpfc_meta.csv"), index=False)
+    um = float(meta.um_per_px.median())
+    u05 = resample(u8, um)
+    np.save(os.path.join(OUT, "dlpfc_he_05um.npy"), u05)
+    print(f"  um/px {um:.4f} -> patch FOV {224 * um:.1f} um; dlpfc_he_05um = centre {112 / um:.1f}px crop -> 224 (112 um)")
+    print(f"  HEST-HVG overlap: 10k cols {hvg_overlap(genes.gene)}/{len(HVG)}, full {hvg_overlap(names)}/{len(HVG)}")
+
+
+# ================================================================== IDC (paired, from HEST-1k)
+HEST_RAW = os.path.join(ROOT, "ext", "external_raw", "hest_idc")
+IDC_IDS = ["TENX95", "TENX99", "NCBI783", "NCBI785"]  # = HEST-Bench IDC
+IDC_MIN_COUNTS = 20
+IDC_MAX_SPOTS = 1500
+CTRL = ("BLANK", "NegControl", "Unassigned", "antisense", "DeprecatedCodeword", "Intergenic")
+
+
+def fetch_patches(sid, barcodes):
+    """Fetch only the selected patches of MahmoodLab/hest patches/<sid>.h5 by HTTP range reads (the full files are
+    5.0 GB together; patches are stored uncompressed, one chunk per patch). Auth: HF_TOKEN env var, read by
+    huggingface_hub itself. Writes patches_subset/<sid>.h5 with img/barcode/coords + the source attrs."""
+    from huggingface_hub import HfFileSystem
+    out = os.path.join(HEST_RAW, "patches_subset", f"{sid}.h5")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    fs = HfFileSystem()
+    src = f"datasets/MahmoodLab/hest/patches/{sid}.h5"
+    pb = all_patch_barcodes(sid)  # cached barcode/coords table of the remote file (row order = h5 row order)
+    coords = pb[["x", "y"]].to_numpy()
+    pos = {b: i for i, b in enumerate(pb.barcode)}
+    rows = np.array([pos[b] for b in barcodes])
+    order = np.argsort(rows)
+    imgs = np.zeros((len(rows), 224, 224, 3), np.uint8)
+    with fs.open(src, "rb", block_size=160 * 1024, cache_type="readahead") as f, h5py.File(f, "r") as h:
+        attrs = dict(h["img"].attrs)
+        for n, k in enumerate(order):
+            imgs[k] = h["img"][rows[k]]
+            if n % 250 == 0:
+                print(f"    {sid}: fetched {n}/{len(rows)} patches", flush=True)
+    with h5py.File(out, "w") as g:
+        g["img"] = imgs
+        g["barcode"] = np.array(barcodes, dtype="S")
+        g["coords"] = coords[rows]
+        g["src_row"] = rows
+        for k, v in attrs.items():
+            g["img"].attrs[k] = v
+    return out
+
+
+def all_patch_barcodes(sid):
+    """Barcodes that have a patch in HEST (read remotely once, cached as a small csv)."""
+    p = os.path.join(HEST_RAW, "patches_subset", f"{sid}_all_barcodes.csv")
+    if not os.path.exists(p):
+        from huggingface_hub import HfFileSystem
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        with HfFileSystem().open(f"datasets/MahmoodLab/hest/patches/{sid}.h5", "rb", block_size=2 ** 22) as f, \
+                h5py.File(f, "r") as h:
+            bc = [b[0].decode() if isinstance(b[0], bytes) else str(b[0]) for b in h["barcode"][:]]
+            pd.DataFrame({"barcode": bc, "x": h["coords"][:, 0], "y": h["coords"][:, 1]}).to_csv(p, index=False)
+    return pd.read_csv(p)
+
+
+def idc():
+    print("== IDC (HEST-1k, paired by barcode)")
+    import anndata as ad
+    ads = {s: ad.read_h5ad(os.path.join(HEST_RAW, "st", f"{s}.h5ad")) for s in IDC_IDS}
+    common = set.intersection(*[set(a.var_names) for a in ads.values()])
+    genes = sorted(g for g in common if not g.startswith(CTRL))
+    rng = np.random.default_rng(0)
+    he, counts, meta = [], [], []
+    for sid in IDC_IDS:
+        a = ads[sid]
+        md = json.load(open(os.path.join(HEST_RAW, "metadata", f"{sid}.json")))
+        X = a[:, genes].X
+        X = np.asarray(X.toarray() if sp.issparse(X) else X, np.float32)
+        tot = X.sum(1)
+        pb = all_patch_barcodes(sid)
+        in_t = a.obs["in_tissue"].astype(bool).values if "in_tissue" in a.obs else np.ones(a.n_obs, bool)
+        has_p = np.isin(a.obs_names, pb.barcode)
+        elig = np.where(in_t & has_p & (tot >= IDC_MIN_COUNTS))[0]
+        sub = len(elig) > IDC_MAX_SPOTS
+        sel = np.sort(rng.choice(elig, IDC_MAX_SPOTS, replace=False)) if sub else elig
+        print(f"  {sid}: spots {a.n_obs}, in_tissue {in_t.sum()}, with patch {has_p.sum()}, "
+              f">= {IDC_MIN_COUNTS} counts {len(elig)} -> kept {len(sel)}{' (random subsample, seed 0)' if sub else ''}")
+        bcs = a.obs_names[sel].tolist()
+        p = os.path.join(HEST_RAW, "patches_subset", f"{sid}.h5")
+        if not os.path.exists(p):
+            fetch_patches(sid, bcs)
+        with h5py.File(p, "r") as g:
+            assert [b.decode() for b in g["barcode"][:]] == bcs, f"{sid}: cached patch subset does not match selection"
+            imgs, coords = g["img"][:], g["coords"][:]
+            src_px, ds, mpp = (float(g["img"].attrs[k]) for k in ("patch_size_src", "downsample", "pixel_size"))
+        # pairing check: patch centre (top-left coords + half the source patch size) vs spot centre
+        cx, cy = coords[:, 0] + src_px / 2, coords[:, 1] + src_px / 2
+        o = a.obs.iloc[sel]
+        dev = np.hypot(cx - o.pxl_col_in_fullres.values, cy - o.pxl_row_in_fullres.values) * mpp
+        um_out = mpp * ds
+        print(f"    patch: {int(src_px)} src px @ {mpp:.4f} um/px -> 224 px @ {um_out:.4f} um/px ({224 * um_out:.0f} um FOV); "
+              f"|patch centre - spot centre| max {dev.max():.2f} um, median {np.median(dev):.2f} um")
+        assert dev.max() < 5, f"{sid}: patch/spot coordinate mismatch"
+        he.append(imgs)
+        counts.append(X[sel])
+        for j, i in enumerate(sel):
+            meta.append(dict(sample=sid, barcode=a.obs_names[i], array_row=int(a.obs.array_row.iloc[i]),
+                             array_col=int(a.obs.array_col.iloc[i]),
+                             x_fullres=float(a.obs.pxl_col_in_fullres.iloc[i]),
+                             y_fullres=float(a.obs.pxl_row_in_fullres.iloc[i]),
+                             organ="Breast", oncotree=md.get("oncotree_code"), um_per_px=um_out,
+                             um_per_px_fullres=mpp, patch_center_dev_um=float(dev[j]), total_counts=float(tot[i]),
+                             subsampled=bool(sub), n_eligible=len(elig),
+                             in_hest_train_cache=sid in HEST_TRAIN))
+    he, counts = np.concatenate(he), np.concatenate(counts)
+    np.save(os.path.join(OUT, "idc_he.npy"), he)
+    np.save(os.path.join(OUT, "idc_counts.npy"), counts)
+    pd.DataFrame({"gene": genes}).to_csv(os.path.join(OUT, "idc_genes.csv"), index=False)
+    pd.DataFrame(meta).to_csv(os.path.join(OUT, "idc_meta.csv"), index=False)
+    for f in ("idc_he_unpaired.npy", "idc_he_unpaired_meta.csv"):  # old, unpaired subsample images: removed
+        if os.path.exists(os.path.join(OUT, f)):
+            os.remove(os.path.join(OUT, f))
+    hest_common = pd.read_csv(os.path.join(ROOT, "cache", "hest", "genes.csv"))["gene"].astype(str)
+    print(f"  genes {len(genes)} (common non-control panel genes); counts {counts.shape}; images {he.shape}")
+    print(f"  IDC panel overlap: HEST HVG {hvg_overlap(genes)}/{len(HVG)}; "
+          f"HEST common genes {int(hest_common.isin(genes).sum())}/{len(hest_common)}")
+
+
+if __name__ == "__main__":
+    todo = sys.argv[1:] or ["dlpfc", "idc"]
+    if "dlpfc" in todo:
+        dlpfc()
+    if "idc" in todo:
+        idc()
+    for f in sorted(os.listdir(OUT)):
+        p = os.path.join(OUT, f)
+        print(f"  {f:28s}", np.load(p, mmap_mode="r").shape if f.endswith(".npy") else f"{len(pd.read_csv(p))} rows")

experiments/37_miphei_diag.py

@@ -0,0 +1,85 @@
+"""MIPHEI-ViT vs GigaTIME virtual mIF on ORION, against the real mIF of the same tiles (tile means).
+
+MIPHEI was trained on the ORION official train split -> 'clean' = official TEST tiles (2 patients, never seen);
+'seen' = official TRAIN tiles (training data of MIPHEI, optimistic). GigaTIME never saw ORION (clean everywhere).
+  (a) per-channel Spearman of tile means (pooled and within-slide)
+  (b) direct real-mean <-> virtual-mean retrieval (rank-gaussianised means, cosine; no learned head), 100-galleries
+Output results/miphei_diag.csv, results/miphei_diag_retrieval.csv
+"""
+import json
+import os
+
+import numpy as np
+import pandas as pd
+from scipy.special import ndtri
+from scipy.stats import rankdata, spearmanr
+
+from common import CACHE, ROOT, SPLITS, load_all_tiles
+
+OUT = os.path.join(ROOT, "results")
+ORION_CH = ["Hoechst", "CD31", "CD45", "CD68", "CD4", "FOXP3", "CD8a", "CD45RO", "CD20", "PD-L1", "CD3e",
+            "CD163", "E-cadherin", "PD-1", "Ki67", "Pan-CK", "SMA"]  # order of orion_real_if_means.npy (25_)
+GT = {"Hoechst": ["DAPI"], "CD68": ["CD68"], "CD4": ["CD4"], "CD8a": ["CD8"], "CD20": ["CD20"], "PD-L1": ["PD-L1"],
+      "CD3e": ["CD3"], "Ki67": ["Ki67"], "Pan-CK": ["CK"], "SMA": ["Actin-D", "Transgelin"]}
+df = load_all_tiles()
+slide = df.in_slide_name.values
+real = np.load(os.path.join(CACHE, "gigatime", "orion_real_if_means.npy"))
+mch = json.load(open(os.path.join(CACHE, "miphei", "channels.json")))["channels"]
+mip = np.load(os.path.join(CACHE, "miphei", "orion_means.npy"))
+gch = json.load(open(os.path.join(CACHE, "gigatime", "channels.json")))["channels"]
+gm = np.load(os.path.join(CACHE, "gigatime", "orion_means.npy"))
+off = pd.read_csv(os.path.join(SPLITS, "orion_official", "fold0.csv")).split.values
+sets = {"clean (official test)": off == "test", "seen (official train)": off == "train"}
+
+
+def within(a, b, m):
+    r, w = [], []
+    for s in np.unique(slide[m]):
+        i = m & (slide == s)
+        if i.sum() >= 15 and a[i].std() > 0 and b[i].std() > 0:
+            r.append(spearmanr(a[i], b[i])[0]); w.append(i.sum())
+    return float(np.average(r, weights=w)) if r else np.nan
+
+
+rows = []
+for ch in mch:
+    r = real[:, ORION_CH.index(ch)]
+    for sname, m in sets.items():
+        row = {"channel": ch, "set": sname,
+               "MIPHEI pooled": spearmanr(mip[m, mch.index(ch)], r[m])[0],
+               "MIPHEI within-slide": within(mip[:, mch.index(ch)], r, m)}
+        if ch in GT:
+            g = gm[:, [gch.index(x) for x in GT[ch]]].mean(1)
+            row.update({"GigaTIME pooled": spearmanr(g[m], r[m])[0], "GigaTIME within-slide": within(g, r, m)})
+        rows.append(row)
+C = pd.DataFrame(rows)
+rg = lambda x: ndtri((np.apply_along_axis(rankdata, 0, x) - 0.5) / len(x))
+
+
+def g100(a, b, draws=50, seed=0):
+    a = a / np.linalg.norm(a, axis=1, keepdims=True); b = b / np.linalg.norm(b, axis=1, keepdims=True)
+    rng = np.random.default_rng(seed); r = []
+    for _ in range(draws):
+        s = rng.choice(len(a), min(100, len(a)), replace=False)
+        sim = a[s] @ b[s].T; d = np.diag(sim)[:, None]
+        r.append(((sim > d).sum(1) == 0).mean() / 2 + ((sim.T > d).sum(1) == 0).mean() / 2)
+    return float(np.mean(r))
+
+
+ten = list(GT)
+Rt = []
+for sname, m in sets.items():
+    R16 = real[m][:, [ORION_CH.index(c) for c in mch]]
+    R10 = real[m][:, [ORION_CH.index(c) for c in ten]]
+    G10 = np.stack([gm[m][:, [gch.index(x) for x in GT[c]]].mean(1) for c in ten], 1)
+    Rt += [{"set": sname, "n": int(m.sum()), "pair": "real <-> MIPHEI (16 ch)", "R@1_g100": g100(rg(R16), rg(mip[m]))},
+           {"set": sname, "n": int(m.sum()), "pair": "real <-> MIPHEI (10 shared ch)", "R@1_g100": g100(rg(R10), rg(mip[m][:, [mch.index(c) for c in ten]]))},
+           {"set": sname, "n": int(m.sum()), "pair": "real <-> GigaTIME (10 shared ch)", "R@1_g100": g100(rg(R10), rg(G10))}]
+Rt = pd.DataFrame(Rt)
+pd.set_option("display.width", 200)
+print(C.round(3).to_string(index=False))
+print("\nmean over channels\n", C.groupby("set")[[c for c in C.columns if c not in ("channel", "set")]].mean().round(3).to_string())
+print("\nmean over the 10 GigaTIME-shared channels\n", C[C.channel.isin(ten)].groupby("set")[[c for c in C.columns if c not in ("channel", "set")]].mean().round(3).to_string())
+print("\n", Rt.round(3).to_string(index=False))
+C.round(4).to_csv(os.path.join(OUT, "miphei_diag.csv"), index=False)
+Rt.round(4).to_csv(os.path.join(OUT, "miphei_diag_retrieval.csv"), index=False)

experiments/38_external_eval.py

@@ -0,0 +1,239 @@
+"""External test sets (never used for training): DLPFC Visium (brain, 12 sections / 3 donors) and HEST IDC
+(breast, Xenium pseudo-spots, 4 samples; 280-gene panel). Prepared by 36_external_prep.py -> cache/external/.
+
+Real data only for training (HEST-1k, train/val by study|patient group, val = early stopping only):
+  dlpfc head : H&E(UNI2) <-> RNA MLP on the 2000 HEST HVGs, trained on HEST WITHOUT the spatialLIBD study
+               (= DLPFC; those spots are in cache/hest) -> DLPFC is fully held out
+  idc head   : same recipe on the IDC panel genes present in the HEST common gene set (IDC samples are not in HEST cache)
+  RNA input  : log1p(CP10k over the HEST common genes / over the panel genes), then z-scored per gene with the
+               statistics of its OWN dataset (HEST training spots vs the external set; label-free platform alignment)
+  5 heads per set (different val groups / seeds) -> mean +- std.
+Virtual data only at test time: MIPHEI-ViT virtual mIF of the external H&E (35_ + 30_ --m16) through the 16-ch mIF
+heads trained on REAL ORION (patient 5-fold, fold k).
+Synthetic text only at test time: GPT captions of the external H&E (27_ --sets dlpfc idc, 32_, 08_ biomedclip)
+through the TCGA text heads (runs/text_clean/uni2_biomedclip_fold<k>, trained on CLEAN real image-text pairs in 33_).
+Pairs: H&E<->RNA (real, external), RNA<->mIF(MIPHEI) (emergent, real x virtual), H&E<->mIF(MIPHEI) (circular),
+       RNA<->text(synthetic) (emergent, real x virtual), H&E<->text(synthetic) (circular),
+       mIF(MIPHEI)<->text(synthetic) (virtual x virtual).
+Metrics: 100-item galleries (50 draws, both directions): R@1/R@5/R@10/MRR + Hungarian pairing accuracy.
+Usage: python 38_external_eval.py feats   (UNI2 features of the external patches, GPU)
+       python 38_external_eval.py eval    -> results/external_long.csv, external_summary.csv
+"""
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, RUNS
+
+E = os.path.join(CACHE, "external")
+H = os.path.join(CACHE, "hest")
+OUT = os.path.join(ROOT, "results")
+SETS = {"dlpfc": "dlpfc_he_05um.npy"}  # IDC dropped from the report 2026-10-18 (user request)
+MODE = sys.argv[1]
+
+if MODE == "feats":
+    import timm
+    dev = "cuda"
+    model = timm.create_model(  # = 04b_extract_anchor_fm.py uni2 (local official UNI2-h checkpoint)
+        "vit_giant_patch14_224", pretrained=False, img_size=224, patch_size=14, depth=24, num_heads=24,
+        init_values=1e-5, embed_dim=1536, mlp_ratio=2.66667 * 2, num_classes=0, no_embed_class=True,
+        mlp_layer=timm.layers.SwiGLUPacked, act_layer=torch.nn.SiLU, reg_tokens=8, dynamic_img_size=True)
+    model.load_state_dict(torch.load("/diskarray/home/tl688/reg2026/model_alldata/uni2-h.bin", map_location="cpu"), strict=True)
+    model = model.eval().to(dev)
+    mean = torch.tensor([0.485, 0.456, 0.406], device=dev).view(1, 3, 1, 1)
+    std = torch.tensor([0.229, 0.224, 0.225], device=dev).view(1, 3, 1, 1)
+    for st, f in SETS.items():
+        X = np.load(os.path.join(E, f), mmap_mode="r")
+        out = np.zeros((len(X), 1536), np.float32)
+        with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16):
+            for s in range(0, len(X), 32):
+                x = torch.from_numpy(np.asarray(X[s:s + 32])).to(dev).permute(0, 3, 1, 2).float() / 255
+                if x.shape[-1] != 224:
+                    x = F.interpolate(x, size=(224, 224), mode="bilinear", antialias=True, align_corners=False)
+                out[s:s + 32] = model((x - mean) / std).float().cpu().numpy()
+        np.save(os.path.join(E, f"{st}_he_uni2.npy"), out)
+        print(st, out.shape, flush=True)
+    # protocol check: DLPFC spots that are also in the HEST cache (same barcode) -> cosine to the HEST UNI2 feature
+    m, hm = pd.read_csv(os.path.join(E, "dlpfc_meta.csv")), pd.read_csv(os.path.join(H, "meta.csv"))
+    hf = np.load(os.path.join(H, "he_uni2.npy"))[:, 0].astype(np.float32)
+    o = np.load(os.path.join(E, "dlpfc_he_uni2.npy"))
+    j = m.reset_index().merge(hm.reset_index(), left_on=["hest_id", "barcode"], right_on=["sample", "spot"], suffixes=("_e", "_h"))
+    if len(j):
+        a, b = F.normalize(torch.from_numpy(o[j.index_e.values]), dim=-1), F.normalize(torch.from_numpy(hf[j.index_h.values]), dim=-1)
+        rnd = F.normalize(torch.from_numpy(hf[np.random.default_rng(0).choice(len(hf), 200, replace=False)]), dim=-1)
+        print(f"protocol check: {len(j)} shared spots, cosine(ours, HEST cache) = {(a * b).sum(1).numpy().round(3)}; "
+              f"mean cosine to random HEST spots {float((a @ rnd.T).mean()):.3f}")
+    sys.exit(0)
+
+# ======================================================================== eval
+dev = "cuda" if torch.cuda.is_available() else "cpu"
+G100, DRAWS, K = 100, 50, 5
+HM = pd.read_csv(os.path.join(H, "meta.csv"))
+hgenes = pd.read_csv(os.path.join(H, "genes.csv")).gene.values
+hcounts = sp.load_npz(os.path.join(H, "counts.npz")).tocsr().astype(np.float32)
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+HE_H = torch.from_numpy(np.load(os.path.join(H, "he_uni2.npy"))[:, 0].astype(np.float32))
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def lognorm(C, total):
+    return np.log1p(C / np.maximum(total, 1)[:, None] * 1e4).astype(np.float32)
+
+
+def zscore(x, ref):
+    mu, sd = ref.mean(0), ref.std(0) + 1e-6
+    return (x - mu) / sd
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float(((s > s.diag()[:, None]).sum(1) == 0).float().mean())
+
+
+def train_head(G, tr, va, seed, epochs=100, bs=256, lr=3e-4):  # = 23_bind_gene.py recipe
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    A, Gd = HE_H.to(dev), torch.from_numpy(G).to(dev)
+    head = mlp(G.shape[1]).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(head.parameters()) + [logit], lr=lr, weight_decay=0.05)
+    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=epochs * max(1, len(tr) // bs), pct_start=0.05)
+    za_va = F.normalize(A[va], dim=-1)
+    best, state = -1, None
+    for ep in range(epochs):
+        head.train()
+        perm = rng.permutation(tr)
+        for s in range(0, len(perm) - bs + 1, bs):
+            b = torch.as_tensor(perm[s:s + bs], device=dev)
+            lg = logit.exp().clamp(max=100) * F.normalize(A[b], dim=-1) @ F.normalize(head(Gd[b]), dim=-1).T
+            t = torch.arange(len(b), device=dev)
+            loss = (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sched.step()
+        head.eval()
+        with torch.no_grad():
+            zg = F.normalize(head(Gd[va]), dim=-1)
+            v = (r1(za_va, zg) + r1(zg, za_va)) / 2
+        if v > best:
+            best, state = v, {k_: x.detach().cpu().clone() for k_, x in head.state_dict().items()}
+    head = head.cpu(); head.load_state_dict(state); head.eval()
+    return head, best
+
+
+def g100(a, b, seed=0):
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+nz = lambda x: F.normalize(torch.as_tensor(np.asarray(x), dtype=torch.float32), dim=-1)
+
+
+def mif_head(k):
+    d = torch.load(os.path.join(RUNS, "bench_fm", f"patient_5fold_fold{k}_uni2_fm-virtues_m16_fm-geneformer_pathgenclip", "heads.pt"), map_location="cpu")
+    h = mlp(d["state"]["if"]["0.weight"].shape[1]); h.load_state_dict(d["state"]["if"]); h.eval()
+    return lambda x: h((torch.as_tensor(x, dtype=torch.float32) - d["if_mean"]) / d["if_std"])
+
+
+def text_head(k):
+    sd = torch.load(os.path.join(RUNS, "text_clean", f"uni2_biomedclip_fold{k}", "head.pt"), map_location="cpu")
+    h = mlp(sd["0.weight"].shape[1], sd["6.weight"].shape[0]); h.load_state_dict(sd); h.eval()
+    return lambda x: h(torch.as_tensor(x, dtype=torch.float32))
+
+
+rows = []
+hset = pd.Series(range(len(hgenes)), index=hgenes)
+htot = np.asarray(hcounts.sum(1)).ravel()
+for st in SETS:
+    em = pd.read_csv(os.path.join(E, f"{st}_meta.csv"))
+    he = np.load(os.path.join(E, f"{st}_he_uni2.npy"))
+    if st == "dlpfc":
+        eg = pd.read_csv(os.path.join(E, "dlpfc_genes_full.csv")).gene.values
+        ec = np.load(os.path.join(E, "dlpfc_counts_full.npy")).astype(np.float32)
+        eidx = pd.Series(range(len(eg)), index=eg)
+        eidx = eidx[~eidx.index.duplicated()]
+        common = [g for g in hgenes if g in eidx.index]
+        genes = [g for g in hvg if g in eidx.index]
+        etot = ec[:, eidx.loc[common].values].sum(1)                      # library size over the HEST common genes
+        excl = HM.group.str.startswith("spatialLIBD").values                # DLPFC study removed from training
+        htot_ = htot
+    else:
+        eg = pd.read_csv(os.path.join(E, "idc_genes.csv")).gene.values
+        ec = np.load(os.path.join(E, "idc_counts.npy")).astype(np.float32)
+        eidx = pd.Series(range(len(eg)), index=eg)
+        eidx = eidx[~eidx.index.duplicated()]
+        genes = [g for g in eg if g in hset.index]
+        etot = ec[:, eidx.loc[genes].values].sum(1)                         # library size over the panel genes
+        htot_ = np.asarray(hcounts[:, hset.loc[genes].values].sum(1)).ravel()
+        excl = HM["sample"].isin(em["sample"].unique()).values
+    Gh = lognorm(hcounts[:, hset.loc[genes].values].toarray(), htot_)
+    Ge = lognorm(ec[:, eidx.loc[genes].values], etot)
+    pool = np.flatnonzero(~excl & (htot_ > 0))
+    print(f"{st}: {len(em)} external spots, {len(genes)} genes, HEST training pool {len(pool)} spots "
+          f"({int(excl.sum())} excluded)", flush=True)
+    f_mv = os.path.join(CACHE, f"if_virtues_m16_virtual_miphei_{st}.npy")
+    mv = np.load(f_mv)[:, 0].astype(np.float32) if os.path.exists(f_mv) else None
+    f_t = os.path.join(CACHE, f"{st}_gentext_text_biomedclip.npy")
+    tx = np.load(f_t)[:, 0].astype(np.float32) if os.path.exists(f_t) else None
+    zh = nz(he)
+    for k in range(K):
+        grp = np.random.default_rng(k).permutation(sorted(HM.group.iloc[pool].unique()))
+        va_g = set(grp[: max(1, len(grp) // 10)])
+        isva = HM.group.iloc[pool].isin(va_g).values
+        tr, va = pool[~isva], pool[isva]
+        for norm in ["z-scored per dataset", "plain"]:
+            Gtr = zscore(Gh, Gh[tr]) if norm != "plain" else Gh
+            Gex = zscore(Ge, Ge) if norm != "plain" else Ge
+            head, best = train_head(Gtr, tr, va, seed=k)
+            with torch.no_grad():
+                zg = nz(head(torch.from_numpy(Gex)))
+            rows.append({"set": st, "fold": k, "rna_input": norm, "part": "real", "pair": "H&E<->RNA",
+                         "kind": "trained pair (external)", "circular": False, "n": len(zh), "val_R@1": best, **g100(zh, zg)})
+            if mv is not None and norm == "z-scored per dataset":
+                with torch.no_grad():
+                    zm = nz(mif_head(k)(mv))
+                for pair, a, b, kind, circ in [("RNA<->mIF(MIPHEI)", zg, zm, "emergent: real x virtual", False),
+                                               ("H&E<->mIF(MIPHEI)", zh, zm, "virtual from this H&E", True)]:
+                    rows.append({"set": st, "fold": k, "rna_input": norm, "part": "virtual", "pair": pair, "kind": kind,
+                                 "circular": circ, "n": len(zh), **g100(a, b)})
+            if tx is not None and norm == "z-scored per dataset":
+                with torch.no_grad():
+                    zt = nz(text_head(k)(tx))
+                    zm = nz(mif_head(k)(mv)) if mv is not None else None
+                for pair, a, b, kind, circ in [("RNA<->text(synthetic)", zg, zt, "emergent: real x virtual", False),
+                                               ("H&E<->text(synthetic)", zh, zt, "virtual from this H&E", True),
+                                               ("mIF(MIPHEI)<->text(synthetic)", zm, zt, "emergent: virtual x virtual", True)]:
+                    if a is not None:
+                        rows.append({"set": st, "fold": k, "rna_input": norm, "part": "virtual", "pair": pair, "kind": kind,
+                                     "circular": circ, "n": len(zh), **g100(a, b)})
+            if norm == "z-scored per dataset":  # embeddings for 39_external_check.py
+                np.savez(os.path.join(E, f"emb_{st}_fold{k}.npz"), he=zh.numpy(), rna=zg.numpy(),
+                         **({"mif": zm.numpy()} if mv is not None else {}), **({"text": zt.numpy()} if tx is not None else {}))
+        print(f"  {st} head {k}: val R@1 {best:.3f}", flush=True)
+
+R = pd.DataFrame(rows)
+R.to_csv(os.path.join(OUT, "external_long.csv"), index=False)
+cols = ["R@1", "R@5", "R@10", "MRR", "pair_acc"]
+S = R.groupby(["set", "part", "pair", "rna_input", "kind", "circular"], sort=False)[cols + ["n"]].agg(["mean", "std"])
+S.round(4).to_csv(os.path.join(OUT, "external_summary.csv"))
+pd.set_option("display.width", 250)
+print(R.groupby(["set", "part", "pair", "rna_input", "circular"], sort=False)[cols + ["n"]].mean().round(3).to_string())

experiments/39_external_check.py

@@ -0,0 +1,75 @@
+"""Sanity checks of the external results (38_external_eval.py; embeddings cache/external/emb_<set>_fold<k>.npz).
+
+Galleries in 38_ mix spots of several samples (IDC 4 samples, DLPFC 12 sections). A model that only recognises the
+SAMPLE (batch / stain / platform effects) and is random within it already beats chance there. Per pair and set:
+  mixed G         : the 38_ metric (G = 100 random spots from all samples), both directions
+  sample oracle   : R@1 of a model that knows the sample exactly and is random within it = mean 1/n_same_sample
+  top-1 same samp : fraction of top-1 hits from the query's own sample (base rate = share of same-sample items)
+  within-sample G : galleries drawn from ONE sample only (IDC G = 100; DLPFC G = 35 since sections have 35-48 spots,
+                    with the mixed metric at G = 35 for comparison) -> spot-level signal beyond sample identity
+  shuffled null   : the mixed metric after permuting one side (checks the metric code; ~1/G)
+Output results/external_check.csv
+"""
+import os
+
+import numpy as np
+import pandas as pd
+
+from common import CACHE, ROOT
+
+E = os.path.join(CACHE, "external")
+PAIRS = [("he", "rna"), ("rna", "mif"), ("he", "mif"), ("rna", "text"), ("he", "text"), ("mif", "text")]
+NAME = {"he": "H&E", "rna": "RNA", "mif": "mIF(MIPHEI)", "text": "text(synthetic)"}
+DRAWS = 200
+
+
+def r1(a, b, s):
+    sim = a[s] @ b[s].T; d = np.diag(sim)[:, None]
+    return ((sim > d).sum(1) == 0).mean() / 2 + ((sim > d.T).sum(0) == 0).mean() / 2, sim
+
+
+def run(a, b, smp, G, within, rng):
+    out = {"R@1": [], "oracle": [], "top1_same": [], "base_same": []}
+    uniq = [u for u in np.unique(smp) if (smp == u).sum() >= G]
+    for _ in range(DRAWS):
+        if within:
+            idx = np.flatnonzero(smp == uniq[rng.integers(len(uniq))])
+            s = rng.choice(idx, G, replace=False)
+        else:
+            s = rng.choice(len(a), G, replace=False)
+        v, sim = r1(a, b, s)
+        g = smp[s]; same = g[:, None] == g[None, :]
+        out["R@1"].append(v)
+        out["oracle"].append((1 / same.sum(1)).mean())
+        out["top1_same"].append((same[np.arange(G), sim.argmax(1)].mean() + same[sim.argmax(0), np.arange(G)].mean()) / 2)
+        out["base_same"].append(same.mean())
+    return {k: float(np.mean(v)) for k, v in out.items()}
+
+
+rows = []
+for st, Gw in [("dlpfc", 35)]:  # IDC dropped from the report 2026-10-18
+    smp = pd.read_csv(os.path.join(E, f"{st}_meta.csv"))["sample"].astype(str).values
+    for k in range(5):
+        Z = np.load(os.path.join(E, f"emb_{st}_fold{k}.npz"))
+        for x, y in PAIRS:
+            if x not in Z or y not in Z:
+                continue
+            a, b = Z[x], Z[y]
+            rng = np.random.default_rng(k)
+            base = {"set": st, "fold": k, "pair": f"{NAME[x]}<->{NAME[y]}"}
+            m = run(a, b, smp, 100, False, rng)
+            rows.append({**base, "gallery": "mixed G=100", "chance": 0.01, **m})
+            rows.append({**base, "gallery": "shuffled null G=100", "chance": 0.01,
+                         **run(a, b[rng.permutation(len(b))], smp, 100, False, rng)})
+            if Gw != 100:
+                rows.append({**base, "gallery": f"mixed G={Gw}", "chance": 1 / Gw, **run(a, b, smp, Gw, False, rng)})
+            rows.append({**base, "gallery": f"within-sample G={Gw}", "chance": 1 / Gw, **run(a, b, smp, Gw, True, rng)})
+
+R = pd.DataFrame(rows)
+R.to_csv(os.path.join(ROOT, "results", "external_check.csv"), index=False)
+S = R.groupby(["set", "pair", "gallery"], sort=False).agg(
+    R1=("R@1", "mean"), R1_sd=("R@1", "std"), chance=("chance", "first"), oracle=("oracle", "mean"),
+    top1_same=("top1_same", "mean"), base_same=("base_same", "mean"))
+S["x_chance"] = S.R1 / S.chance
+pd.set_option("display.width", 250)
+print(S.round(3).to_string())

experiments/40_clean_text.py

@@ -0,0 +1,28 @@
+"""Remove junk texts from the TCGA image-text pairs (cache/alltxt_meta.csv) and list the images left without real text.
+Junk = bare multiple-choice answer letters ("A A B B"), "error error" strings, or < 5 words (answers without question).
+Output cache/alltxt_junk.csv     (idx = row of alltxt_meta.csv, junk, reason)   -> 33_ trains / tests on non-junk only
+       cache/notext_meta.csv     (img_idx, image_path, patient) images whose texts are all junk -> 27_ --sets tcga_notext
+                                 (few-shot GPT captions; VALIDATION ONLY, never trained on)
+"""
+import os
+
+import pandas as pd
+
+from common import CACHE
+
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+I = pd.read_csv(os.path.join(CACHE, "allimg_meta.csv"))
+c = T.caption.astype(str)
+w = c.str.split().str.len()
+reason = pd.Series("", index=T.index)
+reason[w < 5] = "fewer than 5 words"
+reason[c.str.lower().str.fullmatch(r"(\s*error\s*)+")] = "error string"
+reason[c.str.fullmatch(r"\s*([A-Ea-e][\s,.;]*)+")] = "answer letters"
+J = pd.DataFrame({"idx": T.index, "junk": reason != "", "reason": reason})
+J.to_csv(os.path.join(CACHE, "alltxt_junk.csv"), index=False)
+ok_img = set(T.img_idx[~J.junk])
+N = I[~I.index.isin(ok_img)]
+pd.DataFrame({"img_idx": N.index, "image_path": N.image_path, "patient": N.patient}).to_csv(
+    os.path.join(CACHE, "notext_meta.csv"), index=False)
+print(f"texts {len(T)}: junk {J.junk.sum()} {J[J.junk].reason.value_counts().to_dict()}; clean {(~J.junk).sum()}")
+print(f"images {len(I)}: with clean real text {len(ok_img)}, without -> notext_meta.csv {len(N)}")

experiments/41_text_enc_clean.py

@@ -0,0 +1,105 @@
+"""Re-check of the H&E <-> text encoder comparison (20_pairing.py, 28_) under the final protocol with CLEAN real text:
+TCGA splits/final/tcga_fold<k>.csv (patient 5-fold, val = early stopping only), junk texts removed (40_clean_text.py),
+identical text_head recipe to 33_final_eval.py (frozen image anchor, MLP on frozen text features, multi-positive InfoNCE).
+Image anchors x text towers; PathGen-CLIP is left out (trained on PathGen-1.6M = most of this pool -> not held out).
+Output results/text_enc_clean_long.csv, text_enc_clean_summary.csv
+"""
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT
+
+IMGS = ["uni2", "phikonv2", "gigapath", "conch"]
+TXTS = ["biomedclip", "plip", "imagebind", "conchclip"]
+K, G100, DRAWS = 5, 100, 50
+dev = "cuda"
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+t_img = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+
+
+def mlp(din, dout, hid=512):  # = 33_
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):  # = 33_
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def train(X, Tx, tr_t, va_t, epochs=60, bs=256, lr=3e-4, seed=0):  # = 33_ train_text_head
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    th = mlp(Tx.shape[1], X.shape[-1]).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(th.parameters()) + [logit], lr=lr, weight_decay=0.05)
+    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=epochs * max(1, len(tr_t) // bs), pct_start=0.05)
+    va_i, va_tt = one_text_per_image(va_t, np.random.default_rng(0))
+    za_va = F.normalize(X[torch.as_tensor(va_i, device=dev), 0], dim=-1)
+    best, state = -1, None
+    for ep in range(epochs):
+        th.train()
+        perm = rng.permutation(tr_t)
+        for s in range(0, len(perm) - bs + 1, bs):
+            b = torch.as_tensor(perm[s:s + bs], device=dev)
+            ii = t_img[b]
+            zi = F.normalize(X[ii, torch.randint(0, X.shape[1], (len(b),), device=dev)], dim=-1)
+            lg = logit.exp().clamp(max=100) * zi @ F.normalize(th(Tx[b]), dim=-1).T
+            pos = (ii[:, None] == ii[None, :]).float()
+            tgt = pos / pos.sum(1, keepdim=True)
+            loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sched.step()
+        th.eval()
+        with torch.no_grad():
+            sim = za_va @ F.normalize(th(Tx[torch.as_tensor(va_tt, device=dev)]), dim=-1).T
+            v = float((((sim > sim.diag()[:, None]).sum(1) == 0).float().mean() + ((sim.T > sim.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+        if v > best:
+            best, state = v, {k_: x.detach().clone() for k_, x in th.state_dict().items()}
+    th.load_state_dict(state); th.eval()
+    return th, best
+
+
+rows = []
+for ie in IMGS:
+    X = torch.from_numpy(np.load(os.path.join(CACHE, f"allimg_he_{ie}.npy")).astype(np.float32)).to(dev)
+    for te_ in TXTS:
+        Tx = torch.from_numpy(np.load(os.path.join(CACHE, f"alltxt_text_{te_}.npy"))[:, 0].astype(np.float32)).to(dev)
+        for k in range(K):
+            sp = pd.read_csv(os.path.join(ROOT, "splits", "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+            tsp = sp.loc[T.img_idx].values
+            tr_t, va_t, te_t = (T.index[tsp == x].values for x in ["train", "val", "test"])
+            th, best = train(X, Tx, tr_t, va_t)
+            ii, tt = one_text_per_image(te_t, np.random.default_rng(k))
+            with torch.no_grad():
+                za = F.normalize(X[torch.as_tensor(ii, device=dev), 0], dim=-1).cpu()
+                zt = F.normalize(th(Tx[torch.as_tensor(tt, device=dev)]), dim=-1).cpu()
+            rows.append({"img_enc": ie, "txt_enc": te_, "fold": k, "val_R@1": best, "n_test": len(ii), **g100(za, zt)})
+            print(f"{ie} + {te_}  fold {k}: val {best:.3f}  test R@1 {rows[-1]['R@1']:.3f}", flush=True)
+R = pd.DataFrame(rows)
+R.to_csv(os.path.join(ROOT, "results", "text_enc_clean_long.csv"), index=False)
+S = R.groupby(["img_enc", "txt_enc"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "text_enc_clean_summary.csv"))
+pd.set_option("display.width", 200)
+print(R.pivot_table(index="img_enc", columns="txt_enc", values="R@1", sort=False).round(3).to_string())

experiments/42_train_regimes.py

@@ -0,0 +1,299 @@
+"""Training-regime comparison: SEPARATE vs JOINT training of the four-modality model (real pairs only, final protocol).
+
+Real paired data (fold k; train / val (early stopping only) / test, patients / study groups never cross splits):
+  ORION  H&E <-> mIF   splits/patient_5fold/fold<k>.csv   UNI2 (8 views) <-> VirTues 16-ch MIPHEI panel (8 aligned views)
+  HEST   H&E <-> RNA   splits/final/hest_fold<k>.csv      UNI2 <-> log1p CP10k of the 2000 HVGs
+  TCGA   H&E <-> text  splits/final/tcga_fold<k>.csv      UNI2 (8 views) <-> BiomedCLIP, CLEAN real texts (40_)
+Regimes (same MLP heads, InfoNCE, AdamW 3e-4 / wd 0.05 / OneCycle, batch 256, one temperature per pair):
+  S1 separate, frozen anchor     modality head -> frozen UNI2 space; three independent trainings (= current method)
+  S2 separate, two-tower         per pair a trainable H&E head + modality head (CLIP-style); no shared space
+  J1 joint, frozen anchor        S1 heads trained in one loop (sum of the 3 losses), joint early stopping
+  J2 joint, shared H&E head      ONE trainable H&E head shared by all pairs + 3 modality heads (ImageBind with
+                                 a learned anchor projection)
+  J3 J2, size-proportional       one task per step, sampled proportional to its training-set size
+  J4 J2 + anchor distillation    + lambda * MSE between in-batch cosine-similarity matrices of UNI2 and of the projected
+                                 H&E (keeps the shared head close to the UNI2 geometry)
+  J5 joint, residual adapter     H&E = UNI2 + shared zero-initialised MLP adapter (frozen anchor + small shared change)
+  S3 separate, frozen, linear    S1 with a single linear layer per modality (head capacity)
+  J4b J4 with lambda = 10        stronger distillation
+  J6 J2 + uncertainty weighting  learnable per-task log-variance s_t: sum_t exp(-s_t) L_t + s_t (Kendall et al. 2018)
+  S4 separate, hybrid heads      S1 with MLP head for mIF, linear heads for RNA and text
+  J8 two-stage from S4           J7 starting from the S4 heads
+  J7 two-stage                   S1 heads, then joint fine-tuning of S1 heads + residual adapter (lr 1e-4, 1000 steps,
+                                 distillation lambda 1)
+Early stopping: separate = own val R@1; joint = geometric mean of the three val R@1 (every 50 steps).
+Test (fold k): trained pairs H&E<->mIF / RNA / text, and emergent pairs with test-time-only virtual data:
+  ORION mIF<->RNA(STPath virtual), mIF<->text(GPT synthetic); TCGA text<->RNA(STPath virtual), text<->mIF(MIPHEI virtual).
+Usage: python 42_train_regimes.py [--regimes S1 S2 J1 J2 J3 J4 J5] [--folds 0 1 2 3 4]
+Output results/regimes_long.csv, results/regimes_summary.csv
+"""
+import argparse
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, SPLITS
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--regimes", nargs="+", default=["S1", "S2", "S3", "S4", "J1", "J2", "J3", "J4", "J4b", "J5", "J6", "J7", "J8"])
+ap.add_argument("--folds", nargs="+", type=int, default=[0, 1, 2, 3, 4])
+args = ap.parse_args()
+dev = "cuda"
+H = os.path.join(CACHE, "hest")
+G100, DRAWS, BS, LR, EVAL_EVERY = 100, 50, 256, 3e-4, 50
+SEP_STEPS = {"mif": 400, "rna": 2700, "text": 1000}  # = 40 / 100 / 60 epochs of 17_ / 23_ / 33_
+JOINT_STEPS = {"equal": 2000, "prop": 4000}
+f32 = lambda a: torch.from_numpy(np.asarray(a, dtype=np.float32))
+
+# ---------------------------------------------------------------- data (all on GPU)
+HE_O = f32(np.load(os.path.join(CACHE, "he_uni2.npy"))).to(dev)                 # (3187, 8, 1536)
+IF_O = f32(np.load(os.path.join(CACHE, "if_virtues_m16.npy"))).to(dev)          # (3187, 8, 512)
+IF_V_T = f32(np.load(os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy"))[:, 0]).to(dev)
+HE_H = f32(np.load(os.path.join(H, "he_uni2.npy"))[:, 0]).to(dev)               # (9484, 1536)
+RNA_H = f32(np.load(os.path.join(H, "hvg.npy"))).to(dev)                         # (9484, 2000)
+HE_T = f32(np.load(os.path.join(CACHE, "allimg_he_uni2.npy"))).to(dev)           # (6144, 8, 1536)
+TXT = f32(np.load(os.path.join(CACHE, "alltxt_text_biomedclip.npy"))[:, 0]).to(dev)
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+T_IMG = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+GT_O = f32(np.load(os.path.join(CACHE, "orion_gentext_text_biomedclip.npy"))[:, 0]).to(dev)
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def virtual_rna(st):  # = 33_final_eval.py
+    g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+    c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+    c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+    return f32(c[:, pd.Series(range(len(g)), index=g).loc[hvg].values]).to(dev)
+
+
+RNA_V_O, RNA_V_T = virtual_rna("orion"), virtual_rna("tcga_text")
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_
+    a, b = a.cpu(), b.cpu()
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float((((s > s.diag()[:, None]).sum(1) == 0).float().mean() + ((s.T > s.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+
+
+class Model(nn.Module):
+    def __init__(self, anchor, tasks, head="mlp", uw=False):
+        super().__init__()
+        self.anchor = anchor
+        kind = head if isinstance(head, dict) else {t: head for t in ("mif", "rna", "text")}
+        mk = lambda t, d: mlp(d) if kind[t] == "mlp" else nn.Linear(d, 1536)
+        self.heads = nn.ModuleDict({"mif": mk("mif", 512), "rna": mk("rna", 2000), "text": mk("text", 512)})
+        self.uw = nn.ParameterDict({t: nn.Parameter(torch.zeros(())) for t in tasks}) if uw else None
+        self.logit = nn.ParameterDict({t: nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32)) for t in tasks})
+        if anchor == "shared":
+            self.ah = mlp(1536)
+        elif anchor == "pair":
+            self.ah = nn.ModuleDict({t: mlp(1536) for t in tasks})
+        elif anchor == "residual":
+            self.ah = mlp(1536)
+            nn.init.zeros_(self.ah[-1].weight); nn.init.zeros_(self.ah[-1].bias)
+
+    def he(self, x, task):
+        if self.anchor == "frozen":
+            return x
+        if self.anchor == "shared":
+            return self.ah(x)
+        if self.anchor == "pair":
+            return self.ah[task](x)
+        return x + self.ah(x)  # residual
+
+
+class Fold:
+    def __init__(self, k):
+        s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+        self.o = {x: s.tile_id[s.split == x].values for x in ["train", "val", "test"]}
+        self.if_mu = IF_O[self.o["train"]].mean((0, 1)); self.if_sd = IF_O[self.o["train"]].std((0, 1)) + 1e-3
+        h = pd.read_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"))
+        self.h = {x: h.spot_idx[h.split == x].values for x in ["train", "val", "test"]}
+        sp = pd.read_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+        tsp = sp.loc[T.img_idx].values
+        self.t = {x: T.index[tsp == x].values for x in ["train", "val", "test"]}
+        self.t_va = one_text_per_image(self.t["val"], np.random.default_rng(0))
+        self.t_te = one_text_per_image(self.t["test"], np.random.default_rng(k))
+        self.n = {"mif": len(self.o["train"]), "rna": len(self.h["train"]), "text": len(self.t["train"])}
+
+    def nif(self, x):
+        return (x - self.if_mu) / self.if_sd
+
+
+def batch_loss(m, F_, task, rng):
+    if task == "mif":
+        b = torch.as_tensor(rng.choice(F_.o["train"], BS, replace=False), device=dev)
+        d = torch.randint(0, 8, (BS,), device=dev)
+        x_he, x_m = HE_O[b, d], F_.nif(IF_O[b, d]); pos = None
+    elif task == "rna":
+        b = torch.as_tensor(rng.choice(F_.h["train"], BS, replace=False), device=dev)
+        x_he, x_m = HE_H[b], RNA_H[b]; pos = None
+    else:
+        b = torch.as_tensor(rng.choice(F_.t["train"], BS, replace=False), device=dev)
+        ii = T_IMG[b]
+        x_he, x_m = HE_T[ii, torch.randint(0, 8, (BS,), device=dev)], TXT[b]
+        pos = (ii[:, None] == ii[None, :]).float()
+    za = F.normalize(m.he(x_he, task), dim=-1)
+    zm = F.normalize(m.heads[task](x_m), dim=-1)
+    lg = m.logit[task].exp().clamp(max=100) * za @ zm.T
+    if pos is None:
+        t = torch.arange(BS, device=dev)
+        loss = (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+    else:
+        tgt = pos / pos.sum(1, keepdim=True)
+        loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+    if m.uw is not None:
+        loss = torch.exp(-m.uw[task]) * loss + m.uw[task]
+    if m.distill and m.anchor != "frozen":
+        u = F.normalize(x_he, dim=-1)
+        loss = loss + m.distill * F.mse_loss(za @ za.T, u @ u.T)
+    return loss
+
+
+@torch.no_grad()
+def embed(m, F_, task, split):
+    if task == "mif":
+        idx = F_.o[split]
+        return F.normalize(m.he(HE_O[idx, 0], task), dim=-1), F.normalize(m.heads["mif"](F_.nif(IF_O[idx, 0])), dim=-1)
+    if task == "rna":
+        idx = F_.h[split]
+        return F.normalize(m.he(HE_H[idx], task), dim=-1), F.normalize(m.heads["rna"](RNA_H[idx]), dim=-1)
+    ii, tt = F_.t_va if split == "val" else F_.t_te
+    ii, tt = torch.as_tensor(ii, device=dev), torch.as_tensor(tt, device=dev)
+    return F.normalize(m.he(HE_T[ii, 0], task), dim=-1), F.normalize(m.heads["text"](TXT[tt]), dim=-1)
+
+
+def val_scores(m, F_, tasks):
+    m.eval()
+    v = {t: r1(*embed(m, F_, t, "val")) for t in tasks}
+    m.train()
+    return v
+
+
+def train(anchor, tasks, F_, steps, sampling="equal", distill=0.0, seed=0, head="mlp", uw=False, init=None, lr=LR):
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    m = Model(anchor, tasks, head, uw).to(dev); m.distill = distill
+    if init is not None:  # two-stage: start from separately trained heads / temperatures
+        for t, src in init.items():
+            m.heads[t].load_state_dict(src.heads[t].state_dict()); m.logit[t].data.copy_(src.logit[t].data)
+    opt = torch.optim.AdamW(m.parameters(), lr=lr, weight_decay=0.05)
+    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.05)
+    p = np.array([F_.n[t] for t in tasks], float); p /= p.sum()
+    best, state, bv = -1, None, None
+    m.train()
+    for step in range(1, steps + 1):
+        todo = tasks if sampling == "equal" else [tasks[rng.choice(len(tasks), p=p)]]
+        loss = sum(batch_loss(m, F_, t, rng) for t in todo)
+        opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sched.step()
+        if step % EVAL_EVERY == 0 or step == steps:
+            v = val_scores(m, F_, tasks)
+            score = float(np.exp(np.mean(np.log([max(x, 1e-4) for x in v.values()]))))
+            if score > best:
+                best, bv, state = score, v, {k_: x.detach().clone() for k_, x in m.state_dict().items()}
+    m.load_state_dict(state); m.eval()
+    return m, bv
+
+
+@torch.no_grad()
+def evaluate(models, F_):
+    """models: task -> model holding that task's head (same object for joint regimes)"""
+    out = {}
+    for t, lab in [("mif", "H&E<->mIF (ORION)"), ("rna", "H&E<->RNA (HEST)"), ("text", "H&E<->text (TCGA)")]:
+        out[(lab, "trained")] = g100(*embed(models[t], F_, t, "test"))
+    te = F_.o["test"]
+    zm = F.normalize(models["mif"].heads["mif"](F_.nif(IF_O[te, 0])), dim=-1)
+    zr = F.normalize(models["rna"].heads["rna"](RNA_V_O[te]), dim=-1)
+    zt = F.normalize(models["text"].heads["text"](GT_O[te]), dim=-1)
+    out[("mIF<->RNA(virtual) (ORION)", "emergent")] = g100(zm, zr)
+    out[("mIF<->text(synthetic) (ORION)", "emergent")] = g100(zm, zt)
+    ii, tt = F_.t_te
+    ii, tt = torch.as_tensor(ii, device=dev), torch.as_tensor(tt, device=dev)
+    zt = F.normalize(models["text"].heads["text"](TXT[tt]), dim=-1)
+    zr = F.normalize(models["rna"].heads["rna"](RNA_V_T[ii]), dim=-1)
+    zv = F.normalize(models["mif"].heads["mif"](F_.nif(IF_V_T[ii])), dim=-1)
+    out[("text<->RNA(virtual) (TCGA)", "emergent")] = g100(zt, zr)
+    out[("text<->mIF(MIPHEI) (TCGA)", "emergent")] = g100(zt, zv)
+    return out
+
+
+HYB = {"mif": "mlp", "rna": "linear", "text": "linear"}  # per-modality head capacity
+REG = {"S1": dict(anchor="frozen", mode="separate"), "S2": dict(anchor="pair", mode="separate"),
+       "S3": dict(anchor="frozen", mode="separate", head="linear"),
+       "J1": dict(anchor="frozen", mode="joint"), "J2": dict(anchor="shared", mode="joint"),
+       "J3": dict(anchor="shared", mode="joint", sampling="prop"), "J4": dict(anchor="shared", mode="joint", distill=1.0),
+       "J4b": dict(anchor="shared", mode="joint", distill=10.0), "J5": dict(anchor="residual", mode="joint"),
+       "J6": dict(anchor="shared", mode="joint", uw=True), "J7": dict(anchor="residual", mode="two-stage", distill=1.0),
+       "S4": dict(anchor="frozen", mode="separate", head=HYB), "J8": dict(anchor="residual", mode="two-stage", distill=1.0, head=HYB)}
+NAME = {"S1": "separate, frozen anchor", "S2": "separate, two-tower", "J1": "joint, frozen anchor",
+        "J2": "joint, shared H&E head", "J3": "joint, shared H&E head, size-prop.", "J4": "joint, shared H&E head + distill",
+        "J5": "joint, residual adapter", "S3": "separate, frozen anchor, linear head",
+        "J4b": "joint, shared H&E head + distill x10", "J6": "joint, shared H&E head, uncertainty wt.",
+        "J7": "two-stage: separate -> joint residual", "S4": "separate, frozen, MLP mIF + linear RNA/text",
+        "J8": "two-stage from S4 heads -> joint residual"}
+TASKS = ["mif", "rna", "text"]
+f_out = os.path.join(ROOT, "results", "regimes_long.csv")
+rows = pd.read_csv(f_out).to_dict("records") if os.path.exists(f_out) else []
+done = {(r["regime"], r["fold"]) for r in rows}
+for k in args.folds:
+    F_ = Fold(k)
+    for rg in args.regimes:
+        if (rg, k) in done:
+            continue
+        c = REG[rg]
+        if c["mode"] in ("separate", "two-stage"):
+            models, val = {}, {}
+            for t in TASKS:
+                models[t], v = train("frozen" if c["mode"] == "two-stage" else c["anchor"], [t], F_, SEP_STEPS[t],
+                                     head=c.get("head", "mlp"))
+                val.update(v)
+        if c["mode"] == "joint":
+            sm = c.get("sampling", "equal")
+            m, val = train(c["anchor"], TASKS, F_, JOINT_STEPS[sm], sm, c.get("distill", 0.0), uw=c.get("uw", False))
+            models = {t: m for t in TASKS}
+        elif c["mode"] == "two-stage":
+            m, val = train(c["anchor"], TASKS, F_, 1000, "equal", c["distill"], init=models, lr=1e-4, head=c.get("head", "mlp"))
+            models = {t: m for t in TASKS}
+        for (pair, kind), r in evaluate(models, F_).items():
+            rows.append({"regime": rg, "name": NAME[rg], "fold": k, "pair": pair, "kind": kind, **r,
+                         "val_mif": val["mif"], "val_rna": val["rna"], "val_text": val["text"]})
+        pd.DataFrame(rows).to_csv(f_out, index=False)
+        last = [r for r in rows if r["regime"] == rg and r["fold"] == k]
+        print(f"fold {k} {rg:>2} {NAME[rg]:<36} " + "  ".join(f"{r['pair'].split(' (')[0]} {r['R@1']:.3f}" for r in last), flush=True)
+
+R = pd.DataFrame(rows)
+S = R.groupby(["regime", "name", "kind", "pair"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "regimes_summary.csv"))
+pd.set_option("display.width", 250)
+print(R.pivot_table(index=["regime", "name"], columns="pair", values="R@1", sort=False).round(3).to_string())

experiments/43_rnafm_base.py

@@ -0,0 +1,185 @@
+"""Frozen RNA embeddings for every RNA set of rna_sets.py (hest, hest_v, orion_v, tcga_v, dlpfc).
+
+    python 43_rnafm_base.py scgpt | geneformer | genept-bge | genept-bmc
+
+scgpt / geneformer : same preprocessing and readout as 22_extract_gene_fm.py (scGPT-human <cls> on the HEST 2000 HVGs
+                     in vocab, quantile-binned log1p CP10k, <=1200 genes; Geneformer-V2-104M rank-value encoding,
+                     penultimate layer mean over gene tokens). 22_ only embedded HEST; this adds the virtual / external sets.
+genept-*           : GenePT-w (Chen & Zou 2024) with open text encoders: each gene = text embedding of its NCBI + UniProt
+                     summary (GenePT v2 release, Zenodo 10833191, CC BY 4.0, ext/genept/); spot = average of the gene
+                     embeddings weighted by log1p CP10k over the genes that have a summary, L2-normalised.
+                     bge = BAAI/bge-base-en-v1.5 (CLS, 512 tokens), bmc = BiomedCLIP PubMedBERT text tower (256 tokens,
+                     the text tower of the text modality -> RNA lands in the same text space).
+Output cache/rnafm/<model>_<set>.npy (rna_sets.save)
+"""
+import importlib
+import json
+import os
+import pickle
+import sys
+import time
+
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+import torch
+
+from common import CACHE, ROOT
+from rna_sets import SETS, load, save
+
+M15 = importlib.import_module("15_extract_marker_fm")
+name = sys.argv[1]
+sets = sys.argv[2:] or SETS
+dev = "cuda"
+HVG = pd.read_csv(os.path.join(CACHE, "hest", "hvg_genes.csv")).gene.values
+t0 = time.time()
+
+
+def cp10k_log(X, cols):
+    tot = np.asarray(X.sum(1)).ravel()
+    L = (sp.diags(1e4 / np.maximum(tot, 1)) @ X[:, cols]).tocsr()
+    L.data = np.log1p(L.data)
+    return L
+
+
+def scgpt_model():
+    from huggingface_hub import hf_hub_download
+    args = json.load(open(hf_hub_download(M15.SCGPT_REPO, "args.json")))
+    vocab = json.load(open(hf_hub_download(M15.SCGPT_REPO, "vocab.json")))
+    sd = torch.load(hf_hub_download(M15.SCGPT_REPO, "best_model.pt"), map_location="cpu", weights_only=True)
+    sd = {k.replace("self_attn.Wqkv.", "self_attn.in_proj_"): v for k, v in sd.items()}
+    model = M15.ScGPT(len(vocab), args["embsize"], args["nheads"], args["d_hid"], args["nlayers"], vocab[args["pad_token"]])
+    model.load_state_dict(sd, strict=True)
+    return model.eval().to(dev), args, vocab
+
+
+def scgpt(X, genes, model, args, vocab):
+    PAD, CLS = vocab[args["pad_token"]], vocab["<cls>"]
+    hv = set(HVG)
+    gi = np.array([i for i, g in enumerate(genes) if g in hv and g in vocab])
+    gid = np.array([vocab[g] for g in genes[gi]])
+    L = cp10k_log(X, gi)
+    rng = np.random.default_rng(0)
+    seqs = []
+    for r in range(L.shape[0]):
+        nz, v = L.indices[L.indptr[r]:L.indptr[r + 1]], L.data[L.indptr[r]:L.indptr[r + 1]]
+        if len(nz) > 1199:
+            keep = np.sort(rng.choice(len(nz), 1199, replace=False))
+            nz, v = nz[keep], v[keep]
+        vals = M15.scgpt_binning(v, args["n_bins"], rng).astype(np.float32) if len(v) else np.zeros(0, np.float32)
+        seqs.append((np.r_[CLS, gid[nz]], np.r_[np.float32(args["pad_value"]), vals]))
+    print(f"  scGPT: {len(gi)} HVGs in vocab; genes/spot median {np.median([len(s[0]) - 1 for s in seqs])}", flush=True)
+    embs = []
+    with torch.no_grad():
+        for i in range(0, len(seqs), 32):
+            b = seqs[i:i + 32]
+            Lm = max(len(g) for g, _ in b)
+            g = torch.full((len(b), Lm), PAD, dtype=torch.long)
+            v = torch.full((len(b), Lm), float(args["pad_value"]))
+            for j, (gg, vv) in enumerate(b):
+                g[j, :len(gg)], v[j, :len(vv)] = torch.from_numpy(gg), torch.from_numpy(vv)
+            g, v = g.to(dev), v.to(dev)
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                h = model.encode(g, v, g.eq(PAD))
+            embs.append(h[:, 0].float().cpu())
+    E = torch.cat(embs).numpy()
+    return E / np.linalg.norm(E, axis=1, keepdims=True)
+
+
+def gf_model():
+    from transformers import BertModel
+    tokd = pickle.load(open(M15.hf_file(M15.GF_REPO, "geneformer/token_dictionary_gc104M.pkl"), "rb"))
+    medd = pickle.load(open(M15.hf_file(M15.GF_REPO, "geneformer/gene_median_dictionary_gc104M.pkl"), "rb"))
+    n2id = pickle.load(open(M15.hf_file(M15.GF_REPO, "geneformer/gene_name_id_dict_gc104M.pkl"), "rb"))
+    mdir = os.path.dirname(M15.hf_file(M15.GF_REPO, "Geneformer-V2-104M/model.safetensors"))
+    M15.hf_file(M15.GF_REPO, "Geneformer-V2-104M/config.json")
+    model = BertModel.from_pretrained(mdir, add_pooling_layer=False, attn_implementation="sdpa").eval().to(dev)
+    return model, tokd, medd, n2id
+
+
+def geneformer(X, genes, model, tokd, medd, n2id):
+    gi = np.array([i for i, g in enumerate(genes) if g in n2id and n2id[g] in tokd and n2id[g] in medd])
+    tok = np.array([tokd[n2id[g]] for g in genes[gi]])
+    med = np.array([medd[n2id[g]] for g in genes[gi]])
+    CLS, EOS, PAD = tokd["<cls>"], tokd["<eos>"], tokd["<pad>"]
+    tot = np.asarray(X.sum(1)).ravel()
+    Xg = (sp.diags(1e4 / np.maximum(tot, 1)) @ X[:, gi]).tocsr()
+    seqs = []
+    for r in range(Xg.shape[0]):
+        nz, v = Xg.indices[Xg.indptr[r]:Xg.indptr[r + 1]], Xg.data[Xg.indptr[r]:Xg.indptr[r + 1]]
+        v = v / med[nz]
+        order = nz[np.argsort(-v, kind="stable")][:4094]
+        seqs.append([CLS] + tok[order].tolist() + [EOS])
+    print(f"  Geneformer: {len(gi)} genes in dictionary; tokens/spot median {np.median([len(s) for s in seqs])}", flush=True)
+    order_len = np.argsort([len(s) for s in seqs])
+    embs = np.zeros((len(seqs), model.config.hidden_size), np.float32)
+    with torch.no_grad():
+        for k in range(0, len(seqs), 8):
+            idx = order_len[k:k + 8]
+            b = [seqs[i] for i in idx]
+            ids = torch.full((len(b), max(map(len, b))), PAD, dtype=torch.long)
+            for j, s in enumerate(b):
+                ids[j, :len(s)] = torch.tensor(s)
+            ids = ids.to(dev)
+            att = (ids != PAD).long()
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                h = model(input_ids=ids, attention_mask=att, output_hidden_states=True).hidden_states[-2].float()
+            gm = att.clone().float()
+            gm[:, 0] = 0
+            gm[torch.arange(len(b)), att.sum(1) - 1] = 0
+            embs[idx] = ((h * gm[..., None]).sum(1) / gm.sum(1, keepdim=True).clamp(min=1)).cpu().numpy()
+    return embs
+
+
+def gene_text_table(enc):
+    """Text embedding of each gene's NCBI + UniProt summary; cached in cache/rnafm/genetext_<enc>.npz."""
+    f = os.path.join(CACHE, "rnafm", f"genetext_{enc}.npz")
+    if os.path.exists(f):
+        z = np.load(f, allow_pickle=False)
+        return z["genes"], z["E"]
+    summ = json.load(open(os.path.join(ROOT, "ext", "genept", "unz", "GenePT_emebdding_v2", "NCBI_UniProt_summary_of_genes.json")))
+    g = np.array(sorted(summ))
+    txt = [summ[x] for x in g]
+    out = []
+    if enc == "bge":
+        from transformers import AutoModel, AutoTokenizer
+        tk = AutoTokenizer.from_pretrained("BAAI/bge-base-en-v1.5")
+        m = AutoModel.from_pretrained("BAAI/bge-base-en-v1.5").to(dev).eval()
+        f_ = lambda b: m(**{k: v.to(dev) for k, v in tk(b, padding=True, truncation=True, max_length=512, return_tensors="pt").items()}).last_hidden_state[:, 0]
+    else:
+        from biomedclip_text import BiomedCLIPText  # open_clip-free, reproduces cache/alltxt_text_biomedclip.npy (cos 1.000)
+        m = BiomedCLIPText().to(dev).eval()
+        f_ = m
+    with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16):
+        for i in range(0, len(txt), 128):
+            out.append(torch.nn.functional.normalize(f_(txt[i:i + 128]).float(), dim=-1).cpu())
+    E = torch.cat(out).numpy()
+    os.makedirs(os.path.dirname(f), exist_ok=True)
+    np.savez(f, genes=g, E=E)
+    print(f"  gene texts: {len(g)} genes embedded with {enc} {E.shape}", flush=True)
+    return g, E
+
+
+def genept(X, genes, gt, GE):
+    idx = pd.Series(range(len(gt)), index=gt)
+    gi = np.array([i for i, g in enumerate(genes) if g in idx.index])
+    W = cp10k_log(X, gi)
+    print(f"  GenePT: {len(gi)}/{len(genes)} genes with a summary", flush=True)
+    E = W @ GE[idx.loc[genes[gi]].values]
+    E = E / np.maximum(np.asarray(W.sum(1)), 1e-8)
+    return E / np.linalg.norm(E, axis=1, keepdims=True).clip(1e-8)
+
+
+if name == "scgpt":
+    ctx = scgpt_model(); fn = scgpt
+elif name == "geneformer":
+    ctx = gf_model(); fn = geneformer
+elif name.startswith("genept-"):
+    ctx = gene_text_table(name.split("-")[1]); fn = genept
+else:
+    raise ValueError(name)
+for s in sets:
+    X, genes = load(s)
+    print(f"[{name}] {s} {X.shape}", flush=True)
+    save(name, s, fn(X, genes, *ctx))
+    print(f"  {time.time() - t0:.0f}s", flush=True)

experiments/43_rnafm_c2s.py

@@ -0,0 +1,216 @@
+"""Frozen Cell2Sentence (C2S-Scale-Gemma-2-2B) spot embeddings for the RNA sets of rna_sets.py (spot = pseudo-cell).
+
+    python 43_rnafm_c2s.py [set ...]          (default: all of rna_sets.SETS)
+
+Model     : HF vandijklab/C2S-Scale-Gemma-2-2B @ 5ddf28b8f1c81b7ab7a9be192924da82b6c5d512 (cc-by-4.0), a Gemma-2 2B
+            decoder (26 layers, d=2304) instruction-tuned on cell sentences from >800 CellxGene/HCA datasets. Loaded as
+            a plain transformers Gemma2Model (no remote code, strict: no missing / unexpected keys), bf16, eager
+            attention (keeps Gemma-2's attention-logit soft-capping exactly as in training).
+Tokenizer : the repo ships only the sentencepiece tokenizer.model; sentencepiece is not in the shared env, so it was
+            converted once to ext/cell2sentence/tokenizer/fast/tokenizer.json with transformers' own GemmaConverter
+            inside an isolated venv (ext/cell2sentence/venv) and checked token-identical to the sentencepiece
+            GemmaTokenizer on sampled prompts of every set.
+Cell sentence (exactly as cell2sentence src/cell2sentence/utils.py generate_sentences + prompt_formatter.py, repo
+            vandijklab/cell2sentence @ 6599c9d, read-only clone in ext/cell2sentence/src):
+            raw counts -> sc.pp.normalize_total (target = median total counts of the set) -> log1p base 10
+            (tutorial 0; rank-preserving within a spot) -> gene symbols upper-cased (generate_vocabulary) ->
+            nonzero genes of the spot, shuffled with np.random.seed(42) / sklearn.utils.shuffle and stable-sorted by
+            descending expression (= their random tie-breaking) -> top K=200 symbols (default n_genes=200 of
+            tasks.embed_cells / predict_cell_types_of_data and the tutorials), space-separated.
+            Not applied: tutorial-0 QC filters (filter_cells min_genes=200, filter_genes min_cells=3), because they
+            would drop spots / make a spot's sentence depend on the other spots of the set.
+Prompt    : the cell-type-prediction template used by tasks.embed_cells (C2SPromptFormatter task
+            "cell_type_prediction"); that formatter picks one of 15 paraphrases at random, we fix the first one, which
+            is also the template on the C2S-Scale model card:
+            "The following is a list of {K} gene names ordered by descending expression level in a Homo sapiens cell.
+             Your task is to give the cell type which this cell belongs to based on its gene expression.\\n
+             Cell sentence: {sentence}.\\nThe cell type corresponding to these genes is:"   (+ <bos>, no chat template)
+            {K} is always 200, as in the formatter (spots with < 200 expressed genes keep "200").
+Readout   : last hidden layer (post final RMSNorm = hidden_states[-1], as CSModel.embed_cells_batched).
+            "c2s"      : mean over the cell-sentence tokens only (tokens whose char span overlaps the gene list;
+                         official code averages over the whole padded prompt, which mixes in the constant template
+                         and padding).
+            "c2s-last" : hidden state of the last prompt token (":" after "is"), i.e. the state that would generate
+                         the cell type.
+Output    : cache/rnafm/c2s_<set>.npy and c2s-last_<set>.npy, (n_spot, 2304) float16, set order. Log logs/rnafm/c2s.log.
+"""
+import os
+import sys
+import time
+
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+import torch
+from sklearn.utils import shuffle
+
+from common import CACHE, ROOT
+from rna_sets import SETS, load, save
+
+REPO = "vandijklab/C2S-Scale-Gemma-2-2B"
+REV = "5ddf28b8f1c81b7ab7a9be192924da82b6c5d512"
+TOK_DIR = os.path.join(ROOT, "ext", "cell2sentence", "tokenizer", "fast")
+K = 200
+ORGANISM = "Homo sapiens"
+PREFIX = (f"The following is a list of {K} gene names ordered by descending expression level in a {ORGANISM} cell. "
+          "Your task is to give the cell type which this cell belongs to based on its gene expression.\nCell sentence: ")
+SUFFIX = ".\nThe cell type corresponding to these genes is:"
+GPU_GB = 8.0
+TOK_BUDGET = 8 * 800           # tokens per forward pass (halved on OOM; ~7 GB peak at L~750)
+dev = "cuda"
+
+
+def log(*a):
+    print(*a, flush=True)
+
+
+def cell_sentences(X, genes):
+    """C2S normalisation + rank -> list of top-K space-separated (upper-case) gene symbols, plus #expressed genes."""
+    X = X.tocsr().astype(np.float64)
+    tot = np.asarray(X.sum(1)).ravel()
+    target = np.median(tot[tot > 0])                       # sc.pp.normalize_total default target_sum=None
+    Xn = sp.diags(target / np.maximum(tot, 1e-12)) @ X
+    Xn = Xn.tocsr()
+    Xn.data = np.log1p(Xn.data) / np.log(10)               # sc.pp.log1p(base=10)
+    enc = np.char.upper(np.asarray(genes, dtype=str))
+    np.random.seed(42)                                     # generate_sentences(random_state=42)
+    sents, n_expr = [], np.zeros(Xn.shape[0], int)
+    for i in range(Xn.shape[0]):
+        cols = Xn.indices[Xn.indptr[i]:Xn.indptr[i + 1]]
+        vals = Xn.data[Xn.indptr[i]:Xn.indptr[i + 1]]
+        cols, vals = shuffle(cols, vals)
+        order = cols[np.argsort(-vals, kind="stable")]
+        n_expr[i] = len(order)
+        sents.append(" ".join(enc[order[:K]]))
+    return sents, n_expr
+
+
+def tokenize(tok, sents):
+    """Prompt ids + mask of cell-sentence tokens (by char offsets)."""
+    prompts = [PREFIX + s + SUFFIX for s in sents]
+    enc = tok(prompts, return_offsets_mapping=True, add_special_tokens=True)
+    out = []
+    for p, s, ids, off in zip(prompts, sents, enc["input_ids"], enc["offset_mapping"]):
+        a, b = len(PREFIX), len(PREFIX) + len(s)
+        off = np.asarray(off)
+        m = (off[:, 0] < b) & (off[:, 1] > a) & (off[:, 1] > off[:, 0])
+        assert ids[0] == tok.bos_token_id and m.sum() > 0
+        out.append((np.asarray(ids), m))
+    return prompts, out
+
+
+@torch.no_grad()
+def embed(model, seqs, pad_id):
+    n, D = len(seqs), model.config.hidden_size
+    E_mean, E_last = np.zeros((n, D), np.float32), np.zeros((n, D), np.float32)
+    order = np.argsort([len(s[0]) for s in seqs])[::-1]   # longest first: OOM shows up immediately
+    budget, k, t0, done = TOK_BUDGET, 0, time.time(), 0
+    while k < n:
+        Lm = len(seqs[order[k]][0])
+        bs = max(1, budget // Lm)
+        idx = order[k:k + bs]
+        ids = torch.full((len(idx), Lm), pad_id, dtype=torch.long)
+        msk = torch.zeros((len(idx), Lm), dtype=torch.bool)
+        att = torch.zeros((len(idx), Lm), dtype=torch.long)
+        for j, i in enumerate(idx):
+            s, m = seqs[i]
+            ids[j, :len(s)], msk[j, :len(s)], att[j, :len(s)] = torch.from_numpy(s), torch.from_numpy(m), 1
+        try:
+            h = model(input_ids=ids.to(dev), attention_mask=att.to(dev)).last_hidden_state.float()
+        except torch.OutOfMemoryError:
+            torch.cuda.empty_cache()
+            if budget <= Lm:
+                raise
+            budget //= 2
+            log(f"  OOM -> token budget {budget}")
+            continue
+        mk = msk.to(dev).float()
+        E_mean[idx] = ((h * mk[..., None]).sum(1) / mk.sum(1, keepdim=True)).cpu().numpy()
+        last = att.sum(1).to(dev) - 1
+        E_last[idx] = h[torch.arange(len(idx), device=dev), last].cpu().numpy()
+        k += len(idx)
+        if k // 1000 > done:
+            done = k // 1000
+            log(f"  {k}/{n}  {time.time() - t0:.0f}s  bs={len(idx)} L={Lm}  peak {torch.cuda.max_memory_allocated() / 2**30:.2f} GB")
+    return E_mean, E_last
+
+
+def sanity(E, tag, organ=None):
+    rng = np.random.default_rng(0)
+    Z = E / np.linalg.norm(E, axis=1, keepdims=True)
+    s = Z[rng.choice(len(Z), min(200, len(Z)), replace=False)]
+    C = s @ s.T
+    mc = C[~np.eye(len(s), dtype=bool)].mean()
+    msg = f"[{tag}] finite={np.isfinite(E).all()}  mean pairwise cos(200)={mc:.4f}  std/dim={E.std(0).mean():.4f}"
+    if organ is not None:
+        Zc = Z - Z.mean(0)
+        Zc /= np.linalg.norm(Zc, axis=1, keepdims=True)
+        o = np.asarray(organ)
+        same, raw = [], []
+        for ZZ, acc in ((Zc, same), (Z, raw)):
+            Zt = torch.from_numpy(ZZ).to(dev)
+            for i in range(0, len(Zt), 2048):
+                S = Zt[i:i + 2048] @ Zt.T
+                S[torch.arange(S.shape[0]), torch.arange(i, i + S.shape[0])] = -2
+                nn = S.topk(10, dim=1).indices.cpu().numpy()
+                acc.append((o[nn] == o[i:i + S.shape[0], None]).mean(1))
+        cnt = pd.Series(o).value_counts()
+        base = ((cnt * (cnt - 1)).sum()) / (len(o) * (len(o) - 1))
+        msg += (f"  10-NN same-organ: {np.concatenate(raw).mean():.3f} (cos), {np.concatenate(same).mean():.3f}"
+                f" (centred cos)  base rate {base:.3f}")
+    log(msg)
+
+
+def main():
+    from transformers import Gemma2Model, PreTrainedTokenizerFast
+    from huggingface_hub import snapshot_download
+    sets = sys.argv[1:] or SETS
+    torch.cuda.set_per_process_memory_fraction(min(1.0, GPU_GB * 2**30 / torch.cuda.get_device_properties(0).total_memory))
+    tok = PreTrainedTokenizerFast.from_pretrained(TOK_DIR)
+    tok.padding_side = "right"
+    mdir = snapshot_download(REPO, revision=REV)
+    model, info = Gemma2Model.from_pretrained(mdir, dtype=torch.bfloat16, attn_implementation="eager",
+                                              output_loading_info=True)
+    log(f"loaded {REPO}@{REV}: missing={info['missing_keys']} unexpected={info['unexpected_keys']} "
+        f"mismatched={info['mismatched_keys']}")
+    assert not info["missing_keys"] and not info["mismatched_keys"] and not info["unexpected_keys"]
+    model.eval().to(dev)
+    log(f"weights {torch.cuda.memory_allocated() / 2**30:.2f} GB; prompt prefix tokens {len(tok(PREFIX)['input_ids'])}")
+    meta = pd.read_csv(os.path.join(CACHE, "hest", "meta.csv"))
+    tot_seq, tot_t = 0, 0.0
+    for name in sets:
+        t0 = time.time()
+        X, genes = load(name)
+        sents, n_expr = cell_sentences(X, genes)
+        prompts, seqs = tokenize(tok, sents)
+        ntok = np.array([len(s[0]) for s in seqs])
+        nsent = np.array([m.sum() for _, m in seqs])
+        upg = np.char.upper(genes.astype(str))
+        nonhgnc = np.mean([("." in g) or g.startswith("ENSG") for g in upg])
+        log(f"== {name}: {X.shape[0]} spots x {X.shape[1]} genes (all symbols usable as text; {nonhgnc:.1%} non-HGNC-"
+            f"style ids e.g. ENSG / RP11-..), {len(set(upg))} unique upper-case symbols; expressed genes/spot median "
+            f"{np.median(n_expr):.0f}, spots with < {K} expressed genes {np.mean(n_expr < K):.1%}; sentence genes/spot "
+            f"median {np.median(np.minimum(n_expr, K)):.0f}; tokens/prompt median {np.median(ntok):.0f} "
+            f"(sentence tokens {np.median(nsent):.0f}, {np.median(nsent) / K:.2f} tok/gene)")
+        log("  example: " + prompts[0][:400].replace("\n", " | ") + " ...")
+        with open(os.path.join(ROOT, "ext", "cell2sentence", f"prompts_sample_{name}.txt"), "w") as f:  # tokenizer cross-check
+            for i in np.random.default_rng(1).choice(len(prompts), min(200, len(prompts)), replace=False):
+                f.write(repr(prompts[i]) + "\n")
+        t1 = time.time()
+        E_mean, E_last = embed(model, seqs, tok.pad_token_id)
+        dt = time.time() - t1
+        tot_seq, tot_t = tot_seq + len(seqs), tot_t + dt
+        log(f"  forward {dt:.0f}s = {len(seqs) / dt:.2f} seq/s, {ntok.sum() / dt:.0f} tok/s; total {time.time() - t0:.0f}s;"
+            f" peak GPU {torch.cuda.max_memory_allocated() / 2**30:.2f} GB")
+        save("c2s", name, E_mean)
+        save("c2s-last", name, E_last)
+        organ = meta.organ.values if name in ("hest", "hest_v") else None
+        if organ is not None:
+            assert len(organ) == len(E_mean)
+        sanity(E_mean, f"c2s {name}", organ)
+        sanity(E_last, f"c2s-last {name}", organ)
+    log(f"overall throughput {tot_seq / max(tot_t, 1e-9):.2f} seq/s over {tot_seq} sequences ({tot_t / 60:.1f} min)")
+
+
+if __name__ == "__main__":
+    main()

experiments/43_rnafm_nicheformer.py

@@ -0,0 +1,232 @@
+"""Frozen Nicheformer spot embeddings of the 5 RNA sets (each Visium spot / virtual spot = one pseudo-cell).
+
+    python 43_rnafm_nicheformer.py [set ...]          (default: all of rna_sets.SETS)
+
+Model     : theislab/Nicheformer (HF, MIT), revision 0ba4ba162f9ad3d6c8b713761954c31be2fa69ed, file model.safetensors
+            (12-layer post-norm nn.TransformerEncoder, d=512, 16 heads, FFN 1024, learnable PE, context 1500).
+            Downloaded to ext/nicheformer/hf/; the HF remote code (modeling_/tokenization_nicheformer.py) was read but is
+            NOT executed: the forward pass is re-implemented below (same nn.TransformerEncoderLayer stack) and loaded
+            with a strict state-dict load (the unused template `encoder_layer.*`, the MLM head and the `pos` buffer
+            are the only checkpoint keys skipped, and that set is asserted).
+            Technology means: ext/nicheformer/repo/data/model_means/dissociated_mean_script.npy from the official
+            GitHub repo theislab/nicheformer (commit 485cadbc5caa15119adfd54228f8a8af835fcabc); gene order =
+            model.h5ad var (20,310 Ensembl ids, token = column index + 30, == vocab.json).
+Inputs    : rna_sets.load(set) raw(-like) counts + HGNC symbols. Symbols -> Ensembl via cache/external/
+            dlpfc_genes_full.csv (10x GRCh38 reference), fallback Geneformer gene_name_id_dict_gc104M.pkl; genes not in
+            the Nicheformer vocabulary are dropped (several symbols hitting one Ensembl id are summed), i.e. the spot is
+            aligned to model.h5ad like the official `ad.concat([model, adata])` step.
+Tokenising: official NicheformerDataset / tokenization notebooks: counts -> size-factor normalise to 1e4 (over the
+            aligned genes) -> divide by the technology mean (nan->0, rounded half-up, 0 -> 1, as in the notebooks) ->
+            rank the non-zero genes by descending value (stable sort; official uses numba quicksort, so only ties may
+            be ordered differently) -> token = gene index + 30, zero-padded to 1500. Context tokens prepended in the
+            official order [specie, assay, modality] and the sequence truncated to context_length 1500 (=> at most
+            1497 gene tokens). Pad 0 -> 1 and attention padding mask = (token == 1), as in models/_utils.complete_masking
+            with p = 0.
+Context   : Visium was not in the pretraining corpus (SpatialCorpus-110M = dissociated 10x/Smart-seq + MERFISH/CosMx/
+            Xenium/ISS) and has no technology token (the "visium": 9 entry in the get_embeddings notebook is a typo
+            for Xenium = 9 in every tokenisation notebook and the HF vocab). Chosen: specie = human (5),
+            modality = spatial (4), assay = 10x 3' v3 (11) -- Visium v1/v2 is a whole-transcriptome poly(A) 3'
+            sequencing assay, so the closest documented technology is 10x 3' v3, and its matching technology mean is
+            the dissociated (10x) one (the image-based means only cover their 300-1000-gene panels). A sensitivity
+            run with modality = dissociated (3) is logged for hest only (not saved).
+Readout   : official Nicheformer.get_embeddings(layer=-1, with_context=False): last-layer output, drop the 3 context
+            positions, mean over the remaining 1497 positions (this INCLUDES padding positions, exactly as the
+            official code; only spots with < 1497 detected vocabulary genes are affected and the fraction is logged).
+            D = 512, not L2-normalised. bf16 autocast inference.
+Output    : rna_sets.save("nicheformer", set, E) -> cache/rnafm/nicheformer_<set>.npy (n_spot, 512) float16.
+Sanity    : finite, mean pairwise cosine of 200 random spots, and (hest) organ agreement of cosine 10-NN vs base rate.
+"""
+import hashlib
+import json
+import os
+import pickle
+import sys
+import time
+
+import anndata as ad
+import numpy as np
+import pandas as pd
+import scipy.sparse as sp
+import torch
+import torch.nn as nn
+from safetensors.torch import load_file
+
+from common import CACHE, ROOT
+from rna_sets import SETS, load, save
+
+MODEL = "nicheformer"
+REPO, REV = "theislab/Nicheformer", "0ba4ba162f9ad3d6c8b713761954c31be2fa69ed"
+GH_COMMIT = "485cadbc5caa15119adfd54228f8a8af835fcabc"
+EXT = os.path.join(ROOT, "ext", "nicheformer")
+HF_DIR, GH_DIR = os.path.join(EXT, "hf"), os.path.join(EXT, "repo")
+PAD, AUX, CTX_LEN = 1, 30, 1500
+SPECIE_HUMAN, ASSAY_10X3V3, MOD_SPATIAL, MOD_DISSOC = 5, 11, 4, 3
+dev = "cuda"
+torch.manual_seed(0)
+
+
+def sha(p):
+    return hashlib.sha256(open(p, "rb").read()).hexdigest()[:16]
+
+
+def ensure_download():
+    if not os.path.exists(os.path.join(HF_DIR, "model.safetensors")):
+        from huggingface_hub import snapshot_download
+        snapshot_download(REPO, revision=REV, local_dir=HF_DIR)
+    assert os.path.exists(os.path.join(GH_DIR, "data", "model_means", "dissociated_mean_script.npy")), \
+        f"git clone https://github.com/theislab/nicheformer {GH_DIR} (commit {GH_COMMIT})"
+
+
+class Nicheformer(nn.Module):
+    """Re-implementation of NicheformerModel (HF modeling_nicheformer.py / official models/_nicheformer.py)."""
+
+    def __init__(self, cfg):
+        super().__init__()
+        self.embeddings = nn.Embedding(cfg["n_tokens"] + 5, cfg["dim_model"], padding_idx=1)
+        self.positional_embedding = nn.Embedding(cfg["context_length"], cfg["dim_model"])
+        layer = nn.TransformerEncoderLayer(cfg["dim_model"], cfg["nheads"], cfg["dim_feedforward"], dropout=0.0,
+                                           batch_first=True, layer_norm_eps=1e-12)
+        self.encoder = nn.TransformerEncoder(layer, cfg["nlayers"], enable_nested_tensor=False)
+
+    def get_embeddings(self, ids):
+        pad = ids.eq(PAD)
+        x = self.embeddings(ids) + self.positional_embedding(torch.arange(ids.shape[1], device=ids.device))
+        for blk in self.encoder.layers:
+            x = blk(x, src_key_padding_mask=pad, is_causal=False)
+        return x[:, 3:].float().mean(1)        # with_context=False, mean incl. pad positions (official)
+
+
+def load_model():
+    cfg = json.load(open(os.path.join(HF_DIR, "config.json")))
+    assert cfg["learnable_pe"] and cfg["specie"] and cfg["assay"] and cfg["modality"]
+    sd = load_file(os.path.join(HF_DIR, "model.safetensors"))
+    keep = {k[len("nicheformer."):]: v for k, v in sd.items()
+            if k.startswith("nicheformer.") and not k.startswith("nicheformer.encoder_layer.") and k != "nicheformer.pos"}
+    skipped = sorted(set(sd) - {"nicheformer." + k for k in keep})
+    assert all(k.startswith(("nicheformer.encoder_layer.", "classifier_head.")) or k == "nicheformer.pos"
+               for k in skipped), skipped
+    m = Nicheformer(cfg)
+    m.load_state_dict(keep, strict=True)
+    print(f"weights {sha(os.path.join(HF_DIR, 'model.safetensors'))}: loaded {len(keep)} tensors strictly; "
+          f"skipped {len(skipped)} (template encoder_layer / MLM head / pos buffer)", flush=True)
+    return m.eval().to(dev), cfg
+
+
+def gene_reference():
+    ref = ad.read_h5ad(os.path.join(HF_DIR, "model.h5ad"), backed=None)
+    ens = np.asarray(ref.var_names, dtype=str)
+    vocab = json.load(open(os.path.join(HF_DIR, "vocab.json")))
+    assert all(vocab[g] == i + AUX for i, g in enumerate(ens))
+    mean = np.load(os.path.join(GH_DIR, "data", "model_means", "dissociated_mean_script.npy"))
+    assert mean.shape == ens.shape
+    mean = np.nan_to_num(mean)
+    r = np.where((mean % 1) >= 0.5, np.ceil(mean), np.floor(mean))
+    mean = np.where(r == 0, 1, r)
+    sym2ens = dict(pd.read_csv(os.path.join(CACHE, "external", "dlpfc_genes_full.csv"))[["gene", "ensembl_id"]].values)
+    from huggingface_hub import hf_hub_download
+    gf = pickle.load(open(hf_hub_download("ctheodoris/Geneformer", "geneformer/gene_name_id_dict_gc104M.pkl"), "rb"))
+    for k, v in gf.items():
+        sym2ens.setdefault(k, v)
+    return ens, mean, sym2ens
+
+
+def tokenize(X, genes, ens, mean, sym2ens, modality):
+    e2i = {e: i for i, e in enumerate(ens)}
+    src = [(j, e2i[sym2ens[g]]) for j, g in enumerate(genes) if g in sym2ens and sym2ens[g] in e2i]
+    rows, cols = zip(*src)
+    M = sp.csr_matrix((np.ones(len(src), np.float32), (rows, cols)), shape=(len(genes), len(ens)))
+    A = (X @ M).tocsr()                                          # spot x model genes (aligned like concat with model)
+    lib = np.asarray(A.sum(1)).ravel()
+    cov = dict(n_in=len(genes), n_mapped=len(src), n_model_genes=int((np.asarray(M.sum(0)).ravel() > 0).sum()),
+               umi_frac=float(np.median(lib / np.maximum(np.asarray(X.sum(1)).ravel(), 1))))
+    lib = lib + (lib == 0)
+    A = (sp.diags(1e4 / lib) @ A).tocsr()
+    ids = np.full((A.shape[0], CTX_LEN), PAD, np.int64)
+    ids[:, :3] = [SPECIE_HUMAN, ASSAY_10X3V3, modality]
+    ngen = np.zeros(A.shape[0], int)
+    for r in range(A.shape[0]):
+        s, e = A.indptr[r], A.indptr[r + 1]
+        nz, v = A.indices[s:e], A.data[s:e] / mean[A.indices[s:e]]
+        keep = v > 0
+        nz, v = nz[keep], v[keep]
+        order = nz[np.argsort(-v, kind="stable")][:CTX_LEN - 3]
+        ids[r, 3:3 + len(order)] = order + AUX
+        ngen[r] = len(order)
+    cov["expr_genes_median"] = float(np.median(np.diff(A.indptr)))
+    cov["tokens_median"] = float(np.median(ngen))
+    cov["frac_spots_padded"] = float((ngen < CTX_LEN - 3).mean())
+    return ids, cov
+
+
+@torch.no_grad()
+def embed(model, ids, bs=32):
+    out = np.zeros((len(ids), 512), np.float32)
+    i = 0
+    while i < len(ids):
+        try:
+            b = torch.from_numpy(ids[i:i + bs]).to(dev)
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                out[i:i + bs] = model.get_embeddings(b).cpu().numpy()
+            i += bs
+        except torch.OutOfMemoryError:
+            torch.cuda.empty_cache()
+            bs = max(1, bs // 2)
+            print(f"  OOM -> batch {bs}", flush=True)
+    return out
+
+
+def sanity(E, name):
+    rng = np.random.default_rng(0)
+    S = E[rng.choice(len(E), min(200, len(E)), replace=False)]
+    S = S / np.linalg.norm(S, axis=1, keepdims=True)
+    C = S @ S.T
+    mc = float(C[np.triu_indices(len(S), 1)].mean())
+    msg = f"[{MODEL}] {name}: finite={np.isfinite(E).all()} std={E.std(0).mean():.4f} mean-pair-cos(200)={mc:.4f}"
+    if name == "hest":
+        org = pd.read_csv(os.path.join(CACHE, "hest", "meta.csv")).organ.values
+        assert len(org) == len(E)
+        Z = torch.from_numpy(E / np.linalg.norm(E, axis=1, keepdims=True)).to(dev)
+        nn_idx = []
+        for k in range(0, len(Z), 2048):
+            s = Z[k:k + 2048] @ Z.T
+            s[torch.arange(s.shape[0]), torch.arange(k, k + s.shape[0])] = -2
+            nn_idx.append(s.topk(10, dim=1).indices.cpu().numpy())
+        nn_idx = np.concatenate(nn_idx)
+        knn = float((org[nn_idx] == org[:, None]).mean())
+        p = pd.Series(org).value_counts(normalize=True).values
+        msg += f"  10-NN same-organ={knn:.3f} (base rate {float((p ** 2).sum()):.3f})"
+    print(msg, flush=True)
+    return mc
+
+
+def main():
+    sets = sys.argv[1:] or SETS
+    t0 = time.time()
+    ensure_download()
+    model, cfg = load_model()
+    ens, mean, sym2ens = gene_reference()
+    print(f"context tokens: specie=human({SPECIE_HUMAN}) assay=10x 3' v3({ASSAY_10X3V3}) modality=spatial({MOD_SPATIAL}); "
+          f"technology mean = dissociated_mean_script.npy (GitHub {GH_COMMIT[:7]})", flush=True)
+    for name in sets:
+        t1 = time.time()
+        X, genes = load(name)
+        ids, cov = tokenize(X, genes, ens, mean, sym2ens, MOD_SPATIAL)
+        print(f"[{MODEL}] {name}: {X.shape[0]} spots; genes in {cov['n_in']}, mapped to vocab {cov['n_mapped']} "
+              f"({cov['n_mapped'] / cov['n_in']:.1%}) -> {cov['n_model_genes']} model genes; median UMI fraction kept "
+              f"{cov['umi_frac']:.3f}; expressed vocab genes/spot median {cov['expr_genes_median']:.0f}; gene tokens "
+              f"median {cov['tokens_median']:.0f}; spots with padding {cov['frac_spots_padded']:.1%}", flush=True)
+        E = embed(model, ids)
+        sanity(E, name)
+        save(MODEL, name, E)
+        if name == "hest":                                 # sensitivity: modality token = dissociated
+            ids[:, 2] = MOD_DISSOC
+            E2 = embed(model, ids)
+            c = (E * E2).sum(1) / np.linalg.norm(E, axis=1) / np.linalg.norm(E2, axis=1)
+            print(f"  sensitivity modality=dissociated(3): cosine to saved per spot median {np.median(c):.4f}", flush=True)
+            sanity(E2, "hest")
+        print(f"[{MODEL}] {name}: {time.time() - t1:.0f}s", flush=True)
+    print(f"[{MODEL}] done {time.time() - t0:.0f}s; peak GPU {torch.cuda.max_memory_allocated() / 2**30:.2f} GB", flush=True)
+
+
+if __name__ == "__main__":
+    main()

experiments/44_rnafm_eval.py

@@ -0,0 +1,275 @@
+"""RNA encoder comparison + fusion under the final protocol (real pairs only; virtual data only at test time).
+
+RNA encoders (frozen inputs, cache/rnafm/<enc>_<set>.npy from 43_*.py; "hvg" = log1p CP10k of the 2000 HEST HVGs,
+identical to 42_train_regimes.py):
+  hvg, scgpt, geneformer, nicheformer, tahoex1, aidocell, se600m, c2s, c2s-last, genept-bge, genept-bmc (whatever exists)
+  cat:<a>+<b>+...  early fusion: concatenation of the z-scored inputs, each block scaled by 1/sqrt(its dim)
+  ens:<a>+<b>+...  late fusion: mean of the L2-normalised outputs of the separately trained single-encoder heads
+                   (possible because every head maps into the same frozen UNI2 space)
+Head: linear or MLP (42_'s mlp) from the encoder input to the frozen UNI2 space, InfoNCE, AdamW 3e-4 / wd 0.05 /
+OneCycle, batch 256, 2700 steps, early stopping on val R@1 every 50 steps (= 42_ S1/S4 RNA recipe).
+FM inputs are z-scored with the HEST TRAINING-spot mean / std of fold k (applied unchanged to virtual / external sets).
+Per fold k (splits/final/hest_fold<k>.csv, ORION splits/patient_5fold, TCGA splits/final/tcga_fold<k>.csv):
+  trained   H&E<->RNA            HEST test spots (real)
+  emergent  mIF<->RNA(virtual)   ORION test tiles, mIF = S4 mIF MLP head of fold k (trained here once, cached)
+  emergent  text<->RNA(virtual)  TCGA test images, text = S4 linear text head of fold k (clean real text)
+  calib     H&E<->RNA(virtual)   HEST test spots with STPath RNA instead of the real one (circular, reference only)
+--dlpfc: additionally train on HEST WITHOUT the spatialLIBD study (val = 10% of groups) and test H&E<->RNA on the
+         external DLPFC set (482 spots; FM inputs z-scored per dataset as in 38_), mixed galleries G=100.
+Usage: python 44_rnafm_eval.py [--encs ...] [--heads linear mlp] [--folds 0 1 2 3 4] [--dlpfc]
+Output results/rnafm_long.csv (resumable), results/rnafm_summary.csv
+"""
+import argparse
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, SPLITS
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--encs", nargs="+", default=None)
+ap.add_argument("--heads", nargs="+", default=["linear", "mlp"])
+ap.add_argument("--folds", nargs="+", type=int, default=[0, 1, 2, 3, 4])
+ap.add_argument("--dlpfc", action="store_true")
+args = ap.parse_args()
+dev = "cuda"
+H, RF = os.path.join(CACHE, "hest"), os.path.join(CACHE, "rnafm")
+G100, DRAWS, BS, LR, EVAL_EVERY, STEPS = 100, 50, 256, 3e-4, 50, 2700
+f32 = lambda a: torch.from_numpy(np.asarray(a, dtype=np.float32))
+SETS = ["hest", "hest_v", "orion_v", "tcga_v", "dlpfc"]
+
+# ---------------------------------------------------------------- anchors / other modalities (= 42_)
+HE_O = f32(np.load(os.path.join(CACHE, "he_uni2.npy"))).to(dev)
+IF_O = f32(np.load(os.path.join(CACHE, "if_virtues_m16.npy"))).to(dev)
+HE_H = f32(np.load(os.path.join(H, "he_uni2.npy"))[:, 0]).to(dev)
+HE_T = f32(np.load(os.path.join(CACHE, "allimg_he_uni2.npy"))).to(dev)
+HE_D = f32(np.load(os.path.join(CACHE, "external", "dlpfc_he_uni2.npy"))).to(dev)
+HE_D = HE_D[:, 0] if HE_D.dim() == 3 else HE_D
+TXT = f32(np.load(os.path.join(CACHE, "alltxt_text_biomedclip.npy"))[:, 0]).to(dev)
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+T_IMG = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+HM = pd.read_csv(os.path.join(H, "meta.csv"))
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def hvg_inputs():
+    def virt(st):
+        g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+        c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+        c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+        return c[:, pd.Series(range(len(g)), index=g).loc[hvg].values]
+    eg = pd.read_csv(os.path.join(CACHE, "external", "dlpfc_genes_full.csv")).gene.values
+    ec = np.load(os.path.join(CACHE, "external", "dlpfc_counts_full.npy")).astype(np.float32)
+    ei = pd.Series(range(len(eg)), index=eg); ei = ei[~ei.index.duplicated()]
+    d = np.log1p(ec / ec.sum(1, keepdims=True).clip(1) * 1e4)
+    d = np.stack([d[:, ei[g]] if g in ei.index else np.zeros(len(d), np.float32) for g in hvg], 1)
+    return {"hest": np.load(os.path.join(H, "hvg.npy")), "hest_v": virt("hest_val"), "orion_v": virt("orion"),
+            "tcga_v": virt("tcga_text"), "dlpfc": d}
+
+
+def load_enc(e):
+    e = e.split("@")[0]  # "<enc>@<n>": same input, head trained with another seed (seed-ensemble control)
+    if e == "hvg":
+        return {s: f32(x).to(dev) for s, x in hvg_inputs().items()}
+    return {s: f32(np.load(os.path.join(RF, f"{e}_{s}.npy"))).to(dev) for s in SETS}
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_ / 42_
+    a, b = a.cpu(), b.cpu()
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float((((s > s.diag()[:, None]).sum(1) == 0).float().mean() + ((s.T > s.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+
+
+def train_head(X_tr, A_tr, X_va, A_va, kind, steps, seed, pos_tr=None):
+    """modality head X -> frozen anchor space A (rows paired). A_* may be (n, V, 1536) -> random view per sample."""
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    h = (mlp(X_tr.shape[1]) if kind == "mlp" else nn.Linear(X_tr.shape[1], 1536)).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(h.parameters()) + [logit], lr=LR, weight_decay=0.05)
+    sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=LR, total_steps=steps, pct_start=0.05)
+    za_va = F.normalize(A_va[:, 0] if A_va.dim() == 3 else A_va, dim=-1)
+    best, state = -1, None
+    for step in range(1, steps + 1):
+        h.train()
+        b = torch.as_tensor(rng.choice(len(X_tr), BS, replace=False), device=dev)
+        a = A_tr[b, torch.randint(0, A_tr.shape[1], (BS,), device=dev)] if A_tr.dim() == 3 else A_tr[b]
+        lg = logit.exp().clamp(max=100) * F.normalize(a, dim=-1) @ F.normalize(h(X_tr[b]), dim=-1).T
+        if pos_tr is None:
+            t = torch.arange(BS, device=dev)
+            loss = (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+        else:
+            ii = pos_tr[b]; pos = (ii[:, None] == ii[None, :]).float(); tgt = pos / pos.sum(1, keepdim=True)
+            loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+        opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+        if step % EVAL_EVERY == 0 or step == steps:
+            h.eval()
+            with torch.no_grad():
+                v = r1(za_va, F.normalize(h(X_va), dim=-1))
+            if v > best:
+                best, state = v, {k_: x.detach().clone() for k_, x in h.state_dict().items()}
+    h.load_state_dict(state); h.eval()
+    return h, best
+
+
+class Fold:
+    def __init__(self, k):
+        s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+        self.o = {x: s.tile_id[s.split == x].values for x in ["train", "val", "test"]}
+        mu, sd = IF_O[self.o["train"]].mean((0, 1)), IF_O[self.o["train"]].std((0, 1)) + 1e-3
+        self.IF = (IF_O - mu) / sd
+        h = pd.read_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"))
+        self.h = {x: h.spot_idx[h.split == x].values for x in ["train", "val", "test"]}
+        sp = pd.read_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+        tsp = sp.loc[T.img_idx].values
+        self.t = {x: T.index[tsp == x].values for x in ["train", "val", "test"]}
+        self.t_va = one_text_per_image(self.t["val"], np.random.default_rng(0))
+        self.t_te = one_text_per_image(self.t["test"], np.random.default_rng(k))
+        # DLPFC-held-out training pool (= 38_)
+        pool = np.flatnonzero(~HM.group.str.startswith("spatialLIBD").values)
+        grp = np.random.default_rng(k).permutation(sorted(HM.group.iloc[pool].unique()))
+        isva = HM.group.iloc[pool].isin(set(grp[: max(1, len(grp) // 10)])).values
+        self.d = {"train": pool[~isva], "val": pool[isva]}
+
+    def other_heads(self, k):
+        """S4 heads of fold k for mIF (MLP) and text (linear); cached in runs/rnafm/other_fold<k>.pt"""
+        f = os.path.join(ROOT, "runs", "rnafm", f"other_fold{k}.pt")
+        hm, ht = mlp(512).to(dev), nn.Linear(512, 1536).to(dev)
+        if os.path.exists(f):
+            sd = torch.load(f, map_location=dev); hm.load_state_dict(sd["mif"]); ht.load_state_dict(sd["text"])
+            return hm.eval(), ht.eval()
+        o, t = self.o, self.t
+        hm, _ = train_head(self.IF[o["train"]].flatten(0, 1), HE_O[o["train"]].flatten(0, 1),
+                           self.IF[o["val"], 0], HE_O[o["val"], 0], "mlp", 400, k)
+        ii, tt = self.t_va
+        ht, _ = train_head(TXT[t["train"]], HE_T[T_IMG[torch.as_tensor(t["train"], device=dev)]],
+                           TXT[torch.as_tensor(tt, device=dev)], HE_T[torch.as_tensor(ii, device=dev), 0],
+                           "linear", 1000, k, pos_tr=T_IMG[torch.as_tensor(t["train"], device=dev)])
+        os.makedirs(os.path.dirname(f), exist_ok=True)
+        torch.save({"mif": hm.state_dict(), "text": ht.state_dict()}, f)
+        return hm, ht
+
+
+def prep(X, tr, per_dataset_ext=False):
+    """z-score with HEST-train stats (hvg: unchanged, = 42_). Returns dict set -> tensor."""
+    if X is None:
+        return None
+    mu, sd = X["hest"][tr].mean(0), X["hest"][tr].std(0) + 1e-6
+    out = {s: (x - mu) / sd for s, x in X.items()}
+    if per_dataset_ext:
+        out["dlpfc"] = (X["dlpfc"] - X["dlpfc"].mean(0)) / (X["dlpfc"].std(0) + 1e-6)
+    return out
+
+
+def build(enc, fold_tr, dl_tr=None):
+    """input tensors per set for encoder spec enc; hvg kept raw (42_), FMs / fusions z-scored."""
+    parts = enc.split(":", 1)[1].split("+") if enc.startswith("cat:") else [enc]
+    main, ext = [], []
+    for p in parts:
+        X = CACHE_ENC[p]
+        if p.split("@")[0] == "hvg" and len(parts) == 1:
+            main.append(X)
+            if dl_tr is not None:
+                ext.append(prep(X, dl_tr, True))
+            continue
+        w = 1 / np.sqrt(X["hest"].shape[1]) if len(parts) > 1 else 1.0
+        main.append({s: v * w for s, v in prep(X, fold_tr).items()})
+        if dl_tr is not None:
+            ext.append({s: v * w for s, v in prep(X, dl_tr, True).items()})
+    cat = lambda L: {s: torch.cat([d[s] for d in L], 1) for s in SETS}
+    return cat(main), (cat(ext) if dl_tr is not None else None)
+
+
+# ---------------------------------------------------------------- run
+avail = ["hvg"] + sorted({f.rsplit("_", 1)[0] for f in os.listdir(RF) if f.endswith("_hest.npy")
+                          and all(os.path.exists(os.path.join(RF, f.replace("_hest.npy", f"_{s}.npy"))) for s in SETS)})
+encs = sorted(args.encs or avail, key=lambda e: e.startswith("ens:"))  # late fusions after their single heads
+singles = sorted({p for e in encs for p in (e.split(":", 1)[1].split("+") if ":" in e else [e])})
+CACHE_ENC = {e: load_enc(e) for e in singles}
+print("encoders:", {e: CACHE_ENC[e]["hest"].shape[1] for e in singles}, flush=True)
+f_out = os.path.join(ROOT, "results", "rnafm_long.csv")
+rows = pd.read_csv(f_out).to_dict("records") if os.path.exists(f_out) else []
+done = {(r["enc"], r["head"], r["fold"]) for r in rows}
+nz = lambda x: F.normalize(x, dim=-1)
+for k in args.folds:
+    F_ = Fold(k)
+    hm, ht = F_.other_heads(k)
+    te_o = F_.o["test"]; ii, tt = (torch.as_tensor(x, device=dev) for x in F_.t_te)
+    with torch.no_grad():
+        z_mif = nz(hm(F_.IF[te_o, 0])); z_txt = nz(ht(TXT[tt]))
+    heads_out = {}  # (enc, head) -> dict of output embeddings, for late fusion
+    for enc in encs:
+        for hk in args.heads:
+            parts = enc.split(":", 1)[1].split("+") if enc.startswith("ens:") else None
+            need = any(enc in e.split(":", 1)[1].split("+") for e in encs if e.startswith("ens:"))
+            if (enc, hk, k) in done and parts is None and not need:
+                continue
+            tr, va, te = F_.h["train"], F_.h["val"], F_.h["test"]
+            if parts is not None:  # late fusion of single-encoder heads trained in this run
+                if not all((p, hk) in heads_out for p in parts) or (enc, hk, k) in done:
+                    continue
+                Z = {s: nz(sum(heads_out[(p, hk)][s] for p in parts)) for s in heads_out[(parts[0], hk)]}
+                best = float("nan")
+            else:
+                X, Xd = build(enc, tr, F_.d["train"] if args.dlpfc else None)
+                sd_ = k + 1000 * int(enc.split("@")[1]) if "@" in enc else k
+                h, best = train_head(X["hest"][tr], HE_H[tr], X["hest"][va], HE_H[va], hk, STEPS, sd_)
+                with torch.no_grad():
+                    Z = {"hest": nz(h(X["hest"][te])), "hest_v": nz(h(X["hest_v"][te])),
+                         "orion_v": nz(h(X["orion_v"][te_o])), "tcga_v": nz(h(X["tcga_v"][ii]))}
+                if args.dlpfc:
+                    hd, _ = train_head(Xd["hest"][F_.d["train"]], HE_H[F_.d["train"]], Xd["hest"][F_.d["val"]],
+                                       HE_H[F_.d["val"]], hk, STEPS, sd_)
+                    with torch.no_grad():
+                        Z["dlpfc"] = nz(hd(Xd["dlpfc"]))
+                heads_out[(enc, hk)] = Z
+            res = {("H&E<->RNA (HEST)", "trained"): g100(nz(HE_H[te]), Z["hest"]),
+                   ("mIF<->RNA(virtual) (ORION)", "emergent"): g100(z_mif, Z["orion_v"]),
+                   ("text<->RNA(virtual) (TCGA)", "emergent"): g100(z_txt, Z["tcga_v"]),
+                   ("H&E<->RNA(virtual) (HEST)", "calib"): g100(nz(HE_H[te]), Z["hest_v"])}
+            if "dlpfc" in Z:
+                res[("H&E<->RNA (DLPFC external)", "external")] = g100(nz(HE_D), Z["dlpfc"])
+            if (enc, hk, k) in done:  # retrained only to feed a late fusion
+                continue
+            for (pair, kind), r in res.items():
+                rows.append({"enc": enc, "head": hk, "fold": k, "pair": pair, "kind": kind, "val_R@1": best, **r})
+            pd.DataFrame(rows).to_csv(f_out, index=False)
+            print(f"fold {k} {enc:<34} {hk:<6} val {best:.3f}  " +
+                  "  ".join(f"{p.split(' (')[0]} {r['R@1']:.3f}" for (p, _), r in res.items()), flush=True)
+
+R = pd.DataFrame(rows)
+S = R.groupby(["enc", "head", "kind", "pair"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "rnafm_summary.csv"))
+pd.set_option("display.width", 250)
+print(R.pivot_table(index=["enc", "head"], columns="pair", values="R@1", sort=False).round(3).to_string())

experiments/46_text_enc_eval.py

@@ -0,0 +1,231 @@
+"""Text encoder comparison + fusion under the final protocol (clean real TCGA texts only; synthetic / virtual = test only).
+
+Text encoders (frozen, cache/alltxt_text_<enc>.npy and cache/orion_gentext_text_<enc>.npy; 45_ / 45b_ / 08_):
+  biomedclip (current; raw features as in 42_), conchclip, plip, medcpt, biolord, pubmedbert, medembed, bge, qwen3-0.6b,
+  qwen3-4b, keep, quiltnet (whatever exists for both sets). Non-baseline inputs are z-scored with TCGA-train stats.
+  cat:<a>+<b>  early fusion (z-scored blocks scaled by 1/sqrt(dim), one head)
+  ens:<a>+<b>  late fusion (mean of the L2-normalised outputs of the single-encoder heads; same frozen UNI2 space)
+Head: linear or MLP into the frozen UNI2 space, multi-positive InfoNCE (several texts per image), AdamW 3e-4 / wd 0.05 /
+OneCycle, batch 256, 1000 steps, early stopping on val R@1 every 50 steps (= 42_ S4 text recipe).
+Per fold k (splits/final/tcga_fold<k>.csv; one random text per test image, = 33_/42_):
+  trained   H&E<->text                 TCGA test images (real text)
+  emergent  mIF<->text(synthetic)      ORION patient_5fold test tiles, mIF = S4 MLP head (runs/rnafm/other_fold<k>.pt, 44_)
+  emergent  text<->RNA(virtual)        TCGA test images, RNA = hvg linear head of fold k (runs/rnafm/rna_hvg_fold<k>.pt)
+  emergent  text<->mIF(MIPHEI)         TCGA test images, MIPHEI virtual mIF through the same mIF head
+Usage: python 46_text_enc_eval.py [--encs ...] [--heads linear mlp] [--folds 0 1 2 3 4]
+Output results/textenc_long.csv (resumable), results/textenc_summary.csv
+"""
+import argparse
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, SPLITS
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--encs", nargs="+", default=None)
+ap.add_argument("--heads", nargs="+", default=["linear", "mlp"])
+ap.add_argument("--folds", nargs="+", type=int, default=[0, 1, 2, 3, 4])
+args = ap.parse_args()
+dev = "cuda"
+H = os.path.join(CACHE, "hest")
+G100, DRAWS, BS, LR, EVAL_EVERY, STEPS = 100, 50, 256, 3e-4, 50, 1000
+f32 = lambda a: torch.from_numpy(np.asarray(a, dtype=np.float32))
+nz = lambda x: F.normalize(x, dim=-1)
+
+HE_O = f32(np.load(os.path.join(CACHE, "he_uni2.npy"))).to(dev)
+IF_O = f32(np.load(os.path.join(CACHE, "if_virtues_m16.npy"))).to(dev)
+IF_V_T = f32(np.load(os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy"))[:, 0]).to(dev)
+HE_H = f32(np.load(os.path.join(H, "he_uni2.npy"))[:, 0]).to(dev)
+RNA_H = f32(np.load(os.path.join(H, "hvg.npy"))).to(dev)
+HE_T = f32(np.load(os.path.join(CACHE, "allimg_he_uni2.npy"))).to(dev)
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+T_IMG = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def virtual_rna(st):  # = 42_
+    g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+    c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+    c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+    return f32(c[:, pd.Series(range(len(g)), index=g).loc[hvg].values]).to(dev)
+
+
+RNA_V_T = virtual_rna("tcga_text")
+
+
+def load_enc(e):
+    e = e.split("@")[0]  # "<enc>@<n>": same input, head trained with another seed (seed-ensemble control)
+    return {s: f32(np.load(os.path.join(CACHE, f"{s}_text_{e}.npy"))[:, 0]).to(dev) for s in ["alltxt", "orion_gentext"]}
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_ / 42_
+    a, b = a.cpu(), b.cpu()
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float((((s > s.diag()[:, None]).sum(1) == 0).float().mean() + ((s.T > s.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+
+
+def train_head(X_tr, A_tr, X_va, A_va, kind, steps, seed, pos_tr=None):  # = 44_
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    h = (mlp(X_tr.shape[1]) if kind == "mlp" else nn.Linear(X_tr.shape[1], 1536)).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(h.parameters()) + [logit], lr=LR, weight_decay=0.05)
+    sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=LR, total_steps=steps, pct_start=0.05)
+    za_va = nz(A_va[:, 0] if A_va.dim() == 3 else A_va)
+    best, state = -1, None
+    for step in range(1, steps + 1):
+        h.train()
+        b = torch.as_tensor(rng.choice(len(X_tr), BS, replace=False), device=dev)
+        a = A_tr[b, torch.randint(0, A_tr.shape[1], (BS,), device=dev)] if A_tr.dim() == 3 else A_tr[b]
+        lg = logit.exp().clamp(max=100) * nz(a) @ nz(h(X_tr[b])).T
+        if pos_tr is None:
+            t = torch.arange(BS, device=dev)
+            loss = (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+        else:
+            ii = pos_tr[b]; pos = (ii[:, None] == ii[None, :]).float(); tgt = pos / pos.sum(1, keepdim=True)
+            loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+        opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+        if step % EVAL_EVERY == 0 or step == steps:
+            h.eval()
+            with torch.no_grad():
+                v = r1(za_va, nz(h(X_va)))
+            if v > best:
+                best, state = v, {k_: x.detach().clone() for k_, x in h.state_dict().items()}
+    h.load_state_dict(state); h.eval()
+    return h, best
+
+
+class Fold:
+    def __init__(self, k):
+        s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+        self.o = {x: s.tile_id[s.split == x].values for x in ["train", "val", "test"]}
+        self.if_mu, self.if_sd = IF_O[self.o["train"]].mean((0, 1)), IF_O[self.o["train"]].std((0, 1)) + 1e-3
+        h = pd.read_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"))
+        self.h = {x: h.spot_idx[h.split == x].values for x in ["train", "val", "test"]}
+        sp = pd.read_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+        tsp = sp.loc[T.img_idx].values
+        self.t = {x: T.index[tsp == x].values for x in ["train", "val", "test"]}
+        self.t_va = one_text_per_image(self.t["val"], np.random.default_rng(0))
+        self.t_te = one_text_per_image(self.t["test"], np.random.default_rng(k))
+
+    def nif(self, x):
+        return (x - self.if_mu) / self.if_sd
+
+    def mif_head(self, k):
+        f = os.path.join(ROOT, "runs", "rnafm", f"other_fold{k}.pt")  # written by 44_rnafm_eval.py
+        hm = mlp(512).to(dev)
+        hm.load_state_dict(torch.load(f, map_location=dev)["mif"])
+        return hm.eval()
+
+    def rna_head(self, k):
+        f = os.path.join(ROOT, "runs", "rnafm", f"rna_hvg_fold{k}.pt")
+        hr = nn.Linear(2000, 1536).to(dev)
+        if os.path.exists(f):
+            hr.load_state_dict(torch.load(f, map_location=dev))
+            return hr.eval()
+        hr, _ = train_head(RNA_H[self.h["train"]], HE_H[self.h["train"]], RNA_H[self.h["val"]], HE_H[self.h["val"]],
+                           "linear", 2700, k)
+        torch.save(hr.state_dict(), f)
+        return hr
+
+
+def build(enc, tr):
+    parts = enc.split(":", 1)[1].split("+") if enc.startswith("cat:") else [enc]
+    blocks = []
+    for p in parts:
+        X = CACHE_ENC[p]
+        if p.split("@")[0] == "biomedclip" and len(parts) == 1:
+            blocks.append(X); continue  # = 42_ / 33_ (raw BiomedCLIP features)
+        mu, sd = X["alltxt"][tr].mean(0), X["alltxt"][tr].std(0) + 1e-6
+        w = 1 / np.sqrt(X["alltxt"].shape[1]) if len(parts) > 1 else 1.0
+        blocks.append({s: (v - mu) / sd * w for s, v in X.items()})
+    return {s: torch.cat([b[s] for b in blocks], 1) for s in blocks[0]}
+
+
+avail = sorted({f[len("alltxt_text_"):-4] for f in os.listdir(CACHE) if f.startswith("alltxt_text_") and f.endswith(".npy")
+                and os.path.exists(os.path.join(CACHE, "orion_gentext_text_" + f[len("alltxt_text_"):]))})
+encs = sorted(args.encs or avail, key=lambda e: e.startswith("ens:"))  # late fusions after their single heads
+singles = sorted({p for e in encs for p in (e.split(":", 1)[1].split("+") if ":" in e else [e])})
+CACHE_ENC = {e: load_enc(e) for e in singles}
+print("encoders:", {e: CACHE_ENC[e]["alltxt"].shape[1] for e in singles}, flush=True)
+f_out = os.path.join(ROOT, "results", "textenc_long.csv")
+rows = pd.read_csv(f_out).to_dict("records") if os.path.exists(f_out) else []
+done = {(r["enc"], r["head"], r["fold"]) for r in rows}
+for k in args.folds:
+    F_ = Fold(k)
+    hm, hr = F_.mif_head(k), F_.rna_head(k)
+    te_o = F_.o["test"]; ii, tt = (torch.as_tensor(x, device=dev) for x in F_.t_te)
+    vi, vt = (torch.as_tensor(x, device=dev) for x in F_.t_va)
+    trt = torch.as_tensor(F_.t["train"], device=dev)
+    with torch.no_grad():
+        z_mif = nz(hm(F_.nif(IF_O[te_o, 0]))); z_rna = nz(hr(RNA_V_T[ii])); z_vmif = nz(hm(F_.nif(IF_V_T[ii])))
+        z_he = nz(HE_T[ii, 0])
+    heads_out = {}
+    for enc in encs:
+        for hk in args.heads:
+            parts = enc.split(":", 1)[1].split("+") if enc.startswith("ens:") else None
+            need = any(enc in e.split(":", 1)[1].split("+") for e in encs if e.startswith("ens:"))
+            if (enc, hk, k) in done and not (parts is None and need):
+                continue
+            if parts is not None:
+                if not all((p, hk) in heads_out for p in parts):
+                    continue
+                Z = {s: nz(sum(heads_out[(p, hk)][s] for p in parts)) for s in heads_out[(parts[0], hk)]}
+                best = float("nan")
+            else:
+                X = build(enc, trt)
+                sd_ = k + 1000 * int(enc.split("@")[1]) if "@" in enc else k
+                h, best = train_head(X["alltxt"][trt], HE_T[T_IMG[trt]], X["alltxt"][vt], HE_T[vi, 0], hk, STEPS, sd_,
+                                     pos_tr=T_IMG[trt])
+                with torch.no_grad():
+                    Z = {"tcga": nz(h(X["alltxt"][tt])), "orion": nz(h(X["orion_gentext"][te_o]))}
+                heads_out[(enc, hk)] = Z
+            if (enc, hk, k) in done:
+                continue
+            res = {("H&E<->text (TCGA)", "trained"): g100(z_he, Z["tcga"]),
+                   ("mIF<->text(synthetic) (ORION)", "emergent"): g100(z_mif, Z["orion"]),
+                   ("text<->RNA(virtual) (TCGA)", "emergent"): g100(Z["tcga"], z_rna),
+                   ("text<->mIF(MIPHEI) (TCGA)", "emergent"): g100(Z["tcga"], z_vmif)}
+            for (pair, kind), r in res.items():
+                rows.append({"enc": enc, "head": hk, "fold": k, "pair": pair, "kind": kind, "val_R@1": best, **r})
+            pd.DataFrame(rows).to_csv(f_out, index=False)
+            print(f"fold {k} {enc:<34} {hk:<6} val {best:.3f}  " +
+                  "  ".join(f"{p.split(' (')[0]} {r['R@1']:.3f}" for (p, _), r in res.items()), flush=True)
+
+R = pd.DataFrame(rows)
+S = R.groupby(["enc", "head", "kind", "pair"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "textenc_summary.csv"))
+pd.set_option("display.width", 250)
+print(R.pivot_table(index=["enc", "head"], columns="pair", values="R@1", sort=False).round(3).to_string())

experiments/47_final_ens.py

@@ -0,0 +1,227 @@
+"""Final four-modality evaluation with multi-encoder late fusion (frozen UNI2 anchor; real pairs only; virtual /
+synthetic data only at test time). Combines the winners of 44_rnafm_eval.py (RNA) and 46_text_enc_eval.py (text).
+
+Every head maps one frozen encoder into the frozen UNI2 space (S4 recipe of 42_: MLP for mIF, linear for RNA / text), so
+the heads of several encoders of one modality can be averaged at test time:  z_mod = normalise(sum_e normalise(head_e(x_e))).
+Configurations (mIF = VirTues 16-ch + MLP head in all of them):
+  S4        RNA hvg                                   text BiomedCLIP                  (current method)
+  single    RNA AIDO.Cell                             text KEEP                        (best single encoders)
+  ens-small RNA hvg + Tahoe-x1 + SE-600M + AIDO.Cell  text KEEP + BiomedCLIP + Qwen3-Emb-0.6B
+  ens-large RNA all 10 encoders                       text KEEP + BiomedCLIP + Qwen3-0.6B + PubMedBERT-emb + MedCPT
+Pairs per fold k (same splits / test items as 42_ / 44_ / 46_): trained H&E<->mIF (ORION), H&E<->RNA (HEST),
+H&E<->text (TCGA); emergent mIF<->RNA(v), mIF<->text(s) (ORION), text<->RNA(v), text<->mIF(MIPHEI) (TCGA);
+external DLPFC (RNA heads retrained without spatialLIBD, inputs z-scored per dataset): H&E<->RNA, RNA<->text(s),
+RNA<->mIF(MIPHEI).
+Output results/final_ens_long.csv, results/final_ens_summary.csv, results/final_ens_tests.csv (paired vs S4)
+"""
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy import stats
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, SPLITS
+
+dev = "cuda"
+H, RF = os.path.join(CACHE, "hest"), os.path.join(CACHE, "rnafm")
+G100, DRAWS, BS, LR, EVAL_EVERY = 100, 50, 256, 3e-4, 50
+f32 = lambda a: torch.from_numpy(np.asarray(a, dtype=np.float32))
+nz = lambda x: F.normalize(x, dim=-1)
+RNA_ALL = ["hvg", "tahoex1", "se600m", "scgpt", "nicheformer", "aidocell", "geneformer", "c2s", "genept-bge", "genept-bmc"]
+CONF = {"S4": (["hvg"], ["biomedclip"]),
+        "single": (["aidocell"], ["keep"]),
+        "ens-small": (["hvg", "tahoex1", "se600m", "aidocell"], ["keep", "biomedclip", "qwen3-0.6b"]),
+        "ens-large": (RNA_ALL, ["keep", "biomedclip", "qwen3-0.6b", "pubmedbert", "medcpt"])}
+
+# ---------------------------------------------------------------- data
+HE_O = f32(np.load(os.path.join(CACHE, "he_uni2.npy"))).to(dev)
+IF_O = f32(np.load(os.path.join(CACHE, "if_virtues_m16.npy"))).to(dev)
+IF_V_T = f32(np.load(os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy"))[:, 0]).to(dev)
+IF_V_D = f32(np.load(os.path.join(CACHE, "if_virtues_m16_virtual_miphei_dlpfc.npy"))[:, 0]).to(dev)
+HE_H = f32(np.load(os.path.join(H, "he_uni2.npy"))[:, 0]).to(dev)
+HE_T = f32(np.load(os.path.join(CACHE, "allimg_he_uni2.npy"))).to(dev)
+HE_D = f32(np.load(os.path.join(CACHE, "external", "dlpfc_he_uni2.npy"))).to(dev)
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+T_IMG = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+HM = pd.read_csv(os.path.join(H, "meta.csv"))
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def hvg_inputs():  # = 44_
+    def virt(st):
+        g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+        c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+        c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+        return c[:, pd.Series(range(len(g)), index=g).loc[hvg].values]
+    eg = pd.read_csv(os.path.join(CACHE, "external", "dlpfc_genes_full.csv")).gene.values
+    ec = np.load(os.path.join(CACHE, "external", "dlpfc_counts_full.npy")).astype(np.float32)
+    ei = pd.Series(range(len(eg)), index=eg); ei = ei[~ei.index.duplicated()]
+    d = np.log1p(ec / ec.sum(1, keepdims=True).clip(1) * 1e4)
+    d = np.stack([d[:, ei[g]] if g in ei.index else np.zeros(len(d), np.float32) for g in hvg], 1)
+    return {"hest": np.load(os.path.join(H, "hvg.npy")), "orion_v": virt("orion"), "tcga_v": virt("tcga_text"), "dlpfc": d}
+
+
+RNA_IN = {e: ({s: f32(x).to(dev) for s, x in hvg_inputs().items()} if e == "hvg" else
+              {s: f32(np.load(os.path.join(RF, f"{e}_{s}.npy"))).to(dev) for s in ["hest", "orion_v", "tcga_v", "dlpfc"]})
+          for e in RNA_ALL}
+TXT_ENCS = sorted({e for _, x in CONF.values() for e in x})
+TXT_IN = {e: {s: f32(np.load(os.path.join(CACHE, f"{s}_text_{e}.npy"))[:, 0]).to(dev)
+              for s in ["alltxt", "orion_gentext", "dlpfc_gentext"]} for e in TXT_ENCS}
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_ / 42_
+    a, b = a.cpu(), b.cpu()
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float((((s > s.diag()[:, None]).sum(1) == 0).float().mean() + ((s.T > s.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+
+
+def train_head(X_tr, A_tr, X_va, A_va, kind, steps, seed, pos_tr=None):  # = 44_ / 46_
+    torch.manual_seed(seed); rng = np.random.default_rng(seed)
+    h = (mlp(X_tr.shape[1]) if kind == "mlp" else nn.Linear(X_tr.shape[1], 1536)).to(dev)
+    logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+    opt = torch.optim.AdamW(list(h.parameters()) + [logit], lr=LR, weight_decay=0.05)
+    sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=LR, total_steps=steps, pct_start=0.05)
+    za_va = nz(A_va[:, 0] if A_va.dim() == 3 else A_va)
+    best, state = -1, None
+    for step in range(1, steps + 1):
+        h.train()
+        b = torch.as_tensor(rng.choice(len(X_tr), BS, replace=False), device=dev)
+        a = A_tr[b, torch.randint(0, A_tr.shape[1], (BS,), device=dev)] if A_tr.dim() == 3 else A_tr[b]
+        lg = logit.exp().clamp(max=100) * nz(a) @ nz(h(X_tr[b])).T
+        if pos_tr is None:
+            t = torch.arange(BS, device=dev)
+            loss = (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+        else:
+            ii = pos_tr[b]; pos = (ii[:, None] == ii[None, :]).float(); tgt = pos / pos.sum(1, keepdim=True)
+            loss = (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+        opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+        if step % EVAL_EVERY == 0 or step == steps:
+            h.eval()
+            with torch.no_grad():
+                v = r1(za_va, nz(h(X_va)))
+            if v > best:
+                best, state = v, {k_: x.detach().clone() for k_, x in h.state_dict().items()}
+    h.load_state_dict(state); h.eval()
+    return h
+
+
+def zs(X, ref_set, idx, raw, ext=None):
+    """z-score every set with the training rows of ref_set (raw = keep the baseline input unchanged, as in 42_);
+    ext: sets z-scored on themselves (external data, = 38_ / 44_)"""
+    if raw:
+        out = dict(X)
+    else:
+        mu, sd = X[ref_set][idx].mean(0), X[ref_set][idx].std(0) + 1e-6
+        out = {s: (v - mu) / sd for s, v in X.items()}
+    for s in ext or []:
+        out[s] = (X[s] - X[s].mean(0)) / (X[s].std(0) + 1e-6)
+    return out
+
+
+rows = []
+for k in range(5):
+    s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+    o = {x: s.tile_id[s.split == x].values for x in ["train", "val", "test"]}
+    nif = lambda x: (x - IF_O[o["train"]].mean((0, 1))) / (IF_O[o["train"]].std((0, 1)) + 1e-3)
+    h = pd.read_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"))
+    hs = {x: h.spot_idx[h.split == x].values for x in ["train", "val", "test"]}
+    sp = pd.read_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+    tsp = sp.loc[T.img_idx].values
+    ts = {x: T.index[tsp == x].values for x in ["train", "val", "test"]}
+    vi, vt = (torch.as_tensor(x, device=dev) for x in one_text_per_image(ts["val"], np.random.default_rng(0)))
+    ii, tt = (torch.as_tensor(x, device=dev) for x in one_text_per_image(ts["test"], np.random.default_rng(k)))
+    trt = torch.as_tensor(ts["train"], device=dev)
+    pool = np.flatnonzero(~HM.group.str.startswith("spatialLIBD").values)
+    grp = np.random.default_rng(k).permutation(sorted(HM.group.iloc[pool].unique()))
+    isva = HM.group.iloc[pool].isin(set(grp[: max(1, len(grp) // 10)])).values
+    d_tr, d_va = pool[~isva], pool[isva]
+
+    hm = mlp(512).to(dev)
+    hm.load_state_dict(torch.load(os.path.join(ROOT, "runs", "rnafm", f"other_fold{k}.pt"), map_location=dev)["mif"])
+    hm.eval()
+    with torch.no_grad():
+        Zm = {"he_o": nz(HE_O[o["test"], 0]), "orion": nz(hm(nif(IF_O[o["test"], 0]))),
+              "tcga_v": nz(hm(nif(IF_V_T[ii]))), "dlpfc_v": nz(hm(nif(IF_V_D)))}
+    Zr, Zt = {}, {}
+    for e in RNA_ALL:
+        X = zs(RNA_IN[e], "hest", hs["train"], e == "hvg")
+        hr = train_head(X["hest"][hs["train"]], HE_H[hs["train"]], X["hest"][hs["val"]], HE_H[hs["val"]], "linear", 2700, k)
+        Xd = zs(RNA_IN[e], "hest", d_tr, False, ext=["dlpfc"])
+        hd = train_head(Xd["hest"][d_tr], HE_H[d_tr], Xd["hest"][d_va], HE_H[d_va], "linear", 2700, k)
+        with torch.no_grad():
+            Zr[e] = {"hest": nz(hr(X["hest"][hs["test"]])), "orion_v": nz(hr(X["orion_v"][o["test"]])),
+                     "tcga_v": nz(hr(X["tcga_v"][ii])), "dlpfc": nz(hd(Xd["dlpfc"]))}
+        print(f"fold {k} RNA head {e} done", flush=True)
+    for e in TXT_ENCS:
+        X = zs(TXT_IN[e], "alltxt", trt, e == "biomedclip")
+        ht = train_head(X["alltxt"][trt], HE_T[T_IMG[trt]], X["alltxt"][vt], HE_T[vi, 0], "linear", 1000, k, pos_tr=T_IMG[trt])
+        # DLPFC synthetic captions: z-scored with TCGA-train stats (text domain is the same caption style)
+        with torch.no_grad():
+            Zt[e] = {"tcga": nz(ht(X["alltxt"][tt])), "orion_s": nz(ht(X["orion_gentext"][o["test"]])),
+                     "dlpfc_s": nz(ht(X["dlpfc_gentext"]))}
+    he_h, he_t, he_d = nz(HE_H[hs["test"]]), nz(HE_T[ii, 0]), nz(HE_D)
+    for cname, (re_, te_) in CONF.items():
+        R = {s_: nz(sum(Zr[e][s_] for e in re_)) for s_ in Zr[re_[0]]}
+        Tx = {s_: nz(sum(Zt[e][s_] for e in te_)) for s_ in Zt[te_[0]]}
+        res = {("H&E<->mIF (ORION)", "trained"): g100(Zm["he_o"], Zm["orion"]),
+               ("H&E<->RNA (HEST)", "trained"): g100(he_h, R["hest"]),
+               ("H&E<->text (TCGA)", "trained"): g100(he_t, Tx["tcga"]),
+               ("mIF<->RNA(virtual) (ORION)", "emergent"): g100(Zm["orion"], R["orion_v"]),
+               ("mIF<->text(synthetic) (ORION)", "emergent"): g100(Zm["orion"], Tx["orion_s"]),
+               ("text<->RNA(virtual) (TCGA)", "emergent"): g100(Tx["tcga"], R["tcga_v"]),
+               ("text<->mIF(MIPHEI) (TCGA)", "emergent"): g100(Tx["tcga"], Zm["tcga_v"]),
+               ("H&E<->RNA (DLPFC)", "external"): g100(he_d, R["dlpfc"]),
+               ("RNA<->text(synthetic) (DLPFC)", "external"): g100(R["dlpfc"], Tx["dlpfc_s"]),
+               ("RNA<->mIF(MIPHEI) (DLPFC)", "external"): g100(R["dlpfc"], Zm["dlpfc_v"])}
+        for (pair, kind), r in res.items():
+            rows.append({"config": cname, "fold": k, "pair": pair, "kind": kind, **r})
+        print(f"fold {k} {cname:<10} " + "  ".join(f"{p.split(' (')[0]} {r['R@1']:.3f}" for (p, _), r in res.items()), flush=True)
+    pd.DataFrame(rows).to_csv(os.path.join(ROOT, "results", "final_ens_long.csv"), index=False)
+
+R = pd.DataFrame(rows)
+S = R.groupby(["config", "kind", "pair"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "final_ens_summary.csv"))
+tests = []
+for c in CONF:
+    if c == "S4":
+        continue
+    for p in R.pair.unique():
+        a = R[(R.config == c) & (R.pair == p)].sort_values("fold")["R@1"].values
+        b = R[(R.config == "S4") & (R.pair == p)].sort_values("fold")["R@1"].values
+        tests.append({"config": c, "pair": p, "S4": b.mean(), "new": a.mean(), "delta": (a - b).mean(),
+                      "folds_won": int((a > b).sum()), "p_paired_t": stats.ttest_rel(a, b).pvalue})
+pd.DataFrame(tests).round(4).to_csv(os.path.join(ROOT, "results", "final_ens_tests.csv"), index=False)
+pd.set_option("display.width", 250)
+print(R.pivot_table(index="config", columns="pair", values="R@1", sort=False).round(3).T.to_string())
+print(pd.DataFrame(tests).round(4).to_string())

experiments/48_joint_ens.py

@@ -0,0 +1,315 @@
+"""Joint training of the multi-encoder (late-fusion) four-modality model: does the J8 two-stage gain of 42_ stack with
+the encoder fusion of 47_?  Real pairs only; virtual / synthetic data only at test time (same splits / test items).
+
+Stage 1 (separate, frozen UNI2 anchor; = 47_): one head per encoder - VirTues mIF -> MLP; every RNA / text encoder ->
+linear (FM inputs z-scored with train stats, hvg / BiomedCLIP raw).
+Stage 2 (joint, = J8 of 42_): all stage-1 heads + a shared zero-initialised residual adapter on UNI2 (H&E = UNI2 +
+adapter(UNI2)) fine-tuned together, sum of the three pair losses (H&E-mIF, H&E-RNA, H&E-text), similarity
+distillation lambda 1, AdamW lr 1e-4, 1000 steps, joint early stopping on the geometric mean of the three val R@1.
+Configurations (per fold, all from the SAME stage-1 heads):
+  S4        stage 1 only, RNA hvg, text BiomedCLIP                         (single encoders, separate)
+  J8        S4 -> stage 2                                                  (single encoders, joint)
+  E         stage 1 only, ens-small (RNA hvg+Tahoe-x1+SE-600M+AIDO.Cell, text KEEP+BiomedCLIP+Qwen3-0.6B)
+  EJ-fused  E -> stage 2, loss on the FUSED modality embedding normalise(sum_e normalise(head_e(x_e)))
+  EJ-each   E -> stage 2, loss = mean over encoders of each head's own loss (fusion only at test time)
+  EJ-frz    EJ-each with the RNA / text heads frozen (only the H&E adapter, the mIF head and the temperatures move)
+Output results/joint_ens_long.csv, results/joint_ens_summary.csv, results/joint_ens_tests.csv
+"""
+import copy
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy import stats
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, SPLITS
+
+dev = "cuda"
+H, RF = os.path.join(CACHE, "hest"), os.path.join(CACHE, "rnafm")
+G100, DRAWS, BS, LR, EVAL_EVERY = 100, 50, 256, 3e-4, 50
+f32 = lambda a: torch.from_numpy(np.asarray(a, dtype=np.float32))
+nz = lambda x: F.normalize(x, dim=-1)
+RNA_E, TXT_E = ["hvg", "tahoex1", "se600m", "aidocell"], ["keep", "biomedclip", "qwen3-0.6b"]
+CONF = {"S4": (["hvg"], ["biomedclip"], None), "J8": (["hvg"], ["biomedclip"], "fused"),
+        "E": (RNA_E, TXT_E, None), "EJ-fused": (RNA_E, TXT_E, "fused"), "EJ-each": (RNA_E, TXT_E, "each"),
+        "EJ-frz": (RNA_E, TXT_E, "frz")}
+
+# ---------------------------------------------------------------- data (= 47_)
+HE_O = f32(np.load(os.path.join(CACHE, "he_uni2.npy"))).to(dev)
+IF_O = f32(np.load(os.path.join(CACHE, "if_virtues_m16.npy"))).to(dev)
+IF_V_T = f32(np.load(os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy"))[:, 0]).to(dev)
+HE_H = f32(np.load(os.path.join(H, "he_uni2.npy"))[:, 0]).to(dev)
+HE_T = f32(np.load(os.path.join(CACHE, "allimg_he_uni2.npy"))).to(dev)
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+T_IMG = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def hvg_inputs():
+    def virt(st):
+        g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+        c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+        c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+        return c[:, pd.Series(range(len(g)), index=g).loc[hvg].values]
+    return {"hest": np.load(os.path.join(H, "hvg.npy")), "orion_v": virt("orion"), "tcga_v": virt("tcga_text")}
+
+
+RNA_IN = {e: ({s: f32(x).to(dev) for s, x in hvg_inputs().items()} if e == "hvg" else
+              {s: f32(np.load(os.path.join(RF, f"{e}_{s}.npy"))).to(dev) for s in ["hest", "orion_v", "tcga_v"]})
+          for e in RNA_E}
+TXT_IN = {e: {s: f32(np.load(os.path.join(CACHE, f"{s}_text_{e}.npy"))[:, 0]).to(dev)
+              for s in ["alltxt", "orion_gentext"]} for e in TXT_E}
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_ / 42_ / 47_
+    a, b = a.cpu(), b.cpu()
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float((((s > s.diag()[:, None]).sum(1) == 0).float().mean() + ((s.T > s.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+
+
+def clip_loss(lg, pos=None):
+    if pos is None:
+        t = torch.arange(len(lg), device=dev)
+        return (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+    tgt = pos / pos.sum(1, keepdim=True)
+    return (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+
+
+def zs(X, ref_set, idx, raw):
+    if raw:
+        return dict(X)
+    mu, sd = X[ref_set][idx].mean(0), X[ref_set][idx].std(0) + 1e-6
+    return {s: (v - mu) / sd for s, v in X.items()}
+
+
+class Joint(nn.Module):
+    """heads: 'mif' + 'rna:<enc>' + 'text:<enc>'; logits per head + fused 'rna' / 'text'; residual adapter on UNI2"""
+
+    def __init__(self, heads, logits, adapter):
+        super().__init__()
+        self.heads = nn.ModuleDict({k.replace(".", "_"): copy.deepcopy(h) for k, h in heads.items()})
+        self.keys = {k.replace(".", "_"): k for k in heads}
+        self.logit = nn.ParameterDict({k.replace(":", "__").replace(".", "_"): nn.Parameter(v.detach().clone())
+                                       for k, v in logits.items()})
+        self.ah = None
+        if adapter:
+            self.ah = mlp(1536)
+            nn.init.zeros_(self.ah[-1].weight); nn.init.zeros_(self.ah[-1].bias)
+
+    def h(self, k):
+        return self.heads[k.replace(".", "_")]
+
+    def lg(self, k):
+        return self.logit[k.replace(":", "__").replace(".", "_")].exp().clamp(max=100)
+
+    def he(self, x):
+        return x if self.ah is None else x + self.ah(x)
+
+    def fuse(self, mod, encs, X):
+        """X: enc -> input rows; returns normalised fused embedding and the per-encoder embeddings"""
+        z = {e: nz(self.h(f"{mod}:{e}")(X[e])) for e in encs}
+        return nz(sum(z.values())), z
+
+
+f_out = os.path.join(ROOT, "results", "joint_ens_long.csv")
+rows = pd.read_csv(f_out).to_dict("records") if os.path.exists(f_out) else []
+done = {(r["config"], r["fold"]) for r in rows}
+
+for k in range(5):
+    if all((c, k) in done for c in CONF):
+        continue
+    torch.manual_seed(k)
+    s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+    o = {x: s.tile_id[s.split == x].values for x in ["train", "val", "test"]}
+    mu_if, sd_if = IF_O[o["train"]].mean((0, 1)), IF_O[o["train"]].std((0, 1)) + 1e-3
+    nif = lambda x: (x - mu_if) / sd_if
+    hh = pd.read_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"))
+    hs = {x: hh.spot_idx[hh.split == x].values for x in ["train", "val", "test"]}
+    sp = pd.read_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+    tsp = sp.loc[T.img_idx].values
+    ts = {x: T.index[tsp == x].values for x in ["train", "val", "test"]}
+    vi, vt = (torch.as_tensor(x, device=dev) for x in one_text_per_image(ts["val"], np.random.default_rng(0)))
+    ii, tt = (torch.as_tensor(x, device=dev) for x in one_text_per_image(ts["test"], np.random.default_rng(k)))
+    trt = torch.as_tensor(ts["train"], device=dev)
+    Xr = {e: zs(RNA_IN[e], "hest", hs["train"], e == "hvg") for e in RNA_E}
+    Xt = {e: zs(TXT_IN[e], "alltxt", trt, e == "biomedclip") for e in TXT_E}
+    o_tr = torch.as_tensor(o["train"], device=dev)
+    h_tr = torch.as_tensor(hs["train"], device=dev)
+
+    def batch(task, rng):
+        """one training batch: (H&E anchor rows, modality inputs (enc -> rows), multi-positive matrix or None)"""
+        if task == "mif":
+            b = o_tr[torch.as_tensor(rng.choice(len(o_tr), BS, replace=False), device=dev)]
+            d = torch.randint(0, 8, (BS,), device=dev)
+            return HE_O[b, d], {"mif": nif(IF_O[b, d])}, None
+        if task == "rna":
+            b = h_tr[torch.as_tensor(rng.choice(len(h_tr), BS, replace=False), device=dev)]
+            return HE_H[b], {e: Xr[e]["hest"][b] for e in RNA_E}, None
+        b = trt[torch.as_tensor(rng.choice(len(trt), BS, replace=False), device=dev)]
+        im = T_IMG[b]
+        return (HE_T[im, torch.randint(0, 8, (BS,), device=dev)], {e: Xt[e]["alltxt"][b] for e in TXT_E},
+                (im[:, None] == im[None, :]).float())
+
+    VAL = {"mif": (HE_O[o["val"], 0], {"mif": nif(IF_O[o["val"], 0])}),
+           "rna": (HE_H[hs["val"]], {e: Xr[e]["hest"][hs["val"]] for e in RNA_E}),
+           "text": (HE_T[vi, 0], {e: Xt[e]["alltxt"][vt] for e in TXT_E})}
+
+    # ------------------------------------------------ stage 1: one head per encoder, frozen anchor (= 47_)
+    def train_head(task, enc, kind, steps):
+        torch.manual_seed(k); rng = np.random.default_rng(k)
+        din = VAL[task][1][enc].shape[1]
+        h = (mlp(din) if kind == "mlp" else nn.Linear(din, 1536)).to(dev)
+        logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+        opt = torch.optim.AdamW(list(h.parameters()) + [logit], lr=LR, weight_decay=0.05)
+        sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=LR, total_steps=steps, pct_start=0.05)
+        za_va = nz(VAL[task][0])
+        best, state = -1, None
+        for step in range(1, steps + 1):
+            h.train()
+            a, X, pos = batch(task, rng)
+            loss = clip_loss(logit.exp().clamp(max=100) * nz(a) @ nz(h(X[enc])).T, pos)
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+            if step % EVAL_EVERY == 0 or step == steps:
+                h.eval()
+                with torch.no_grad():
+                    v = r1(za_va, nz(h(VAL[task][1][enc])))
+                if v > best:
+                    best, state = v, ({k_: x.detach().clone() for k_, x in h.state_dict().items()}, logit.detach().clone())
+        h.load_state_dict(state[0]); h.eval()
+        return h, state[1]
+
+    HEADS, LOGITS = {}, {}
+    HEADS["mif"], LOGITS["mif"] = train_head("mif", "mif", "mlp", 400)
+    for e in RNA_E:
+        HEADS[f"rna:{e}"], LOGITS[f"rna:{e}"] = train_head("rna", e, "linear", 2700)
+    for e in TXT_E:
+        HEADS[f"text:{e}"], LOGITS[f"text:{e}"] = train_head("text", e, "linear", 1000)
+    print(f"fold {k} stage 1 done", flush=True)
+
+    # ------------------------------------------------ stage 2: joint residual fine-tune (= J8 of 42_)
+    def val_score(m, re_, te_):
+        m.eval()
+        with torch.no_grad():
+            v = {"mif": r1(nz(m.he(VAL["mif"][0])), nz(m.h("mif")(VAL["mif"][1]["mif"]))),
+                 "rna": r1(nz(m.he(VAL["rna"][0])), m.fuse("rna", re_, VAL["rna"][1])[0]),
+                 "text": r1(nz(m.he(VAL["text"][0])), m.fuse("text", te_, VAL["text"][1])[0])}
+        m.train()
+        return float(np.exp(np.mean(np.log([max(x, 1e-4) for x in v.values()])))), v
+
+    def joint(re_, te_, mode, steps=1000, lr=1e-4, distill=1.0):
+        keys = ["mif"] + [f"rna:{e}" for e in re_] + [f"text:{e}" for e in te_]
+        lg = {kk: LOGITS[kk] for kk in keys}
+        lg["rna"] = torch.stack([LOGITS[f"rna:{e}"] for e in re_]).mean()
+        lg["text"] = torch.stack([LOGITS[f"text:{e}"] for e in te_]).mean()
+        m = Joint({kk: HEADS[kk] for kk in keys}, lg, adapter=True).to(dev)
+        torch.manual_seed(k); rng = np.random.default_rng(k)
+        if mode == "frz":
+            for kk in keys[1:]:
+                m.h(kk).requires_grad_(False)
+        opt = torch.optim.AdamW([p for p in m.parameters() if p.requires_grad], lr=lr, weight_decay=0.05)
+        sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.05)
+        best, state, bv = -1, None, None
+        m.train()
+        for step in range(1, steps + 1):
+            loss = 0
+            for task in ("mif", "rna", "text"):
+                a, X, pos = batch(task, rng)
+                za = nz(m.he(a))
+                if task == "mif":
+                    loss = loss + clip_loss(m.lg("mif") * za @ nz(m.h("mif")(X["mif"])).T, pos)
+                else:
+                    encs = re_ if task == "rna" else te_
+                    X = {e: X[e] for e in encs}
+                    zf, ze = m.fuse(task, encs, X)
+                    if mode == "fused":
+                        loss = loss + clip_loss(m.lg(task) * za @ zf.T, pos)
+                    else:
+                        loss = loss + sum(clip_loss(m.lg(f"{task}:{e}") * za @ ze[e].T, pos) for e in encs) / len(encs)
+                u = nz(a)
+                loss = loss + distill * F.mse_loss(za @ za.T, u @ u.T)
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+            if step % EVAL_EVERY == 0 or step == steps:
+                sc, v = val_score(m, re_, te_)
+                if sc > best:
+                    best, bv, state = sc, v, {k_: x.detach().clone() for k_, x in m.state_dict().items()}
+        m.load_state_dict(state); m.eval()
+        return m, bv
+
+    @torch.no_grad()
+    def evaluate(m, re_, te_):
+        R = lambda st, idx: m.fuse("rna", re_, {e: Xr[e][st][idx] for e in re_})[0]
+        Tx = lambda st, idx: m.fuse("text", te_, {e: Xt[e][st][idx] for e in te_})[0]
+        zm = nz(m.h("mif")(nif(IF_O[o["test"], 0])))
+        zt = Tx("alltxt", tt)
+        return {("H&E<->mIF (ORION)", "trained"): g100(nz(m.he(HE_O[o["test"], 0])), zm),
+                ("H&E<->RNA (HEST)", "trained"): g100(nz(m.he(HE_H[hs["test"]])), R("hest", hs["test"])),
+                ("H&E<->text (TCGA)", "trained"): g100(nz(m.he(HE_T[ii, 0])), zt),
+                ("mIF<->RNA(virtual) (ORION)", "emergent"): g100(zm, R("orion_v", o["test"])),
+                ("mIF<->text(synthetic) (ORION)", "emergent"): g100(zm, Tx("orion_gentext", o["test"])),
+                ("text<->RNA(virtual) (TCGA)", "emergent"): g100(zt, R("tcga_v", ii)),
+                ("text<->mIF(MIPHEI) (TCGA)", "emergent"): g100(zt, nz(m.h("mif")(nif(IF_V_T[ii]))))}
+
+    for cname, (re_, te_, mode) in CONF.items():
+        if (cname, k) in done:
+            continue
+        if mode is None:  # separate: stage-1 heads, no adapter
+            keys = ["mif"] + [f"rna:{e}" for e in re_] + [f"text:{e}" for e in te_]
+            m = Joint({kk: HEADS[kk] for kk in keys}, {kk: LOGITS[kk] for kk in keys}, adapter=False).to(dev).eval()
+            val = {}
+        else:
+            m, val = joint(re_, te_, mode)
+        for (pair, kind), r in evaluate(m, re_, te_).items():
+            rows.append({"config": cname, "fold": k, "pair": pair, "kind": kind, **r,
+                         **{f"val_{t}": v for t, v in val.items()}})
+        pd.DataFrame(rows).to_csv(f_out, index=False)
+        print(f"fold {k} {cname:<9} " + "  ".join(f"{r['pair'].split(' (')[0]} {r['R@1']:.3f}"
+                                                  for r in rows if r["config"] == cname and r["fold"] == k), flush=True)
+
+R = pd.DataFrame(rows)
+S = R.groupby(["config", "kind", "pair"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "joint_ens_summary.csv"))
+tests = []
+for c, ref in [("J8", "S4"), ("E", "S4"), ("EJ-fused", "S4"), ("EJ-each", "S4"), ("EJ-fused", "E"), ("EJ-each", "E"),
+               ("EJ-fused", "J8"), ("EJ-each", "EJ-fused"), ("EJ-frz", "S4"),
+               ("EJ-frz", "E"), ("EJ-frz", "EJ-each")]:
+    for p in R.pair.unique():
+        a = R[(R.config == c) & (R.pair == p)].sort_values("fold")["R@1"].values
+        b = R[(R.config == ref) & (R.pair == p)].sort_values("fold")["R@1"].values
+        tests.append({"config": c, "vs": ref, "pair": p, "ref": b.mean(), "new": a.mean(), "delta": (a - b).mean(),
+                      "folds_won": int((a > b).sum()), "p_paired_t": stats.ttest_rel(a, b).pvalue})
+pd.DataFrame(tests).round(4).to_csv(os.path.join(ROOT, "results", "joint_ens_tests.csv"), index=False)
+pd.set_option("display.width", 250)
+print(R.pivot_table(index="config", columns="pair", values="R@1", sort=False).round(3).T.to_string())
+print(R.pivot_table(index="config", columns="pair", values="pair_acc", sort=False).round(3).T.to_string())
+print(pd.DataFrame(tests).round(4).to_string())

experiments/49_hest_patches.py

@@ -0,0 +1,147 @@
+"""H&E patches of the 9,484 HEST training-pool spots (cache/hest/meta.csv), so that image encoders other than the
+precomputed UNI2 / GigaPath features can be run on them (e.g. MedVTok).
+
+Run:  HF_TOKEN=... .venv/bin/python -I 49_hest_patches.py [--workers 16] [--only <sample> ...]
+Source: MahmoodLab/hest wsis/<sample>.tif (pyramidal tiled TIFF), level 0, read with HTTP range requests on the
+resolved CDN URL (RangeFile) - only the tiles under the needed spots. (The patches/<sample>.h5 files cannot be read
+selectively: their barcode chunks are interleaved with the image chunks, i.e. finding the rows reads the whole file.)
+Patch = 224 x 224 level-0 px centred on the spot centre (obsm['spatial'] = full-res (x, y), cache/hest/spot_coords.csv),
+no resizing: this is exactly the image behind the cached UNI2 features (re-encoding these crops with UNI2-h gives
+cos 1.000 to cache/hest/he_uni2.npy, checked on TENX152 / MEND139; 0.5-um/px resampled crops give only 0.66-0.81).
+Pixels outside the slide are white. Resumable per sample.
+Output cache/hest/patches/<sample>.npz (img, idx) and, once all are present, cache/hest/he_img.npy (9484, 224, 224, 3)
+uint8 in meta.csv row order (+ cache/hest/he_img_missing.csv, spots whose slide could not be read).
+"""
+import io
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+import pandas as pd
+import requests
+import tifffile
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+H = os.path.join(ROOT, "cache", "hest")
+OUT = os.path.join(H, "patches")
+os.makedirs(OUT, exist_ok=True)
+WORKERS = int(sys.argv[sys.argv.index("--workers") + 1]) if "--workers" in sys.argv else 16
+ONLY = sys.argv[sys.argv.index("--only") + 1:] if "--only" in sys.argv else None
+M = pd.read_csv(os.path.join(H, "meta.csv"))
+C = pd.read_csv(os.path.join(H, "spot_coords.csv"))
+assert (C.idx.values == M.idx.values).all()
+
+
+class RangeFile(io.RawIOBase):
+    """Read-only, seekable file over HTTP range requests (block cache), for h5py. huggingface_hub's HfFileSystem hung
+    in this environment (2026-10-28), plain range requests on the resolved CDN URL are fast."""
+    BLOCK = 256 * 1024
+
+    def __init__(self, url):
+        hdr = {"Authorization": f"Bearer {os.environ['HF_TOKEN']}"}
+        r = requests.get(url, headers={**hdr, "Range": "bytes=0-0"}, timeout=60)
+        r.raise_for_status()
+        self.url = r.url  # signed CDN URL after redirects
+        self.size = int(r.headers["Content-Range"].split("/")[1])
+        self.pos, self.cache, self.s = 0, {}, requests.Session()
+
+    def _block(self, i):
+        if i not in self.cache:
+            a = i * self.BLOCK
+            for t in range(5):
+                try:
+                    r = self.s.get(self.url, headers={"Range": f"bytes={a}-{min(self.size, a + self.BLOCK) - 1}"}, timeout=60)
+                    r.raise_for_status()
+                    break
+                except requests.RequestException:
+                    if t == 4:
+                        raise
+            self.cache[i] = r.content
+        return self.cache[i]
+
+    def readable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+    def tell(self):
+        return self.pos
+
+    def seek(self, off, whence=0):
+        self.pos = off if whence == 0 else self.pos + off if whence == 1 else self.size + off
+        return self.pos
+
+    def readinto(self, b):
+        n = min(len(b), max(0, self.size - self.pos))
+        out, p = bytearray(), self.pos
+        while len(out) < n:
+            blk = self._block(p // self.BLOCK)
+            o = p % self.BLOCK
+            take = blk[o:o + n - len(out)]
+            out += take; p += len(take)
+        b[:n] = out
+        self.pos = p
+        return n
+
+
+def fetch(sid):
+    out = os.path.join(OUT, f"{sid}.npz")
+    if os.path.exists(out):
+        return sid, "cached"
+    c = C[C["sample"] == sid]
+    f = RangeFile(f"https://huggingface.co/datasets/MahmoodLab/hest/resolve/main/wsis/{sid}.tif")
+    with tifffile.TiffFile(f) as tf:
+        pg = tf.pages[0]
+        Hh, W = pg.shape[:2]
+        th, tw = pg.tilelength, pg.tilewidth
+        ntx = -(-W // tw)
+        S = 224
+        tiles = {}
+
+        def tile(ty, tx):
+            if (ty, tx) not in tiles:
+                i = ty * ntx + tx
+                f.seek(pg.dataoffsets[i])
+                a = pg.decode(f.read(pg.databytecounts[i]), i, jpegtables=pg.jpegtables)[0]
+                tiles[(ty, tx)] = a.reshape(th, tw, -1)[..., :3]
+            return tiles[(ty, tx)]
+
+        imgs = []
+        for x, y in zip(c.x.values, c.y.values):
+            x0, y0 = int(round(x - S / 2)), int(round(y - S / 2))
+            reg = np.full((S, S, 3), 255, np.uint8)
+            for ty in range(max(0, y0 // th), min(-(-Hh // th), (y0 + S - 1) // th + 1)):
+                for tx in range(max(0, x0 // tw), min(ntx, (x0 + S - 1) // tw + 1)):
+                    a = tile(ty, tx)
+                    ys, xs = max(y0, ty * th), max(x0, tx * tw)
+                    ye, xe = min(y0 + S, (ty + 1) * th, Hh), min(x0 + S, (tx + 1) * tw, W)
+                    reg[ys - y0:ye - y0, xs - x0:xe - x0] = a[ys - ty * th:ye - ty * th, xs - tx * tw:xe - tx * tw]
+            imgs.append(reg)
+    np.savez(out + ".tmp.npz", img=np.stack(imgs), idx=c.idx.values)
+    os.replace(out + ".tmp.npz", out)
+    return sid, f"{len(imgs)} spots, crop {S}px, {len(tiles)} tiles, {sum(len(v) for v in f.cache.values()) / 1e6:.0f} MB"
+
+
+samples = ONLY or M["sample"].unique().tolist()
+with ThreadPoolExecutor(WORKERS) as ex:
+    futs = {s: ex.submit(fetch, s) for s in samples}
+    for n, (sid, fu) in enumerate(futs.items()):
+        try:
+            print(f"[{n + 1}/{len(samples)}] {sid}: {fu.result()[1]}", flush=True)
+        except Exception as e:  # noqa: BLE001 - reported, spots become 'missing'
+            print(f"[{n + 1}/{len(samples)}] {sid}: FAILED {type(e).__name__}: {e}", flush=True)
+if ONLY:
+    sys.exit()
+
+img = np.zeros((len(M), 224, 224, 3), np.uint8)
+have = np.zeros(len(M), bool)
+for sid in M["sample"].unique():
+    p = os.path.join(OUT, f"{sid}.npz")
+    if os.path.exists(p):
+        z = np.load(p)
+        img[z["idx"]] = z["img"]; have[z["idx"]] = True
+np.save(os.path.join(H, "he_img.npy"), img)
+pd.DataFrame({"idx": np.flatnonzero(~have)}).to_csv(os.path.join(H, "he_img_missing.csv"), index=False)
+print(f"saved he_img.npy {img.shape}; missing {(~have).sum()} spots")

experiments/51_medvtok_eval.py

@@ -0,0 +1,373 @@
+"""MedVTok as image (H&E) information for the fused four-modality model, plus the adapter-only variant of EJ-frz.
+Real pairs only; virtual / synthetic data only at test time; same splits / test items as 42_ / 47_ / 48_.
+RNA = ens-small (hvg + Tahoe-x1 + SE-600M + AIDO.Cell), text = KEEP + BiomedCLIP + Qwen3-Emb-0.6B, mIF = VirTues MLP.
+
+Configurations (per fold; all UNI2 configs start from the same stage-1 heads):
+  E           separate, frozen UNI2 anchor                                         (= 47_ ens-small / 48_ E)
+  EJ-frz      E -> joint stage 2: residual adapter on UNI2 + mIF head, RNA / text heads frozen       (= 48_)
+  EJ-ad       EJ-frz with the mIF head frozen as well (only the adapter + temperatures train)
+  MV-E        separate, frozen MedVTok anchor: every head maps into the MedVTok space (1152-d) instead of UNI2
+  EJ-frz+MV   EJ-frz whose adapter also sees MedVTok: H&E = UNI2 + A([UNI2, c * MedVTok])   (c matches the mean
+              feature norms on ORION train; zero-initialised output, so it starts exactly at E)
+  EJ-ad+MV    EJ-ad with the same MedVTok-informed adapter
+  EJ-ad+RP    control for EJ-ad+MV: the extra adapter input is a fixed random projection of UNI2 to 1152-d (same
+              shape / scale as MedVTok, no new information) -> separates "MedVTok information" from "wider adapter"
+Stage 2 (= J8 / 48_): sum of the three pair losses (each RNA / text head its own loss, averaged over encoders),
+similarity distillation to UNI2 (lambda 1), AdamW lr 1e-4, 1000 steps, early stopping on the geometric mean of the
+three val R@1 (fused embeddings).
+External DLPFC H&E<->RNA for E and MV-E (RNA heads retrained without spatialLIBD, inputs z-scored per dataset, = 47_).
+Output results/medvtok_long.csv, results/medvtok_summary.csv, results/medvtok_tests.csv
+"""
+import copy
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from scipy import stats
+from scipy.optimize import linear_sum_assignment
+
+from common import CACHE, ROOT, SPLITS
+
+dev = "cuda"
+H, RF, EXT = os.path.join(CACHE, "hest"), os.path.join(CACHE, "rnafm"), os.path.join(CACHE, "external")
+G100, DRAWS, BS, LR, EVAL_EVERY = 100, 50, 256, 3e-4, 50
+f32 = lambda a: torch.from_numpy(np.asarray(a, dtype=np.float32))
+nz = lambda x: F.normalize(x, dim=-1)
+RNA_E, TXT_E = ["hvg", "tahoex1", "se600m", "aidocell"], ["keep", "biomedclip", "qwen3-0.6b"]
+# name: (anchor, stage-2 mode, adapter input)
+CONF = {"E": ("uni2", None, None), "EJ-frz": ("uni2", "frz", "u"), "EJ-ad": ("uni2", "ad", "u"),
+        "MV-E": ("medvtok", None, None), "EJ-frz+MV": ("uni2", "frz", "cat"), "EJ-ad+MV": ("uni2", "ad", "cat"),
+        "EJ-ad+RP": ("uni2", "ad", "rp")}
+
+# ---------------------------------------------------------------- data
+IF_O = f32(np.load(os.path.join(CACHE, "if_virtues_m16.npy"))).to(dev)
+IF_V_T = f32(np.load(os.path.join(CACHE, "if_virtues_m16_virtual_miphei_tcga_text.npy"))[:, 0]).to(dev)
+AN = {"uni2": {"o": np.load(os.path.join(CACHE, "he_uni2.npy")), "h": np.load(os.path.join(H, "he_uni2.npy"))[:, 0],
+               "t": np.load(os.path.join(CACHE, "allimg_he_uni2.npy")), "d": np.load(os.path.join(EXT, "dlpfc_he_uni2.npy"))},
+      "medvtok": {"o": np.load(os.path.join(CACHE, "he_medvtok.npy")), "h": np.load(os.path.join(H, "he_medvtok.npy"))[:, 0],
+                  "t": np.load(os.path.join(CACHE, "allimg_he_medvtok.npy")),
+                  "d": np.load(os.path.join(EXT, "dlpfc_he_medvtok.npy"))}}
+AN = {a: {s: f32(x).to(dev) for s, x in d.items()} for a, d in AN.items()}
+DIM = {a: AN[a]["o"].shape[-1] for a in AN}
+MISS = pd.read_csv(os.path.join(H, "he_img_missing.csv")).idx.values  # HEST spots without a HEST patch (no MedVTok)
+T = pd.read_csv(os.path.join(CACHE, "alltxt_meta.csv"))
+T_IMG = torch.as_tensor(T.img_idx.values, device=dev)
+T = T[~pd.read_csv(os.path.join(CACHE, "alltxt_junk.csv")).junk.values]
+HM = pd.read_csv(os.path.join(H, "meta.csv"))
+RPM = torch.randn(1536, DIM["medvtok"], generator=torch.Generator().manual_seed(0)).to(dev) / np.sqrt(1536)
+hvg = pd.read_csv(os.path.join(H, "hvg_genes.csv")).gene.values
+
+
+def hvg_inputs():  # = 47_
+    def virt(st):
+        g = pd.read_csv(os.path.join(CACHE, "stpath", f"{st}_genes.csv")).gene.values
+        c = np.expm1(np.load(os.path.join(CACHE, "stpath", f"{st}_pred.npy")).astype(np.float32)).clip(0)
+        c = np.log1p(c / c.sum(1, keepdims=True) * 1e4)
+        return c[:, pd.Series(range(len(g)), index=g).loc[hvg].values]
+    eg = pd.read_csv(os.path.join(EXT, "dlpfc_genes_full.csv")).gene.values
+    ec = np.load(os.path.join(EXT, "dlpfc_counts_full.npy")).astype(np.float32)
+    ei = pd.Series(range(len(eg)), index=eg); ei = ei[~ei.index.duplicated()]
+    d = np.log1p(ec / ec.sum(1, keepdims=True).clip(1) * 1e4)
+    d = np.stack([d[:, ei[g]] if g in ei.index else np.zeros(len(d), np.float32) for g in hvg], 1)
+    return {"hest": np.load(os.path.join(H, "hvg.npy")), "orion_v": virt("orion"), "tcga_v": virt("tcga_text"), "dlpfc": d}
+
+
+RNA_IN = {e: ({s: f32(x).to(dev) for s, x in hvg_inputs().items()} if e == "hvg" else
+              {s: f32(np.load(os.path.join(RF, f"{e}_{s}.npy"))).to(dev) for s in ["hest", "orion_v", "tcga_v", "dlpfc"]})
+          for e in RNA_E}
+TXT_IN = {e: {s: f32(np.load(os.path.join(CACHE, f"{s}_text_{e}.npy"))[:, 0]).to(dev)
+              for s in ["alltxt", "orion_gentext"]} for e in TXT_E}
+
+
+def mlp(din, dout=1536, hid=512):
+    return nn.Sequential(nn.Linear(din, hid), nn.GELU(), nn.Dropout(0.1), nn.Linear(hid, hid), nn.GELU(),
+                         nn.LayerNorm(hid), nn.Linear(hid, dout))
+
+
+def one_text_per_image(t_ids, rng):
+    pick = T.loc[t_ids].groupby("img_idx").apply(lambda g: g.index[rng.integers(len(g))], include_groups=False)
+    return pick.index.values, pick.values
+
+
+def g100(a, b, seed=0):  # = 33_ / 42_ / 47_ / 48_
+    a, b = a.cpu(), b.cpu()
+    rng = np.random.default_rng(seed)
+    out = {m: [] for m in ["R@1", "R@5", "R@10", "MRR", "pair_acc"]}
+    for _ in range(DRAWS):
+        s = torch.from_numpy(rng.choice(len(a), min(G100, len(a)), replace=False))
+        sim = (a[s] @ b[s].T).numpy(); d = np.diag(sim)
+        for S in (sim, sim.T):
+            rank = (S > d[:, None]).sum(1)
+            for k_ in (1, 5, 10):
+                out[f"R@{k_}"].append((rank < k_).mean() / 2)
+            out["MRR"].append((1 / (rank + 1)).mean() / 2)
+        ri, ci = linear_sum_assignment(-sim)
+        out["pair_acc"].append((ci[np.argsort(ri)] == np.arange(len(s))).mean())
+    return {m: float(np.sum(v) / DRAWS) for m, v in out.items()}
+
+
+def r1(a, b):
+    s = a @ b.T
+    return float((((s > s.diag()[:, None]).sum(1) == 0).float().mean() + ((s.T > s.diag()[:, None]).sum(1) == 0).float().mean()) / 2)
+
+
+def clip_loss(lg, pos=None):
+    if pos is None:
+        t = torch.arange(len(lg), device=dev)
+        return (F.cross_entropy(lg, t) + F.cross_entropy(lg.T, t)) / 2
+    tgt = pos / pos.sum(1, keepdim=True)
+    return (torch.sum(-tgt * F.log_softmax(lg, 1), 1).mean() + torch.sum(-tgt * F.log_softmax(lg.T, 1), 1).mean()) / 2
+
+
+def zs(X, ref_set, idx, raw, ext=None):  # = 47_
+    out = dict(X) if raw else {s: (v - X[ref_set][idx].mean(0)) / (X[ref_set][idx].std(0) + 1e-6) for s, v in X.items()}
+    for s in ext or []:
+        out[s] = (X[s] - X[s].mean(0)) / (X[s].std(0) + 1e-6)
+    return out
+
+
+class Joint(nn.Module):
+    """heads 'mif', 'rna:<enc>', 'text:<enc>' (deep copies of stage-1 heads); adapter None / 'u' (UNI2 -> residual) /
+    'cat' ([UNI2, c * MedVTok] -> residual on UNI2)"""
+
+    def __init__(self, heads, logits, adapter=None, c_mv=1.0, anchor="uni2"):
+        super().__init__()
+        self.anchor = anchor
+        self.heads = nn.ModuleDict({k.replace(".", "_"): copy.deepcopy(h) for k, h in heads.items()})
+        self.logit = nn.ParameterDict({k.replace(":", "__").replace(".", "_"): nn.Parameter(v.detach().clone())
+                                       for k, v in logits.items()})
+        self.adapter, self.c_mv, self.ah = adapter, c_mv, None
+        if adapter:
+            self.ah = nn.Sequential(nn.Linear(1536 + (DIM["medvtok"] if adapter in ("cat", "rp") else 0), 512), *mlp(1536)[1:])
+            nn.init.zeros_(self.ah[-1].weight); nn.init.zeros_(self.ah[-1].bias)
+
+    def h(self, k):
+        return self.heads[k.replace(".", "_")]
+
+    def lg(self, k):
+        return self.logit[k.replace(":", "__").replace(".", "_")].exp().clamp(max=100)
+
+    def he(self, a):
+        """a: dict anchor -> rows (same images / views)"""
+        if self.adapter is None:
+            return a[self.anchor]
+        x = a["uni2"]
+        if self.adapter == "u":
+            inp = x
+        elif self.adapter == "cat":
+            inp = torch.cat([x, self.c_mv * a["medvtok"]], -1)
+        else:  # "rp": random projection of UNI2, rescaled to the same mean norm as c * MedVTok
+            rp = x @ RPM
+            inp = torch.cat([x, rp * (self.c_mv * self.mv_norm / self.rp_norm)], -1)
+        return x + self.ah(inp)
+
+    def fuse(self, mod, encs, X):
+        z = {e: nz(self.h(f"{mod}:{e}")(X[e])) for e in encs}
+        return nz(sum(z.values())), z
+
+
+f_out = os.path.join(ROOT, "results", "medvtok_long.csv")
+rows = pd.read_csv(f_out).to_dict("records") if os.path.exists(f_out) else []
+done = {(r["config"], r["fold"]) for r in rows}
+
+for k in range(5):
+    if all((c, k) in done for c in CONF):
+        continue
+    s = pd.read_csv(os.path.join(SPLITS, "patient_5fold", f"fold{k}.csv"))
+    o = {x: s.tile_id[s.split == x].values for x in ["train", "val", "test"]}
+    mu_if, sd_if = IF_O[o["train"]].mean((0, 1)), IF_O[o["train"]].std((0, 1)) + 1e-3
+    nif = lambda x: (x - mu_if) / sd_if
+    hh = pd.read_csv(os.path.join(SPLITS, "final", f"hest_fold{k}.csv"))
+    # spots without a HEST patch have no MedVTok feature: dropped from every split for every config (same items)
+    hs = {x: np.setdiff1d(hh.spot_idx[hh.split == x].values, MISS) for x in ["train", "val", "test"]}
+    sp = pd.read_csv(os.path.join(SPLITS, "final", f"tcga_fold{k}.csv")).set_index("img_idx").split
+    tsp = sp.loc[T.img_idx].values
+    ts = {x: T.index[tsp == x].values for x in ["train", "val", "test"]}
+    vi, vt = (torch.as_tensor(x, device=dev) for x in one_text_per_image(ts["val"], np.random.default_rng(0)))
+    ii, tt = (torch.as_tensor(x, device=dev) for x in one_text_per_image(ts["test"], np.random.default_rng(k)))
+    trt = torch.as_tensor(ts["train"], device=dev)
+    Xr = {e: zs(RNA_IN[e], "hest", hs["train"], e == "hvg") for e in RNA_E}
+    Xt = {e: zs(TXT_IN[e], "alltxt", trt, e == "biomedclip") for e in TXT_E}
+    o_tr, h_tr = torch.as_tensor(o["train"], device=dev), torch.as_tensor(hs["train"], device=dev)
+    # DLPFC heads: HEST without spatialLIBD (= 47_)
+    pool = np.setdiff1d(np.flatnonzero(~HM.group.str.startswith("spatialLIBD").values), MISS)
+    grp = np.random.default_rng(k).permutation(sorted(HM.group.iloc[pool].unique()))
+    isva = HM.group.iloc[pool].isin(set(grp[: max(1, len(grp) // 10)])).values
+    d_tr, d_va = pool[~isva], pool[isva]
+    c_mv = float(AN["uni2"]["o"][o["train"]].norm(dim=-1).mean() / AN["medvtok"]["o"][o["train"]].norm(dim=-1).mean())
+    mv_norm = float(AN["medvtok"]["o"][o["train"]].norm(dim=-1).mean())
+    rp_norm = float((AN["uni2"]["o"][o["train"]] @ RPM).norm(dim=-1).mean())
+
+    def anc(key, idx, views=None):
+        """anchor rows of all anchor encoders for the same items / views"""
+        out = {}
+        for a in AN:
+            x = AN[a][key]
+            out[a] = x[idx] if x.dim() == 2 else x[idx, views if views is not None else 0]
+        return out
+
+    def batch(task, rng):
+        if task == "mif":
+            b = o_tr[torch.as_tensor(rng.choice(len(o_tr), BS, replace=False), device=dev)]
+            d = torch.randint(0, 8, (BS,), device=dev)
+            return anc("o", b, d), {"mif": nif(IF_O[b, d])}, None
+        if task == "rna":
+            b = h_tr[torch.as_tensor(rng.choice(len(h_tr), BS, replace=False), device=dev)]
+            return anc("h", b), {e: Xr[e]["hest"][b] for e in RNA_E}, None
+        b = trt[torch.as_tensor(rng.choice(len(trt), BS, replace=False), device=dev)]
+        im = T_IMG[b]
+        return (anc("t", im, torch.randint(0, 8, (BS,), device=dev)), {e: Xt[e]["alltxt"][b] for e in TXT_E},
+                (im[:, None] == im[None, :]).float())
+
+    VAL = {"mif": (anc("o", o["val"]), {"mif": nif(IF_O[o["val"], 0])}),
+           "rna": (anc("h", hs["val"]), {e: Xr[e]["hest"][hs["val"]] for e in RNA_E}),
+           "text": (anc("t", vi), {e: Xt[e]["alltxt"][vt] for e in TXT_E})}
+
+    # ------------------------------------------------ stage 1 (frozen anchor, one head per encoder)
+    def train_head(task, enc, kind, steps, an, X_tr_fn=None, X_va=None, A_va=None):
+        torch.manual_seed(k); rng = np.random.default_rng(k)
+        din = VAL[task][1][enc].shape[1]
+        h = (mlp(din, DIM[an]) if kind == "mlp" else nn.Linear(din, DIM[an])).to(dev)
+        logit = nn.Parameter(torch.tensor(np.log(1 / 0.07), dtype=torch.float32, device=dev))
+        opt = torch.optim.AdamW(list(h.parameters()) + [logit], lr=LR, weight_decay=0.05)
+        sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=LR, total_steps=steps, pct_start=0.05)
+        za_va = nz(VAL[task][0][an] if A_va is None else A_va)
+        x_va = VAL[task][1][enc] if X_va is None else X_va
+        best, state = -1, None
+        for step in range(1, steps + 1):
+            h.train()
+            a, X, pos = batch(task, rng) if X_tr_fn is None else X_tr_fn(rng)
+            loss = clip_loss(logit.exp().clamp(max=100) * nz(a[an]) @ nz(h(X[enc])).T, pos)
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+            if step % EVAL_EVERY == 0 or step == steps:
+                h.eval()
+                with torch.no_grad():
+                    v = r1(za_va, nz(h(x_va)))
+                if v > best:
+                    best, state = v, ({k_: x.detach().clone() for k_, x in h.state_dict().items()}, logit.detach().clone())
+        h.load_state_dict(state[0]); h.eval()
+        return h, state[1]
+
+    HEADS, LOGITS = {}, {}
+    for an in AN:
+        HEADS[an], LOGITS[an] = {}, {}
+        HEADS[an]["mif"], LOGITS[an]["mif"] = train_head("mif", "mif", "mlp", 400, an)
+        for e in RNA_E:
+            HEADS[an][f"rna:{e}"], LOGITS[an][f"rna:{e}"] = train_head("rna", e, "linear", 2700, an)
+        for e in TXT_E:
+            HEADS[an][f"text:{e}"], LOGITS[an][f"text:{e}"] = train_head("text", e, "linear", 1000, an)
+    print(f"fold {k} stage 1 done (c_mv {c_mv:.3f})", flush=True)
+
+    # DLPFC external: RNA heads without spatialLIBD, per anchor
+    ZD = {}
+    for an in AN:
+        zd = []
+        for e in RNA_E:
+            Xd = zs(RNA_IN[e], "hest", d_tr, False, ext=["dlpfc"])
+            d_tr_t = torch.as_tensor(d_tr, device=dev)
+
+            def dl_batch(rng, Xd=Xd, e=e, d_tr_t=d_tr_t):
+                b = d_tr_t[torch.as_tensor(rng.choice(len(d_tr_t), BS, replace=False), device=dev)]
+                return anc("h", b), {e: Xd["hest"][b]}, None
+            hd, _ = train_head("rna", e, "linear", 2700, an, dl_batch, Xd["hest"][d_va], AN[an]["h"][d_va])
+            with torch.no_grad():
+                zd.append(nz(hd(Xd["dlpfc"])))
+        ZD[an] = nz(sum(zd))
+
+    # ------------------------------------------------ stage 2
+    keys = ["mif"] + [f"rna:{e}" for e in RNA_E] + [f"text:{e}" for e in TXT_E]
+
+    def val_score(m):
+        m.eval()
+        with torch.no_grad():
+            v = {"mif": r1(nz(m.he(VAL["mif"][0])), nz(m.h("mif")(VAL["mif"][1]["mif"]))),
+                 "rna": r1(nz(m.he(VAL["rna"][0])), m.fuse("rna", RNA_E, VAL["rna"][1])[0]),
+                 "text": r1(nz(m.he(VAL["text"][0])), m.fuse("text", TXT_E, VAL["text"][1])[0])}
+        m.train()
+        return float(np.exp(np.mean(np.log([max(x, 1e-4) for x in v.values()])))), v
+
+    def joint(mode, adapter, steps=1000, lr=1e-4, distill=1.0):
+        m = Joint(HEADS["uni2"], LOGITS["uni2"], adapter, c_mv).to(dev)
+        m.mv_norm, m.rp_norm = mv_norm, rp_norm
+        for kk in keys[1:] if mode == "frz" else keys:
+            m.h(kk).requires_grad_(False)
+        torch.manual_seed(k); rng = np.random.default_rng(k)
+        opt = torch.optim.AdamW([p for p in m.parameters() if p.requires_grad], lr=lr, weight_decay=0.05)
+        sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.05)
+        best, state, bv = -1, None, None
+        m.train()
+        for step in range(1, steps + 1):
+            loss = 0
+            for task in ("mif", "rna", "text"):
+                a, X, pos = batch(task, rng)
+                za = nz(m.he(a))
+                if task == "mif":
+                    loss = loss + clip_loss(m.lg("mif") * za @ nz(m.h("mif")(X["mif"])).T, pos)
+                else:
+                    encs = RNA_E if task == "rna" else TXT_E
+                    _, ze = m.fuse(task, encs, X)
+                    loss = loss + sum(clip_loss(m.lg(f"{task}:{e}") * za @ ze[e].T, pos) for e in encs) / len(encs)
+                u = nz(a["uni2"])
+                loss = loss + distill * F.mse_loss(za @ za.T, u @ u.T)
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sch.step()
+            if step % EVAL_EVERY == 0 or step == steps:
+                sc, v = val_score(m)
+                if sc > best:
+                    best, bv, state = sc, v, {k_: x.detach().clone() for k_, x in m.state_dict().items()}
+        m.load_state_dict(state); m.eval()
+        return m, bv
+
+    @torch.no_grad()
+    def evaluate(m, an):
+        R = lambda st, idx: m.fuse("rna", RNA_E, {e: Xr[e][st][idx] for e in RNA_E})[0]
+        Tx = lambda st, idx: m.fuse("text", TXT_E, {e: Xt[e][st][idx] for e in TXT_E})[0]
+        zm = nz(m.h("mif")(nif(IF_O[o["test"], 0])))
+        zt = Tx("alltxt", tt)
+        res = {("H&E<->mIF (ORION)", "trained"): g100(nz(m.he(anc("o", o["test"]))), zm),
+               ("H&E<->RNA (HEST)", "trained"): g100(nz(m.he(anc("h", hs["test"]))), R("hest", hs["test"])),
+               ("H&E<->text (TCGA)", "trained"): g100(nz(m.he(anc("t", ii))), zt),
+               ("mIF<->RNA(virtual) (ORION)", "emergent"): g100(zm, R("orion_v", o["test"])),
+               ("mIF<->text(synthetic) (ORION)", "emergent"): g100(zm, Tx("orion_gentext", o["test"])),
+               ("text<->RNA(virtual) (TCGA)", "emergent"): g100(zt, R("tcga_v", ii)),
+               ("text<->mIF(MIPHEI) (TCGA)", "emergent"): g100(zt, nz(m.h("mif")(nif(IF_V_T[ii]))))}
+        if m.adapter is None:
+            res[("H&E<->RNA (DLPFC)", "external")] = g100(nz(AN[an]["d"]), ZD[an])
+        return res
+
+    for cname, (an, mode, adapter) in CONF.items():
+        if (cname, k) in done:
+            continue
+        if mode is None:
+            m = Joint(HEADS[an], LOGITS[an], anchor=an).to(dev).eval()  # separate: H&E = the frozen anchor itself
+            val = {}
+        else:
+            m, val = joint(mode, adapter)
+        for (pair, kind), r in evaluate(m, an).items():
+            rows.append({"config": cname, "fold": k, "pair": pair, "kind": kind, **r,
+                         **{f"val_{t}": v for t, v in val.items()}})
+        pd.DataFrame(rows).to_csv(f_out, index=False)
+        print(f"fold {k} {cname:<10} " + "  ".join(f"{r['pair'].split(' (')[0]} {r['R@1']:.3f}"
+                                                   for r in rows if r["config"] == cname and r["fold"] == k), flush=True)
+
+R = pd.DataFrame(rows)
+S = R.groupby(["config", "kind", "pair"], sort=False)[["R@1", "R@10", "pair_acc"]].agg(["mean", "std"]).round(4)
+S.to_csv(os.path.join(ROOT, "results", "medvtok_summary.csv"))
+tests = []
+for c, ref in [("EJ-frz", "E"), ("EJ-ad", "E"), ("EJ-ad", "EJ-frz"), ("MV-E", "E"), ("EJ-frz+MV", "EJ-frz"),
+               ("EJ-ad+MV", "EJ-ad"), ("EJ-frz+MV", "E"), ("EJ-ad+MV", "E"),
+               ("EJ-ad+RP", "EJ-ad"), ("EJ-ad+MV", "EJ-ad+RP")]:
+    for p in R.pair.unique():
+        a = R[(R.config == c) & (R.pair == p)].sort_values("fold")["R@1"].values
+        b = R[(R.config == ref) & (R.pair == p)].sort_values("fold")["R@1"].values
+        if len(a) != len(b) or len(a) < 2:
+            continue
+        tests.append({"config": c, "vs": ref, "pair": p, "ref": b.mean(), "new": a.mean(), "delta": (a - b).mean(),
+                      "folds_won": int((a > b).sum()), "p_paired_t": stats.ttest_rel(a, b).pvalue})
+pd.DataFrame(tests).round(4).to_csv(os.path.join(ROOT, "results", "medvtok_tests.csv"), index=False)
+pd.set_option("display.width", 250)
+print(R.pivot_table(index="config", columns="pair", values="R@1", sort=False).round(3).T.to_string())
+print(R.pivot_table(index="config", columns="pair", values="pair_acc", sort=False).round(3).T.to_string())
+print(pd.DataFrame(tests).round(4).to_string())

experiments/README_v1_orion.md

@@ -0,0 +1,31 @@
+# ImageBind-style 4-modality binding on ORION (H&E / IF / nuclei / markers)
+
+Data: `../ORION/ORIONCRC_dataset_tile_20x` (1% subsample), 3187 paired 333×333 tiles, 41 slides / 40 patients.
+- **he**: H&E RGB → *anchor*, frozen ImageBind-Huge vision encoder (official `imagebind_huge.pth`), embeddings precomputed for the 8 dihedral views
+- **if**: 17-ch multiplex IF → ResNet18 (ImageNet init, 17-ch stem) + linear head to 1024
+- **nuclei**: instance mask → 2-ch (mask, boundary) → ResNet18
+- **markers**: 17 `*_prop` + 16 log1p `*_count` columns → MLP
+
+Loss: `anchor` = symmetric InfoNCE(he, m) for m ∈ {if, nuclei, markers} (ImageBind; IF↔nuclei, IF↔markers, nuclei↔markers are *emergent*);
+`allpairs` = ablation adding the 3 non-anchor pairs. 40 epochs, AdamW 3e-4, bs 128, learnable temperature, model selection on val mean R@1.
+
+Splits (`splits/`): `orion_official` (your csvs; 2954/124/109 tiles, 2 test slides), `random_tile` (tile-level 80/10/10, leaky upper bound),
+`patient_5fold` (patient-grouped 5-fold CV), `batch_lobo` (leave-one-IF-acquisition-batch-out, 4 folds: C59kX/mX/nX/qX).
+
+Metrics (test split, all 12 directed pairs): R@1/5/10, MRR, MedR, one-to-one **pairing accuracy** (Hungarian),
+plus `*_g100` = same on random 100-tile galleries (comparable across splits; chance R@1 = 0.01), and a `test_tissue` subset (≥20 nuclei).
+
+Pipeline: `01_analyze_data.py` → `02_make_splits.py` → `03_cache_data.py` → `04_extract_anchor.py` → `run_all.sh` (`05_train_eval.py` ×22) → `06_summarize.py`.
+Outputs: `results/` (data_summary.json, figures, summary_by_pair.csv, summary_headline.csv, all_runs_long.csv), `runs/*/metrics.json`, test embeddings.
+Env: `.venv` (scileo + tifffile/timm/imagecodecs).
+
+## H&E encoder ablation (frozen anchor; `--anchor`, `04b_extract_anchor_fm.py`, `run_fm.sh <anchor>`)
+| anchor | weights | dim |
+|---|---|---|
+| imagebind | ImageBind-Huge vision (natural images) | 1024 |
+| uni2 | UNI2-h, local copy `reg2026/model_alldata/uni2-h.bin` (HF repo gated for this account) | 1536 |
+| plip | vinid/plip — open pathology CLIP, stand-in for CONCH | 512 |
+| phikonv2 | owkin/phikon-v2 ViT-L — open stand-in for GigaPath | 1024 |
+| conch / gigapath | code ready; HF repos gated (GatedRepoError) — request access, then `04b_extract_anchor_fm.py conch` + `./run_fm.sh conch` | 512 / 1536 |
+
+Results: `results/anchor_ablation_R1_g100.csv`, `results/anchor_ablation_pair_acc.csv`.

experiments/prompts.json

@@ -0,0 +1,9 @@
+{
+ "immune":        {"label": "CD45_count",   "text": "Dense lymphocytic inflammatory infiltrate composed of numerous small round lymphocytes and immune cells."},
+ "tumor_epith":   {"label": "Pan-CK_count", "text": "Malignant glandular epithelium of colorectal adenocarcinoma with crowded, atypical, hyperchromatic nuclei."},
+ "stroma":        {"label": "SMA_count",    "text": "Fibrous desmoplastic stroma with spindle-shaped fibroblasts and smooth muscle bundles."},
+ "proliferation": {"label": "Ki67_count",   "text": "Highly proliferative tumor with frequent mitotic figures and high cellular turnover."},
+ "vessels":       {"label": "CD31_count",   "text": "Numerous small blood vessels and capillaries lined by endothelial cells."},
+ "macrophage":    {"label": "CD68_count",   "text": "Sheets of foamy histiocytes and macrophages within the tissue."},
+ "cellularity":   {"label": "n_nuclei",     "text": "Hypercellular tissue densely packed with nuclei."}
+}

experiments/run_all.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Full experiment grid: 4 split schemes (11 folds) x 2 losses. Two workers on one GPU.
+cd "$(dirname "$0")"; mkdir -p logs
+jobs=()
+for loss in anchor allpairs; do
+  jobs+=("orion_official 0 $loss" "random_tile 0 $loss")
+  for k in 0 1 2 3 4; do jobs+=("patient_5fold $k $loss"); done
+  for k in 0 1 2 3; do jobs+=("batch_lobo $k $loss"); done
+done
+printf '%s\n' "${jobs[@]}" | xargs -P 2 -L 1 bash -c '
+  [ -f runs/${0}_fold${1}_${2}/metrics.json ] && exit 0
+  .venv/bin/python 05_train_eval.py --split $0 --fold $1 --loss $2 > logs/${0}_fold${1}_${2}.log 2>&1 \
+    && echo "DONE $0 $1 $2" || echo "FAILED $0 $1 $2"'
+.venv/bin/python 06_summarize.py > logs/summary.log 2>&1; echo ALLDONE

experiments/run_bench.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Encoder benchmark: 3 configs (one encoder per modality each; modalities train independently) x 6 split/folds.
+cd "$(dirname "$0")"; mkdir -p logs/bench
+cfgs=("resnet18 linear plip" "convnext_tiny mlp biomedclip" "chanset ftt imagebind")
+jobs=()
+for c in "${cfgs[@]}"; do
+  jobs+=("orion_official 0 $c")
+  for k in 0 1 2 3 4; do jobs+=("patient_5fold $k $c"); done
+done
+printf '%s\n' "${jobs[@]}" | xargs -P 2 -L 1 bash -c '
+  tag=${0}_fold${1}_uni2_${2}_${3}_${4}
+  [ -f runs/bench/$tag/test_embeddings.pt ] && exit 0
+  .venv/bin/python 09_bench.py --split $0 --fold $1 --if_enc $2 --mk_enc $3 --txt_enc $4 > logs/bench/$tag.log 2>&1 \
+    && echo "DONE $tag" || echo "FAILED $tag"'
+echo ALLDONE