| """External H&E<->RNA test sets: DLPFC (spatialLIBD Visium) and breast IDC (HEST Xenium pseudo-spots). |
| |
| Run: .venv/bin/python -I 36_external_prep.py [dlpfc] [idc] (default: both) |
| Needs the public spatialLIBD files in ext/external_raw/dlpfc_spatialLIBD/ (downloaded once, data only): |
| <sid>_filtered_feature_bc_matrix.h5 https://spatial-dlpfc.s3.us-east-2.amazonaws.com/h5/ |
| <sid>_tissue_positions_list.txt https://raw.githubusercontent.com/LieberInstitute/HumanPilot/master/10X/<sid>/ |
| <sid>_scalefactors_json.json (same) |
| |
| Findings this script relies on (verified, numbers printed again at run time): |
| DLPFC visium_gene_expression_list.pkl = log1p(count * 3034 / total_counts_all_33538_genes) (3034 = median |
| total over all 47,681 spots, i.e. scanpy normalize_total default) restricted to the top-10000 |
| scanpy 'seurat'-flavour HVGs computed on all 12 samples concatenated, columns in 10x feature (genome) order. |
| Rows = h5 barcode order of each sample, samples concatenated in sorted order (manifest spot_idx). |
| Images = ImageNet-normalised uint8 RGB, native 224x224 full-res crop centred on the spot. |
| Full-res scale: Visium centre-to-centre pitch 100 um = 138.0 px -> 0.7246 um/px (HEST: 0.725); |
| note the 'pxl_row/col' columns of the HumanPilot tissue_positions_list are (x, y), not (row, col). |
| IDC rebuilt from HEST-1k (MahmoodLab/hest): st/<id>.h5ad + the selected 224-px, 0.5 um/px patches of |
| patches/<id>.h5 for TENX95, TENX99, NCBI783, NCBI785 (= HEST-Bench IDC), paired by barcode and checked by |
| patch-centre vs spot-centre distance. Spots: in tissue, with a patch, >= 20 panel counts; > 1500 -> random |
| 1500 (seed 0). The earlier IDC built from ../IDC/UNI was unusable: subsample.py drew its image indices |
| independently of its adata indices (key '/k/__list__' vs '/k/__arr__', hash()-randomised seed). |
| First run fetches patches with HTTP range reads: needs HF_TOKEN in the environment. |
| Outputs cache/external/: see the OUT dict printed at the end. |
| """ |
| import json |
| import os |
| import pickle |
| import sys |
|
|
| import h5py |
| import numpy as np |
| import pandas as pd |
| import scipy.sparse as sp |
| from PIL import Image |
|
|
| ROOT = os.path.dirname(os.path.abspath(__file__)) |
| DATA = os.path.abspath(os.path.join(ROOT, "..")) |
| RAW = os.path.join(ROOT, "ext", "external_raw", "dlpfc_spatialLIBD") |
| OUT = os.path.join(ROOT, "cache", "external") |
| os.makedirs(OUT, exist_ok=True) |
| IMNET_MEAN = np.array([0.485, 0.456, 0.406], np.float32) |
| IMNET_STD = np.array([0.229, 0.224, 0.225], np.float32) |
| TARGET_UM = 0.5 |
| HVG = pd.read_csv(os.path.join(ROOT, "cache", "hest", "hvg_genes.csv"))["gene"].astype(str).tolist() |
| HEST_META = pd.read_csv(os.path.join(DATA, "HEST", "HEST_v1_1_0.csv")) |
| HEST_TRAIN = set(pd.read_csv(os.path.join(ROOT, "cache", "hest", "meta.csv"), usecols=["sample"])["sample"]) |
|
|
|
|
| def load_pickle(p): |
| with open(p, "rb") as f: |
| return pickle.load(f) |
|
|
|
|
| def denorm(x): |
| """(N,3,H,W) ImageNet-normalised float -> (N,H,W,3) uint8; returns max dequantisation error (in 0..255 units).""" |
| v = (np.asarray(x, np.float32).transpose(0, 2, 3, 1) * IMNET_STD + IMNET_MEAN) * 255.0 |
| u = np.clip(np.round(v), 0, 255).astype(np.uint8) |
| return u, float(np.abs(v - u).max()) |
|
|
|
|
| def resample(u8, um_native, fov_um=224 * TARGET_UM, out=224): |
| """Centre crop fov_um (if available) and resize to out x out => TARGET_UM um/px.""" |
| res = [] |
| for im in u8: |
| side = fov_um / um_native |
| h = im.shape[0] |
| if side > h: |
| side = h |
| c = (h - side) / 2 |
| res.append(np.asarray(Image.fromarray(im).resize((out, out), Image.BICUBIC, box=(c, c, c + side, c + side)))) |
| return np.stack(res) |
|
|
|
|
| def hvg_overlap(genes): |
| g = set(genes) |
| return int(sum(x in g for x in HVG)) |
|
|
|
|
| |
| def read_h5(sid): |
| with h5py.File(os.path.join(RAW, f"{sid}_filtered_feature_bc_matrix.h5"), "r") as f: |
| g = f["matrix"] |
| M = sp.csc_matrix((g["data"][:], g["indices"][:], g["indptr"][:]), shape=g["shape"][:]).T.tocsr() |
| bcs = [b.decode() for b in g["barcodes"][:]] |
| names = [b.decode() for b in g["features"]["name"][:]] |
| ids = [b.decode() for b in g["features"]["id"][:]] |
| return M, bcs, names, ids |
|
|
|
|
| def seurat_hvg(L, n_top): |
| """scanpy.pp.highly_variable_genes(flavor='seurat', n_top_genes=n_top) on log1p data L (csr).""" |
| E = L.copy() |
| E.data = np.expm1(E.data) |
| n = E.shape[0] |
| mean = np.asarray(E.mean(0)).ravel() |
| var = (np.asarray(E.multiply(E).mean(0)).ravel() - mean ** 2) * n / (n - 1) |
| with np.errstate(all="ignore"): |
| disp = var / mean |
| disp[disp == 0] = np.nan |
| disp = np.log(disp) |
| df = pd.DataFrame({"m": np.log1p(mean), "d": disp}) |
| df["bin"] = pd.cut(df.m, bins=20) |
| g = df.groupby("bin", observed=False)["d"] |
| bm, bs, one = g.transform("mean"), g.transform("std"), g.transform("count") == 1 |
| bs, bm = bs.where(~one, bm), bm.where(~one, 0) |
| dn = ((df.d - bm) / bs).to_numpy() |
| v = np.sort(dn[~np.isnan(dn)])[::-1] |
| cut = v[n_top - 1] |
| print(f" HVG cut-off normalised dispersion {cut:.5f}; next {v[n_top]:.5f} (gap {cut - v[n_top]:.2e})") |
| return np.where(np.nan_to_num(dn) >= cut)[0] |
|
|
|
|
| def dlpfc(): |
| print("== DLPFC") |
| P = os.path.join(DATA, "DLPFC_visium", "GPFM") |
| man = json.load(open(os.path.join(DATA, "_manifest", "dlpfc.json"))) |
| samples = man["samples"] |
| X = load_pickle(os.path.join(P, "visium_gene_expression_list.pkl")) |
| X = np.asarray(X.numpy() if hasattr(X, "numpy") else X, np.float64) |
| gidx = np.asarray(pd.read_csv(os.path.join(DATA, "DLPFC_visium", "global_spot_index.csv"))["global_spot_idx"]) |
| assert np.array_equal(gidx, man["global_spot_idx"]) and X.shape == (len(gidx), 10000) |
|
|
| Ms, rows, names, ids = [], [], None, None |
| sf_by, pos_by, bcs_by = {}, {}, {} |
| for sid in samples: |
| M, bcs, names, ids = read_h5(sid) |
| assert M.shape[0] == man["n_spots"][sid], sid |
| Ms.append(M) |
| bcs_by[sid] = bcs |
| sf_by[sid] = json.load(open(os.path.join(RAW, f"{sid}_scalefactors_json.json"))) |
| pos = pd.read_csv(os.path.join(RAW, f"{sid}_tissue_positions_list.txt"), header=None, |
| names=["barcode", "in_tissue", "array_row", "array_col", "px_x", "px_y"]).set_index("barcode") |
| pos_by[sid] = pos |
| for i in man["spot_idx"][sid]: |
| rows.append((sid, i, bcs[i])) |
| M = sp.vstack(Ms).tocsr().astype(np.float64) |
| tot = np.asarray(M.sum(1)).ravel() |
| target = float(np.median(tot)) |
| A = M[gidx] |
| print(f" all spots {M.shape}, median total counts {target}") |
|
|
| |
| L = sp.diags(target / tot) @ M |
| L.data = np.log1p(L.data) |
| |
| sel = seurat_hvg(L, 10000) |
| assert len(sel) == 10000 |
| Lsel = L[gidx][:, sel].toarray() |
| err = np.abs(Lsel - X).max() |
| |
| Ad = A.toarray() |
| K = np.round(np.expm1(X) * (tot[gidx] / target)[:, None]) |
| buckets = {} |
| for j in range(Ad.shape[1]): |
| buckets.setdefault(Ad[:, j].tobytes(), []).append(j) |
| cand = [buckets.get(K[:, c].tobytes(), []) for c in range(10000)] |
| n_unique = sum(len(c) == 1 for c in cand) |
| n_consistent = sum(sel[c] in cand[c] for c in range(10000)) |
| n_unique_ok = sum(len(cand[c]) == 1 and cand[c][0] == sel[c] for c in range(10000)) |
| print(f" max |log-expr(reconstructed) - pickle| = {err:.2e}") |
| print(f" exact-value column matches: {n_unique}/10000 columns match exactly one h5 gene " |
| f"({n_unique_ok} of them = HVG reconstruction); {n_consistent}/10000 HVG genes value-consistent; " |
| f"{int((K.sum(0) == 0).sum())} columns all-zero in the 482 spots") |
| assert err < 1e-5 and n_consistent == 10000 |
|
|
| genes = pd.DataFrame({"col": np.arange(10000), "gene": np.array(names)[sel], "ensembl_id": np.array(ids)[sel], |
| "h5_feature_idx": sel, |
| "match": ["exact_unique" if len(c) == 1 else "exact_ambiguous_resolved_by_hvg" for c in cand]}) |
| genes["duplicated_symbol"] = genes.gene.duplicated(keep=False) |
| genes.to_csv(os.path.join(OUT, "dlpfc_genes.csv"), index=False) |
| np.save(os.path.join(OUT, "dlpfc_counts.npy"), Ad[:, sel].astype(np.float32)) |
| |
| np.save(os.path.join(OUT, "dlpfc_counts_full.npy"), Ad.astype(np.float32)) |
| pd.DataFrame({"gene": names, "ensembl_id": ids, "in_10k": np.isin(np.arange(len(names)), sel)}).to_csv( |
| os.path.join(OUT, "dlpfc_genes_full.csv"), index=False) |
|
|
| |
| im = load_pickle(os.path.join(P, "visium_image_list.pkl")) |
| im = im.numpy() if hasattr(im, "numpy") else np.asarray(im) |
| u8, qerr = denorm(im) |
| print(f" images {im.shape}: ImageNet-normalised, dequantisation error {qerr:.1e} (exact uint8)") |
| np.save(os.path.join(OUT, "dlpfc_he.npy"), u8) |
|
|
| hest_ids = dict(zip(HEST_META.subseries.astype(str), HEST_META.id)) |
| meta = [] |
| for k, (sid, i, bc) in enumerate(rows): |
| p = pos_by[sid] |
| q = p.reset_index() |
| nb = q.merge(q, on="array_row") |
| nb = nb[nb.array_col_y == nb.array_col_x + 2] |
| pitch = float(np.median(np.hypot(nb.px_x_x - nb.px_x_y, nb.px_y_x - nb.px_y_y))) |
| hid = hest_ids.get(sid) |
| meta.append(dict(sample=sid, spot_idx=i, global_spot_idx=int(gidx[k]), barcode=bc, |
| array_row=int(p.loc[bc, "array_row"]), array_col=int(p.loc[bc, "array_col"]), |
| x_fullres=float(p.loc[bc, "px_x"]), y_fullres=float(p.loc[bc, "px_y"]), |
| organ="Brain", tissue="dorsolateral prefrontal cortex", |
| um_per_px=100.0 / pitch, fov_um=224 * 100.0 / pitch, |
| spot_diameter_fullres=sf_by[sid]["spot_diameter_fullres"], |
| total_counts=float(tot[gidx[k]]), hest_id=hid, in_hest_train_cache=hid in HEST_TRAIN)) |
| meta = pd.DataFrame(meta) |
| meta.to_csv(os.path.join(OUT, "dlpfc_meta.csv"), index=False) |
| um = float(meta.um_per_px.median()) |
| u05 = resample(u8, um) |
| np.save(os.path.join(OUT, "dlpfc_he_05um.npy"), u05) |
| print(f" um/px {um:.4f} -> patch FOV {224 * um:.1f} um; dlpfc_he_05um = centre {112 / um:.1f}px crop -> 224 (112 um)") |
| print(f" HEST-HVG overlap: 10k cols {hvg_overlap(genes.gene)}/{len(HVG)}, full {hvg_overlap(names)}/{len(HVG)}") |
|
|
|
|
| |
| HEST_RAW = os.path.join(ROOT, "ext", "external_raw", "hest_idc") |
| IDC_IDS = ["TENX95", "TENX99", "NCBI783", "NCBI785"] |
| IDC_MIN_COUNTS = 20 |
| IDC_MAX_SPOTS = 1500 |
| CTRL = ("BLANK", "NegControl", "Unassigned", "antisense", "DeprecatedCodeword", "Intergenic") |
|
|
|
|
| def fetch_patches(sid, barcodes): |
| """Fetch only the selected patches of MahmoodLab/hest patches/<sid>.h5 by HTTP range reads (the full files are |
| 5.0 GB together; patches are stored uncompressed, one chunk per patch). Auth: HF_TOKEN env var, read by |
| huggingface_hub itself. Writes patches_subset/<sid>.h5 with img/barcode/coords + the source attrs.""" |
| from huggingface_hub import HfFileSystem |
| out = os.path.join(HEST_RAW, "patches_subset", f"{sid}.h5") |
| os.makedirs(os.path.dirname(out), exist_ok=True) |
| fs = HfFileSystem() |
| src = f"datasets/MahmoodLab/hest/patches/{sid}.h5" |
| pb = all_patch_barcodes(sid) |
| coords = pb[["x", "y"]].to_numpy() |
| pos = {b: i for i, b in enumerate(pb.barcode)} |
| rows = np.array([pos[b] for b in barcodes]) |
| order = np.argsort(rows) |
| imgs = np.zeros((len(rows), 224, 224, 3), np.uint8) |
| with fs.open(src, "rb", block_size=160 * 1024, cache_type="readahead") as f, h5py.File(f, "r") as h: |
| attrs = dict(h["img"].attrs) |
| for n, k in enumerate(order): |
| imgs[k] = h["img"][rows[k]] |
| if n % 250 == 0: |
| print(f" {sid}: fetched {n}/{len(rows)} patches", flush=True) |
| with h5py.File(out, "w") as g: |
| g["img"] = imgs |
| g["barcode"] = np.array(barcodes, dtype="S") |
| g["coords"] = coords[rows] |
| g["src_row"] = rows |
| for k, v in attrs.items(): |
| g["img"].attrs[k] = v |
| return out |
|
|
|
|
| def all_patch_barcodes(sid): |
| """Barcodes that have a patch in HEST (read remotely once, cached as a small csv).""" |
| p = os.path.join(HEST_RAW, "patches_subset", f"{sid}_all_barcodes.csv") |
| if not os.path.exists(p): |
| from huggingface_hub import HfFileSystem |
| os.makedirs(os.path.dirname(p), exist_ok=True) |
| with HfFileSystem().open(f"datasets/MahmoodLab/hest/patches/{sid}.h5", "rb", block_size=2 ** 22) as f, \ |
| h5py.File(f, "r") as h: |
| bc = [b[0].decode() if isinstance(b[0], bytes) else str(b[0]) for b in h["barcode"][:]] |
| pd.DataFrame({"barcode": bc, "x": h["coords"][:, 0], "y": h["coords"][:, 1]}).to_csv(p, index=False) |
| return pd.read_csv(p) |
|
|
|
|
| def idc(): |
| print("== IDC (HEST-1k, paired by barcode)") |
| import anndata as ad |
| ads = {s: ad.read_h5ad(os.path.join(HEST_RAW, "st", f"{s}.h5ad")) for s in IDC_IDS} |
| common = set.intersection(*[set(a.var_names) for a in ads.values()]) |
| genes = sorted(g for g in common if not g.startswith(CTRL)) |
| rng = np.random.default_rng(0) |
| he, counts, meta = [], [], [] |
| for sid in IDC_IDS: |
| a = ads[sid] |
| md = json.load(open(os.path.join(HEST_RAW, "metadata", f"{sid}.json"))) |
| X = a[:, genes].X |
| X = np.asarray(X.toarray() if sp.issparse(X) else X, np.float32) |
| tot = X.sum(1) |
| pb = all_patch_barcodes(sid) |
| in_t = a.obs["in_tissue"].astype(bool).values if "in_tissue" in a.obs else np.ones(a.n_obs, bool) |
| has_p = np.isin(a.obs_names, pb.barcode) |
| elig = np.where(in_t & has_p & (tot >= IDC_MIN_COUNTS))[0] |
| sub = len(elig) > IDC_MAX_SPOTS |
| sel = np.sort(rng.choice(elig, IDC_MAX_SPOTS, replace=False)) if sub else elig |
| print(f" {sid}: spots {a.n_obs}, in_tissue {in_t.sum()}, with patch {has_p.sum()}, " |
| f">= {IDC_MIN_COUNTS} counts {len(elig)} -> kept {len(sel)}{' (random subsample, seed 0)' if sub else ''}") |
| bcs = a.obs_names[sel].tolist() |
| p = os.path.join(HEST_RAW, "patches_subset", f"{sid}.h5") |
| if not os.path.exists(p): |
| fetch_patches(sid, bcs) |
| with h5py.File(p, "r") as g: |
| assert [b.decode() for b in g["barcode"][:]] == bcs, f"{sid}: cached patch subset does not match selection" |
| imgs, coords = g["img"][:], g["coords"][:] |
| src_px, ds, mpp = (float(g["img"].attrs[k]) for k in ("patch_size_src", "downsample", "pixel_size")) |
| |
| cx, cy = coords[:, 0] + src_px / 2, coords[:, 1] + src_px / 2 |
| o = a.obs.iloc[sel] |
| dev = np.hypot(cx - o.pxl_col_in_fullres.values, cy - o.pxl_row_in_fullres.values) * mpp |
| um_out = mpp * ds |
| print(f" patch: {int(src_px)} src px @ {mpp:.4f} um/px -> 224 px @ {um_out:.4f} um/px ({224 * um_out:.0f} um FOV); " |
| f"|patch centre - spot centre| max {dev.max():.2f} um, median {np.median(dev):.2f} um") |
| assert dev.max() < 5, f"{sid}: patch/spot coordinate mismatch" |
| he.append(imgs) |
| counts.append(X[sel]) |
| for j, i in enumerate(sel): |
| meta.append(dict(sample=sid, barcode=a.obs_names[i], array_row=int(a.obs.array_row.iloc[i]), |
| array_col=int(a.obs.array_col.iloc[i]), |
| x_fullres=float(a.obs.pxl_col_in_fullres.iloc[i]), |
| y_fullres=float(a.obs.pxl_row_in_fullres.iloc[i]), |
| organ="Breast", oncotree=md.get("oncotree_code"), um_per_px=um_out, |
| um_per_px_fullres=mpp, patch_center_dev_um=float(dev[j]), total_counts=float(tot[i]), |
| subsampled=bool(sub), n_eligible=len(elig), |
| in_hest_train_cache=sid in HEST_TRAIN)) |
| he, counts = np.concatenate(he), np.concatenate(counts) |
| np.save(os.path.join(OUT, "idc_he.npy"), he) |
| np.save(os.path.join(OUT, "idc_counts.npy"), counts) |
| pd.DataFrame({"gene": genes}).to_csv(os.path.join(OUT, "idc_genes.csv"), index=False) |
| pd.DataFrame(meta).to_csv(os.path.join(OUT, "idc_meta.csv"), index=False) |
| for f in ("idc_he_unpaired.npy", "idc_he_unpaired_meta.csv"): |
| if os.path.exists(os.path.join(OUT, f)): |
| os.remove(os.path.join(OUT, f)) |
| hest_common = pd.read_csv(os.path.join(ROOT, "cache", "hest", "genes.csv"))["gene"].astype(str) |
| print(f" genes {len(genes)} (common non-control panel genes); counts {counts.shape}; images {he.shape}") |
| print(f" IDC panel overlap: HEST HVG {hvg_overlap(genes)}/{len(HVG)}; " |
| f"HEST common genes {int(hest_common.isin(genes).sum())}/{len(hest_common)}") |
|
|
|
|
| if __name__ == "__main__": |
| todo = sys.argv[1:] or ["dlpfc", "idc"] |
| if "dlpfc" in todo: |
| dlpfc() |
| if "idc" in todo: |
| idc() |
| for f in sorted(os.listdir(OUT)): |
| p = os.path.join(OUT, f) |
| print(f" {f:28s}", np.load(p, mmap_mode="r").shape if f.endswith(".npy") else f"{len(pd.read_csv(p))} rows") |
|
|
