File size: 6,389 Bytes
9a5439c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """H&E patches of the 9,484 HEST training-pool spots (cache/hest/meta.csv), so that image encoders other than the
precomputed UNI2 / GigaPath features can be run on them (e.g. MedVTok).
Run: HF_TOKEN=... .venv/bin/python -I 49_hest_patches.py [--workers 16] [--only <sample> ...]
Source: MahmoodLab/hest wsis/<sample>.tif (pyramidal tiled TIFF), level 0, read with HTTP range requests on the
resolved CDN URL (RangeFile) - only the tiles under the needed spots. (The patches/<sample>.h5 files cannot be read
selectively: their barcode chunks are interleaved with the image chunks, i.e. finding the rows reads the whole file.)
Patch = 224 x 224 level-0 px centred on the spot centre (obsm['spatial'] = full-res (x, y), cache/hest/spot_coords.csv),
no resizing: this is exactly the image behind the cached UNI2 features (re-encoding these crops with UNI2-h gives
cos 1.000 to cache/hest/he_uni2.npy, checked on TENX152 / MEND139; 0.5-um/px resampled crops give only 0.66-0.81).
Pixels outside the slide are white. Resumable per sample.
Output cache/hest/patches/<sample>.npz (img, idx) and, once all are present, cache/hest/he_img.npy (9484, 224, 224, 3)
uint8 in meta.csv row order (+ cache/hest/he_img_missing.csv, spots whose slide could not be read).
"""
import io
import os
import sys
from concurrent.futures import ThreadPoolExecutor
import numpy as np
import pandas as pd
import requests
import tifffile
ROOT = os.path.dirname(os.path.abspath(__file__))
H = os.path.join(ROOT, "cache", "hest")
OUT = os.path.join(H, "patches")
os.makedirs(OUT, exist_ok=True)
WORKERS = int(sys.argv[sys.argv.index("--workers") + 1]) if "--workers" in sys.argv else 16
ONLY = sys.argv[sys.argv.index("--only") + 1:] if "--only" in sys.argv else None
M = pd.read_csv(os.path.join(H, "meta.csv"))
C = pd.read_csv(os.path.join(H, "spot_coords.csv"))
assert (C.idx.values == M.idx.values).all()
class RangeFile(io.RawIOBase):
"""Read-only, seekable file over HTTP range requests (block cache), for h5py. huggingface_hub's HfFileSystem hung
in this environment (2026-10-28), plain range requests on the resolved CDN URL are fast."""
BLOCK = 256 * 1024
def __init__(self, url):
hdr = {"Authorization": f"Bearer {os.environ['HF_TOKEN']}"}
r = requests.get(url, headers={**hdr, "Range": "bytes=0-0"}, timeout=60)
r.raise_for_status()
self.url = r.url # signed CDN URL after redirects
self.size = int(r.headers["Content-Range"].split("/")[1])
self.pos, self.cache, self.s = 0, {}, requests.Session()
def _block(self, i):
if i not in self.cache:
a = i * self.BLOCK
for t in range(5):
try:
r = self.s.get(self.url, headers={"Range": f"bytes={a}-{min(self.size, a + self.BLOCK) - 1}"}, timeout=60)
r.raise_for_status()
break
except requests.RequestException:
if t == 4:
raise
self.cache[i] = r.content
return self.cache[i]
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.pos
def seek(self, off, whence=0):
self.pos = off if whence == 0 else self.pos + off if whence == 1 else self.size + off
return self.pos
def readinto(self, b):
n = min(len(b), max(0, self.size - self.pos))
out, p = bytearray(), self.pos
while len(out) < n:
blk = self._block(p // self.BLOCK)
o = p % self.BLOCK
take = blk[o:o + n - len(out)]
out += take; p += len(take)
b[:n] = out
self.pos = p
return n
def fetch(sid):
out = os.path.join(OUT, f"{sid}.npz")
if os.path.exists(out):
return sid, "cached"
c = C[C["sample"] == sid]
f = RangeFile(f"https://huggingface.co/datasets/MahmoodLab/hest/resolve/main/wsis/{sid}.tif")
with tifffile.TiffFile(f) as tf:
pg = tf.pages[0]
Hh, W = pg.shape[:2]
th, tw = pg.tilelength, pg.tilewidth
ntx = -(-W // tw)
S = 224
tiles = {}
def tile(ty, tx):
if (ty, tx) not in tiles:
i = ty * ntx + tx
f.seek(pg.dataoffsets[i])
a = pg.decode(f.read(pg.databytecounts[i]), i, jpegtables=pg.jpegtables)[0]
tiles[(ty, tx)] = a.reshape(th, tw, -1)[..., :3]
return tiles[(ty, tx)]
imgs = []
for x, y in zip(c.x.values, c.y.values):
x0, y0 = int(round(x - S / 2)), int(round(y - S / 2))
reg = np.full((S, S, 3), 255, np.uint8)
for ty in range(max(0, y0 // th), min(-(-Hh // th), (y0 + S - 1) // th + 1)):
for tx in range(max(0, x0 // tw), min(ntx, (x0 + S - 1) // tw + 1)):
a = tile(ty, tx)
ys, xs = max(y0, ty * th), max(x0, tx * tw)
ye, xe = min(y0 + S, (ty + 1) * th, Hh), min(x0 + S, (tx + 1) * tw, W)
reg[ys - y0:ye - y0, xs - x0:xe - x0] = a[ys - ty * th:ye - ty * th, xs - tx * tw:xe - tx * tw]
imgs.append(reg)
np.savez(out + ".tmp.npz", img=np.stack(imgs), idx=c.idx.values)
os.replace(out + ".tmp.npz", out)
return sid, f"{len(imgs)} spots, crop {S}px, {len(tiles)} tiles, {sum(len(v) for v in f.cache.values()) / 1e6:.0f} MB"
samples = ONLY or M["sample"].unique().tolist()
with ThreadPoolExecutor(WORKERS) as ex:
futs = {s: ex.submit(fetch, s) for s in samples}
for n, (sid, fu) in enumerate(futs.items()):
try:
print(f"[{n + 1}/{len(samples)}] {sid}: {fu.result()[1]}", flush=True)
except Exception as e: # noqa: BLE001 - reported, spots become 'missing'
print(f"[{n + 1}/{len(samples)}] {sid}: FAILED {type(e).__name__}: {e}", flush=True)
if ONLY:
sys.exit()
img = np.zeros((len(M), 224, 224, 3), np.uint8)
have = np.zeros(len(M), bool)
for sid in M["sample"].unique():
p = os.path.join(OUT, f"{sid}.npz")
if os.path.exists(p):
z = np.load(p)
img[z["idx"]] = z["img"]; have[z["idx"]] = True
np.save(os.path.join(H, "he_img.npy"), img)
pd.DataFrame({"idx": np.flatnonzero(~have)}).to_csv(os.path.join(H, "he_img_missing.csv"), index=False)
print(f"saved he_img.npy {img.shape}; missing {(~have).sum()} spots")
|