Spaces:
Running on Zero
Running on Zero
| """Link mode: a region of an open-data segment, fetched by the Space itself, for people who only have a screenshot. | |
| Most screenshots in community questions come from a released segment. Instead of guessing from the picture (single | |
| layers score at chance, see crackle.py), the person names the segment and the spot, and this module reads that | |
| region's real layers from one of the segment's surface volumes in the Vesuvius Challenge open-data bucket, pools | |
| them toward the readers' 9 um grid and hands the stack to the usual check. Nothing is uploaded and nothing is kept. | |
| scroll, segment = parse("PHerc0841/segments/20260220213127-w00") # or a bucket URL, or "PHerc0841 SEGMENT" | |
| vols = volumes(scroll, segment) # its surface volumes, cheapest first | |
| stack, um, note = region(scroll, segment, vols[0]["name"], x, y) # x, y: the center, in that volume's pixels | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import os | |
| import re | |
| import urllib.parse | |
| import numpy as np | |
| import crackle | |
| import engine | |
| import volio | |
| BUCKET = volio.BUCKET | |
| SIDE = 384 # default region side on the readers' grid: about 3.6 mm at 9.4 um | |
| MIN_SIDE, MAX_SIDE = 320, 640 # 320 px leaves the 100,000 papyrus pixels the gate needs; 640 is the examples' size | |
| MAX_FETCH_MB = 900 # a region whose chunks add up to more is refused before any read | |
| # Scrolls the Space does not fetch, comma-separated. Chase chose on 22 Oct 2026 to allow every scroll in the open data, | |
| # so none is listed by default. | |
| BLOCKED = tuple(s.strip() for s in os.environ.get("CRACKLE_LINK_BLOCK", "").split(",") if s.strip()) | |
| _SCROLL = r"(PHerc[A-Za-z0-9]+)" | |
| _SEGMENT = r"([A-Za-z0-9][A-Za-z0-9_.-]{0,127})" | |
| class LinkError(crackle.CheckError): | |
| """Shown to the person as it is.""" | |
| def parse(text): | |
| """(scroll, segment) from 'SCROLL/segments/SEGMENT', a bucket https or s3 URL to it (or anything inside it), or | |
| 'SCROLL SEGMENT'.""" | |
| t = str(text or "").strip() | |
| t = re.sub(r"^(https?://[^/]+|s3://[^/]+)/", "", t) | |
| t = urllib.parse.unquote(t).strip("/ ") | |
| m = re.match(rf"^{_SCROLL}/segments/{_SEGMENT}(?:/.*)?$", t) or re.match(rf"^{_SCROLL}[ /]+{_SEGMENT}$", t) | |
| if not m: | |
| raise LinkError("Give the segment as SCROLL/segments/SEGMENT, for example " | |
| "PHerc0841/segments/20260220213127-w00, or paste its open-data URL.") | |
| scroll, segment = m.group(1), m.group(2) | |
| if scroll in BLOCKED: | |
| raise LinkError(f"{scroll} is not offered in link mode here. Export the layers yourself and upload them.") | |
| return scroll, segment | |
| def volume_um(name): | |
| """The pixel size of a surface volume from its name: '9.366um-...' is 9.366 um; a '-L1' level doubles it.""" | |
| m = re.match(r"^(\d+(?:\.\d+)?)um", name) | |
| if not m: | |
| return None | |
| lvl = re.search(r"-L(\d+)\.zarr/?$", name) | |
| return round(float(m.group(1)) * (2 ** int(lvl.group(1)) if lvl else 1), 4) | |
| def _open(url, fetch, opener): | |
| try: | |
| return opener(url, fetch) | |
| except volio.VolumeError as exc: | |
| raise LinkError(f"Could not read {url.rsplit('/', 1)[-1]} from the open data: {str(exc)[:200]}") from None | |
| def _ls(prefix, fetch=volio._get): | |
| """The sub-folders of a bucket prefix (S3 ListObjectsV2 with a delimiter).""" | |
| out, token = [], None | |
| while True: | |
| q = {"list-type": "2", "prefix": prefix, "delimiter": "/"} | |
| if token: | |
| q["continuation-token"] = token | |
| try: | |
| raw = fetch(f"{BUCKET}/?{urllib.parse.urlencode(q)}") | |
| except volio.VolumeError as exc: | |
| raise LinkError(f"Could not list the open data just now ({str(exc)[:200]}). Try again in a minute.") from None | |
| text = (raw or b"").decode("utf-8", "replace") | |
| out += [p for p in re.findall(r"<Prefix>([^<]*)</Prefix>", text) if p != prefix] | |
| m = re.search(r"<NextContinuationToken>([^<]*)</NextContinuationToken>", text) | |
| if not m: | |
| return out | |
| token = m.group(1) | |
| def _cost(shape, chunks, um, side=SIDE): | |
| """(source side px, pool, pooled layers, MB the region's chunks hold) for a side x side region on the 9.4 um grid.""" | |
| pool = volio.pool_factor(um) | |
| src = max(1, round(side * crackle.MODEL_UM / um)) | |
| n = math.ceil(src / chunks[1] + 1) * math.ceil(src / chunks[2] + 1) | |
| return src, pool, shape[0] // pool, n * math.prod(chunks) / 1e6 | |
| def volumes(scroll, segment, fetch=volio._get, opener=volio.open_zarr): | |
| """The segment's surface volumes the check can use, cheapest region first: name, url, um, shape, pool, layers | |
| after pooling and the MB a default region reads. Volumes with too few layers after pooling are listed as unusable.""" | |
| base = f"{scroll}/segments/{segment}/surface-volumes/" | |
| names = [p[len(base):].rstrip("/") for p in _ls(base, fetch) if p.startswith(base) and len(p) > len(base)] | |
| if not names: | |
| raise LinkError(f"No surface volumes found for {scroll}/segments/{segment} in the open data. Check the " | |
| f"segment name, or export the layers yourself and upload them.") | |
| out = [] | |
| for name in names: | |
| um = volume_um(name) | |
| url = f"{BUCKET}/{base}{name}" | |
| try: | |
| arr = opener(url, fetch) | |
| except volio.VolumeError as exc: | |
| out.append({"name": name, "url": url, "um": um, "usable": False, "why": str(exc)[:200]}) | |
| continue | |
| if um is None: | |
| out.append({"name": name, "url": url, "um": None, "usable": False, "why": "no pixel size in its name"}) | |
| continue | |
| src, pool, layers, mb = _cost(arr.shape, arr.chunks, um) | |
| ok = layers >= engine.DEPTH and getattr(arr, "dtype", np.dtype("u1")) == np.dtype("u1") | |
| out.append({"name": name, "url": url, "um": um, "shape": list(arr.shape), "chunks": list(arr.chunks), | |
| "pool": pool, "layers": layers, "region_mb": round(mb), "usable": ok, | |
| "why": "" if ok else f"{layers} layers after pooling, the readers need {engine.DEPTH}"}) | |
| out.sort(key=lambda v: (not v["usable"], v.get("region_mb", 1e9))) | |
| return out | |
| def label(v): | |
| """One line for a volume picker.""" | |
| if not v["usable"]: | |
| return f"{v['name']} (not usable: {v['why']})" | |
| h, w = v["shape"][1:] | |
| return f"{v['name'].split('-volume')[0].removesuffix('.zarr')}, {w} x {h} px, {v['region_mb']} MB a check" | |
| def region(scroll, segment, name, x, y, side=SIDE, fetch=volio._get, opener=volio.open_zarr): | |
| """(stack uint8 layers x rows x columns on about the 9.4 um grid, um per pixel after pooling, note) for the | |
| side x side region centered on column x, row y of the named surface volume (its own pixels). A region that runs | |
| past an edge is moved inside it and the note says so.""" | |
| side = int(side) | |
| if not MIN_SIDE <= side <= MAX_SIDE: | |
| raise LinkError(f"size {side}: use {MIN_SIDE} to {MAX_SIDE} px") | |
| if not re.match(r"^[A-Za-z0-9][A-Za-z0-9_.-]*\.zarr$", name or ""): | |
| raise LinkError("Pick one of the segment's surface volumes first.") | |
| um = volume_um(name) | |
| if um is None: | |
| raise LinkError(f"{name} has no pixel size in its name") | |
| url = f"{BUCKET}/{scroll}/segments/{segment}/surface-volumes/{name}" | |
| arr = _open(url, fetch, opener) | |
| src, pool, layers, mb = _cost(arr.shape, arr.chunks, um, side) | |
| if layers < engine.DEPTH: | |
| raise LinkError(f"{name} has {arr.shape[0]} layers, {layers} after pooling to about 9 um; the readers need " | |
| f"{engine.DEPTH}. Pick another surface volume.") | |
| if mb > MAX_FETCH_MB: | |
| raise LinkError(f"That region would read about {mb:.0f} MB from {name}; pick a smaller size or a coarser volume.") | |
| L, H, W = arr.shape | |
| if src > min(H, W): | |
| raise LinkError(f"{name} is {W} x {H} px, smaller than a {src} px region") | |
| try: | |
| x, y = int(round(float(x))), int(round(float(y))) | |
| except (TypeError, ValueError): | |
| raise LinkError("Give the spot as whole numbers: x (column) and y (row) in the volume's pixels.") from None | |
| if not (0 <= x < W and 0 <= y < H): | |
| raise LinkError(f"x {x}, y {y} is outside {name}, which is {W} x {H} px (x 0 to {W - 1}, y 0 to {H - 1})") | |
| x0 = min(max(0, x - src // 2), W - src) | |
| y0 = min(max(0, y - src // 2), H - src) | |
| try: | |
| data = arr.read((0, L), (y0, y0 + src), (x0, x0 + src)) | |
| except volio.VolumeError as exc: | |
| raise LinkError(f"Could not read that region from the open data: {str(exc)[:200]}") from None | |
| data = volio.to_uint8(data) | |
| # pooled k layers at a time, so the float copy stays small (a 640 px region of a 2.4 um volume is 0.7 GB raw) | |
| stack = np.concatenate([volio.pool(data[i:i + pool], pool) for i in range(0, L // pool * pool, pool)]) | |
| if stack.shape[0] > engine.MAX_LAYERS: | |
| c = stack.shape[0] // 2 | |
| stack = stack[c - engine.MAX_LAYERS // 2:c - engine.MAX_LAYERS // 2 + engine.MAX_LAYERS] | |
| moved = (x0 + src // 2, y0 + src // 2) != (x, y) | |
| note = (f"open data {scroll}/segments/{segment}/surface-volumes/{name}, center x {x0 + src // 2}, " | |
| f"y {y0 + src // 2}, {src} px at {um} um" + (f", pooled {pool} x {pool} x {pool}" if pool > 1 else "") | |
| + (" (moved inside the volume's edge)" if moved else "")) | |
| return np.ascontiguousarray(stack), round(um * pool, 4), note | |