Spaces:
Sleeping
Sleeping
Download src/io.py from farrell236/OCT_Segmentation_a2: direct link, hf CLI and curl.
- Browser
- Download file 9.34 kB
-
https://huggingface.co/spaces/farrell236/OCT_Segmentation_a2/resolve/main/src/io.py
- Command line
-
hf download hf://spaces/farrell236/OCT_Segmentation_a2/src/io.py
-
curl -L -o io.py https://huggingface.co/spaces/farrell236/OCT_Segmentation_a2/resolve/main/src/io.py
9.34 kB
| """Structural OCT pixels, derived cache paths, and prediction receipts.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| from pathlib import Path, PurePosixPath | |
| import numpy as np | |
| from pydicom.filereader import read_partial | |
| from pydicom.pixels import iter_pixels | |
| from pydicom.tag import Tag | |
| OCT_SOP = "1.2.840.10008.5.1.4.1.1.77.1.5.4" | |
| def safe_path(root, value): | |
| p = PurePosixPath(value) | |
| if p.is_absolute() or ".." in p.parts or not value or "\\" in value: | |
| raise ValueError("Unsafe relative dataset path") | |
| return Path(root).joinpath(*p.parts) | |
| def read_input(path): | |
| rows = [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()] | |
| if not rows: | |
| raise ValueError("Empty input manifest") | |
| for r in rows: | |
| if set(r) != {"case_id", "device", "oct_paths"}: | |
| raise ValueError("Prediction input must be label free") | |
| if len({r["case_id"] for r in rows}) != len(rows): | |
| raise ValueError("Duplicate inference case ids") | |
| return rows | |
| def metadata_header(path, specific_tags=None): | |
| """Stop before PerFrameFunctionalGroups: no payload or unused sequence parsing. | |
| All source identification, geometry and SharedFunctionalGroups precede this tag. | |
| Per-frame geometry is inspected separately by the discovery/audit programs. | |
| """ | |
| tags = [Tag(x) for x in specific_tags] if specific_tags else None | |
| with Path(path).open("rb") as fp: | |
| return read_partial( | |
| fp, | |
| stop_when=lambda tag, vr, length: int(tag) >= 0x52009230, | |
| specific_tags=tags, | |
| ) | |
| def structural_header(path): | |
| d = metadata_header( | |
| path, | |
| [ | |
| "SOPClassUID", | |
| "Modality", | |
| "NumberOfFrames", | |
| "Rows", | |
| "Columns", | |
| "PhotometricInterpretation", | |
| "ConcatenationFrameOffsetNumber", | |
| ], | |
| ) | |
| if str(d.SOPClassUID) != OCT_SOP or str(d.Modality) != "OPT": | |
| raise ValueError("Input is not structural OCT") | |
| return d | |
| def normalize_frame(frame): | |
| """No fitted statistics: deterministic robust intensity scaling per input B-scan.""" | |
| x = np.asarray(frame, dtype=np.float32) | |
| sample = x[::4, ::4] | |
| lo, hi = np.percentile(sample, [1, 99.5]) | |
| return np.clip((x - lo) / max(float(hi - lo), 1e-6), 0, 1).astype("float32") | |
| def resize_frames(frames, height, width, device="cpu"): | |
| import torch | |
| from torch.nn import functional as F | |
| x = torch.from_numpy(np.stack([normalize_frame(f) for f in frames]))[:, None].to( | |
| device | |
| ) | |
| return F.interpolate(x, size=(height, width), mode="bilinear", align_corners=True) | |
| def context_batch(pixels, start, end, radius, device): | |
| """Build float32 image channels with edge replication in encoded B order. | |
| Prepared pixels are NumPy arrays or CPU tensors with shape [B, 1, H, W]. | |
| Context is selected from the whole volume, including across batch edges. | |
| """ | |
| import torch | |
| if radius: | |
| indices = np.clip( | |
| np.arange(start, end)[:, None] + np.arange(-radius, radius + 1)[None, :], | |
| 0, | |
| len(pixels) - 1, | |
| ) | |
| part = pixels[indices][:, :, 0] | |
| else: | |
| part = pixels[start:end] | |
| image = torch.from_numpy(part) if isinstance(part, np.ndarray) else part | |
| return image.to(device=device, dtype=torch.float32) | |
| def source_frames(path, indices=None): | |
| d = structural_header(path) | |
| plugin = "pylibjpeg" if d.file_meta.TransferSyntaxUID.is_compressed else "" | |
| yield from iter_pixels(path, indices=indices, decoding_plugin=plugin) | |
| def resized_context(path, centers, num_frames, height, width, radius): | |
| """Only image channels, ordered previous->center->next in encoded B order.""" | |
| offsets = range(-radius, radius + 1) | |
| needed = sorted( | |
| {max(0, min(num_frames - 1, b + offset)) for b in centers for offset in offsets} | |
| ) | |
| frames = list(source_frames(path, indices=needed)) | |
| if len(frames) != len(needed): | |
| raise ValueError("Incomplete context pixel decoding") | |
| images = resize_frames(frames, height, width).numpy()[:, 0] | |
| lookup = {b: i for i, b in enumerate(needed)} | |
| return { | |
| b: images[ | |
| [lookup[max(0, min(num_frames - 1, b + offset))] for offset in offsets] | |
| ] | |
| for b in centers | |
| } | |
| def enforce_input_opens(root, rows): | |
| """Audit and reject Python reads of dataset files outside declared OCT inputs.""" | |
| import os | |
| import socket | |
| import sys | |
| prefix = os.path.abspath(root) + os.sep | |
| allowed = { | |
| os.path.abspath(safe_path(root, p)) for r in rows for p in r["oct_paths"] | |
| } | |
| stats = { | |
| "declared_oct_files": len(allowed), | |
| "oct_open_events": 0, | |
| "denied_events": 0, | |
| "reference_npz_reads": 0, | |
| "network_attempts": 0, | |
| } | |
| def audit(event, args): | |
| if event in [ | |
| "socket.getaddrinfo", | |
| "socket.gethostbyname", | |
| "socket.gethostbyaddr", | |
| ] or ( | |
| event == "socket.connect" | |
| and args[0].family in [socket.AF_INET, socket.AF_INET6] | |
| ): | |
| stats["network_attempts"] += 1 | |
| raise PermissionError("Network access is forbidden during inference") | |
| if event != "open" or not isinstance(args[0], (str, bytes, os.PathLike)): | |
| return | |
| path = os.path.abspath(os.fsdecode(args[0])) | |
| flags = args[2] or 0 | |
| writing = bool(flags & (os.O_WRONLY | os.O_RDWR)) | |
| if path.startswith(prefix): | |
| if path not in allowed or writing: | |
| stats["denied_events"] += 1 | |
| raise PermissionError("Inference attempted a non-input dataset access") | |
| stats["oct_open_events"] += 1 | |
| if path.endswith(".npz") and not writing: | |
| stats["reference_npz_reads"] += 1 | |
| raise PermissionError( | |
| "Reference array reads are forbidden during inference" | |
| ) | |
| sys.addaudithook(audit) | |
| return stats | |
| _PIXEL_WORKER_AUDIT = None | |
| def initialize_pixel_worker(root, rows): | |
| """Spawn workers have independent read guards and never initialize CUDA.""" | |
| global _PIXEL_WORKER_AUDIT | |
| import torch | |
| torch.set_num_threads(1) | |
| _PIXEL_WORKER_AUDIT = enforce_input_opens(root, rows) | |
| def prepare_pixel_packet(arguments): | |
| """Only declared OCT paths are available; return pixels and audit deltas.""" | |
| if _PIXEL_WORKER_AUDIT is None: | |
| raise RuntimeError("Missing inference worker file guard") | |
| row, root, cfg = arguments | |
| fields = [ | |
| "oct_open_events", | |
| "denied_events", | |
| "reference_npz_reads", | |
| "network_attempts", | |
| ] | |
| before = {k: _PIXEL_WORKER_AUDIT[k] for k in fields} | |
| from src.inference.predict import prepare_pixels | |
| prepared = prepare_pixels(row, root, cfg) | |
| # NumPy packets avoid implicit Torch shared-memory dependencies across workers. | |
| packet = [ | |
| (part[0].numpy(), part[1], part[2], *[x.numpy() for x in part[3:]]) | |
| for part in prepared | |
| ] | |
| delta = {k: _PIXEL_WORKER_AUDIT[k] - before[k] for k in fields} | |
| return packet, delta | |
| def cache_layout(directory): | |
| meta = Path(directory) / "cache-metadata.json" | |
| layout = ( | |
| json.loads(meta.read_text()).get("layout", "flat_v1") | |
| if meta.exists() | |
| else "flat_v1" | |
| ) | |
| if layout not in ["flat_v1", "case_prefix2_v1"]: | |
| raise ValueError("Unknown training-cache layout") | |
| return layout | |
| def frame_path(directory, case_id, frame, layout="flat_v1"): | |
| root = Path(directory) | |
| if layout == "case_prefix2_v1": | |
| root = root / case_id[:2] | |
| elif layout != "flat_v1": | |
| raise ValueError("Unknown training-cache layout") | |
| return root / (case_id + f"-{frame}.npz") | |
| def completion_fields(directory): | |
| directory = Path(directory) | |
| files = list(directory.glob("*.npz")) | |
| inference = json.loads((directory / "inference-receipt.json").read_text()) | |
| if inference["cases"] != len(files): | |
| raise ValueError("Prediction completion count mismatch") | |
| return { | |
| "predicted_cases": len(files), | |
| "input_open_audit": inference["input_open_audit"], | |
| "prediction_files_sha256": None, | |
| "prediction_digest_deferred_to_cpu_score": True, | |
| "prediction_digest_order": "sorted filename bytes then ASCII SHA256 of file bytes", | |
| } | |
| def prediction_digest(directory): | |
| combined = hashlib.sha256() | |
| files = sorted(Path(directory).glob("*.npz")) | |
| for file in files: | |
| digest = hashlib.sha256() | |
| with file.open("rb") as stream: | |
| for block in iter(lambda: stream.read(1048576), b""): | |
| digest.update(block) | |
| combined.update(file.name.encode()) | |
| combined.update(digest.hexdigest().encode()) | |
| return combined.hexdigest(), len(files) | |
| def collect_source_hashes(): | |
| """Identify executable project sources with paths relative to the bundle.""" | |
| root = Path(__file__).resolve().parents[1] | |
| paths = [ | |
| *root.glob("*.py"), | |
| *(root / "src").rglob("*.py"), | |
| *(root / "tools").glob("*.py"), | |
| *(root / "research" / "experiments").glob("*.py"), | |
| ] | |
| return { | |
| str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() | |
| for path in sorted(paths) | |
| } | |