"""Structural OCT pixels, derived cache paths, and prediction receipts.""" from __future__ import annotations import hashlib import json from pathlib import Path, PurePosixPath import numpy as np from pydicom.filereader import read_partial from pydicom.pixels import iter_pixels from pydicom.tag import Tag OCT_SOP = "1.2.840.10008.5.1.4.1.1.77.1.5.4" def safe_path(root, value): p = PurePosixPath(value) if p.is_absolute() or ".." in p.parts or not value or "\\" in value: raise ValueError("Unsafe relative dataset path") return Path(root).joinpath(*p.parts) def read_input(path): rows = [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()] if not rows: raise ValueError("Empty input manifest") for r in rows: if set(r) != {"case_id", "device", "oct_paths"}: raise ValueError("Prediction input must be label free") if len({r["case_id"] for r in rows}) != len(rows): raise ValueError("Duplicate inference case ids") return rows def metadata_header(path, specific_tags=None): """Stop before PerFrameFunctionalGroups: no payload or unused sequence parsing. All source identification, geometry and SharedFunctionalGroups precede this tag. Per-frame geometry is inspected separately by the discovery/audit programs. """ tags = [Tag(x) for x in specific_tags] if specific_tags else None with Path(path).open("rb") as fp: return read_partial( fp, stop_when=lambda tag, vr, length: int(tag) >= 0x52009230, specific_tags=tags, ) def structural_header(path): d = metadata_header( path, [ "SOPClassUID", "Modality", "NumberOfFrames", "Rows", "Columns", "PhotometricInterpretation", "ConcatenationFrameOffsetNumber", ], ) if str(d.SOPClassUID) != OCT_SOP or str(d.Modality) != "OPT": raise ValueError("Input is not structural OCT") return d def normalize_frame(frame): """No fitted statistics: deterministic robust intensity scaling per input B-scan.""" x = np.asarray(frame, dtype=np.float32) sample = x[::4, ::4] lo, hi = np.percentile(sample, [1, 99.5]) return np.clip((x - lo) / max(float(hi - lo), 1e-6), 0, 1).astype("float32") def resize_frames(frames, height, width, device="cpu"): import torch from torch.nn import functional as F x = torch.from_numpy(np.stack([normalize_frame(f) for f in frames]))[:, None].to( device ) return F.interpolate(x, size=(height, width), mode="bilinear", align_corners=True) def context_batch(pixels, start, end, radius, device): """Build float32 image channels with edge replication in encoded B order. Prepared pixels are NumPy arrays or CPU tensors with shape [B, 1, H, W]. Context is selected from the whole volume, including across batch edges. """ import torch if radius: indices = np.clip( np.arange(start, end)[:, None] + np.arange(-radius, radius + 1)[None, :], 0, len(pixels) - 1, ) part = pixels[indices][:, :, 0] else: part = pixels[start:end] image = torch.from_numpy(part) if isinstance(part, np.ndarray) else part return image.to(device=device, dtype=torch.float32) def source_frames(path, indices=None): d = structural_header(path) plugin = "pylibjpeg" if d.file_meta.TransferSyntaxUID.is_compressed else "" yield from iter_pixels(path, indices=indices, decoding_plugin=plugin) def resized_context(path, centers, num_frames, height, width, radius): """Only image channels, ordered previous->center->next in encoded B order.""" offsets = range(-radius, radius + 1) needed = sorted( {max(0, min(num_frames - 1, b + offset)) for b in centers for offset in offsets} ) frames = list(source_frames(path, indices=needed)) if len(frames) != len(needed): raise ValueError("Incomplete context pixel decoding") images = resize_frames(frames, height, width).numpy()[:, 0] lookup = {b: i for i, b in enumerate(needed)} return { b: images[ [lookup[max(0, min(num_frames - 1, b + offset))] for offset in offsets] ] for b in centers } def enforce_input_opens(root, rows): """Audit and reject Python reads of dataset files outside declared OCT inputs.""" import os import socket import sys prefix = os.path.abspath(root) + os.sep allowed = { os.path.abspath(safe_path(root, p)) for r in rows for p in r["oct_paths"] } stats = { "declared_oct_files": len(allowed), "oct_open_events": 0, "denied_events": 0, "reference_npz_reads": 0, "network_attempts": 0, } def audit(event, args): if event in [ "socket.getaddrinfo", "socket.gethostbyname", "socket.gethostbyaddr", ] or ( event == "socket.connect" and args[0].family in [socket.AF_INET, socket.AF_INET6] ): stats["network_attempts"] += 1 raise PermissionError("Network access is forbidden during inference") if event != "open" or not isinstance(args[0], (str, bytes, os.PathLike)): return path = os.path.abspath(os.fsdecode(args[0])) flags = args[2] or 0 writing = bool(flags & (os.O_WRONLY | os.O_RDWR)) if path.startswith(prefix): if path not in allowed or writing: stats["denied_events"] += 1 raise PermissionError("Inference attempted a non-input dataset access") stats["oct_open_events"] += 1 if path.endswith(".npz") and not writing: stats["reference_npz_reads"] += 1 raise PermissionError( "Reference array reads are forbidden during inference" ) sys.addaudithook(audit) return stats _PIXEL_WORKER_AUDIT = None def initialize_pixel_worker(root, rows): """Spawn workers have independent read guards and never initialize CUDA.""" global _PIXEL_WORKER_AUDIT import torch torch.set_num_threads(1) _PIXEL_WORKER_AUDIT = enforce_input_opens(root, rows) def prepare_pixel_packet(arguments): """Only declared OCT paths are available; return pixels and audit deltas.""" if _PIXEL_WORKER_AUDIT is None: raise RuntimeError("Missing inference worker file guard") row, root, cfg = arguments fields = [ "oct_open_events", "denied_events", "reference_npz_reads", "network_attempts", ] before = {k: _PIXEL_WORKER_AUDIT[k] for k in fields} from src.inference.predict import prepare_pixels prepared = prepare_pixels(row, root, cfg) # NumPy packets avoid implicit Torch shared-memory dependencies across workers. packet = [ (part[0].numpy(), part[1], part[2], *[x.numpy() for x in part[3:]]) for part in prepared ] delta = {k: _PIXEL_WORKER_AUDIT[k] - before[k] for k in fields} return packet, delta def cache_layout(directory): meta = Path(directory) / "cache-metadata.json" layout = ( json.loads(meta.read_text()).get("layout", "flat_v1") if meta.exists() else "flat_v1" ) if layout not in ["flat_v1", "case_prefix2_v1"]: raise ValueError("Unknown training-cache layout") return layout def frame_path(directory, case_id, frame, layout="flat_v1"): root = Path(directory) if layout == "case_prefix2_v1": root = root / case_id[:2] elif layout != "flat_v1": raise ValueError("Unknown training-cache layout") return root / (case_id + f"-{frame}.npz") def completion_fields(directory): directory = Path(directory) files = list(directory.glob("*.npz")) inference = json.loads((directory / "inference-receipt.json").read_text()) if inference["cases"] != len(files): raise ValueError("Prediction completion count mismatch") return { "predicted_cases": len(files), "input_open_audit": inference["input_open_audit"], "prediction_files_sha256": None, "prediction_digest_deferred_to_cpu_score": True, "prediction_digest_order": "sorted filename bytes then ASCII SHA256 of file bytes", } def prediction_digest(directory): combined = hashlib.sha256() files = sorted(Path(directory).glob("*.npz")) for file in files: digest = hashlib.sha256() with file.open("rb") as stream: for block in iter(lambda: stream.read(1048576), b""): digest.update(block) combined.update(file.name.encode()) combined.update(digest.hexdigest().encode()) return combined.hexdigest(), len(files) def collect_source_hashes(): """Identify executable project sources with paths relative to the bundle.""" root = Path(__file__).resolve().parents[1] paths = [ *root.glob("*.py"), *(root / "src").rglob("*.py"), *(root / "tools").glob("*.py"), *(root / "research" / "experiments").glob("*.py"), ] return { str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(paths) }