"""Local DICOM viewing and inference adapter for the retained OCT submission. The scientific model, normalization, head combinations and saved postprocessing are imported from the researcher's original files. This module adds upload, session and display handling; it does not retrain or alter the checkpoint. """ from __future__ import annotations from src.paths import PROJECT_ROOT from collections import OrderedDict from dataclasses import dataclass, field from pathlib import Path from threading import Lock, RLock from typing import Iterator import hashlib import html import json import math import os import shutil import tempfile import time import uuid import numpy as np from PIL import Image, ImageDraw import pydicom import torch from torch.nn import functional as F from src.models import build_model, output_indices from src.inference.context import OFFSETS, iter_normalized_contexts from src.inference.runtime import configure_inference, autocast_options from src.io import volume_headers, read_frame, read_frames, normalize_frame from src.inference.decoding import positions_from_logits from src.inference.combination import combine_named_heads, finish_positions ROOT = PROJECT_ROOT CHECKPOINT = ROOT / "artifacts/checkpoint.pt" STRUCTURAL_OCT = "1.2.840.10008.5.1.4.1.1.77.1.5.4" FAMILIES = { "heidelberg_spectralis": "Heidelberg Spectralis", "topcon_maestro2": "Topcon Maestro2", "topcon_triton": "Topcon Triton", "zeiss_cirrus": "Zeiss Cirrus", } AUTO = "auto" NAMES = { "SCT_280677004": "ILM", "DCM_128289": "RNFL · outer", "DCM_128290": "GCL · outer", "DCM_128291": "IPL · outer", "DCM_128292": "INL · outer", "DCM_128293": "OPL · outer", "SCT_76710003": "ELM", "DCM_128320": "EZ · inner", "DCM_128321": "EZ · midline", "DCM_128323": "Interdigitation zone · inner", "DCM_128297": "RPE · inner", "DCM_128299": "RPE · outer", "DCM_128300": "Bruch’s membrane · outer", "DCM_128298": "RPE · centre", "DCM_128301": "Choroid–sclera interface", } PALETTE = [ "#22d3ee", "#a3e635", "#fbbf24", "#fb7185", "#c084fc", "#34d399", "#f472b6", "#60a5fa", "#f97316", "#e879f9", "#2dd4bf", "#fde047", "#a5b4fc", "#fda4af", "#d9f99d", ] COLORS = {key: PALETTE[i % len(PALETTE)] for i, key in enumerate(NAMES)} class ViewerError(ValueError): """A recoverable error suitable for displaying in the viewer.""" @dataclass class Prediction: positions: np.ndarray # K, number of predicted B-scans, original A keys: list[str] frames: list[int] family: str compute: str complete_volume: bool seconds: float export: Path @dataclass class Volume: identity: str directory: Path paths: list[Path] headers: list shape: tuple[int, int, int] detected_family: str | None axial_mm: float | None lateral_mm: float | None filename: str prediction: Prediction | None = None view_frame: int = 0 view_settings: tuple | None = None frames: OrderedDict = field(default_factory=OrderedDict) cancelled: bool = False updated: float = field(default_factory=time.monotonic) def frame(self, index: int) -> np.ndarray: index = max(0, min(int(index), self.shape[0] - 1)) self.updated = time.monotonic() if index not in self.frames: try: pixels = np.asarray(read_frame(self.headers, index)) except Exception as exc: raise ViewerError( "Could not decode this DICOM. Install requirements.txt " "for JPEG2000 and lossless JPEG support. " + str(exc) ) from exc if pixels.shape != self.shape[1:]: raise ViewerError( "Decoded image dimensions do not match the DICOM header." ) self.frames[index] = pixels while len(self.frames) > 8: self.frames.popitem(last=False) self.frames.move_to_end(index) return self.frames[index] class SessionStore: def __init__(self): self.volumes: dict[str, Volume] = {} self.lock = RLock() def add(self, volume: Volume) -> str: with self.lock: self.reap() self.volumes[volume.identity] = volume return volume.identity def get(self, identity: str | None) -> Volume: with self.lock: if not identity or identity not in self.volumes: raise ViewerError("Upload a structural OCT DICOM first.") volume = self.volumes[identity] volume.updated = time.monotonic() return volume def remove(self, identity: str | None): with self.lock: volume = self.volumes.pop(identity, None) if volume: volume.cancelled = True shutil.rmtree(volume.directory, ignore_errors=True) def reap(self): for identity, volume in list(self.volumes.items()): if time.monotonic() - volume.updated > 7200: self.remove(identity) STORE = SessionStore() def detect_family(header) -> str | None: description = " ".join( str(header.get(tag, "")) for tag in ("Manufacturer", "ManufacturerModelName", "SeriesDescription") ).lower() if "spectralis" in description or "heidelberg" in description: return "heidelberg_spectralis" if "maestro" in description: return "topcon_maestro2" if "triton" in description: return "topcon_triton" if "cirrus" in description or "zeiss" in description: return "zeiss_cirrus" return None def pixel_spacing(header) -> tuple[float | None, float | None]: candidates = [] for key in ("SharedFunctionalGroupsSequence", "PerFrameFunctionalGroupsSequence"): sequence = header.get(key, []) if sequence: measures = sequence[0].get("PixelMeasuresSequence", []) if measures: candidates.append(measures[0].get("PixelSpacing")) candidates.append(header.get("PixelSpacing")) for values in candidates: if values is not None and len(values) == 2: row, col = map(float, values) if math.isfinite(row) and math.isfinite(col) and row > 0 and col > 0: return row, col return None, None def load_upload(files: list[str] | str) -> Volume: if isinstance(files, str): files = [files] if not files: raise ViewerError("Choose a .dcm file to begin.") directory = Path(tempfile.mkdtemp(prefix="myxos-oct-viewer-")) os.chmod(directory, 0o700) try: paths = [] all_headers = [] for index, uploaded in enumerate(files): source = Path(uploaded) if not source.is_file(): raise ViewerError( "An uploaded DICOM is no longer available. Upload it again." ) # A private copy decouples inference from Gradio's upload-cache lifetime. path = directory / f"part-{index:03d}.dcm" shutil.copyfile(source, path) os.chmod(path, 0o600) header = pydicom.dcmread(path, stop_before_pixels=True) if str(header.get("SOPClassUID", "")) != STRUCTURAL_OCT: raise ViewerError( "This is not a structural OCT DICOM. Upload the OCT volume, rather than an OCTA or segmentation export." ) if int(header.get("SamplesPerPixel", 1)) != 1: raise ViewerError( "The selected checkpoint expects grayscale structural OCT." ) all_headers.append(header) paths.append(path) for tag in ("StudyInstanceUID", "SeriesInstanceUID", "ConcatenationUID"): identifiers = {str(h.get(tag)) for h in all_headers if h.get(tag)} if len(identifiers) > 1: raise ViewerError( "Upload one OCT volume, or the contiguous DICOM parts of that same volume." ) headers, shape = volume_headers(directory, [p.name for p in paths]) if not all(int(n) > 0 for n in shape): raise ViewerError("The DICOM contains an empty volume.") axial, lateral = pixel_spacing(all_headers[0]) volume = Volume( uuid.uuid4().hex, directory, paths, headers, tuple(shape), detect_family(all_headers[0]), axial, lateral, Path(files[0]).name, ) # Fail early when a required codec is missing. volume.frame((shape[0] - 1) // 2) return volume except ViewerError: shutil.rmtree(directory, ignore_errors=True) raise except Exception as exc: shutil.rmtree(directory, ignore_errors=True) raise ViewerError("Unable to load this OCT volume: " + str(exc)) from exc def resolve_family(volume: Volume, selected: str) -> str: family = volume.detected_family if selected == AUTO else selected if family not in FAMILIES: raise ViewerError( "The scanner could not be identified. Select its scanner family below." ) return family def boundary_choices(family: str | None) -> list[tuple[str, str]]: cfg = json.loads((ROOT / "artifacts/config.json").read_text()) keys = cfg["device_keys"].get(family, []) return [(NAMES.get(key, key), key) for key in keys] def compute_choices() -> list[tuple[str, str]]: choices = [("Automatic", "auto"), ("CPU", "cpu")] if torch.cuda.is_available(): choices.append(("NVIDIA GPU · CUDA", "cuda")) if torch.backends.mps.is_available(): choices.append(("Apple GPU · MPS", "mps")) return choices def resolve_compute(selected: str) -> str: if selected == "auto": return ( "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu") ) if selected not in {value for _, value in compute_choices()}: raise ViewerError("That compute device is not available on this machine.") return selected def selected_contexts(headers, frames: list[int], shapes: list[tuple]): total = sum(int(ds.get("NumberOfFrames", 1)) for _, ds in headers) indices = set() for frame in frames: for shape in shapes: indices.update( max(0, min(total - 1, frame + offset)) for offset in OFFSETS[shape[2]] ) if shape[2] == 6: indices.update([0, total // 2, total - 1]) raw = read_frames(headers, sorted(indices)) normalized = {} for frame in frames: images = [] for h, a, channels in shapes: context = [ max(0, min(total - 1, frame + offset)) for offset in OFFSETS[channels] ] if channels == 6: context += [0, total // 2, total - 1] channels_out = [] for index in context: key = index, h, a if key not in normalized: normalized[key] = normalize_frame(raw[index], (h, a)) channels_out.append(normalized[key]) images.append(np.stack(channels_out)) yield images class InferenceEngine: def __init__(self): # A plain Lock can span generator yields, unlike thread-owned RLock. self.lock = Lock() self.checkpoint = None self.models = {} self.device = None def load(self, device: str): if self.checkpoint is None: self.checkpoint = torch.load( CHECKPOINT, map_location="cpu", weights_only=True ) if self.checkpoint["config"]["architecture"] != "BoundaryEnsemble": raise ViewerError( "This viewer expects the retained Researcher 1 ensemble checkpoint." ) if self.device != device: self.models.clear() self.device = device if torch.cuda.is_available(): torch.cuda.empty_cache() if torch.backends.mps.is_available(): torch.mps.empty_cache() return self.checkpoint def members(self, checkpoint, family): cfg = checkpoint["config"] combination = cfg["combination"] active = None if "head_weights_by_device" in combination: active = { name for rule in combination["head_weights_by_device"][family].values() for name in rule } elif "head_source_by_device" in combination: active = set(combination["head_source_by_device"][family].values()) result = [] for saved in checkpoint["members"]: if active is not None and saved["name"] not in active: continue if saved["name"] not in self.models: model = build_model(saved["config"]) model.load_state_dict(saved["model"], strict=True) self.models[saved["name"]] = ( model.to(self.device).eval().requires_grad_(False) ) result.append( { "name": saved["name"], "config": saved["config"], "model": self.models[saved["name"]], } ) return result def predict( self, volume: Volume, family: str, compute: str, batch_size: int, full_volume: bool, frame_index: int, ) -> Iterator[dict]: device = resolve_compute(compute) batch_size = max(1, min(32, int(batch_size))) b, h, a = volume.shape frames = ( list(range(b)) if full_volume else [max(0, min(b - 1, int(frame_index)))] ) started = time.monotonic() volume.cancelled = False with self.lock: configure_inference("fp32") checkpoint = self.load(device) members = self.members(checkpoint, family) cfg = checkpoint["config"] keys = cfg["device_keys"][family] shapes = sorted( { ( m["config"]["height"], m["config"]["width"], m["config"].get("input_channels", 1), ) for m in members } ) buffers = [ np.empty((len(keys), len(frames), a), np.float32) for _ in members ] iterator = ( iter_normalized_contexts(volume.headers, shapes) if full_volume else selected_contexts(volume.headers, frames, shapes) ) batch = [] completed = 0 for images in iterator: if volume.cancelled: raise ViewerError("Segmentation cancelled.") batch.append(images) if len(batch) < batch_size and completed + len(batch) < len(frames): continue # Do not keep thread-local inference contexts open across yields. with torch.inference_mode(), torch.autocast( **autocast_options(device, "fp32") ): tensors = [ torch.from_numpy(np.stack([item[i] for item in batch])).to( device ) for i in range(len(shapes)) ] for member, buffer in zip(members, buffers): own = member["config"] shape = ( own["height"], own["width"], own.get("input_channels", 1), ) indices = output_indices(own, family) logits = member["model"](tensors[shapes.index(shape)])[ :, indices ] positions = positions_from_logits(logits, own, family) positions = F.interpolate( positions, size=a, mode="linear", align_corners=False ) positions = ((positions + 0.5) * h / own["height"] - 0.5).clamp( 0, h - 1 ) order = [own["device_keys"][family].index(key) for key in keys] values = ( positions.float().cpu().numpy().transpose(1, 0, 2)[order] ) buffer[:, completed : completed + len(batch)] = values del logits, positions completed += len(batch) batch = [] volume.updated = time.monotonic() yield { "completed": completed, "total": len(frames), "device": device, "seconds": time.monotonic() - started, } if volume.cancelled: raise ViewerError("Segmentation cancelled.") components = [ finish_positions( values, family, h, member["config"].get("inference", {}).get("postprocess", {}), ) for values, member in zip(buffers, members) ] positions = combine_named_heads( components, [m["name"] for m in members], keys, family, cfg["combination"], ) positions = finish_positions( positions, family, h, cfg.get("inference", {}).get("postprocess", {}) ) if completed != len(frames) or not np.isfinite(positions).all(): raise ViewerError( "Inference did not produce complete finite boundary predictions." ) suffix = "volume" if full_volume else f"preview-bscan-{frames[0] + 1:04d}" output = volume.directory / f"boundaries-{suffix}.npz" arrays = {"positions": positions, "boundary_keys": np.asarray(keys)} if not full_volume: arrays["frame_indices"] = np.asarray(frames, dtype=np.int32) np.savez_compressed(output, **arrays) prediction = Prediction( positions, list(keys), frames, family, device, full_volume, time.monotonic() - started, output, ) volume.prediction = prediction yield {"result": prediction} ENGINE = InferenceEngine() def display_dimensions(volume: Volume, physical: bool) -> tuple[int, int]: _, h, a = volume.shape ratio = h / a if physical and volume.axial_mm and volume.lateral_mm: ratio *= volume.axial_mm / volume.lateral_mm width = 1000 height = max(1, round(width * ratio)) scale = min(1.0, 1400 / max(width, height)) return max(1, round(width * scale)), max(1, round(height * scale)) def render( volume: Volume, index: int, selected_keys: list[str] | None = None, overlay: bool = True, opacity: float = 0.9, line_width: int = 2, labels: bool = False, contrast: float = 1.0, brightness: float = 0.0, physical: bool = True, ) -> tuple[np.ndarray, np.ndarray | None]: index = max(0, min(volume.shape[0] - 1, int(index))) image = np.asarray(volume.frame(index), dtype=np.float32) finite = image[np.isfinite(image)] lo, hi = np.quantile(finite, [0.01, 0.995]) if finite.size else (0, 1) gray = np.nan_to_num((image - lo) / max(float(hi - lo), 1e-6)) gray = np.clip((gray - 0.5) * float(contrast) + 0.5 + float(brightness), 0, 1) width, height = display_dimensions(volume, physical) base = ( Image.fromarray(np.uint8(np.round(gray * 255))) .convert("RGB") .resize((width, height), Image.Resampling.BILINEAR) ) raw = np.asarray(base) prediction = volume.prediction if prediction is None or index not in prediction.frames: return raw, None if not overlay: return raw, raw.copy() frame = prediction.frames.index(index) selected = set(prediction.keys if selected_keys is None else selected_keys) layer = base.convert("RGBA") annotation = Image.new("RGBA", base.size) draw = ImageDraw.Draw(annotation) _, native_h, native_a = volume.shape x = np.linspace(0, width - 1, native_a) for key, values in zip(prediction.keys, prediction.positions[:, frame]): if key not in selected: continue y = np.clip(values, 0, native_h - 1) * (height - 1) / max(1, native_h - 1) color = COLORS.get(key, "#ffffff") rgb = tuple(int(color[i : i + 2], 16) for i in (1, 3, 5)) points = list(zip(x.tolist(), y.tolist())) if len(points) > 1: draw.line( points, fill=(*rgb, round(255 * float(opacity))), width=max(1, int(line_width)), joint="curve", ) if labels: text = NAMES.get(key, key) anchor = max( 0, min( height - 14, round(float(np.median(y[: max(1, native_a // 8)])) - 13), ), ) box = draw.textbbox((8, anchor), text) draw.rectangle( (box[0] - 3, box[1] - 2, box[2] + 3, box[3] + 2), fill=(8, 15, 26, 210) ) draw.text((8, anchor), text, fill=(*rgb, 255)) return raw, np.asarray(Image.alpha_composite(layer, annotation).convert("RGB")) def legend(keys: list[str]) -> str: if not keys: return "Boundary colours appear after a scan is loaded." items = [] for key in keys: items.append( f'● {html.escape(NAMES.get(key, key))}' ) return " · ".join(items) def volume_info(volume: Volume, family: str | None) -> str: b, h, a = volume.shape scanner = FAMILIES.get(family, "Select scanner family") spacing = f" · axial {volume.axial_mm * 1000:.2f} µm" if volume.axial_mm else "" return ( f"**{html.escape(scanner)}** \n{b:,} B-scans · {h:,} × {a:,} pixels{spacing}" ) def checkpoint_identity() -> str: return hashlib.sha256(CHECKPOINT.read_bytes()).hexdigest()