"""Bounded parallel, label-free OCT decoding for optional faster inference.""" from pathlib import Path import json import time import numpy as np import torch from torch.nn import functional as F from torch.utils.data import IterableDataset, DataLoader, get_worker_info from src.inference.decoding import positions_from_logits from src.inference.combination import finish_positions from src.models import output_indices from src.inference.runtime import autocast_options from src.io import volume_headers from src.inference.context import iter_normalized_contexts class OCTBatches(IterableDataset): def __init__(self, rows, root, shapes, batch_size): self.rows, self.root = rows, root self.shapes, self.batch_size = shapes, batch_size def __iter__(self): worker = get_worker_info() worker_index, workers = (worker.id, worker.num_workers) if worker else (0, 1) for index in range(worker_index, len(self.rows), workers): row = self.rows[index] headers, geometry = volume_headers(self.root, row["oct_paths"]) batches = [[] for _ in self.shapes] completed = 0 def item(): nonlocal completed n = len(batches[0]) if completed + n > geometry[0]: raise ValueError( "Decoded frames exceed declared structural geometry" ) result = { "index": index, "offset": completed, "geometry": geometry, "images": [np.stack(batch) for batch in batches], } completed += n for batch in batches: batch.clear() return result for images in iter_normalized_contexts(headers, self.shapes): for frame, batch in zip(images, batches): batch.append(frame) if len(batches[0]) == self.batch_size: yield item() if batches[0]: yield item() if completed != geometry[0]: raise ValueError("Incomplete structural frame coverage") def stream_predictions( rows, root, candidates, device, batch_size, workers, tta=False, precision="fp32" ): """Candidates contain loaded models/configs/destinations; no reference inputs.""" shapes = sorted( { ( c["config"]["height"], c["config"]["width"], c["config"].get("input_channels", 1), ) for c in candidates } ) dataset = OCTBatches(rows, root, shapes, batch_size) loader = DataLoader( dataset, batch_size=None, num_workers=workers, pin_memory=device.startswith("cuda"), **({"prefetch_factor": 2} if workers else {}) ) pending, done = {}, 0 start = time.monotonic() for batch in loader: index, offset = int(batch["index"]), int(batch["offset"]) b, h, a = map(int, batch["geometry"]) row = rows[index] n = len(batch["images"][0]) if index not in pending: pending[index] = {"seen": np.zeros(b, bool), "buffers": []} for candidate in candidates: config = candidate["config"] keys = config["device_keys"][row["device"]] shape = ( config["height"], config["width"], config.get("input_channels", 1), ) pending[index]["buffers"].append( { "candidate": candidate, "shape_index": shapes.index(shape), "height": shape[0], "keys": keys, "indices": output_indices(config, row["device"]), "positions": np.empty((len(keys), b, a), np.float32), } ) state = pending[index] if state["seen"][offset : offset + n].any() or offset + n > b: raise ValueError("Duplicate or out-of-range structural frames") tensors = [image.to(device, non_blocking=True) for image in batch["images"]] with torch.inference_mode(), torch.autocast( **autocast_options(device, precision) ): for buffer in state["buffers"]: x = tensors[buffer["shape_index"]] model = buffer["candidate"]["model"] pred = positions_from_logits( model(x)[:, buffer["indices"]], buffer["candidate"]["config"], row["device"], ) if tta: pred = ( pred + positions_from_logits( model(x.flip(-1))[:, buffer["indices"]], buffer["candidate"]["config"], row["device"], ).flip(-1) ) * 0.5 pred = F.interpolate(pred, size=a, mode="linear", align_corners=False) pred = ((pred + 0.5) * h / buffer["height"] - 0.5).clamp(0, h - 1) buffer["positions"][:, offset : offset + n] = ( pred.float().cpu().numpy().transpose(1, 0, 2) ) state["seen"][offset : offset + n] = True if state["seen"].all(): for buffer in state["buffers"]: values = finish_positions( buffer["positions"], row["device"], h, buffer["candidate"]["config"] .get("inference", {}) .get("postprocess", {}), ) if not np.isfinite(values).all(): raise ValueError("Nonfinite candidate prediction") destination = Path(buffer["candidate"]["destination"]) path = destination / (row["case_id"] + ".npz") temporary = path.with_suffix(".tmp.npz") np.savez_compressed( temporary, positions=values, boundary_keys=np.asarray(buffer["keys"]), ) temporary.replace(path) del pending[index] done += 1 if done % 10 == 0: print( json.dumps( { "completed_cases": done, "elapsed_seconds": time.monotonic() - start, "decode_workers": workers, } ), flush=True, ) if pending or done != len(rows): raise ValueError("Incomplete inference manifest coverage") report = { "completed_cases": done, "elapsed_seconds": time.monotonic() - start, "decode_workers": workers, } print(json.dumps(report), flush=True) return report