"""Fresh-process, label-free boundary inference using structural OCT pixels only.""" from __future__ import annotations import argparse import hashlib import json import re import time from pathlib import Path import numpy as np import torch from torch.nn import functional as F from src.inference import apply_policy from src.io import ( context_batch, enforce_input_opens, read_input, resize_frames, safe_path, source_frames, structural_header, ) from src.models import make_model from src.models.unet import decode_logits def load_model(checkpoint, device): ck = torch.load(checkpoint, map_location="cpu", weights_only=False) cfg = ck["config"] net = make_model(cfg) net.load_state_dict(ck["model"]) net.to(device).eval() return net, cfg @torch.inference_mode() def predict_case(row, root, net, cfg, device, batch_size=12, tta=False): if cfg.get("context_radius", 0) or cfg.get("architecture") in [ "boundary_ensemble_v1", "boundary_profile_refine_v1", "boundary_zero_atom_v1", "boundary_volume_refine_v1", ]: return predict_prepared( row, prepare_pixels(row, root, cfg), net, cfg, device, batch_size, tta ) family = row["device"] if family not in cfg["boundary_keys"]: raise ValueError("Unsupported device family") chunks = [] height = None width = None # Multiple files, when admitted, must already be in canonical concatenation order. for rel in row["oct_paths"]: path = safe_path(root, rel) d = structural_header(path) h, w = int(d.Rows), int(d.Columns) b = int(d.NumberOfFrames) if height is not None and (h, w) != (height, width): raise ValueError("Inconsistent source file grids") height, width = h, w frames = [] outs = [] n = 0 def apply(frames): x = resize_frames(frames, cfg["height"], cfg["width"], device) with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): z = net(x, family) p = decode_logits( z, cfg.get("local_radius", 6), cfg.get("decode_method", "local_centroid"), ) if tta: p = ( p + decode_logits( net(x.flip(-1), family), cfg.get("local_radius", 6), cfg.get("decode_method", "local_centroid"), ).flip(-1) ) / 2 p = F.interpolate(p, size=w, mode="linear", align_corners=True) p = p * ((h - 1) / (cfg["height"] - 1)) return p.clamp(0, h - 1).cpu().numpy().astype("float32") for frame in source_frames(path): frames.append(frame) n += 1 if len(frames) == batch_size: outs.append(apply(frames)) frames = [] if frames: outs.append(apply(frames)) if n != b: raise ValueError("Decoded source frame count mismatch") chunks.append(np.concatenate(outs, axis=0).transpose(1, 0, 2)) return np.concatenate(chunks, axis=1), np.asarray(cfg["boundary_keys"][family]) def prepare_pixels(row, root, cfg): """Bounded per-volume decoding, CPU normalization/resizing; no label access.""" from src.models.refiners import fine_preparation_shape fine_shape = fine_preparation_shape(cfg) result = [] for rel in row["oct_paths"]: path = safe_path(root, rel) d = structural_header(path) B, H, A = int(d.NumberOfFrames), int(d.Rows), int(d.Columns) blocks = [] fine_blocks = [] frames = [] n = 0 for frame in source_frames(path): frames.append(frame) n += 1 if len(frames) == 24: blocks.append( resize_frames(frames, cfg["height"], cfg["width"], "cpu").to( torch.float16 ) ) if fine_shape: fine_blocks.append( resize_frames(frames, *fine_shape, "cpu").to(torch.float16) ) frames = [] if frames: blocks.append( resize_frames(frames, cfg["height"], cfg["width"], "cpu").to( torch.float16 ) ) if fine_shape: fine_blocks.append( resize_frames(frames, *fine_shape, "cpu").to(torch.float16) ) if n != B: raise ValueError("Decoded source frame count mismatch") block = (torch.cat(blocks, dim=0), H, A) result.append((*block, torch.cat(fine_blocks, dim=0)) if fine_shape else block) return result @torch.inference_mode() def predict_prepared( row, prepared, net, cfg, device, batch_size=24, tta=False, progress=None ): if cfg.get("architecture") == "boundary_volume_refine_v1": from src.inference.refinement import predict_volume return predict_volume(row, prepared, net, cfg, device, batch_size, tta) if cfg.get("architecture") == "boundary_zero_atom_v1": from src.inference.density import predict_zero_atom return predict_zero_atom(row, prepared, net, cfg, device, batch_size, tta) if cfg.get("architecture") == "boundary_profile_refine_v1": from src.inference.refinement import predict_cascade return predict_cascade(row, prepared, net, cfg, device, batch_size, tta) if cfg.get("architecture") == "boundary_ensemble_v1": if cfg.get("reduce") == "zero_probability_mean": from src.inference.density import predict_zero_density_ensemble return predict_zero_density_ensemble( row, prepared, net, cfg, device, batch_size, tta ) if cfg.get("reduce") == "probability_mean": from src.inference.density import predict_density_ensemble return predict_density_ensemble( row, prepared, net, cfg, device, batch_size, tta, progress=progress ) arrays = [] keys = None for i, (component, model) in enumerate(zip(cfg["components"], net.nets)): child = component["config"] values, child_keys = predict_prepared( row, prepared, model, child, device, batch_size, tta or child.get("tta_horizontal", False), progress=progress if i == len(cfg["components"]) - 1 else None, ) values = apply_policy(values, row["device"], child.get("prediction_policy")) values = apply_policy(values, row["device"], component.get("post_policy")) if keys is None: keys = child_keys if set(keys) != set(child_keys): raise ValueError("Ensemble output vocabulary mismatch") values = values[[list(child_keys).index(k) for k in keys]] if arrays and values.shape != arrays[0].shape: raise ValueError("Ensemble output geometry mismatch") arrays.append(values) if cfg.get("reduce", "mean") == "median": return np.median(np.stack(arrays, axis=0), axis=0).astype("float32"), keys if cfg.get("reduce") == "edge_expert": threshold = cfg.get("edge_threshold", 3.0) if len(arrays) != 2 or not np.isfinite(threshold) or threshold < 0: raise ValueError("Invalid ordinary-coordinate edge expert") edge = 0.0 if cfg.get("edge_snap_zero", True) else arrays[1] return ( np.where(arrays[1] <= threshold, edge, arrays[0]).astype("float32"), keys, ) positions = np.zeros_like(arrays[0], dtype="float32") for component, values in zip(cfg["components"], arrays): positions += np.float32(component["weight"]) * values return np.clip(positions, 0, prepared[0][1] - 1), keys prepared = [p[:3] for p in prepared] family = row["device"] if family not in cfg["boundary_keys"]: raise ValueError("Unsupported device family") chunks = [] geometry = None radius = cfg.get("context_radius", 0) if radius and len(prepared) > 1: H, A = prepared[0][1:] if any((h, w) != (H, A) for _, h, w in prepared): raise ValueError("Inconsistent concatenated source grids") arrays = [x for x, _, _ in prepared] combined = ( np.concatenate(arrays, axis=0) if isinstance(arrays[0], np.ndarray) else torch.cat(arrays, dim=0) ) prepared = [(combined, H, A)] for pixels, H, A in prepared: if geometry is not None and geometry != (H, A): raise ValueError("Inconsistent source file grids") geometry = (H, A) outs = [] for start in range(0, len(pixels), batch_size): x = context_batch( pixels, start, min(start + batch_size, len(pixels)), radius, device ) with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): p = decode_logits( net(x, family), cfg.get("local_radius", 6), cfg.get("decode_method", "local_centroid"), ) if tta: p = ( p + decode_logits( net(x.flip(-1), family), cfg.get("local_radius", 6), cfg.get("decode_method", "local_centroid"), ).flip(-1) ) / 2 p = F.interpolate(p, size=A, mode="linear", align_corners=True) * ( (H - 1) / (cfg["height"] - 1) ) outs.append(p.clamp(0, H - 1).cpu().numpy().astype("float32")) if progress: completed = sum(part.shape[1] for part in chunks) progress( completed + min(start + batch_size, len(pixels)), sum(len(packet[0]) for packet in prepared), ) chunks.append(np.concatenate(outs, axis=0).transpose(1, 0, 2)) return np.concatenate(chunks, axis=1), np.asarray(cfg["boundary_keys"][family]) def ordered_prefetch(rows, root, cfg, workers): """At most workers+1 prepared volumes; Python open guard covers all threads.""" import collections import concurrent.futures with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: iterator = iter(rows) pending = collections.deque() for _ in range(min(workers, len(rows))): row = next(iterator) pending.append((row, pool.submit(prepare_pixels, row, root, cfg))) while pending: row, future = pending.popleft() pixels = future.result() nextrow = next(iterator, None) if nextrow is not None: pending.append( (nextrow, pool.submit(prepare_pixels, nextrow, root, cfg)) ) yield row, pixels def ordered_process_prefetch(rows, root, cfg, workers, open_audit): """Bounded spawn workers decode OCT independently of the CUDA process.""" import collections import concurrent.futures import multiprocessing from src.io import initialize_pixel_worker, prepare_pixel_packet open_audit.update( decoder_backend="processes_spawn", decoder_processes_configured=workers, worker_oct_open_events=0, ) with concurrent.futures.ProcessPoolExecutor( max_workers=workers, mp_context=multiprocessing.get_context("spawn"), initializer=initialize_pixel_worker, initargs=(str(root), rows), ) as pool: iterator = iter(rows) pending = collections.deque() for _ in range(min(workers, len(rows))): row = next(iterator) pending.append((row, pool.submit(prepare_pixel_packet, (row, root, cfg)))) while pending: row, future = pending.popleft() pixels, delta = future.result() for key, value in delta.items(): open_audit[key] += value open_audit["worker_oct_open_events"] += delta["oct_open_events"] nextrow = next(iterator, None) if nextrow is not None: pending.append( (nextrow, pool.submit(prepare_pixel_packet, (nextrow, root, cfg))) ) yield row, pixels def main(): p = argparse.ArgumentParser() p.add_argument("--input-manifest", type=Path, required=True) p.add_argument("--data-root", type=Path, required=True) p.add_argument("--checkpoint", type=Path, required=True) p.add_argument("--output-dir", type=Path, required=True) p.add_argument("--batch-size", type=int, default=64) p.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") p.add_argument("--tta", action="store_true") p.add_argument("--threads", type=int, default=1) p.add_argument("--decode-workers", type=int, default=16) p.add_argument( "--decode-backend", choices=["threads", "processes"], default="processes" ) a = p.parse_args() torch.set_num_threads(a.threads) device = torch.device(a.device) net, cfg = load_model(a.checkpoint, device) rows = read_input(a.input_manifest) open_audit = enforce_input_opens(a.data_root, rows) a.output_dir.mkdir(parents=True, exist_ok=False) t = time.time() tta = a.tta or cfg.get("tta_horizontal", False) workers = max(1, min(a.decode_workers, len(rows))) if a.decode_workers: stream = ( ordered_process_prefetch(rows, a.data_root, cfg, workers, open_audit) if a.decode_backend == "processes" else ordered_prefetch(rows, a.data_root, cfg, workers) ) else: stream = ((r, None) for r in rows) for i, (r, prepared) in enumerate(stream): if not re.fullmatch("[A-Za-z0-9_-]{1,64}", r["case_id"]): raise ValueError("Unsafe case id") positions, keys = ( predict_prepared(r, prepared, net, cfg, device, a.batch_size, tta) if prepared is not None else predict_case(r, a.data_root, net, cfg, device, a.batch_size, tta) ) positions = apply_policy(positions, r["device"], cfg.get("prediction_policy")) if not np.isfinite(positions).all(): raise ValueError("Nonfinite prediction") height = ( prepared[0][1] if prepared is not None else int(structural_header(safe_path(a.data_root, r["oct_paths"][0])).Rows) ) positions = np.clip(positions, 0, height - 1).astype("float32") np.savez_compressed( a.output_dir / (r["case_id"] + ".npz"), positions=positions, boundary_keys=keys, ) if (i + 1) % 10 == 0 or i + 1 == len(rows): print( json.dumps( { "predicted_cases": i + 1, "total": len(rows), "seconds": round(time.time() - t, 2), } ), flush=True, ) source_hashes = { name: hashlib.sha256( (Path(__file__).resolve().parents[2] / name).read_bytes() ).hexdigest() for name in [ "predict.py", "src/inference/predict.py", "src/io.py", "src/models/unet.py", "src/models/__init__.py", "src/inference/density.py", "src/models/refiners.py", "src/inference/refinement.py", "src/models/volume.py", "src/inference/__init__.py", "src/protocol.py", ] } (a.output_dir / "inference-receipt.json").write_text( json.dumps( { "cases": len(rows), "seconds": time.time() - t, "checkpoint": a.checkpoint.name, "config": cfg, "source_hashes": source_hashes, "torch": torch.__version__, "execution_device": str(device), "batch_size": a.batch_size, "threads": a.threads, "input_fields": ["case_id", "device", "oct_paths"], "segmentation_access": False, "input_open_audit": open_audit, "tta": tta, "prediction_policy": cfg.get("prediction_policy"), "decode_workers": a.decode_workers, "decode_backend": a.decode_backend, "preprocessing_storage_dtype": ( "float16" if a.decode_workers else "float32" ), }, indent=2, ) ) if __name__ == "__main__": main()