"""Density ensembles, zero-coordinate mixtures, and encoded-B smoothing.""" import argparse import hashlib import json import re import time from pathlib import Path import numpy as np import torch from torch.nn import functional as F from src.inference import apply_policy from src.io import context_batch, enforce_input_opens, read_input from src.models.unet import decode_logits, probability_median PLAIN = { "boundary_unet_v1", "boundary_unet_axial_v1", "boundary_unet_global_v1", "boundary_unet_spatial_v1", } WITH_ZERO = PLAIN | {"boundary_zero_atom_v1"} @torch.inference_mode() def predict_density_ensemble( row, prepared, net, cfg, device, batch_size, tta, progress=None ): if cfg.get("density_decoder") != "posterior_median": raise ValueError("Unsupported probability-mixture decision") packets = [p[:3] for p in prepared] H, A = packets[0][1:] if any(p[1:] != (H, A) for p in packets): raise ValueError("Inconsistent source grids") items = [p[0] for p in packets] pixels = ( items[0] if len(items) == 1 else ( np.concatenate(items, 0) if isinstance(items[0], np.ndarray) else torch.cat(items, 0) ) ) family = row["device"] out = [] weights = [c["weight"] for c in cfg["components"]] if ( any(not np.isfinite(w) or w <= 0 for w in weights) or abs(sum(weights) - 1) > 1e-6 ): raise ValueError("Invalid probability-mixture weights") for start in range(0, len(pixels), batch_size): end = min(start + batch_size, len(pixels)) mixture = None for component, model in zip(cfg["components"], net.nets): child = component["config"] radius = child.get("context_radius", 0) if ( child["architecture"] not in PLAIN or child.get("prediction_policy") or component.get("post_policy") or child.get("tta_horizontal", False) ): raise ValueError( "Density mixture requires plain, unfiltered component logits" ) x = context_batch(pixels, start, end, radius, device) with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): z = model(x, family) p = z.float().softmax(2) if tta: p = (p + model(x.flip(-1), family).float().softmax(2).flip(-1)) / 2 weighted = p * float(component["weight"]) mixture = weighted if mixture is None else mixture + weighted if cfg.get("density_native_columns", False): mixture = F.interpolate( mixture, size=(mixture.shape[-2], A), mode="bilinear", align_corners=True, ) mixture = mixture / mixture.sum(2, keepdim=True).clamp_min(1e-12) positions = probability_median(mixture) positions = F.interpolate( positions, size=A, mode="linear", align_corners=True ) * ((H - 1) / (cfg["height"] - 1)) out.append(positions.clamp(0, H - 1).cpu().numpy().astype("float32")) if progress: progress(end, len(pixels)) return np.concatenate(out, 0).transpose(1, 0, 2), np.asarray( cfg["boundary_keys"][family] ) @torch.inference_mode() def predict_zero_density_ensemble(row, prepared, net, cfg, device, batch_size, tta): """Average unconditional densities and literal-zero atoms before deciding.""" if cfg.get("density_decoder") != "joint_zero_positive_median" or not cfg.get( "density_native_columns" ): raise ValueError("Zero-density mixtures require a native-column joint median") packets = [p[:3] for p in prepared] H, A = packets[0][1:] if any(p[1:] != (H, A) for p in packets): raise ValueError("Inconsistent source grids") items = [p[0] for p in packets] pixels = ( items[0] if len(items) == 1 else ( np.concatenate(items, 0) if isinstance(items[0], np.ndarray) else torch.cat(items, 0) ) ) family = row["device"] out = [] weights = [c["weight"] for c in cfg["components"]] if ( any(not np.isfinite(w) or w <= 0 for w in weights) or abs(sum(weights) - 1) > 1e-6 ): raise ValueError("Invalid zero-density mixture weights") for start in range(0, len(pixels), batch_size): end = min(start + batch_size, len(pixels)) total_density = None total_atom = None for component, model in zip(cfg["components"], net.nets): child = component["config"] radius = child.get("context_radius", 0) if ( child["architecture"] not in WITH_ZERO or child.get("prediction_policy") or component.get("post_policy") or child.get("tta_horizontal", False) ): raise ValueError("Unsupported filtered zero-density component") x = context_batch(pixels, start, end, radius, device) def component_mass(image): result = model(image, family) if child["architecture"] == "boundary_zero_atom_v1": logits, zero_logits = result atom = F.interpolate( zero_logits.float(), size=A, mode="linear", align_corners=True ).sigmoid() else: logits = result atom = torch.zeros( (len(image), logits.shape[1], A), device=device, dtype=torch.float32, ) probability = logits.float().softmax(2) probability = F.interpolate( probability, size=(probability.shape[-2], A), mode="bilinear", align_corners=True, ) probability = probability / probability.sum(2, keepdim=True).clamp_min( 1e-12 ) return probability * (1 - atom[:, :, None, :]), atom with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): density, atom = component_mass(x) if tta: flipped_density, flipped_atom = component_mass(x.flip(-1)) density = (density + flipped_density.flip(-1)) / 2 atom = (atom + flipped_atom.flip(-1)) / 2 density = density * float(component["weight"]) atom = atom * float(component["weight"]) total_density = ( density if total_density is None else total_density + density ) total_atom = atom if total_atom is None else total_atom + atom positions = zero_mixture_median(total_density, total_atom) * ( (H - 1) / (cfg["height"] - 1) ) out.append(positions.clamp(0, H - 1).cpu().numpy().astype("float32")) return np.concatenate(out, 0).transpose(1, 0, 2), np.asarray( cfg["boundary_keys"][family] ) def zero_mixture_median(probability, zero_probability): """Median of a literal-zero atom and a conditional row-bin density.""" p = probability.float() p = p / p.sum(-2, keepdim=True).clamp_min(1e-12) q = zero_probability.float() tau = ((0.5 - q) / (1 - q).clamp_min(1e-12)).clamp(0, 1).unsqueeze(-2) cdf = p.cumsum(-2) index = (cdf >= tau).to(torch.int64).argmax(-2, keepdim=True) previous = cdf.gather(-2, (index - 1).clamp_min(0)) previous = torch.where(index == 0, torch.zeros_like(previous), previous) mass = p.gather(-2, index).clamp_min(1e-12) value = ( (index.float() - 0.5 + ((tau - previous) / mass).clamp(0, 1)) .clamp(0, p.shape[-2] - 1) .squeeze(-2) ) return torch.where(q >= 0.5, torch.zeros_like(value), value) @torch.inference_mode() def predict_joint_zero_median(row, prepared, net, cfg, device, batch_size, tta): """Combine distributions before deciding, including native-column mapping.""" packets = [p[:3] for p in prepared] H, A = packets[0][1:] family = row["device"] radius = cfg.get("context_radius", 0) if any(p[1:] != (H, A) for p in packets): raise ValueError("Inconsistent zero-mixture grids") arrays = [p[0] for p in packets] pixels = ( arrays[0] if len(arrays) == 1 else ( np.concatenate(arrays, 0) if isinstance(arrays[0], np.ndarray) else torch.cat(arrays, 0) ) ) outs = [] for start in range(0, len(pixels), batch_size): end = min(start + batch_size, len(pixels)) x = context_batch(pixels, start, end, radius, device) with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): z, q = net(x, family) probability = z.float().softmax(2) q = F.interpolate( q.float(), size=A, mode="linear", align_corners=True ).sigmoid() if tta: zf, qf = net(x.flip(-1), family) probability = (probability + zf.float().softmax(2).flip(-1)) / 2 q = ( q + F.interpolate( qf.float(), size=A, mode="linear", align_corners=True ) .sigmoid() .flip(-1) ) / 2 probability = F.interpolate( probability, size=(probability.shape[-2], A), mode="bilinear", align_corners=True, ) values = zero_mixture_median(probability, q) * ((H - 1) / (cfg["height"] - 1)) outs.append(values.clamp(0, H - 1).cpu().numpy().astype("float32")) return np.concatenate(outs, 0).transpose(1, 0, 2), np.asarray( cfg["boundary_keys"][family] ) @torch.inference_mode() def predict_zero_atom(row, prepared, net, cfg, device, batch_size, tta): decision = cfg.get("zero_decision", "threshold_positive") if decision == "mixture_median": return predict_joint_zero_median( row, prepared, net, cfg, device, batch_size, tta ) if decision != "threshold_positive": raise ValueError("Unknown coordinate-zero decision") prepared = [p[:3] for p in prepared] H, A = prepared[0][1:] family = row["device"] radius = cfg.get("context_radius", 0) threshold = cfg.get("zero_threshold", 0.5) if not 0 < threshold < 1 or any(p[1:] != (H, A) for p in prepared): raise ValueError("Invalid zero-atom geometry/threshold") arrays = [p[0] for p in prepared] pixels = ( arrays[0] if len(arrays) == 1 else ( np.concatenate(arrays, 0) if isinstance(arrays[0], np.ndarray) else torch.cat(arrays, 0) ) ) outs = [] child = cfg["coarse_config"] def run(x): z, q = net(x, family) p = decode_logits( z, child.get("local_radius", 6), child.get("decode_method", "local_centroid"), ) p = F.interpolate(p, size=A, mode="linear", align_corners=True) * ( (H - 1) / (cfg["height"] - 1) ) q = F.interpolate( q.float(), size=A, mode="linear", align_corners=True ).sigmoid() return p, q for start in range(0, len(pixels), batch_size): end = min(start + batch_size, len(pixels)) x = context_batch(pixels, start, end, radius, device) with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): p, q = run(x) if tta: pf, qf = run(x.flip(-1)) p = (p + pf.flip(-1)) / 2 q = (q + qf.flip(-1)) / 2 values = torch.where(q >= threshold, torch.zeros_like(p), p) outs.append(values.clamp(0, H - 1).cpu().numpy().astype("float32")) return np.concatenate(outs, 0).transpose(1, 0, 2), np.asarray( cfg["boundary_keys"][family] ) def kernel(sigma): sigma = float(sigma) if not np.isfinite(sigma) or not 0 <= sigma <= 2: raise ValueError("B density sigma must lie in [0,2]") radius = int(4 * sigma + 0.5) if radius == 0: return np.array([0], dtype="int64"), np.array([1.0], dtype="float64") offsets = np.arange(-radius, radius + 1, dtype="int64") weights = np.exp(-0.5 * (offsets / sigma) ** 2) weights /= weights.sum() return offsets, weights def mix_b(probability, start, end, block_start, frames, sigma): """Central B frames from a halo block; replicate encoded-volume endpoints.""" offsets, weights = kernel(sigma) result = None if len(offsets) == 1: return probability[start - block_start : end - block_start] for offset, weight in zip(offsets, weights): indices = np.clip(np.arange(start, end) + offset, 0, frames - 1) - block_start if indices.min() < 0 or indices.max() >= len(probability): raise ValueError("Probability halo is incomplete") part = probability.index_select( 0, torch.as_tensor(indices, device=probability.device) ) value = part * float(weight) result = value if result is None else result + value return result / result.sum(2, keepdim=True).clamp_min(1e-12) @torch.inference_mode() def predict_bprob(row, prepared, net, cfg, device, batch_size=64, progress=None): if ( cfg["architecture"] not in PLAIN or cfg.get("inference_wrapper") != "b_probability_smooth_v1" or cfg.get("decode_method") != "posterior_median" or cfg.get("tta_horizontal", False) ): raise ValueError("Unsupported B-density model configuration") packets = [p[:3] for p in prepared] height, width = packets[0][1:] if any(p[1:] != (height, width) for p in packets): raise ValueError("Inconsistent structural source grids") arrays = [p[0] for p in packets] pixels = ( arrays[0] if len(arrays) == 1 else ( np.concatenate(arrays, 0) if isinstance(arrays[0], np.ndarray) else torch.cat(arrays, 0) ) ) if not len(pixels): raise ValueError("Empty structural volume") sigma = cfg["density_b_sigma"] offsets, _ = kernel(sigma) halo = int(offsets[-1]) context = cfg.get("context_radius", 0) family = row["device"] output = [] if family not in cfg["boundary_keys"]: raise ValueError("Unsupported device family") for start in range(0, len(pixels), batch_size): end = min(start + batch_size, len(pixels)) lo = max(0, start - halo) hi = min(len(pixels), end + halo) image = context_batch(pixels, lo, hi, context, device) with torch.autocast( device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda" ): probability = net(image, family).float().softmax(2) probability = mix_b(probability, start, end, lo, len(pixels), sigma) positions = probability_median(probability) positions = F.interpolate( positions, size=width, mode="linear", align_corners=True ) * ((height - 1) / (cfg["height"] - 1)) output.append(positions.clamp(0, height - 1).cpu().numpy().astype("float32")) if progress: progress(end, len(pixels)) return np.concatenate(output, 0).transpose(1, 0, 2), np.asarray( cfg["boundary_keys"][family] ) def bprob_main(): from src.inference.predict import ( load_model, ordered_prefetch, ordered_process_prefetch, prepare_pixels, ) p = argparse.ArgumentParser() for key in ["input-manifest", "data-root", "checkpoint", "output-dir"]: p.add_argument("--" + key, type=Path, required=True) p.add_argument("--batch-size", type=int, default=64) p.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") p.add_argument("--threads", type=int, default=1) p.add_argument("--decode-workers", type=int, default=16) p.add_argument( "--decode-backend", choices=["threads", "processes"], default="processes" ) a = p.parse_args() if a.batch_size < 1 or a.decode_workers < 0: raise ValueError("Invalid inference batch/worker count") torch.set_num_threads(a.threads) device = torch.device(a.device) net, cfg = load_model(a.checkpoint, device) offsets, weights = kernel(cfg["density_b_sigma"]) rows = read_input(a.input_manifest) audit = enforce_input_opens(a.data_root, rows) a.output_dir.mkdir(parents=True, exist_ok=False) start = time.time() workers = max(1, min(a.decode_workers, len(rows))) stream = ( ( ordered_process_prefetch(rows, a.data_root, cfg, workers, audit) if a.decode_backend == "processes" else ordered_prefetch(rows, a.data_root, cfg, workers) ) if a.decode_workers else ((row, prepare_pixels(row, a.data_root, cfg)) for row in rows) ) for i, (row, prepared) in enumerate(stream): if not re.fullmatch("[A-Za-z0-9_-]{1,64}", row["case_id"]): raise ValueError("Unsafe case identifier") positions, keys = predict_bprob(row, prepared, net, cfg, device, a.batch_size) positions = apply_policy(positions, row["device"], cfg.get("prediction_policy")) height = prepared[0][1] if not np.isfinite(positions).all(): raise ValueError("Nonfinite prediction") np.savez_compressed( a.output_dir / (row["case_id"] + ".npz"), positions=np.clip(positions, 0, height - 1).astype("float32"), boundary_keys=keys, ) if (i + 1) % 10 == 0 or i + 1 == len(rows): print( json.dumps( { "predicted_cases": i + 1, "total": len(rows), "seconds": round(time.time() - start, 2), } ), flush=True, ) files = [ "predict.py", "src/inference/density.py", "src/inference/predict.py", "src/io.py", "src/models/unet.py", "src/models/__init__.py", "src/inference/__init__.py", "src/protocol.py", ] root = Path(__file__).resolve().parents[2] receipt = { "cases": len(rows), "seconds": time.time() - start, "checkpoint": a.checkpoint.name, "config": cfg, "source_hashes": { f: hashlib.sha256((root / f).read_bytes()).hexdigest() for f in files }, "torch": torch.__version__, "execution_device": str(device), "batch_size": a.batch_size, "threads": a.threads, "input_fields": ["case_id", "device", "oct_paths"], "segmentation_access": False, "input_open_audit": audit, "tta": False, "prediction_policy": cfg.get("prediction_policy"), "decode_workers": a.decode_workers, "decode_backend": a.decode_backend, "preprocessing_storage_dtype": "float16", "inference_wrapper": "b_probability_smooth_v1", "density_b_sigma": cfg["density_b_sigma"], "encoded_B_kernel_offsets": offsets.tolist(), "encoded_B_kernel_weights": weights.tolist(), "density_filter_precedes_median": True, } (a.output_dir / "inference-receipt.json").write_text( json.dumps(receipt, indent=2) + "\n" )