"""NumPy profile model, checkpoint I/O, and label-free inference.""" from __future__ import annotations import argparse import json import re import time from pathlib import Path import numpy as np from src.io import ( enforce_input_opens, normalize_frame, read_input, safe_path, source_frames, structural_header, ) def resize_axis(x, n, axis): old = x.shape[axis] if old == n: return x grid = np.linspace(0, old - 1, n, dtype=np.float32) lo = np.floor(grid).astype(int) hi = np.minimum(lo + 1, old - 1) w = (grid - lo).astype(np.float32) shape = [1] * x.ndim shape[axis] = n w = w.reshape(shape) return (np.take(x, lo, axis=axis) * (1 - w) + np.take(x, hi, axis=axis) * w).astype( np.float32 ) def box_filter(x, n, axis): if n <= 1: return x if n % 2 == 0: n += 1 pads = [(0, 0)] * x.ndim pads[axis] = (n // 2, n - 1 - n // 2) padded = np.pad(x, pads, mode="edge") zero_shape = list(padded.shape) zero_shape[axis] = 1 cumulative = np.concatenate( [ np.zeros(zero_shape, np.float32), np.cumsum(padded, axis=axis, dtype=np.float32), ], axis=axis, ) upper = [slice(None)] * x.ndim lower = [slice(None)] * x.ndim upper[axis] = slice(n, None) lower[axis] = slice(None, -n) return (cumulative[tuple(upper)] - cumulative[tuple(lower)]) / n def frame_features(frame, cfg): x = normalize_frame(frame) if cfg.get("antialias", True): x = box_filter(x, max(1, int(round(x.shape[0] / cfg["profile_height"]))), 0) x = box_filter(x, max(1, int(round(x.shape[1] / cfg["width"]))), 1) x = resize_axis(resize_axis(x, cfg["profile_height"], 0), cfg["width"], 1) width = x.shape[1] radius = cfg["context_radius"] cols = np.arange(width) features = np.concatenate( [ x[:, np.clip(cols + offset, 0, width - 1)].T for offset in [-radius, 0, radius] ], axis=1, ) return features.astype(np.float32) class ProfileMLP: def __init__(self, dims, rng): self.weights = [ (rng.standard_normal((a, b)) * np.sqrt(2 / a)).astype(np.float32) for a, b in zip(dims[:-1], dims[1:]) ] self.biases = [np.zeros(b, np.float32) for b in dims[1:]] self.weights[-1] *= 0.05 self.biases[-1][:] = 0.5 def forward(self, x, retain=False): a = x * 2 - 1 acts = [a] pre = [] for i, (w, b) in enumerate(zip(self.weights, self.biases)): z = a @ w + b pre.append(z) a = np.maximum(z, 0) if i < len(self.weights) - 1 else z acts.append(a) return (a, (acts, pre)) if retain else a def gradients_from_cache(self, dy, cache): acts, pre = cache grads = [] for i in range(len(self.weights) - 1, -1, -1): dw = acts[i].T @ dy db = dy.sum(axis=0) grads.append((dw, db)) if i: dy = dy @ self.weights[i].T dy *= pre[i - 1] > 0 return list(reversed(grads)) class Adam: def __init__(self, net, weight_decay=1e-4): self.m = [ (np.zeros_like(w), np.zeros_like(b)) for w, b in zip(net.weights, net.biases) ] self.v = [ (np.zeros_like(w), np.zeros_like(b)) for w, b in zip(net.weights, net.biases) ] self.t = 0 self.weight_decay = weight_decay def step(self, net, grads, lr, clip=5.0): norm = np.sqrt(sum(float((g * g).sum()) for pair in grads for g in pair)) scale = min(1.0, clip / max(norm, 1e-12)) self.t += 1 beta1 = 0.9 beta2 = 0.999 for i, pair in enumerate(grads): for j, (param, grad) in enumerate( zip([net.weights[i], net.biases[i]], pair) ): grad = grad * scale self.m[i][j][:] = beta1 * self.m[i][j] + (1 - beta1) * grad self.v[i][j][:] = beta2 * self.v[i][j] + (1 - beta2) * grad * grad update = (self.m[i][j] / (1 - beta1**self.t)) / ( np.sqrt(self.v[i][j] / (1 - beta2**self.t)) + 1e-8 ) if j == 0: param *= 1 - lr * self.weight_decay param -= lr * update return float(norm) def save_checkpoint(path, models, cfg, provenance): arrays = { "config_json": np.asarray(json.dumps(cfg, sort_keys=True)), "provenance_json": np.asarray(json.dumps(provenance, sort_keys=True)), } for device, net in models.items(): for i, (w, b) in enumerate(zip(net.weights, net.biases)): arrays[device + "__w" + str(i)] = w arrays[device + "__b" + str(i)] = b path = Path(path) tmp = path.with_suffix(".tmp.npz") np.savez_compressed(tmp, **arrays) tmp.replace(path) def load_checkpoint(path): with np.load(path, allow_pickle=False) as data: cfg = json.loads(str(data["config_json"])) provenance = json.loads(str(data["provenance_json"])) models = {} for device, keys in cfg["boundary_keys"].items(): dims = [cfg["profile_height"] * 3] + cfg["hidden"] + [len(keys)] net = ProfileMLP(dims, np.random.default_rng(0)) for i in range(len(net.weights)): net.weights[i] = data[device + "__w" + str(i)].copy() net.biases[i] = data[device + "__b" + str(i)].copy() models[device] = net return models, cfg, provenance def predict_case(row, root, models, cfg, batch_size): device = row["device"] net = models[device] chunks = [] shape = None for rel in row["oct_paths"]: path = safe_path(root, rel) h = structural_header(path) B, H, A = int(h.NumberOfFrames), int(h.Rows), int(h.Columns) if shape is not None and shape != (H, A): raise ValueError("Inconsistent source grids") shape = (H, A) frames = [] parts = [] n = 0 def apply(frames): x = np.stack([frame_features(f, cfg) for f in frames]) pred = net.forward(x.reshape(-1, x.shape[-1])).reshape( len(x), cfg["width"], -1 ) pred = np.clip(pred, 0, 1) * (H - 1) pred = pred.transpose(2, 0, 1) return resize_axis(pred, A, 2) for frame in source_frames(path): frames.append(frame) n += 1 if len(frames) == batch_size: parts.append(apply(frames)) frames = [] if frames: parts.append(apply(frames)) if n != B: raise ValueError("Source frame count changed") chunks.append(np.concatenate(parts, axis=1)) return np.concatenate(chunks, axis=1).astype(np.float32), np.asarray( cfg["boundary_keys"][device] ) def predict_main(): p = argparse.ArgumentParser() p.add_argument("--input-manifest", type=Path, required=True) p.add_argument("--data-root", type=Path, required=True) p.add_argument("--checkpoint", type=Path, required=True) p.add_argument("--output-dir", type=Path, required=True) p.add_argument("--batch-size", type=int, default=16) a = p.parse_args() models, cfg, provenance = load_checkpoint(a.checkpoint) rows = read_input(a.input_manifest) open_audit = enforce_input_opens(a.data_root, rows) a.output_dir.mkdir(parents=True, exist_ok=False) t = time.time() for i, row in enumerate(rows): if not re.fullmatch("[A-Za-z0-9_-]{1,64}", row["case_id"]): raise ValueError("Unsafe case id") positions, keys = predict_case(row, a.data_root, models, cfg, a.batch_size) if not np.isfinite(positions).all(): raise ValueError("Nonfinite prediction") np.savez_compressed( a.output_dir / (row["case_id"] + ".npz"), positions=positions, boundary_keys=keys, ) if (i + 1) % 10 == 0 or i + 1 == len(rows): print( json.dumps( { "predicted_cases": i + 1, "total": len(rows), "seconds": round(time.time() - t, 2), } ), flush=True, ) (a.output_dir / "inference-receipt.json").write_text( json.dumps( { "cases": len(rows), "seconds": time.time() - t, "config": cfg, "numpy": np.__version__, "input_fields": ["case_id", "device", "oct_paths"], "segmentation_access": False, "input_open_audit": open_audit, }, indent=2, ) )