OCT_Segmentation_a2 / src /profile.py
farrell236's picture
Deploy OCT viewer runtime and bundled checkpoint
bf88397 verified
Raw History Blame Contribute Delete
8.97 kB
"""NumPy profile model, checkpoint I/O, and label-free inference."""
from __future__ import annotations
import argparse
import json
import re
import time
from pathlib import Path
import numpy as np
from src.io import (
enforce_input_opens,
normalize_frame,
read_input,
safe_path,
source_frames,
structural_header,
)
def resize_axis(x, n, axis):
old = x.shape[axis]
if old == n:
return x
grid = np.linspace(0, old - 1, n, dtype=np.float32)
lo = np.floor(grid).astype(int)
hi = np.minimum(lo + 1, old - 1)
w = (grid - lo).astype(np.float32)
shape = [1] * x.ndim
shape[axis] = n
w = w.reshape(shape)
return (np.take(x, lo, axis=axis) * (1 - w) + np.take(x, hi, axis=axis) * w).astype(
np.float32
)
def box_filter(x, n, axis):
if n <= 1:
return x
if n % 2 == 0:
n += 1
pads = [(0, 0)] * x.ndim
pads[axis] = (n // 2, n - 1 - n // 2)
padded = np.pad(x, pads, mode="edge")
zero_shape = list(padded.shape)
zero_shape[axis] = 1
cumulative = np.concatenate(
[
np.zeros(zero_shape, np.float32),
np.cumsum(padded, axis=axis, dtype=np.float32),
],
axis=axis,
)
upper = [slice(None)] * x.ndim
lower = [slice(None)] * x.ndim
upper[axis] = slice(n, None)
lower[axis] = slice(None, -n)
return (cumulative[tuple(upper)] - cumulative[tuple(lower)]) / n
def frame_features(frame, cfg):
x = normalize_frame(frame)
if cfg.get("antialias", True):
x = box_filter(x, max(1, int(round(x.shape[0] / cfg["profile_height"]))), 0)
x = box_filter(x, max(1, int(round(x.shape[1] / cfg["width"]))), 1)
x = resize_axis(resize_axis(x, cfg["profile_height"], 0), cfg["width"], 1)
width = x.shape[1]
radius = cfg["context_radius"]
cols = np.arange(width)
features = np.concatenate(
[
x[:, np.clip(cols + offset, 0, width - 1)].T
for offset in [-radius, 0, radius]
],
axis=1,
)
return features.astype(np.float32)
class ProfileMLP:
def __init__(self, dims, rng):
self.weights = [
(rng.standard_normal((a, b)) * np.sqrt(2 / a)).astype(np.float32)
for a, b in zip(dims[:-1], dims[1:])
]
self.biases = [np.zeros(b, np.float32) for b in dims[1:]]
self.weights[-1] *= 0.05
self.biases[-1][:] = 0.5
def forward(self, x, retain=False):
a = x * 2 - 1
acts = [a]
pre = []
for i, (w, b) in enumerate(zip(self.weights, self.biases)):
z = a @ w + b
pre.append(z)
a = np.maximum(z, 0) if i < len(self.weights) - 1 else z
acts.append(a)
return (a, (acts, pre)) if retain else a
def gradients_from_cache(self, dy, cache):
acts, pre = cache
grads = []
for i in range(len(self.weights) - 1, -1, -1):
dw = acts[i].T @ dy
db = dy.sum(axis=0)
grads.append((dw, db))
if i:
dy = dy @ self.weights[i].T
dy *= pre[i - 1] > 0
return list(reversed(grads))
class Adam:
def __init__(self, net, weight_decay=1e-4):
self.m = [
(np.zeros_like(w), np.zeros_like(b))
for w, b in zip(net.weights, net.biases)
]
self.v = [
(np.zeros_like(w), np.zeros_like(b))
for w, b in zip(net.weights, net.biases)
]
self.t = 0
self.weight_decay = weight_decay
def step(self, net, grads, lr, clip=5.0):
norm = np.sqrt(sum(float((g * g).sum()) for pair in grads for g in pair))
scale = min(1.0, clip / max(norm, 1e-12))
self.t += 1
beta1 = 0.9
beta2 = 0.999
for i, pair in enumerate(grads):
for j, (param, grad) in enumerate(
zip([net.weights[i], net.biases[i]], pair)
):
grad = grad * scale
self.m[i][j][:] = beta1 * self.m[i][j] + (1 - beta1) * grad
self.v[i][j][:] = beta2 * self.v[i][j] + (1 - beta2) * grad * grad
update = (self.m[i][j] / (1 - beta1**self.t)) / (
np.sqrt(self.v[i][j] / (1 - beta2**self.t)) + 1e-8
)
if j == 0:
param *= 1 - lr * self.weight_decay
param -= lr * update
return float(norm)
def save_checkpoint(path, models, cfg, provenance):
arrays = {
"config_json": np.asarray(json.dumps(cfg, sort_keys=True)),
"provenance_json": np.asarray(json.dumps(provenance, sort_keys=True)),
}
for device, net in models.items():
for i, (w, b) in enumerate(zip(net.weights, net.biases)):
arrays[device + "__w" + str(i)] = w
arrays[device + "__b" + str(i)] = b
path = Path(path)
tmp = path.with_suffix(".tmp.npz")
np.savez_compressed(tmp, **arrays)
tmp.replace(path)
def load_checkpoint(path):
with np.load(path, allow_pickle=False) as data:
cfg = json.loads(str(data["config_json"]))
provenance = json.loads(str(data["provenance_json"]))
models = {}
for device, keys in cfg["boundary_keys"].items():
dims = [cfg["profile_height"] * 3] + cfg["hidden"] + [len(keys)]
net = ProfileMLP(dims, np.random.default_rng(0))
for i in range(len(net.weights)):
net.weights[i] = data[device + "__w" + str(i)].copy()
net.biases[i] = data[device + "__b" + str(i)].copy()
models[device] = net
return models, cfg, provenance
def predict_case(row, root, models, cfg, batch_size):
device = row["device"]
net = models[device]
chunks = []
shape = None
for rel in row["oct_paths"]:
path = safe_path(root, rel)
h = structural_header(path)
B, H, A = int(h.NumberOfFrames), int(h.Rows), int(h.Columns)
if shape is not None and shape != (H, A):
raise ValueError("Inconsistent source grids")
shape = (H, A)
frames = []
parts = []
n = 0
def apply(frames):
x = np.stack([frame_features(f, cfg) for f in frames])
pred = net.forward(x.reshape(-1, x.shape[-1])).reshape(
len(x), cfg["width"], -1
)
pred = np.clip(pred, 0, 1) * (H - 1)
pred = pred.transpose(2, 0, 1)
return resize_axis(pred, A, 2)
for frame in source_frames(path):
frames.append(frame)
n += 1
if len(frames) == batch_size:
parts.append(apply(frames))
frames = []
if frames:
parts.append(apply(frames))
if n != B:
raise ValueError("Source frame count changed")
chunks.append(np.concatenate(parts, axis=1))
return np.concatenate(chunks, axis=1).astype(np.float32), np.asarray(
cfg["boundary_keys"][device]
)
def predict_main():
p = argparse.ArgumentParser()
p.add_argument("--input-manifest", type=Path, required=True)
p.add_argument("--data-root", type=Path, required=True)
p.add_argument("--checkpoint", type=Path, required=True)
p.add_argument("--output-dir", type=Path, required=True)
p.add_argument("--batch-size", type=int, default=16)
a = p.parse_args()
models, cfg, provenance = load_checkpoint(a.checkpoint)
rows = read_input(a.input_manifest)
open_audit = enforce_input_opens(a.data_root, rows)
a.output_dir.mkdir(parents=True, exist_ok=False)
t = time.time()
for i, row in enumerate(rows):
if not re.fullmatch("[A-Za-z0-9_-]{1,64}", row["case_id"]):
raise ValueError("Unsafe case id")
positions, keys = predict_case(row, a.data_root, models, cfg, a.batch_size)
if not np.isfinite(positions).all():
raise ValueError("Nonfinite prediction")
np.savez_compressed(
a.output_dir / (row["case_id"] + ".npz"),
positions=positions,
boundary_keys=keys,
)
if (i + 1) % 10 == 0 or i + 1 == len(rows):
print(
json.dumps(
{
"predicted_cases": i + 1,
"total": len(rows),
"seconds": round(time.time() - t, 2),
}
),
flush=True,
)
(a.output_dir / "inference-receipt.json").write_text(
json.dumps(
{
"cases": len(rows),
"seconds": time.time() - t,
"config": cfg,
"numpy": np.__version__,
"input_fields": ["case_id", "device", "oct_paths"],
"segmentation_access": False,
"input_open_audit": open_audit,
},
indent=2,
)
)