farrell236's picture
Deploy OCT viewer runtime and bundled checkpoint
bf88397 verified
Raw History Blame Contribute Delete
17.4 kB
"""Fresh-process, label-free boundary inference using structural OCT pixels only."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import time
from pathlib import Path
import numpy as np
import torch
from torch.nn import functional as F
from src.inference import apply_policy
from src.io import (
context_batch,
enforce_input_opens,
read_input,
resize_frames,
safe_path,
source_frames,
structural_header,
)
from src.models import make_model
from src.models.unet import decode_logits
def load_model(checkpoint, device):
ck = torch.load(checkpoint, map_location="cpu", weights_only=False)
cfg = ck["config"]
net = make_model(cfg)
net.load_state_dict(ck["model"])
net.to(device).eval()
return net, cfg
@torch.inference_mode()
def predict_case(row, root, net, cfg, device, batch_size=12, tta=False):
if cfg.get("context_radius", 0) or cfg.get("architecture") in [
"boundary_ensemble_v1",
"boundary_profile_refine_v1",
"boundary_zero_atom_v1",
"boundary_volume_refine_v1",
]:
return predict_prepared(
row, prepare_pixels(row, root, cfg), net, cfg, device, batch_size, tta
)
family = row["device"]
if family not in cfg["boundary_keys"]:
raise ValueError("Unsupported device family")
chunks = []
height = None
width = None
# Multiple files, when admitted, must already be in canonical concatenation order.
for rel in row["oct_paths"]:
path = safe_path(root, rel)
d = structural_header(path)
h, w = int(d.Rows), int(d.Columns)
b = int(d.NumberOfFrames)
if height is not None and (h, w) != (height, width):
raise ValueError("Inconsistent source file grids")
height, width = h, w
frames = []
outs = []
n = 0
def apply(frames):
x = resize_frames(frames, cfg["height"], cfg["width"], device)
with torch.autocast(
device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda"
):
z = net(x, family)
p = decode_logits(
z,
cfg.get("local_radius", 6),
cfg.get("decode_method", "local_centroid"),
)
if tta:
p = (
p
+ decode_logits(
net(x.flip(-1), family),
cfg.get("local_radius", 6),
cfg.get("decode_method", "local_centroid"),
).flip(-1)
) / 2
p = F.interpolate(p, size=w, mode="linear", align_corners=True)
p = p * ((h - 1) / (cfg["height"] - 1))
return p.clamp(0, h - 1).cpu().numpy().astype("float32")
for frame in source_frames(path):
frames.append(frame)
n += 1
if len(frames) == batch_size:
outs.append(apply(frames))
frames = []
if frames:
outs.append(apply(frames))
if n != b:
raise ValueError("Decoded source frame count mismatch")
chunks.append(np.concatenate(outs, axis=0).transpose(1, 0, 2))
return np.concatenate(chunks, axis=1), np.asarray(cfg["boundary_keys"][family])
def prepare_pixels(row, root, cfg):
"""Bounded per-volume decoding, CPU normalization/resizing; no label access."""
from src.models.refiners import fine_preparation_shape
fine_shape = fine_preparation_shape(cfg)
result = []
for rel in row["oct_paths"]:
path = safe_path(root, rel)
d = structural_header(path)
B, H, A = int(d.NumberOfFrames), int(d.Rows), int(d.Columns)
blocks = []
fine_blocks = []
frames = []
n = 0
for frame in source_frames(path):
frames.append(frame)
n += 1
if len(frames) == 24:
blocks.append(
resize_frames(frames, cfg["height"], cfg["width"], "cpu").to(
torch.float16
)
)
if fine_shape:
fine_blocks.append(
resize_frames(frames, *fine_shape, "cpu").to(torch.float16)
)
frames = []
if frames:
blocks.append(
resize_frames(frames, cfg["height"], cfg["width"], "cpu").to(
torch.float16
)
)
if fine_shape:
fine_blocks.append(
resize_frames(frames, *fine_shape, "cpu").to(torch.float16)
)
if n != B:
raise ValueError("Decoded source frame count mismatch")
block = (torch.cat(blocks, dim=0), H, A)
result.append((*block, torch.cat(fine_blocks, dim=0)) if fine_shape else block)
return result
@torch.inference_mode()
def predict_prepared(
row, prepared, net, cfg, device, batch_size=24, tta=False, progress=None
):
if cfg.get("architecture") == "boundary_volume_refine_v1":
from src.inference.refinement import predict_volume
return predict_volume(row, prepared, net, cfg, device, batch_size, tta)
if cfg.get("architecture") == "boundary_zero_atom_v1":
from src.inference.density import predict_zero_atom
return predict_zero_atom(row, prepared, net, cfg, device, batch_size, tta)
if cfg.get("architecture") == "boundary_profile_refine_v1":
from src.inference.refinement import predict_cascade
return predict_cascade(row, prepared, net, cfg, device, batch_size, tta)
if cfg.get("architecture") == "boundary_ensemble_v1":
if cfg.get("reduce") == "zero_probability_mean":
from src.inference.density import predict_zero_density_ensemble
return predict_zero_density_ensemble(
row, prepared, net, cfg, device, batch_size, tta
)
if cfg.get("reduce") == "probability_mean":
from src.inference.density import predict_density_ensemble
return predict_density_ensemble(
row, prepared, net, cfg, device, batch_size, tta, progress=progress
)
arrays = []
keys = None
for i, (component, model) in enumerate(zip(cfg["components"], net.nets)):
child = component["config"]
values, child_keys = predict_prepared(
row,
prepared,
model,
child,
device,
batch_size,
tta or child.get("tta_horizontal", False),
progress=progress if i == len(cfg["components"]) - 1 else None,
)
values = apply_policy(values, row["device"], child.get("prediction_policy"))
values = apply_policy(values, row["device"], component.get("post_policy"))
if keys is None:
keys = child_keys
if set(keys) != set(child_keys):
raise ValueError("Ensemble output vocabulary mismatch")
values = values[[list(child_keys).index(k) for k in keys]]
if arrays and values.shape != arrays[0].shape:
raise ValueError("Ensemble output geometry mismatch")
arrays.append(values)
if cfg.get("reduce", "mean") == "median":
return np.median(np.stack(arrays, axis=0), axis=0).astype("float32"), keys
if cfg.get("reduce") == "edge_expert":
threshold = cfg.get("edge_threshold", 3.0)
if len(arrays) != 2 or not np.isfinite(threshold) or threshold < 0:
raise ValueError("Invalid ordinary-coordinate edge expert")
edge = 0.0 if cfg.get("edge_snap_zero", True) else arrays[1]
return (
np.where(arrays[1] <= threshold, edge, arrays[0]).astype("float32"),
keys,
)
positions = np.zeros_like(arrays[0], dtype="float32")
for component, values in zip(cfg["components"], arrays):
positions += np.float32(component["weight"]) * values
return np.clip(positions, 0, prepared[0][1] - 1), keys
prepared = [p[:3] for p in prepared]
family = row["device"]
if family not in cfg["boundary_keys"]:
raise ValueError("Unsupported device family")
chunks = []
geometry = None
radius = cfg.get("context_radius", 0)
if radius and len(prepared) > 1:
H, A = prepared[0][1:]
if any((h, w) != (H, A) for _, h, w in prepared):
raise ValueError("Inconsistent concatenated source grids")
arrays = [x for x, _, _ in prepared]
combined = (
np.concatenate(arrays, axis=0)
if isinstance(arrays[0], np.ndarray)
else torch.cat(arrays, dim=0)
)
prepared = [(combined, H, A)]
for pixels, H, A in prepared:
if geometry is not None and geometry != (H, A):
raise ValueError("Inconsistent source file grids")
geometry = (H, A)
outs = []
for start in range(0, len(pixels), batch_size):
x = context_batch(
pixels, start, min(start + batch_size, len(pixels)), radius, device
)
with torch.autocast(
device_type="cuda", dtype=torch.float16, enabled=device.type == "cuda"
):
p = decode_logits(
net(x, family),
cfg.get("local_radius", 6),
cfg.get("decode_method", "local_centroid"),
)
if tta:
p = (
p
+ decode_logits(
net(x.flip(-1), family),
cfg.get("local_radius", 6),
cfg.get("decode_method", "local_centroid"),
).flip(-1)
) / 2
p = F.interpolate(p, size=A, mode="linear", align_corners=True) * (
(H - 1) / (cfg["height"] - 1)
)
outs.append(p.clamp(0, H - 1).cpu().numpy().astype("float32"))
if progress:
completed = sum(part.shape[1] for part in chunks)
progress(
completed + min(start + batch_size, len(pixels)),
sum(len(packet[0]) for packet in prepared),
)
chunks.append(np.concatenate(outs, axis=0).transpose(1, 0, 2))
return np.concatenate(chunks, axis=1), np.asarray(cfg["boundary_keys"][family])
def ordered_prefetch(rows, root, cfg, workers):
"""At most workers+1 prepared volumes; Python open guard covers all threads."""
import collections
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
iterator = iter(rows)
pending = collections.deque()
for _ in range(min(workers, len(rows))):
row = next(iterator)
pending.append((row, pool.submit(prepare_pixels, row, root, cfg)))
while pending:
row, future = pending.popleft()
pixels = future.result()
nextrow = next(iterator, None)
if nextrow is not None:
pending.append(
(nextrow, pool.submit(prepare_pixels, nextrow, root, cfg))
)
yield row, pixels
def ordered_process_prefetch(rows, root, cfg, workers, open_audit):
"""Bounded spawn workers decode OCT independently of the CUDA process."""
import collections
import concurrent.futures
import multiprocessing
from src.io import initialize_pixel_worker, prepare_pixel_packet
open_audit.update(
decoder_backend="processes_spawn",
decoder_processes_configured=workers,
worker_oct_open_events=0,
)
with concurrent.futures.ProcessPoolExecutor(
max_workers=workers,
mp_context=multiprocessing.get_context("spawn"),
initializer=initialize_pixel_worker,
initargs=(str(root), rows),
) as pool:
iterator = iter(rows)
pending = collections.deque()
for _ in range(min(workers, len(rows))):
row = next(iterator)
pending.append((row, pool.submit(prepare_pixel_packet, (row, root, cfg))))
while pending:
row, future = pending.popleft()
pixels, delta = future.result()
for key, value in delta.items():
open_audit[key] += value
open_audit["worker_oct_open_events"] += delta["oct_open_events"]
nextrow = next(iterator, None)
if nextrow is not None:
pending.append(
(nextrow, pool.submit(prepare_pixel_packet, (nextrow, root, cfg)))
)
yield row, pixels
def main():
p = argparse.ArgumentParser()
p.add_argument("--input-manifest", type=Path, required=True)
p.add_argument("--data-root", type=Path, required=True)
p.add_argument("--checkpoint", type=Path, required=True)
p.add_argument("--output-dir", type=Path, required=True)
p.add_argument("--batch-size", type=int, default=64)
p.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
p.add_argument("--tta", action="store_true")
p.add_argument("--threads", type=int, default=1)
p.add_argument("--decode-workers", type=int, default=16)
p.add_argument(
"--decode-backend", choices=["threads", "processes"], default="processes"
)
a = p.parse_args()
torch.set_num_threads(a.threads)
device = torch.device(a.device)
net, cfg = load_model(a.checkpoint, device)
rows = read_input(a.input_manifest)
open_audit = enforce_input_opens(a.data_root, rows)
a.output_dir.mkdir(parents=True, exist_ok=False)
t = time.time()
tta = a.tta or cfg.get("tta_horizontal", False)
workers = max(1, min(a.decode_workers, len(rows)))
if a.decode_workers:
stream = (
ordered_process_prefetch(rows, a.data_root, cfg, workers, open_audit)
if a.decode_backend == "processes"
else ordered_prefetch(rows, a.data_root, cfg, workers)
)
else:
stream = ((r, None) for r in rows)
for i, (r, prepared) in enumerate(stream):
if not re.fullmatch("[A-Za-z0-9_-]{1,64}", r["case_id"]):
raise ValueError("Unsafe case id")
positions, keys = (
predict_prepared(r, prepared, net, cfg, device, a.batch_size, tta)
if prepared is not None
else predict_case(r, a.data_root, net, cfg, device, a.batch_size, tta)
)
positions = apply_policy(positions, r["device"], cfg.get("prediction_policy"))
if not np.isfinite(positions).all():
raise ValueError("Nonfinite prediction")
height = (
prepared[0][1]
if prepared is not None
else int(structural_header(safe_path(a.data_root, r["oct_paths"][0])).Rows)
)
positions = np.clip(positions, 0, height - 1).astype("float32")
np.savez_compressed(
a.output_dir / (r["case_id"] + ".npz"),
positions=positions,
boundary_keys=keys,
)
if (i + 1) % 10 == 0 or i + 1 == len(rows):
print(
json.dumps(
{
"predicted_cases": i + 1,
"total": len(rows),
"seconds": round(time.time() - t, 2),
}
),
flush=True,
)
source_hashes = {
name: hashlib.sha256(
(Path(__file__).resolve().parents[2] / name).read_bytes()
).hexdigest()
for name in [
"predict.py",
"src/inference/predict.py",
"src/io.py",
"src/models/unet.py",
"src/models/__init__.py",
"src/inference/density.py",
"src/models/refiners.py",
"src/inference/refinement.py",
"src/models/volume.py",
"src/inference/__init__.py",
"src/protocol.py",
]
}
(a.output_dir / "inference-receipt.json").write_text(
json.dumps(
{
"cases": len(rows),
"seconds": time.time() - t,
"checkpoint": a.checkpoint.name,
"config": cfg,
"source_hashes": source_hashes,
"torch": torch.__version__,
"execution_device": str(device),
"batch_size": a.batch_size,
"threads": a.threads,
"input_fields": ["case_id", "device", "oct_paths"],
"segmentation_access": False,
"input_open_audit": open_audit,
"tta": tta,
"prediction_policy": cfg.get("prediction_policy"),
"decode_workers": a.decode_workers,
"decode_backend": a.decode_backend,
"preprocessing_storage_dtype": (
"float16" if a.decode_workers else "float32"
),
},
indent=2,
)
)
if __name__ == "__main__":
main()