farrell236's picture
Deploy native Gradio OCT inference app
8fca640 verified
Raw History Blame Contribute Delete
7.19 kB
"""Bounded parallel, label-free OCT decoding for optional faster inference."""
from pathlib import Path
import json
import time
import numpy as np
import torch
from torch.nn import functional as F
from torch.utils.data import IterableDataset, DataLoader, get_worker_info
from src.inference.decoding import positions_from_logits
from src.inference.combination import finish_positions
from src.models import output_indices
from src.inference.runtime import autocast_options
from src.io import volume_headers
from src.inference.context import iter_normalized_contexts
class OCTBatches(IterableDataset):
def __init__(self, rows, root, shapes, batch_size):
self.rows, self.root = rows, root
self.shapes, self.batch_size = shapes, batch_size
def __iter__(self):
worker = get_worker_info()
worker_index, workers = (worker.id, worker.num_workers) if worker else (0, 1)
for index in range(worker_index, len(self.rows), workers):
row = self.rows[index]
headers, geometry = volume_headers(self.root, row["oct_paths"])
batches = [[] for _ in self.shapes]
completed = 0
def item():
nonlocal completed
n = len(batches[0])
if completed + n > geometry[0]:
raise ValueError(
"Decoded frames exceed declared structural geometry"
)
result = {
"index": index,
"offset": completed,
"geometry": geometry,
"images": [np.stack(batch) for batch in batches],
}
completed += n
for batch in batches:
batch.clear()
return result
for images in iter_normalized_contexts(headers, self.shapes):
for frame, batch in zip(images, batches):
batch.append(frame)
if len(batches[0]) == self.batch_size:
yield item()
if batches[0]:
yield item()
if completed != geometry[0]:
raise ValueError("Incomplete structural frame coverage")
def stream_predictions(
rows, root, candidates, device, batch_size, workers, tta=False, precision="fp32"
):
"""Candidates contain loaded models/configs/destinations; no reference inputs."""
shapes = sorted(
{
(
c["config"]["height"],
c["config"]["width"],
c["config"].get("input_channels", 1),
)
for c in candidates
}
)
dataset = OCTBatches(rows, root, shapes, batch_size)
loader = DataLoader(
dataset,
batch_size=None,
num_workers=workers,
pin_memory=device.startswith("cuda"),
**({"prefetch_factor": 2} if workers else {})
)
pending, done = {}, 0
start = time.monotonic()
for batch in loader:
index, offset = int(batch["index"]), int(batch["offset"])
b, h, a = map(int, batch["geometry"])
row = rows[index]
n = len(batch["images"][0])
if index not in pending:
pending[index] = {"seen": np.zeros(b, bool), "buffers": []}
for candidate in candidates:
config = candidate["config"]
keys = config["device_keys"][row["device"]]
shape = (
config["height"],
config["width"],
config.get("input_channels", 1),
)
pending[index]["buffers"].append(
{
"candidate": candidate,
"shape_index": shapes.index(shape),
"height": shape[0],
"keys": keys,
"indices": output_indices(config, row["device"]),
"positions": np.empty((len(keys), b, a), np.float32),
}
)
state = pending[index]
if state["seen"][offset : offset + n].any() or offset + n > b:
raise ValueError("Duplicate or out-of-range structural frames")
tensors = [image.to(device, non_blocking=True) for image in batch["images"]]
with torch.inference_mode(), torch.autocast(
**autocast_options(device, precision)
):
for buffer in state["buffers"]:
x = tensors[buffer["shape_index"]]
model = buffer["candidate"]["model"]
pred = positions_from_logits(
model(x)[:, buffer["indices"]],
buffer["candidate"]["config"],
row["device"],
)
if tta:
pred = (
pred
+ positions_from_logits(
model(x.flip(-1))[:, buffer["indices"]],
buffer["candidate"]["config"],
row["device"],
).flip(-1)
) * 0.5
pred = F.interpolate(pred, size=a, mode="linear", align_corners=False)
pred = ((pred + 0.5) * h / buffer["height"] - 0.5).clamp(0, h - 1)
buffer["positions"][:, offset : offset + n] = (
pred.float().cpu().numpy().transpose(1, 0, 2)
)
state["seen"][offset : offset + n] = True
if state["seen"].all():
for buffer in state["buffers"]:
values = finish_positions(
buffer["positions"],
row["device"],
h,
buffer["candidate"]["config"]
.get("inference", {})
.get("postprocess", {}),
)
if not np.isfinite(values).all():
raise ValueError("Nonfinite candidate prediction")
destination = Path(buffer["candidate"]["destination"])
path = destination / (row["case_id"] + ".npz")
temporary = path.with_suffix(".tmp.npz")
np.savez_compressed(
temporary,
positions=values,
boundary_keys=np.asarray(buffer["keys"]),
)
temporary.replace(path)
del pending[index]
done += 1
if done % 10 == 0:
print(
json.dumps(
{
"completed_cases": done,
"elapsed_seconds": time.monotonic() - start,
"decode_workers": workers,
}
),
flush=True,
)
if pending or done != len(rows):
raise ValueError("Incomplete inference manifest coverage")
report = {
"completed_cases": done,
"elapsed_seconds": time.monotonic() - start,
"decode_workers": workers,
}
print(json.dumps(report), flush=True)
return report