farrell236's picture
Deploy OCT viewer runtime and bundled checkpoint
bf88397 verified
Raw History Blame Contribute Delete
9.34 kB
"""Structural OCT pixels, derived cache paths, and prediction receipts."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path, PurePosixPath
import numpy as np
from pydicom.filereader import read_partial
from pydicom.pixels import iter_pixels
from pydicom.tag import Tag
OCT_SOP = "1.2.840.10008.5.1.4.1.1.77.1.5.4"
def safe_path(root, value):
p = PurePosixPath(value)
if p.is_absolute() or ".." in p.parts or not value or "\\" in value:
raise ValueError("Unsafe relative dataset path")
return Path(root).joinpath(*p.parts)
def read_input(path):
rows = [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
if not rows:
raise ValueError("Empty input manifest")
for r in rows:
if set(r) != {"case_id", "device", "oct_paths"}:
raise ValueError("Prediction input must be label free")
if len({r["case_id"] for r in rows}) != len(rows):
raise ValueError("Duplicate inference case ids")
return rows
def metadata_header(path, specific_tags=None):
"""Stop before PerFrameFunctionalGroups: no payload or unused sequence parsing.
All source identification, geometry and SharedFunctionalGroups precede this tag.
Per-frame geometry is inspected separately by the discovery/audit programs.
"""
tags = [Tag(x) for x in specific_tags] if specific_tags else None
with Path(path).open("rb") as fp:
return read_partial(
fp,
stop_when=lambda tag, vr, length: int(tag) >= 0x52009230,
specific_tags=tags,
)
def structural_header(path):
d = metadata_header(
path,
[
"SOPClassUID",
"Modality",
"NumberOfFrames",
"Rows",
"Columns",
"PhotometricInterpretation",
"ConcatenationFrameOffsetNumber",
],
)
if str(d.SOPClassUID) != OCT_SOP or str(d.Modality) != "OPT":
raise ValueError("Input is not structural OCT")
return d
def normalize_frame(frame):
"""No fitted statistics: deterministic robust intensity scaling per input B-scan."""
x = np.asarray(frame, dtype=np.float32)
sample = x[::4, ::4]
lo, hi = np.percentile(sample, [1, 99.5])
return np.clip((x - lo) / max(float(hi - lo), 1e-6), 0, 1).astype("float32")
def resize_frames(frames, height, width, device="cpu"):
import torch
from torch.nn import functional as F
x = torch.from_numpy(np.stack([normalize_frame(f) for f in frames]))[:, None].to(
device
)
return F.interpolate(x, size=(height, width), mode="bilinear", align_corners=True)
def context_batch(pixels, start, end, radius, device):
"""Build float32 image channels with edge replication in encoded B order.
Prepared pixels are NumPy arrays or CPU tensors with shape [B, 1, H, W].
Context is selected from the whole volume, including across batch edges.
"""
import torch
if radius:
indices = np.clip(
np.arange(start, end)[:, None] + np.arange(-radius, radius + 1)[None, :],
0,
len(pixels) - 1,
)
part = pixels[indices][:, :, 0]
else:
part = pixels[start:end]
image = torch.from_numpy(part) if isinstance(part, np.ndarray) else part
return image.to(device=device, dtype=torch.float32)
def source_frames(path, indices=None):
d = structural_header(path)
plugin = "pylibjpeg" if d.file_meta.TransferSyntaxUID.is_compressed else ""
yield from iter_pixels(path, indices=indices, decoding_plugin=plugin)
def resized_context(path, centers, num_frames, height, width, radius):
"""Only image channels, ordered previous->center->next in encoded B order."""
offsets = range(-radius, radius + 1)
needed = sorted(
{max(0, min(num_frames - 1, b + offset)) for b in centers for offset in offsets}
)
frames = list(source_frames(path, indices=needed))
if len(frames) != len(needed):
raise ValueError("Incomplete context pixel decoding")
images = resize_frames(frames, height, width).numpy()[:, 0]
lookup = {b: i for i, b in enumerate(needed)}
return {
b: images[
[lookup[max(0, min(num_frames - 1, b + offset))] for offset in offsets]
]
for b in centers
}
def enforce_input_opens(root, rows):
"""Audit and reject Python reads of dataset files outside declared OCT inputs."""
import os
import socket
import sys
prefix = os.path.abspath(root) + os.sep
allowed = {
os.path.abspath(safe_path(root, p)) for r in rows for p in r["oct_paths"]
}
stats = {
"declared_oct_files": len(allowed),
"oct_open_events": 0,
"denied_events": 0,
"reference_npz_reads": 0,
"network_attempts": 0,
}
def audit(event, args):
if event in [
"socket.getaddrinfo",
"socket.gethostbyname",
"socket.gethostbyaddr",
] or (
event == "socket.connect"
and args[0].family in [socket.AF_INET, socket.AF_INET6]
):
stats["network_attempts"] += 1
raise PermissionError("Network access is forbidden during inference")
if event != "open" or not isinstance(args[0], (str, bytes, os.PathLike)):
return
path = os.path.abspath(os.fsdecode(args[0]))
flags = args[2] or 0
writing = bool(flags & (os.O_WRONLY | os.O_RDWR))
if path.startswith(prefix):
if path not in allowed or writing:
stats["denied_events"] += 1
raise PermissionError("Inference attempted a non-input dataset access")
stats["oct_open_events"] += 1
if path.endswith(".npz") and not writing:
stats["reference_npz_reads"] += 1
raise PermissionError(
"Reference array reads are forbidden during inference"
)
sys.addaudithook(audit)
return stats
_PIXEL_WORKER_AUDIT = None
def initialize_pixel_worker(root, rows):
"""Spawn workers have independent read guards and never initialize CUDA."""
global _PIXEL_WORKER_AUDIT
import torch
torch.set_num_threads(1)
_PIXEL_WORKER_AUDIT = enforce_input_opens(root, rows)
def prepare_pixel_packet(arguments):
"""Only declared OCT paths are available; return pixels and audit deltas."""
if _PIXEL_WORKER_AUDIT is None:
raise RuntimeError("Missing inference worker file guard")
row, root, cfg = arguments
fields = [
"oct_open_events",
"denied_events",
"reference_npz_reads",
"network_attempts",
]
before = {k: _PIXEL_WORKER_AUDIT[k] for k in fields}
from src.inference.predict import prepare_pixels
prepared = prepare_pixels(row, root, cfg)
# NumPy packets avoid implicit Torch shared-memory dependencies across workers.
packet = [
(part[0].numpy(), part[1], part[2], *[x.numpy() for x in part[3:]])
for part in prepared
]
delta = {k: _PIXEL_WORKER_AUDIT[k] - before[k] for k in fields}
return packet, delta
def cache_layout(directory):
meta = Path(directory) / "cache-metadata.json"
layout = (
json.loads(meta.read_text()).get("layout", "flat_v1")
if meta.exists()
else "flat_v1"
)
if layout not in ["flat_v1", "case_prefix2_v1"]:
raise ValueError("Unknown training-cache layout")
return layout
def frame_path(directory, case_id, frame, layout="flat_v1"):
root = Path(directory)
if layout == "case_prefix2_v1":
root = root / case_id[:2]
elif layout != "flat_v1":
raise ValueError("Unknown training-cache layout")
return root / (case_id + f"-{frame}.npz")
def completion_fields(directory):
directory = Path(directory)
files = list(directory.glob("*.npz"))
inference = json.loads((directory / "inference-receipt.json").read_text())
if inference["cases"] != len(files):
raise ValueError("Prediction completion count mismatch")
return {
"predicted_cases": len(files),
"input_open_audit": inference["input_open_audit"],
"prediction_files_sha256": None,
"prediction_digest_deferred_to_cpu_score": True,
"prediction_digest_order": "sorted filename bytes then ASCII SHA256 of file bytes",
}
def prediction_digest(directory):
combined = hashlib.sha256()
files = sorted(Path(directory).glob("*.npz"))
for file in files:
digest = hashlib.sha256()
with file.open("rb") as stream:
for block in iter(lambda: stream.read(1048576), b""):
digest.update(block)
combined.update(file.name.encode())
combined.update(digest.hexdigest().encode())
return combined.hexdigest(), len(files)
def collect_source_hashes():
"""Identify executable project sources with paths relative to the bundle."""
root = Path(__file__).resolve().parents[1]
paths = [
*root.glob("*.py"),
*(root / "src").rglob("*.py"),
*(root / "tools").glob("*.py"),
*(root / "research" / "experiments").glob("*.py"),
]
return {
str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in sorted(paths)
}