Spaces:
Sleeping
Sleeping
Download src/profile.py from farrell236/OCT_Segmentation_a2: direct link, hf CLI and curl.
- Browser
- Download file 8.97 kB
-
https://huggingface.co/spaces/farrell236/OCT_Segmentation_a2/resolve/main/src/profile.py
- Command line
-
hf download hf://spaces/farrell236/OCT_Segmentation_a2/src/profile.py
-
curl -L -o profile.py https://huggingface.co/spaces/farrell236/OCT_Segmentation_a2/resolve/main/src/profile.py
8.97 kB
| """NumPy profile model, checkpoint I/O, and label-free inference.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| from src.io import ( | |
| enforce_input_opens, | |
| normalize_frame, | |
| read_input, | |
| safe_path, | |
| source_frames, | |
| structural_header, | |
| ) | |
| def resize_axis(x, n, axis): | |
| old = x.shape[axis] | |
| if old == n: | |
| return x | |
| grid = np.linspace(0, old - 1, n, dtype=np.float32) | |
| lo = np.floor(grid).astype(int) | |
| hi = np.minimum(lo + 1, old - 1) | |
| w = (grid - lo).astype(np.float32) | |
| shape = [1] * x.ndim | |
| shape[axis] = n | |
| w = w.reshape(shape) | |
| return (np.take(x, lo, axis=axis) * (1 - w) + np.take(x, hi, axis=axis) * w).astype( | |
| np.float32 | |
| ) | |
| def box_filter(x, n, axis): | |
| if n <= 1: | |
| return x | |
| if n % 2 == 0: | |
| n += 1 | |
| pads = [(0, 0)] * x.ndim | |
| pads[axis] = (n // 2, n - 1 - n // 2) | |
| padded = np.pad(x, pads, mode="edge") | |
| zero_shape = list(padded.shape) | |
| zero_shape[axis] = 1 | |
| cumulative = np.concatenate( | |
| [ | |
| np.zeros(zero_shape, np.float32), | |
| np.cumsum(padded, axis=axis, dtype=np.float32), | |
| ], | |
| axis=axis, | |
| ) | |
| upper = [slice(None)] * x.ndim | |
| lower = [slice(None)] * x.ndim | |
| upper[axis] = slice(n, None) | |
| lower[axis] = slice(None, -n) | |
| return (cumulative[tuple(upper)] - cumulative[tuple(lower)]) / n | |
| def frame_features(frame, cfg): | |
| x = normalize_frame(frame) | |
| if cfg.get("antialias", True): | |
| x = box_filter(x, max(1, int(round(x.shape[0] / cfg["profile_height"]))), 0) | |
| x = box_filter(x, max(1, int(round(x.shape[1] / cfg["width"]))), 1) | |
| x = resize_axis(resize_axis(x, cfg["profile_height"], 0), cfg["width"], 1) | |
| width = x.shape[1] | |
| radius = cfg["context_radius"] | |
| cols = np.arange(width) | |
| features = np.concatenate( | |
| [ | |
| x[:, np.clip(cols + offset, 0, width - 1)].T | |
| for offset in [-radius, 0, radius] | |
| ], | |
| axis=1, | |
| ) | |
| return features.astype(np.float32) | |
| class ProfileMLP: | |
| def __init__(self, dims, rng): | |
| self.weights = [ | |
| (rng.standard_normal((a, b)) * np.sqrt(2 / a)).astype(np.float32) | |
| for a, b in zip(dims[:-1], dims[1:]) | |
| ] | |
| self.biases = [np.zeros(b, np.float32) for b in dims[1:]] | |
| self.weights[-1] *= 0.05 | |
| self.biases[-1][:] = 0.5 | |
| def forward(self, x, retain=False): | |
| a = x * 2 - 1 | |
| acts = [a] | |
| pre = [] | |
| for i, (w, b) in enumerate(zip(self.weights, self.biases)): | |
| z = a @ w + b | |
| pre.append(z) | |
| a = np.maximum(z, 0) if i < len(self.weights) - 1 else z | |
| acts.append(a) | |
| return (a, (acts, pre)) if retain else a | |
| def gradients_from_cache(self, dy, cache): | |
| acts, pre = cache | |
| grads = [] | |
| for i in range(len(self.weights) - 1, -1, -1): | |
| dw = acts[i].T @ dy | |
| db = dy.sum(axis=0) | |
| grads.append((dw, db)) | |
| if i: | |
| dy = dy @ self.weights[i].T | |
| dy *= pre[i - 1] > 0 | |
| return list(reversed(grads)) | |
| class Adam: | |
| def __init__(self, net, weight_decay=1e-4): | |
| self.m = [ | |
| (np.zeros_like(w), np.zeros_like(b)) | |
| for w, b in zip(net.weights, net.biases) | |
| ] | |
| self.v = [ | |
| (np.zeros_like(w), np.zeros_like(b)) | |
| for w, b in zip(net.weights, net.biases) | |
| ] | |
| self.t = 0 | |
| self.weight_decay = weight_decay | |
| def step(self, net, grads, lr, clip=5.0): | |
| norm = np.sqrt(sum(float((g * g).sum()) for pair in grads for g in pair)) | |
| scale = min(1.0, clip / max(norm, 1e-12)) | |
| self.t += 1 | |
| beta1 = 0.9 | |
| beta2 = 0.999 | |
| for i, pair in enumerate(grads): | |
| for j, (param, grad) in enumerate( | |
| zip([net.weights[i], net.biases[i]], pair) | |
| ): | |
| grad = grad * scale | |
| self.m[i][j][:] = beta1 * self.m[i][j] + (1 - beta1) * grad | |
| self.v[i][j][:] = beta2 * self.v[i][j] + (1 - beta2) * grad * grad | |
| update = (self.m[i][j] / (1 - beta1**self.t)) / ( | |
| np.sqrt(self.v[i][j] / (1 - beta2**self.t)) + 1e-8 | |
| ) | |
| if j == 0: | |
| param *= 1 - lr * self.weight_decay | |
| param -= lr * update | |
| return float(norm) | |
| def save_checkpoint(path, models, cfg, provenance): | |
| arrays = { | |
| "config_json": np.asarray(json.dumps(cfg, sort_keys=True)), | |
| "provenance_json": np.asarray(json.dumps(provenance, sort_keys=True)), | |
| } | |
| for device, net in models.items(): | |
| for i, (w, b) in enumerate(zip(net.weights, net.biases)): | |
| arrays[device + "__w" + str(i)] = w | |
| arrays[device + "__b" + str(i)] = b | |
| path = Path(path) | |
| tmp = path.with_suffix(".tmp.npz") | |
| np.savez_compressed(tmp, **arrays) | |
| tmp.replace(path) | |
| def load_checkpoint(path): | |
| with np.load(path, allow_pickle=False) as data: | |
| cfg = json.loads(str(data["config_json"])) | |
| provenance = json.loads(str(data["provenance_json"])) | |
| models = {} | |
| for device, keys in cfg["boundary_keys"].items(): | |
| dims = [cfg["profile_height"] * 3] + cfg["hidden"] + [len(keys)] | |
| net = ProfileMLP(dims, np.random.default_rng(0)) | |
| for i in range(len(net.weights)): | |
| net.weights[i] = data[device + "__w" + str(i)].copy() | |
| net.biases[i] = data[device + "__b" + str(i)].copy() | |
| models[device] = net | |
| return models, cfg, provenance | |
| def predict_case(row, root, models, cfg, batch_size): | |
| device = row["device"] | |
| net = models[device] | |
| chunks = [] | |
| shape = None | |
| for rel in row["oct_paths"]: | |
| path = safe_path(root, rel) | |
| h = structural_header(path) | |
| B, H, A = int(h.NumberOfFrames), int(h.Rows), int(h.Columns) | |
| if shape is not None and shape != (H, A): | |
| raise ValueError("Inconsistent source grids") | |
| shape = (H, A) | |
| frames = [] | |
| parts = [] | |
| n = 0 | |
| def apply(frames): | |
| x = np.stack([frame_features(f, cfg) for f in frames]) | |
| pred = net.forward(x.reshape(-1, x.shape[-1])).reshape( | |
| len(x), cfg["width"], -1 | |
| ) | |
| pred = np.clip(pred, 0, 1) * (H - 1) | |
| pred = pred.transpose(2, 0, 1) | |
| return resize_axis(pred, A, 2) | |
| for frame in source_frames(path): | |
| frames.append(frame) | |
| n += 1 | |
| if len(frames) == batch_size: | |
| parts.append(apply(frames)) | |
| frames = [] | |
| if frames: | |
| parts.append(apply(frames)) | |
| if n != B: | |
| raise ValueError("Source frame count changed") | |
| chunks.append(np.concatenate(parts, axis=1)) | |
| return np.concatenate(chunks, axis=1).astype(np.float32), np.asarray( | |
| cfg["boundary_keys"][device] | |
| ) | |
| def predict_main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--input-manifest", type=Path, required=True) | |
| p.add_argument("--data-root", type=Path, required=True) | |
| p.add_argument("--checkpoint", type=Path, required=True) | |
| p.add_argument("--output-dir", type=Path, required=True) | |
| p.add_argument("--batch-size", type=int, default=16) | |
| a = p.parse_args() | |
| models, cfg, provenance = load_checkpoint(a.checkpoint) | |
| rows = read_input(a.input_manifest) | |
| open_audit = enforce_input_opens(a.data_root, rows) | |
| a.output_dir.mkdir(parents=True, exist_ok=False) | |
| t = time.time() | |
| for i, row in enumerate(rows): | |
| if not re.fullmatch("[A-Za-z0-9_-]{1,64}", row["case_id"]): | |
| raise ValueError("Unsafe case id") | |
| positions, keys = predict_case(row, a.data_root, models, cfg, a.batch_size) | |
| if not np.isfinite(positions).all(): | |
| raise ValueError("Nonfinite prediction") | |
| np.savez_compressed( | |
| a.output_dir / (row["case_id"] + ".npz"), | |
| positions=positions, | |
| boundary_keys=keys, | |
| ) | |
| if (i + 1) % 10 == 0 or i + 1 == len(rows): | |
| print( | |
| json.dumps( | |
| { | |
| "predicted_cases": i + 1, | |
| "total": len(rows), | |
| "seconds": round(time.time() - t, 2), | |
| } | |
| ), | |
| flush=True, | |
| ) | |
| (a.output_dir / "inference-receipt.json").write_text( | |
| json.dumps( | |
| { | |
| "cases": len(rows), | |
| "seconds": time.time() - t, | |
| "config": cfg, | |
| "numpy": np.__version__, | |
| "input_fields": ["case_id", "device", "oct_paths"], | |
| "segmentation_access": False, | |
| "input_open_audit": open_audit, | |
| }, | |
| indent=2, | |
| ) | |
| ) | |