"""Shared I/O, pointwise normalization and metrics for the Pipe experiment.""" from __future__ import annotations import hashlib import json import os from pathlib import Path import random import sys import tempfile import numpy as np import torch import yaml ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) def load_config(path) -> dict: with Path(path).open(encoding="utf-8") as handle: config = yaml.safe_load(handle) if not isinstance(config, dict): raise ValueError("Configuration must be a mapping") required = {"data", "model", "training", "runtime", "output", "provenance"} missing = required.difference(config) if missing: raise ValueError(f"Missing configuration sections: {sorted(missing)}") return config def read_pipe(config): """Read original CPU float32 X/Y and Q0 in fixed sample order, without downloads.""" data = config["data"] directory = Path(data["directory"]) ntrain, ntest = int(data["ntrain"]), int(data["ntest"]) count = ntrain + ntest if ntrain != 1000 or ntest != 200 or int(data["target_channel"]) != 0: raise ValueError("The formal Pipe experiment requires train1000/test200 and Q0") mesh = tuple(data["mesh"]) if mesh != (129, 129): raise ValueError("The formal Pipe mesh must remain 129 by 129") x = np.load(directory / "Pipe_X.npy", mmap_mode="r", allow_pickle=False) y = np.load(directory / "Pipe_Y.npy", mmap_mode="r", allow_pickle=False) q = np.load(directory / "Pipe_Q.npy", mmap_mode="r", allow_pickle=False) if x.shape != (2310, *mesh) or y.shape != x.shape: raise ValueError(f"Unexpected coordinate shapes: X={x.shape}, Y={y.shape}") if q.shape != (2310, 3, *mesh): raise ValueError(f"Unexpected Pipe_Q shape: {q.shape}") coordinates = np.empty((count, *mesh, 2), dtype=np.float32) coordinates[..., 0] = x[:count] coordinates[..., 1] = y[:count] target = np.array(q[:count, 0], dtype=np.float32, copy=True) if not np.isfinite(coordinates).all() or not np.isfinite(target).all(): raise ValueError("Pipe coordinates and Q0 must be finite") return torch.from_numpy(coordinates), torch.from_numpy(target) class Normalizer: """Train-only pointwise statistics along the sample dimension.""" def __init__(self, mean, std, eps=1e-5): self.mean = mean self.std = std self.eps = float(eps) @classmethod def fit(cls, x, eps=1e-5): if x.shape[0] < 2: raise ValueError("Unbiased normalization needs at least two samples") return cls(x.mean(dim=0), x.std(dim=0, unbiased=True), eps) def encode(self, x): return (x - self.mean) / (self.std + self.eps) def decode(self, x): return x * (self.std + self.eps) + self.mean def to(self, device): self.mean = self.mean.to(device) self.std = self.std.to(device) return self def state_dict(self): return {"mean": self.mean.detach().cpu(), "std": self.std.detach().cpu(), "eps": self.eps} @classmethod def from_state_dict(cls, state, device="cpu"): return cls(torch.as_tensor(state["mean"]).to(device), torch.as_tensor(state["std"]).to(device), state["eps"]) def relative_l2(pred, target, reduction="mean"): """Per-sample spatial norm ratio, in the same physical units as target.""" if pred.shape != target.shape or pred.ndim < 2: raise ValueError("Prediction and target need matching batch/spatial dimensions") denominator = torch.linalg.vector_norm(target.reshape(target.shape[0], -1), dim=1) if torch.any(denominator == 0): raise ValueError("Relative L2 is undefined for a zero-norm target") numerator = torch.linalg.vector_norm((pred-target).reshape(target.shape[0], -1), dim=1) ratios = numerator / denominator if reduction == "none": return ratios if reduction == "sum": return ratios.sum() if reduction == "mean": return ratios.mean() raise ValueError("Reduction must be mean, sum or none") def seed_all(seed, deterministic=False): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = deterministic def require_compute_node(): if not torch.cuda.is_available(): raise RuntimeError("An allocated accelerator is required") def atomic_json(path, payload): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = None try: with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, prefix=path.name + ".", suffix=".tmp", delete=False) as handle: temporary = Path(handle.name) json.dump(payload, handle, indent=2, ensure_ascii=False, allow_nan=False) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) finally: if temporary is not None and temporary.exists(): temporary.unlink() def load_checkpoint(path, device="cpu") -> dict: try: checkpoint = torch.load(path, map_location=device, weights_only=False) except TypeError: checkpoint = torch.load(path, map_location=device) if not isinstance(checkpoint, dict): raise ValueError("Expected a checkpoint dictionary") return checkpoint def source_hashes(root): root = Path(root) hashes = {} for directory in ("model", "scripts", "config"): for path in sorted((root / directory).rglob("*")): if path.is_file() and path.suffix in {".py", ".yaml", ".yml", ".sh"}: hashes[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest() return hashes def build_model(config): from model.DSENO import DSENO model = DSENO(**config["model"]) model.memory_format = getattr(torch, config["runtime"].get("memory_format", "channels_last")) return model