D-SENO / scripts /common.py
yushuang88's picture
Upload folder using huggingface_hub
b144968 verified
Raw History Blame Contribute Delete
6.26 kB
"""Shared I/O, pointwise normalization and metrics for the Pipe experiment."""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import random
import sys
import tempfile
import numpy as np
import torch
import yaml
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
def load_config(path) -> dict:
with Path(path).open(encoding="utf-8") as handle:
config = yaml.safe_load(handle)
if not isinstance(config, dict):
raise ValueError("Configuration must be a mapping")
required = {"data", "model", "training", "runtime", "output", "provenance"}
missing = required.difference(config)
if missing:
raise ValueError(f"Missing configuration sections: {sorted(missing)}")
return config
def read_pipe(config):
"""Read original CPU float32 X/Y and Q0 in fixed sample order, without downloads."""
data = config["data"]
directory = Path(data["directory"])
ntrain, ntest = int(data["ntrain"]), int(data["ntest"])
count = ntrain + ntest
if ntrain != 1000 or ntest != 200 or int(data["target_channel"]) != 0:
raise ValueError("The formal Pipe experiment requires train1000/test200 and Q0")
mesh = tuple(data["mesh"])
if mesh != (129, 129):
raise ValueError("The formal Pipe mesh must remain 129 by 129")
x = np.load(directory / "Pipe_X.npy", mmap_mode="r", allow_pickle=False)
y = np.load(directory / "Pipe_Y.npy", mmap_mode="r", allow_pickle=False)
q = np.load(directory / "Pipe_Q.npy", mmap_mode="r", allow_pickle=False)
if x.shape != (2310, *mesh) or y.shape != x.shape:
raise ValueError(f"Unexpected coordinate shapes: X={x.shape}, Y={y.shape}")
if q.shape != (2310, 3, *mesh):
raise ValueError(f"Unexpected Pipe_Q shape: {q.shape}")
coordinates = np.empty((count, *mesh, 2), dtype=np.float32)
coordinates[..., 0] = x[:count]
coordinates[..., 1] = y[:count]
target = np.array(q[:count, 0], dtype=np.float32, copy=True)
if not np.isfinite(coordinates).all() or not np.isfinite(target).all():
raise ValueError("Pipe coordinates and Q0 must be finite")
return torch.from_numpy(coordinates), torch.from_numpy(target)
class Normalizer:
"""Train-only pointwise statistics along the sample dimension."""
def __init__(self, mean, std, eps=1e-5):
self.mean = mean
self.std = std
self.eps = float(eps)
@classmethod
def fit(cls, x, eps=1e-5):
if x.shape[0] < 2:
raise ValueError("Unbiased normalization needs at least two samples")
return cls(x.mean(dim=0), x.std(dim=0, unbiased=True), eps)
def encode(self, x):
return (x - self.mean) / (self.std + self.eps)
def decode(self, x):
return x * (self.std + self.eps) + self.mean
def to(self, device):
self.mean = self.mean.to(device)
self.std = self.std.to(device)
return self
def state_dict(self):
return {"mean": self.mean.detach().cpu(),
"std": self.std.detach().cpu(), "eps": self.eps}
@classmethod
def from_state_dict(cls, state, device="cpu"):
return cls(torch.as_tensor(state["mean"]).to(device),
torch.as_tensor(state["std"]).to(device), state["eps"])
def relative_l2(pred, target, reduction="mean"):
"""Per-sample spatial norm ratio, in the same physical units as target."""
if pred.shape != target.shape or pred.ndim < 2:
raise ValueError("Prediction and target need matching batch/spatial dimensions")
denominator = torch.linalg.vector_norm(target.reshape(target.shape[0], -1), dim=1)
if torch.any(denominator == 0):
raise ValueError("Relative L2 is undefined for a zero-norm target")
numerator = torch.linalg.vector_norm((pred-target).reshape(target.shape[0], -1), dim=1)
ratios = numerator / denominator
if reduction == "none":
return ratios
if reduction == "sum":
return ratios.sum()
if reduction == "mean":
return ratios.mean()
raise ValueError("Reduction must be mean, sum or none")
def seed_all(seed, deterministic=False):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = deterministic
def require_compute_node():
if not torch.cuda.is_available():
raise RuntimeError("An allocated accelerator is required")
def atomic_json(path, payload):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = None
try:
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
prefix=path.name + ".", suffix=".tmp",
delete=False) as handle:
temporary = Path(handle.name)
json.dump(payload, handle, indent=2, ensure_ascii=False, allow_nan=False)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
finally:
if temporary is not None and temporary.exists():
temporary.unlink()
def load_checkpoint(path, device="cpu") -> dict:
try:
checkpoint = torch.load(path, map_location=device, weights_only=False)
except TypeError:
checkpoint = torch.load(path, map_location=device)
if not isinstance(checkpoint, dict):
raise ValueError("Expected a checkpoint dictionary")
return checkpoint
def source_hashes(root):
root = Path(root)
hashes = {}
for directory in ("model", "scripts", "config"):
for path in sorted((root / directory).rglob("*")):
if path.is_file() and path.suffix in {".py", ".yaml", ".yml", ".sh"}:
hashes[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest()
return hashes
def build_model(config):
from model.DSENO import DSENO
model = DSENO(**config["model"])
model.memory_format = getattr(torch, config["runtime"].get("memory_format", "channels_last"))
return model