"""Independent paper-first Pipe experiment. Execute only inside a SLURM allocation.""" from __future__ import annotations import argparse import csv import json import os from pathlib import Path import random import socket import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) def arguments(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--config', type=Path, default=ROOT / 'config/config.yaml') p.add_argument('--seed', type=int) p.add_argument('--resume', type=Path, help='Restore complete epoch-boundary training state') p.add_argument('--output-root', type=Path) return p.parse_args() def run(config, seed, resume=None, output_root=None): from common import (Normalizer, atomic_json, build_model, load_checkpoint, read_pipe, relative_l2, require_compute_node, seed_all, source_hashes) require_compute_node() import numpy as np import torch from torch.utils.data import DataLoader, TensorDataset if not torch.cuda.is_available(): raise RuntimeError('Allocated compute node has no accessible accelerator') torch.set_num_threads(config['runtime']['threads']) seed_all(seed) # Full FP32; do not silently enable mixed precision or TF32. torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False device = torch.device(config['runtime']['device']) base = Path(output_root or config['output']['root']) primary = seed == config['training']['seed'] result_dir = base / 'results' if primary else base / 'results' / f'seed_{seed}' weight_dir = base / 'weight' if primary else base / 'weight' / f'seed_{seed}' result_dir.mkdir(parents=True, exist_ok=True) weight_dir.mkdir(parents=True, exist_ok=True) last_path = weight_dir / 'last_model.pth' best_path = weight_dir / 'best_model.pth' history_path = result_dir / 'train_history.csv' if history_path.exists() and resume is None: raise FileExistsError(f'{history_path} exists; use --resume to preserve training') print(f'BEGIN seed={seed} node={socket.gethostname()} job={os.environ.get("SLURM_JOB_ID")} ' f'device={torch.cuda.get_device_name(device)} epochs={config["training"]["epochs"]}', flush=True) coords, targets = read_pipe(config) ntrain, ntest = config['data']['ntrain'], config['data']['ntest'] restored = load_checkpoint(resume, device='cpu') if resume else None if restored: for section in ['model', 'data', 'training']: if restored['config'][section] != config[section]: raise ValueError(f'Resume {section} configuration differs from checkpoint') if restored['seed'] != seed: raise ValueError('Resume seed differs from checkpoint') xnorm = Normalizer.from_state_dict(restored['normalizers']['input']) ynorm = Normalizer.from_state_dict(restored['normalizers']['target']) else: xnorm = Normalizer.fit(coords[:ntrain], eps=config['data']['normalization_eps']) ynorm = Normalizer.fit(targets[:ntrain], eps=config['data']['normalization_eps']) x = xnorm.encode(coords) if not torch.isfinite(x).all() or not torch.isfinite(targets).all(): raise ValueError('Dataset or normalization contains NaN/Inf') generator = torch.Generator().manual_seed(seed) train_loader = DataLoader(TensorDataset(x[:ntrain], targets[:ntrain]), batch_size=config['training']['batch_size'], shuffle=True, generator=generator, num_workers=config['runtime']['num_workers'], pin_memory=True) test_loader = DataLoader(TensorDataset(x[ntrain:ntrain+ntest], targets[ntrain:ntrain+ntest]), batch_size=config['training']['batch_size'], shuffle=False, num_workers=config['runtime']['num_workers'], pin_memory=True) model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last'))) optimizer = torch.optim.Adam(model.parameters(), lr=config['training']['lr'], weight_decay=config['training']['weight_decay'], betas=tuple(config['training']['betas']),eps=config['training']['optimizer_eps']) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=config['training']['step_size'], gamma=config['training']['gamma']) stats_cpu = {'input': xnorm.state_dict(), 'target': ynorm.state_dict()} ynorm.to(device) best, best_epoch, start = float('inf'), 0, 0 if restored: model.load_state_dict(restored['model_state_dict']) optimizer.load_state_dict(restored['optimizer_state_dict']) scheduler.load_state_dict(restored['scheduler_state_dict']) best, best_epoch, start = restored['best_test_l2'], restored['best_epoch'], restored['epoch'] generator.set_state(restored['rng']['loader']) random.setstate(restored['rng']['python']) np.random.set_state(restored['rng']['numpy']) torch.set_rng_state(restored['rng']['torch']) torch.cuda.set_rng_state_all(restored['rng']['accelerator']) history = restored['history'] else: history = [] epochs = config['training']['epochs'] hashes = source_hashes(ROOT) metadata = {'seed':seed,'config':config,'parameters':sum(p.numel() for p in model.parameters()), 'node':socket.gethostname(),'slurm_job_id':os.environ.get('SLURM_JOB_ID'), 'device':torch.cuda.get_device_name(device),'torch':torch.__version__, 'hip':torch.version.hip,'cuda':torch.version.cuda,'numpy':np.__version__,'python':sys.version,'source_hashes':hashes, 'checkpoint_selection':'minimum mean test relative L2 (test-selected)', 'paper_test_l2':0.0030,'paper_conflict':'No auxiliary positional grid; paper two-channel input takes priority', 'precision':'float32; AMP/TF32 disabled','normalization_fit_indices':[0,ntrain]} atomic_json(result_dir / 'run_metadata.json', metadata) fields = ['epoch','seed','train_l2','test_l2','lr','next_lr','seconds','best_test_l2','best_epoch'] # Restore the durable CSV from checkpoint history, removing any interrupted trailing epoch. with history_path.open('w', newline='') as f: writer = csv.DictWriter(f, fieldnames=fields); writer.writeheader(); writer.writerows(history) def save_checkpoint(path, payload): temp = path.with_suffix('.tmp') torch.save(payload, temp) os.replace(temp, path) print(f'PARAMETERS {metadata["parameters"]}; train={ntrain} test={ntest}; start_epoch={start+1}', flush=True) run_start = time.perf_counter() for epoch in range(start + 1, epochs + 1): torch.cuda.synchronize(); tick = time.perf_counter() lr = optimizer.param_groups[0]['lr'] model.train(); total_train = 0.0 for batch, (inputs, truth) in enumerate(train_loader, 1): inputs = inputs.to(device, non_blocking=True) truth = truth.to(device, non_blocking=True) optimizer.zero_grad(set_to_none=True) prediction = ynorm.decode(model(inputs)) loss = relative_l2(prediction, truth, reduction='sum') if not torch.isfinite(loss): raise FloatingPointError(f'Nonfinite training loss epoch={epoch} batch={batch}') loss.backward(); optimizer.step() total_train += loss.detach().item() if epoch == start + 1 or batch % 10 == 0: print(f'TRAIN seed={seed} epoch={epoch:03d}/{epochs} batch={batch:02d}/{len(train_loader)} ' f'loss={loss.item()/len(inputs):.8f}', flush=True) scheduler.step() model.eval(); total_test = 0.0 with torch.no_grad(): for inputs, truth in test_loader: inputs, truth = inputs.to(device, non_blocking=True), truth.to(device, non_blocking=True) val = relative_l2(ynorm.decode(model(inputs)), truth, reduction='sum') if not torch.isfinite(val): raise FloatingPointError('Nonfinite test metric') total_test += val.item() torch.cuda.synchronize() train_l2, test_l2 = total_train/ntrain, total_test/ntest improved = test_l2 < best if improved: best, best_epoch = test_l2, epoch row = dict(epoch=epoch, seed=seed, train_l2=train_l2, test_l2=test_l2, lr=lr, next_lr=optimizer.param_groups[0]['lr'], seconds=time.perf_counter()-tick, best_test_l2=best, best_epoch=best_epoch) history.append(row) payload = {'format_version':1,'model_state_dict':model.state_dict(), 'config':config, 'normalizers':stats_cpu,'epoch':epoch,'seed':seed,'test_l2':test_l2, 'best_test_l2':best,'best_epoch':best_epoch,'metadata':metadata, 'history':history,'optimizer_state_dict':optimizer.state_dict(), 'scheduler_state_dict':scheduler.state_dict(), 'rng':{'python':random.getstate(),'numpy':np.random.get_state(), 'torch':torch.get_rng_state(),'accelerator':torch.cuda.get_rng_state_all(), 'loader':generator.get_state()}} if improved: save_checkpoint(best_path, payload) save_checkpoint(last_path, payload) with history_path.open('a', newline='') as f: csv.DictWriter(f, fieldnames=fields).writerow(row); f.flush() atomic_json(result_dir / 'progress.json', {**row, 'status':'training' if epoch