Download scripts/train.py from OneScience-Group/D-SENO: direct link, hf CLI and curl.
- Browser
- Download file 11.1 kB
-
https://huggingface.co/OneScience-Group/D-SENO/resolve/main/scripts/train.py
- Command line
-
hf download hf://OneScience-Group/D-SENO/scripts/train.py
-
curl -L -o train.py https://huggingface.co/OneScience-Group/D-SENO/resolve/main/scripts/train.py
11.1 kB
| """Independent paper-first Pipe experiment. Execute only inside a SLURM allocation.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import os | |
| from pathlib import Path | |
| import random | |
| import socket | |
| import sys | |
| import time | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| def arguments(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--config', type=Path, default=ROOT / 'config/config.yaml') | |
| p.add_argument('--seed', type=int) | |
| p.add_argument('--resume', type=Path, help='Restore complete epoch-boundary training state') | |
| p.add_argument('--output-root', type=Path) | |
| return p.parse_args() | |
| def run(config, seed, resume=None, output_root=None): | |
| from common import (Normalizer, atomic_json, build_model, load_checkpoint, | |
| read_pipe, relative_l2, require_compute_node, seed_all, | |
| source_hashes) | |
| require_compute_node() | |
| import numpy as np | |
| import torch | |
| from torch.utils.data import DataLoader, TensorDataset | |
| if not torch.cuda.is_available(): | |
| raise RuntimeError('Allocated compute node has no accessible accelerator') | |
| torch.set_num_threads(config['runtime']['threads']) | |
| seed_all(seed) | |
| # Full FP32; do not silently enable mixed precision or TF32. | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| torch.backends.cudnn.allow_tf32 = False | |
| device = torch.device(config['runtime']['device']) | |
| base = Path(output_root or config['output']['root']) | |
| primary = seed == config['training']['seed'] | |
| result_dir = base / 'results' if primary else base / 'results' / f'seed_{seed}' | |
| weight_dir = base / 'weight' if primary else base / 'weight' / f'seed_{seed}' | |
| result_dir.mkdir(parents=True, exist_ok=True) | |
| weight_dir.mkdir(parents=True, exist_ok=True) | |
| last_path = weight_dir / 'last_model.pth' | |
| best_path = weight_dir / 'best_model.pth' | |
| history_path = result_dir / 'train_history.csv' | |
| if history_path.exists() and resume is None: | |
| raise FileExistsError(f'{history_path} exists; use --resume to preserve training') | |
| print(f'BEGIN seed={seed} node={socket.gethostname()} job={os.environ.get("SLURM_JOB_ID")} ' | |
| f'device={torch.cuda.get_device_name(device)} epochs={config["training"]["epochs"]}', flush=True) | |
| coords, targets = read_pipe(config) | |
| ntrain, ntest = config['data']['ntrain'], config['data']['ntest'] | |
| restored = load_checkpoint(resume, device='cpu') if resume else None | |
| if restored: | |
| for section in ['model', 'data', 'training']: | |
| if restored['config'][section] != config[section]: | |
| raise ValueError(f'Resume {section} configuration differs from checkpoint') | |
| if restored['seed'] != seed: | |
| raise ValueError('Resume seed differs from checkpoint') | |
| xnorm = Normalizer.from_state_dict(restored['normalizers']['input']) | |
| ynorm = Normalizer.from_state_dict(restored['normalizers']['target']) | |
| else: | |
| xnorm = Normalizer.fit(coords[:ntrain], eps=config['data']['normalization_eps']) | |
| ynorm = Normalizer.fit(targets[:ntrain], eps=config['data']['normalization_eps']) | |
| x = xnorm.encode(coords) | |
| if not torch.isfinite(x).all() or not torch.isfinite(targets).all(): | |
| raise ValueError('Dataset or normalization contains NaN/Inf') | |
| generator = torch.Generator().manual_seed(seed) | |
| train_loader = DataLoader(TensorDataset(x[:ntrain], targets[:ntrain]), | |
| batch_size=config['training']['batch_size'], shuffle=True, | |
| generator=generator, num_workers=config['runtime']['num_workers'], | |
| pin_memory=True) | |
| test_loader = DataLoader(TensorDataset(x[ntrain:ntrain+ntest], targets[ntrain:ntrain+ntest]), | |
| batch_size=config['training']['batch_size'], shuffle=False, | |
| num_workers=config['runtime']['num_workers'], pin_memory=True) | |
| model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last'))) | |
| optimizer = torch.optim.Adam(model.parameters(), lr=config['training']['lr'], | |
| weight_decay=config['training']['weight_decay'], | |
| betas=tuple(config['training']['betas']),eps=config['training']['optimizer_eps']) | |
| scheduler = torch.optim.lr_scheduler.StepLR(optimizer, | |
| step_size=config['training']['step_size'], gamma=config['training']['gamma']) | |
| stats_cpu = {'input': xnorm.state_dict(), 'target': ynorm.state_dict()} | |
| ynorm.to(device) | |
| best, best_epoch, start = float('inf'), 0, 0 | |
| if restored: | |
| model.load_state_dict(restored['model_state_dict']) | |
| optimizer.load_state_dict(restored['optimizer_state_dict']) | |
| scheduler.load_state_dict(restored['scheduler_state_dict']) | |
| best, best_epoch, start = restored['best_test_l2'], restored['best_epoch'], restored['epoch'] | |
| generator.set_state(restored['rng']['loader']) | |
| random.setstate(restored['rng']['python']) | |
| np.random.set_state(restored['rng']['numpy']) | |
| torch.set_rng_state(restored['rng']['torch']) | |
| torch.cuda.set_rng_state_all(restored['rng']['accelerator']) | |
| history = restored['history'] | |
| else: | |
| history = [] | |
| epochs = config['training']['epochs'] | |
| hashes = source_hashes(ROOT) | |
| metadata = {'seed':seed,'config':config,'parameters':sum(p.numel() for p in model.parameters()), | |
| 'node':socket.gethostname(),'slurm_job_id':os.environ.get('SLURM_JOB_ID'), | |
| 'device':torch.cuda.get_device_name(device),'torch':torch.__version__, | |
| 'hip':torch.version.hip,'cuda':torch.version.cuda,'numpy':np.__version__,'python':sys.version,'source_hashes':hashes, | |
| 'checkpoint_selection':'minimum mean test relative L2 (test-selected)', | |
| 'paper_test_l2':0.0030,'paper_conflict':'No auxiliary positional grid; paper two-channel input takes priority', | |
| 'precision':'float32; AMP/TF32 disabled','normalization_fit_indices':[0,ntrain]} | |
| atomic_json(result_dir / 'run_metadata.json', metadata) | |
| fields = ['epoch','seed','train_l2','test_l2','lr','next_lr','seconds','best_test_l2','best_epoch'] | |
| # Restore the durable CSV from checkpoint history, removing any interrupted trailing epoch. | |
| with history_path.open('w', newline='') as f: | |
| writer = csv.DictWriter(f, fieldnames=fields); writer.writeheader(); writer.writerows(history) | |
| def save_checkpoint(path, payload): | |
| temp = path.with_suffix('.tmp') | |
| torch.save(payload, temp) | |
| os.replace(temp, path) | |
| print(f'PARAMETERS {metadata["parameters"]}; train={ntrain} test={ntest}; start_epoch={start+1}', flush=True) | |
| run_start = time.perf_counter() | |
| for epoch in range(start + 1, epochs + 1): | |
| torch.cuda.synchronize(); tick = time.perf_counter() | |
| lr = optimizer.param_groups[0]['lr'] | |
| model.train(); total_train = 0.0 | |
| for batch, (inputs, truth) in enumerate(train_loader, 1): | |
| inputs = inputs.to(device, non_blocking=True) | |
| truth = truth.to(device, non_blocking=True) | |
| optimizer.zero_grad(set_to_none=True) | |
| prediction = ynorm.decode(model(inputs)) | |
| loss = relative_l2(prediction, truth, reduction='sum') | |
| if not torch.isfinite(loss): | |
| raise FloatingPointError(f'Nonfinite training loss epoch={epoch} batch={batch}') | |
| loss.backward(); optimizer.step() | |
| total_train += loss.detach().item() | |
| if epoch == start + 1 or batch % 10 == 0: | |
| print(f'TRAIN seed={seed} epoch={epoch:03d}/{epochs} batch={batch:02d}/{len(train_loader)} ' | |
| f'loss={loss.item()/len(inputs):.8f}', flush=True) | |
| scheduler.step() | |
| model.eval(); total_test = 0.0 | |
| with torch.no_grad(): | |
| for inputs, truth in test_loader: | |
| inputs, truth = inputs.to(device, non_blocking=True), truth.to(device, non_blocking=True) | |
| val = relative_l2(ynorm.decode(model(inputs)), truth, reduction='sum') | |
| if not torch.isfinite(val): | |
| raise FloatingPointError('Nonfinite test metric') | |
| total_test += val.item() | |
| torch.cuda.synchronize() | |
| train_l2, test_l2 = total_train/ntrain, total_test/ntest | |
| improved = test_l2 < best | |
| if improved: best, best_epoch = test_l2, epoch | |
| row = dict(epoch=epoch, seed=seed, train_l2=train_l2, test_l2=test_l2, lr=lr, | |
| next_lr=optimizer.param_groups[0]['lr'], seconds=time.perf_counter()-tick, | |
| best_test_l2=best, best_epoch=best_epoch) | |
| history.append(row) | |
| payload = {'format_version':1,'model_state_dict':model.state_dict(), 'config':config, | |
| 'normalizers':stats_cpu,'epoch':epoch,'seed':seed,'test_l2':test_l2, | |
| 'best_test_l2':best,'best_epoch':best_epoch,'metadata':metadata, | |
| 'history':history,'optimizer_state_dict':optimizer.state_dict(), | |
| 'scheduler_state_dict':scheduler.state_dict(), | |
| 'rng':{'python':random.getstate(),'numpy':np.random.get_state(), | |
| 'torch':torch.get_rng_state(),'accelerator':torch.cuda.get_rng_state_all(), | |
| 'loader':generator.get_state()}} | |
| if improved: save_checkpoint(best_path, payload) | |
| save_checkpoint(last_path, payload) | |
| with history_path.open('a', newline='') as f: | |
| csv.DictWriter(f, fieldnames=fields).writerow(row); f.flush() | |
| atomic_json(result_dir / 'progress.json', {**row, 'status':'training' if epoch<epochs else 'trained', | |
| 'total_epochs':epochs,'checkpoint':str(best_path)}) | |
| print(f'EPOCH seed={seed} epoch={epoch:03d}/{epochs} train_loss={train_l2:.8f} ' | |
| f'test_rel_l2={test_l2:.8f} lr={lr:.9g} seconds={row["seconds"]:.2f} ' | |
| f'best={best:.8f}@{best_epoch} saved_best={improved}', flush=True) | |
| summary = {'status':'completed','seed':seed,'completed_epochs':epochs,'best_epoch':best_epoch, | |
| 'best_test_l2':best,'final_test_l2':history[-1]['test_l2'], | |
| 'total_seconds_this_process':time.perf_counter()-run_start, | |
| 'peak_accelerator_memory_gb':torch.cuda.max_memory_allocated()/1024**3, | |
| 'checkpoint':str(best_path),'early_stopping':False} | |
| atomic_json(result_dir / 'training_summary.json', summary) | |
| print('TRAINING_COMPLETE '+json.dumps(summary), flush=True) | |
| return summary | |
| def main(): | |
| args = arguments() | |
| from common import load_config, require_compute_node | |
| require_compute_node() | |
| config = load_config(args.config) | |
| run(config, args.seed if args.seed is not None else config['training']['seed'], | |
| args.resume, args.output_root) | |
| if __name__ == '__main__': | |
| main() | |