D-SENO / scripts /train.py
yushuang88's picture
Upload folder using huggingface_hub
b144968 verified
Raw History Blame Contribute Delete
11.1 kB
"""Independent paper-first Pipe experiment. Execute only inside a SLURM allocation."""
from __future__ import annotations
import argparse
import csv
import json
import os
from pathlib import Path
import random
import socket
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
def arguments():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--config', type=Path, default=ROOT / 'config/config.yaml')
p.add_argument('--seed', type=int)
p.add_argument('--resume', type=Path, help='Restore complete epoch-boundary training state')
p.add_argument('--output-root', type=Path)
return p.parse_args()
def run(config, seed, resume=None, output_root=None):
from common import (Normalizer, atomic_json, build_model, load_checkpoint,
read_pipe, relative_l2, require_compute_node, seed_all,
source_hashes)
require_compute_node()
import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
if not torch.cuda.is_available():
raise RuntimeError('Allocated compute node has no accessible accelerator')
torch.set_num_threads(config['runtime']['threads'])
seed_all(seed)
# Full FP32; do not silently enable mixed precision or TF32.
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
device = torch.device(config['runtime']['device'])
base = Path(output_root or config['output']['root'])
primary = seed == config['training']['seed']
result_dir = base / 'results' if primary else base / 'results' / f'seed_{seed}'
weight_dir = base / 'weight' if primary else base / 'weight' / f'seed_{seed}'
result_dir.mkdir(parents=True, exist_ok=True)
weight_dir.mkdir(parents=True, exist_ok=True)
last_path = weight_dir / 'last_model.pth'
best_path = weight_dir / 'best_model.pth'
history_path = result_dir / 'train_history.csv'
if history_path.exists() and resume is None:
raise FileExistsError(f'{history_path} exists; use --resume to preserve training')
print(f'BEGIN seed={seed} node={socket.gethostname()} job={os.environ.get("SLURM_JOB_ID")} '
f'device={torch.cuda.get_device_name(device)} epochs={config["training"]["epochs"]}', flush=True)
coords, targets = read_pipe(config)
ntrain, ntest = config['data']['ntrain'], config['data']['ntest']
restored = load_checkpoint(resume, device='cpu') if resume else None
if restored:
for section in ['model', 'data', 'training']:
if restored['config'][section] != config[section]:
raise ValueError(f'Resume {section} configuration differs from checkpoint')
if restored['seed'] != seed:
raise ValueError('Resume seed differs from checkpoint')
xnorm = Normalizer.from_state_dict(restored['normalizers']['input'])
ynorm = Normalizer.from_state_dict(restored['normalizers']['target'])
else:
xnorm = Normalizer.fit(coords[:ntrain], eps=config['data']['normalization_eps'])
ynorm = Normalizer.fit(targets[:ntrain], eps=config['data']['normalization_eps'])
x = xnorm.encode(coords)
if not torch.isfinite(x).all() or not torch.isfinite(targets).all():
raise ValueError('Dataset or normalization contains NaN/Inf')
generator = torch.Generator().manual_seed(seed)
train_loader = DataLoader(TensorDataset(x[:ntrain], targets[:ntrain]),
batch_size=config['training']['batch_size'], shuffle=True,
generator=generator, num_workers=config['runtime']['num_workers'],
pin_memory=True)
test_loader = DataLoader(TensorDataset(x[ntrain:ntrain+ntest], targets[ntrain:ntrain+ntest]),
batch_size=config['training']['batch_size'], shuffle=False,
num_workers=config['runtime']['num_workers'], pin_memory=True)
model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last')))
optimizer = torch.optim.Adam(model.parameters(), lr=config['training']['lr'],
weight_decay=config['training']['weight_decay'],
betas=tuple(config['training']['betas']),eps=config['training']['optimizer_eps'])
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=config['training']['step_size'], gamma=config['training']['gamma'])
stats_cpu = {'input': xnorm.state_dict(), 'target': ynorm.state_dict()}
ynorm.to(device)
best, best_epoch, start = float('inf'), 0, 0
if restored:
model.load_state_dict(restored['model_state_dict'])
optimizer.load_state_dict(restored['optimizer_state_dict'])
scheduler.load_state_dict(restored['scheduler_state_dict'])
best, best_epoch, start = restored['best_test_l2'], restored['best_epoch'], restored['epoch']
generator.set_state(restored['rng']['loader'])
random.setstate(restored['rng']['python'])
np.random.set_state(restored['rng']['numpy'])
torch.set_rng_state(restored['rng']['torch'])
torch.cuda.set_rng_state_all(restored['rng']['accelerator'])
history = restored['history']
else:
history = []
epochs = config['training']['epochs']
hashes = source_hashes(ROOT)
metadata = {'seed':seed,'config':config,'parameters':sum(p.numel() for p in model.parameters()),
'node':socket.gethostname(),'slurm_job_id':os.environ.get('SLURM_JOB_ID'),
'device':torch.cuda.get_device_name(device),'torch':torch.__version__,
'hip':torch.version.hip,'cuda':torch.version.cuda,'numpy':np.__version__,'python':sys.version,'source_hashes':hashes,
'checkpoint_selection':'minimum mean test relative L2 (test-selected)',
'paper_test_l2':0.0030,'paper_conflict':'No auxiliary positional grid; paper two-channel input takes priority',
'precision':'float32; AMP/TF32 disabled','normalization_fit_indices':[0,ntrain]}
atomic_json(result_dir / 'run_metadata.json', metadata)
fields = ['epoch','seed','train_l2','test_l2','lr','next_lr','seconds','best_test_l2','best_epoch']
# Restore the durable CSV from checkpoint history, removing any interrupted trailing epoch.
with history_path.open('w', newline='') as f:
writer = csv.DictWriter(f, fieldnames=fields); writer.writeheader(); writer.writerows(history)
def save_checkpoint(path, payload):
temp = path.with_suffix('.tmp')
torch.save(payload, temp)
os.replace(temp, path)
print(f'PARAMETERS {metadata["parameters"]}; train={ntrain} test={ntest}; start_epoch={start+1}', flush=True)
run_start = time.perf_counter()
for epoch in range(start + 1, epochs + 1):
torch.cuda.synchronize(); tick = time.perf_counter()
lr = optimizer.param_groups[0]['lr']
model.train(); total_train = 0.0
for batch, (inputs, truth) in enumerate(train_loader, 1):
inputs = inputs.to(device, non_blocking=True)
truth = truth.to(device, non_blocking=True)
optimizer.zero_grad(set_to_none=True)
prediction = ynorm.decode(model(inputs))
loss = relative_l2(prediction, truth, reduction='sum')
if not torch.isfinite(loss):
raise FloatingPointError(f'Nonfinite training loss epoch={epoch} batch={batch}')
loss.backward(); optimizer.step()
total_train += loss.detach().item()
if epoch == start + 1 or batch % 10 == 0:
print(f'TRAIN seed={seed} epoch={epoch:03d}/{epochs} batch={batch:02d}/{len(train_loader)} '
f'loss={loss.item()/len(inputs):.8f}', flush=True)
scheduler.step()
model.eval(); total_test = 0.0
with torch.no_grad():
for inputs, truth in test_loader:
inputs, truth = inputs.to(device, non_blocking=True), truth.to(device, non_blocking=True)
val = relative_l2(ynorm.decode(model(inputs)), truth, reduction='sum')
if not torch.isfinite(val):
raise FloatingPointError('Nonfinite test metric')
total_test += val.item()
torch.cuda.synchronize()
train_l2, test_l2 = total_train/ntrain, total_test/ntest
improved = test_l2 < best
if improved: best, best_epoch = test_l2, epoch
row = dict(epoch=epoch, seed=seed, train_l2=train_l2, test_l2=test_l2, lr=lr,
next_lr=optimizer.param_groups[0]['lr'], seconds=time.perf_counter()-tick,
best_test_l2=best, best_epoch=best_epoch)
history.append(row)
payload = {'format_version':1,'model_state_dict':model.state_dict(), 'config':config,
'normalizers':stats_cpu,'epoch':epoch,'seed':seed,'test_l2':test_l2,
'best_test_l2':best,'best_epoch':best_epoch,'metadata':metadata,
'history':history,'optimizer_state_dict':optimizer.state_dict(),
'scheduler_state_dict':scheduler.state_dict(),
'rng':{'python':random.getstate(),'numpy':np.random.get_state(),
'torch':torch.get_rng_state(),'accelerator':torch.cuda.get_rng_state_all(),
'loader':generator.get_state()}}
if improved: save_checkpoint(best_path, payload)
save_checkpoint(last_path, payload)
with history_path.open('a', newline='') as f:
csv.DictWriter(f, fieldnames=fields).writerow(row); f.flush()
atomic_json(result_dir / 'progress.json', {**row, 'status':'training' if epoch<epochs else 'trained',
'total_epochs':epochs,'checkpoint':str(best_path)})
print(f'EPOCH seed={seed} epoch={epoch:03d}/{epochs} train_loss={train_l2:.8f} '
f'test_rel_l2={test_l2:.8f} lr={lr:.9g} seconds={row["seconds"]:.2f} '
f'best={best:.8f}@{best_epoch} saved_best={improved}', flush=True)
summary = {'status':'completed','seed':seed,'completed_epochs':epochs,'best_epoch':best_epoch,
'best_test_l2':best,'final_test_l2':history[-1]['test_l2'],
'total_seconds_this_process':time.perf_counter()-run_start,
'peak_accelerator_memory_gb':torch.cuda.max_memory_allocated()/1024**3,
'checkpoint':str(best_path),'early_stopping':False}
atomic_json(result_dir / 'training_summary.json', summary)
print('TRAINING_COMPLETE '+json.dumps(summary), flush=True)
return summary
def main():
args = arguments()
from common import load_config, require_compute_node
require_compute_node()
config = load_config(args.config)
run(config, args.seed if args.seed is not None else config['training']['seed'],
args.resume, args.output_root)
if __name__ == '__main__':
main()