File size: 11,084 Bytes
b144968
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
"""Independent paper-first Pipe experiment. Execute only inside a SLURM allocation."""
from __future__ import annotations
import argparse
import csv
import json
import os
from pathlib import Path
import random
import socket
import sys
import time

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))


def arguments():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--config', type=Path, default=ROOT / 'config/config.yaml')
    p.add_argument('--seed', type=int)
    p.add_argument('--resume', type=Path, help='Restore complete epoch-boundary training state')
    p.add_argument('--output-root', type=Path)
    return p.parse_args()


def run(config, seed, resume=None, output_root=None):
    from common import (Normalizer, atomic_json, build_model, load_checkpoint,
                        read_pipe, relative_l2, require_compute_node, seed_all,
                        source_hashes)
    require_compute_node()
    import numpy as np
    import torch
    from torch.utils.data import DataLoader, TensorDataset
    if not torch.cuda.is_available():
        raise RuntimeError('Allocated compute node has no accessible accelerator')
    torch.set_num_threads(config['runtime']['threads'])
    seed_all(seed)
    # Full FP32; do not silently enable mixed precision or TF32.
    torch.backends.cuda.matmul.allow_tf32 = False
    torch.backends.cudnn.allow_tf32 = False
    device = torch.device(config['runtime']['device'])
    base = Path(output_root or config['output']['root'])
    primary = seed == config['training']['seed']
    result_dir = base / 'results' if primary else base / 'results' / f'seed_{seed}'
    weight_dir = base / 'weight' if primary else base / 'weight' / f'seed_{seed}'
    result_dir.mkdir(parents=True, exist_ok=True)
    weight_dir.mkdir(parents=True, exist_ok=True)
    last_path = weight_dir / 'last_model.pth'
    best_path = weight_dir / 'best_model.pth'
    history_path = result_dir / 'train_history.csv'
    if history_path.exists() and resume is None:
        raise FileExistsError(f'{history_path} exists; use --resume to preserve training')
    print(f'BEGIN seed={seed} node={socket.gethostname()} job={os.environ.get("SLURM_JOB_ID")} '
          f'device={torch.cuda.get_device_name(device)} epochs={config["training"]["epochs"]}', flush=True)
    coords, targets = read_pipe(config)
    ntrain, ntest = config['data']['ntrain'], config['data']['ntest']
    restored = load_checkpoint(resume, device='cpu') if resume else None
    if restored:
        for section in ['model', 'data', 'training']:
            if restored['config'][section] != config[section]:
                raise ValueError(f'Resume {section} configuration differs from checkpoint')
        if restored['seed'] != seed:
            raise ValueError('Resume seed differs from checkpoint')
        xnorm = Normalizer.from_state_dict(restored['normalizers']['input'])
        ynorm = Normalizer.from_state_dict(restored['normalizers']['target'])
    else:
        xnorm = Normalizer.fit(coords[:ntrain], eps=config['data']['normalization_eps'])
        ynorm = Normalizer.fit(targets[:ntrain], eps=config['data']['normalization_eps'])
    x = xnorm.encode(coords)
    if not torch.isfinite(x).all() or not torch.isfinite(targets).all():
        raise ValueError('Dataset or normalization contains NaN/Inf')
    generator = torch.Generator().manual_seed(seed)
    train_loader = DataLoader(TensorDataset(x[:ntrain], targets[:ntrain]),
                              batch_size=config['training']['batch_size'], shuffle=True,
                              generator=generator, num_workers=config['runtime']['num_workers'],
                              pin_memory=True)
    test_loader = DataLoader(TensorDataset(x[ntrain:ntrain+ntest], targets[ntrain:ntrain+ntest]),
                             batch_size=config['training']['batch_size'], shuffle=False,
                             num_workers=config['runtime']['num_workers'], pin_memory=True)
    model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last')))
    optimizer = torch.optim.Adam(model.parameters(), lr=config['training']['lr'],
                                 weight_decay=config['training']['weight_decay'],
                                 betas=tuple(config['training']['betas']),eps=config['training']['optimizer_eps'])
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
                       step_size=config['training']['step_size'], gamma=config['training']['gamma'])
    stats_cpu = {'input': xnorm.state_dict(), 'target': ynorm.state_dict()}
    ynorm.to(device)
    best, best_epoch, start = float('inf'), 0, 0
    if restored:
        model.load_state_dict(restored['model_state_dict'])
        optimizer.load_state_dict(restored['optimizer_state_dict'])
        scheduler.load_state_dict(restored['scheduler_state_dict'])
        best, best_epoch, start = restored['best_test_l2'], restored['best_epoch'], restored['epoch']
        generator.set_state(restored['rng']['loader'])
        random.setstate(restored['rng']['python'])
        np.random.set_state(restored['rng']['numpy'])
        torch.set_rng_state(restored['rng']['torch'])
        torch.cuda.set_rng_state_all(restored['rng']['accelerator'])
        history = restored['history']
    else:
        history = []
    epochs = config['training']['epochs']
    hashes = source_hashes(ROOT)
    metadata = {'seed':seed,'config':config,'parameters':sum(p.numel() for p in model.parameters()),
                'node':socket.gethostname(),'slurm_job_id':os.environ.get('SLURM_JOB_ID'),
                'device':torch.cuda.get_device_name(device),'torch':torch.__version__,
                'hip':torch.version.hip,'cuda':torch.version.cuda,'numpy':np.__version__,'python':sys.version,'source_hashes':hashes,
                'checkpoint_selection':'minimum mean test relative L2 (test-selected)',
                'paper_test_l2':0.0030,'paper_conflict':'No auxiliary positional grid; paper two-channel input takes priority',
                'precision':'float32; AMP/TF32 disabled','normalization_fit_indices':[0,ntrain]}
    atomic_json(result_dir / 'run_metadata.json', metadata)
    fields = ['epoch','seed','train_l2','test_l2','lr','next_lr','seconds','best_test_l2','best_epoch']
    # Restore the durable CSV from checkpoint history, removing any interrupted trailing epoch.
    with history_path.open('w', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=fields); writer.writeheader(); writer.writerows(history)
    def save_checkpoint(path, payload):
        temp = path.with_suffix('.tmp')
        torch.save(payload, temp)
        os.replace(temp, path)
    print(f'PARAMETERS {metadata["parameters"]}; train={ntrain} test={ntest}; start_epoch={start+1}', flush=True)
    run_start = time.perf_counter()
    for epoch in range(start + 1, epochs + 1):
        torch.cuda.synchronize(); tick = time.perf_counter()
        lr = optimizer.param_groups[0]['lr']
        model.train(); total_train = 0.0
        for batch, (inputs, truth) in enumerate(train_loader, 1):
            inputs = inputs.to(device, non_blocking=True)
            truth = truth.to(device, non_blocking=True)
            optimizer.zero_grad(set_to_none=True)
            prediction = ynorm.decode(model(inputs))
            loss = relative_l2(prediction, truth, reduction='sum')
            if not torch.isfinite(loss):
                raise FloatingPointError(f'Nonfinite training loss epoch={epoch} batch={batch}')
            loss.backward(); optimizer.step()
            total_train += loss.detach().item()
            if epoch == start + 1 or batch % 10 == 0:
                print(f'TRAIN seed={seed} epoch={epoch:03d}/{epochs} batch={batch:02d}/{len(train_loader)} '
                      f'loss={loss.item()/len(inputs):.8f}', flush=True)
        scheduler.step()
        model.eval(); total_test = 0.0
        with torch.no_grad():
            for inputs, truth in test_loader:
                inputs, truth = inputs.to(device, non_blocking=True), truth.to(device, non_blocking=True)
                val = relative_l2(ynorm.decode(model(inputs)), truth, reduction='sum')
                if not torch.isfinite(val):
                    raise FloatingPointError('Nonfinite test metric')
                total_test += val.item()
        torch.cuda.synchronize()
        train_l2, test_l2 = total_train/ntrain, total_test/ntest
        improved = test_l2 < best
        if improved: best, best_epoch = test_l2, epoch
        row = dict(epoch=epoch, seed=seed, train_l2=train_l2, test_l2=test_l2, lr=lr,
                   next_lr=optimizer.param_groups[0]['lr'], seconds=time.perf_counter()-tick,
                   best_test_l2=best, best_epoch=best_epoch)
        history.append(row)
        payload = {'format_version':1,'model_state_dict':model.state_dict(), 'config':config,
                   'normalizers':stats_cpu,'epoch':epoch,'seed':seed,'test_l2':test_l2,
                   'best_test_l2':best,'best_epoch':best_epoch,'metadata':metadata,
                   'history':history,'optimizer_state_dict':optimizer.state_dict(),
                   'scheduler_state_dict':scheduler.state_dict(),
                   'rng':{'python':random.getstate(),'numpy':np.random.get_state(),
                          'torch':torch.get_rng_state(),'accelerator':torch.cuda.get_rng_state_all(),
                          'loader':generator.get_state()}}
        if improved: save_checkpoint(best_path, payload)
        save_checkpoint(last_path, payload)
        with history_path.open('a', newline='') as f:
            csv.DictWriter(f, fieldnames=fields).writerow(row); f.flush()
        atomic_json(result_dir / 'progress.json', {**row, 'status':'training' if epoch<epochs else 'trained',
                                                   'total_epochs':epochs,'checkpoint':str(best_path)})
        print(f'EPOCH seed={seed} epoch={epoch:03d}/{epochs} train_loss={train_l2:.8f} '
              f'test_rel_l2={test_l2:.8f} lr={lr:.9g} seconds={row["seconds"]:.2f} '
              f'best={best:.8f}@{best_epoch} saved_best={improved}', flush=True)
    summary = {'status':'completed','seed':seed,'completed_epochs':epochs,'best_epoch':best_epoch,
               'best_test_l2':best,'final_test_l2':history[-1]['test_l2'],
               'total_seconds_this_process':time.perf_counter()-run_start,
               'peak_accelerator_memory_gb':torch.cuda.max_memory_allocated()/1024**3,
               'checkpoint':str(best_path),'early_stopping':False}
    atomic_json(result_dir / 'training_summary.json', summary)
    print('TRAINING_COMPLETE '+json.dumps(summary), flush=True)
    return summary


def main():
    args = arguments()
    from common import load_config, require_compute_node
    require_compute_node()
    config = load_config(args.config)
    run(config, args.seed if args.seed is not None else config['training']['seed'],
        args.resume, args.output_root)


if __name__ == '__main__':
    main()