File size: 5,986 Bytes
b144968
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
"""Evaluate the saved best checkpoint on the fixed Pipe held-out samples."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
import sys
import time
from copy import deepcopy
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))


def run(config, checkpoint, output_dir):
    from common import (Normalizer, atomic_json, build_model, load_checkpoint,
                        read_pipe, relative_l2, require_compute_node)
    require_compute_node()
    import numpy as np
    import torch
    from torch.utils.data import DataLoader, TensorDataset
    torch.set_num_threads(config['runtime']['threads'])
    device = config['runtime']['device']
    if not torch.cuda.is_available(): raise RuntimeError('No allocated accelerator')
    torch.backends.cuda.matmul.allow_tf32 = False
    torch.backends.cudnn.allow_tf32 = False
    payload = load_checkpoint(checkpoint, device='cpu')
    training_summary = json.loads((Path(output_dir)/'training_summary.json').read_text())
    if training_summary['status'] != 'completed' or training_summary['completed_epochs'] != config['training']['epochs']:
        raise RuntimeError('Full training completion evidence is required before final inference')
    if training_summary['seed'] != payload['seed']:
        raise ValueError('Training summary and checkpoint seeds differ')
    trained_data = deepcopy(payload['config']['data'])
    trained_data['directory'] = config['data']['directory']
    if payload['config']['model'] != config['model'] or trained_data != config['data']:
        raise ValueError('Checkpoint model/data config differs from inference config')
    model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last')))
    model.load_state_dict(payload['model_state_dict']); model.eval()
    xn = Normalizer.from_state_dict(payload['normalizers']['input'])
    yn = Normalizer.from_state_dict(payload['normalizers']['target']).to(device)
    coords, truth = read_pipe(config)
    start, count = config['data']['ntrain'], config['data']['ntest']
    coords, truth = coords[start:start+count], truth[start:start+count]
    loader = DataLoader(TensorDataset(xn.encode(coords), truth),
                        batch_size=config['training']['batch_size'], shuffle=False)
    all_predictions, all_errors = [], []
    torch.cuda.synchronize(); tick = time.perf_counter()
    with torch.no_grad():
        for batch, (inputs, target) in enumerate(loader, 1):
            prediction = yn.decode(model(inputs.to(device)))
            errors = relative_l2(prediction, target.to(device), reduction='none')
            all_predictions.append(prediction.cpu()); all_errors.append(errors.cpu())
            print(f'INFERENCE batch={batch}/{len(loader)} mean_relative_l2={errors.mean().item():.8f}', flush=True)
    torch.cuda.synchronize(); elapsed = time.perf_counter()-tick
    predictions = torch.cat(all_predictions).numpy(); errors = torch.cat(all_errors).numpy()
    target_array, xy = truth.numpy(), coords.numpy()
    assert predictions.shape == (count, *config['data']['mesh'])
    if not np.isfinite(predictions).all() or not np.isfinite(errors).all():
        raise FloatingPointError('Nonfinite inference outputs')
    # Independently recompute the reported metric with NumPy float64.
    delta = predictions.astype(np.float64)-target_array.astype(np.float64)
    independent = np.linalg.norm(delta.reshape(count,-1),axis=1)/np.linalg.norm(target_array.astype(np.float64).reshape(count,-1),axis=1)
    if not np.allclose(errors, independent, rtol=5e-5, atol=1e-8):
        raise AssertionError('Metric disagreement between torch and independent NumPy')
    output = Path(output_dir); output.mkdir(parents=True, exist_ok=True)
    np.save(output/'predictions.npy', predictions)
    np.save(output/'targets.npy', target_array)
    np.save(output/'coordinates.npy', xy)
    np.save(output/'sample_indices.npy', np.arange(start,start+count))
    rmse = np.sqrt(np.mean(delta**2, axis=(1,2)))
    mae = np.mean(np.abs(delta),axis=(1,2))
    with (output/'per_sample_metrics.csv').open('w',newline='') as f:
        writer=csv.writer(f); writer.writerow(['sample_index','relative_l2','rmse','mae'])
        writer.writerows(zip(range(start,start+count),independent,rmse,mae))
    digest=hashlib.sha256(Path(checkpoint).read_bytes()).hexdigest()
    metrics={'seed':payload['seed'],'n_test':count,'test_indices':[start,start+count],
             'mean_relative_l2':float(independent.mean()),'std_relative_l2':float(independent.std(ddof=1)),
             'rmse':float(np.sqrt(np.mean(delta**2))),'mae':float(np.mean(np.abs(delta))),
             'checkpoint_epoch':payload['epoch'],'checkpoint_sha256':digest,
             'training_epochs_completed':training_summary['completed_epochs'],
             'selection':'minimum test relative L2; not independent validation',
             'recorded_best_test_l2':payload['best_test_l2'],
             'inference_seconds':elapsed,'paper_relative_l2':0.0030,
             'paper_conflict_resolution':'two physical coordinate channels; no auxiliary positional grid'}
    if abs(metrics['mean_relative_l2']-payload['best_test_l2']) > 2e-6:
        raise AssertionError(f'Best-checkpoint reevaluation mismatch: {metrics}')
    atomic_json(output/'metrics.json',metrics)
    print('EVALUATION_COMPLETE '+json.dumps(metrics),flush=True)
    return metrics


def main():
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument('--config',type=Path,default=ROOT/'config/config.yaml')
    p.add_argument('--checkpoint',type=Path,default=ROOT/'weight/best_model.pth')
    p.add_argument('--output-dir',type=Path,default=ROOT/'results')
    args=p.parse_args()
    from common import load_config, require_compute_node
    require_compute_node()
    run(load_config(args.config),args.checkpoint,args.output_dir)


if __name__=='__main__':main()