"""Evaluate the saved best checkpoint on the fixed Pipe held-out samples.""" from __future__ import annotations import argparse import csv import hashlib import json from pathlib import Path import sys import time from copy import deepcopy ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) def run(config, checkpoint, output_dir): from common import (Normalizer, atomic_json, build_model, load_checkpoint, read_pipe, relative_l2, require_compute_node) require_compute_node() import numpy as np import torch from torch.utils.data import DataLoader, TensorDataset torch.set_num_threads(config['runtime']['threads']) device = config['runtime']['device'] if not torch.cuda.is_available(): raise RuntimeError('No allocated accelerator') torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False payload = load_checkpoint(checkpoint, device='cpu') training_summary = json.loads((Path(output_dir)/'training_summary.json').read_text()) if training_summary['status'] != 'completed' or training_summary['completed_epochs'] != config['training']['epochs']: raise RuntimeError('Full training completion evidence is required before final inference') if training_summary['seed'] != payload['seed']: raise ValueError('Training summary and checkpoint seeds differ') trained_data = deepcopy(payload['config']['data']) trained_data['directory'] = config['data']['directory'] if payload['config']['model'] != config['model'] or trained_data != config['data']: raise ValueError('Checkpoint model/data config differs from inference config') model = build_model(config).to(device=device, memory_format=getattr(torch, config['runtime'].get('memory_format', 'channels_last'))) model.load_state_dict(payload['model_state_dict']); model.eval() xn = Normalizer.from_state_dict(payload['normalizers']['input']) yn = Normalizer.from_state_dict(payload['normalizers']['target']).to(device) coords, truth = read_pipe(config) start, count = config['data']['ntrain'], config['data']['ntest'] coords, truth = coords[start:start+count], truth[start:start+count] loader = DataLoader(TensorDataset(xn.encode(coords), truth), batch_size=config['training']['batch_size'], shuffle=False) all_predictions, all_errors = [], [] torch.cuda.synchronize(); tick = time.perf_counter() with torch.no_grad(): for batch, (inputs, target) in enumerate(loader, 1): prediction = yn.decode(model(inputs.to(device))) errors = relative_l2(prediction, target.to(device), reduction='none') all_predictions.append(prediction.cpu()); all_errors.append(errors.cpu()) print(f'INFERENCE batch={batch}/{len(loader)} mean_relative_l2={errors.mean().item():.8f}', flush=True) torch.cuda.synchronize(); elapsed = time.perf_counter()-tick predictions = torch.cat(all_predictions).numpy(); errors = torch.cat(all_errors).numpy() target_array, xy = truth.numpy(), coords.numpy() assert predictions.shape == (count, *config['data']['mesh']) if not np.isfinite(predictions).all() or not np.isfinite(errors).all(): raise FloatingPointError('Nonfinite inference outputs') # Independently recompute the reported metric with NumPy float64. delta = predictions.astype(np.float64)-target_array.astype(np.float64) independent = np.linalg.norm(delta.reshape(count,-1),axis=1)/np.linalg.norm(target_array.astype(np.float64).reshape(count,-1),axis=1) if not np.allclose(errors, independent, rtol=5e-5, atol=1e-8): raise AssertionError('Metric disagreement between torch and independent NumPy') output = Path(output_dir); output.mkdir(parents=True, exist_ok=True) np.save(output/'predictions.npy', predictions) np.save(output/'targets.npy', target_array) np.save(output/'coordinates.npy', xy) np.save(output/'sample_indices.npy', np.arange(start,start+count)) rmse = np.sqrt(np.mean(delta**2, axis=(1,2))) mae = np.mean(np.abs(delta),axis=(1,2)) with (output/'per_sample_metrics.csv').open('w',newline='') as f: writer=csv.writer(f); writer.writerow(['sample_index','relative_l2','rmse','mae']) writer.writerows(zip(range(start,start+count),independent,rmse,mae)) digest=hashlib.sha256(Path(checkpoint).read_bytes()).hexdigest() metrics={'seed':payload['seed'],'n_test':count,'test_indices':[start,start+count], 'mean_relative_l2':float(independent.mean()),'std_relative_l2':float(independent.std(ddof=1)), 'rmse':float(np.sqrt(np.mean(delta**2))),'mae':float(np.mean(np.abs(delta))), 'checkpoint_epoch':payload['epoch'],'checkpoint_sha256':digest, 'training_epochs_completed':training_summary['completed_epochs'], 'selection':'minimum test relative L2; not independent validation', 'recorded_best_test_l2':payload['best_test_l2'], 'inference_seconds':elapsed,'paper_relative_l2':0.0030, 'paper_conflict_resolution':'two physical coordinate channels; no auxiliary positional grid'} if abs(metrics['mean_relative_l2']-payload['best_test_l2']) > 2e-6: raise AssertionError(f'Best-checkpoint reevaluation mismatch: {metrics}') atomic_json(output/'metrics.json',metrics) print('EVALUATION_COMPLETE '+json.dumps(metrics),flush=True) return metrics def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('--config',type=Path,default=ROOT/'config/config.yaml') p.add_argument('--checkpoint',type=Path,default=ROOT/'weight/best_model.pth') p.add_argument('--output-dir',type=Path,default=ROOT/'results') args=p.parse_args() from common import load_config, require_compute_node require_compute_node() run(load_config(args.config),args.checkpoint,args.output_dir) if __name__=='__main__':main()