"""Produce reproducible figures and a paper comparison from saved test predictions.""" from __future__ import annotations import csv import json import shutil from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import torch from common import ROOT, arguments, configuration, output_path def main(): parser = arguments(__doc__) parser.add_argument("--sample", type=int, default=19) args = parser.parse_args() cfg = configuration(args) directory = output_path(cfg["output"]["results_dir"]) metrics = json.loads((directory / "metrics.json").read_text()) summary = json.loads((directory / "training_summary.json").read_text()) with (directory / "history.csv").open() as stream: history = list(csv.DictReader(stream)) if len(history) != 500 or [int(row["epoch"]) for row in history] != list(range(1, 501)): raise ValueError("Results require one contiguous full 500-epoch training history") data = torch.load(directory / "test_predictions.pt", map_location="cpu", weights_only=True) if len(data["prediction"]) != 100: raise ValueError("Expected all 100 test predictions") epochs = np.array([int(row["epoch"]) for row in history]) energy = np.array([float(row["train_energy"]) for row in history]) error = np.array([float(row["test_relative_l2"]) for row in history]) * 100 fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) axes[0].plot(epochs, energy, label="Train variational energy") axes[0].plot(epochs, [float(row["test_energy"]) for row in history], label="Test variational energy", alpha=0.8) axes[0].set(xlabel="Epoch", ylabel="Energy", title="Unlabeled training objective") axes[0].legend() axes[1].semilogy(epochs, error, label="Test relative L2") axes[1].axhline(cfg["reference"]["table1_mean_percent"], color="black", linestyle="--", label="Paper Table 1 mean") axes[1].scatter([metrics["checkpoint_epoch"]], [metrics["relative_l2_mean_percent"]], color="red", s=30, label="Selected checkpoint", zorder=3) axes[1].set(xlabel="Epoch", ylabel="Mean relative L2 (%)", title="100 fixed test cases") axes[1].legend() for extension in ("png", "pdf"): fig.savefig(directory / f"training_curves.{extension}", dpi=180) plt.close(fig) errors = data["relative_l2"].numpy() * 100 fig, axes = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True) axes[0].hist(errors, bins=16, color="#2879a5", edgecolor="white") axes[0].axvline(errors.mean(), color="red", label=f"Mean {errors.mean():.3f}%") axes[0].set(xlabel="Relative L2 (%)", ylabel="Test cases", title="Test error distribution") axes[0].legend() axes[1].boxplot(errors, labels=["VINO reproduction"]) axes[1].axhline(cfg["reference"]["table1_mean_percent"], linestyle="--", color="black", label="Paper mean") axes[1].set(ylabel="Relative L2 (%)") axes[1].legend() fig.savefig(directory / "test_error_distribution.png", dpi=180) plt.close(fig) order = np.argsort(errors) selections = list(dict.fromkeys([args.sample, int(order[0]), int(order[len(order)//2]), int(order[-1])])) for index in selections: if index < 0 or index >= 100: raise ValueError("Sample index must be in [0,99]") coefficient = data["nodal_permeability"][index, ..., 0].numpy() reference = data["reference"][index, ..., 0].numpy() predicted = data["prediction"][index, ..., 0].numpy() low, high = min(reference.min(), predicted.min()), max(reference.max(), predicted.max()) fig, axes = plt.subplots(1, 4, figsize=(14, 3.7), constrained_layout=True) for ax, value, title, cmap, limits in zip(axes, (coefficient, reference, predicted, np.abs(predicted-reference)), ("Nodal permeability", "Reference head", "VINO head", "Absolute error"), ("viridis", "viridis", "viridis", "magma"), ((None, None), (low, high), (low, high), (0, None))): plot = ax.imshow(value.T, origin="lower", extent=[0,1,0,1], cmap=cmap, vmin=limits[0], vmax=limits[1]) ax.set(xlabel="x", ylabel="y", title=title) fig.colorbar(plot, ax=ax, shrink=0.8) fig.suptitle(f"Original test case {index}: relative L2 = {errors[index]:.4f}%") fig.savefig(directory / f"test_case_{index:03d}.png", dpi=180) plt.close(fig) report = f"""# VINO Darcy Flow experiment Completed **500 epochs**, using the original **1000 training / 100 test** samples at **64×64** resolution. | Measurement | Reproduction | Paper Table 1 | |---|---:|---:| | Test relative L2 mean | {metrics['relative_l2_mean_percent']:.6f}% | 0.93% | | Test relative L2 sample standard deviation | {metrics['relative_l2_std_percent']:.6f}% | 0.31% | | Training wall time | {summary['training_seconds']:.2f} s | 244.2 s (A100) | Selected checkpoint: epoch **{metrics['checkpoint_epoch']}**, `weight/best_model.pt`. Final epoch 500 mean relative L2: **{summary['last_test_relative_l2']*100:.6f}%**. Zero Dirichlet boundary maximum absolute value: **{metrics['boundary_max_abs']}**. Mean error difference from paper: **{metrics['relative_l2_mean_percent']-0.93:+.6f} percentage points**. Training uses only permeability and the variational energy. U_train never enters optimization. The test labels select the best checkpoint as explicitly requested; the reported best-checkpoint test score therefore includes model selection on that same test set. The epoch-500 score is also provided. The standard deviation is across 100 test samples, not across independent training seeds. Paper: https://arxiv.org/html/2411.06587 (Section 2.1.3, Table 1, Section 4.2). Implementation is independent. Official source was viewed online only for paper omissions; no repository code was downloaded or copied. Exact Q1 rectangular-element integration follows the paper rather than the ambiguous manual Darcy expansion in the official helper. The coefficient is bilinearly interpolated within each element after the official missing-detail nodal averaging step. Network and optimizer details, assumptions, and source conflicts are recorded in `reproduction_spec.md`. The paper describes a permeability contrast of 4, whereas the required author file contains the two values 4 and 12 (contrast 3). The original data were used without rescaling or replacement, as requested. Hardware/software are in `environment.json`; timing includes per-epoch testing and checkpoint I/O and is not a direct hardware-controlled comparison. Dataset SHA256: `{metrics['data_sha256']}`. Files: `history.csv`, `metrics.json`, `per_sample_metrics.csv`, `test_predictions.pt`, `training_curves.png`, `test_error_distribution.png`, and selected `test_case_*.png`. """ (directory / "report.md").write_text(report) paper_spec = ROOT.parents[2] / ".paper2code_work/2411.06587/reproduction_spec.md" if paper_spec.exists(): shutil.copyfile(paper_spec, directory / "reproduction_spec.md") print(report, flush=True) if __name__ == "__main__": main()