| """Verify complete test outputs, visualize them, and compare documented metrics.""" |
| import argparse |
| import json |
| from pathlib import Path |
|
|
| import matplotlib |
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
| import numpy as np |
| import pandas as pd |
| from common import (load_config, ensure_compute_node, load_fixed_dataset, |
| metrics_for, print_metrics, write_json) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--config", required=True) |
| args = parser.parse_args() |
| config = load_config(args.config) |
| ensure_compute_node(config) |
| results = Path(config["output"]["results_dir"]) |
| figures = results / "figures" |
| figures.mkdir(parents=True, exist_ok=True) |
| data, _ = load_fixed_dataset(config) |
| targets = config["data"]["targets"] |
| key = config["data"]["join_key"] |
| test = data.loc[data[config["data"]["split_column"]] == "test"] |
| predictions = pd.read_csv(results / "inference_predictions.csv") |
| if predictions[key].tolist() != test[key].tolist() or len(predictions) != 922: |
| raise ValueError("Visualization must include all 922 original fixed test cases") |
| true = predictions[[f"{t}_true" for t in targets]].to_numpy() |
| predicted = predictions[[f"{t}_pred" for t in targets]].to_numpy() |
| np.testing.assert_allclose(true, test[targets].to_numpy(), atol=1e-12, rtol=1e-14) |
| metrics = metrics_for(true, predicted, targets) |
| print_metrics("verified-test", metrics, len(test)) |
| stored = json.loads((results / "metrics.json").read_text()) |
| for t in targets: |
| for name in ["MAE", "RMSE", "R2"]: |
| np.testing.assert_allclose(metrics[t][name], stored[t][name], rtol=1e-10, atol=1e-10) |
| plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 10, |
| "axes.spines.top": False, "axes.spines.right": False, |
| "figure.facecolor": "white", "savefig.facecolor": "white"}) |
| labels = [r"$Nu_{avg}$", r"$\Delta P$ (Pa)"] |
| colors = ["#007A87", "#C96822"] |
| outputs = [] |
|
|
| def save(fig, name): |
| for suffix in ["png", "pdf"]: |
| path = figures / f"{name}.{suffix}" |
| fig.savefig(path, dpi=config["output"]["figure_dpi"], bbox_inches="tight") |
| outputs.append(str(path.relative_to(results))) |
| plt.close(fig) |
| print(f"[figure] Saved {name}.png/.pdf", flush=True) |
|
|
| fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), constrained_layout=True) |
| for i, (axis, target) in enumerate(zip(axes, targets)): |
| axis.scatter(true[:, i], predicted[:, i], s=13, alpha=.58, color=colors[i], edgecolors="none") |
| low = min(true[:, i].min(), predicted[:, i].min()) |
| high = max(true[:, i].max(), predicted[:, i].max()) |
| pad = (high - low) * .035 |
| axis.plot([low-pad, high+pad], [low-pad, high+pad], "--", color="#303A44", linewidth=1, label="1:1") |
| axis.set(xlim=(low-pad, high+pad), ylim=(low-pad, high+pad), xlabel=f"CFD {labels[i]}", |
| ylabel=f"Random Forest {labels[i]}", title=f"{labels[i]} | fixed test, n=922") |
| axis.set_aspect("equal", adjustable="box") |
| m = metrics[target] |
| annotation = f"MAE = {m['MAE']:.6f}\nRMSE = {m['RMSE']:.6f}\n" + r"$R^2$" + f" = {m['R2']:.8f}" |
| axis.text(.04, .96, annotation, transform=axis.transAxes, va="top", |
| bbox={"facecolor": "white", "alpha": .9, "edgecolor": "#DFE4E7"}) |
| axis.grid(alpha=.16) |
| save(fig, "parity") |
|
|
| residual = predicted - true |
| fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) |
| for i, axis in enumerate(axes): |
| axis.scatter(true[:, i], residual[:, i], s=13, alpha=.55, color=colors[i], edgecolors="none") |
| axis.axhline(0, color="#303A44", linestyle="--", linewidth=1) |
| axis.set(xlabel=f"CFD {labels[i]}", ylabel=f"Prediction - CFD {labels[i]}", title=f"{labels[i]} residuals | n=922") |
| axis.grid(alpha=.16) |
| save(fig, "residuals") |
|
|
| fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) |
| for i, axis in enumerate(axes): |
| axis.hist(residual[:, i], bins=45, color=colors[i], edgecolor="white", linewidth=.4) |
| axis.axvline(0, color="#303A44", linestyle="--", linewidth=1) |
| axis.set(xlabel=f"Prediction - CFD {labels[i]}", ylabel="Test case count", |
| title=f"{labels[i]} residual distribution | n=922") |
| axis.grid(axis="y", alpha=.16) |
| save(fig, "residual_histograms") |
|
|
| importance = pd.read_csv(results / "feature_importances.csv") |
| if len(importance) != 7 or importance["feature"].tolist() != config["data"]["features"]: |
| raise ValueError("Feature-importance schema mismatch") |
| fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) |
| for i, (axis, target) in enumerate(zip(axes, targets)): |
| np.testing.assert_allclose(importance[target].sum(), 1.0, atol=1e-12) |
| order = importance[target].sort_values().index |
| axis.barh(importance.loc[order, "feature"], importance.loc[order, target], color=colors[i]) |
| axis.set(xlabel="Impurity-based feature importance", title=f"{labels[i]} | independent forest") |
| axis.grid(axis="x", alpha=.16) |
| save(fig, "feature_importances") |
|
|
| comparison = [] |
| for target in targets: |
| for name in ["MAE", "RMSE", "R2"]: |
| measured = metrics[target][name] |
| reference = config["reference"]["reported_test_metrics"][target][name] |
| comparison.append({"target": target, "metric": name, "reproduced": measured, |
| "author_supplement": reference, "difference": measured-reference, |
| "matches_reported_6dp": bool(abs(measured-reference) <= 0.0000005001)}) |
| comparison_frame = pd.DataFrame(comparison) |
| comparison_frame.to_csv(results / "reference_comparison.csv", index=False) |
| per_re = [] |
| for re in sorted(predictions["Re"].unique()): |
| mask = predictions["Re"].to_numpy() == re |
| for target, values in metrics_for(true[mask], predicted[mask], targets).items(): |
| per_re.append({"Re": float(re), "n_test": int(mask.sum()), "target": target, **values}) |
| pd.DataFrame(per_re).to_csv(results / "test_metrics_by_Re.csv", index=False) |
| manifest = json.loads((results / "run_manifest.json").read_text()) |
| reload_check = json.loads((results / "reload_validation.json").read_text()) |
| all_match = bool(comparison_frame["matches_reported_6dp"].all()) |
| report = ["# WavyChannel Random Forest 随机 80/20 复现实验", "", |
| "全部 3686 条训练样本拟合两个独立的 800 树森林；全部 922 条固定测试样本评估。", |
| "按 case_uid 一对一匹配原始 CSV，未重新划分、未采样、未标准化、未转换目标。", "", |
| "| 目标 | MAE | RMSE | R2 |", "|---|---:|---:|---:|"] |
| for t in targets: |
| m = metrics[t] |
| report.append(f"| {t} | {m['MAE']:.10f} | {m['RMSE']:.10f} | {m['R2']:.10f} |") |
| report += ["", "DelP_Pa 的 MAE/RMSE 单位为 Pa，Nuavg 无量纲。", |
| f"六项指标与作者正式补充说明的六位小数一致：{all_match}。逐项数值见 reference_comparison.csv。", |
| f"计算节点：{manifest['runtime']['host']}；SLURM 作业：{manifest['runtime']['slurm_job_id']}。", |
| f"拟合时间（秒）：{manifest['fit_seconds']}。", |
| f"独立进程重载最大预测差：{reload_check['max_abs_prediction_difference']:.3g}。", "", |
| "## 来源与复现范围", "", |
| "[论文](https://www.sciencedirect.com/science/article/pii/S305058522600025X)；" |
| "[作者公开数据](https://data.mendeley.com/datasets/5b5n3cg32n/4)。", |
| "正式超参数来自作者随公开数据附带的 07_tables/random80_20/training_summary.md：" |
| "800 棵树、min_samples_leaf=2、random_state=42、n_jobs=-1。", |
| "出版社全文/PDF 返回 403，FULL API 返回 401，未完成论文正文/图表/附录逐页审核。" |
| "其余参数显式沿用 scikit-learn 1.6.0 默认值，属于已记录假设；" |
| "不能声称已逐项核实论文全文。未下载、复制或运行官方 Git 代码。", |
| "best_model.pth 是 joblib 格式，包含两个 sklearn 随机森林；" |
| "best 表示唯一完成正式拟合的模型，没有依据测试集挑选模型。", |
| "随机森林没有 epoch；未执行早停、神经网络或额外拟合。", "", |
| "## 图表", ""] |
| for name in ["parity", "residuals", "residual_histograms", "feature_importances"]: |
| report.append(f"") |
| (results / "report.md").write_text("\n".join(report) + "\n") |
| summary = {"status": "completed", "test_rows": len(test), "test_metrics": metrics, |
| "all_6_metrics_match_author_report_6dp": all_match, "figure_files": outputs, |
| "reload_validation": reload_check, "full_manuscript_audit": False, |
| "evidence_limitations": "Publisher full text unavailable; formal RF parameters verified from author data supplement."} |
| write_json(results / "experiment_summary.json", summary) |
| print(f"[result] All 922 test cases verified. Six-metric reference match={all_match}", flush=True) |
| print(f"[result] Report: {results / 'report.md'}", flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
