Blame for scripts/result.py

"""Verify complete test outputs, visualize them, and compare documented metrics."""
import argparse
import json
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from common import (load_config, ensure_compute_node, load_fixed_dataset,
                    metrics_for, print_metrics, write_json)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--config", required=True)
    args = parser.parse_args()
    config = load_config(args.config)
    ensure_compute_node(config)
    results = Path(config["output"]["results_dir"])
    figures = results / "figures"
    figures.mkdir(parents=True, exist_ok=True)
    data, _ = load_fixed_dataset(config)
    targets = config["data"]["targets"]
    key = config["data"]["join_key"]
    test = data.loc[data[config["data"]["split_column"]] == "test"]
    predictions = pd.read_csv(results / "inference_predictions.csv")
    if predictions[key].tolist() != test[key].tolist() or len(predictions) != 922:
        raise ValueError("Visualization must include all 922 original fixed test cases")
    true = predictions[[f"{t}_true" for t in targets]].to_numpy()
    predicted = predictions[[f"{t}_pred" for t in targets]].to_numpy()
    np.testing.assert_allclose(true, test[targets].to_numpy(), atol=1e-12, rtol=1e-14)
    metrics = metrics_for(true, predicted, targets)
    print_metrics("verified-test", metrics, len(test))
    stored = json.loads((results / "metrics.json").read_text())
    for t in targets:
        for name in ["MAE", "RMSE", "R2"]:
            np.testing.assert_allclose(metrics[t][name], stored[t][name], rtol=1e-10, atol=1e-10)
    plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 10,
                         "axes.spines.top": False, "axes.spines.right": False,
                         "figure.facecolor": "white", "savefig.facecolor": "white"})
    labels = [r"$Nu_{avg}$", r"$\Delta P$ (Pa)"]
    colors = ["#007A87", "#C96822"]
    outputs = []

    def save(fig, name):
        for suffix in ["png", "pdf"]:
            path = figures / f"{name}.{suffix}"
            fig.savefig(path, dpi=config["output"]["figure_dpi"], bbox_inches="tight")
            outputs.append(str(path.relative_to(results)))
        plt.close(fig)
        print(f"[figure] Saved {name}.png/.pdf", flush=True)

    fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), constrained_layout=True)
    for i, (axis, target) in enumerate(zip(axes, targets)):
        axis.scatter(true[:, i], predicted[:, i], s=13, alpha=.58, color=colors[i], edgecolors="none")
        low = min(true[:, i].min(), predicted[:, i].min())
        high = max(true[:, i].max(), predicted[:, i].max())
        pad = (high - low) * .035
        axis.plot([low-pad, high+pad], [low-pad, high+pad], "--", color="#303A44", linewidth=1, label="1:1")
        axis.set(xlim=(low-pad, high+pad), ylim=(low-pad, high+pad), xlabel=f"CFD {labels[i]}",
                 ylabel=f"Random Forest {labels[i]}", title=f"{labels[i]} | fixed test, n=922")
        axis.set_aspect("equal", adjustable="box")
        m = metrics[target]
        annotation = f"MAE = {m['MAE']:.6f}\nRMSE = {m['RMSE']:.6f}\n" + r"$R^2$" + f" = {m['R2']:.8f}"
        axis.text(.04, .96, annotation, transform=axis.transAxes, va="top",
                  bbox={"facecolor": "white", "alpha": .9, "edgecolor": "#DFE4E7"})
        axis.grid(alpha=.16)
    save(fig, "parity")

    residual = predicted - true
    fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
    for i, axis in enumerate(axes):
        axis.scatter(true[:, i], residual[:, i], s=13, alpha=.55, color=colors[i], edgecolors="none")
        axis.axhline(0, color="#303A44", linestyle="--", linewidth=1)
        axis.set(xlabel=f"CFD {labels[i]}", ylabel=f"Prediction - CFD {labels[i]}", title=f"{labels[i]} residuals | n=922")
        axis.grid(alpha=.16)
    save(fig, "residuals")

    fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
    for i, axis in enumerate(axes):
        axis.hist(residual[:, i], bins=45, color=colors[i], edgecolor="white", linewidth=.4)
        axis.axvline(0, color="#303A44", linestyle="--", linewidth=1)
        axis.set(xlabel=f"Prediction - CFD {labels[i]}", ylabel="Test case count",
                 title=f"{labels[i]} residual distribution | n=922")
        axis.grid(axis="y", alpha=.16)
    save(fig, "residual_histograms")

    importance = pd.read_csv(results / "feature_importances.csv")
    if len(importance) != 7 or importance["feature"].tolist() != config["data"]["features"]:
        raise ValueError("Feature-importance schema mismatch")
    fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
    for i, (axis, target) in enumerate(zip(axes, targets)):
        np.testing.assert_allclose(importance[target].sum(), 1.0, atol=1e-12)
        order = importance[target].sort_values().index
        axis.barh(importance.loc[order, "feature"], importance.loc[order, target], color=colors[i])
        axis.set(xlabel="Impurity-based feature importance", title=f"{labels[i]} | independent forest")
        axis.grid(axis="x", alpha=.16)
    save(fig, "feature_importances")

    comparison = []
    for target in targets:
        for name in ["MAE", "RMSE", "R2"]:
            measured = metrics[target][name]
            reference = config["reference"]["reported_test_metrics"][target][name]
            comparison.append({"target": target, "metric": name, "reproduced": measured,
                               "author_supplement": reference, "difference": measured-reference,
                               "matches_reported_6dp": bool(abs(measured-reference) <= 0.0000005001)})
    comparison_frame = pd.DataFrame(comparison)
    comparison_frame.to_csv(results / "reference_comparison.csv", index=False)
    per_re = []
    for re in sorted(predictions["Re"].unique()):
        mask = predictions["Re"].to_numpy() == re
        for target, values in metrics_for(true[mask], predicted[mask], targets).items():
            per_re.append({"Re": float(re), "n_test": int(mask.sum()), "target": target, **values})
    pd.DataFrame(per_re).to_csv(results / "test_metrics_by_Re.csv", index=False)
    manifest = json.loads((results / "run_manifest.json").read_text())
    reload_check = json.loads((results / "reload_validation.json").read_text())
    all_match = bool(comparison_frame["matches_reported_6dp"].all())
    report = ["# WavyChannel Random Forest 随机 80/20 复现实验", "",
              "全部 3686 条训练样本拟合两个独立的 800 树森林；全部 922 条固定测试样本评估。",
              "按 case_uid 一对一匹配原始 CSV，未重新划分、未采样、未标准化、未转换目标。", "",
              "| 目标 | MAE | RMSE | R2 |", "|---|---:|---:|---:|"]
    for t in targets:
        m = metrics[t]
        report.append(f"| {t} | {m['MAE']:.10f} | {m['RMSE']:.10f} | {m['R2']:.10f} |")
    report += ["", "DelP_Pa 的 MAE/RMSE 单位为 Pa，Nuavg 无量纲。",
               f"六项指标与作者正式补充说明的六位小数一致：{all_match}。逐项数值见 reference_comparison.csv。",
               f"计算节点：{manifest['runtime']['host']}；SLURM 作业：{manifest['runtime']['slurm_job_id']}。",
               f"拟合时间（秒）：{manifest['fit_seconds']}。",
               f"独立进程重载最大预测差：{reload_check['max_abs_prediction_difference']:.3g}。", "",
               "## 来源与复现范围", "",
               "[论文](https://www.sciencedirect.com/science/article/pii/S305058522600025X)；"
               "[作者公开数据](https://data.mendeley.com/datasets/5b5n3cg32n/4)。",
               "正式超参数来自作者随公开数据附带的 07_tables/random80_20/training_summary.md："
               "800 棵树、min_samples_leaf=2、random_state=42、n_jobs=-1。",
               "出版社全文/PDF 返回 403，FULL API 返回 401，未完成论文正文/图表/附录逐页审核。"
               "其余参数显式沿用 scikit-learn 1.6.0 默认值，属于已记录假设；"
               "不能声称已逐项核实论文全文。未下载、复制或运行官方 Git 代码。",
               "best_model.pth 是 joblib 格式，包含两个 sklearn 随机森林；"
               "best 表示唯一完成正式拟合的模型，没有依据测试集挑选模型。",
               "随机森林没有 epoch；未执行早停、神经网络或额外拟合。", "",
               "## 图表", ""]
    for name in ["parity", "residuals", "residual_histograms", "feature_importances"]:
        report.append(f"![{name}](figures/{name}.png)")
    (results / "report.md").write_text("\n".join(report) + "\n")
    summary = {"status": "completed", "test_rows": len(test), "test_metrics": metrics,
               "all_6_metrics_match_author_report_6dp": all_match, "figure_files": outputs,
               "reload_validation": reload_check, "full_manuscript_audit": False,
               "evidence_limitations": "Publisher full text unavailable; formal RF parameters verified from author data supplement."}
    write_json(results / "experiment_summary.json", summary)
    print(f"[result] All 922 test cases verified. Six-metric reference match={all_match}", flush=True)
    print(f"[result] Report: {results / 'report.md'}", flush=True)


if __name__ == "__main__":
    main()