scripts/result.py

"""Compute paper metrics for every scenario and draw maps and error boxes."""

import json
import sys
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import yaml


ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from model.climemu_s2l import GRID_SHAPE, area_weights, weighted_rmse


REGIONS = {
    "North America": (15, 75, 190, 310), "Europe": (35, 70, 350, 45),
    "South Asia": (5, 35, 60, 100), "East Asia": (20, 55, 100, 150),
    "Arctic": (66, 90, 0, 360), "Northwest Asia": (50, 75, 45, 120),
    "Northern Africa": (15, 35, 340, 55), "Southern Africa": (-35, 0, 10, 55),
    "South America": (-60, 15, 275, 330), "Australia": (-45, -10, 110, 155),
}


def region_mask(latitude, longitude, bounds):
    south, north, west, east = bounds
    latitude_mask = (latitude[:, None] >= south) & (latitude[:, None] <= north)
    longitude_mask = ((longitude[None, :] >= west) & (longitude[None, :] <= east)
                      if west <= east else (longitude[None, :] >= west) | (longitude[None, :] <= east))
    return latitude_mask & longitude_mask


def weighted_mean(field, weights, mask=None):
    selected = np.ones(field.shape, dtype=bool) if mask is None else mask
    local_weights = weights[selected]
    return float(np.sum(field[selected] * local_weights) / np.sum(local_weights))


def main():
    config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
    data = np.load(ROOT / config["paths"]["inference"])
    latitude, longitude = data["latitude_deg"], data["longitude_deg"]
    weights = area_weights(latitude, GRID_SHAPE[1])
    targets = data["long_response"]
    scenario_ids = [str(item) for item in data["scenario_ids"]]
    masks = {name: region_mask(latitude, longitude, bounds) for name, bounds in REGIONS.items()}
    report = {"grid": list(GRID_SHAPE), "scenario_count": 21, "regions": list(REGIONS), "methods": {}}
    box_values = {}
    for method in ("ridge", "gpr"):
        predictions = data[f"{method}_prediction"]
        scenarios = []
        for index, scenario_id in enumerate(scenario_ids):
            target, prediction = targets[index], predictions[index]
            regional = {name: abs(weighted_mean(prediction, weights, mask) - weighted_mean(target, weights, mask))
                        for name, mask in masks.items()}
            scenarios.append({"scenario_id": scenario_id,
                              "area_weighted_grid_rmse": weighted_rmse(target, prediction, weights),
                              "global_mean_absolute_error": abs(weighted_mean(prediction, weights) - weighted_mean(target, weights)),
                              "regional_mean_absolute_error": regional})
        grid_errors = [item["area_weighted_grid_rmse"] for item in scenarios]
        global_errors = [item["global_mean_absolute_error"] for item in scenarios]
        report["methods"][method] = {"scenarios": scenarios,
            "summary": {"mean_grid_rmse": float(np.mean(grid_errors)), "median_grid_rmse": float(np.median(grid_errors)),
                        "mean_global_absolute_error": float(np.mean(global_errors)),
                        "mean_regional_absolute_error": {name: float(np.mean([item["regional_mean_absolute_error"][name]
                                                                             for item in scenarios])) for name in REGIONS}}}
        box_values[method] = [grid_errors, global_errors] + [[item["regional_mean_absolute_error"][name] for item in scenarios]
                                                             for name in REGIONS]
    numeric = [value for method in report["methods"].values() for value in
               (method["summary"]["mean_grid_rmse"], method["summary"]["mean_global_absolute_error"])]
    if not np.isfinite(numeric).all():
        raise FloatingPointError("evaluation metrics are not finite")
    output = ROOT / config["paths"]["evaluation_dir"]
    output.mkdir(parents=True, exist_ok=True)
    (output / "metrics.json").write_text(json.dumps(report, indent=2) + "\n")
    plot_index = scenario_ids.index(config["evaluation"]["plot_scenario_id"])
    fields = [targets[plot_index], data["ridge_prediction"][plot_index], data["gpr_prediction"][plot_index],
              data["ridge_prediction"][plot_index] - targets[plot_index], data["gpr_prediction"][plot_index] - targets[plot_index]]
    titles = ["HadGEM3 target", "Dual Ridge", "Shared-kernel GPR", "Ridge error", "GPR error"]
    figure, axes = plt.subplots(2, 3, figsize=(14, 7), constrained_layout=True)
    for axis, field, title in zip(axes.flat, fields, titles):
        limit = max(abs(np.percentile(field, 1)), abs(np.percentile(field, 99)))
        image = axis.imshow(field, origin="lower", extent=(0, 360, -90, 90), cmap="RdBu_r", vmin=-limit, vmax=limit, aspect="auto")
        axis.set(title=title, xlabel="Longitude", ylabel="Latitude"); figure.colorbar(image, ax=axis, shrink=0.75)
    axes.flat[-1].axis("off")
    figure.suptitle(config["evaluation"]["plot_scenario_id"] + " full 145x192 fields")
    figure.savefig(output / "spatial_fields.png", dpi=150); plt.close(figure)
    labels = ["Grid RMSE", "Global"] + list(REGIONS)
    figure, axes = plt.subplots(1, 2, figsize=(15, 5), sharey=False, constrained_layout=True)
    for axis, method, color in zip(axes, ("ridge", "gpr"), ("#2a6f97", "#c45d35")):
        boxes = axis.boxplot(box_values[method], patch_artist=True, showmeans=True)
        for patch in boxes["boxes"]: patch.set_facecolor(color); patch.set_alpha(0.65)
        axis.set_xticks(range(1, len(labels) + 1), labels, rotation=55, ha="right")
        axis.set(title=method.upper(), ylabel="Absolute error / RMSE (deg C)"); axis.grid(axis="y", alpha=0.25)
    figure.savefig(output / "error_boxplots.png", dpi=150); plt.close(figure)
    print(f"evaluation={output.relative_to(ROOT)} scenarios=21 regions=10 methods=2")


if __name__ == "__main__":
    main()