"""Compute paper metrics for every scenario and draw maps and error boxes.""" import json import sys from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import yaml ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from model.climemu_s2l import GRID_SHAPE, area_weights, weighted_rmse REGIONS = { "North America": (15, 75, 190, 310), "Europe": (35, 70, 350, 45), "South Asia": (5, 35, 60, 100), "East Asia": (20, 55, 100, 150), "Arctic": (66, 90, 0, 360), "Northwest Asia": (50, 75, 45, 120), "Northern Africa": (15, 35, 340, 55), "Southern Africa": (-35, 0, 10, 55), "South America": (-60, 15, 275, 330), "Australia": (-45, -10, 110, 155), } def region_mask(latitude, longitude, bounds): south, north, west, east = bounds latitude_mask = (latitude[:, None] >= south) & (latitude[:, None] <= north) longitude_mask = ((longitude[None, :] >= west) & (longitude[None, :] <= east) if west <= east else (longitude[None, :] >= west) | (longitude[None, :] <= east)) return latitude_mask & longitude_mask def weighted_mean(field, weights, mask=None): selected = np.ones(field.shape, dtype=bool) if mask is None else mask local_weights = weights[selected] return float(np.sum(field[selected] * local_weights) / np.sum(local_weights)) def main(): config = yaml.safe_load((ROOT / "conf/config.yaml").read_text()) data = np.load(ROOT / config["paths"]["inference"]) latitude, longitude = data["latitude_deg"], data["longitude_deg"] weights = area_weights(latitude, GRID_SHAPE[1]) targets = data["long_response"] scenario_ids = [str(item) for item in data["scenario_ids"]] masks = {name: region_mask(latitude, longitude, bounds) for name, bounds in REGIONS.items()} report = {"grid": list(GRID_SHAPE), "scenario_count": 21, "regions": list(REGIONS), "methods": {}} box_values = {} for method in ("ridge", "gpr"): predictions = data[f"{method}_prediction"] scenarios = [] for index, scenario_id in enumerate(scenario_ids): target, prediction = targets[index], predictions[index] regional = {name: abs(weighted_mean(prediction, weights, mask) - weighted_mean(target, weights, mask)) for name, mask in masks.items()} scenarios.append({"scenario_id": scenario_id, "area_weighted_grid_rmse": weighted_rmse(target, prediction, weights), "global_mean_absolute_error": abs(weighted_mean(prediction, weights) - weighted_mean(target, weights)), "regional_mean_absolute_error": regional}) grid_errors = [item["area_weighted_grid_rmse"] for item in scenarios] global_errors = [item["global_mean_absolute_error"] for item in scenarios] report["methods"][method] = {"scenarios": scenarios, "summary": {"mean_grid_rmse": float(np.mean(grid_errors)), "median_grid_rmse": float(np.median(grid_errors)), "mean_global_absolute_error": float(np.mean(global_errors)), "mean_regional_absolute_error": {name: float(np.mean([item["regional_mean_absolute_error"][name] for item in scenarios])) for name in REGIONS}}} box_values[method] = [grid_errors, global_errors] + [[item["regional_mean_absolute_error"][name] for item in scenarios] for name in REGIONS] numeric = [value for method in report["methods"].values() for value in (method["summary"]["mean_grid_rmse"], method["summary"]["mean_global_absolute_error"])] if not np.isfinite(numeric).all(): raise FloatingPointError("evaluation metrics are not finite") output = ROOT / config["paths"]["evaluation_dir"] output.mkdir(parents=True, exist_ok=True) (output / "metrics.json").write_text(json.dumps(report, indent=2) + "\n") plot_index = scenario_ids.index(config["evaluation"]["plot_scenario_id"]) fields = [targets[plot_index], data["ridge_prediction"][plot_index], data["gpr_prediction"][plot_index], data["ridge_prediction"][plot_index] - targets[plot_index], data["gpr_prediction"][plot_index] - targets[plot_index]] titles = ["HadGEM3 target", "Dual Ridge", "Shared-kernel GPR", "Ridge error", "GPR error"] figure, axes = plt.subplots(2, 3, figsize=(14, 7), constrained_layout=True) for axis, field, title in zip(axes.flat, fields, titles): limit = max(abs(np.percentile(field, 1)), abs(np.percentile(field, 99))) image = axis.imshow(field, origin="lower", extent=(0, 360, -90, 90), cmap="RdBu_r", vmin=-limit, vmax=limit, aspect="auto") axis.set(title=title, xlabel="Longitude", ylabel="Latitude"); figure.colorbar(image, ax=axis, shrink=0.75) axes.flat[-1].axis("off") figure.suptitle(config["evaluation"]["plot_scenario_id"] + " full 145x192 fields") figure.savefig(output / "spatial_fields.png", dpi=150); plt.close(figure) labels = ["Grid RMSE", "Global"] + list(REGIONS) figure, axes = plt.subplots(1, 2, figsize=(15, 5), sharey=False, constrained_layout=True) for axis, method, color in zip(axes, ("ridge", "gpr"), ("#2a6f97", "#c45d35")): boxes = axis.boxplot(box_values[method], patch_artist=True, showmeans=True) for patch in boxes["boxes"]: patch.set_facecolor(color); patch.set_alpha(0.65) axis.set_xticks(range(1, len(labels) + 1), labels, rotation=55, ha="right") axis.set(title=method.upper(), ylabel="Absolute error / RMSE (deg C)"); axis.grid(axis="y", alpha=0.25) figure.savefig(output / "error_boxplots.png", dpi=150); plt.close(figure) print(f"evaluation={output.relative_to(ROOT)} scenarios=21 regions=10 methods=2") if __name__ == "__main__": main()