File size: 5,979 Bytes
87f2bd3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | """Compute paper metrics for every scenario and draw maps and error boxes."""
import json
import sys
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import yaml
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from model.climemu_s2l import GRID_SHAPE, area_weights, weighted_rmse
REGIONS = {
"North America": (15, 75, 190, 310), "Europe": (35, 70, 350, 45),
"South Asia": (5, 35, 60, 100), "East Asia": (20, 55, 100, 150),
"Arctic": (66, 90, 0, 360), "Northwest Asia": (50, 75, 45, 120),
"Northern Africa": (15, 35, 340, 55), "Southern Africa": (-35, 0, 10, 55),
"South America": (-60, 15, 275, 330), "Australia": (-45, -10, 110, 155),
}
def region_mask(latitude, longitude, bounds):
south, north, west, east = bounds
latitude_mask = (latitude[:, None] >= south) & (latitude[:, None] <= north)
longitude_mask = ((longitude[None, :] >= west) & (longitude[None, :] <= east)
if west <= east else (longitude[None, :] >= west) | (longitude[None, :] <= east))
return latitude_mask & longitude_mask
def weighted_mean(field, weights, mask=None):
selected = np.ones(field.shape, dtype=bool) if mask is None else mask
local_weights = weights[selected]
return float(np.sum(field[selected] * local_weights) / np.sum(local_weights))
def main():
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text())
data = np.load(ROOT / config["paths"]["inference"])
latitude, longitude = data["latitude_deg"], data["longitude_deg"]
weights = area_weights(latitude, GRID_SHAPE[1])
targets = data["long_response"]
scenario_ids = [str(item) for item in data["scenario_ids"]]
masks = {name: region_mask(latitude, longitude, bounds) for name, bounds in REGIONS.items()}
report = {"grid": list(GRID_SHAPE), "scenario_count": 21, "regions": list(REGIONS), "methods": {}}
box_values = {}
for method in ("ridge", "gpr"):
predictions = data[f"{method}_prediction"]
scenarios = []
for index, scenario_id in enumerate(scenario_ids):
target, prediction = targets[index], predictions[index]
regional = {name: abs(weighted_mean(prediction, weights, mask) - weighted_mean(target, weights, mask))
for name, mask in masks.items()}
scenarios.append({"scenario_id": scenario_id,
"area_weighted_grid_rmse": weighted_rmse(target, prediction, weights),
"global_mean_absolute_error": abs(weighted_mean(prediction, weights) - weighted_mean(target, weights)),
"regional_mean_absolute_error": regional})
grid_errors = [item["area_weighted_grid_rmse"] for item in scenarios]
global_errors = [item["global_mean_absolute_error"] for item in scenarios]
report["methods"][method] = {"scenarios": scenarios,
"summary": {"mean_grid_rmse": float(np.mean(grid_errors)), "median_grid_rmse": float(np.median(grid_errors)),
"mean_global_absolute_error": float(np.mean(global_errors)),
"mean_regional_absolute_error": {name: float(np.mean([item["regional_mean_absolute_error"][name]
for item in scenarios])) for name in REGIONS}}}
box_values[method] = [grid_errors, global_errors] + [[item["regional_mean_absolute_error"][name] for item in scenarios]
for name in REGIONS]
numeric = [value for method in report["methods"].values() for value in
(method["summary"]["mean_grid_rmse"], method["summary"]["mean_global_absolute_error"])]
if not np.isfinite(numeric).all():
raise FloatingPointError("evaluation metrics are not finite")
output = ROOT / config["paths"]["evaluation_dir"]
output.mkdir(parents=True, exist_ok=True)
(output / "metrics.json").write_text(json.dumps(report, indent=2) + "\n")
plot_index = scenario_ids.index(config["evaluation"]["plot_scenario_id"])
fields = [targets[plot_index], data["ridge_prediction"][plot_index], data["gpr_prediction"][plot_index],
data["ridge_prediction"][plot_index] - targets[plot_index], data["gpr_prediction"][plot_index] - targets[plot_index]]
titles = ["HadGEM3 target", "Dual Ridge", "Shared-kernel GPR", "Ridge error", "GPR error"]
figure, axes = plt.subplots(2, 3, figsize=(14, 7), constrained_layout=True)
for axis, field, title in zip(axes.flat, fields, titles):
limit = max(abs(np.percentile(field, 1)), abs(np.percentile(field, 99)))
image = axis.imshow(field, origin="lower", extent=(0, 360, -90, 90), cmap="RdBu_r", vmin=-limit, vmax=limit, aspect="auto")
axis.set(title=title, xlabel="Longitude", ylabel="Latitude"); figure.colorbar(image, ax=axis, shrink=0.75)
axes.flat[-1].axis("off")
figure.suptitle(config["evaluation"]["plot_scenario_id"] + " full 145x192 fields")
figure.savefig(output / "spatial_fields.png", dpi=150); plt.close(figure)
labels = ["Grid RMSE", "Global"] + list(REGIONS)
figure, axes = plt.subplots(1, 2, figsize=(15, 5), sharey=False, constrained_layout=True)
for axis, method, color in zip(axes, ("ridge", "gpr"), ("#2a6f97", "#c45d35")):
boxes = axis.boxplot(box_values[method], patch_artist=True, showmeans=True)
for patch in boxes["boxes"]: patch.set_facecolor(color); patch.set_alpha(0.65)
axis.set_xticks(range(1, len(labels) + 1), labels, rotation=55, ha="right")
axis.set(title=method.upper(), ylabel="Absolute error / RMSE (deg C)"); axis.grid(axis="y", alpha=0.25)
figure.savefig(output / "error_boxplots.png", dpi=150); plt.close(figure)
print(f"evaluation={output.relative_to(ROOT)} scenarios=21 regions=10 methods=2")
if __name__ == "__main__":
main()
|