File size: 8,805 Bytes
d7c810d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | """Products built from the out-of-fold predictions (runs/v1 -> runs/v1/products).
Residuals for trends use ONE reference per cell-month (the all-years climatology), not the per-block
training climatology used for evaluation. With per-block climatologies a real trend would be inflated
(each held-out block's reference comes from the other years). pred_anom (the weather effect) still comes
from models that never saw that block, so no target information leaks into it.
Outputs (cell level, plus block and district aggregates):
expected_oof.parquet cell_id, year, month, observed, expected, p10, p90, residual
residual_trend*.parquet Sen slope / CI / Mann-Kendall (Hamed-Rao) / BH q / class of summer residuals
season_vs_expected*.parquet summer mean residual per year -> below / within / above usual for its weather
aqua_terra_blocks.parquet per-block summer NDVI trend from Terra and Aqua (2003-2025)
products_summary.json headline shares, placebo, Aqua-Terra agreement
"""
from __future__ import annotations
import json
import sys
import time
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, "/root/trendex-ml/src")
from trendex_ml.stats.trend import bh_fdr, classify, mk_vectorized, trend_many # noqa: E402
CUBE = Path("/root/trendex-ml/data/cube")
RUN = Path("/root/trendex-ml/runs/v1")
OUT = RUN / "products"
YEARS = np.arange(2001, 2026)
BLOCKS = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)]
SUMMER = [6, 7, 8]
ALPHA, MIN_YEARS, N_PLACEBO = 0.05, 18, 200
def log(msg):
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
def trend_table(wide: pd.DataFrame, key: str) -> pd.DataFrame:
r = trend_many(wide.values, wide.columns.values.astype(float), method="hamed_rao", min_n=MIN_YEARS)
df = pd.DataFrame(r)
df.insert(0, key, wide.index.values)
df["q"] = bh_fdr(df["p"].values)
df["class"] = classify(df["slope"].values, df["q"].values, df["n"].values, ALPHA, MIN_YEARS)
df["slope_per_decade"] = df["slope"] * 10
return df
def main() -> None:
OUT.mkdir(parents=True, exist_ok=True)
cells = pd.read_parquet(CUBE / "cells.parquet")
oof = pd.read_parquet(RUN / "expected_oof.parquet")
conf = {int(k): v for k, v in json.loads((RUN / "conformal.json").read_text()).items()}
# recover pred_anom = expected - clim_fold (clim of the other 20 years), then re-reference to clim_all
ndvi = pd.read_parquet(CUBE / "monthly", columns=["cell_id", "month", "ndvi", "ndvi_ok", "ndvi_aqua", "year"])
ndvi["year"] = ndvi["year"].astype(int)
ndvi = ndvi[ndvi["cell_id"].isin(oof["cell_id"].unique()) & ndvi["year"].between(2001, 2025)]
ok = ndvi[ndvi["ndvi_ok"] & ndvi["month"].between(5, 9)]
clim_all = ok.groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_all")
parts = []
for blk in BLOCKS:
cf = ok[~ok["year"].isin(blk)].groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_fold").reset_index()
sub = oof[oof["year"].isin(blk)].merge(cf, on=["cell_id", "month"], how="left")
parts.append(sub)
oof = pd.concat(parts, ignore_index=True).merge(clim_all.reset_index(), on=["cell_id", "month"], how="left")
pred_anom = oof["expected"] - oof["clim_fold"]
oof["expected"] = oof["clim_all"] + pred_anom
lo = oof["month"].map(lambda m: conf[int(m)][0])
hi = oof["month"].map(lambda m: conf[int(m)][1])
oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi
oof["residual"] = oof["observed"] - oof["expected"]
oof = oof[["cell_id", "year", "month", "observed", "expected", "p10", "p90", "residual"]]
oof.to_parquet(OUT / "expected_oof.parquet", index=False)
log(f"expected_oof: {len(oof)} rows")
# summer residual series
summer = oof[oof["month"].isin(SUMMER)].groupby(["cell_id", "year"])["residual"].mean().rename("res").reset_index()
summer = summer.merge(cells[["cell_id", "block_id", "district"]], on="cell_id")
wide = summer.pivot(index="cell_id", columns="year", values="res").reindex(columns=YEARS)
# residual trends: cell, block, district
rt = trend_table(wide, "cell_id")
rt.to_parquet(OUT / "residual_trend.parquet", index=False)
blk_w = summer.groupby(["block_id", "year"])["res"].mean().unstack().reindex(columns=YEARS)
trend_table(blk_w, "block_id").to_parquet(OUT / "residual_trend_block.parquet", index=False)
dis_w = summer.groupby(["district", "year"])["res"].mean().unstack().reindex(columns=YEARS)
rtd = trend_table(dis_w, "district")
rtd.to_parquet(OUT / "residual_trend_district.parquet", index=False)
shares = rt["class"].value_counts(normalize=True).round(4).to_dict()
log(f"residual trend classes (cells): {shares}")
# placebo: shuffle years within each cell; how many cells would be flagged by chance?
rng = np.random.default_rng(0)
W = wide.values.astype("float32")
flagged_raw, flagged_q = [], []
for _ in range(N_PLACEBO):
perm = np.argsort(rng.random(W.shape), axis=1)
Wp = np.take_along_axis(W, perm, axis=1)
_, _, _, p = mk_vectorized(Wp)
flagged_raw.append(float(np.nanmean(p < ALPHA)))
flagged_q.append(float(np.nanmean(bh_fdr(p) < ALPHA)))
_, _, _, p_obs = mk_vectorized(W)
placebo = {"permutations": N_PLACEBO, "test": "original MK (mk_vectorized), same for observed and placebo",
"observed_share_p<0.05": float(np.nanmean(p_obs < ALPHA)),
"observed_share_q<0.05": float(np.nanmean(bh_fdr(p_obs) < ALPHA)),
"placebo_share_p<0.05_mean": float(np.mean(flagged_raw)),
"placebo_share_p<0.05_p95": float(np.quantile(flagged_raw, 0.95)),
"placebo_share_q<0.05_mean": float(np.mean(flagged_q)),
"placebo_share_q<0.05_p95": float(np.quantile(flagged_q, 0.95))}
log(f"placebo {placebo}")
# season vs expected: summer mean residual vs the empirical P10/P90 of all cell-years
q10, q90 = summer["res"].quantile([0.1, 0.9])
summer["season_class"] = np.select([summer["res"] < q10, summer["res"] > q90], ["below", "above"], "within")
summer.rename(columns={"res": "summer_residual"}).to_parquet(OUT / "season_vs_expected.parquet", index=False)
sb = summer.groupby(["block_id", "year"])["res"].mean().rename("summer_residual").reset_index()
bq10, bq90 = sb["summer_residual"].quantile([0.1, 0.9])
sb["season_class"] = np.select([sb["summer_residual"] < bq10, sb["summer_residual"] > bq90], ["below", "above"], "within")
sb.to_parquet(OUT / "season_vs_expected_block.parquet", index=False)
# Aqua vs Terra: per-block summer NDVI trend 2003-2025, both satellites (raw NDVI, not residuals)
s = ndvi[ndvi["month"].isin(SUMMER) & ndvi["year"].between(2003, 2025)].merge(cells[["cell_id", "block_id"]], on="cell_id")
s.loc[~s["ndvi_ok"], "ndvi"] = np.nan
yrs = np.arange(2003, 2026)
terra = s.groupby(["block_id", "year"])["ndvi"].mean().unstack().reindex(columns=yrs)
aqua = s.groupby(["block_id", "year"])["ndvi_aqua"].mean().unstack().reindex(columns=yrs).reindex(terra.index)
tt, ta = trend_table(terra, "block_id"), trend_table(aqua, "block_id")
at = tt[["block_id", "slope", "q", "class"]].merge(ta[["block_id", "slope", "q", "class"]], on="block_id",
suffixes=("_terra", "_aqua"))
at.to_parquet(OUT / "aqua_terra_blocks.parquet", index=False)
sig_t = at["class_terra"].isin(["growth", "decline"])
agreement = {"blocks": int(len(at)),
"slope_sign_agreement": float(np.mean(np.sign(at["slope_terra"]) == np.sign(at["slope_aqua"]))),
"terra_significant_blocks": int(sig_t.sum()),
"same_class_where_terra_significant": float(np.mean(at.loc[sig_t, "class_terra"] == at.loc[sig_t, "class_aqua"])) if sig_t.any() else None,
"terra_class_shares": at["class_terra"].value_counts(normalize=True).round(4).to_dict(),
"aqua_class_shares": at["class_aqua"].value_counts(normalize=True).round(4).to_dict()}
log(f"aqua vs terra {agreement}")
summary = {"residual_trend_cell_class_shares": shares,
"residual_trend_district": rtd[["district", "slope_per_decade", "ci_lo", "ci_hi", "p", "q", "class"]]
.round(5).to_dict(orient="records"),
"season_band_summer_residual_q10_q90": [float(q10), float(q90)],
"placebo": placebo, "aqua_vs_terra_blocks": agreement,
"wording": "change not explained by precipitation, temperature, snow and soil moisture (not a cause)"}
(OUT / "products_summary.json").write_text(json.dumps(summary, indent=2, default=float))
log("PRODUCTS_DONE")
if __name__ == "__main__":
main()
|