"""Train and evaluate the expected-greenness model exactly as locked in configs/model_v1.yaml. python scripts/30_train.py --git-commit [--tune-minutes 25] Writes runs/v1/: metrics.json, expected_oof.parquet, model files for the Space, feature importance. Nothing here looks at the test years before the final evaluation step. """ from __future__ import annotations import argparse import json import time from pathlib import Path import numpy as np import pandas as pd import xgboost as xgb from sklearn.linear_model import Ridge from sklearn.model_selection import GroupKFold CUBE = Path("/root/trendex-ml/data/cube") RUN = Path("/root/trendex-ml/runs/v1") Y0, NY = 2000, 26 YEARS = np.arange(2001, 2026) MONTHS = np.arange(5, 10) STATIC = ["elev_mean", "elev_std", "slope_mean", "steep_share", "aspect_sin", "aspect_cos", "tpi", "grass_frac", "barren_frac", "lc_type1"] WEATHER = ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev", "snow_spring", "snow_m", "soil_m", "swe_apr"] FEATURES = STATIC + ["month", "clim_ndvi"] + WEATHER ABLATION = {"precip": ["precip_m", "precip_3m", "precip_winter"], "+lst": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev"], "+snow": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev", "snow_spring", "snow_m"], "+gldas (full)": WEATHER} SEEDS = [0, 1, 2, 3, 4] FIXED = {"tree_method": "hist", "device": "cuda", "objective": "reg:squarederror"} def log(msg: str) -> None: print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) # ----------------------------------------------------------------------------- data def load(): cells = pd.read_parquet(CUBE / "cells.parquet") sel = cells[cells["in_oblast"] & cells["mask_reason"].isna()].reset_index(drop=True) cols = ["cell_id", "month", "ndvi", "ndvi_ok", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe", "year"] m = pd.read_parquet(CUBE / "monthly", columns=cols) m["year"] = m["year"].astype(int) m = m[m["cell_id"].isin(sel["cell_id"])] pos = pd.Series(np.arange(len(sel)), index=sel["cell_id"].values) ci, yi, mi = pos[m["cell_id"].values].values, m["year"].values - Y0, m["month"].values - 1 def cube(col): a = np.full((len(sel), NY, 12), np.nan, np.float32) a[ci, yi, mi] = m[col].values.astype("float32") return a arr = {k: cube(k) for k in ["ndvi", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"]} ok = np.zeros((len(sel), NY, 12), bool) ok[ci, yi, mi] = m["ndvi_ok"].values arr["ndvi"][~ok] = np.nan return sel, arr def build_features(sel: pd.DataFrame, arr: dict) -> dict: """All features as arrays shaped (cells, 25 years, 5 months). Weather anomalies vs full-period mean.""" yy, mm = YEARS - Y0, MONTHS - 1 take = lambda a, dm=0: a[:, yy][:, :, mm - dm] # noqa: E731 anom_m = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell-month anom_y = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell (annual) rep = lambda x: np.repeat(x[:, :, None], len(MONTHS), axis=2) # noqa: E731 p, ld, ln, sn, so, sw = (arr[k] for k in ["precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"]) with np.errstate(all="ignore"): raw_m = {"precip_m": take(p), "precip_3m": take(p) + take(p, 1) + take(p, 2), "lst_day_m": take(ld), "lst_night_m": take(ln), "lst_day_prev": take(ld, 1), "snow_m": take(sn), "soil_m": take(so)} raw_y = {"precip_winter": p[:, yy - 1, 9:12].sum(-1) + p[:, yy, 0:4].sum(-1), "snow_spring": np.nanmean(sn[:, yy, 2:5], axis=-1), "swe_apr": sw[:, yy, 3]} f = {k: anom_m(v) for k, v in raw_m.items()} | {k: rep(anom_y(v)) for k, v in raw_y.items()} # weather climatology for inference: raw value - this mean = the anomaly the model sees wc = [pd.DataFrame({"cell_id": sel["cell_id"].values, "month": int(mo)} | {f"{k}_mean": np.nanmean(v[:, :, j], axis=1) for k, v in raw_m.items()} | {f"{k}_mean": np.nanmean(v, axis=1) for k, v in raw_y.items()}) for j, mo in enumerate(MONTHS)] pd.concat(wc, ignore_index=True).to_parquet(RUN / "weather_climatology.parquet", index=False) shape = (len(sel), len(YEARS), len(MONTHS)) for k in STATIC: f[k] = np.broadcast_to(sel[k].values.astype("float32")[:, None, None], shape) f["month"] = np.broadcast_to(MONTHS.astype("float32")[None, None, :], shape) f["ndvi"] = take(arr["ndvi"]) f["year"] = np.broadcast_to(YEARS[None, :, None], shape) f["cell"] = np.broadcast_to(np.arange(len(sel))[:, None, None], shape) sb = (sel["block_row"].values // 3) * 1000 + sel["block_col"].values // 3 f["superblock"] = np.broadcast_to(sb[:, None, None], shape) return f def clim_fold(ndvi: np.ndarray, train_years: np.ndarray) -> np.ndarray: """Cell-month NDVI climatology from training years only; leave-own-year-out for training rows.""" tm = np.isin(YEARS, train_years) s = np.nansum(np.where(tm[None, :, None], ndvi, np.nan), axis=1, keepdims=True) n = np.sum(np.isfinite(ndvi) & tm[None, :, None], axis=1, keepdims=True).astype("float32") own = np.where(np.isfinite(ndvi) & tm[None, :, None], ndvi, 0.0) own_n = (np.isfinite(ndvi) & tm[None, :, None]).astype("float32") with np.errstate(all="ignore"): return ((s - own) / (n - own_n)).astype("float32") def table(f: dict, train_years: np.ndarray, features: list[str]) -> pd.DataFrame: clim = clim_fold(f["ndvi"], train_years) d = {k: f[k].ravel() for k in ["cell", "year", "superblock"]} for k in features: d[k] = (clim if k == "clim_ndvi" else f[k]).ravel() d["clim_ndvi"] = clim.ravel() d["target"] = (f["ndvi"] - clim).ravel() d["ndvi"] = f["ndvi"].ravel() df = pd.DataFrame(d) return df[np.isfinite(df["target"])].reset_index(drop=True) # ----------------------------------------------------------------------------- models def fit_xgb(Xtr, ytr, params, n_estimators, seed, Xva=None, yva=None): m = xgb.XGBRegressor(**FIXED, **params, n_estimators=n_estimators, random_state=seed, early_stopping_rounds=50 if Xva is not None else None) m.fit(Xtr, ytr, eval_set=[(Xva, yva)] if Xva is not None else None, verbose=False) return m class RidgeModel: def __init__(self, alpha: float): self.alpha = alpha def fit(self, X: pd.DataFrame, y): self.cols = list(X.columns) self.med = X.median() Z = X.fillna(self.med) self.mu, self.sd = Z.mean(), Z.std().replace(0, 1) self.r = Ridge(alpha=self.alpha).fit(((Z - self.mu) / self.sd).values, y) return self def predict(self, X: pd.DataFrame): Z = X[self.cols].fillna(self.med) return self.r.predict(((Z - self.mu) / self.sd).values) def to_json(self) -> dict: return {"kind": "ridge", "alpha": self.alpha, "features": self.cols, "fill": self.med.to_dict(), "mean": self.mu.to_dict(), "std": self.sd.to_dict(), "coef": self.r.coef_.tolist(), "intercept": float(self.r.intercept_)} def split(f, train_years, test_years, features): t = table(f, train_years, features) tr = t[np.isin(t["year"], train_years)].reset_index(drop=True) te = t[np.isin(t["year"], test_years)].reset_index(drop=True) return tr, te # ----------------------------------------------------------------------------- metrics def boot_skill(sse_a: np.ndarray, sse_b: np.ndarray, rng, n=2000) -> dict: """Skill 1 - SSE_a/SSE_b with a bootstrap over independent units (years or superblocks).""" point = 1 - sse_a.sum() / sse_b.sum() idx = rng.integers(0, len(sse_a), size=(n, len(sse_a))) dist = 1 - sse_a[idx].sum(1) / sse_b[idx].sum(1) return {"point": float(point), "ci_lo": float(np.quantile(dist, 0.025)), "ci_hi": float(np.quantile(dist, 0.975)), "units": int(len(sse_a))} def tercile_bacc(y_true, y_pred, cuts) -> float: t = np.digitize(y_true, cuts) p = np.digitize(y_pred, cuts) return float(np.mean([np.mean(p[t == k] == k) for k in range(3) if np.any(t == k)])) def score(te: pd.DataFrame, preds: dict, unit: str, cuts, rng) -> dict: y = te["target"].values out = {"n_rows": int(len(y))} sse = {} for name, p in preds.items(): err = y - p out[name] = {"rmse": float(np.sqrt(np.mean(err ** 2))), "mae": float(np.mean(np.abs(err))), "anomaly_corr": float(np.corrcoef(y, p)[0, 1]) if np.std(p) > 0 else None, "tercile_bacc": tercile_bacc(y, p, cuts)} sse[name] = pd.Series(err ** 2).groupby(te[unit].values).sum().values out["skill_xgb_vs_ridge"] = boot_skill(sse["xgb"], sse["ridge"], rng) out["skill_xgb_vs_clim"] = boot_skill(sse["xgb"], sse["clim"], rng) out["skill_ridge_vs_clim"] = boot_skill(sse["ridge"], sse["clim"], rng) return out # ----------------------------------------------------------------------------- main def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--git-commit", required=True) ap.add_argument("--tune-minutes", type=float, default=25) ap.add_argument("--tune-trials", type=int, default=40) a = ap.parse_args() RUN.mkdir(parents=True, exist_ok=True) rng = np.random.default_rng(0) metrics: dict = {"protocol": "configs/model_v1.yaml", "git_commit": a.git_commit, "features": FEATURES} sel, arr = load() f = build_features(sel, arr) log(f"cells {len(sel)}, feature arrays {f['ndvi'].shape}") metrics["data"] = {"cells": len(sel), "years": [int(YEARS[0]), int(YEARS[-1])], "months": MONTHS.tolist()} # --- 1. tuning on the inner split only (2001-2015 -> 2016-2019) inner_tr, inner_va = split(f, np.arange(2001, 2016), np.arange(2016, 2020), FEATURES) Xi, yi_, Xv, yv = inner_tr[FEATURES], inner_tr["target"], inner_va[FEATURES], inner_va["target"] alphas = [0.1, 1, 10, 100, 1000] ridge_rmse = {al: float(np.sqrt(np.mean((yv - RidgeModel(al).fit(Xi, yi_).predict(Xv)) ** 2))) for al in alphas} alpha = min(ridge_rmse, key=ridge_rmse.get) log(f"ridge alpha {alpha} inner rmse {ridge_rmse[alpha]:.4f}") import optuna optuna.logging.set_verbosity(optuna.logging.WARNING) def objective(trial): params = {"max_depth": trial.suggest_int("max_depth", 4, 10), "learning_rate": trial.suggest_float("learning_rate", 0.02, 0.2, log=True), "min_child_weight": trial.suggest_float("min_child_weight", 1, 200, log=True), "subsample": trial.suggest_float("subsample", 0.5, 1.0), "colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0), "reg_lambda": trial.suggest_float("reg_lambda", 1e-3, 30, log=True), "reg_alpha": trial.suggest_float("reg_alpha", 1e-3, 10, log=True)} m = fit_xgb(Xi, yi_, params, 3000, 0, Xv, yv) trial.set_user_attr("n_estimators", int(m.best_iteration) + 1) return float(np.sqrt(np.mean((yv - m.predict(Xv)) ** 2))) study = optuna.create_study(direction="minimize", sampler=optuna.samplers.TPESampler(seed=0)) study.optimize(objective, n_trials=a.tune_trials, timeout=a.tune_minutes * 60) best, n_est = study.best_params, study.best_trial.user_attrs["n_estimators"] log(f"tuning: {len(study.trials)} trials, best inner rmse {study.best_value:.4f}, n_estimators {n_est}") metrics["tuning"] = {"trials": len(study.trials), "best_params": best, "n_estimators": n_est, "inner_rmse_xgb": study.best_value, "ridge_alpha": alpha, "inner_rmse_ridge": ridge_rmse} cuts = np.quantile(yi_, [1 / 3, 2 / 3]) # conformal residual quantiles per month from the inner split (model never saw 2016-2019) m_inner = fit_xgb(Xi, yi_, best, n_est, 0) res_inner = yv.values - m_inner.predict(Xv) conf_inner = {int(mo): [float(np.quantile(res_inner[inner_va["month"] == mo], q)) for q in (0.1, 0.9)] for mo in MONTHS} # --- 2. temporal test (train 2001-2019, test 2020-2025), 5 seeds tr, te = split(f, np.arange(2001, 2020), np.arange(2020, 2026), FEATURES) ridge = RidgeModel(alpha).fit(tr[FEATURES], tr["target"]) p_ridge = ridge.predict(te[FEATURES]) seed_scores, p_xgb_seeds = [], [] for s in SEEDS: m = fit_xgb(tr[FEATURES], tr["target"], best, n_est, s) p = m.predict(te[FEATURES]) p_xgb_seeds.append(p) seed_scores.append(float(1 - np.mean((te["target"] - p) ** 2) / np.mean((te["target"] - p_ridge) ** 2))) p_xgb = np.mean(p_xgb_seeds, axis=0) temporal = score(te, {"xgb": p_xgb, "ridge": p_ridge, "clim": np.zeros(len(te))}, "year", cuts, rng) temporal["seed_skill_vs_ridge"] = {"values": seed_scores, "mean": float(np.mean(seed_scores)), "sd": float(np.std(seed_scores))} lo, hi = np.array([conf_inner[int(mo)] for mo in te["month"]]).T res = te["target"].values - p_xgb temporal["conformal_coverage_p10_p90"] = float(np.mean((res >= lo) & (res <= hi))) metrics["temporal_test"] = temporal log(f"temporal: skill vs ridge {temporal['skill_xgb_vs_ridge']}") # ablation on the temporal split (one seed) abl = {} for name, wcols in ABLATION.items(): cols = STATIC + ["month", "clim_ndvi"] + wcols m = fit_xgb(tr[cols], tr["target"], best, n_est, 0) abl[name] = float(1 - np.mean((te["target"] - m.predict(te[cols])) ** 2) / np.mean(te["target"] ** 2)) metrics["ablation_skill_vs_clim_temporal"] = abl log(f"ablation {abl}") # --- 3. spatial test: GroupKFold over 33x33-cell superblocks, all years full = table(f, YEARS, FEATURES) preds = {"xgb": np.zeros(len(full)), "ridge": np.zeros(len(full))} for k, (itr, ite) in enumerate(GroupKFold(5).split(full, groups=full["superblock"])): Xtr, ytr, Xte = full.loc[itr, FEATURES], full.loc[itr, "target"], full.loc[ite, FEATURES] preds["xgb"][ite] = fit_xgb(Xtr, ytr, best, n_est, 0).predict(Xte) preds["ridge"][ite] = RidgeModel(alpha).fit(Xtr, ytr).predict(Xte) preds["clim"] = np.zeros(len(full)) spatial = score(full, preds, "superblock", cuts, rng) metrics["spatial_test"] = spatial log(f"spatial: skill vs ridge {spatial['skill_xgb_vs_ridge']}") # --- 4. gate (pre-registered) passed = temporal["skill_xgb_vs_ridge"]["ci_lo"] > 0 and spatial["skill_xgb_vs_ridge"]["ci_lo"] > 0 chosen = "xgb" if passed else "ridge" metrics["gate"] = {"rule": "CI lower bound of skill_xgb_vs_ridge > 0 on temporal AND spatial tests", "passed": bool(passed), "shipped_model": chosen} log(f"GATE passed={passed} -> shipping {chosen}") # --- 5. cross-fitted out-of-fold predictions (5 contiguous year blocks) with the chosen model blocks = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)] oof = [] for blk in blocks: trn = np.setdiff1d(YEARS, blk) trb, teb = split(f, trn, blk, FEATURES) model = fit_xgb(trb[FEATURES], trb["target"], best, n_est, 0) if chosen == "xgb" else \ RidgeModel(alpha).fit(trb[FEATURES], trb["target"]) teb = teb.assign(pred_anom=model.predict(teb[FEATURES])) oof.append(teb[["cell", "year", "month", "ndvi", "clim_ndvi", "target", "pred_anom"]]) oof = pd.concat(oof, ignore_index=True) res_all = oof["target"] - oof["pred_anom"] conf_prod = {int(mo): [float(np.quantile(res_all[oof["month"] == mo], q)) for q in (0.1, 0.9)] for mo in MONTHS} lo = oof["month"].map(lambda mo: conf_prod[int(mo)][0]) hi = oof["month"].map(lambda mo: conf_prod[int(mo)][1]) oof["cell_id"] = sel["cell_id"].values[oof["cell"].values] oof["expected"] = oof["clim_ndvi"] + oof["pred_anom"] oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi oof["residual"] = res_all oof[["cell_id", "year", "month", "ndvi", "expected", "p10", "p90", "residual"]].rename( columns={"ndvi": "observed"}).to_parquet(RUN / "expected_oof.parquet", index=False) metrics["conformal_p10_p90_by_month"] = conf_prod log(f"oof rows {len(oof)}") # --- 6. production model: refit on all years; climatology tables for inference full_all = full if chosen == "xgb": prod = fit_xgb(full_all[FEATURES], full_all["target"], best, n_est, 0) prod.save_model(RUN / "model_a.ubj") contrib = prod.get_booster().predict(xgb.DMatrix(full_all[FEATURES].sample(200_000, random_state=0)), pred_contribs=True) imp = pd.Series(np.abs(contrib[:, :-1]).mean(0), index=FEATURES).sort_values(ascending=False) else: prod = RidgeModel(alpha).fit(full_all[FEATURES], full_all["target"]) (RUN / "ridge_a.json").write_text(json.dumps(prod.to_json())) imp = pd.Series(np.abs(prod.r.coef_), index=FEATURES).sort_values(ascending=False) metrics["feature_importance_mean_abs"] = imp.round(6).to_dict() yy = YEARS - Y0 clim_tab = [] for k, mo in enumerate(MONTHS): rec = {"cell_id": sel["cell_id"].values, "month": int(mo), "clim_ndvi": np.nanmean(arr["ndvi"][:, yy, mo - 1], axis=1)} clim_tab.append(pd.DataFrame(rec)) pd.concat(clim_tab).to_parquet(RUN / "climatology.parquet", index=False) (RUN / "feature_schema.json").write_text(json.dumps({ "features": FEATURES, "target": "ndvi_anomaly = ndvi - clim_ndvi", "months": MONTHS.tolist(), "weather_anomaly_baseline": "cell-month mean 2001-2025 (see climatology.parquet / weather_climatology.parquet)", "shipped_model": chosen}, indent=2)) (RUN / "conformal.json").write_text(json.dumps(conf_prod, indent=2)) (RUN / "metrics.json").write_text(json.dumps(metrics, indent=2, default=float)) log("TRAIN_DONE") if __name__ == "__main__": main()