Download training_code/scripts/45_package.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 7.51 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/45_package.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/45_package.py
-
curl -L -o 45_package.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/45_package.py
7.51 kB
| """Assemble the HF model repo folder runs/v1/hf_model: model files, predict.py, golden test, model card. | |
| Golden test: 100 fixed feature rows + the outputs of predict.py; the serving side must reproduce them. | |
| Also checks that predict.py (plain Booster, CPU) gives the same numbers as the training wrapper. | |
| """ | |
| from __future__ import annotations | |
| import importlib.util | |
| import json | |
| import shutil | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| import xgboost as xgb | |
| ROOT = Path("/root/trendex-ml") | |
| RUN = ROOT / "runs" / "v1" | |
| HF = RUN / "hf_model" | |
| def load_module(path: Path, name: str): | |
| spec = importlib.util.spec_from_file_location(name, path) | |
| mod = importlib.util.module_from_spec(spec) | |
| sys.modules[name] = mod | |
| spec.loader.exec_module(mod) | |
| return mod | |
| def main() -> None: | |
| HF.mkdir(parents=True, exist_ok=True) | |
| for f in ["model_a.ubj", "feature_schema.json", "climatology.parquet", "weather_climatology.parquet", | |
| "conformal.json", "metrics.json"]: | |
| shutil.copy(RUN / f, HF / f) | |
| shutil.copy(ROOT / "src" / "trendex_ml" / "predict.py", HF / "predict.py") | |
| shutil.copy(ROOT / "configs" / "model_v1.yaml", HF / "model_v1.yaml") | |
| # golden rows rebuilt with the training code itself | |
| tr = load_module(ROOT / "scripts" / "30_train.py", "train_mod") | |
| sel, arr = tr.load() | |
| f = tr.build_features(sel, arr) | |
| t = tr.table(f, tr.YEARS, tr.FEATURES).sample(100, random_state=0).reset_index(drop=True) | |
| t.insert(0, "cell_id", sel["cell_id"].values[t["cell"].values]) | |
| golden_in = t[["cell_id", "year"] + tr.FEATURES] | |
| (HF / "golden").mkdir(exist_ok=True) | |
| golden_in.to_parquet(HF / "golden" / "inputs.parquet", index=False) | |
| pm = load_module(HF / "predict.py", "predict_mod") | |
| model = pm.load(HF) | |
| out = model.predict(golden_in) | |
| out.to_parquet(HF / "golden" / "expected.parquet", index=False) | |
| ref = xgb.XGBRegressor() | |
| ref.load_model(RUN / "model_a.ubj") | |
| max_diff = float(np.max(np.abs(ref.predict(golden_in[tr.FEATURES]) - out["pred_anom"].values))) | |
| print(f"golden: 100 rows, max |predict.py - training wrapper| = {max_diff:.2e}") | |
| assert max_diff < 1e-5, "predict.py disagrees with the training model" | |
| (HF / "test_golden.py").write_text('''"""pytest test_golden.py - the serving code must reproduce these outputs exactly.""" | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| from predict import load | |
| HERE = Path(__file__).parent | |
| def test_golden(): | |
| m = load(HERE) | |
| got = m.predict(pd.read_parquet(HERE / "golden" / "inputs.parquet")) | |
| want = pd.read_parquet(HERE / "golden" / "expected.parquet") | |
| assert np.allclose(got.values, want.values, atol=1e-6) | |
| ''') | |
| write_card() | |
| print("PACKAGE_DONE", sorted(p.name for p in HF.iterdir())) | |
| def write_card() -> None: | |
| m = json.loads((RUN / "metrics.json").read_text()) | |
| t, s = m["temporal_test"], m["spatial_test"] | |
| fmt = lambda d: f"{d['point']:.3f} (95% CI {d['ci_lo']:.3f} to {d['ci_hi']:.3f}, {d['units']} units)" # noqa: E731 | |
| row = lambda name, d: f"| {name} | {d['xgb']['rmse']:.4f} | {d['ridge']['rmse']:.4f} | {d['clim']['rmse']:.4f} | " \ | |
| f"{d['xgb']['anomaly_corr']:.3f} | {d['ridge']['anomaly_corr']:.3f} | {d['xgb']['tercile_bacc']:.3f} |" # noqa: E731 | |
| abl = " -> ".join(f"{k}: {v:.3f}" for k, v in m["ablation_skill_vs_clim_temporal"].items()) | |
| imp = ", ".join(list(m["feature_importance_mean_abs"].keys())[:6]) | |
| card = f"""--- | |
| license: other | |
| library_name: xgboost | |
| tags: [nasa, modis, ndvi, imerg, gldas, pastures, kyrgyzstan, tabular-regression] | |
| datasets: [TrendeX90/naryn-cube] | |
| --- | |
| # TrendeX expected-greenness model (v1) | |
| Predicts the **summer greenness (NDVI) anomaly** of every 1 km pasture cell in Naryn oblast, Kyrgyzstan | |
| (May-September, 2001-2025) **from weather and terrain alone**. The gap between observed and expected NDVI is | |
| change **not explained by precipitation, temperature, snow and soil moisture**. It is not a cause and says | |
| nothing about forage quality. Built for NASA Space Apps 2026 ("Be An Earth System Trend Detective!"). | |
| ## Inputs | |
| - Weather anomalies vs each cell's 2001-2025 mean for the same month: GPM IMERG precipitation (month, 3-month, | |
| Oct-Apr), MODIS LST day/night (month, previous month), MODIS snow (Mar-May, month), GLDAS soil moisture | |
| (month) and April snow water. | |
| - Terrain from NASADEM (elevation, spread, slope, steep share, aspect, TPI) and land cover (MCD12Q1 2020). | |
| - The cell's own NDVI climatology for that month. **Not used on purpose:** year, coordinates, past NDVI. | |
| ## How it was tested (protocol locked in git before the first fit: `model_v1.yaml`, commit {m['git_commit']}) | |
| - Tuning only on 2001-2015 -> 2016-2019. The 2020-2025 years were used once, for the final test. | |
| - Spatial test: 5-fold GroupKFold over ~30 x 30 km super-blocks (all years; easier than the temporal test | |
| because the same years' weather appears in training). | |
| - Baselines: climatology (anomaly = 0) and ridge regression on the same features. | |
| - Pre-registered gate: ship XGBoost only if the 95% CI lower bound of its skill over ridge is > 0 on both tests. | |
| **Gate passed: {m['gate']['passed']}. Shipped: {m['gate']['shipped_model']}.** | |
| | Test | RMSE xgb | RMSE ridge | RMSE climatology | anomaly corr xgb | anomaly corr ridge | tercile bal. acc. xgb | | |
| |---|---|---|---|---|---|---| | |
| {row("Temporal (2020-2025)", t)} | |
| {row("Spatial (unseen areas)", s)} | |
| - Skill vs ridge, temporal: {fmt(t['skill_xgb_vs_ridge'])}. Five seeds: {t['seed_skill_vs_ridge']['mean']:.3f} +/- {t['seed_skill_vs_ridge']['sd']:.3f}. | |
| - Skill vs ridge, spatial: {fmt(s['skill_xgb_vs_ridge'])}. | |
| - Skill vs climatology, temporal: {fmt(t['skill_xgb_vs_clim'])}. | |
| - What each NASA dataset adds (skill vs climatology, temporal test): {abl}. | |
| - Most used inputs: {imp}. | |
| ## Limits (read before using) | |
| - The model explains about a fifth of the year-to-year NDVI anomaly on unseen years; most variation is not weather-explained. | |
| - P10-P90 band: on 2020-2025 it covered **{t['conformal_coverage_p10_p90']:.0%} instead of 80%**, so treat it as too narrow. | |
| - Weather comes from coarse grids (IMERG 0.1 deg, GLDAS 0.25 deg) interpolated to 1 km. | |
| - NDVI is greenness, not forage: shrub encroachment can look like improvement. | |
| - Land cover is a fixed 2020 map; MODIS Terra drifted in recent years (see Aqua-Terra check in the dataset products). | |
| - What-if runs: drier and hotter-than-normal conditions lower expected July NDVI, but wetter-than-normal | |
| precipitation does not raise it (slightly lowers it on average). Treat what-if outputs as model | |
| sensitivities, not causal effects. | |
| - No field validation. Relative to Naryn pastures only. | |
| ## Use | |
| ```python | |
| from predict import load | |
| m = load(".") # needs only numpy, pandas, xgboost (CPU) | |
| out = m.predict(features_df) # columns as in feature_schema.json | |
| scen = m.what_if(cells, cell_ids, month=7, precip_pct=-30) # cells = naryn-cube cells.parquet | |
| ``` | |
| `pytest test_golden.py` reproduces the 100 golden rows. | |
| ## Data and tools | |
| MOD13A3.061, MYD13A3.061, MOD11A2.061, MOD10A2.061, MCD12Q1.061, NASADEM_HGT.001 (LP DAAC / NSIDC via earthaccess), | |
| GPM_3IMERGM.07 and GLDAS_NOAH025_M.2.1 (GES DISC). XGBoost, Optuna, scikit-learn, pymannkendall. | |
| AI use: implementation was assisted by Claude (Anthropic); design decisions and the product idea are the team's. | |
| """ | |
| (HF / "README.md").write_text(card) | |
| if __name__ == "__main__": | |
| main() | |