"""Push the cube (and later products/model) to Hugging Face. python scripts/50_push_hf.py cube # data/cube + data/aoi -> TrendeX90/naryn-cube (private dataset) Uses the token in ~/.cache/huggingface/token (a fine-grained token with write access to TrendeX90 only). """ from __future__ import annotations import argparse import json from pathlib import Path from huggingface_hub import HfApi ROOT = Path("/root/trendex-ml/data") CUBE_REPO = "TrendeX90/naryn-cube" MODEL_REPO = "TrendeX90/expected-greenness" RUN = Path("/root/trendex-ml/runs/v1") CARD = """--- license: other pretty_name: TrendeX Naryn pasture cube tags: [nasa, modis, ndvi, imerg, gldas, kyrgyzstan, pastures] --- # TrendeX Naryn cube ({status}) Shared data cube for TrendeX (NASA Space Apps 2026, "Be An Earth System Trend Detective!"): Naryn oblast, Kyrgyzstan, plus a 20 km buffer and two known-place boxes (Suusamyr valley, Ak-Shyirak). **synthetic = false** - every value comes from NASA data listed below. ## Grid Global MODIS sinusoidal 1 km grid (`+proj=sinu +R=6371007.181`), pixel 926.625 m, equal-area. `cell_id = global_row * 43200 + global_col`. Blocks: 11 x 11 cells (`block_id` = "row-col", ints in `block_row`, `block_col`). ## Files - `cells.parquet` - one row per cell: position, district (5 official districts; null outside), terrain (NASADEM), land cover (MCD12Q1 2020), `mask_reason` (water / ice / urban / too_steep; null = pasture candidate). - `monthly/year=YYYY/part.parquet` - one row per cell x month, 2000-02..2025-12 (`pd.read_parquet("monthly/")`). - `aoi/` - oblast, district and request polygons (GeoJSON, EPSG:4326). - `manifest.json` - units, dataset versions and DOIs, sources, notes. `qa_report.json` - sanity checks. ## Units {units} ## Data sources {datasets} Downloaded with earthaccess from LP DAAC (MODIS, NASADEM) and GES DISC (IMERG, GLDAS). ## Notes {notes} """ def push_cube(api: HfApi, message: str) -> None: manifest = json.loads((ROOT / "cube" / "manifest.json").read_text()) status = "PARTIAL - NDVI only" if manifest.get("partial") else "full" card = CARD.format( status=status, units="\n".join(f"- `{k}`: {v}" for k, v in manifest.get("units", {}).items()), datasets="\n".join(f"- {k} v{v}" for k, v in manifest.get("datasets", {}).items()), notes="\n".join(f"- {n}" for n in manifest.get("notes", []))) (ROOT / "cube" / "README.md").write_text(card) if not api.repo_exists(CUBE_REPO, repo_type="dataset"): # tokens without org create rights still work api.create_repo(CUBE_REPO, repo_type="dataset", private=True) api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "cube"), commit_message=message or f"cube ({status})") api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "aoi"), path_in_repo="aoi", allow_patterns=["naryn_oblast.geojson", "naryn_districts.geojson", "aoi_request.geojson", "aoi_summary.json"], commit_message="area of interest polygons") print(f"PUSHED {CUBE_REPO} ({status})", flush=True) def push_products(api: HfApi, message: str) -> None: """runs/v1/products + metrics.json -> naryn-cube/products/ (what the backend reads).""" api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(RUN / "products"), path_in_repo="products", commit_message=message or "model products v1") api.upload_file(path_or_fileobj=str(RUN / "metrics.json"), path_in_repo="products/metrics.json", repo_id=CUBE_REPO, repo_type="dataset", commit_message="model metrics v1") print(f"PUSHED {CUBE_REPO}/products", flush=True) def push_model(api: HfApi, message: str) -> None: if not api.repo_exists(MODEL_REPO): api.create_repo(MODEL_REPO, private=True) api.upload_folder(repo_id=MODEL_REPO, folder_path=str(RUN / "hf_model"), ignore_patterns=["__pycache__/*", "*.pyc", ".pytest_cache/*"], commit_message=message or "expected-greenness v1") refs = [t.name for t in api.list_repo_refs(MODEL_REPO).tags] if "v1" not in refs: api.create_tag(MODEL_REPO, tag="v1", tag_message="protocol model_v1, gate passed") print(f"PUSHED {MODEL_REPO} (tag v1)", flush=True) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("what", choices=["cube", "products", "model", "all"]) ap.add_argument("-m", "--message", default="") a = ap.parse_args() api = HfApi() print("as:", api.whoami().get("name"), flush=True) if a.what in ("cube", "all"): push_cube(api, a.message) if a.what in ("products", "all"): push_products(api, a.message) if a.what in ("model", "all"): push_model(api, a.message) if __name__ == "__main__": main()