Download training_code/scripts/50_push_hf.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 4.88 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/50_push_hf.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/50_push_hf.py
-
curl -L -o 50_push_hf.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/50_push_hf.py
4.88 kB
| """Push the cube (and later products/model) to Hugging Face. | |
| python scripts/50_push_hf.py cube # data/cube + data/aoi -> TrendeX90/naryn-cube (private dataset) | |
| Uses the token in ~/.cache/huggingface/token (a fine-grained token with write access to TrendeX90 only). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| from huggingface_hub import HfApi | |
| ROOT = Path("/root/trendex-ml/data") | |
| CUBE_REPO = "TrendeX90/naryn-cube" | |
| MODEL_REPO = "TrendeX90/expected-greenness" | |
| RUN = Path("/root/trendex-ml/runs/v1") | |
| CARD = """--- | |
| license: other | |
| pretty_name: TrendeX Naryn pasture cube | |
| tags: [nasa, modis, ndvi, imerg, gldas, kyrgyzstan, pastures] | |
| --- | |
| # TrendeX Naryn cube ({status}) | |
| Shared data cube for TrendeX (NASA Space Apps 2026, "Be An Earth System Trend Detective!"): | |
| Naryn oblast, Kyrgyzstan, plus a 20 km buffer and two known-place boxes (Suusamyr valley, Ak-Shyirak). | |
| **synthetic = false** - every value comes from NASA data listed below. | |
| ## Grid | |
| Global MODIS sinusoidal 1 km grid (`+proj=sinu +R=6371007.181`), pixel 926.625 m, equal-area. | |
| `cell_id = global_row * 43200 + global_col`. Blocks: 11 x 11 cells (`block_id` = "row-col", ints in `block_row`, `block_col`). | |
| ## Files | |
| - `cells.parquet` - one row per cell: position, district (5 official districts; null outside), terrain (NASADEM), | |
| land cover (MCD12Q1 2020), `mask_reason` (water / ice / urban / too_steep; null = pasture candidate). | |
| - `monthly/year=YYYY/part.parquet` - one row per cell x month, 2000-02..2025-12 (`pd.read_parquet("monthly/")`). | |
| - `aoi/` - oblast, district and request polygons (GeoJSON, EPSG:4326). | |
| - `manifest.json` - units, dataset versions and DOIs, sources, notes. `qa_report.json` - sanity checks. | |
| ## Units | |
| {units} | |
| ## Data sources | |
| {datasets} | |
| Downloaded with earthaccess from LP DAAC (MODIS, NASADEM) and GES DISC (IMERG, GLDAS). | |
| ## Notes | |
| {notes} | |
| """ | |
| def push_cube(api: HfApi, message: str) -> None: | |
| manifest = json.loads((ROOT / "cube" / "manifest.json").read_text()) | |
| status = "PARTIAL - NDVI only" if manifest.get("partial") else "full" | |
| card = CARD.format( | |
| status=status, | |
| units="\n".join(f"- `{k}`: {v}" for k, v in manifest.get("units", {}).items()), | |
| datasets="\n".join(f"- {k} v{v}" for k, v in manifest.get("datasets", {}).items()), | |
| notes="\n".join(f"- {n}" for n in manifest.get("notes", []))) | |
| (ROOT / "cube" / "README.md").write_text(card) | |
| if not api.repo_exists(CUBE_REPO, repo_type="dataset"): # tokens without org create rights still work | |
| api.create_repo(CUBE_REPO, repo_type="dataset", private=True) | |
| api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "cube"), | |
| commit_message=message or f"cube ({status})") | |
| api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "aoi"), path_in_repo="aoi", | |
| allow_patterns=["naryn_oblast.geojson", "naryn_districts.geojson", "aoi_request.geojson", | |
| "aoi_summary.json"], commit_message="area of interest polygons") | |
| print(f"PUSHED {CUBE_REPO} ({status})", flush=True) | |
| def push_products(api: HfApi, message: str) -> None: | |
| """runs/v1/products + metrics.json -> naryn-cube/products/ (what the backend reads).""" | |
| api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(RUN / "products"), | |
| path_in_repo="products", commit_message=message or "model products v1") | |
| api.upload_file(path_or_fileobj=str(RUN / "metrics.json"), path_in_repo="products/metrics.json", | |
| repo_id=CUBE_REPO, repo_type="dataset", commit_message="model metrics v1") | |
| print(f"PUSHED {CUBE_REPO}/products", flush=True) | |
| def push_model(api: HfApi, message: str) -> None: | |
| if not api.repo_exists(MODEL_REPO): | |
| api.create_repo(MODEL_REPO, private=True) | |
| api.upload_folder(repo_id=MODEL_REPO, folder_path=str(RUN / "hf_model"), ignore_patterns=["__pycache__/*", "*.pyc", ".pytest_cache/*"], | |
| commit_message=message or "expected-greenness v1") | |
| refs = [t.name for t in api.list_repo_refs(MODEL_REPO).tags] | |
| if "v1" not in refs: | |
| api.create_tag(MODEL_REPO, tag="v1", tag_message="protocol model_v1, gate passed") | |
| print(f"PUSHED {MODEL_REPO} (tag v1)", flush=True) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("what", choices=["cube", "products", "model", "all"]) | |
| ap.add_argument("-m", "--message", default="") | |
| a = ap.parse_args() | |
| api = HfApi() | |
| print("as:", api.whoami().get("name"), flush=True) | |
| if a.what in ("cube", "all"): | |
| push_cube(api, a.message) | |
| if a.what in ("products", "all"): | |
| push_products(api, a.message) | |
| if a.what in ("model", "all"): | |
| push_model(api, a.message) | |
| if __name__ == "__main__": | |
| main() | |