File size: 4,880 Bytes
d7c810d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """Push the cube (and later products/model) to Hugging Face.
python scripts/50_push_hf.py cube # data/cube + data/aoi -> TrendeX90/naryn-cube (private dataset)
Uses the token in ~/.cache/huggingface/token (a fine-grained token with write access to TrendeX90 only).
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from huggingface_hub import HfApi
ROOT = Path("/root/trendex-ml/data")
CUBE_REPO = "TrendeX90/naryn-cube"
MODEL_REPO = "TrendeX90/expected-greenness"
RUN = Path("/root/trendex-ml/runs/v1")
CARD = """---
license: other
pretty_name: TrendeX Naryn pasture cube
tags: [nasa, modis, ndvi, imerg, gldas, kyrgyzstan, pastures]
---
# TrendeX Naryn cube ({status})
Shared data cube for TrendeX (NASA Space Apps 2026, "Be An Earth System Trend Detective!"):
Naryn oblast, Kyrgyzstan, plus a 20 km buffer and two known-place boxes (Suusamyr valley, Ak-Shyirak).
**synthetic = false** - every value comes from NASA data listed below.
## Grid
Global MODIS sinusoidal 1 km grid (`+proj=sinu +R=6371007.181`), pixel 926.625 m, equal-area.
`cell_id = global_row * 43200 + global_col`. Blocks: 11 x 11 cells (`block_id` = "row-col", ints in `block_row`, `block_col`).
## Files
- `cells.parquet` - one row per cell: position, district (5 official districts; null outside), terrain (NASADEM),
land cover (MCD12Q1 2020), `mask_reason` (water / ice / urban / too_steep; null = pasture candidate).
- `monthly/year=YYYY/part.parquet` - one row per cell x month, 2000-02..2025-12 (`pd.read_parquet("monthly/")`).
- `aoi/` - oblast, district and request polygons (GeoJSON, EPSG:4326).
- `manifest.json` - units, dataset versions and DOIs, sources, notes. `qa_report.json` - sanity checks.
## Units
{units}
## Data sources
{datasets}
Downloaded with earthaccess from LP DAAC (MODIS, NASADEM) and GES DISC (IMERG, GLDAS).
## Notes
{notes}
"""
def push_cube(api: HfApi, message: str) -> None:
manifest = json.loads((ROOT / "cube" / "manifest.json").read_text())
status = "PARTIAL - NDVI only" if manifest.get("partial") else "full"
card = CARD.format(
status=status,
units="\n".join(f"- `{k}`: {v}" for k, v in manifest.get("units", {}).items()),
datasets="\n".join(f"- {k} v{v}" for k, v in manifest.get("datasets", {}).items()),
notes="\n".join(f"- {n}" for n in manifest.get("notes", [])))
(ROOT / "cube" / "README.md").write_text(card)
if not api.repo_exists(CUBE_REPO, repo_type="dataset"): # tokens without org create rights still work
api.create_repo(CUBE_REPO, repo_type="dataset", private=True)
api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "cube"),
commit_message=message or f"cube ({status})")
api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "aoi"), path_in_repo="aoi",
allow_patterns=["naryn_oblast.geojson", "naryn_districts.geojson", "aoi_request.geojson",
"aoi_summary.json"], commit_message="area of interest polygons")
print(f"PUSHED {CUBE_REPO} ({status})", flush=True)
def push_products(api: HfApi, message: str) -> None:
"""runs/v1/products + metrics.json -> naryn-cube/products/ (what the backend reads)."""
api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(RUN / "products"),
path_in_repo="products", commit_message=message or "model products v1")
api.upload_file(path_or_fileobj=str(RUN / "metrics.json"), path_in_repo="products/metrics.json",
repo_id=CUBE_REPO, repo_type="dataset", commit_message="model metrics v1")
print(f"PUSHED {CUBE_REPO}/products", flush=True)
def push_model(api: HfApi, message: str) -> None:
if not api.repo_exists(MODEL_REPO):
api.create_repo(MODEL_REPO, private=True)
api.upload_folder(repo_id=MODEL_REPO, folder_path=str(RUN / "hf_model"), ignore_patterns=["__pycache__/*", "*.pyc", ".pytest_cache/*"],
commit_message=message or "expected-greenness v1")
refs = [t.name for t in api.list_repo_refs(MODEL_REPO).tags]
if "v1" not in refs:
api.create_tag(MODEL_REPO, tag="v1", tag_message="protocol model_v1, gate passed")
print(f"PUSHED {MODEL_REPO} (tag v1)", flush=True)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("what", choices=["cube", "products", "model", "all"])
ap.add_argument("-m", "--message", default="")
a = ap.parse_args()
api = HfApi()
print("as:", api.whoami().get("name"), flush=True)
if a.what in ("cube", "all"):
push_cube(api, a.message)
if a.what in ("products", "all"):
push_products(api, a.message)
if a.what in ("model", "all"):
push_model(api, a.message)
if __name__ == "__main__":
main()
|