File size: 4,880 Bytes
d7c810d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
"""Push the cube (and later products/model) to Hugging Face.

    python scripts/50_push_hf.py cube        # data/cube + data/aoi -> TrendeX90/naryn-cube (private dataset)

Uses the token in ~/.cache/huggingface/token (a fine-grained token with write access to TrendeX90 only).
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

from huggingface_hub import HfApi

ROOT = Path("/root/trendex-ml/data")
CUBE_REPO = "TrendeX90/naryn-cube"
MODEL_REPO = "TrendeX90/expected-greenness"
RUN = Path("/root/trendex-ml/runs/v1")

CARD = """---
license: other
pretty_name: TrendeX Naryn pasture cube
tags: [nasa, modis, ndvi, imerg, gldas, kyrgyzstan, pastures]
---
# TrendeX Naryn cube ({status})

Shared data cube for TrendeX (NASA Space Apps 2026, "Be An Earth System Trend Detective!"):
Naryn oblast, Kyrgyzstan, plus a 20 km buffer and two known-place boxes (Suusamyr valley, Ak-Shyirak).
**synthetic = false** - every value comes from NASA data listed below.

## Grid
Global MODIS sinusoidal 1 km grid (`+proj=sinu +R=6371007.181`), pixel 926.625 m, equal-area.
`cell_id = global_row * 43200 + global_col`. Blocks: 11 x 11 cells (`block_id` = "row-col", ints in `block_row`, `block_col`).

## Files
- `cells.parquet` - one row per cell: position, district (5 official districts; null outside), terrain (NASADEM),
  land cover (MCD12Q1 2020), `mask_reason` (water / ice / urban / too_steep; null = pasture candidate).
- `monthly/year=YYYY/part.parquet` - one row per cell x month, 2000-02..2025-12 (`pd.read_parquet("monthly/")`).
- `aoi/` - oblast, district and request polygons (GeoJSON, EPSG:4326).
- `manifest.json` - units, dataset versions and DOIs, sources, notes. `qa_report.json` - sanity checks.

## Units
{units}

## Data sources
{datasets}

Downloaded with earthaccess from LP DAAC (MODIS, NASADEM) and GES DISC (IMERG, GLDAS).

## Notes
{notes}
"""


def push_cube(api: HfApi, message: str) -> None:
    manifest = json.loads((ROOT / "cube" / "manifest.json").read_text())
    status = "PARTIAL - NDVI only" if manifest.get("partial") else "full"
    card = CARD.format(
        status=status,
        units="\n".join(f"- `{k}`: {v}" for k, v in manifest.get("units", {}).items()),
        datasets="\n".join(f"- {k} v{v}" for k, v in manifest.get("datasets", {}).items()),
        notes="\n".join(f"- {n}" for n in manifest.get("notes", [])))
    (ROOT / "cube" / "README.md").write_text(card)
    if not api.repo_exists(CUBE_REPO, repo_type="dataset"):  # tokens without org create rights still work
        api.create_repo(CUBE_REPO, repo_type="dataset", private=True)
    api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "cube"),
                      commit_message=message or f"cube ({status})")
    api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "aoi"), path_in_repo="aoi",
                      allow_patterns=["naryn_oblast.geojson", "naryn_districts.geojson", "aoi_request.geojson",
                                      "aoi_summary.json"], commit_message="area of interest polygons")
    print(f"PUSHED {CUBE_REPO} ({status})", flush=True)


def push_products(api: HfApi, message: str) -> None:
    """runs/v1/products + metrics.json -> naryn-cube/products/ (what the backend reads)."""
    api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(RUN / "products"),
                      path_in_repo="products", commit_message=message or "model products v1")
    api.upload_file(path_or_fileobj=str(RUN / "metrics.json"), path_in_repo="products/metrics.json",
                    repo_id=CUBE_REPO, repo_type="dataset", commit_message="model metrics v1")
    print(f"PUSHED {CUBE_REPO}/products", flush=True)


def push_model(api: HfApi, message: str) -> None:
    if not api.repo_exists(MODEL_REPO):
        api.create_repo(MODEL_REPO, private=True)
    api.upload_folder(repo_id=MODEL_REPO, folder_path=str(RUN / "hf_model"), ignore_patterns=["__pycache__/*", "*.pyc", ".pytest_cache/*"],
                      commit_message=message or "expected-greenness v1")
    refs = [t.name for t in api.list_repo_refs(MODEL_REPO).tags]
    if "v1" not in refs:
        api.create_tag(MODEL_REPO, tag="v1", tag_message="protocol model_v1, gate passed")
    print(f"PUSHED {MODEL_REPO} (tag v1)", flush=True)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("what", choices=["cube", "products", "model", "all"])
    ap.add_argument("-m", "--message", default="")
    a = ap.parse_args()
    api = HfApi()
    print("as:", api.whoami().get("name"), flush=True)
    if a.what in ("cube", "all"):
        push_cube(api, a.message)
    if a.what in ("products", "all"):
        push_products(api, a.message)
    if a.what in ("model", "all"):
        push_model(api, a.message)


if __name__ == "__main__":
    main()