"""Bulk download of every NASA layer for the cube. python scripts/10_download.py submit # AppEEARS: one area task per product, ids saved to JSON python scripts/10_download.py fetch # AppEEARS: poll tasks, download finished bundles (re-runnable) python scripts/10_download.py gesdisc # GES DISC: IMERG + GLDAS monthly files (re-runnable) Credentials come from ~/.netrc (urs.earthdata.nasa.gov). Every step skips work already done, so it is safe to re-run after a crash or a lost SSH session. """ from __future__ import annotations import argparse import json import netrc import sys import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path import requests APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api" ROOT = Path("/root/trendex-ml/data") AOI = ROOT / "aoi" / "aoi_request.geojson" RAW = ROOT / "raw" TASK_IDS = RAW / "appeears_task_ids.json" LAST_YEAR = 2025 def full(start: str, end: str = f"12-31-{LAST_YEAR}") -> dict: return {"startDate": start, "endDate": end} def months_each_year(start_mmdd: str, end_mmdd: str, first_year: int) -> dict: """Recurring window, e.g. Mar-Sep of every year.""" return {"startDate": start_mmdd, "endDate": end_mmdd, "recurring": True, "yearRange": [first_year, LAST_YEAR]} # product -> (date window, layers). LST and snow are only needed Mar-Sep (model features). PRODUCTS = { "MOD13A3": (full("02-01-2000"), [("MOD13A3.061", "_1_km_monthly_NDVI"), ("MOD13A3.061", "_1_km_monthly_pixel_reliability")]), "MYD13A3": (full("07-01-2002"), [("MYD13A3.061", "_1_km_monthly_NDVI"), ("MYD13A3.061", "_1_km_monthly_pixel_reliability")]), "MOD11A2": (months_each_year("03-01", "09-30", 2000), [("MOD11A2.061", "LST_Day_1km"), ("MOD11A2.061", "LST_Night_1km"), ("MOD11A2.061", "QC_Day"), ("MOD11A2.061", "QC_Night")]), "MOD10A2": (months_each_year("03-01", "09-30", 2000), [("MOD10A2.061", "Maximum_Snow_Extent")]), "MCD12Q1": (full("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1"), ("MCD12Q1.061", "LW")]), "NASADEM": (full("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]), } ORDER = ["MOD13A3", "NASADEM", "MCD12Q1", "MOD11A2", "MOD10A2", "MYD13A3"] # what the formula needs first LPDAAC = { # fallback: raw HDF4 tiles straight from LP DAAC / NSIDC if the AppEEARS queue is too slow "MOD13A3": ("MOD13A3", "061", "2000-02-01"), "MYD13A3": ("MYD13A3", "061", "2002-07-01"), "MOD11A2": ("MOD11A2", "061", "2000-02-18"), "MOD10A2": ("MOD10A2", "61", "2000-02-24"), "MCD12Q1": ("MCD12Q1", "061", "2020-01-01"), "NASADEM": ("NASADEM_HGT", "001", "2000-02-11"), } GESDISC = { # short_name: (version, first month) "GPM_3IMERGM": ("07", "2000-06-01"), "GLDAS_NOAH025_M": ("2.1", "2000-01-01"), } def headers() -> dict: auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov") if not auth: sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc") r = requests.post(f"{APPEEARS}/login", auth=(auth[0], auth[2]), timeout=60) r.raise_for_status() return {"Authorization": f"Bearer {r.json()['token']}"} def submit() -> None: RAW.mkdir(parents=True, exist_ok=True) ids = json.loads(TASK_IDS.read_text()) if TASK_IDS.exists() else {} geo = json.loads(AOI.read_text()) h = headers() for name in ORDER: if name in ids: print(f"[submit] {name}: already submitted {ids[name]}") continue window, layers = PRODUCTS[name] body = {"task_type": "area", "task_name": f"trendex-{name}", "params": {"dates": [window], "layers": [{"product": p, "layer": l} for p, l in layers], "geo": geo, "output": {"format": {"type": "geotiff"}, "projection": "native"}}} r = requests.post(f"{APPEEARS}/task", json=body, headers=h, timeout=120) if r.status_code >= 400: print(f"[submit] {name}: FAILED {r.status_code} {r.text[:400]}") continue ids[name] = r.json()["task_id"] TASK_IDS.write_text(json.dumps(ids, indent=2)) print(f"[submit] {name}: {ids[name]}") class Session: """AppEEARS client: one login, re-login on 401, polite back-off on 429/5xx (it rate-limits hard).""" def __init__(self) -> None: self.h = headers() def get(self, url: str, stream: bool = False, tries: int = 8) -> requests.Response: for i in range(tries): try: r = requests.get(url, headers=self.h, stream=stream, allow_redirects=True, timeout=300) except requests.RequestException as exc: print(f" network error ({exc}); retry in {20 * (i + 1)}s", flush=True) time.sleep(20 * (i + 1)) continue if r.status_code == 401: self.h = headers() continue if r.status_code == 429 or r.status_code >= 500: wait = int(r.headers.get("Retry-After", 0) or 60 * (i + 1)) print(f" HTTP {r.status_code}; waiting {wait}s", flush=True) r.close() time.sleep(wait) continue r.raise_for_status() return r raise RuntimeError(f"giving up on {url}") def _download(s: Session, url: str, path: Path, size: int | None) -> None: if path.exists() and (size is None or path.stat().st_size == size): return tmp = path.with_suffix(path.suffix + ".part") with s.get(url, stream=True) as r: with open(tmp, "wb") as fh: for chunk in r.iter_content(1 << 20): fh.write(chunk) tmp.rename(path) def fetch(poll_s: int, max_hours: float) -> None: ids = json.loads(TASK_IDS.read_text()) done_marker = RAW / "appeears_done.json" done = json.loads(done_marker.read_text()) if done_marker.exists() else {} deadline = time.time() + max_hours * 3600 s = Session() while time.time() < deadline: pending = [n for n in ids if n not in done] if not pending: break # one request for every task's status instead of one per task statuses = {t["task_id"]: t.get("status") for t in s.get(f"{APPEEARS}/task?limit=100").json()} for name in pending: tid = ids[name] status = statuses.get(tid) if status in ("error", "expired", "deleted"): print(f"[fetch] {name}: task {status}; resubmit with `submit` after removing it from {TASK_IDS.name}") done[name] = status continue if status != "done": continue files = s.get(f"{APPEEARS}/bundle/{tid}").json()["files"] dest = RAW / "appeears" / name dest.mkdir(parents=True, exist_ok=True) jobs = [(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", dest / Path(f["file_name"]).name, f.get("file_size")) for f in files] with ThreadPoolExecutor(6) as pool: list(pool.map(lambda j: _download(s, j[0], j[1], j[2]), jobs)) done[name] = "done" done_marker.write_text(json.dumps(done, indent=2)) print(f"[fetch] {name}: downloaded {len(jobs)} files", flush=True) left = [n for n in ids if n not in done] if left: shown = {n: statuses.get(ids[n]) for n in left} print(f"[fetch] waiting on {shown} ({time.strftime('%H:%M:%S')})", flush=True) time.sleep(poll_s) print("FETCH_COMPLETE" if all(done.get(n) == "done" for n in ids) else f"FETCH_STATUS {done}") def aoi_bbox() -> tuple[float, float, float, float]: geo = json.loads(AOI.read_text()) xs = [c[0] for f in geo["features"] for c in f["geometry"]["coordinates"][0]] ys = [c[1] for f in geo["features"] for c in f["geometry"]["coordinates"][0]] return (min(xs), min(ys), max(xs), max(ys)) def lpdaac(products: list[str]) -> None: import earthaccess earthaccess.login(strategy="netrc") bbox = aoi_bbox() for name in products: short, ver, first = LPDAAC[name] last = "2020-12-31" if name == "MCD12Q1" else f"{LAST_YEAR}-12-31" found = earthaccess.search_data(short_name=short, version=ver, temporal=(first, last), bounding_box=bbox) dest = RAW / "earthaccess" / name dest.mkdir(parents=True, exist_ok=True) print(f"[lpdaac] {name}: {len(found)} granules", flush=True) earthaccess.download(found, local_path=str(dest), threads=16) print(f"[lpdaac] {name}: {len(list(dest.iterdir()))} files on disk", flush=True) print("LPDAAC_COMPLETE", products) def gesdisc() -> None: import earthaccess earthaccess.login(strategy="netrc") bbox = aoi_bbox() for short, (ver, first) in GESDISC.items(): found = earthaccess.search_data(short_name=short, version=ver, temporal=(first, f"{LAST_YEAR}-12-31"), bounding_box=bbox) dest = RAW / "gesdisc" / short dest.mkdir(parents=True, exist_ok=True) print(f"[gesdisc] {short} v{ver}: {len(found)} monthly granules", flush=True) earthaccess.download(found, local_path=str(dest), threads=16) print(f"[gesdisc] {short}: {len(list(dest.iterdir()))} files on disk", flush=True) print("GESDISC_COMPLETE") def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("step", choices=["submit", "fetch", "gesdisc", "lpdaac"]) ap.add_argument("--products", nargs="*", default=["MOD13A3"]) ap.add_argument("--poll-s", type=int, default=60) ap.add_argument("--max-hours", type=float, default=6) a = ap.parse_args() {"submit": submit, "gesdisc": gesdisc, "lpdaac": lambda: lpdaac(a.products), "fetch": lambda: fetch(a.poll_s, a.max_hours)}[a.step]() if __name__ == "__main__": main()