Konuktor92's picture
Add full training code (open source) for model v1
d7c810d verified
Raw History Blame
10.2 kB
"""Bulk download of every NASA layer for the cube.
python scripts/10_download.py submit # AppEEARS: one area task per product, ids saved to JSON
python scripts/10_download.py fetch # AppEEARS: poll tasks, download finished bundles (re-runnable)
python scripts/10_download.py gesdisc # GES DISC: IMERG + GLDAS monthly files (re-runnable)
Credentials come from ~/.netrc (urs.earthdata.nasa.gov). Every step skips work already done,
so it is safe to re-run after a crash or a lost SSH session.
"""
from __future__ import annotations
import argparse
import json
import netrc
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import requests
APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api"
ROOT = Path("/root/trendex-ml/data")
AOI = ROOT / "aoi" / "aoi_request.geojson"
RAW = ROOT / "raw"
TASK_IDS = RAW / "appeears_task_ids.json"
LAST_YEAR = 2025
def full(start: str, end: str = f"12-31-{LAST_YEAR}") -> dict:
return {"startDate": start, "endDate": end}
def months_each_year(start_mmdd: str, end_mmdd: str, first_year: int) -> dict:
"""Recurring window, e.g. Mar-Sep of every year."""
return {"startDate": start_mmdd, "endDate": end_mmdd, "recurring": True,
"yearRange": [first_year, LAST_YEAR]}
# product -> (date window, layers). LST and snow are only needed Mar-Sep (model features).
PRODUCTS = {
"MOD13A3": (full("02-01-2000"), [("MOD13A3.061", "_1_km_monthly_NDVI"),
("MOD13A3.061", "_1_km_monthly_pixel_reliability")]),
"MYD13A3": (full("07-01-2002"), [("MYD13A3.061", "_1_km_monthly_NDVI"),
("MYD13A3.061", "_1_km_monthly_pixel_reliability")]),
"MOD11A2": (months_each_year("03-01", "09-30", 2000), [("MOD11A2.061", "LST_Day_1km"),
("MOD11A2.061", "LST_Night_1km"),
("MOD11A2.061", "QC_Day"),
("MOD11A2.061", "QC_Night")]),
"MOD10A2": (months_each_year("03-01", "09-30", 2000), [("MOD10A2.061", "Maximum_Snow_Extent")]),
"MCD12Q1": (full("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1"),
("MCD12Q1.061", "LW")]),
"NASADEM": (full("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]),
}
ORDER = ["MOD13A3", "NASADEM", "MCD12Q1", "MOD11A2", "MOD10A2", "MYD13A3"] # what the formula needs first
LPDAAC = { # fallback: raw HDF4 tiles straight from LP DAAC / NSIDC if the AppEEARS queue is too slow
"MOD13A3": ("MOD13A3", "061", "2000-02-01"),
"MYD13A3": ("MYD13A3", "061", "2002-07-01"),
"MOD11A2": ("MOD11A2", "061", "2000-02-18"),
"MOD10A2": ("MOD10A2", "61", "2000-02-24"),
"MCD12Q1": ("MCD12Q1", "061", "2020-01-01"),
"NASADEM": ("NASADEM_HGT", "001", "2000-02-11"),
}
GESDISC = { # short_name: (version, first month)
"GPM_3IMERGM": ("07", "2000-06-01"),
"GLDAS_NOAH025_M": ("2.1", "2000-01-01"),
}
def headers() -> dict:
auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov")
if not auth:
sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc")
r = requests.post(f"{APPEEARS}/login", auth=(auth[0], auth[2]), timeout=60)
r.raise_for_status()
return {"Authorization": f"Bearer {r.json()['token']}"}
def submit() -> None:
RAW.mkdir(parents=True, exist_ok=True)
ids = json.loads(TASK_IDS.read_text()) if TASK_IDS.exists() else {}
geo = json.loads(AOI.read_text())
h = headers()
for name in ORDER:
if name in ids:
print(f"[submit] {name}: already submitted {ids[name]}")
continue
window, layers = PRODUCTS[name]
body = {"task_type": "area", "task_name": f"trendex-{name}",
"params": {"dates": [window],
"layers": [{"product": p, "layer": l} for p, l in layers],
"geo": geo,
"output": {"format": {"type": "geotiff"}, "projection": "native"}}}
r = requests.post(f"{APPEEARS}/task", json=body, headers=h, timeout=120)
if r.status_code >= 400:
print(f"[submit] {name}: FAILED {r.status_code} {r.text[:400]}")
continue
ids[name] = r.json()["task_id"]
TASK_IDS.write_text(json.dumps(ids, indent=2))
print(f"[submit] {name}: {ids[name]}")
class Session:
"""AppEEARS client: one login, re-login on 401, polite back-off on 429/5xx (it rate-limits hard)."""
def __init__(self) -> None:
self.h = headers()
def get(self, url: str, stream: bool = False, tries: int = 8) -> requests.Response:
for i in range(tries):
try:
r = requests.get(url, headers=self.h, stream=stream, allow_redirects=True, timeout=300)
except requests.RequestException as exc:
print(f" network error ({exc}); retry in {20 * (i + 1)}s", flush=True)
time.sleep(20 * (i + 1))
continue
if r.status_code == 401:
self.h = headers()
continue
if r.status_code == 429 or r.status_code >= 500:
wait = int(r.headers.get("Retry-After", 0) or 60 * (i + 1))
print(f" HTTP {r.status_code}; waiting {wait}s", flush=True)
r.close()
time.sleep(wait)
continue
r.raise_for_status()
return r
raise RuntimeError(f"giving up on {url}")
def _download(s: Session, url: str, path: Path, size: int | None) -> None:
if path.exists() and (size is None or path.stat().st_size == size):
return
tmp = path.with_suffix(path.suffix + ".part")
with s.get(url, stream=True) as r:
with open(tmp, "wb") as fh:
for chunk in r.iter_content(1 << 20):
fh.write(chunk)
tmp.rename(path)
def fetch(poll_s: int, max_hours: float) -> None:
ids = json.loads(TASK_IDS.read_text())
done_marker = RAW / "appeears_done.json"
done = json.loads(done_marker.read_text()) if done_marker.exists() else {}
deadline = time.time() + max_hours * 3600
s = Session()
while time.time() < deadline:
pending = [n for n in ids if n not in done]
if not pending:
break
# one request for every task's status instead of one per task
statuses = {t["task_id"]: t.get("status") for t in s.get(f"{APPEEARS}/task?limit=100").json()}
for name in pending:
tid = ids[name]
status = statuses.get(tid)
if status in ("error", "expired", "deleted"):
print(f"[fetch] {name}: task {status}; resubmit with `submit` after removing it from {TASK_IDS.name}")
done[name] = status
continue
if status != "done":
continue
files = s.get(f"{APPEEARS}/bundle/{tid}").json()["files"]
dest = RAW / "appeears" / name
dest.mkdir(parents=True, exist_ok=True)
jobs = [(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", dest / Path(f["file_name"]).name,
f.get("file_size")) for f in files]
with ThreadPoolExecutor(6) as pool:
list(pool.map(lambda j: _download(s, j[0], j[1], j[2]), jobs))
done[name] = "done"
done_marker.write_text(json.dumps(done, indent=2))
print(f"[fetch] {name}: downloaded {len(jobs)} files", flush=True)
left = [n for n in ids if n not in done]
if left:
shown = {n: statuses.get(ids[n]) for n in left}
print(f"[fetch] waiting on {shown} ({time.strftime('%H:%M:%S')})", flush=True)
time.sleep(poll_s)
print("FETCH_COMPLETE" if all(done.get(n) == "done" for n in ids) else f"FETCH_STATUS {done}")
def aoi_bbox() -> tuple[float, float, float, float]:
geo = json.loads(AOI.read_text())
xs = [c[0] for f in geo["features"] for c in f["geometry"]["coordinates"][0]]
ys = [c[1] for f in geo["features"] for c in f["geometry"]["coordinates"][0]]
return (min(xs), min(ys), max(xs), max(ys))
def lpdaac(products: list[str]) -> None:
import earthaccess
earthaccess.login(strategy="netrc")
bbox = aoi_bbox()
for name in products:
short, ver, first = LPDAAC[name]
last = "2020-12-31" if name == "MCD12Q1" else f"{LAST_YEAR}-12-31"
found = earthaccess.search_data(short_name=short, version=ver, temporal=(first, last), bounding_box=bbox)
dest = RAW / "earthaccess" / name
dest.mkdir(parents=True, exist_ok=True)
print(f"[lpdaac] {name}: {len(found)} granules", flush=True)
earthaccess.download(found, local_path=str(dest), threads=16)
print(f"[lpdaac] {name}: {len(list(dest.iterdir()))} files on disk", flush=True)
print("LPDAAC_COMPLETE", products)
def gesdisc() -> None:
import earthaccess
earthaccess.login(strategy="netrc")
bbox = aoi_bbox()
for short, (ver, first) in GESDISC.items():
found = earthaccess.search_data(short_name=short, version=ver,
temporal=(first, f"{LAST_YEAR}-12-31"), bounding_box=bbox)
dest = RAW / "gesdisc" / short
dest.mkdir(parents=True, exist_ok=True)
print(f"[gesdisc] {short} v{ver}: {len(found)} monthly granules", flush=True)
earthaccess.download(found, local_path=str(dest), threads=16)
print(f"[gesdisc] {short}: {len(list(dest.iterdir()))} files on disk", flush=True)
print("GESDISC_COMPLETE")
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("step", choices=["submit", "fetch", "gesdisc", "lpdaac"])
ap.add_argument("--products", nargs="*", default=["MOD13A3"])
ap.add_argument("--poll-s", type=int, default=60)
ap.add_argument("--max-hours", type=float, default=6)
a = ap.parse_args()
{"submit": submit, "gesdisc": gesdisc, "lpdaac": lambda: lpdaac(a.products),
"fetch": lambda: fetch(a.poll_s, a.max_hours)}[a.step]()
if __name__ == "__main__":
main()