File size: 10,225 Bytes
7e6579d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
"""Bulk download of every NASA layer for the cube.

    python scripts/10_download.py submit    # AppEEARS: one area task per product, ids saved to JSON
    python scripts/10_download.py fetch     # AppEEARS: poll tasks, download finished bundles (re-runnable)
    python scripts/10_download.py gesdisc   # GES DISC: IMERG + GLDAS monthly files (re-runnable)

Credentials come from ~/.netrc (urs.earthdata.nasa.gov). Every step skips work already done,
so it is safe to re-run after a crash or a lost SSH session.
"""
from __future__ import annotations

import argparse
import json
import netrc
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests

APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api"
ROOT = Path("/root/trendex-ml/data")
AOI = ROOT / "aoi" / "aoi_request.geojson"
RAW = ROOT / "raw"
TASK_IDS = RAW / "appeears_task_ids.json"
LAST_YEAR = 2025


def full(start: str, end: str = f"12-31-{LAST_YEAR}") -> dict:
    return {"startDate": start, "endDate": end}


def months_each_year(start_mmdd: str, end_mmdd: str, first_year: int) -> dict:
    """Recurring window, e.g. Mar-Sep of every year."""
    return {"startDate": start_mmdd, "endDate": end_mmdd, "recurring": True,
            "yearRange": [first_year, LAST_YEAR]}


# product -> (date window, layers). LST and snow are only needed Mar-Sep (model features).
PRODUCTS = {
    "MOD13A3": (full("02-01-2000"), [("MOD13A3.061", "_1_km_monthly_NDVI"),
                                     ("MOD13A3.061", "_1_km_monthly_pixel_reliability")]),
    "MYD13A3": (full("07-01-2002"), [("MYD13A3.061", "_1_km_monthly_NDVI"),
                                     ("MYD13A3.061", "_1_km_monthly_pixel_reliability")]),
    "MOD11A2": (months_each_year("03-01", "09-30", 2000), [("MOD11A2.061", "LST_Day_1km"),
                                                           ("MOD11A2.061", "LST_Night_1km"),
                                                           ("MOD11A2.061", "QC_Day"),
                                                           ("MOD11A2.061", "QC_Night")]),
    "MOD10A2": (months_each_year("03-01", "09-30", 2000), [("MOD10A2.061", "Maximum_Snow_Extent")]),
    "MCD12Q1": (full("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1"),
                                                   ("MCD12Q1.061", "LW")]),
    "NASADEM": (full("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]),
}
ORDER = ["MOD13A3", "NASADEM", "MCD12Q1", "MOD11A2", "MOD10A2", "MYD13A3"]  # what the formula needs first

LPDAAC = {  # fallback: raw HDF4 tiles straight from LP DAAC / NSIDC if the AppEEARS queue is too slow
    "MOD13A3": ("MOD13A3", "061", "2000-02-01"),
    "MYD13A3": ("MYD13A3", "061", "2002-07-01"),
    "MOD11A2": ("MOD11A2", "061", "2000-02-18"),
    "MOD10A2": ("MOD10A2", "61", "2000-02-24"),
    "MCD12Q1": ("MCD12Q1", "061", "2020-01-01"),
    "NASADEM": ("NASADEM_HGT", "001", "2000-02-11"),
}

GESDISC = {  # short_name: (version, first month)
    "GPM_3IMERGM": ("07", "2000-06-01"),
    "GLDAS_NOAH025_M": ("2.1", "2000-01-01"),
}


def headers() -> dict:
    auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov")
    if not auth:
        sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc")
    r = requests.post(f"{APPEEARS}/login", auth=(auth[0], auth[2]), timeout=60)
    r.raise_for_status()
    return {"Authorization": f"Bearer {r.json()['token']}"}


def submit() -> None:
    RAW.mkdir(parents=True, exist_ok=True)
    ids = json.loads(TASK_IDS.read_text()) if TASK_IDS.exists() else {}
    geo = json.loads(AOI.read_text())
    h = headers()
    for name in ORDER:
        if name in ids:
            print(f"[submit] {name}: already submitted {ids[name]}")
            continue
        window, layers = PRODUCTS[name]
        body = {"task_type": "area", "task_name": f"trendex-{name}",
                "params": {"dates": [window],
                           "layers": [{"product": p, "layer": l} for p, l in layers],
                           "geo": geo,
                           "output": {"format": {"type": "geotiff"}, "projection": "native"}}}
        r = requests.post(f"{APPEEARS}/task", json=body, headers=h, timeout=120)
        if r.status_code >= 400:
            print(f"[submit] {name}: FAILED {r.status_code} {r.text[:400]}")
            continue
        ids[name] = r.json()["task_id"]
        TASK_IDS.write_text(json.dumps(ids, indent=2))
        print(f"[submit] {name}: {ids[name]}")


class Session:
    """AppEEARS client: one login, re-login on 401, polite back-off on 429/5xx (it rate-limits hard)."""

    def __init__(self) -> None:
        self.h = headers()

    def get(self, url: str, stream: bool = False, tries: int = 8) -> requests.Response:
        for i in range(tries):
            try:
                r = requests.get(url, headers=self.h, stream=stream, allow_redirects=True, timeout=300)
            except requests.RequestException as exc:
                print(f"   network error ({exc}); retry in {20 * (i + 1)}s", flush=True)
                time.sleep(20 * (i + 1))
                continue
            if r.status_code == 401:
                self.h = headers()
                continue
            if r.status_code == 429 or r.status_code >= 500:
                wait = int(r.headers.get("Retry-After", 0) or 60 * (i + 1))
                print(f"   HTTP {r.status_code}; waiting {wait}s", flush=True)
                r.close()
                time.sleep(wait)
                continue
            r.raise_for_status()
            return r
        raise RuntimeError(f"giving up on {url}")


def _download(s: Session, url: str, path: Path, size: int | None) -> None:
    if path.exists() and (size is None or path.stat().st_size == size):
        return
    tmp = path.with_suffix(path.suffix + ".part")
    with s.get(url, stream=True) as r:
        with open(tmp, "wb") as fh:
            for chunk in r.iter_content(1 << 20):
                fh.write(chunk)
    tmp.rename(path)


def fetch(poll_s: int, max_hours: float) -> None:
    ids = json.loads(TASK_IDS.read_text())
    done_marker = RAW / "appeears_done.json"
    done = json.loads(done_marker.read_text()) if done_marker.exists() else {}
    deadline = time.time() + max_hours * 3600
    s = Session()
    while time.time() < deadline:
        pending = [n for n in ids if n not in done]
        if not pending:
            break
        # one request for every task's status instead of one per task
        statuses = {t["task_id"]: t.get("status") for t in s.get(f"{APPEEARS}/task?limit=100").json()}
        for name in pending:
            tid = ids[name]
            status = statuses.get(tid)
            if status in ("error", "expired", "deleted"):
                print(f"[fetch] {name}: task {status}; resubmit with `submit` after removing it from {TASK_IDS.name}")
                done[name] = status
                continue
            if status != "done":
                continue
            files = s.get(f"{APPEEARS}/bundle/{tid}").json()["files"]
            dest = RAW / "appeears" / name
            dest.mkdir(parents=True, exist_ok=True)
            jobs = [(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", dest / Path(f["file_name"]).name,
                     f.get("file_size")) for f in files]
            with ThreadPoolExecutor(6) as pool:
                list(pool.map(lambda j: _download(s, j[0], j[1], j[2]), jobs))
            done[name] = "done"
            done_marker.write_text(json.dumps(done, indent=2))
            print(f"[fetch] {name}: downloaded {len(jobs)} files", flush=True)
        left = [n for n in ids if n not in done]
        if left:
            shown = {n: statuses.get(ids[n]) for n in left}
            print(f"[fetch] waiting on {shown} ({time.strftime('%H:%M:%S')})", flush=True)
            time.sleep(poll_s)
    print("FETCH_COMPLETE" if all(done.get(n) == "done" for n in ids) else f"FETCH_STATUS {done}")


def aoi_bbox() -> tuple[float, float, float, float]:
    geo = json.loads(AOI.read_text())
    xs = [c[0] for f in geo["features"] for c in f["geometry"]["coordinates"][0]]
    ys = [c[1] for f in geo["features"] for c in f["geometry"]["coordinates"][0]]
    return (min(xs), min(ys), max(xs), max(ys))


def lpdaac(products: list[str]) -> None:
    import earthaccess
    earthaccess.login(strategy="netrc")
    bbox = aoi_bbox()
    for name in products:
        short, ver, first = LPDAAC[name]
        last = "2020-12-31" if name == "MCD12Q1" else f"{LAST_YEAR}-12-31"
        found = earthaccess.search_data(short_name=short, version=ver, temporal=(first, last), bounding_box=bbox)
        dest = RAW / "earthaccess" / name
        dest.mkdir(parents=True, exist_ok=True)
        print(f"[lpdaac] {name}: {len(found)} granules", flush=True)
        earthaccess.download(found, local_path=str(dest), threads=16)
        print(f"[lpdaac] {name}: {len(list(dest.iterdir()))} files on disk", flush=True)
    print("LPDAAC_COMPLETE", products)


def gesdisc() -> None:
    import earthaccess
    earthaccess.login(strategy="netrc")
    bbox = aoi_bbox()
    for short, (ver, first) in GESDISC.items():
        found = earthaccess.search_data(short_name=short, version=ver,
                                        temporal=(first, f"{LAST_YEAR}-12-31"), bounding_box=bbox)
        dest = RAW / "gesdisc" / short
        dest.mkdir(parents=True, exist_ok=True)
        print(f"[gesdisc] {short} v{ver}: {len(found)} monthly granules", flush=True)
        earthaccess.download(found, local_path=str(dest), threads=16)
        print(f"[gesdisc] {short}: {len(list(dest.iterdir()))} files on disk", flush=True)
    print("GESDISC_COMPLETE")


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("step", choices=["submit", "fetch", "gesdisc", "lpdaac"])
    ap.add_argument("--products", nargs="*", default=["MOD13A3"])
    ap.add_argument("--poll-s", type=int, default=60)
    ap.add_argument("--max-hours", type=float, default=6)
    a = ap.parse_args()
    {"submit": submit, "gesdisc": gesdisc, "lpdaac": lambda: lpdaac(a.products),
     "fetch": lambda: fetch(a.poll_s, a.max_hours)}[a.step]()


if __name__ == "__main__":
    main()