Time Series Forecasting
PyTorch
English
pluvium
weather
weather-forecasting
climate
meteorology
earth-science
precipitation
rainfall
extreme-rainfall
nowcasting
probabilistic-forecasting
tropical-cyclones
hurricane
multi-model-ensemble
post-processing
online-learning
geospatial
caribbean
naturev1
ihelix
graphcast
aifs
Download tests/test_program.py from CorxLabs/Pluvium-500: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/CorxLabs/Pluvium-500/resolve/main/tests/test_program.py
- Command line
-
hf download hf://CorxLabs/Pluvium-500/tests/test_program.py
-
curl -L -o test_program.py https://huggingface.co/CorxLabs/Pluvium-500/resolve/main/tests/test_program.py
13.6 kB
| # Copyright 2026 Nathan. Apache-2.0. | |
| """The training program's settings: the years it trains on and holds out follow what the reanalysis archive really | |
| holds, and the GPU memory settings are read from the environment and checked. Needs neither a network nor a GPU.""" | |
| import numpy as np | |
| import pytest | |
| from pluvium import program | |
| from pluvium.program import Program, ProgramConfig, choose_years | |
| from pluvium.sources import era5 | |
| from pluvium.util import hour_of | |
| class Store: | |
| """Stands in for the hourly ERA5 store: only its time axis is asked for.""" | |
| def __init__(self, first: str, last: str) -> None: | |
| self.first, self.last = hour_of(first), hour_of(last) | |
| def hours(self) -> np.ndarray: | |
| return np.arange(self.first, self.last + 1, dtype=np.int64) | |
| def test_whole_years_are_read_from_the_archive_itself(): | |
| # the real store on 15 November 2026: named 1959-2022, and ending on 31 December 2021 | |
| assert era5.hourly_years(Store("2019-01-01T00", "2021-12-31T23")) == (2019, 2021) | |
| assert era5.hourly_years(Store("2019-01-01T00", "2022-06-30T23")) == (2019, 2021) # a year cut short is not whole | |
| assert era5.hourly_years(Store("2019-03-01T00", "2022-12-31T23")) == (2020, 2022) # nor one that starts late | |
| class Unreachable: | |
| def hours(self): | |
| raise OSError("no network") | |
| assert era5.hourly_years(Unreachable()) == (1959, 2021) # what the code remembers | |
| assert era5.HOURLY_LAST == hour_of("2021-12-31T23") | |
| def test_years_asked_for_are_fitted_to_the_archive(tmp_path, monkeypatch): | |
| monkeypatch.setattr(era5, "hourly_years", lambda store=None: (1959, 2021)) | |
| def years(**settings): | |
| made = Program(ProgramConfig(root=str(tmp_path / str(len(settings))), hub_repo=None, **settings)) | |
| made.check_years() | |
| return made.years, made.cfg.holdout_year | |
| assert ProgramConfig().holdout_year == 2021 and choose_years(24) == (2015, 2016, 2017, 2018, 2019, 2020) | |
| assert years() == ((2015, 2016, 2017, 2018, 2019, 2020), 2021) | |
| # 2022 was the default held-out year until 0.4.3: the archive does not have it, and the run stopped while | |
| # staging it, after every training year had been downloaded | |
| assert years(holdout_year=2022) == ((2015, 2016, 2017, 2018, 2019, 2020), 2021) | |
| assert years(train_years=(2019, 2020, 2021, 2022), holdout_year=2022) == ((2019, 2020), 2021) | |
| assert years(train_years=(2018, 2019), holdout_year=2020) == ((2018, 2019), 2020) # a valid choice is kept | |
| with pytest.raises(ValueError, match="no training year is left"): | |
| years(train_years=(2021,)) | |
| monkeypatch.setattr(era5, "hourly_years", lambda store=None: (1959, 2023)) # the archive grows | |
| assert years(holdout_year=2023, train_years=(2021, 2022)) == ((2021, 2022), 2023) | |
| def test_memory_settings_come_from_the_environment_and_are_checked(tmp_path, monkeypatch): | |
| for name in ("GPU_HOURS", "PLUVIUM_ROOT", "HF_REPO", "HF_TOKEN", "PLUVIUM_SIZE", "PLUVIUM_COUNTRIES", "PLUVIUM_BOX", | |
| "PLUVIUM_VRAM", "PLUVIUM_VRAM_SHARE", "PLUVIUM_BATCH"): | |
| monkeypatch.delenv(name, raising=False) | |
| cfg = ProgramConfig.from_env(root=str(tmp_path), hub_repo=None) | |
| assert (cfg.vram, cfg.vram_share, cfg.batch_size) == ("fill", 0.95, 0) # fill the card by default | |
| monkeypatch.setenv("PLUVIUM_VRAM_SHARE", "0.9") | |
| monkeypatch.setenv("PLUVIUM_VRAM", "fastest") | |
| monkeypatch.setenv("PLUVIUM_BATCH", "6") | |
| cfg = ProgramConfig.from_env(root=str(tmp_path), hub_repo=None) | |
| cfg.hub_repo = None | |
| assert (cfg.vram, cfg.vram_share, cfg.batch_size) == ("fastest", 0.9, 6) | |
| train = Program(cfg).train_cfg(1.0) | |
| assert (train.memory, train.memory_share, train.batch_size) == ("fastest", 0.9, 6) | |
| for wrong in ({"vram": "all"}, {"vram_share": 1.0}, {"vram_share": 0.1}): | |
| with pytest.raises(ValueError): | |
| Program(ProgramConfig(root=str(tmp_path), hub_repo=None, **wrong)).train_cfg(1.0) | |
| source = open(program.__file__).read() # run as a module, it starts | |
| assert callable(program.main) and source.rstrip().endswith('if __name__ == "__main__":\n main()') | |
| def test_staged_data_go_up_in_a_few_commits_and_only_what_the_hub_lacks(tmp_path): | |
| """The Hub takes at most 128 commits an hour in a repository. One commit per staged file (until 0.5.2) used that | |
| up with the data alone, and most of them were refused with HTTP 429.""" | |
| import hashlib | |
| from types import SimpleNamespace | |
| from pluvium.program import FILES_PER_COMMIT, mirror_files | |
| files = [] | |
| for k in range(2 * FILES_PER_COMMIT + 5): | |
| path = tmp_path / f"f{k}.npy" | |
| path.write_bytes(bytes([k % 256]) * 100) | |
| files.append((path, f"data/f{k}.npy")) | |
| same = files[0][0].read_bytes() | |
| old = tmp_path / "old.npy" | |
| old.write_bytes(b"older content") | |
| class Api: | |
| def __init__(self, fail=False): | |
| self.commits, self.fail = [], fail | |
| def get_paths_info(self, repo, paths, repo_type="model"): | |
| if self.fail: | |
| raise RuntimeError("429 Too Many Requests") | |
| return [SimpleNamespace(path="data/f0.npy", lfs={"sha256": hashlib.sha256(same).hexdigest()}), # held | |
| SimpleNamespace(path="data/f1.npy", lfs={"sha256": hashlib.sha256(b"older").hexdigest()})] # stale | |
| def create_commit(self, repo, repo_type, commit_message, operations): | |
| self.commits.append([op.path_in_repo for op in operations]) | |
| mirror = SimpleNamespace(can_push=True, repo_id="org/repo", token="t") | |
| api = Api() | |
| assert mirror_files(mirror, files + [(tmp_path / "missing.npy", "data/missing.npy")], "data", api=api) == len(files) - 1 | |
| assert [len(c) for c in api.commits] == [FILES_PER_COMMIT, FILES_PER_COMMIT, 4] # three commits, not 125 | |
| pushed = [p for c in api.commits for p in c] | |
| assert "data/f0.npy" not in pushed and "data/f1.npy" in pushed and "data/missing.npy" not in pushed | |
| assert mirror_files(mirror, files, "data", api=Api(fail=True)) == 0 # refused: said, not raised | |
| assert mirror_files(SimpleNamespace(can_push=False), files, "data", api=Api()) == 0 # no token: nothing to do | |
| def test_a_year_with_only_one_of_its_two_rain_files_is_staged_again(tmp_path, monkeypatch): | |
| """The Hub once held ``targets_2016.npy`` without ``mesh_2016.npy`` (the uploads of an earlier run were refused | |
| half way). The stager asked only the targets file, took 2016 as done, and a training run on a 96 GB card stopped | |
| when it first needed the file that was never made.""" | |
| import pluvium.sources.cmorph as C | |
| from pluvium.util import year_hours | |
| class Points: | |
| def __init__(self, n): | |
| self.n, self.extra = n, {"row": np.zeros(n, int), "col": np.zeros(n, int)} | |
| def __len__(self): | |
| return self.n | |
| targets, mesh, fetched = Points(3), Points(5), [] | |
| for name in ("cmorph_window", "mesh_operator", "default_half_widths", "_init"): | |
| monkeypatch.setattr(C, name, lambda *a, **k: None) | |
| monkeypatch.setattr(C, "_one_hour", lambda hour: (fetched.append(hour) or hour, np.full(3, 7, np.uint16), np.full(5, 9, np.uint16))) | |
| first, _ = year_hours(2016) | |
| period = (first, first + 48) | |
| folder = tmp_path / "cmorph" | |
| C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False) | |
| assert len(fetched) == 48 and (folder / "targets_2016.npy").exists() and (folder / "mesh_2016.npy").exists() | |
| fetched.clear() | |
| C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False) | |
| assert fetched == [] # a whole year is not fetched twice | |
| (folder / "mesh_2016.npy").unlink() # half the year, as it came from the Hub | |
| cache = C.stage_cmorph(tmp_path, targets, mesh, period=period, workers=1, verbose=False) | |
| assert len(fetched) == 48 and (folder / "mesh_2016.npy").exists() | |
| assert cache.mesh(np.arange(first, first + 3)).shape == (3, 5) and np.isfinite(cache.mesh(np.arange(first, first + 3))).all() | |
| def test_the_background_fetch_does_not_keep_a_finished_run_alive(tmp_path): | |
| import multiprocessing | |
| import time | |
| prog = Program(ProgramConfig(root=str(tmp_path), hub_repo=None)) | |
| prog.stop_prefetch() # nothing running: nothing to do | |
| prog._prefetch = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(120,)) | |
| prog._prefetch.start() | |
| began = time.time() | |
| prog.stop_prefetch() | |
| assert not prog._prefetch.is_alive() and time.time() - began < 30 | |
| def test_a_molab_session_is_planned_to_be_published_before_molab_stops_it(tmp_path, monkeypatch): | |
| """molab stops a notebook 12 hours after it starts. The first full run there trained for 11.6 of its 24 hours and | |
| was stopped before it was scored and published. A run there now plans to be done before the session ends.""" | |
| import time | |
| from pluvium import program as P | |
| for name in ("PLUVIUM_SESSION_HOURS", "PLUVIUM_SESSION_START", "PLUVIUM_SESSION_WHY"): | |
| monkeypatch.delenv(name, raising=False) | |
| molab = "/tmp/uv-venv/lib/python3.13/site-packages/pluvium/program.py" # as the tracebacks there show | |
| assert P.on_molab({}, places=[molab], commands=[]) | |
| assert P.on_molab({}, places=["/usr/lib"], commands=["/usr/bin/python -m marimo edit notebook.py"]) | |
| assert P.on_molab({"MARIMO_MANAGE_SCRIPT_METADATA": "true"}, places=["/usr/lib"], commands=[]) | |
| assert not P.on_molab({"HOME": "/root"}, places=["/usr/lib"], commands=["bash", "python -m pluvium.program"]) | |
| assert P.session_limit({"PLUVIUM_SESSION_HOURS": "6"}) == (6.0, "PLUVIUM_SESSION_HOURS") | |
| assert P.session_limit({"PLUVIUM_SESSION_HOURS": "0"})[0] is None # 0: no end, even on molab | |
| began = time.time() - 40 * 60 | |
| assert P.session_start(12, {"PLUVIUM_SESSION_START": str(began)}) == began | |
| found = P.session_start(12, {}) # the earliest process of this session | |
| assert time.time() - 12 * 3600 < found <= time.time() | |
| monkeypatch.setattr(P, "on_molab", lambda env=None, places=None, commands=None: True) | |
| assert P.session_limit({}) == (12.0, "molab stops a notebook after 12 hours") | |
| # the notebook started 40 minutes before the program: published 11.5 hours after that, not 24 hours after | |
| monkeypatch.setenv("PLUVIUM_SESSION_START", str(began)) | |
| prog = Program(ProgramConfig(root=str(tmp_path), hub_repo=None, gpu_hours=24)) | |
| assert prog.session is not None and abs(prog.deadline - (began + 11.5 * 3600)) < 1 | |
| assert 10.7 < prog.left() < 10.9 and prog.reserve() == 1.5 | |
| # no end known: 24 hours from the program's start, as before | |
| monkeypatch.setattr(P, "on_molab", lambda env=None, places=None, commands=None: False) | |
| prog = Program(ProgramConfig(root=str(tmp_path), hub_repo=None, gpu_hours=24)) | |
| assert prog.session is None and 23.9 < prog.left() <= 24 and prog.reserve() == 1.0 | |
| def test_the_supervisor_finds_the_session_once_and_every_try_gets_it(tmp_path, monkeypatch): | |
| import sys | |
| import time | |
| from pluvium.program import supervise | |
| seen = tmp_path / "env.txt" | |
| child = [sys.executable, "-c", "import os, sys; open(sys.argv[1], 'a').write(os.environ['PLUVIUM_SESSION_HOURS'] " | |
| "+ ' ' + os.environ['PLUVIUM_SESSION_START'] + '\\n'); sys.exit(0 if os.environ['PLUVIUM_ATTEMPT'] == '2' " | |
| "else 1)", str(seen)] | |
| monkeypatch.delenv("PLUVIUM_SESSION_START", raising=False) | |
| monkeypatch.setenv("PLUVIUM_SESSION_HOURS", "12") | |
| assert supervise([], 0.95, command=child) == 0 | |
| first, second = seen.read_text().splitlines() | |
| assert first == second and first.split()[0] == "12" and float(first.split()[1]) <= time.time() | |
| def test_a_torn_cached_gfs_run_is_fetched_again_and_a_new_one_is_written_whole(tmp_path): | |
| """Stage B's GFS runs were cached with a plain write. A process stopped in the middle of one (molab's 12 hours, a | |
| restart, the end of a run) left a torn file, and every later read of that run failed.""" | |
| from pluvium.sources.nwp import NWPReader | |
| from pluvium.util import atomic_save | |
| reader = NWPReader("gfs", [17.0, 18.0], [-78.0, -77.0], [17.0, 18.0], [-78.0, -77.0], cache=tmp_path, workers=1) | |
| torn = tmp_path / "gfs_2025-01-01T00+6.npz" | |
| torn.write_bytes(b"PK\x03\x04 cut off here") | |
| class Fetching(Exception): | |
| pass | |
| def fetch(*args, **kwargs): | |
| raise Fetching | |
| reader._surface_at = fetch | |
| with pytest.raises(Fetching): # it went past the cache to fetch the run | |
| reader.frames(hour_of("2025-01-01T00"), offset=6) | |
| assert not torn.exists() | |
| atomic_save(tmp_path / "x.npz", lambda q: np.save(q, np.ones(3))) # a sibling per process, gone after | |
| assert sorted(p.name for p in tmp_path.iterdir()) == ["x.npz"] | |
| def test_a_torn_record_of_finished_stages_does_not_stop_the_next_run(tmp_path): | |
| (tmp_path / "program.json").write_text('{"geography": {"done": "2026-11-17 1') # stopped while writing it | |
| prog = Program(ProgramConfig(root=str(tmp_path), hub_repo=None)) | |
| assert prog.state == {} | |
| prog.mark("geography", mesh=3) | |
| assert Program(ProgramConfig(root=str(tmp_path), hub_repo=None)).state["geography"]["mesh"] == 3 | |
| assert sorted(p.name for p in tmp_path.iterdir()) == ["program.json"] # no sibling left behind | |