Spaces:
Running on Zero
Running on Zero
fix: the memory check stops waiting when less than a millisecond of its bound is left, so a clock that barely moves cannot keep it reading
a5caa41 unverified Download tests/test_memory_settle.py from techfreakworm/LTX2.5-Studio: direct link, hf CLI and curl.
- Browser
- Download file 13 kB
-
https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_memory_settle.py
- Command line
-
hf download hf://spaces/techfreakworm/LTX2.5-Studio/tests/test_memory_settle.py
-
curl -L -o test_memory_settle.py https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_memory_settle.py
13 kB
| """Spec: a judgement on the live gauge is made on a settled gauge (F73, items 1 and 2). | |
| After the local runner frees device memory, the kernel un-wires its pages over a | |
| few hundred milliseconds (measured on the operator's Mac, 2026-11-16: 24 GB of | |
| text encoder on MPS, the gauge read every 0.1 s after the release gave 62.3, | |
| 59.3, 46.3, 38.3 GB, then flat). The transformer's preflight read it inside that | |
| window and refused a run that fitted (committed 67.3 + need 25.1 = 92.4 GB > 90; | |
| eight seconds later the gauge read 38.8). So, on the live gauge only: | |
| * after a release, the gauge is read every 50 ms until two consecutive readings | |
| are within 0.25 GB, and that reading is the one judged and logged; | |
| * a FAIL is read again every 0.1 s and passes as soon as a reading fits; only a | |
| failure that persists is refused, with the last reading; | |
| * both waits share ``LTX25_MEMORY_SETTLE_SECONDS`` (default 2.0; 0 switches them | |
| off). A projected judgement, no gauge or no limit: one reading or none, as | |
| before. | |
| Every gauge here is a scripted sequence on a clock that only sleeping moves, so | |
| nothing really sleeps. Runs in both environments (no torch). | |
| """ | |
| from __future__ import annotations | |
| import pickle | |
| import pytest | |
| from ltx25_studio import memory | |
| from ltx25_studio.errors import ConfigError | |
| SETTLE = "LTX25_MEMORY_SETTLE_SECONDS" | |
| def _defaults(monkeypatch): | |
| monkeypatch.delenv(SETTLE, raising=False) | |
| monkeypatch.delenv("LTX25_MEMORY_LIMIT_GB", raising=False) | |
| class Gauge: | |
| """Scripted readings (the last one repeats) on a clock that only sleeping moves.""" | |
| def __init__(self, *readings): | |
| self.readings = list(readings) | |
| self.reads = 0 | |
| self.now = 0.0 | |
| self.slept: list[float] = [] | |
| def read(self): | |
| value = self.readings[min(self.reads, len(self.readings) - 1)] | |
| self.reads += 1 | |
| return value | |
| def clock(self): | |
| return self.now | |
| def sleep(self, seconds): | |
| assert seconds > 0 | |
| self.slept.append(seconds) | |
| self.now += seconds | |
| def judge(self, phase="load transformer", *, need_gb=25.1, limit_gb=90.0, **extra): | |
| return memory.preflight( | |
| phase, | |
| need_gb=need_gb, | |
| limit_gb=limit_gb, | |
| read=self.read, | |
| clock=self.clock, | |
| sleep=self.sleep, | |
| **extra, | |
| ) | |
| class CoarseClock(Gauge): | |
| """A clock that moves in whole milliseconds: a shorter sleep does not move it.""" | |
| def sleep(self, seconds): | |
| assert len(self.slept) < 10_000, "the judgement kept waiting on a clock that did not move" | |
| self.slept.append(seconds) | |
| self.now += round(seconds, 3) | |
| class Rising(Gauge): | |
| """A gauge that never settles: 1 GB more at every reading.""" | |
| def __init__(self, start): | |
| super().__init__(start) | |
| self.start = start | |
| def read(self): | |
| self.reads += 1 | |
| return self.start + self.reads - 1 | |
| # -------------------------------------------------------------------------- | |
| # item 1: after a release, the settled reading is the one judged and logged | |
| # -------------------------------------------------------------------------- | |
| def test_after_a_release_the_gauge_is_read_until_it_stops_falling_and_that_reading_is_judged(): | |
| gauge = Gauge(62.3, 59.3, 46.3, 38.3, 38.3) | |
| result = gauge.judge(released=True) | |
| assert result.committed == 38.3 and result.verdict == "OK" | |
| assert result.settled.describe() == "memory settled in 0.20 s: 62.3 → 38.3 GB" | |
| assert result.describe() == ( | |
| "preflight[load transformer]: committed 38.3 + need 25.1 = 63.4 GB <= 90.0 GB OK " | |
| "(headroom 26.6)" | |
| ) | |
| assert gauge.reads == 5 and gauge.slept == [memory.SETTLE_STEP_SECONDS] * 4 | |
| def test_the_sessions_refused_run_is_judged_on_the_settled_gauge(): | |
| """Run 1 of the 19:02 session: 67.3 GB right after the release, 38.8 once settled.""" | |
| gauge = Gauge(67.3, 64.1, 51.2, 40.1, 38.8, 38.8) | |
| result = gauge.judge(released=True) | |
| assert result.verdict == "OK" | |
| assert result.committed == 38.8 | |
| assert result.settled.first == 67.3 and result.settled.last == 38.8 | |
| assert result.settled.steady | |
| result.require() | |
| def test_a_release_followed_by_a_reading_that_fits_waits_only_for_the_settle(): | |
| gauge = Gauge(40.0, 40.1) | |
| result = gauge.judge(released=True) | |
| assert result.verdict == "OK" and result.committed == 40.1 | |
| assert gauge.slept == [memory.SETTLE_STEP_SECONDS], "one settle step, no reading again" | |
| assert result.first is None | |
| def test_without_a_release_a_reading_that_fits_is_one_reading_and_no_wait(): | |
| gauge = Gauge(62.3, 38.3) | |
| result = gauge.judge() | |
| assert result.committed == 62.3 and result.verdict == "OK" | |
| assert (gauge.reads, gauge.slept, result.settled) == (1, [], None) | |
| def test_the_settle_is_bounded_and_says_so_when_the_gauge_keeps_moving(): | |
| gauge = Rising(40.0) | |
| result = gauge.judge(released=True, limit_gb=500.0) | |
| assert result.verdict == "OK" | |
| assert not result.settled.steady | |
| assert sum(gauge.slept) <= memory.DEFAULT_SETTLE_SECONDS + 1e-9 | |
| assert result.settled.seconds == pytest.approx(memory.DEFAULT_SETTLE_SECONDS) | |
| assert result.committed == result.settled.last == 40.0 + gauge.reads - 1 | |
| assert result.settled.describe() == ( | |
| f"memory still moving after 2.00 s: 40.0 → {result.committed:.1f} GB; " | |
| "judged on the last reading" | |
| ) | |
| # -------------------------------------------------------------------------- | |
| # item 2: a refusal is never based on a falling gauge | |
| # -------------------------------------------------------------------------- | |
| def test_a_failing_reading_is_read_again_and_passes_as_soon_as_one_fits(): | |
| gauge = Gauge(67.3, 66.0, 63.0, 44.8) | |
| result = gauge.judge() | |
| assert result.verdict == "OK" and result.committed == 63.0 | |
| assert gauge.reads == 3, "it stops at the first reading that fits" | |
| assert gauge.slept == [memory.REREAD_STEP_SECONDS] * 2 | |
| assert result.describe() == ( | |
| "preflight[load transformer]: committed 67.3 + need 25.1 = 92.4 GB > 90.0 GB FAIL " | |
| "(short by 2.4); read again for 0.2 s: committed 63.0 + need 25.1 = 88.1 GB " | |
| "<= 90.0 GB OK (headroom 1.9)" | |
| ) | |
| result.require() | |
| def test_a_settle_that_stops_before_the_gauge_starts_falling_is_caught_by_reading_again(): | |
| """Two readings taken before the kernel starts un-wiring agree; the FAIL is read again.""" | |
| gauge = Gauge(67.3, 67.3, 59.3, 46.3, 38.3) | |
| result = gauge.judge(released=True) | |
| assert result.verdict == "OK" and result.committed == 59.3 | |
| assert result.settled.describe() == "memory settled in 0.05 s: 67.3 → 67.3 GB" | |
| assert result.first.committed == 67.3 and result.first.verdict == "FAIL" | |
| assert "; read again for 0.1 s: committed 59.3 + need 25.1" in result.describe() | |
| def test_a_failure_that_persists_is_refused_after_the_bound_with_the_last_reading(released): | |
| gauge = Gauge(67.3, 67.1, 66.9) | |
| result = gauge.judge(released=released) | |
| assert result.verdict == "FAIL" | |
| assert sum(gauge.slept) <= memory.DEFAULT_SETTLE_SECONDS + 1e-9, "slept past the bound" | |
| assert sum(gauge.slept) == pytest.approx(memory.DEFAULT_SETTLE_SECONDS) | |
| assert result.committed == 66.9, "the refusal carries the last reading" | |
| line = result.describe() | |
| first = "committed 67.3 + need 25.1 = 92.4 GB > 90.0 GB FAIL (short by 2.4)" | |
| last = "committed 66.9 + need 25.1 = 92.0 GB > 90.0 GB FAIL (short by 2.0)" | |
| if released: | |
| # 67.3 then 67.1 agree within 0.25 GB: the settle stops there, and that fails first. | |
| assert result.settled.first == 67.3 and result.settled.last == 67.1 | |
| first = "committed 67.1 + need 25.1 = 92.2 GB > 90.0 GB FAIL (short by 2.2)" | |
| assert line.startswith(f"preflight[load transformer]: {first}; read again for ") | |
| assert line.endswith(f": {last}") | |
| with pytest.raises(memory.MemoryPreflightError) as caught: | |
| result.require() | |
| message = str(caught.value) | |
| assert "66.9 GB is already in use" in message and "67.3" not in message | |
| assert "LTX25_MEMORY_LIMIT_GB" in message | |
| # -------------------------------------------------------------------------- | |
| # what does not change | |
| # -------------------------------------------------------------------------- | |
| def test_no_gauge_is_report_only_exactly_as_before(monkeypatch, released): | |
| monkeypatch.setenv(SETTLE, "not even a number") # never read without a gauge | |
| gauge = Gauge(None) | |
| result = gauge.judge("x", need_gb=5.0, released=released) | |
| assert result.verdict == "REPORT" | |
| assert ( | |
| result.describe() | |
| == "preflight[x]: committed n/a + need 5.0 GB (not measured here; report only)" | |
| ) | |
| assert (gauge.reads, gauge.slept, result.settled, result.first) == (1, [], None, None) | |
| def test_no_limit_is_report_only_exactly_as_before(): | |
| gauge = Gauge(95.0, 40.0) | |
| result = gauge.judge("x", need_gb=5.0, limit_gb=None, released=True) | |
| assert result.verdict == "REPORT" | |
| assert ( | |
| result.describe() | |
| == "preflight[x]: committed 95.0 + need 5.0 = 100.0 GB (no limit; report only)" | |
| ) | |
| assert (gauge.reads, gauge.slept) == (1, []) | |
| def test_a_projected_judgement_never_reads_the_gauge_or_waits(): | |
| gauge = Gauge(10.0) | |
| result = gauge.judge(committed_gb=95.0, released=True) | |
| assert result.verdict == "FAIL" and result.committed == 95.0 | |
| assert (gauge.reads, gauge.slept, result.first) == (0, [], None) | |
| def test_zero_seconds_is_the_single_reading_of_before(monkeypatch): | |
| monkeypatch.setenv(SETTLE, "0") | |
| gauge = Gauge(67.3, 38.3) | |
| result = gauge.judge(released=True) | |
| assert result.verdict == "FAIL" and result.committed == 67.3 | |
| assert (gauge.reads, gauge.slept, result.settled, result.first) == (1, [], None, None) | |
| assert result.describe() == ( | |
| "preflight[load transformer]: committed 67.3 + need 25.1 = 92.4 GB > 90.0 GB FAIL " | |
| "(short by 2.4)" | |
| ) | |
| def test_the_bound_comes_from_the_environment(monkeypatch): | |
| monkeypatch.setenv(SETTLE, "0.5") | |
| gauge = Gauge(67.3) | |
| result = gauge.judge(released=True) | |
| assert result.verdict == "FAIL" | |
| assert sum(gauge.slept) == pytest.approx(0.5) | |
| def test_less_than_a_millisecond_left_of_the_bound_is_no_time_to_wait(monkeypatch): | |
| """A wait that kept asking for the last 0.4 ms never ended on a clock that did not move.""" | |
| monkeypatch.setenv(SETTLE, "0.2004") | |
| gauge = CoarseClock(67.3) | |
| result = gauge.judge(released=True) | |
| assert result.verdict == "FAIL" | |
| assert gauge.slept == [0.05, 0.1, pytest.approx(0.0504)] | |
| assert sum(gauge.slept) <= 0.2004 + 1e-9 | |
| def test_a_bad_settle_value_is_refused_like_the_other_memory_variables(monkeypatch, value): | |
| monkeypatch.setenv(SETTLE, value) | |
| with pytest.raises(ConfigError, match=SETTLE) as caught: | |
| memory.settle_seconds() | |
| assert caught.value.variable == SETTLE and caught.value.value == value | |
| with pytest.raises(ConfigError, match=SETTLE): | |
| Gauge(40.0).judge(released=True) | |
| def test_every_settle_value_from_zero_to_a_minute_is_accepted(monkeypatch, value, seconds): | |
| monkeypatch.setenv(SETTLE, value) | |
| assert memory.settle_seconds() == seconds | |
| def test_the_defaults_are_the_measured_ones(): | |
| assert memory.settle_seconds() == memory.DEFAULT_SETTLE_SECONDS == 2.0 | |
| assert memory.SETTLE_STEP_SECONDS == 0.05 | |
| assert memory.SETTLE_TOLERANCE_GB == 0.25 | |
| assert memory.REREAD_STEP_SECONDS == 0.1 | |
| def test_a_judgement_read_again_survives_the_fork_boundary(): | |
| result = Gauge(67.3, 67.3, 66.9).judge(released=True) | |
| assert result.first is not None and result.settled is not None | |
| back = pickle.loads(pickle.dumps(result)) | |
| error = memory.MemoryPreflightError(result) | |
| clone = pickle.loads(pickle.dumps(error)) | |
| assert back == result and back.describe() == result.describe() | |
| assert type(clone) is memory.MemoryPreflightError and str(clone) == str(error) | |
| assert clone.result == result | |
| def test_the_live_judgement_reads_the_modules_gauge_on_the_modules_clock(monkeypatch): | |
| """What the runner uses when nothing is injected: ``committed_gb``, ``_clock``, ``_sleep``.""" | |
| gauge = Gauge(62.3, 38.3, 38.3) | |
| monkeypatch.setattr(memory, "committed_gb", lambda **_: gauge.read()) | |
| monkeypatch.setattr(memory, "_clock", gauge.clock) | |
| monkeypatch.setattr(memory, "_sleep", gauge.sleep) | |
| result = memory.preflight("load transformer", need_gb=25.1, limit_gb=90.0, released=True) | |
| assert result.committed == 38.3 and gauge.reads == 3 | |
| assert gauge.slept == [memory.SETTLE_STEP_SECONDS] * 2 | |