Spaces:
Running on Zero
Running on Zero
File size: 12,968 Bytes
ba2d582 a5caa41 ba2d582 a5caa41 ba2d582 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 | """Spec: a judgement on the live gauge is made on a settled gauge (F73, items 1 and 2).
After the local runner frees device memory, the kernel un-wires its pages over a
few hundred milliseconds (measured on the operator's Mac, 2026-11-16: 24 GB of
text encoder on MPS, the gauge read every 0.1 s after the release gave 62.3,
59.3, 46.3, 38.3 GB, then flat). The transformer's preflight read it inside that
window and refused a run that fitted (committed 67.3 + need 25.1 = 92.4 GB > 90;
eight seconds later the gauge read 38.8). So, on the live gauge only:
* after a release, the gauge is read every 50 ms until two consecutive readings
are within 0.25 GB, and that reading is the one judged and logged;
* a FAIL is read again every 0.1 s and passes as soon as a reading fits; only a
failure that persists is refused, with the last reading;
* both waits share ``LTX25_MEMORY_SETTLE_SECONDS`` (default 2.0; 0 switches them
off). A projected judgement, no gauge or no limit: one reading or none, as
before.
Every gauge here is a scripted sequence on a clock that only sleeping moves, so
nothing really sleeps. Runs in both environments (no torch).
"""
from __future__ import annotations
import pickle
import pytest
from ltx25_studio import memory
from ltx25_studio.errors import ConfigError
SETTLE = "LTX25_MEMORY_SETTLE_SECONDS"
@pytest.fixture(autouse=True)
def _defaults(monkeypatch):
monkeypatch.delenv(SETTLE, raising=False)
monkeypatch.delenv("LTX25_MEMORY_LIMIT_GB", raising=False)
class Gauge:
"""Scripted readings (the last one repeats) on a clock that only sleeping moves."""
def __init__(self, *readings):
self.readings = list(readings)
self.reads = 0
self.now = 0.0
self.slept: list[float] = []
def read(self):
value = self.readings[min(self.reads, len(self.readings) - 1)]
self.reads += 1
return value
def clock(self):
return self.now
def sleep(self, seconds):
assert seconds > 0
self.slept.append(seconds)
self.now += seconds
def judge(self, phase="load transformer", *, need_gb=25.1, limit_gb=90.0, **extra):
return memory.preflight(
phase,
need_gb=need_gb,
limit_gb=limit_gb,
read=self.read,
clock=self.clock,
sleep=self.sleep,
**extra,
)
class CoarseClock(Gauge):
"""A clock that moves in whole milliseconds: a shorter sleep does not move it."""
def sleep(self, seconds):
assert len(self.slept) < 10_000, "the judgement kept waiting on a clock that did not move"
self.slept.append(seconds)
self.now += round(seconds, 3)
class Rising(Gauge):
"""A gauge that never settles: 1 GB more at every reading."""
def __init__(self, start):
super().__init__(start)
self.start = start
def read(self):
self.reads += 1
return self.start + self.reads - 1
# --------------------------------------------------------------------------
# item 1: after a release, the settled reading is the one judged and logged
# --------------------------------------------------------------------------
def test_after_a_release_the_gauge_is_read_until_it_stops_falling_and_that_reading_is_judged():
gauge = Gauge(62.3, 59.3, 46.3, 38.3, 38.3)
result = gauge.judge(released=True)
assert result.committed == 38.3 and result.verdict == "OK"
assert result.settled.describe() == "memory settled in 0.20 s: 62.3 → 38.3 GB"
assert result.describe() == (
"preflight[load transformer]: committed 38.3 + need 25.1 = 63.4 GB <= 90.0 GB OK "
"(headroom 26.6)"
)
assert gauge.reads == 5 and gauge.slept == [memory.SETTLE_STEP_SECONDS] * 4
def test_the_sessions_refused_run_is_judged_on_the_settled_gauge():
"""Run 1 of the 19:02 session: 67.3 GB right after the release, 38.8 once settled."""
gauge = Gauge(67.3, 64.1, 51.2, 40.1, 38.8, 38.8)
result = gauge.judge(released=True)
assert result.verdict == "OK"
assert result.committed == 38.8
assert result.settled.first == 67.3 and result.settled.last == 38.8
assert result.settled.steady
result.require()
def test_a_release_followed_by_a_reading_that_fits_waits_only_for_the_settle():
gauge = Gauge(40.0, 40.1)
result = gauge.judge(released=True)
assert result.verdict == "OK" and result.committed == 40.1
assert gauge.slept == [memory.SETTLE_STEP_SECONDS], "one settle step, no reading again"
assert result.first is None
def test_without_a_release_a_reading_that_fits_is_one_reading_and_no_wait():
gauge = Gauge(62.3, 38.3)
result = gauge.judge()
assert result.committed == 62.3 and result.verdict == "OK"
assert (gauge.reads, gauge.slept, result.settled) == (1, [], None)
def test_the_settle_is_bounded_and_says_so_when_the_gauge_keeps_moving():
gauge = Rising(40.0)
result = gauge.judge(released=True, limit_gb=500.0)
assert result.verdict == "OK"
assert not result.settled.steady
assert sum(gauge.slept) <= memory.DEFAULT_SETTLE_SECONDS + 1e-9
assert result.settled.seconds == pytest.approx(memory.DEFAULT_SETTLE_SECONDS)
assert result.committed == result.settled.last == 40.0 + gauge.reads - 1
assert result.settled.describe() == (
f"memory still moving after 2.00 s: 40.0 → {result.committed:.1f} GB; "
"judged on the last reading"
)
# --------------------------------------------------------------------------
# item 2: a refusal is never based on a falling gauge
# --------------------------------------------------------------------------
def test_a_failing_reading_is_read_again_and_passes_as_soon_as_one_fits():
gauge = Gauge(67.3, 66.0, 63.0, 44.8)
result = gauge.judge()
assert result.verdict == "OK" and result.committed == 63.0
assert gauge.reads == 3, "it stops at the first reading that fits"
assert gauge.slept == [memory.REREAD_STEP_SECONDS] * 2
assert result.describe() == (
"preflight[load transformer]: committed 67.3 + need 25.1 = 92.4 GB > 90.0 GB FAIL "
"(short by 2.4); read again for 0.2 s: committed 63.0 + need 25.1 = 88.1 GB "
"<= 90.0 GB OK (headroom 1.9)"
)
result.require()
def test_a_settle_that_stops_before_the_gauge_starts_falling_is_caught_by_reading_again():
"""Two readings taken before the kernel starts un-wiring agree; the FAIL is read again."""
gauge = Gauge(67.3, 67.3, 59.3, 46.3, 38.3)
result = gauge.judge(released=True)
assert result.verdict == "OK" and result.committed == 59.3
assert result.settled.describe() == "memory settled in 0.05 s: 67.3 → 67.3 GB"
assert result.first.committed == 67.3 and result.first.verdict == "FAIL"
assert "; read again for 0.1 s: committed 59.3 + need 25.1" in result.describe()
@pytest.mark.parametrize("released", [False, True])
def test_a_failure_that_persists_is_refused_after_the_bound_with_the_last_reading(released):
gauge = Gauge(67.3, 67.1, 66.9)
result = gauge.judge(released=released)
assert result.verdict == "FAIL"
assert sum(gauge.slept) <= memory.DEFAULT_SETTLE_SECONDS + 1e-9, "slept past the bound"
assert sum(gauge.slept) == pytest.approx(memory.DEFAULT_SETTLE_SECONDS)
assert result.committed == 66.9, "the refusal carries the last reading"
line = result.describe()
first = "committed 67.3 + need 25.1 = 92.4 GB > 90.0 GB FAIL (short by 2.4)"
last = "committed 66.9 + need 25.1 = 92.0 GB > 90.0 GB FAIL (short by 2.0)"
if released:
# 67.3 then 67.1 agree within 0.25 GB: the settle stops there, and that fails first.
assert result.settled.first == 67.3 and result.settled.last == 67.1
first = "committed 67.1 + need 25.1 = 92.2 GB > 90.0 GB FAIL (short by 2.2)"
assert line.startswith(f"preflight[load transformer]: {first}; read again for ")
assert line.endswith(f": {last}")
with pytest.raises(memory.MemoryPreflightError) as caught:
result.require()
message = str(caught.value)
assert "66.9 GB is already in use" in message and "67.3" not in message
assert "LTX25_MEMORY_LIMIT_GB" in message
# --------------------------------------------------------------------------
# what does not change
# --------------------------------------------------------------------------
@pytest.mark.parametrize("released", [False, True])
def test_no_gauge_is_report_only_exactly_as_before(monkeypatch, released):
monkeypatch.setenv(SETTLE, "not even a number") # never read without a gauge
gauge = Gauge(None)
result = gauge.judge("x", need_gb=5.0, released=released)
assert result.verdict == "REPORT"
assert (
result.describe()
== "preflight[x]: committed n/a + need 5.0 GB (not measured here; report only)"
)
assert (gauge.reads, gauge.slept, result.settled, result.first) == (1, [], None, None)
def test_no_limit_is_report_only_exactly_as_before():
gauge = Gauge(95.0, 40.0)
result = gauge.judge("x", need_gb=5.0, limit_gb=None, released=True)
assert result.verdict == "REPORT"
assert (
result.describe()
== "preflight[x]: committed 95.0 + need 5.0 = 100.0 GB (no limit; report only)"
)
assert (gauge.reads, gauge.slept) == (1, [])
def test_a_projected_judgement_never_reads_the_gauge_or_waits():
gauge = Gauge(10.0)
result = gauge.judge(committed_gb=95.0, released=True)
assert result.verdict == "FAIL" and result.committed == 95.0
assert (gauge.reads, gauge.slept, result.first) == (0, [], None)
def test_zero_seconds_is_the_single_reading_of_before(monkeypatch):
monkeypatch.setenv(SETTLE, "0")
gauge = Gauge(67.3, 38.3)
result = gauge.judge(released=True)
assert result.verdict == "FAIL" and result.committed == 67.3
assert (gauge.reads, gauge.slept, result.settled, result.first) == (1, [], None, None)
assert result.describe() == (
"preflight[load transformer]: committed 67.3 + need 25.1 = 92.4 GB > 90.0 GB FAIL "
"(short by 2.4)"
)
def test_the_bound_comes_from_the_environment(monkeypatch):
monkeypatch.setenv(SETTLE, "0.5")
gauge = Gauge(67.3)
result = gauge.judge(released=True)
assert result.verdict == "FAIL"
assert sum(gauge.slept) == pytest.approx(0.5)
def test_less_than_a_millisecond_left_of_the_bound_is_no_time_to_wait(monkeypatch):
"""A wait that kept asking for the last 0.4 ms never ended on a clock that did not move."""
monkeypatch.setenv(SETTLE, "0.2004")
gauge = CoarseClock(67.3)
result = gauge.judge(released=True)
assert result.verdict == "FAIL"
assert gauge.slept == [0.05, 0.1, pytest.approx(0.0504)]
assert sum(gauge.slept) <= 0.2004 + 1e-9
@pytest.mark.parametrize("value", ["soon", "-1", "nan", "inf", "61", "1e9"])
def test_a_bad_settle_value_is_refused_like_the_other_memory_variables(monkeypatch, value):
monkeypatch.setenv(SETTLE, value)
with pytest.raises(ConfigError, match=SETTLE) as caught:
memory.settle_seconds()
assert caught.value.variable == SETTLE and caught.value.value == value
with pytest.raises(ConfigError, match=SETTLE):
Gauge(40.0).judge(released=True)
@pytest.mark.parametrize(
("value", "seconds"), [("0", 0.0), ("0.25", 0.25), ("2", 2.0), ("60", 60.0)]
)
def test_every_settle_value_from_zero_to_a_minute_is_accepted(monkeypatch, value, seconds):
monkeypatch.setenv(SETTLE, value)
assert memory.settle_seconds() == seconds
def test_the_defaults_are_the_measured_ones():
assert memory.settle_seconds() == memory.DEFAULT_SETTLE_SECONDS == 2.0
assert memory.SETTLE_STEP_SECONDS == 0.05
assert memory.SETTLE_TOLERANCE_GB == 0.25
assert memory.REREAD_STEP_SECONDS == 0.1
def test_a_judgement_read_again_survives_the_fork_boundary():
result = Gauge(67.3, 67.3, 66.9).judge(released=True)
assert result.first is not None and result.settled is not None
back = pickle.loads(pickle.dumps(result))
error = memory.MemoryPreflightError(result)
clone = pickle.loads(pickle.dumps(error))
assert back == result and back.describe() == result.describe()
assert type(clone) is memory.MemoryPreflightError and str(clone) == str(error)
assert clone.result == result
def test_the_live_judgement_reads_the_modules_gauge_on_the_modules_clock(monkeypatch):
"""What the runner uses when nothing is injected: ``committed_gb``, ``_clock``, ``_sleep``."""
gauge = Gauge(62.3, 38.3, 38.3)
monkeypatch.setattr(memory, "committed_gb", lambda **_: gauge.read())
monkeypatch.setattr(memory, "_clock", gauge.clock)
monkeypatch.setattr(memory, "_sleep", gauge.sleep)
result = memory.preflight("load transformer", need_gb=25.1, limit_gb=90.0, released=True)
assert result.committed == 38.3 and gauge.reads == 3
assert gauge.slept == [memory.SETTLE_STEP_SECONDS] * 2
|