LTX2.5-Studio / tests /test_errors_memory.py
techfreakworm's picture
fix: a memory refusal is its own failure category, "memory", with no GPU time used, not a bug in the app
04c841e unverified
Raw History Blame Contribute Delete
11.3 kB
"""A memory refusal is said as one (F73, item 3).
``memory.MemoryPreflightError`` inherited the root's category, "bug", and
``gpu_used = True``, so the session's refused run read "Something went wrong in
this app — Any GPU time this run used is billed. Press Generate again." It has
its own category now, appended to ``errors.CATEGORIES`` the way "limit" and
"queue" were: ``"memory"``, this machine's memory check declined a step and
nothing was loaded for it, so no GPU time was used.
Pinned for a refusal raised at every step of a run by the real staged runner on
the tiny checkpoint, through ``generate()``, and across the dispatch boundary as
``test_errors_fork_boundary.py`` pins the other errors: the same class, category
and words on the far side of a pickle round trip, bare and in spaces 0.52's
wrapper.
"""
from __future__ import annotations
import html
import pickle
import pytest
from ltx25_studio import errors, memory
from ltx25_studio import generate as generate_module
SIZE = 64
FRAMES = 9
SEED = 1234
PROMPT = "a red ball bouncing on a wooden floor"
#: The category a card is keyed on (a string, so the UI needs no import), and no GPU time.
MEMORY_REFUSAL = errors.Failure("memory", False)
#: The session's refused run, in the words a user reads.
SESSION_WORDS = (
"Not enough memory for 'load transformer': 67.3 GB is already in use and this step needs "
"about 25.1 GB more, 92.4 GB in all, which is over the 90.0 GB limit by 2.4 GB. Nothing "
"was loaded. Close other apps, pick a smaller size or fewer frames, or raise "
"LTX25_MEMORY_LIMIT_GB if you have measured that this machine can take it."
)
def _session_refusal():
return memory.MemoryPreflightError(
memory.preflight("load transformer", need_gb=25.1, limit_gb=90.0, committed_gb=67.3)
)
# --------------------------------------------------------------------------
# the category
# --------------------------------------------------------------------------
def test_memory_is_its_own_category_appended_after_the_others():
assert errors.MEMORY == "memory"
assert errors.CATEGORIES == (
"quota",
"timeout",
"oom",
"input",
"adapter",
"enhancer",
"bug",
"limit",
"queue",
"memory",
)
@pytest.mark.parametrize(
"error",
[
_session_refusal(),
memory.MemoryPreflightError(reason="could not run vm_stat to measure memory: (stand-in)"),
],
ids=["over the limit", "no reading"],
)
def test_a_memory_refusal_is_categorised_as_memory_with_no_gpu_time_used(error):
assert (error.category, error.gpu_used) == ("memory", False)
assert errors.categorise(error) == MEMORY_REFUSAL
def test_the_refusal_keeps_its_words_and_the_users_knob():
assert str(_session_refusal()) == SESSION_WORDS
# --------------------------------------------------------------------------
# a refusal at each step of a run, from the real staged runner
# --------------------------------------------------------------------------
class ArmedGauge:
"""The gauge reads 200 GB while one step is judged and 30 GB otherwise.
``_LocalWeights._preflight`` is wrapped only to say which step is being
judged; the judgement, any reading again and the refusal are the real ones.
The clock moves only when the preflight sleeps.
"""
def __init__(self, monkeypatch, step):
from ltx25_studio import local_runtime
self.step = step
self.armed = False
self.now = 0.0
real = local_runtime._LocalWeights._preflight
def judged(weights, phase, need_gb):
self.armed = phase.startswith(step)
try:
return real(weights, phase, need_gb)
finally:
self.armed = False
monkeypatch.setattr(local_runtime._LocalWeights, "_preflight", judged)
monkeypatch.setattr(memory, "committed_gb", lambda **_: 200.0 if self.armed else 30.0)
monkeypatch.setattr(memory, "_clock", lambda: self.now, raising=False)
monkeypatch.setattr(memory, "_sleep", self.sleep, raising=False)
monkeypatch.setenv("LTX25_MEMORY_LIMIT_GB", "90")
monkeypatch.delenv("LTX25_MEMORY_SETTLE_SECONDS", raising=False)
def sleep(self, seconds):
self.now += seconds
@pytest.fixture
def tiny_runtime():
pytest.importorskip("torch")
pytest.importorskip("diffusers")
from ltx25_studio import local_runtime
local_runtime.clear_local_cache()
yield local_runtime
local_runtime.clear_local_cache()
@pytest.fixture(scope="module")
def composite(tmp_path_factory):
"""The tiny checkpoint plus tiny stage-2 models, as one snapshot directory."""
pytest.importorskip("torch")
pytest.importorskip("diffusers")
from local_fakes import tiny_snapshot
from test_real_pipeline_tiny import load_smoke
path, missing = load_smoke().composite_snapshot(
tiny_snapshot(), tmp_path_factory.mktemp("composite") / "snapshot", fetch=False
)
if missing:
pytest.skip(missing)
return path
def _tiny_adapter(tmp_path):
import torch
from local_fakes import tiny_snapshot
from test_real_pipeline_tiny import load_smoke
from ltx25_studio import local_runtime
adapter = tmp_path / "look.safetensors"
donor = local_runtime.load_transformer(tiny_snapshot(), dtype=torch.float32, device="cpu")
load_smoke().make_lora(adapter, donor)
return {"source": str(adapter), "path": str(adapter), "weight": 1.0, "report": object()}
def _request(mode, tmp_path, **fields):
from test_real_pipeline_tiny import load_smoke
media = tmp_path / "media"
media.mkdir(exist_ok=True)
request = dict(mode=mode, seed=SEED, width=SIZE, height=SIZE, num_frames=FRAMES)
request.update(load_smoke()._media_for(mode, media, SIZE, FRAMES))
request.update(fields)
return request
def _factory(local_runtime, snapshot):
from local_fakes import install_tiny_audio_encode_shim
def factory(*, pipeline, **_):
runner = local_runtime.build_local_runner(
kind=pipeline,
device="cpu",
dtype="float32",
snapshot_dir=snapshot,
residency="staged",
)
install_tiny_audio_encode_shim(runner.pipeline.audio_vae)
return runner
return factory
#: step -> (the phase it is judged as, mode, extra request fields)
STEPS = {
"small set": ("load small set", "text_to_video", {}),
"text encoder": ("load text encoder", "text_to_video", {}),
"transformer": ("load transformer", "text_to_video", {}),
"adapter attach": ("attach LoRA look.safetensors", "text_to_video", {"loras": "adapter"}),
"clip encode": (f"encode clip {SIZE}x{SIZE}x{FRAMES}", "video_to_video", {}),
"denoise": (f"denoise {SIZE}x{SIZE}x{FRAMES}", "text_to_video", {}),
"stage-2 load": (
"load latent_upsampler and diffusion_decoder",
"image_to_video",
{"enable_stage_2": True},
),
}
@pytest.mark.parametrize("step", list(STEPS))
def test_a_refusal_at_each_step_of_a_run_is_categorised_as_memory(
step, tiny_runtime, tmp_path, monkeypatch, request
):
from local_fakes import tiny_snapshot
phase, mode, fields = STEPS[step]
snapshot = tiny_snapshot()
if step == "stage-2 load":
snapshot = request.getfixturevalue("composite")
# The stage-2 parts resolve through the snapshot directory.
monkeypatch.setenv("LTX25_MODEL_SNAPSHOT_DIR", str(snapshot))
if fields.get("loras") == "adapter":
pytest.importorskip("peft")
fields = {"loras": (_tiny_adapter(tmp_path),)}
ArmedGauge(monkeypatch, phase)
with pytest.raises(memory.MemoryPreflightError) as caught:
generate_module.generate(
**_request(mode, tmp_path, **fields),
pipeline_factory=_factory(tiny_runtime, snapshot),
output_root=tmp_path / "out",
retention_bytes=10**12,
)
error = caught.value
assert str(error).startswith(f"Not enough memory for '{phase}': 200.0 GB is already in use")
assert "Nothing was loaded." in str(error) and "LTX25_MEMORY_LIMIT_GB" in str(error)
assert errors.categorise(error) == MEMORY_REFUSAL
# --------------------------------------------------------------------------
# across the dispatch boundary
# --------------------------------------------------------------------------
class ZeroGPUTaskException(Exception):
"""spaces 0.52's wrapper: the worker's own error rides on ``.exception``."""
def __init__(self, exception):
super().__init__(type(exception).__name__)
self.exception = exception
def _crossed(exc, *, wrapped):
"""What the parent raises: the worker's error after the result queue's pickle trip."""
back = pickle.loads(pickle.dumps(exc))
return ZeroGPUTaskException(back) if wrapped else back
@pytest.mark.parametrize("wrapped", [False, True], ids=["spaces 0.51", "spaces 0.52"])
def test_a_memory_refusal_crosses_the_dispatch_with_its_category_and_words(
monkeypatch, tmp_path, wrapped
):
raised = _session_refusal()
def dispatch(**_kwargs):
raise _crossed(raised, wrapped=wrapped)
monkeypatch.setattr(generate_module, "_GPU_DISPATCH", dispatch)
with pytest.raises(memory.MemoryPreflightError) as caught:
generate_module.generate(mode="text_to_video", prompt="p", output_root=tmp_path)
arrived = caught.value
assert type(arrived) is memory.MemoryPreflightError and arrived is not raised
assert str(arrived) == SESSION_WORDS
assert arrived.result == raised.result
assert errors.categorise(arrived) == MEMORY_REFUSAL
def test_the_card_shows_the_refusal_word_for_word_on_both_sides_of_the_boundary():
"""The card's headline is the UI's; its detail must be the refusal, before and after."""
from ltx25_studio.ui import render
raised = _session_refusal()
rendered = render.failure(raised)
assert SESSION_WORDS in rendered or html.escape(SESSION_WORDS) in rendered
assert render.failure(_crossed(raised, wrapped=False)) == rendered
def test_a_refusal_raised_by_the_local_runner_in_the_worker_reaches_the_caller_as_itself(
tiny_runtime, tmp_path, monkeypatch
):
from local_fakes import tiny_snapshot
ArmedGauge(monkeypatch, "load transformer")
real = generate_module.generate
def dispatch(**kwargs):
token = generate_module._IN_GPU_WORKER.set(True)
try:
return real(**kwargs)
except BaseException as exc:
raise _crossed(exc, wrapped=True) from None
finally:
generate_module._IN_GPU_WORKER.reset(token)
monkeypatch.setattr(generate_module, "_GPU_DISPATCH", dispatch)
with pytest.raises(memory.MemoryPreflightError) as caught:
generate_module.generate(
**_request("text_to_video", tmp_path),
pipeline_factory=_factory(tiny_runtime, tiny_snapshot()),
output_root=tmp_path / "out",
retention_bytes=10**12,
)
assert str(caught.value).startswith(
"Not enough memory for 'load transformer': 200.0 GB is already in use"
)
assert errors.categorise(caught.value) == MEMORY_REFUSAL