Spaces:
Running on Zero
Running on Zero
fix: a memory refusal is its own failure category, "memory", with no GPU time used, not a bug in the app
04c841e unverified Download tests/test_errors_memory.py from techfreakworm/LTX2.5-Studio: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_errors_memory.py
- Command line
-
hf download hf://spaces/techfreakworm/LTX2.5-Studio/tests/test_errors_memory.py
-
curl -L -o test_errors_memory.py https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_errors_memory.py
11.3 kB
| """A memory refusal is said as one (F73, item 3). | |
| ``memory.MemoryPreflightError`` inherited the root's category, "bug", and | |
| ``gpu_used = True``, so the session's refused run read "Something went wrong in | |
| this app — Any GPU time this run used is billed. Press Generate again." It has | |
| its own category now, appended to ``errors.CATEGORIES`` the way "limit" and | |
| "queue" were: ``"memory"``, this machine's memory check declined a step and | |
| nothing was loaded for it, so no GPU time was used. | |
| Pinned for a refusal raised at every step of a run by the real staged runner on | |
| the tiny checkpoint, through ``generate()``, and across the dispatch boundary as | |
| ``test_errors_fork_boundary.py`` pins the other errors: the same class, category | |
| and words on the far side of a pickle round trip, bare and in spaces 0.52's | |
| wrapper. | |
| """ | |
| from __future__ import annotations | |
| import html | |
| import pickle | |
| import pytest | |
| from ltx25_studio import errors, memory | |
| from ltx25_studio import generate as generate_module | |
| SIZE = 64 | |
| FRAMES = 9 | |
| SEED = 1234 | |
| PROMPT = "a red ball bouncing on a wooden floor" | |
| #: The category a card is keyed on (a string, so the UI needs no import), and no GPU time. | |
| MEMORY_REFUSAL = errors.Failure("memory", False) | |
| #: The session's refused run, in the words a user reads. | |
| SESSION_WORDS = ( | |
| "Not enough memory for 'load transformer': 67.3 GB is already in use and this step needs " | |
| "about 25.1 GB more, 92.4 GB in all, which is over the 90.0 GB limit by 2.4 GB. Nothing " | |
| "was loaded. Close other apps, pick a smaller size or fewer frames, or raise " | |
| "LTX25_MEMORY_LIMIT_GB if you have measured that this machine can take it." | |
| ) | |
| def _session_refusal(): | |
| return memory.MemoryPreflightError( | |
| memory.preflight("load transformer", need_gb=25.1, limit_gb=90.0, committed_gb=67.3) | |
| ) | |
| # -------------------------------------------------------------------------- | |
| # the category | |
| # -------------------------------------------------------------------------- | |
| def test_memory_is_its_own_category_appended_after_the_others(): | |
| assert errors.MEMORY == "memory" | |
| assert errors.CATEGORIES == ( | |
| "quota", | |
| "timeout", | |
| "oom", | |
| "input", | |
| "adapter", | |
| "enhancer", | |
| "bug", | |
| "limit", | |
| "queue", | |
| "memory", | |
| ) | |
| def test_a_memory_refusal_is_categorised_as_memory_with_no_gpu_time_used(error): | |
| assert (error.category, error.gpu_used) == ("memory", False) | |
| assert errors.categorise(error) == MEMORY_REFUSAL | |
| def test_the_refusal_keeps_its_words_and_the_users_knob(): | |
| assert str(_session_refusal()) == SESSION_WORDS | |
| # -------------------------------------------------------------------------- | |
| # a refusal at each step of a run, from the real staged runner | |
| # -------------------------------------------------------------------------- | |
| class ArmedGauge: | |
| """The gauge reads 200 GB while one step is judged and 30 GB otherwise. | |
| ``_LocalWeights._preflight`` is wrapped only to say which step is being | |
| judged; the judgement, any reading again and the refusal are the real ones. | |
| The clock moves only when the preflight sleeps. | |
| """ | |
| def __init__(self, monkeypatch, step): | |
| from ltx25_studio import local_runtime | |
| self.step = step | |
| self.armed = False | |
| self.now = 0.0 | |
| real = local_runtime._LocalWeights._preflight | |
| def judged(weights, phase, need_gb): | |
| self.armed = phase.startswith(step) | |
| try: | |
| return real(weights, phase, need_gb) | |
| finally: | |
| self.armed = False | |
| monkeypatch.setattr(local_runtime._LocalWeights, "_preflight", judged) | |
| monkeypatch.setattr(memory, "committed_gb", lambda **_: 200.0 if self.armed else 30.0) | |
| monkeypatch.setattr(memory, "_clock", lambda: self.now, raising=False) | |
| monkeypatch.setattr(memory, "_sleep", self.sleep, raising=False) | |
| monkeypatch.setenv("LTX25_MEMORY_LIMIT_GB", "90") | |
| monkeypatch.delenv("LTX25_MEMORY_SETTLE_SECONDS", raising=False) | |
| def sleep(self, seconds): | |
| self.now += seconds | |
| def tiny_runtime(): | |
| pytest.importorskip("torch") | |
| pytest.importorskip("diffusers") | |
| from ltx25_studio import local_runtime | |
| local_runtime.clear_local_cache() | |
| yield local_runtime | |
| local_runtime.clear_local_cache() | |
| def composite(tmp_path_factory): | |
| """The tiny checkpoint plus tiny stage-2 models, as one snapshot directory.""" | |
| pytest.importorskip("torch") | |
| pytest.importorskip("diffusers") | |
| from local_fakes import tiny_snapshot | |
| from test_real_pipeline_tiny import load_smoke | |
| path, missing = load_smoke().composite_snapshot( | |
| tiny_snapshot(), tmp_path_factory.mktemp("composite") / "snapshot", fetch=False | |
| ) | |
| if missing: | |
| pytest.skip(missing) | |
| return path | |
| def _tiny_adapter(tmp_path): | |
| import torch | |
| from local_fakes import tiny_snapshot | |
| from test_real_pipeline_tiny import load_smoke | |
| from ltx25_studio import local_runtime | |
| adapter = tmp_path / "look.safetensors" | |
| donor = local_runtime.load_transformer(tiny_snapshot(), dtype=torch.float32, device="cpu") | |
| load_smoke().make_lora(adapter, donor) | |
| return {"source": str(adapter), "path": str(adapter), "weight": 1.0, "report": object()} | |
| def _request(mode, tmp_path, **fields): | |
| from test_real_pipeline_tiny import load_smoke | |
| media = tmp_path / "media" | |
| media.mkdir(exist_ok=True) | |
| request = dict(mode=mode, seed=SEED, width=SIZE, height=SIZE, num_frames=FRAMES) | |
| request.update(load_smoke()._media_for(mode, media, SIZE, FRAMES)) | |
| request.update(fields) | |
| return request | |
| def _factory(local_runtime, snapshot): | |
| from local_fakes import install_tiny_audio_encode_shim | |
| def factory(*, pipeline, **_): | |
| runner = local_runtime.build_local_runner( | |
| kind=pipeline, | |
| device="cpu", | |
| dtype="float32", | |
| snapshot_dir=snapshot, | |
| residency="staged", | |
| ) | |
| install_tiny_audio_encode_shim(runner.pipeline.audio_vae) | |
| return runner | |
| return factory | |
| #: step -> (the phase it is judged as, mode, extra request fields) | |
| STEPS = { | |
| "small set": ("load small set", "text_to_video", {}), | |
| "text encoder": ("load text encoder", "text_to_video", {}), | |
| "transformer": ("load transformer", "text_to_video", {}), | |
| "adapter attach": ("attach LoRA look.safetensors", "text_to_video", {"loras": "adapter"}), | |
| "clip encode": (f"encode clip {SIZE}x{SIZE}x{FRAMES}", "video_to_video", {}), | |
| "denoise": (f"denoise {SIZE}x{SIZE}x{FRAMES}", "text_to_video", {}), | |
| "stage-2 load": ( | |
| "load latent_upsampler and diffusion_decoder", | |
| "image_to_video", | |
| {"enable_stage_2": True}, | |
| ), | |
| } | |
| def test_a_refusal_at_each_step_of_a_run_is_categorised_as_memory( | |
| step, tiny_runtime, tmp_path, monkeypatch, request | |
| ): | |
| from local_fakes import tiny_snapshot | |
| phase, mode, fields = STEPS[step] | |
| snapshot = tiny_snapshot() | |
| if step == "stage-2 load": | |
| snapshot = request.getfixturevalue("composite") | |
| # The stage-2 parts resolve through the snapshot directory. | |
| monkeypatch.setenv("LTX25_MODEL_SNAPSHOT_DIR", str(snapshot)) | |
| if fields.get("loras") == "adapter": | |
| pytest.importorskip("peft") | |
| fields = {"loras": (_tiny_adapter(tmp_path),)} | |
| ArmedGauge(monkeypatch, phase) | |
| with pytest.raises(memory.MemoryPreflightError) as caught: | |
| generate_module.generate( | |
| **_request(mode, tmp_path, **fields), | |
| pipeline_factory=_factory(tiny_runtime, snapshot), | |
| output_root=tmp_path / "out", | |
| retention_bytes=10**12, | |
| ) | |
| error = caught.value | |
| assert str(error).startswith(f"Not enough memory for '{phase}': 200.0 GB is already in use") | |
| assert "Nothing was loaded." in str(error) and "LTX25_MEMORY_LIMIT_GB" in str(error) | |
| assert errors.categorise(error) == MEMORY_REFUSAL | |
| # -------------------------------------------------------------------------- | |
| # across the dispatch boundary | |
| # -------------------------------------------------------------------------- | |
| class ZeroGPUTaskException(Exception): | |
| """spaces 0.52's wrapper: the worker's own error rides on ``.exception``.""" | |
| def __init__(self, exception): | |
| super().__init__(type(exception).__name__) | |
| self.exception = exception | |
| def _crossed(exc, *, wrapped): | |
| """What the parent raises: the worker's error after the result queue's pickle trip.""" | |
| back = pickle.loads(pickle.dumps(exc)) | |
| return ZeroGPUTaskException(back) if wrapped else back | |
| def test_a_memory_refusal_crosses_the_dispatch_with_its_category_and_words( | |
| monkeypatch, tmp_path, wrapped | |
| ): | |
| raised = _session_refusal() | |
| def dispatch(**_kwargs): | |
| raise _crossed(raised, wrapped=wrapped) | |
| monkeypatch.setattr(generate_module, "_GPU_DISPATCH", dispatch) | |
| with pytest.raises(memory.MemoryPreflightError) as caught: | |
| generate_module.generate(mode="text_to_video", prompt="p", output_root=tmp_path) | |
| arrived = caught.value | |
| assert type(arrived) is memory.MemoryPreflightError and arrived is not raised | |
| assert str(arrived) == SESSION_WORDS | |
| assert arrived.result == raised.result | |
| assert errors.categorise(arrived) == MEMORY_REFUSAL | |
| def test_the_card_shows_the_refusal_word_for_word_on_both_sides_of_the_boundary(): | |
| """The card's headline is the UI's; its detail must be the refusal, before and after.""" | |
| from ltx25_studio.ui import render | |
| raised = _session_refusal() | |
| rendered = render.failure(raised) | |
| assert SESSION_WORDS in rendered or html.escape(SESSION_WORDS) in rendered | |
| assert render.failure(_crossed(raised, wrapped=False)) == rendered | |
| def test_a_refusal_raised_by_the_local_runner_in_the_worker_reaches_the_caller_as_itself( | |
| tiny_runtime, tmp_path, monkeypatch | |
| ): | |
| from local_fakes import tiny_snapshot | |
| ArmedGauge(monkeypatch, "load transformer") | |
| real = generate_module.generate | |
| def dispatch(**kwargs): | |
| token = generate_module._IN_GPU_WORKER.set(True) | |
| try: | |
| return real(**kwargs) | |
| except BaseException as exc: | |
| raise _crossed(exc, wrapped=True) from None | |
| finally: | |
| generate_module._IN_GPU_WORKER.reset(token) | |
| monkeypatch.setattr(generate_module, "_GPU_DISPATCH", dispatch) | |
| with pytest.raises(memory.MemoryPreflightError) as caught: | |
| generate_module.generate( | |
| **_request("text_to_video", tmp_path), | |
| pipeline_factory=_factory(tiny_runtime, tiny_snapshot()), | |
| output_root=tmp_path / "out", | |
| retention_bytes=10**12, | |
| ) | |
| assert str(caught.value).startswith( | |
| "Not enough memory for 'load transformer': 200.0 GB is already in use" | |
| ) | |
| assert errors.categorise(caught.value) == MEMORY_REFUSAL | |