File size: 14,532 Bytes
ed13532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d229cdc
ed13532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d229cdc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ed13532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5e06a43
 
 
 
 
 
 
 
 
 
ed13532
 
 
 
 
 
 
 
 
 
 
5e06a43
 
ed13532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5e06a43
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
"""The diffusion decoder is built once, at start-up, with the other parts (#17).

On the Space the default decode is the diffusion decoder. It used to be built
lazily inside each GPU worker and lost with it, so every run loaded its 0.83 GB
and resolved its NATTEN kernel again. ``app.py`` now builds it at start-up, after
the pipeline and before ``demo.launch()``, so its ``.to("cuda")`` is registered
for packing with the rest; each worker's runner finds that instance through its
decoder factory. A failed start-up build logs one line and leaves today's per-run
build in place.

Two tiers: the CUDA path against the fake diffusers tree the stage-graph spec
uses (``.to("cuda")``, the NATTEN processor), and the worker path end to end on
the real tiny checkpoint, through ``generate()``.
"""

from __future__ import annotations

import ast
import logging

import pytest
from conftest import REPO_ROOT
from test_stage_graph_contract import FakeModel, LoadRecorder, install_fake_stack

from ltx25_studio import config, model_files, pipelines

DECODER = "LTX2VideoDiffusionDecoderModel"


@pytest.fixture(autouse=True)
def _no_cached_pipelines():
    pipelines.clear_cache()
    yield
    pipelines.clear_cache()


# --------------------------------------------------------------------------
# the CUDA path, on the fake diffusers tree
# --------------------------------------------------------------------------


class DecodePipelineRecorder:
    """``LTX2VideoDiffusionDecodePipeline``: records each build, returns a marked object."""

    def __init__(self):
        self.built = []

    def __call__(self, **kwargs):
        made = FakeModel()
        made.parts = kwargs
        self.built.append(made)
        return made


class NattenOnce:
    """The NATTEN processor: the first construction fails as a Hub outage would."""

    def __init__(self):
        self.calls = 0

    def __call__(self):
        self.calls += 1
        if self.calls == 1:
            raise RuntimeError("Cannot reach the Hub to resolve shi-labs/natten")
        return "natten-processor"


@pytest.fixture
def space(tmp_path, monkeypatch):
    """The ZeroGPU start-up's pieces on the fake tree: a cache holding the decoder at
    its own revision, the two runners ``app.py`` builds, and the recorders."""

    def build(**classes):
        monkeypatch.delenv(config.ENV_MODEL_SNAPSHOT_DIR, raising=False)
        monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
        repo = tmp_path / f"models--{model_files.REPO_ID.replace('/', '--')}" / "snapshots"
        snapshot = repo / model_files.REVISION
        snapshot.mkdir(parents=True)
        (repo / model_files.DIFFUSION_DECODER_REVISION / "diffusion_decoder").mkdir(parents=True)
        model = FakeModel()
        decode = classes.pop("LTX2VideoDiffusionDecodePipeline", None) or DecodePipelineRecorder()
        base = FakeModel()
        base.components = {}  # what `derive_pipeline` shares into the text2video runner
        recorders = install_fake_stack(
            monkeypatch,
            snapshot,
            LTX2ConditionPipeline=LoadRecorder(result=base),
            LTX2VideoDiffusionDecoderModel=LoadRecorder(result=model),
            LTX2VideoDiffusionDecodePipeline=decode,
            **classes,
        )
        runners = {
            kind: pipelines.get_pipeline(pipeline=kind, device="cuda")
            for kind in ("condition", "text2video")
        }
        return model, decode, recorders, runners

    return build


def test_the_startup_build_loads_the_decoder_once_and_moves_it_to_cuda(space, caplog):
    model, decode, recorders, _runners = space()
    assert not recorders[DECODER].calls, "building the runners must still not load the decoder"

    with caplog.at_level(logging.INFO, logger="ltx25_studio"):
        built = pipelines.build_decoder_at_startup(device="cuda")

    assert built is True
    assert len(recorders[DECODER].calls) == 1
    assert model.moved_to == "cuda", "the .to('cuda') that ZeroGPU registers for packing"
    assert model.processor is not None, "the NATTEN processor is set at start-up too"
    assert model.tiled is True
    assert len(decode.built) == 1
    lines = [r.getMessage() for r in caplog.records if "diffusion decoder" in r.getMessage()]
    assert any("built at start-up" in line and "GiB" in line for line in lines), lines


def test_each_runner_finds_the_startup_decoder_instead_of_building_one(space):
    _model, decode, recorders, runners = space()
    pipelines.build_decoder_at_startup(device="cuda")
    startup_decoder = decode.built[0]

    for runner in runners.values():
        decoder, why_conv = runner._single_stage_decoder("auto", {})
        assert (decoder, why_conv) == ("diffusion", None)
        assert runner.decoder is startup_decoder

    assert len(recorders[DECODER].calls) == 1, "a run loaded the decoder again"
    assert len(decode.built) == 1


def test_a_second_startup_call_builds_nothing_more(space):
    _model, decode, recorders, _runners = space()

    assert pipelines.build_decoder_at_startup(device="cuda") is True
    assert pipelines.build_decoder_at_startup(device="cuda") is True

    assert len(recorders[DECODER].calls) == 1 and len(decode.built) == 1


def test_a_failed_startup_build_logs_one_line_and_the_run_builds_its_own(space, caplog):
    natten = NattenOnce()
    _model, decode, recorders, runners = space(
        LTX2VideoVaeNeighborhoodNattenProcessor=natten
    )

    with caplog.at_level(logging.INFO, logger="ltx25_studio"):
        built = pipelines.build_decoder_at_startup(device="cuda")

    assert built is False
    warnings = [r for r in caplog.records if r.levelno >= logging.WARNING]
    assert len(warnings) == 1, [r.getMessage() for r in warnings]
    message = warnings[0].getMessage()
    assert "NOT built at start-up" in message and "each run builds its own" in message, message
    assert "Cannot reach the Hub" in message and "\n" not in message, message
    assert not decode.built, "a half-built decoder was kept"

    decoder, why_conv = runners["text2video"]._single_stage_decoder("auto", {})

    assert (decoder, why_conv) == ("diffusion", None), "the per-run build is gone"
    assert len(recorders[DECODER].calls) == 2 and len(decode.built) == 1


class WrapperOnce(DecodePipelineRecorder):
    """``LTX2VideoDiffusionDecodePipeline`` whose first construction fails."""

    def __call__(self, **kwargs):
        if not getattr(self, "failed", False):
            self.failed = True
            raise TypeError("LTX2VideoDiffusionDecodePipeline got an unexpected component")
        return super().__call__(**kwargs)


def test_a_failure_after_the_load_leaves_the_decoder_unmoved_and_the_run_builds_it(space):
    """ZeroGPU packs whatever is moved at start-up and then prunes the file it came from.
    A decoder moved but never cached would leave the per-run build a pruned file, so the
    move comes last: a build that fails at any step after the load has moved nothing."""
    wrapper = WrapperOnce()
    model, _decode, recorders, runners = space(LTX2VideoDiffusionDecodePipeline=wrapper)

    assert pipelines.build_decoder_at_startup(device="cuda") is False

    assert model.moved_to is None, "the decoder was moved to cuda by a build that then failed"
    assert "__decoder__" not in pipelines._CACHE

    decoder, why_conv = runners["condition"]._single_stage_decoder("auto", {})

    assert (decoder, why_conv) == ("diffusion", None)
    assert runners["condition"].decoder is wrapper.built[0]
    assert model.moved_to == "cuda" and len(recorders[DECODER].calls) == 2


def test_a_failed_startup_build_never_raises(space, monkeypatch):
    """Whatever goes wrong, start-up goes on: the Space must come up as before."""
    _model, _decode, _recorders, _runners = space()

    def broken(*args, **kwargs):
        raise ImportError("kernels is not installed")

    monkeypatch.setattr(pipelines, "_decoder_part", broken)

    assert pipelines.build_decoder_at_startup(device="cuda") is False


def test_the_app_builds_the_decoder_at_module_scope_after_the_pipeline():
    """Where ``app.py`` asks for it: in the CUDA branch, after both runners, at module
    scope -- inside ``@spaces.GPU`` it would never be packed."""
    tree = ast.parse((REPO_ROOT / "app.py").read_text())
    branch = next(
        node
        for node in tree.body
        if isinstance(node, ast.If) and "get_pipeline" in ast.unparse(node)
    )
    calls = [
        ast.unparse(statement.value)
        for statement in branch.body
        if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Call)
    ]
    assert calls[-1] == "pipelines.build_decoder_at_startup(device='cuda')", calls
    assert sum("get_pipeline" in call for call in calls[:-1]) == 2, calls


def test_the_startup_report_names_the_decoder_beside_the_pipeline(monkeypatch):
    from ltx25_studio import device, zerogpu

    monkeypatch.setattr(zerogpu, "host_has_cuda", lambda: True)
    monkeypatch.setenv(config.ENV_DEVICE, "cuda")
    runtime_config = config.load_config()
    report = pipelines.startup_report(
        runtime_config, missing_files=(), resolution=device.resolve(None, runtime_config)
    )

    line = next(line for line in report.splitlines() if line.strip().startswith("runner"))
    assert "diffusion decoder" in line and "start" in line, line


# --------------------------------------------------------------------------
# the worker path, end to end, on the real tiny checkpoint
# --------------------------------------------------------------------------


def _tiny_composite(tmp_path_factory):
    pytest.importorskip("torch")
    pytest.importorskip("diffusers")
    from test_real_pipeline_tiny import _tiny_snapshot_cached, load_smoke

    if not _tiny_snapshot_cached():
        pytest.skip(f"{model_files.TINY_REPO_ID} is not in the HF cache")
    smoke = load_smoke()
    snapshot, missing = smoke.composite_snapshot(
        smoke.tiny_snapshot(fetch=False), tmp_path_factory.mktemp("tiny") / "snapshot", fetch=False
    )
    if missing:
        pytest.skip(missing)
    return snapshot


@pytest.fixture(scope="module")
def tiny_composite_dir(tmp_path_factory):
    return _tiny_composite(tmp_path_factory)


@pytest.fixture
def tiny_space(tiny_composite_dir, monkeypatch):
    """``app.py``'s start-up on the tiny checkpoint, on the CPU: both runners, then the
    decoder. Records every decoder load and every decode call."""
    from diffusers.pipelines.ltx2.pipeline_ltx2_diffusion_decode import (
        LTX2VideoDiffusionDecodePipeline,
    )

    monkeypatch.setenv(config.ENV_MODEL_SNAPSHOT_DIR, str(tiny_composite_dir))
    # The process's device, as LTX25_DEVICE sets it: the worker's own runner lookup
    # (`get_pipeline` with no device) resolves it.
    monkeypatch.setenv(config.ENV_DEVICE, "cpu")
    loads, decodes = [], []
    real_load = pipelines._load_part

    def counting_load(cls, component, **kwargs):
        if component == "diffusion_decoder":
            loads.append(component)
        return real_load(cls, component, **kwargs)

    monkeypatch.setattr(pipelines, "_load_part", counting_load)
    real_call = LTX2VideoDiffusionDecodePipeline.__call__

    def recording_call(self, *args, **kwargs):
        decodes.append(self)
        return real_call(self, *args, **kwargs)

    monkeypatch.setattr(LTX2VideoDiffusionDecodePipeline, "__call__", recording_call)
    for kind in ("condition", "text2video"):
        pipelines.get_pipeline(pipeline=kind, device="cpu")
    return loads, decodes


def _run(tmp_path, name):
    """One run as the GPU worker makes it: inside the worker context ``app.run_generation``
    sets (so no dispatch another test left registered can intercept it), through
    ``generate()`` and its default runner lookup (``pipelines.get_pipeline``), no factory
    injected. 128x128x9 is the tiny decoder's minimum and within the CPU decode bound."""
    from ltx25_studio import generate

    token = generate._IN_GPU_WORKER.set(True)
    try:
        return generate.generate(
            mode="text_to_video",
            prompt="a lighthouse at dusk",
            seed=7,
            width=128,
            height=128,
            num_frames=9,
            frame_rate=24.0,
            decoder="diffusion",
            output_root=tmp_path / name,
            retention_bytes=10**9,
        )
    finally:
        generate._IN_GPU_WORKER.reset(token)


def _decode_is_timed(result) -> None:
    """The run's time line still has its decode phase, after sampling, and adds up."""
    from test_run_report_lines import _adds_up, took_lines

    (line,) = took_lines(result)
    labels = [label for label, _seconds in _adds_up(line)]
    assert "decoding the frames" in labels, line
    assert labels.index("decoding the frames") > labels.index("sampling"), line


def test_tiny_the_decoder_is_built_once_and_every_run_uses_that_instance(tiny_space, tmp_path):
    loads, decodes = tiny_space

    assert pipelines.build_decoder_at_startup(device="cpu") is True
    startup_decoder = pipelines._CACHE["__decoder__"]
    first = _run(tmp_path, "one")
    second = _run(tmp_path, "two")

    assert loads == ["diffusion_decoder"], f"the decoder was loaded {len(loads)} times"
    assert decodes == [startup_decoder, startup_decoder]
    assert first.output_path.is_file() and second.output_path.is_file()
    _decode_is_timed(first)
    _decode_is_timed(second)


def test_tiny_a_failed_startup_build_falls_back_to_the_per_run_build(
    tiny_space, tmp_path, monkeypatch, caplog
):
    loads, decodes = tiny_space
    real_part = pipelines._decoder_part
    failures = []

    def fails_at_startup(*args, **kwargs):
        if not failures:
            failures.append(1)
            raise OSError("the decoder's file could not be read")
        return real_part(*args, **kwargs)

    monkeypatch.setattr(pipelines, "_decoder_part", fails_at_startup)

    with caplog.at_level(logging.WARNING, logger="ltx25_studio"):
        assert pipelines.build_decoder_at_startup(device="cpu") is False
    result = _run(tmp_path, "one")

    fell_back = [
        r for r in caplog.records if r.getMessage().startswith("diffusion decoder NOT built")
    ]
    assert len(fell_back) == 1, [r.getMessage() for r in caplog.records]
    assert loads == ["diffusion_decoder"], "the run did not build its own decoder"
    assert len(decodes) == 1 and decodes[0] is pipelines._CACHE["__decoder__"]
    assert result.output_path.is_file()
    _decode_is_timed(result)