Spaces:
Running on Zero
Running on Zero
fix: everything a user reads after Generate is in the UI's words, and audio too short for the set length is refused in them
8a7fa0c unverified Download tests/test_encoder_contract.py from techfreakworm/LTX2.5-Studio: direct link, hf CLI and curl.
- Browser
- Download file 22.5 kB
-
https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_encoder_contract.py
- Command line
-
hf download hf://spaces/techfreakworm/LTX2.5-Studio/tests/test_encoder_contract.py
-
curl -L -o test_encoder_contract.py https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_encoder_contract.py
22.5 kB
| """Spec for what ``generate()`` owes the encoder — PLAN §5.3, §5.4, §5.7. | |
| **The injected-encoder seam hides a whole class of defect.** Every existing | |
| generate() spec passes ``encoder=`` a fake, and a fake accepts whatever it is | |
| handed. So ``generate()`` could forward an argument set that ``outputs.write_output`` | |
| cannot actually write with, and every one of those tests would still pass while | |
| the production path failed deterministically for a mode nobody exercised. | |
| That is not hypothetical. ``outputs`` refuses to invent two numbers, both on | |
| purpose (PLAN §5.4): the **sampling rate**, which comes from | |
| ``vocoder.config.output_sampling_rate``, and the **frame rate**, which is the | |
| rate the pipeline generated at. A container written at a wrong or absent rate is | |
| pitch-shifted or plays at the wrong speed — and reports no error at all. So the | |
| writer raises instead, and whatever ``generate()`` fails to thread through | |
| becomes a run that dies at the last step. | |
| This file therefore tests the **real** encoder, not an injected one: | |
| * ``encoder=None`` — the production path — for every mode in the registry, with | |
| real arrays, through the real ``av`` writer, and a real file at the end of it; | |
| * the muxing itself, demuxed back and inspected, because "wrote an mp4" and | |
| "wrote an mp4 with the audio in it" are the same result to everything except | |
| the user's ears (PLAN: a silent video is the §5.3 failure one layer out); | |
| * the refusal, when the pipeline genuinely cannot supply what audio needs. | |
| Still no GPU, no weights, no network: the pipeline is faked, the media stack is | |
| ``av``, which is already a pinned runtime dependency. | |
| """ | |
| from __future__ import annotations | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import pytest | |
| from conftest import FakeAudioVae, audio_file, inputs_for | |
| from test_exports_contract import MODE_IDS, factory_returning | |
| from ltx25_studio import generate as generate_module | |
| from ltx25_studio import modes, outputs | |
| from ltx25_studio.errors import EncodeError | |
| av = pytest.importorskip("av", reason="the media stack is a pinned runtime dependency") | |
| np = pytest.importorskip("numpy") | |
| #: What the fake vocoder reports. Distinctive so a hardcoded 44100 anywhere in | |
| #: the path shows up as a mismatch rather than blending in. | |
| VOCODER_SAMPLING_RATE = 48000 | |
| #: Small enough to keep the suite fast, large enough to be a real encode. | |
| FRAME_COUNT = 2 | |
| FRAME_SIZE = 64 | |
| AUDIO_CHANNELS = 2 | |
| AUDIO_SAMPLES = VOCODER_SAMPLING_RATE // 10 # 0.1s | |
| class RealOutput: | |
| """A pipeline output the real writer can actually encode. | |
| Stereo, because the vocoder is stereo and PLAN forbids a mono downmix | |
| anywhere in the path — a fake that handed back mono could not detect one. | |
| """ | |
| def __init__(self): | |
| self.frames = np.zeros((FRAME_COUNT, FRAME_SIZE, FRAME_SIZE, 3), dtype=np.uint8) | |
| self.audio = np.zeros((AUDIO_CHANNELS, AUDIO_SAMPLES), dtype=np.float32) | |
| class VocoderPipe: | |
| """A pipeline that exposes the vocoder ``generate()`` must read the rate off. | |
| It has an audio VAE too, because the audio-input modes encode their upload. | |
| """ | |
| def __init__(self, sampling_rate=VOCODER_SAMPLING_RATE): | |
| self.vocoder = SimpleNamespace(config=SimpleNamespace(output_sampling_rate=sampling_rate)) | |
| self.audio_vae = FakeAudioVae() | |
| self.calls: list[dict] = [] | |
| def __call__(self, **kwargs): | |
| self.calls.append(kwargs) | |
| return RealOutput() | |
| class VocoderlessPipe: | |
| """A pipeline with nothing to read the sampling rate off (it can still encode an upload).""" | |
| def __init__(self): | |
| self.audio_vae = FakeAudioVae() | |
| def __call__(self, **kwargs): | |
| return RealOutput() | |
| def _uploads_load_without_torch(audio_input_ready): | |
| """The audio modes get a real upload; its loader is stubbed so no torch is needed.""" | |
| def run_production(mode_id, tmp_path, *, pipe=None, **overrides): | |
| """``encoder=None`` — the real ``outputs.write_output``, no fake in the way. | |
| Real inputs for every kind the mode requires. A caller in another module | |
| must also request the ``audio_input_ready`` fixture for the audio modes. | |
| """ | |
| return generate_module.generate( | |
| mode=mode_id, | |
| seed=7, | |
| width=FRAME_SIZE, | |
| height=FRAME_SIZE, | |
| pipeline_factory=factory_returning(pipe if pipe is not None else VocoderPipe()), | |
| encoder=None, | |
| output_root=tmp_path, | |
| retention_bytes=10**9, | |
| **{**inputs_for(mode_id), **overrides}, | |
| ) | |
| # -------------------------------------------------------------------------- | |
| # the production path has to reach a file, for every mode | |
| # -------------------------------------------------------------------------- | |
| def test_the_production_encoder_can_actually_write_this_modes_artifact(mode_id, tmp_path): | |
| """The seam every injected-encoder test steps over. | |
| A fake encoder accepts any argument set. The real one raises on the ones it | |
| was not given, and PLAN §5.4 makes that deliberate: it will not guess a rate | |
| it was supposed to be handed. So this drives the writer for real — if | |
| ``generate()`` stops forwarding something ``write_output`` requires, this is | |
| the test that says so, whichever argument it happens to be. | |
| """ | |
| spec = modes.get_mode(mode_id) | |
| result = run_production(mode_id, tmp_path) | |
| for track in spec.exports: | |
| path = Path(getattr(result, f"{track}_path")) | |
| assert path.is_file() and path.stat().st_size > 0, ( | |
| f"mode {mode_id!r} declares exports={list(spec.exports)} but the production " | |
| f"writer produced no usable {track} file at {path}." | |
| ) | |
| def test_an_audio_exporting_mode_is_handed_the_vocoders_sampling_rate(mode_id, tmp_path): | |
| """PLAN §5.4: the rate comes off the loaded vocoder, never from a default here. | |
| ``outputs._write_audio`` raises on a missing rate rather than assuming one, | |
| so a mode that exports audio and is not handed the rate fails at the very | |
| last step of an otherwise complete — and already paid for — generation. | |
| """ | |
| seen: dict = {} | |
| real_write = outputs.write_output | |
| def spy(output, **kwargs): | |
| seen.update(kwargs) | |
| return real_write(output, **kwargs) | |
| original = outputs.write_output | |
| outputs.write_output = spy | |
| try: | |
| run_production(mode_id, tmp_path) | |
| finally: | |
| outputs.write_output = original | |
| assert seen.get("sampling_rate") == VOCODER_SAMPLING_RATE, ( | |
| f"mode {mode_id!r} exports audio, but the encoder was handed " | |
| f"sampling_rate={seen.get('sampling_rate')!r} instead of the vocoder's " | |
| f"{VOCODER_SAMPLING_RATE}. outputs._write_audio refuses to guess a rate, so this " | |
| f"mode dies on the production path." | |
| ) | |
| def test_a_video_exporting_mode_is_handed_a_frame_rate(mode_id, tmp_path): | |
| """The same trap, the other number. | |
| ``outputs._write_video`` raises on a missing frame rate for exactly the | |
| reason §5.4 gives: a container written at the wrong rate plays at the wrong | |
| speed and reports no error. It is the caller's to supply. | |
| """ | |
| seen: dict = {} | |
| real_write = outputs.write_output | |
| def spy(output, **kwargs): | |
| seen.update(kwargs) | |
| return real_write(output, **kwargs) | |
| original = outputs.write_output | |
| outputs.write_output = spy | |
| try: | |
| run_production(mode_id, tmp_path) | |
| finally: | |
| outputs.write_output = original | |
| frame_rate = seen.get("frame_rate") | |
| assert frame_rate is not None, ( | |
| f"mode {mode_id!r} exports video, but the encoder was handed no frame_rate. " | |
| f"outputs._write_video refuses to assume one, so this mode dies at the last step " | |
| f"of an already-paid-for generation." | |
| ) | |
| assert float(frame_rate) > 0, f"frame_rate must be positive, got {frame_rate!r}" | |
| # -------------------------------------------------------------------------- | |
| # muxing: an mp4 with no audio in it is not a success | |
| # -------------------------------------------------------------------------- | |
| JOINT_MODES = tuple(m for m in MODE_IDS if set(modes.MODES[m].exports) == {"audio", "video"}) | |
| def _streams(path): | |
| with av.open(str(path)) as container: | |
| return ( | |
| [s for s in container.streams if s.type == "video"], | |
| [s for s in container.streams if s.type == "audio"], | |
| ) | |
| def test_a_joint_mode_writes_a_container_that_really_carries_the_audio(mode_id, tmp_path): | |
| """Demuxed and inspected, because the file plays either way. | |
| Writing the frames and dropping the audio produces a video that opens, plays, | |
| and is wrong: no error, no warning, a plausible-looking artifact. That is the | |
| §5.3 silent-failure class one layer further out, and the only way to tell is | |
| to open the file back up. | |
| """ | |
| result = run_production(mode_id, tmp_path) | |
| video_streams, audio_streams = _streams(result.video_path) | |
| assert len(video_streams) == 1, f"{mode_id!r} wrote {len(video_streams)} video streams" | |
| assert len(audio_streams) == 1, ( | |
| f"mode {mode_id!r} exports BOTH audio and video, and the container it wrote at " | |
| f"{result.video_path} has {len(audio_streams)} audio streams. This is a silent video " | |
| f"reported as a success — LTX-2.5's headline capability, dropped without an error." | |
| ) | |
| def test_the_drop_warning_fires_exactly_where_a_track_is_actually_dropped(mode_id, tmp_path): | |
| """``dropped_audio_warning`` had no test at all. Both directions, now pinned. | |
| The model is jointly audio-visual with no video-off switch, so every | |
| video-exporting run renders audio. A mode that does not export it discards | |
| it and MUST say so -- the user paid GPU-seconds for that track. A mode that | |
| does export it keeps it and must NOT say so. | |
| The false direction is the one that just bit: ``audio_to_video`` conditioned | |
| on an audio upload, dropped the generated track, and told the user to "use | |
| the matching audio+video mode to keep it" -- when its ``audio_counterpart`` | |
| is None and no such mode has ever existed. A stale warning naming a remedy | |
| that does not exist sends the user hunting for a mode nobody built, which is | |
| worse than silence, not better. | |
| """ | |
| result = run_production(mode_id, tmp_path) | |
| dropped = [note for note in result.warnings if "was left out" in note] | |
| if "audio" in modes.MODES[mode_id].exports: | |
| assert not dropped, ( | |
| f"{mode_id!r} exports audio and keeps its track, yet reports it as dropped: {dropped}" | |
| ) | |
| else: | |
| assert dropped, ( | |
| f"{mode_id!r} exports video only, so this run rendered an audio track and threw it " | |
| f"away, and the result says nothing about it: {list(result.warnings)}" | |
| ) | |
| def test_the_muxed_audio_track_contains_actual_samples(mode_id, tmp_path): | |
| """An audio stream declared and never written is the same lie with a header.""" | |
| result = run_production(mode_id, tmp_path) | |
| decoded = 0 | |
| with av.open(str(result.video_path)) as container: | |
| stream = next(s for s in container.streams if s.type == "audio") | |
| for frame in container.decode(stream): | |
| decoded += frame.samples | |
| assert decoded > 0, ( | |
| f"mode {mode_id!r} wrote an audio stream into {result.video_path} that decodes to " | |
| f"zero samples — a silent track with a header on it." | |
| ) | |
| def test_the_muxed_audio_keeps_every_channel(mode_id, tmp_path): | |
| """PLAN: the vocoder is stereo and a mono downmix is a silent quality regression.""" | |
| result = run_production(mode_id, tmp_path) | |
| with av.open(str(result.video_path)) as container: | |
| stream = next(s for s in container.streams if s.type == "audio") | |
| channels = stream.codec_context.layout.nb_channels | |
| assert channels == AUDIO_CHANNELS, ( | |
| f"mode {mode_id!r} was handed {AUDIO_CHANNELS}-channel audio and wrote {channels}. " | |
| f"Nothing in this path may downmix." | |
| ) | |
| def test_a_supplied_upload_reaches_the_container_as_48k_stereo_at_its_own_pitch( | |
| tmp_path, monkeypatch | |
| ): | |
| """The mode-11 substitution, end to end through the real ``av`` writer. | |
| Every other test of this substitution stops at the injected-encoder seam, | |
| and a fake accepts whatever it is handed -- which is the exact blind spot | |
| this file exists for. So this one drives ``outputs.write_output`` for real | |
| and opens the file back up. | |
| The upload is a 16 kHz tone and the vocoder reports 48 kHz. The track in the | |
| container is converted to the soundtrack's own format, 48 kHz stereo AAC, | |
| and the tone must come back at the SAME frequency: muxing 16 kHz samples | |
| as if they were 48 kHz would triple the pitch with no error anywhere. | |
| """ | |
| from conftest import STUB_SAMPLING_RATE | |
| from ltx25_studio import audio as audio_module | |
| tone_hz = 440.0 | |
| samples = round(121 / 24.0 * STUB_SAMPLING_RATE) | |
| t = np.arange(samples) / STUB_SAMPLING_RATE | |
| upload = np.stack([0.5 * np.sin(2 * np.pi * tone_hz * t)] * 2).astype(np.float32) | |
| monkeypatch.setattr(audio_module, "load_waveform", lambda source: (upload, STUB_SAMPLING_RATE)) | |
| result = run_production("audio_to_video", tmp_path, audio=audio_file()) | |
| with av.open(str(result.video_path)) as container: | |
| stream = next(s for s in container.streams if s.type == "audio") | |
| rate = stream.codec_context.sample_rate | |
| channels = stream.codec_context.channels | |
| decoded = np.concatenate([frame.to_ndarray() for frame in container.decode(stream)], 1) | |
| assert (rate, channels) == (VOCODER_SAMPLING_RATE, AUDIO_CHANNELS), ( | |
| f"the upload was muxed at {rate} Hz with {channels} channel(s); the soundtrack is " | |
| f"{VOCODER_SAMPLING_RATE} Hz stereo, which is also what verify_streams checks" | |
| ) | |
| spectrum = np.abs(np.fft.rfft(decoded[0] * np.hanning(decoded.shape[1]))) | |
| measured = float(np.fft.rfftfreq(decoded.shape[1], 1.0 / rate)[np.argmax(spectrum)]) | |
| assert abs(measured - tone_hz) <= 1.0, ( | |
| f"a {tone_hz:.0f} Hz upload came back at {measured:.1f} Hz: the track was resampled " | |
| f"wrongly or written at the wrong rate, a pitch shift that reports no error" | |
| ) | |
| assert any("16 kHz" in note and "48 kHz" in note for note in result.warnings), ( | |
| f"the upload's rate was changed and the result does not say so: {result.warnings}" | |
| ) | |
| def test_an_audio_only_mode_writes_a_real_track_at_the_vocoders_rate(mode_id, tmp_path): | |
| """The audio-only export, demuxed: right rate, right channel count, real samples.""" | |
| result = run_production(mode_id, tmp_path) | |
| with av.open(str(result.audio_path)) as container: | |
| stream = next(s for s in container.streams if s.type == "audio") | |
| assert stream.codec_context.sample_rate == VOCODER_SAMPLING_RATE, ( | |
| f"mode {mode_id!r} wrote its audio at {stream.codec_context.sample_rate} Hz " | |
| f"instead of the vocoder's {VOCODER_SAMPLING_RATE} Hz — a pitch-shifted file " | |
| f"that reports no error." | |
| ) | |
| assert stream.codec_context.layout.nb_channels == AUDIO_CHANNELS | |
| assert sum(frame.samples for frame in container.decode(stream)) > 0 | |
| # -------------------------------------------------------------------------- | |
| # the refusal: better a typed error than a guessed number | |
| # -------------------------------------------------------------------------- | |
| def test_an_audio_mode_with_no_vocoder_to_read_refuses_rather_than_assuming(mode_id, tmp_path): | |
| """PLAN §5.4: an assumed rate resamples every export, silently and forever. | |
| The failure must be typed and must say what was missing, so the next person | |
| reads "no vocoder to read output_sampling_rate from" rather than a ``None`` | |
| crashing several layers down inside ``av``. | |
| """ | |
| with pytest.raises(EncodeError) as caught: | |
| run_production(mode_id, tmp_path, pipe=VocoderlessPipe()) | |
| message = str(caught.value) | |
| assert "sampling_rate" in message or "sampling rate" in message, ( | |
| f"the refusal must name the missing rate; got {message!r}" | |
| ) | |
| def test_the_writer_still_refuses_a_frame_rate_it_was_not_given(tmp_path): | |
| """Pins the refusal itself, so the guard cannot be softened into a default. | |
| If ``write_output`` ever grows a fallback frame rate, every wrong-speed video | |
| it then writes becomes undetectable — which is precisely why the number is | |
| the caller's to supply. | |
| """ | |
| with pytest.raises(EncodeError) as caught: | |
| outputs.write_output( | |
| RealOutput(), | |
| run_dir=tmp_path / "run-nofps", | |
| mode="text_to_video", | |
| exports=("video",), | |
| seed=1, | |
| sampling_rate=VOCODER_SAMPLING_RATE, | |
| ) | |
| assert "frame rate" in str(caught.value) | |
| def test_the_writer_still_refuses_a_sampling_rate_it_was_not_given(tmp_path): | |
| """The same guard on the audio side.""" | |
| with pytest.raises(EncodeError) as caught: | |
| outputs.write_output( | |
| RealOutput(), | |
| run_dir=tmp_path / "run-norate", | |
| mode="text_to_audio", | |
| exports=("audio",), | |
| seed=1, | |
| ) | |
| assert "sampling rate" in str(caught.value) | |
| # -------------------------------------------------------------------------- | |
| # the dtype the model actually produces | |
| # | |
| # Every fixture above hands the writer `np.float32`. The pipeline hands it | |
| # `torch.bfloat16` -- WEIGHT_DTYPE, documented as not configurable -- and numpy | |
| # has no bfloat16, so `.numpy()` raised and SIX OF ELEVEN MODES generated | |
| # successfully and died at the last step. 1926 tests passed over it, because the | |
| # fixture that made the tests possible encoded the assumption that hid the bug. | |
| # -------------------------------------------------------------------------- | |
| class BFloat16Tensor: | |
| """A tensor whose dtype numpy cannot represent, behaving as torch does. | |
| Reproduces the exact failure rather than approximating it: `.numpy()` raises | |
| ``TypeError`` the way torch does for bfloat16, and `.float()` is the only way | |
| through. `test_the_bfloat16_fake_is_faithful_to_torch` is what stops this | |
| class quietly drifting into a shape that could not fail. | |
| """ | |
| def __init__(self, array): | |
| self._array = array | |
| self.dtype = "torch.bfloat16" | |
| def detach(self): | |
| return self | |
| def cpu(self): | |
| return self | |
| def numpy(self): | |
| raise TypeError("Got unsupported ScalarType BFloat16") | |
| def float(self): | |
| # torch returns a TENSOR, not an ndarray -- `.float().numpy()` is two | |
| # steps. Returning the array here made the fake unfaithful in exactly | |
| # the direction that would have let a broken downcast pass. | |
| return _Float32Tensor(self._array) | |
| class _Float32Tensor: | |
| """What ``.float()`` gives back: still a tensor, now numpy-representable.""" | |
| def __init__(self, array): | |
| self._array = array | |
| self.dtype = "torch.float32" | |
| def detach(self): | |
| return self | |
| def cpu(self): | |
| return self | |
| def numpy(self): | |
| return self._array | |
| def bfloat16_output(): | |
| out = RealOutput() | |
| out.audio = BFloat16Tensor(out.audio) | |
| return out | |
| class BFloat16Pipe(VocoderPipe): | |
| """The pipeline as it really behaves: audio comes back as bfloat16.""" | |
| def __call__(self, **kwargs): | |
| self.calls.append(kwargs) | |
| return bfloat16_output() | |
| def test_an_audio_mode_can_be_written_when_the_audio_is_bfloat16(mode_id, tmp_path): | |
| """The default precision must reach the encoder, for every audio mode.""" | |
| result = run_production(mode_id, tmp_path, pipe=BFloat16Pipe()) | |
| written = result.audio_path or result.video_path | |
| assert written is not None and written.exists(), ( | |
| f"{mode_id} produced no artifact from bfloat16 audio" | |
| ) | |
| assert written.stat().st_size > 0, f"{mode_id} wrote an empty file from bfloat16 audio" | |
| def test_the_downcast_lands_on_float32_not_float16(): | |
| """A silent narrowing in the module whose job is 'never silently wrong'.""" | |
| source = np.zeros((AUDIO_CHANNELS, AUDIO_SAMPLES), dtype=np.float32) | |
| converted = outputs._as_array(BFloat16Tensor(source)) | |
| assert converted.dtype == np.float32, ( | |
| f"bfloat16 audio was converted to {converted.dtype}, not float32" | |
| ) | |
| def test_a_dtype_numpy_cannot_take_is_the_helpers_problem_not_the_callers(): | |
| """`_as_array` promises 'whatever we were given -> a numpy array'. | |
| Fixing this at one caller would leave the other latent, which is how a | |
| shared helper with two callers hid it in the first place. | |
| """ | |
| converted = outputs._as_array(BFloat16Tensor(np.zeros((2, 8), dtype=np.float32))) | |
| assert isinstance(converted, np.ndarray), ( | |
| f"_as_array returned {type(converted).__name__}, so the promise it makes is not kept" | |
| ) | |
| def test_the_bfloat16_fake_is_faithful_to_torch(): | |
| """The fixture is itself untested by the tests it makes possible. | |
| That is the gap this whole section exists to close, so it is closed rather | |
| than described: against real torch, assert the two properties | |
| :class:`BFloat16Tensor` claims -- that `.numpy()` raises on bfloat16, and | |
| that `.float()` yields float32. If either stops holding, the fake above is | |
| testing a failure mode that no longer exists. | |
| """ | |
| torch = pytest.importorskip("torch", reason="fidelity check needs real torch") | |
| real = torch.zeros((AUDIO_CHANNELS, 16), dtype=torch.bfloat16) | |
| with pytest.raises(TypeError): | |
| real.numpy() | |
| assert real.float().numpy().dtype == np.float32 | |
| assert outputs._as_array(real).dtype == np.float32, ( | |
| "a real bfloat16 tensor did not survive _as_array" | |
| ) | |