Spaces:
Running on Zero
Running on Zero
test: a dark black-and-white night and a soft snowfall scene get no note from the picture check
0b00a5b unverified Download tests/test_frame_structure.py from techfreakworm/LTX2.5-Studio: direct link, hf CLI and curl.
- Browser
- Download file 21.7 kB
-
https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_frame_structure.py
- Command line
-
hf download hf://spaces/techfreakworm/LTX2.5-Studio/tests/test_frame_structure.py
-
curl -L -o test_frame_structure.py https://huggingface.co/spaces/techfreakworm/LTX2.5-Studio/resolve/main/tests/test_frame_structure.py
21.7 kB
| """Spec for the picture check: ``outputs.assess_picture`` and the notes every result carries. | |
| Every finished video passes through a cheap check that tells the creator when the picture is | |
| probably broken. It names three conditions, each from statistics measured on up to 16 evenly | |
| spaced frames after resizing them to one analysis size (so the same picture gives the same answer | |
| at any resolution): | |
| * ``flat``: no picture at all, one even tone (the operator's grey failure); | |
| * ``veiled``: a real picture under a grey veil: no blacks, no bright highlights, almost no | |
| colour (the audio-to-video path before its fix); | |
| * ``static``: coloured speckle with no shape larger than a blob (the broken Space build). | |
| The check is a hint, never a refusal: the file is always written, a good picture gets no note, | |
| and the note says what was seen without quoting numbers. | |
| Fixtures in ``tests/fixtures/structure/`` are real frames from local runs; their README lists the | |
| sources. The only real colour-static frames are Space outputs that are not copied into the repo, | |
| so static is synthesised below as random colour blobs and checked against what was measured on | |
| those frames (correlation at an eighth of the frame at most 0.033, colourfulness at least 62.9). | |
| """ | |
| from __future__ import annotations | |
| import base64 | |
| import io | |
| import json | |
| import os | |
| import re | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import pytest | |
| from ltx25_studio import generate as generate_module | |
| from ltx25_studio import outputs | |
| np = pytest.importorskip("numpy") | |
| Image = pytest.importorskip("PIL.Image") | |
| REPO = Path(__file__).resolve().parents[1] | |
| FIXTURES = Path(__file__).resolve().parent / "fixtures" / "structure" | |
| FRAMES = FIXTURES / "frames.json" | |
| SAMPLING_RATE = 48000 | |
| STATIC_SIZE = 256 | |
| EXPECTED = {"good": (), "bad_grey": ("flat",), "bad_washed": ("veiled",)} | |
| def fixture_entries(): | |
| return {entry["name"]: entry for entry in json.loads(FRAMES.read_text())["fixtures"]} | |
| def fixture_names(): | |
| return sorted(fixture_entries()) | |
| def label_of(name): | |
| return name.rsplit("_", 1)[0] | |
| def load(name): | |
| data = base64.b64decode(fixture_entries()[name]["jpeg_base64"]) | |
| with Image.open(io.BytesIO(data)) as image: | |
| return np.asarray(image.convert("RGB")) | |
| def stack(frame, count=4): | |
| return np.repeat(frame[None], count, axis=0) | |
| def blob_static(height, width, *, seed=0, token=32, frames=1): | |
| """Blobby colour static: random colours per 32-px block (the model's latent cell), a finer | |
| 16-px layer on top, softened edges. No two blocks agree, so nothing spans the frame.""" | |
| rng = np.random.default_rng(seed) | |
| kernel = np.ones(4, np.float32) / 4 | |
| def soften(row): | |
| return np.convolve(row, kernel, mode="same") | |
| out = [] | |
| for _ in range(frames): | |
| def layer(size): | |
| rows, cols = -(-height // size) + 1, -(-width // size) + 1 | |
| grid = rng.random((rows, cols, 3)).astype(np.float32) | |
| big = np.repeat(np.repeat(grid, size, 0), size, 1) | |
| dy, dx = rng.integers(0, size, 2) | |
| return big[dy : dy + height, dx : dx + width] | |
| coarse, fine = layer(token), layer(token // 2) | |
| luma = 255 * (0.7 * coarse[..., 0] + 0.3 * fine[..., 0]) | |
| hue = (coarse[..., 1:] - 0.5) * 2 * 255 * 0.22 | |
| rgb = np.stack( | |
| [luma + hue[..., 0], luma - 0.5 * hue[..., 0] - 0.5 * hue[..., 1], luma + hue[..., 1]], | |
| axis=-1, | |
| ) | |
| for axis in (0, 1): | |
| rgb = np.apply_along_axis(soften, axis, rgb) | |
| out.append(np.clip(rgb, 0, 255).astype(np.uint8)) | |
| return np.stack(out) | |
| def resized(frame, long_side): | |
| height, width = frame.shape[:2] | |
| scale = long_side / max(height, width) | |
| size = (max(1, round(width * scale)), max(1, round(height * scale))) | |
| return np.asarray(Image.fromarray(frame).resize(size, Image.BICUBIC)) | |
| # --------------------------------------------------------------------------- | |
| # the fixtures | |
| # --------------------------------------------------------------------------- | |
| def test_the_fixtures_are_small_and_each_one_is_documented(): | |
| entries = fixture_entries() | |
| readme = (FIXTURES / "README.md").read_text() | |
| assert 0 < len(entries) <= 24 | |
| assert FRAMES.stat().st_size <= 400 * 1024 | |
| binary = {".jpg", ".jpeg", ".png", ".npy", ".npz", ".mp4"} | |
| assert not [path.name for path in FIXTURES.iterdir() if path.suffix.lower() in binary] | |
| for name, entry in entries.items(): | |
| assert f"| {name} |" in readme, f"{name} is not in the fixtures README" | |
| assert entry["label"] == label_of(name) and entry["label"] in EXPECTED, name | |
| assert entry["source"], name | |
| assert {label_of(name) for name in entries} == set(EXPECTED) | |
| def test_every_fixture_is_classified_as_labelled(name): | |
| check = outputs.assess_picture(stack(load(name))) | |
| assert check.conditions == EXPECTED[label_of(name)], check.measurements | |
| assert bool(check.notes) == bool(EXPECTED[label_of(name)]) | |
| # --------------------------------------------------------------------------- | |
| # static, synthesised (see the module docstring) | |
| # --------------------------------------------------------------------------- | |
| def test_blobby_colour_static_is_named_static(height, width): | |
| check = outputs.assess_picture(blob_static(height, width, frames=3)) | |
| measured = check.measurements | |
| assert measured.long_range_correlation < 0.06, ( | |
| "the stand-in must decorrelate like the real frames" | |
| ) | |
| assert 45 < measured.colourfulness < 100, "and be about as colourful as them" | |
| assert check.conditions == ("static",) | |
| def test_pixel_noise_is_static_too(): | |
| noise = np.random.default_rng(11).integers(0, 256, (3, 128, 128, 3), dtype=np.uint8) | |
| assert outputs.assess_picture(noise).conditions == ("static",) | |
| def test_grey_speckle_is_not_called_coloured(): | |
| """Snow, gravel, a starfield: texture with no colour is not the coloured mush.""" | |
| frames = blob_static(256, 256, frames=2) | |
| grey = np.repeat(frames.mean(axis=-1, keepdims=True), 3, axis=-1).astype(np.uint8) | |
| assert outputs.assess_picture(grey).conditions == () | |
| # --------------------------------------------------------------------------- | |
| # the same picture at any resolution gives the same answer | |
| # --------------------------------------------------------------------------- | |
| def test_a_fixture_at_1024_px_measures_like_it_does_at_256(name): | |
| small = load(name) | |
| large = resized(small, 1024) | |
| assert max(large.shape[:2]) == 1024 | |
| at_256 = outputs.assess_picture(stack(small)) | |
| at_1024 = outputs.assess_picture(stack(large)) | |
| assert at_1024.conditions == at_256.conditions | |
| # Within 1 % of the 0..255 scale or 5 %: bicubic enlargement rings a little at edges. | |
| for field in ("luma_std", "black_level", "white_level", "colourfulness"): | |
| assert getattr(at_1024.measurements, field) == pytest.approx( | |
| getattr(at_256.measurements, field), rel=0.05, abs=2.5 | |
| ), field | |
| assert at_1024.measurements.long_range_correlation == pytest.approx( | |
| at_256.measurements.long_range_correlation, abs=0.03 | |
| ) | |
| def test_static_at_256_px_and_its_1024_px_enlargement_are_both_static(): | |
| small = blob_static(256, 256)[0] | |
| large = np.repeat(np.repeat(small, 4, axis=0), 4, axis=1) | |
| assert outputs.assess_picture(stack(small)).conditions == ("static",) | |
| assert outputs.assess_picture(stack(large)).conditions == ("static",) | |
| # --------------------------------------------------------------------------- | |
| # legitimate pictures that sit near a condition get no note | |
| # --------------------------------------------------------------------------- | |
| def test_a_dark_contrasty_frame_yields_no_note(name): | |
| frame = load(name) | |
| check = outputs.assess_picture(stack(frame)) | |
| assert float(frame.mean()) < 90, "the fixture really is dark" | |
| assert check.notes == () and check.conditions == () | |
| def test_a_low_texture_sky_and_water_frame_yields_no_note(name): | |
| check = outputs.assess_picture(stack(load(name))) | |
| assert check.notes == () and check.conditions == () | |
| def test_a_dark_low_key_picture_is_not_called_washed_out(): | |
| """The dark portrait at 45 % brightness has no highlights and almost no colour, like a | |
| veiled frame; what it lacks is the veil's lifted blacks.""" | |
| frame = (load("good_10").astype(np.float32) * 0.45).astype(np.uint8) | |
| check = outputs.assess_picture(stack(frame)) | |
| measured = check.measurements | |
| assert measured.white_level < outputs.VEILED_MAX_WHITE_LEVEL | |
| assert measured.colourfulness < outputs.VEILED_MAX_COLOURFULNESS | |
| assert measured.black_level < outputs.VEILED_MIN_BLACK_LEVEL | |
| assert check.conditions == () | |
| def test_a_bright_overcast_picture_is_not_called_washed_out(name): | |
| """The overcast beach is low in contrast, its blacks are lifted and it has little colour, | |
| like a veiled frame; what it lacks is the veil's dimmed highlights.""" | |
| check = outputs.assess_picture(stack(load(name))) | |
| measured = check.measurements | |
| assert measured.black_level >= outputs.VEILED_MIN_BLACK_LEVEL | |
| assert measured.colourfulness < outputs.VEILED_MAX_COLOURFULNESS | |
| assert measured.white_level >= outputs.VEILED_MAX_WHITE_LEVEL | |
| assert check.conditions == () and check.notes == () | |
| def test_a_dark_black_and_white_night_with_true_blacks_yields_no_note(): | |
| """Dim highlights and no colour, like a veiled frame, and a narrow brightness spread; what | |
| it has that a veil takes away is true blacks.""" | |
| check = outputs.assess_picture(stack(load("good_15"))) | |
| measured = check.measurements | |
| assert measured.white_level < outputs.VEILED_MAX_WHITE_LEVEL | |
| assert measured.colourfulness < outputs.VEILED_MAX_COLOURFULNESS | |
| assert measured.black_level < outputs.VEILED_MIN_BLACK_LEVEL / 10 | |
| assert measured.luma_std > outputs.FLAT_MAX_LUMA_STD * 2 | |
| assert check.conditions == () and check.notes == () | |
| def test_a_soft_snowfall_scene_yields_no_note(): | |
| """Soft and low in detail by content: kept out of every condition, not just one.""" | |
| check = outputs.assess_picture(stack(load("good_16"))) | |
| measured = check.measurements | |
| assert measured.black_level < outputs.VEILED_MIN_BLACK_LEVEL | |
| assert measured.white_level > outputs.VEILED_MAX_WHITE_LEVEL | |
| assert measured.long_range_correlation > outputs.STATIC_MAX_CORRELATION | |
| assert check.conditions == () and check.notes == () | |
| def test_a_black_and_white_picture_yields_no_note(): | |
| real = load("good_14") | |
| frame = load("good_1") | |
| grey = np.repeat(frame.mean(axis=-1, keepdims=True), 3, axis=-1).astype(np.uint8) | |
| for picture in (real, grey): | |
| check = outputs.assess_picture(stack(picture)) | |
| assert check.measurements.colourfulness < 5.0 | |
| assert check.conditions == () | |
| def test_one_odd_frame_cannot_swing_the_answer(): | |
| """A fade to black or a cut: the aggregate is a median over the sampled frames.""" | |
| good = stack(load("good_2"), 8) | |
| spoiled = np.concatenate([good, np.zeros((2, *good.shape[1:]), dtype=np.uint8)]) | |
| assert outputs.assess_picture(spoiled).conditions == () | |
| static = blob_static(256, 256, frames=6) | |
| rescued = np.concatenate([static, resized(load("good_11"), 256)[None]]) | |
| assert outputs.assess_picture(rescued).conditions == ("static",) | |
| def test_a_flat_video_is_named_flat_however_it_drifts(): | |
| frames = np.stack([np.full((64, 64, 3), 20 + index * 24, dtype=np.uint8) for index in range(9)]) | |
| check = outputs.assess_picture(frames) | |
| assert check.conditions == ("flat",) | |
| assert check.measurements.long_range_correlation is None, "no variation, nothing to correlate" | |
| def test_an_empty_stack_measures_nothing_and_says_nothing(): | |
| check = outputs.assess_picture(np.zeros((0, 64, 64, 3), dtype=np.uint8)) | |
| assert check.measurements.frames == 0 and check.measurements.sampled == 0 | |
| assert check.measurements.luma_std is None | |
| assert check.conditions == () and check.notes == () | |
| # --------------------------------------------------------------------------- | |
| # the notes | |
| # --------------------------------------------------------------------------- | |
| FORBIDDEN = ( | |
| "luma", | |
| "band", | |
| "std", | |
| "correlation", | |
| "colourfulness", | |
| "ratio", | |
| "threshold", | |
| "percentile", | |
| "p99", | |
| "p1", | |
| "banked", | |
| "artifact", | |
| "dataset", | |
| "statistic", | |
| "structure", | |
| "measured", | |
| "median", | |
| "failed", | |
| "failure", | |
| "error", | |
| "broken", | |
| ) | |
| def every_note(): | |
| notes = {condition.name: condition.note for condition in outputs.PICTURE_CONDITIONS} | |
| assert set(notes) == {"flat", "veiled", "static"} | |
| return notes | |
| def test_a_note_says_what_was_seen_and_that_it_is_a_hint(name): | |
| note = every_note()[name] | |
| lowered = note.lower() | |
| assert not re.search(r"\d", note), f"no numbers or ranges: {note}" | |
| for word in FORBIDDEN: | |
| assert not re.search(rf"\b{re.escape(word)}\b", lowered), f"{word!r} in {note}" | |
| assert "hint" in lowered and "saved" in lowered | |
| assert "if that is not what you asked for" in lowered | |
| def test_each_condition_says_something_different(): | |
| notes = every_note() | |
| assert "flat" in notes["flat"].lower() | |
| assert "contrast" in notes["veiled"].lower() and "colour" in notes["veiled"].lower() | |
| assert "speckle" in notes["static"].lower() | |
| def test_the_notes_a_result_carries_are_the_checks_own(): | |
| frame = load("bad_washed_1") | |
| check = outputs.assess_picture(stack(frame)) | |
| assert outputs.structure_notes(check) == check.notes == (every_note()["veiled"],) | |
| measured = outputs.measure_structure(stack(frame)) | |
| assert outputs.structure_notes(measured) == check.notes | |
| assert outputs.structure_notes(outputs.assess_picture(stack(load("good_3")))) == () | |
| # --------------------------------------------------------------------------- | |
| # bounded and torch-free | |
| # --------------------------------------------------------------------------- | |
| class CountingFrames: | |
| """A 481-frame video that records which frames were read.""" | |
| def __init__(self, frame, count): | |
| self.frame, self.count, self.read = frame, count, [] | |
| def __len__(self): | |
| return self.count | |
| def __getitem__(self, index): | |
| self.read.append(index) | |
| return self.frame | |
| def test_a_481_frame_video_is_judged_on_at_most_16_frames(): | |
| frames = CountingFrames(load("good_4"), 481) | |
| check = outputs.assess_picture(frames) | |
| assert len(set(frames.read)) <= outputs.STRUCTURE_SAMPLE_FRAMES == 16 | |
| assert 0 in frames.read and 480 in frames.read, "evenly spaced, both ends" | |
| assert check.measurements.frames == 481 and check.measurements.sampled == 16 | |
| frames.read.clear() | |
| outputs.measure_structure(frames, sample=None) | |
| assert len(set(frames.read)) <= 16, "the older name is bounded the same way" | |
| def test_a_large_frame_is_measured_at_the_analysis_size(): | |
| frame = resized(load("good_2"), 1920) | |
| check = outputs.assess_picture(stack(frame, 2)) | |
| assert outputs.STRUCTURE_ANALYSIS_SIZE == 256 | |
| assert check.conditions == () | |
| def test_the_check_needs_neither_torch_nor_a_media_stack(): | |
| """numpy only. (PIL is already loaded by gradio through ``errors``; the call adds nothing.)""" | |
| code = ( | |
| "import sys, numpy\n" | |
| "from ltx25_studio import outputs\n" | |
| "def roots(): return {m.split('.')[0] for m in sys.modules}\n" | |
| "before = roots()\n" | |
| "outputs.assess_picture(numpy.zeros((2, 64, 64, 3), numpy.uint8))\n" | |
| "heavy = {'torch', 'diffusers', 'transformers', 'av', 'PIL', 'cv2', 'scipy'}\n" | |
| "print(sorted((roots() - before) & heavy), sorted(roots() & {'torch', 'diffusers', 'av'}))\n" | |
| ) | |
| env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} | |
| result = subprocess.run( | |
| [sys.executable, "-c", code], cwd=REPO, env=env, capture_output=True, text=True, check=True | |
| ) | |
| assert result.stdout.strip() == "[] []", result.stdout + result.stderr | |
| # --------------------------------------------------------------------------- | |
| # thresholds, and the numbers that put them there | |
| # --------------------------------------------------------------------------- | |
| # Measured by this check on the labelled dataset (42 files: 27 good, 3 grey, 9 veiled, 3 static). | |
| WORST_GOOD_LUMA_STD = 14.93 | |
| WORST_GREY_LUMA_STD = 4.30 | |
| WORST_VEILED_BLACK_LEVEL = 57.6 | |
| WORST_VEILED_WHITE_LEVEL = 115.1 | |
| # veiled needs both ends of the range pulled in, so a good file is kept out by either end: | |
| # good files with lifted blacks (p1 >= 40) all keep highlights of at least this (the beach), | |
| GOOD_LIFTED_BLACKS_MIN_WHITE_LEVEL = 176.8 | |
| # and good files with dim highlights (p99 < 135) all keep blacks of at most this (the night). | |
| GOOD_DIM_HIGHLIGHTS_MAX_BLACK_LEVEL = 1.0 | |
| WORST_VEILED_COLOURFULNESS = 13.8 | |
| WORST_GOOD_CORRELATION = 0.188 | |
| WORST_STATIC_CORRELATION = 0.033 | |
| WORST_STATIC_COLOURFULNESS = 62.9 | |
| def test_the_thresholds_sit_between_the_measured_classes_with_room_either_side(): | |
| assert WORST_GREY_LUMA_STD * 1.3 < outputs.FLAT_MAX_LUMA_STD < WORST_GOOD_LUMA_STD / 2.4 | |
| assert outputs.VEILED_MIN_BLACK_LEVEL * 1.4 < WORST_VEILED_BLACK_LEVEL | |
| assert GOOD_DIM_HIGHLIGHTS_MAX_BLACK_LEVEL * 10 < outputs.VEILED_MIN_BLACK_LEVEL | |
| assert WORST_VEILED_WHITE_LEVEL * 1.15 < outputs.VEILED_MAX_WHITE_LEVEL | |
| assert outputs.VEILED_MAX_WHITE_LEVEL * 1.3 < GOOD_LIFTED_BLACKS_MIN_WHITE_LEVEL | |
| assert WORST_VEILED_COLOURFULNESS * 1.4 < outputs.VEILED_MAX_COLOURFULNESS | |
| assert WORST_STATIC_CORRELATION + 0.05 < outputs.STATIC_MAX_CORRELATION | |
| assert outputs.STATIC_MAX_CORRELATION + 0.05 < WORST_GOOD_CORRELATION | |
| assert outputs.STATIC_MIN_COLOURFULNESS * 1.8 < WORST_STATIC_COLOURFULNESS | |
| def test_the_older_names_still_resolve(): | |
| for name in ( | |
| "MIN_STRUCTURE_RATIO", | |
| "MAX_HIGH_FREQUENCY_RATIO", | |
| "LOW_BLOCK_STD", | |
| "LOW_LUMA_BAND", | |
| ): | |
| assert isinstance(getattr(outputs, name), float) | |
| static = outputs.measure_structure(blob_static(256, 256, frames=2)) | |
| assert "static" in outputs.structure_failure(static) | |
| assert outputs.structure_warnings(static) == outputs.structure_notes(static) | |
| good = outputs.measure_structure(stack(load("good_5"))) | |
| assert outputs.structure_failure(good) is None | |
| assert good.structure_ratio is not None and good.high_frequency_ratio is not None | |
| # --------------------------------------------------------------------------- | |
| # the app: a note, never a refusal | |
| # --------------------------------------------------------------------------- | |
| class StaticPipe: | |
| """A pipeline that generates colour static and can report a sampling rate.""" | |
| def __init__(self): | |
| self.vocoder = SimpleNamespace(config=SimpleNamespace(output_sampling_rate=SAMPLING_RATE)) | |
| def __call__(self, **kwargs): | |
| return StaticOutput(audio=False) | |
| class StaticOutput: | |
| """A pipeline output that is colour static, with stereo audio to go with it.""" | |
| def __init__(self, frames=None, audio=True): | |
| self.frames = blob_static(STATIC_SIZE, STATIC_SIZE, frames=9) if frames is None else frames | |
| self.audio = np.zeros((2, SAMPLING_RATE // 10), dtype=np.float32) if audio else None | |
| def test_the_writer_saves_a_static_video_and_says_what_it_saw(tmp_path): | |
| notes: list[str] = [] | |
| written = outputs.write_output( | |
| StaticOutput(audio=False), | |
| run_dir=tmp_path / "run", | |
| mode="text_to_video", | |
| exports=("video",), | |
| seed=1, | |
| frame_rate=24, | |
| warnings=notes, | |
| ) | |
| artifact = written["video"] | |
| assert artifact.exists() and artifact.stat().st_size > 0, "the file must survive the note" | |
| assert notes == [every_note()["static"]] | |
| def test_the_writer_stays_silent_about_a_good_video(tmp_path): | |
| notes: list[str] = [] | |
| outputs.write_output( | |
| StaticOutput(frames=stack(load("good_9")[:160, :256], 9), audio=False), | |
| run_dir=tmp_path / "run", | |
| mode="text_to_video", | |
| exports=("video",), | |
| seed=1, | |
| frame_rate=24, | |
| warnings=notes, | |
| ) | |
| assert notes == [] | |
| def test_a_note_does_not_cost_the_run_its_audio_track(tmp_path): | |
| """``write_output`` once inferred "the audio was muxed" from an empty notes list.""" | |
| notes: list[str] = [] | |
| written = outputs.write_output( | |
| StaticOutput(), | |
| run_dir=tmp_path / "run", | |
| mode="text_to_audio_video", | |
| exports=("video", "audio"), | |
| seed=1, | |
| frame_rate=24, | |
| sampling_rate=SAMPLING_RATE, | |
| warnings=notes, | |
| ) | |
| assert set(written) == {"video", "audio"}, f"the audio track went missing: {notes}" | |
| assert every_note()["static"] in notes | |
| def test_the_note_reaches_the_result_the_user_is_handed(tmp_path): | |
| from conftest import inputs_for | |
| result = generate_module.generate( | |
| mode="text_to_video", | |
| seed=7, | |
| **inputs_for("text_to_video"), | |
| width=STATIC_SIZE, | |
| height=STATIC_SIZE, | |
| pipeline_factory=lambda **kwargs: StaticPipe(), | |
| encoder=None, | |
| output_root=tmp_path, | |
| retention_bytes=10**9, | |
| ) | |
| assert result.output_path.exists() | |
| assert every_note()["static"] in result.warnings, result.warnings | |