Spaces:
Running on Zero
Running on Zero
Download vae_rhythm/engine/pairs.py from hugging-apps/vae-rhythm: direct link, hf CLI and curl.
- Browser
- Download file 6.57 kB
-
https://huggingface.co/spaces/hugging-apps/vae-rhythm/resolve/main/vae_rhythm/engine/pairs.py
- Command line
-
hf download hf://spaces/hugging-apps/vae-rhythm/vae_rhythm/engine/pairs.py
-
curl -L -o pairs.py https://huggingface.co/spaces/hugging-apps/vae-rhythm/resolve/main/vae_rhythm/engine/pairs.py
6.57 kB
| """VAE-pair engine (paper Sec. 4.1, Eq. 2). | |
| For each real clip x the engine emits a pair that differs only by one VAE round trip: | |
| x+ = c(s_G(x); theta) labelled real | |
| x- = c(D(E(s_G(x))); theta) labelled generated | |
| s_G rescales to a reconstruction short side G, D(E(.)) is one round trip through a frozen | |
| video VAE drawn from {Wan2.1, HunyuanVideo, CogVideoX} (4x) and LTX-Video (8x), and c is the | |
| post-processing chain of :mod:`vae_rhythm.engine.chain` with the *same* random parameters | |
| for both members. Content, resolution, frame rate, codec and every random choice are shared. | |
| Paper setting (two passes over the 4,000 AIGVDBench training reals, merged; see | |
| ``scripts/make_paper_pairs.sh``): | |
| pass A G in {256 x4, 360 x2, 480 x2, 720}, S in {480, 720, 1080} with upscaling allowed, | |
| interpolation 20 %, duplicate-frame conversion 25 %, 4,000 pairs, seed 0 | |
| pass B G in {256 x3, 360}, interpolation 10 %, duplicate-frame conversion 10 %, | |
| 2,000 pairs, seed 13 | |
| Pairs are split 9:1 into train / val by a hash of the source clip id (``split_of``). | |
| """ | |
| from __future__ import annotations | |
| import csv | |
| import json | |
| import os | |
| import subprocess | |
| import time | |
| import zlib | |
| import numpy as np | |
| from ..vaes import VAE, crop_to_multiple, to_tensor, to_uint8 | |
| from ..video import FFPROBE, read_frames | |
| from .chain import downscale, run_chain, sample_chain | |
| def split_of(pair_id: str) -> str: | |
| """Deterministic 9:1 train / val split by source clip.""" | |
| return "val" if zlib.crc32(pair_id.encode()) % 10 == 0 else "train" | |
| def vae_prep(frames: np.ndarray, vae: VAE, max_T: int | None = None) -> np.ndarray: | |
| """Shared crop / trim: H, W to multiples of 16 and T to the VAE's valid frame count.""" | |
| fr = crop_to_multiple(frames, 16) | |
| T = vae.valid_T(len(fr) if max_T is None else min(len(fr), max_T)) | |
| return fr[:T] | |
| def vae_pass(frames: np.ndarray, vae: VAE) -> np.ndarray: | |
| """One round trip (posterior mode). On CUDA OOM, reconstruct the first 41 frames and | |
| keep the rest unchanged (rare; recorded through ``vae_T`` in the manifest).""" | |
| import torch | |
| try: | |
| y = vae.roundtrip(to_tensor(frames)) | |
| except torch.cuda.OutOfMemoryError: | |
| torch.cuda.empty_cache() | |
| T = vae.valid_T(min(len(frames), 41)) | |
| y = vae.roundtrip(to_tensor(frames[:T])) | |
| if T < len(frames): | |
| y = torch.cat([y, to_tensor(frames[T:]).to(y.device)], 2) | |
| return to_uint8(y) | |
| def _fps(path: str) -> float: | |
| out = subprocess.run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate", | |
| "-of", "json", path], capture_output=True, text=True).stdout | |
| num, den = json.loads(out)["streams"][0]["r_frame_rate"].split("/") | |
| return float(num) / float(den) | |
| def make_pairs(real_paths: list[str], out: str, vaes=("wan21", "hunyuan", "cogx", "ltx"), gen_scales=(480,), | |
| out_scales=(360, 480, 720), allow_upscale: bool = False, interp_prob: float = 0.2, | |
| dup_prob: float = 0.0, crf_max: int = 28, frames: int = 81, n: int = 0, seed: int = 0, | |
| tag: str = "", shard: int = 0, nshards: int = 1, device: str = "cuda") -> str: | |
| """Build pairs for ``real_paths`` into ``out/pos``, ``out/neg`` and a manifest CSV; returns its path.""" | |
| import torch | |
| torch.set_num_threads(8) | |
| paths = sorted(real_paths) | |
| np.random.default_rng(seed).shuffle(paths) | |
| if n: | |
| paths = paths[:n] | |
| # random parameters are fixed per clip index before sharding, so the shard count does not change them | |
| plan = [(p, np.random.default_rng([seed, i])) for i, p in enumerate(paths)][shard::nshards] | |
| models = {} | |
| for v in vaes: | |
| models[v] = VAE(v, device, "bf16") | |
| if hasattr(models[v].vae, "enable_tiling"): | |
| models[v].vae.enable_tiling() | |
| os.makedirs(os.path.join(out, "pos"), exist_ok=True) | |
| os.makedirs(os.path.join(out, "neg"), exist_ok=True) | |
| mname = os.path.join(out, "manifest.csv" if nshards == 1 else f"manifest_shard{shard}.csv") | |
| rows, nfail, t0 = [], 0, time.time() | |
| for i, (fpath, rng) in enumerate(plan): | |
| vid = os.path.splitext(os.path.basename(fpath))[0] | |
| try: | |
| fps = _fps(fpath) | |
| G = int(rng.choice([int(g) for g in gen_scales])) | |
| p = sample_chain(rng, G, fps, tuple(out_scales), allow_upscale, interp_prob, dup_prob, crf_max) | |
| k = p["dup_k"] | |
| raw = read_frames(fpath, num=frames * k) | |
| while k > 1 and len(raw) < 41 * k: # source too short: fall back to a smaller factor | |
| k -= 1 | |
| p["dup_k"] = k | |
| if k == 1: | |
| p["dup"] = 0 | |
| if len(raw) < 41: | |
| raise RuntimeError("too short") | |
| if k > 1: # duplicate-frame mode: decimate the source first | |
| raw, fps = raw[::k], fps / k | |
| H, W = raw.shape[1:3] | |
| if not allow_upscale: | |
| p["S"] = min(int(p["S"]), min(H, W)) | |
| vname = str(rng.choice(sorted(models))) | |
| base = vae_prep(downscale(raw, G), models[vname]) | |
| pos, neg = os.path.join(out, "pos", vid + ".mp4"), os.path.join(out, "neg", vid + ".mp4") | |
| if not os.path.exists(pos): | |
| run_chain(base, fps, p, pos) | |
| if not os.path.exists(neg): | |
| run_chain(vae_pass(base, models[vname]), fps, p, neg) | |
| common = dict(dataset="synth", pair_id=vid, split=split_of(vid), native_short=min(H, W), fps=round(fps, 3), | |
| vae=vname, G=G, vae_T=len(base), **p) | |
| rows.append(dict(subset="pos" + tag, label=0, video_id=f"pos{tag}/{vid}", path=pos, **common)) | |
| rows.append(dict(subset="neg" + tag, label=1, video_id=f"neg{tag}/{vid}", path=neg, **common)) | |
| except Exception as e: # noqa: BLE001 | |
| nfail += 1 | |
| print("FAIL", fpath, str(e)[:160], flush=True) | |
| if (i + 1) % 50 == 0 or i + 1 == len(plan): | |
| _dump(rows, mname) | |
| print(f"{i + 1}/{len(plan)} clips, {len(rows) // 2} pairs, {nfail} failed, {time.time() - t0:.0f}s", flush=True) | |
| return mname | |
| def _dump(rows: list[dict], path: str) -> None: | |
| if not rows: | |
| return | |
| keys = list(dict.fromkeys(k for r in rows for k in r)) | |
| tmp = path + ".tmp" | |
| with open(tmp, "w", newline="") as fh: | |
| w = csv.DictWriter(fh, fieldnames=keys) | |
| w.writeheader() | |
| w.writerows(rows) | |
| os.replace(tmp, path) | |