Spaces:
Running on Zero
Running on Zero
Download vae_rhythm/video.py from hugging-apps/vae-rhythm: direct link, hf CLI and curl.
- Browser
- Download file 8.69 kB
-
https://huggingface.co/spaces/hugging-apps/vae-rhythm/resolve/main/vae_rhythm/video.py
- Command line
-
hf download hf://spaces/hugging-apps/vae-rhythm/vae_rhythm/video.py
-
curl -L -o video.py https://huggingface.co/spaces/hugging-apps/vae-rhythm/resolve/main/vae_rhythm/video.py
8.69 kB
| """Video input/output and the scan protocol's preprocessing. | |
| Protocol (paper Sec. 3.1 / 4.3): every clip, real or generated, is re-encoded with the | |
| same command before scanning -- libx264, CRF 23, a fixed GOP of 32 without scene-cut | |
| keyframes, two B-frames, no frame-rate change -- after an area downscale that is applied | |
| only when the short side exceeds the scan cap (480 for the census, 720 for the plug-in). | |
| Equal compression history for both classes keeps codec differences from masquerading | |
| as rhythm, and the fixed GOP keeps the codec's own period away from the VAE's. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| from pathlib import Path | |
| import numpy as np | |
| FFMPEG = os.getenv("VAE_RHYTHM_FFMPEG", "ffmpeg") | |
| FFPROBE = os.getenv("VAE_RHYTHM_FFPROBE", "ffprobe") | |
| X264_ARGS = ["-c:v", "libx264", "-preset", "medium", "-pix_fmt", "yuv420p", "-bf", "2", "-sc_threshold", "0"] | |
| CRF, GOP = 23, 32 | |
| _PASSTHROUGH: list[str] | None = None | |
| def passthrough_args() -> list[str]: | |
| """Keep every decoded frame: ``-fps_mode passthrough`` (ffmpeg >= 5.1) or ``-vsync passthrough``.""" | |
| global _PASSTHROUGH | |
| if _PASSTHROUGH is None: | |
| modern = True | |
| try: | |
| first = subprocess.run([FFMPEG, "-hide_banner", "-version"], capture_output=True, timeout=20, | |
| check=False).stdout.decode(errors="ignore").split("\n", 1)[0].split() | |
| version = first[2] if len(first) > 2 and first[1] == "version" else "" | |
| digits = version.lstrip("nN").split("-")[0].split(".") | |
| if digits and digits[0].isdigit(): | |
| major = int(digits[0]) | |
| minor = int(digits[1]) if len(digits) > 1 and digits[1].isdigit() else 0 | |
| modern = (major, minor) >= (5, 1) | |
| except (OSError, subprocess.TimeoutExpired): | |
| pass | |
| _PASSTHROUGH = ["-fps_mode", "passthrough"] if modern else ["-vsync", "passthrough"] | |
| return list(_PASSTHROUGH) | |
| def check_ffmpeg() -> None: | |
| for tool in (FFMPEG, FFPROBE): | |
| if shutil.which(tool) is None and not Path(tool).is_file(): | |
| raise RuntimeError(f"{tool} not found. Install ffmpeg (e.g. `conda install -c conda-forge ffmpeg`) " | |
| "or point VAE_RHYTHM_FFMPEG / VAE_RHYTHM_FFPROBE to the binaries.") | |
| def probe(path: str | os.PathLike) -> dict: | |
| """Width, height, frame rate and duration of the first video stream.""" | |
| out = subprocess.run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries", | |
| "stream=width,height,avg_frame_rate,r_frame_rate,nb_frames:format=duration", | |
| "-of", "json", str(path)], capture_output=True, check=True).stdout | |
| data = json.loads(out) | |
| st = data["streams"][0] | |
| fps = None | |
| for key in ("avg_frame_rate", "r_frame_rate"): | |
| try: | |
| num, den = str(st.get(key)).split("/") | |
| if float(den) > 0 and float(num) > 0: | |
| fps = float(num) / float(den) | |
| break | |
| except (ValueError, ZeroDivisionError): | |
| continue | |
| nb = st.get("nb_frames") | |
| return {"width": int(st["width"]), "height": int(st["height"]), "fps": fps, | |
| "nb_frames": int(nb) if str(nb).isdigit() else None, | |
| "duration": float(data.get("format", {}).get("duration") or 0) or None} | |
| def count_frames(path: str | os.PathLike) -> int: | |
| """Number of video packets (= frames for the protocol-encoded files this package writes).""" | |
| out = subprocess.run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-count_packets", "-show_entries", | |
| "stream=nb_read_packets", "-of", "csv=p=0", str(path)], capture_output=True, check=True) | |
| return int(out.stdout.decode().strip().split(",")[0]) | |
| def target_size(w: int, h: int, short_side: int) -> tuple[int, int]: | |
| """Downscale so that the short side equals ``short_side`` (never upscale); even sizes.""" | |
| if short_side <= 0 or min(w, h) <= short_side: | |
| return w - w % 2, h - h % 2 | |
| if w >= h: | |
| nh, nw = short_side, int(round(w * short_side / h)) | |
| else: | |
| nw, nh = short_side, int(round(h * short_side / w)) | |
| return nw - nw % 2, nh - nh % 2 | |
| # x264's rate control looks 40 frames ahead at preset medium, so encoding the first N + 40 frames gives the | |
| # same first N frames as encoding the whole video (checked bit for bit); 64 leaves a margin. | |
| LOOKAHEAD_MARGIN = 64 | |
| def prepare(src: str | os.PathLike, dst: str | os.PathLike, short_side: int = 720, | |
| crf: int = CRF, gop: int = GOP, max_frames: int | None = None, portable: bool = False) -> Path: | |
| """Re-encode ``src`` with the protocol (area downscale only above ``short_side``, | |
| x264 CRF 23, GOP 32, no scene-cut keyframes, frame rate unchanged). | |
| ``max_frames`` encodes only a prefix (pass the frames you will read + LOOKAHEAD_MARGIN). | |
| ``portable`` runs x264 single-threaded without assembly: identical output on every machine. | |
| By default x264 uses its SIMD code and all cores, as in the paper; its output then differs | |
| slightly between CPU types and thread counts, which can move Ppk of clips with a weak rhythm.""" | |
| info = probe(src) | |
| w, h = target_size(info["width"], info["height"], short_side) | |
| dst = Path(dst) | |
| dst.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = dst.with_name(dst.stem + ".part.mp4") | |
| extra = ["-frames:v", str(int(max_frames))] if max_frames else [] | |
| if portable: | |
| extra += ["-threads", "1", "-x264-params", "asm=0"] | |
| cmd = [FFMPEG, "-nostdin", "-y", "-v", "error", "-i", str(src), "-an", *passthrough_args(), | |
| "-vf", f"scale={w}:{h}:flags=area", *X264_ARGS, "-crf", str(crf), "-g", str(gop), | |
| "-keyint_min", str(gop), *extra, str(tmp)] | |
| p = subprocess.run(cmd, capture_output=True, text=True) | |
| if p.returncode != 0: | |
| raise RuntimeError(f"ffmpeg failed on {src}: {p.stderr[-500:]}") | |
| os.replace(tmp, dst) | |
| return dst | |
| def read_frames(path: str | os.PathLike, num: int | None = None, start: int = 0, stride: int = 1) -> np.ndarray: | |
| """Decode uint8 RGB frames (T, H, W, 3) with PyAV; ``num=None`` reads to the end.""" | |
| import av | |
| out = [] | |
| with av.open(str(path)) as c: | |
| st = c.streams.video[0] | |
| st.thread_type = "AUTO" | |
| st.thread_count = 4 | |
| i = 0 | |
| for fr in c.decode(st): | |
| if i >= start and (i - start) % stride == 0: | |
| out.append(fr.to_ndarray(format="rgb24")) | |
| if num is not None and len(out) >= num: | |
| break | |
| i += 1 | |
| if not out: | |
| raise RuntimeError(f"No frames decoded from {path}") | |
| return np.stack(out) | |
| def write_frames(frames: np.ndarray, path: str | os.PathLike, fps: float, crf: int = CRF, gop: int = GOP) -> None: | |
| """Encode uint8 frames (T, H, W, 3) with the protocol's x264 settings (H, W even).""" | |
| T, H, W, _ = frames.shape | |
| if H % 2 or W % 2: | |
| raise ValueError(f"Frame size must be even for yuv420p, got {W}x{H}") | |
| cmd = [FFMPEG, "-nostdin", "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", f"{W}x{H}", | |
| "-r", f"{fps:g}", "-i", "-", "-an", *passthrough_args(), *X264_ARGS, "-crf", str(crf), "-g", str(gop), | |
| "-keyint_min", str(gop), str(path)] | |
| p = subprocess.run(cmd, input=np.ascontiguousarray(frames).tobytes(), capture_output=True) | |
| if p.returncode != 0: | |
| raise RuntimeError(p.stderr.decode()[-2000:]) | |
| def motion_energy(frames: np.ndarray) -> float: | |
| """Mean absolute frame difference (uint8 scale); the rhythm is unreliable below ~2.""" | |
| f = frames.astype(np.float32) | |
| return float(np.abs(f[1:] - f[:-1]).mean()) if len(f) > 1 else 0.0 | |
| class PreparedVideo: | |
| """Context manager: protocol-encoded copy of a video in a temporary directory.""" | |
| def __init__(self, src: str | os.PathLike, short_side: int = 720, keep: str | os.PathLike | None = None, | |
| max_frames: int | None = None, portable: bool = False): | |
| self.src, self.short_side, self.keep = src, short_side, keep | |
| self.max_frames, self.portable = max_frames, portable | |
| self._tmp = None | |
| def __enter__(self) -> Path: | |
| kw = dict(max_frames=self.max_frames, portable=self.portable) | |
| if self.keep: | |
| return prepare(self.src, self.keep, self.short_side, **kw) | |
| self._tmp = tempfile.TemporaryDirectory(prefix="vae_rhythm_") | |
| return prepare(self.src, Path(self._tmp.name) / "prepared.mp4", self.short_side, **kw) | |
| def __exit__(self, *exc) -> None: | |
| if self._tmp is not None: | |
| self._tmp.cleanup() | |