vae-rhythm / vae_rhythm /video.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
ed519a7 verified
Raw History Blame Contribute Delete
8.69 kB
"""Video input/output and the scan protocol's preprocessing.
Protocol (paper Sec. 3.1 / 4.3): every clip, real or generated, is re-encoded with the
same command before scanning -- libx264, CRF 23, a fixed GOP of 32 without scene-cut
keyframes, two B-frames, no frame-rate change -- after an area downscale that is applied
only when the short side exceeds the scan cap (480 for the census, 720 for the plug-in).
Equal compression history for both classes keeps codec differences from masquerading
as rhythm, and the fixed GOP keeps the codec's own period away from the VAE's.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import tempfile
from pathlib import Path
import numpy as np
FFMPEG = os.getenv("VAE_RHYTHM_FFMPEG", "ffmpeg")
FFPROBE = os.getenv("VAE_RHYTHM_FFPROBE", "ffprobe")
X264_ARGS = ["-c:v", "libx264", "-preset", "medium", "-pix_fmt", "yuv420p", "-bf", "2", "-sc_threshold", "0"]
CRF, GOP = 23, 32
_PASSTHROUGH: list[str] | None = None
def passthrough_args() -> list[str]:
"""Keep every decoded frame: ``-fps_mode passthrough`` (ffmpeg >= 5.1) or ``-vsync passthrough``."""
global _PASSTHROUGH
if _PASSTHROUGH is None:
modern = True
try:
first = subprocess.run([FFMPEG, "-hide_banner", "-version"], capture_output=True, timeout=20,
check=False).stdout.decode(errors="ignore").split("\n", 1)[0].split()
version = first[2] if len(first) > 2 and first[1] == "version" else ""
digits = version.lstrip("nN").split("-")[0].split(".")
if digits and digits[0].isdigit():
major = int(digits[0])
minor = int(digits[1]) if len(digits) > 1 and digits[1].isdigit() else 0
modern = (major, minor) >= (5, 1)
except (OSError, subprocess.TimeoutExpired):
pass
_PASSTHROUGH = ["-fps_mode", "passthrough"] if modern else ["-vsync", "passthrough"]
return list(_PASSTHROUGH)
def check_ffmpeg() -> None:
for tool in (FFMPEG, FFPROBE):
if shutil.which(tool) is None and not Path(tool).is_file():
raise RuntimeError(f"{tool} not found. Install ffmpeg (e.g. `conda install -c conda-forge ffmpeg`) "
"or point VAE_RHYTHM_FFMPEG / VAE_RHYTHM_FFPROBE to the binaries.")
def probe(path: str | os.PathLike) -> dict:
"""Width, height, frame rate and duration of the first video stream."""
out = subprocess.run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries",
"stream=width,height,avg_frame_rate,r_frame_rate,nb_frames:format=duration",
"-of", "json", str(path)], capture_output=True, check=True).stdout
data = json.loads(out)
st = data["streams"][0]
fps = None
for key in ("avg_frame_rate", "r_frame_rate"):
try:
num, den = str(st.get(key)).split("/")
if float(den) > 0 and float(num) > 0:
fps = float(num) / float(den)
break
except (ValueError, ZeroDivisionError):
continue
nb = st.get("nb_frames")
return {"width": int(st["width"]), "height": int(st["height"]), "fps": fps,
"nb_frames": int(nb) if str(nb).isdigit() else None,
"duration": float(data.get("format", {}).get("duration") or 0) or None}
def count_frames(path: str | os.PathLike) -> int:
"""Number of video packets (= frames for the protocol-encoded files this package writes)."""
out = subprocess.run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-count_packets", "-show_entries",
"stream=nb_read_packets", "-of", "csv=p=0", str(path)], capture_output=True, check=True)
return int(out.stdout.decode().strip().split(",")[0])
def target_size(w: int, h: int, short_side: int) -> tuple[int, int]:
"""Downscale so that the short side equals ``short_side`` (never upscale); even sizes."""
if short_side <= 0 or min(w, h) <= short_side:
return w - w % 2, h - h % 2
if w >= h:
nh, nw = short_side, int(round(w * short_side / h))
else:
nw, nh = short_side, int(round(h * short_side / w))
return nw - nw % 2, nh - nh % 2
# x264's rate control looks 40 frames ahead at preset medium, so encoding the first N + 40 frames gives the
# same first N frames as encoding the whole video (checked bit for bit); 64 leaves a margin.
LOOKAHEAD_MARGIN = 64
def prepare(src: str | os.PathLike, dst: str | os.PathLike, short_side: int = 720,
crf: int = CRF, gop: int = GOP, max_frames: int | None = None, portable: bool = False) -> Path:
"""Re-encode ``src`` with the protocol (area downscale only above ``short_side``,
x264 CRF 23, GOP 32, no scene-cut keyframes, frame rate unchanged).
``max_frames`` encodes only a prefix (pass the frames you will read + LOOKAHEAD_MARGIN).
``portable`` runs x264 single-threaded without assembly: identical output on every machine.
By default x264 uses its SIMD code and all cores, as in the paper; its output then differs
slightly between CPU types and thread counts, which can move Ppk of clips with a weak rhythm."""
info = probe(src)
w, h = target_size(info["width"], info["height"], short_side)
dst = Path(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
tmp = dst.with_name(dst.stem + ".part.mp4")
extra = ["-frames:v", str(int(max_frames))] if max_frames else []
if portable:
extra += ["-threads", "1", "-x264-params", "asm=0"]
cmd = [FFMPEG, "-nostdin", "-y", "-v", "error", "-i", str(src), "-an", *passthrough_args(),
"-vf", f"scale={w}:{h}:flags=area", *X264_ARGS, "-crf", str(crf), "-g", str(gop),
"-keyint_min", str(gop), *extra, str(tmp)]
p = subprocess.run(cmd, capture_output=True, text=True)
if p.returncode != 0:
raise RuntimeError(f"ffmpeg failed on {src}: {p.stderr[-500:]}")
os.replace(tmp, dst)
return dst
def read_frames(path: str | os.PathLike, num: int | None = None, start: int = 0, stride: int = 1) -> np.ndarray:
"""Decode uint8 RGB frames (T, H, W, 3) with PyAV; ``num=None`` reads to the end."""
import av
out = []
with av.open(str(path)) as c:
st = c.streams.video[0]
st.thread_type = "AUTO"
st.thread_count = 4
i = 0
for fr in c.decode(st):
if i >= start and (i - start) % stride == 0:
out.append(fr.to_ndarray(format="rgb24"))
if num is not None and len(out) >= num:
break
i += 1
if not out:
raise RuntimeError(f"No frames decoded from {path}")
return np.stack(out)
def write_frames(frames: np.ndarray, path: str | os.PathLike, fps: float, crf: int = CRF, gop: int = GOP) -> None:
"""Encode uint8 frames (T, H, W, 3) with the protocol's x264 settings (H, W even)."""
T, H, W, _ = frames.shape
if H % 2 or W % 2:
raise ValueError(f"Frame size must be even for yuv420p, got {W}x{H}")
cmd = [FFMPEG, "-nostdin", "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", f"{W}x{H}",
"-r", f"{fps:g}", "-i", "-", "-an", *passthrough_args(), *X264_ARGS, "-crf", str(crf), "-g", str(gop),
"-keyint_min", str(gop), str(path)]
p = subprocess.run(cmd, input=np.ascontiguousarray(frames).tobytes(), capture_output=True)
if p.returncode != 0:
raise RuntimeError(p.stderr.decode()[-2000:])
def motion_energy(frames: np.ndarray) -> float:
"""Mean absolute frame difference (uint8 scale); the rhythm is unreliable below ~2."""
f = frames.astype(np.float32)
return float(np.abs(f[1:] - f[:-1]).mean()) if len(f) > 1 else 0.0
class PreparedVideo:
"""Context manager: protocol-encoded copy of a video in a temporary directory."""
def __init__(self, src: str | os.PathLike, short_side: int = 720, keep: str | os.PathLike | None = None,
max_frames: int | None = None, portable: bool = False):
self.src, self.short_side, self.keep = src, short_side, keep
self.max_frames, self.portable = max_frames, portable
self._tmp = None
def __enter__(self) -> Path:
kw = dict(max_frames=self.max_frames, portable=self.portable)
if self.keep:
return prepare(self.src, self.keep, self.short_side, **kw)
self._tmp = tempfile.TemporaryDirectory(prefix="vae_rhythm_")
return prepare(self.src, Path(self._tmp.name) / "prepared.mp4", self.short_side, **kw)
def __exit__(self, *exc) -> None:
if self._tmp is not None:
self._tmp.cleanup()