multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
ed519a7 verified
Raw History Blame Contribute Delete
4.86 kB
"""Open video VAEs used as rulers (phase scan) and as round-trip operators (VAE-pair engine).
Only encode -> decode round trips are used; no diffusion model is loaded.
name diffusers class weights (Hugging Face) time space valid frame counts
wan21 AutoencoderKLWan Wan-AI/Wan2.1-T2V-1.3B-Diffusers/vae 4x 8x 1 + 4n
wan22 AutoencoderKLWan Wan-AI/Wan2.2-TI2V-5B-Diffusers/vae 4x 16x 1 + 4n
hunyuan AutoencoderKLHunyuanVideo hunyuanvideo-community/HunyuanVideo/vae 4x 8x 1 + 4n
cogx AutoencoderKLCogVideoX THUDM/CogVideoX-5b/vae 4x 8x 1 + 8n (*)
ltx AutoencoderKLLTXVideo Lightricks/LTX-Video/vae 8x 32x 1 + 8n
(*) The diffusers CogVideoX decoder only decodes the first frame on its own when the number
of latent frames is odd, so windows must have 1 + 8n frames (9, 17, 25, 33).
Inputs and outputs are float tensors (B, 3, T, H, W) in [-1, 1]; H and W must be multiples
of the spatial factor (use :func:`crop_to_multiple` first). The posterior mode is used, so
round trips are deterministic.
"""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
SPECS = {
# name: (diffusers class, repository, temporal factor, spatial factor, frame-count step)
"wan21": ("AutoencoderKLWan", "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", 4, 8, 4),
"wan22": ("AutoencoderKLWan", "Wan-AI/Wan2.2-TI2V-5B-Diffusers", 4, 16, 4),
"hunyuan": ("AutoencoderKLHunyuanVideo", "hunyuanvideo-community/HunyuanVideo", 4, 8, 4),
"cogx": ("AutoencoderKLCogVideoX", "THUDM/CogVideoX-5b", 4, 8, 8),
"ltx": ("AutoencoderKLLTXVideo", "Lightricks/LTX-Video", 8, 32, 8),
}
DTYPES = {"fp32": "float32", "bf16": "bfloat16", "fp16": "float16"}
@dataclass
class VAESpec:
name: str
cls: str
repo: str
t_factor: int
s_factor: int
t_step: int
def spec(name: str) -> VAESpec:
if name not in SPECS:
raise KeyError(f"Unknown VAE {name!r}; choose from {sorted(SPECS)}")
return VAESpec(name, *SPECS[name])
class VAE:
"""Thin wrapper: ``VAE("wan21").roundtrip(x)``.
``source`` may override the Hugging Face repository with a local diffusers folder
(either the model root containing ``vae/`` or the ``vae/`` folder itself).
"""
def __init__(self, name: str = "wan21", device: str = "cuda", dtype: str = "bf16", source: str | None = None):
import torch
import diffusers
from pathlib import Path
s = spec(name)
self.name, self.t_factor, self.s_factor, self.t_step = s.name, s.t_factor, s.s_factor, s.t_step
self.device = device
self.dtype = getattr(torch, DTYPES.get(dtype, dtype))
repo = source or s.repo
kwargs = {"torch_dtype": self.dtype}
if not (Path(repo).is_dir() and (Path(repo) / "config.json").is_file()):
kwargs["subfolder"] = "vae"
self.vae = getattr(diffusers, s.cls).from_pretrained(repo, **kwargs).to(device).eval()
self.vae.requires_grad_(False)
def valid_T(self, T: int) -> int:
"""Largest valid frame count (1 + n * t_step) not above T."""
return ((T - 1) // self.t_step) * self.t_step + 1
def roundtrip(self, x):
"""x: (B, 3, T, H, W) in [-1, 1] -> reconstruction of the same shape (float32)."""
import torch
with torch.inference_mode():
x = x.to(self.device, self.dtype)
post = self.vae.encode(x).latent_dist
z = post.mode() if hasattr(post, "mode") else post.mean
y = self.vae.decode(z, temb=None).sample if self.name == "ltx" else self.vae.decode(z).sample
if y.shape[2] != x.shape[2]:
raise RuntimeError(f"{self.name}: decoded {y.shape[2]} frames from {x.shape[2]}; "
f"windows must have 1 + {self.t_step}n frames")
return y.float()
def clear_cache(self) -> None:
clear = getattr(self.vae, "clear_cache", None)
if callable(clear):
clear()
def crop_to_multiple(frames: np.ndarray, m: int) -> np.ndarray:
"""Centre-crop uint8 frames (T, H, W, 3) so that H and W are multiples of m."""
T, H, W, _ = frames.shape
h, w = H - H % m, W - W % m
y0, x0 = (H - h) // 2, (W - w) // 2
return frames[:, y0:y0 + h, x0:x0 + w]
def to_tensor(frames: np.ndarray):
"""uint8 (T, H, W, 3) -> float tensor (1, 3, T, H, W) in [-1, 1]."""
import torch
t = torch.from_numpy(np.ascontiguousarray(frames)).permute(3, 0, 1, 2).float().unsqueeze(0)
return t / 127.5 - 1.0
def to_uint8(y) -> np.ndarray:
"""float tensor (1, 3, T, H, W) in [-1, 1] -> uint8 frames (T, H, W, 3)."""
return ((y.clamp(-1, 1) + 1) * 127.5).round().byte()[0].permute(1, 2, 3, 0).cpu().numpy()