Buckets:
| """MiniMax-H3 video VAE -- causal 3D-CNN encoder + non-causal 36-layer ViT decoder. | |
| 16x spatial / 4x temporal compression, 24 latent channels, per-channel | |
| latents_mean/latents_std normalization. The temporal geometry is fixed by | |
| ``clip_length`` (17 pixel frames per encoder chunk) and ``token_drop`` (3 | |
| trailing latent frames dropped after encoding): ``17n + 5`` pixel frames | |
| map to ``5n + 2`` latent frames. ``decode`` mirrors the chunking -- 5-token | |
| chunks with a 2-token overlap, cross-faded over 5 pixel frames. A single | |
| frame (image / i2v keyframe) skips the temporal chunking entirely and maps | |
| to a single latent frame. | |
| The reference pixel convention is ImageNet-normalized RGB over [0, 1]; | |
| ``encode``/``decode`` here speak ai-toolkit's [-1, 1] and convert inside. | |
| Spatial tiling (256 px tiles, >= 64 px overlap, linearly blended) is ON by | |
| default for both encode and decode -- the model was released that way and | |
| the released frames are the blended-tile ones. The checkpoint is float32 | |
| and the released decode recipe is fp16 autocast over the fp32 weights, with | |
| the token embedder, output projection, and every norm kept in fp32. | |
| """ | |
| import math | |
| from contextlib import nullcontext | |
| import torch | |
| from toolkit.models.v2._mixin import OstrisModelMixin | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.utils.checkpoint import checkpoint | |
| IMAGENET_MEAN = (0.485, 0.456, 0.406) | |
| IMAGENET_STD = (0.229, 0.224, 0.225) | |
| LATENTS_MEAN = [ | |
| 0.858090341091156, | |
| -0.9606591463088989, | |
| 1.0661640167236328, | |
| -0.5090325474739075, | |
| -0.2727581858634949, | |
| -1.3675414323806763, | |
| -0.2553254961967468, | |
| -0.26907554268836975, | |
| -0.5376840829849243, | |
| -0.0464097298681736, | |
| 0.6657370328903198, | |
| 0.19690127670764923, | |
| -0.5460608005523682, | |
| -0.4035342037677765, | |
| -0.23683024942874908, | |
| 0.25928452610969543, | |
| -0.30133944749832153, | |
| 0.211341992020607, | |
| -1.1206848621368408, | |
| 0.3581933379173279, | |
| -0.04225143790245056, | |
| 0.2604829967021942, | |
| 0.22864092886447906, | |
| 0.7056031823158264, | |
| ] | |
| LATENTS_STD = [ | |
| 1.2223774194717407, | |
| 1.2767263650894165, | |
| 1.68317747116088865, | |
| 1.7549455165863037, | |
| 1.5636216402053833, | |
| 2.194143533706665, | |
| 0.96531379222869875, | |
| 1.05698859691619875, | |
| 0.841948926448822, | |
| 0.7729952931404114, | |
| 1.8955937623977661, | |
| 0.946841835975647, | |
| 0.7996809482574463, | |
| 0.44988900423049925, | |
| 0.7197399735450745, | |
| 0.69362932443618775, | |
| 2.961095094680786, | |
| 2.7694199085235595, | |
| 3.0496184825897215, | |
| 2.1088054180145265, | |
| 3.276226282119751, | |
| 3.1627357006073, | |
| 2.28168129920959475, | |
| 2.6127843856811525, | |
| ] | |
| # --------------------------------------------------------------------------- | |
| # causal 3D-CNN encoder | |
| # --------------------------------------------------------------------------- | |
| class CausalConv3d(nn.Conv3d): | |
| """Conv3d with symmetric reflect spatial padding and causal (front-only, | |
| zeros) temporal padding of ``kernel_t - 1`` frames.""" | |
| def __init__(self, in_channels, out_channels, kernel_size, stride=1, spatial_pad=0): | |
| super().__init__( | |
| in_channels, out_channels, kernel_size=kernel_size, stride=stride | |
| ) | |
| self.spatial_pad = spatial_pad | |
| def forward(self, x): | |
| if self.spatial_pad > 0: | |
| p = self.spatial_pad | |
| x = F.pad(x, (p, p, p, p, 0, 0), mode="reflect") | |
| t_pad = self.kernel_size[0] - 1 | |
| if t_pad > 0: | |
| x = F.pad(x, (0, 0, 0, 0, t_pad, 0)) | |
| return F.conv3d(x, self.weight, self.bias, stride=self.stride) | |
| class FrameGroupNorm(nn.GroupNorm): | |
| """GroupNorm with statistics computed per frame (``use_t_isolated_gn``): | |
| the temporal axis is folded into the batch axis.""" | |
| def forward(self, x): | |
| b, c, t, h, w = x.shape | |
| x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) | |
| x = super().forward(x) | |
| return x.view(b, t, c, h, w).permute(0, 2, 1, 3, 4) | |
| class ResnetBlock3d(nn.Module): | |
| def __init__(self, in_channels, out_channels, norm_num_groups=32): | |
| super().__init__() | |
| self.norm1 = FrameGroupNorm(norm_num_groups, in_channels, eps=1e-6) | |
| self.conv1 = CausalConv3d(in_channels, out_channels, 3, spatial_pad=1) | |
| self.norm2 = FrameGroupNorm(norm_num_groups, out_channels, eps=1e-6) | |
| self.conv2 = CausalConv3d(out_channels, out_channels, 3, spatial_pad=1) | |
| if in_channels != out_channels: | |
| self.nin_shortcut = CausalConv3d(in_channels, out_channels, 1) | |
| else: | |
| self.nin_shortcut = None | |
| def forward(self, x): | |
| h = self.conv1(F.silu(self.norm1(x))) | |
| h = self.conv2(F.silu(self.norm2(h))) | |
| if self.nin_shortcut is not None: | |
| x = self.nin_shortcut(x) | |
| return x + h | |
| class Downsample3d(nn.Module): | |
| """Strided 3x3x3 conv; a spatial stride of 2 is preceded by an asymmetric | |
| bottom/right reflect pad of 1 so the output is exactly ceil(size / 2).""" | |
| def __init__(self, channels, time_stride, space_stride): | |
| super().__init__() | |
| self.space_stride = space_stride | |
| self.conv = CausalConv3d( | |
| channels, channels, 3, stride=(time_stride, space_stride, space_stride) | |
| ) | |
| def forward(self, x): | |
| if self.space_stride == 2: | |
| x = F.pad(x, (0, 1, 0, 1, 0, 0), mode="reflect") | |
| return self.conv(x) | |
| class Encoder3d(nn.Module): | |
| def __init__( | |
| self, | |
| in_channels, | |
| out_channels, | |
| block_out_channels, | |
| layers_per_block, | |
| space_down, | |
| time_down, | |
| norm_num_groups, | |
| ): | |
| super().__init__() | |
| block_in = (block_out_channels[0],) + tuple(block_out_channels[:-1]) | |
| self.conv_in = CausalConv3d(in_channels, block_in[0], 3, spatial_pad=1) | |
| self.down = nn.ModuleList() | |
| for i in range(len(block_out_channels)): | |
| level = nn.Module() | |
| level.block = nn.ModuleList( | |
| ResnetBlock3d( | |
| block_in[i] if j == 0 else block_out_channels[i], | |
| block_out_channels[i], | |
| norm_num_groups=norm_num_groups, | |
| ) | |
| for j in range(layers_per_block) | |
| ) | |
| if space_down[i] * time_down[i] > 1: | |
| level.downsample = Downsample3d( | |
| block_out_channels[i], time_down[i], space_down[i] | |
| ) | |
| self.down.append(level) | |
| self.norm_out = FrameGroupNorm( | |
| norm_num_groups, block_out_channels[-1], eps=1e-6 | |
| ) | |
| self.conv_out = CausalConv3d( | |
| block_out_channels[-1], out_channels, 3, spatial_pad=1 | |
| ) | |
| def forward(self, x): | |
| h = self.conv_in(x) | |
| for level in self.down: | |
| for block in level.block: | |
| h = block(h) | |
| if hasattr(level, "downsample"): | |
| h = level.downsample(h) | |
| return self.conv_out(F.silu(self.norm_out(h))) | |
| # --------------------------------------------------------------------------- | |
| # non-causal ViT decoder | |
| # --------------------------------------------------------------------------- | |
| class RMSNormFP32(nn.RMSNorm): | |
| """The reference normalizes in float32 regardless of the compute dtype | |
| (or the storage dtype — the Comfy repack ships this VAE in fp16).""" | |
| def forward(self, x): | |
| weight = None if self.weight is None else self.weight.float() | |
| out = F.rms_norm(x.float(), self.normalized_shape, weight, self.eps) | |
| return out.to(x.dtype) | |
| class RotaryEmbedding3d(nn.Module): | |
| """3-axis rotary table. Coordinates are length-normalized to [-1, 1) per | |
| axis and scaled by 2*pi; the per-axis angles are concatenated and then | |
| duplicated, so the first ``rope_dim_ratio * head_dim`` channels of every | |
| head are rotated (rotate-half convention).""" | |
| def __init__(self, dim, theta=100.0, num_axes=3): | |
| super().__init__() | |
| inv_freq = 1.0 / theta ** torch.arange( | |
| 0, 1, 2 * num_axes / dim, dtype=torch.float32 | |
| ) | |
| self.register_buffer("inv_freq", inv_freq, persistent=False) | |
| def forward(self, position_ids): | |
| angles = ( | |
| 2.0 | |
| * math.pi | |
| * position_ids[:, :, :, None].float() | |
| * self.inv_freq[None, None, None, :] | |
| ) | |
| angles = angles.flatten(2, 3).tile(2).unsqueeze(2) # (B, S, 1, rot_dim) | |
| return angles.cos(), angles.sin() | |
| def _rotate_half(x): | |
| x1, x2 = x.chunk(2, dim=-1) | |
| return torch.cat([-x2, x1], dim=-1) | |
| def _apply_rope(x, cos, sin): | |
| rot = cos.shape[-1] | |
| x_rot, x_pass = x[..., :rot], x[..., rot:] | |
| x_rot = x_rot * cos + _rotate_half(x_rot) * sin | |
| return torch.cat([x_rot, x_pass], dim=-1) | |
| class Attention(nn.Module): | |
| def __init__(self, heads, head_dim, eps=1e-5): | |
| super().__init__() | |
| self.heads = heads | |
| self.head_dim = head_dim | |
| inner = heads * head_dim | |
| self.norm_q = RMSNormFP32(head_dim, eps=eps, elementwise_affine=False) | |
| self.norm_k = RMSNormFP32(head_dim, eps=eps, elementwise_affine=False) | |
| self.to_qkv = nn.Linear(inner, inner * 3, bias=True) | |
| self.to_out = nn.Linear(inner, inner, bias=True) | |
| def forward(self, x, rotary_emb): | |
| b, s, _ = x.shape | |
| # per-head-interleaved qkv layout: each head's 3 * head_dim slab is | |
| # split into q | k | v, NOT [Q_all | K_all | V_all] | |
| qkv = self.to_qkv(x).view(b, s, self.heads, 3 * self.head_dim) | |
| q, k, v = qkv.chunk(3, dim=-1) | |
| q = self.norm_q(q) | |
| k = self.norm_k(k) | |
| cos, sin = rotary_emb | |
| cos = cos.to(q.dtype) | |
| sin = sin.to(q.dtype) | |
| q = _apply_rope(q, cos, sin) | |
| k = _apply_rope(k, cos, sin) | |
| out = F.scaled_dot_product_attention( | |
| q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) | |
| ) | |
| return self.to_out(out.transpose(1, 2).reshape(b, s, -1)) | |
| class FeedForward(nn.Module): | |
| """Gated SiLU: w1 output chunks into gate | value, silu(gate) * value.""" | |
| def __init__(self, dim, mult=4): | |
| super().__init__() | |
| inner = dim * mult | |
| self.w1 = nn.Linear(dim, inner * 2, bias=True) | |
| self.w2 = nn.Linear(inner, dim, bias=True) | |
| def forward(self, x): | |
| gate, x = self.w1(x).chunk(2, dim=-1) | |
| return self.w2(F.silu(gate) * x) | |
| class TransformerBlock(nn.Module): | |
| def __init__(self, heads, head_dim, ffn_mult=4, eps=1e-5): | |
| super().__init__() | |
| dim = heads * head_dim | |
| self.norm1 = RMSNormFP32(dim, eps=eps, elementwise_affine=True) | |
| self.attn = Attention(heads, head_dim, eps=eps) | |
| self.scale1 = nn.Parameter(torch.zeros(dim)) | |
| self.norm2 = RMSNormFP32(dim, eps=eps, elementwise_affine=True) | |
| self.ff = FeedForward(dim, mult=ffn_mult) | |
| self.scale2 = nn.Parameter(torch.zeros(dim)) | |
| def forward(self, x, rotary_emb): | |
| x = x + self.attn(self.norm1(x), rotary_emb) * self.scale1 | |
| return x + self.ff(self.norm2(x)) * self.scale2 | |
| class ViTDecoder3d(nn.Module): | |
| """Every latent voxel is one token; 4 learned register tokens plus one | |
| all-zero token are appended (all at rope coordinate 0), attended over with | |
| full self-attention, and dropped before the patch projection expands each | |
| token into a patch_size_t x patch_size x patch_size pixel block.""" | |
| def __init__( | |
| self, | |
| in_channels, | |
| out_channels, | |
| patch_size, | |
| patch_size_t, | |
| num_layers, | |
| heads, | |
| head_dim, | |
| num_register_tokens=4, | |
| ffn_mult=4, | |
| rope_theta=100.0, | |
| rope_dim_ratio=0.75, | |
| eps=1e-5, | |
| ): | |
| super().__init__() | |
| dim = heads * head_dim | |
| self.patch_size = patch_size | |
| self.patch_size_t = patch_size_t | |
| self.out_channels = out_channels | |
| self.num_register_tokens = num_register_tokens | |
| self.rope = RotaryEmbedding3d(int(head_dim * rope_dim_ratio), theta=rope_theta) | |
| self.x_embedder = nn.Linear(in_channels, dim) | |
| self.register_tokens = nn.Parameter(torch.zeros(1, num_register_tokens, dim)) | |
| # unused at inference; kept so the checkpoint loads strict | |
| self.register_buffer("mask_token", torch.zeros(1, 1, dim)) | |
| self.transformer_blocks = nn.ModuleList( | |
| TransformerBlock(heads, head_dim, ffn_mult=ffn_mult, eps=eps) | |
| for _ in range(num_layers) | |
| ) | |
| self.norm_out = nn.LayerNorm(dim, eps=eps) | |
| self.proj_out = nn.Linear( | |
| dim, out_channels * patch_size_t * patch_size * patch_size | |
| ) | |
| # on by default: only engages when gradients are enabled (pixel-space | |
| # losses through decode); no_grad sampling never takes the branch | |
| self.gradient_checkpointing = True | |
| def forward(self, z): | |
| b, c, t, h, w = z.shape | |
| tokens = z.permute(0, 2, 3, 4, 1).reshape(b, t * h * w, c) | |
| # released recipe: token embedder / output projection / norms compute | |
| # fp32 even when the matmuls run fp16 (autocast or fp16 storage); the | |
| # block stack then runs at the storage dtype | |
| with torch.autocast(device_type=z.device.type, enabled=False): | |
| x = F.linear( | |
| tokens.float(), | |
| self.x_embedder.weight.float(), | |
| self.x_embedder.bias.float(), | |
| ) | |
| x = x.to(self.x_embedder.weight.dtype) | |
| num_patches = x.shape[1] | |
| x = torch.cat( | |
| [ | |
| x, | |
| self.register_tokens.expand(b, -1, -1).to(x.dtype), | |
| torch.zeros_like(x[:, :1]), | |
| ], | |
| dim=1, | |
| ) | |
| grids = [ | |
| 2.0 * (torch.arange(0.5, size, dtype=torch.float32, device=z.device) / size) | |
| - 1.0 | |
| for size in (t, h, w) | |
| ] | |
| position_ids = torch.stack( | |
| torch.meshgrid(*grids, indexing="ij"), dim=-1 | |
| ).flatten(0, 2) | |
| position_ids = position_ids.unsqueeze(0).expand(b, -1, -1) | |
| suffix_ids = position_ids.new_zeros((b, self.num_register_tokens + 1, 3)) | |
| with torch.autocast(device_type=z.device.type, enabled=False): | |
| rotary_emb = self.rope(torch.cat([position_ids, suffix_ids], dim=1)) | |
| for block in self.transformer_blocks: | |
| if torch.is_grad_enabled() and self.gradient_checkpointing: | |
| # pixel-space losses backprop through all 36 blocks; without | |
| # checkpointing the stored activations OOM at video sizes. | |
| # Inference runs under no_grad, where this branch is never | |
| # taken, so sampling pays nothing. | |
| x = checkpoint(block, x, rotary_emb, use_reentrant=False) | |
| else: | |
| x = block(x, rotary_emb) | |
| with torch.autocast(device_type=z.device.type, enabled=False): | |
| x = F.layer_norm( | |
| x.float(), | |
| self.norm_out.normalized_shape, | |
| self.norm_out.weight.float(), | |
| self.norm_out.bias.float(), | |
| self.norm_out.eps, | |
| ) | |
| x = F.linear(x, self.proj_out.weight.float(), self.proj_out.bias.float()) | |
| x = x[:, :num_patches] | |
| pt, ps = self.patch_size_t, self.patch_size | |
| x = x.view(b, t, h, w, self.out_channels, pt, ps, ps) | |
| x = x.permute(0, 4, 1, 5, 2, 6, 3, 7) | |
| return x.reshape(b, self.out_channels, t * pt, h * ps, w * ps) | |
| # --------------------------------------------------------------------------- | |
| # full VAE | |
| # --------------------------------------------------------------------------- | |
| class MiniMaxH3VideoVAE(nn.Module, OstrisModelMixin): | |
| def load_from_state_dict(cls, state_dict, dtype=None, **kwargs): | |
| """Comfy single-file load: the normalization stats ride along in the | |
| file and land in the non-persistent fp32 buffers.""" | |
| state_dict = dict(state_dict) | |
| stats = { | |
| k: state_dict.pop(k).float() | |
| for k in ("latents_mean", "latents_std") | |
| if k in state_dict | |
| } | |
| model = cls() | |
| model.load_state_dict(state_dict, strict=True, assign=True) | |
| for k, v in stats.items(): | |
| getattr(model, k).copy_(v) | |
| model.eval().requires_grad_(False) | |
| return model | |
| def __init__( | |
| self, | |
| in_channels=3, | |
| out_channels=3, | |
| latent_channels=24, | |
| block_out_channels=(128, 256, 256, 512, 512, 1024), | |
| layers_per_block=2, | |
| space_down=(2, 2, 2, 2, 1, 1), | |
| time_down=(1, 2, 2, 1, 1, 1), | |
| norm_num_groups=32, | |
| decoder_num_layers=36, | |
| decoder_heads=32, | |
| decoder_head_dim=64, | |
| decoder_num_register_tokens=4, | |
| decoder_ffn_mult=4, | |
| rope_theta=100.0, | |
| rope_dim_ratio=0.75, | |
| clip_length=17, | |
| token_drop=3, | |
| latents_mean=None, | |
| latents_std=None, | |
| tile_size=256, | |
| tile_overlap_min=64, | |
| tiling=True, | |
| ): | |
| super().__init__() | |
| self.latent_channels = latent_channels | |
| self.spatial_compression = int(math.prod(space_down)) | |
| self.temporal_compression = int(math.prod(time_down)) | |
| self.clip_length = clip_length | |
| self.token_drop = token_drop | |
| # clip_length is not a multiple of the temporal ratio, so each decoded | |
| # chunk carries an implicit leading pad; token_drop leaves an overlap | |
| # between consecutive chunks that gets cross-faded in pixel space | |
| self.frame_pre_padding = (-clip_length) % self.temporal_compression | |
| self.tokens_chunk_size = math.ceil(clip_length / self.temporal_compression) | |
| self.token_overlap = (-token_drop) % self.tokens_chunk_size | |
| self.frame_overlap = max( | |
| self.token_overlap * self.temporal_compression - self.frame_pre_padding, 0 | |
| ) | |
| self.use_tiling = tiling | |
| self.tile_size = tile_size | |
| self.tile_overlap_min = tile_overlap_min | |
| self.encoder = Encoder3d( | |
| in_channels=in_channels, | |
| out_channels=2 * latent_channels, | |
| block_out_channels=block_out_channels, | |
| layers_per_block=layers_per_block, | |
| space_down=space_down, | |
| time_down=time_down, | |
| norm_num_groups=norm_num_groups, | |
| ) | |
| self.quant_conv = nn.Conv3d(2 * latent_channels, 2 * latent_channels, 1) | |
| self.post_quant_conv = nn.Conv3d(latent_channels, latent_channels, 1) | |
| self.decoder = ViTDecoder3d( | |
| in_channels=latent_channels, | |
| out_channels=out_channels, | |
| patch_size=self.spatial_compression, | |
| patch_size_t=self.temporal_compression, | |
| num_layers=decoder_num_layers, | |
| heads=decoder_heads, | |
| head_dim=decoder_head_dim, | |
| num_register_tokens=decoder_num_register_tokens, | |
| ffn_mult=decoder_ffn_mult, | |
| rope_theta=rope_theta, | |
| rope_dim_ratio=rope_dim_ratio, | |
| ) | |
| if latents_mean is None: | |
| latents_mean = ( | |
| LATENTS_MEAN | |
| if latent_channels == len(LATENTS_MEAN) | |
| else [0.0] * latent_channels | |
| ) | |
| if latents_std is None: | |
| latents_std = ( | |
| LATENTS_STD | |
| if latent_channels == len(LATENTS_STD) | |
| else [1.0] * latent_channels | |
| ) | |
| self.register_buffer( | |
| "latents_mean", | |
| torch.tensor(latents_mean, dtype=torch.float32), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "latents_std", | |
| torch.tensor(latents_std, dtype=torch.float32), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "pixel_mean", | |
| torch.tensor(IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1, 1), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "pixel_std", | |
| torch.tensor(IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1, 1), | |
| persistent=False, | |
| ) | |
| def device(self): | |
| return next(self.parameters()).device | |
| def dtype(self): | |
| return next(self.parameters()).dtype | |
| def enable_gradient_checkpointing(self, enable: bool = True): | |
| self.decoder.gradient_checkpointing = enable | |
| def disable_gradient_checkpointing(self): | |
| self.enable_gradient_checkpointing(False) | |
| def latent_frames(num_pixel_frames: int) -> int: | |
| """17n + 5 pixel frames -> 5n + 2 latent frames; 1 -> 1 (keyframe).""" | |
| if num_pixel_frames == 1: | |
| return 1 | |
| return 5 * math.ceil(num_pixel_frames / 17) - 3 | |
| def pixel_frames(num_latent_frames: int) -> int: | |
| """5n + 2 latent frames -> 17n + 5 pixel frames; 1 -> 1 (keyframe).""" | |
| if num_latent_frames == 1: | |
| return 1 | |
| n, rem = divmod(num_latent_frames - 2, 5) | |
| if rem != 0 or n < 0: | |
| raise ValueError( | |
| f"latent frame count must be 5n + 2 or 1, got {num_latent_frames}" | |
| ) | |
| return 17 * n + 5 | |
| # -- spatial tiling ------------------------------------------------------ | |
| def _split_tiles(self, length): | |
| """Lay tile_size-wide tiles over length pixels; the slack beyond the | |
| minimum overlap is spread round-robin in whole spatial_compression | |
| steps so every tile boundary stays latent-aligned.""" | |
| tile_size = self.tile_size | |
| if tile_size >= length: | |
| return [0], [length], [] | |
| num_tiles = math.ceil(length / tile_size) | |
| while tile_size * num_tiles - self.tile_overlap_min * (num_tiles - 1) < length: | |
| num_tiles += 1 | |
| overlaps = [self.tile_overlap_min] * (num_tiles - 1) | |
| remaining = tile_size * num_tiles - sum(overlaps) - length | |
| for i in range(remaining // self.spatial_compression): | |
| overlaps[i % (num_tiles - 1)] += self.spatial_compression | |
| starts = [0] | |
| for i in range(num_tiles - 1): | |
| starts.append(starts[-1] + tile_size - overlaps[i]) | |
| return starts, [tile_size] * num_tiles, overlaps | |
| def _blend(a, b, blend_extent, dim): | |
| blend_extent = min(a.shape[dim], b.shape[dim], blend_extent) | |
| positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype) | |
| shape = [1] * b.ndim | |
| shape[dim] = blend_extent | |
| weight_b = (positions / blend_extent).view(shape) | |
| slice_a = [slice(None)] * a.ndim | |
| slice_a[dim] = slice(-blend_extent, None) | |
| slice_b = [slice(None)] * b.ndim | |
| slice_b[dim] = slice(0, blend_extent) | |
| blended = a[tuple(slice_a)] * (1 - weight_b) + b[tuple(slice_b)] * weight_b | |
| if blend_extent == b.shape[dim]: | |
| return blended | |
| slice_rest = [slice(None)] * b.ndim | |
| slice_rest[dim] = slice(blend_extent, None) | |
| return torch.cat([blended, b[tuple(slice_rest)]], dim=dim) | |
| def _stitch_tiles(self, rows, y_overlaps, x_overlaps): | |
| result_rows = [] | |
| for i, row in enumerate(rows): | |
| result_row = [] | |
| for j, tile in enumerate(row): | |
| if i > 0: | |
| tile = self._blend(rows[i - 1][j], tile, y_overlaps[i - 1], dim=-2) | |
| if j > 0: | |
| tile = self._blend(row[j - 1], tile, x_overlaps[j - 1], dim=-1) | |
| if i < len(rows) - 1: | |
| tile = tile[..., : -y_overlaps[i], :] | |
| if j < len(row) - 1: | |
| tile = tile[..., :, : -x_overlaps[j]] | |
| result_row.append(tile) | |
| result_rows.append(torch.cat(result_row, dim=-1)) | |
| return torch.cat(result_rows, dim=-2) | |
| def _encode_clip(self, x): | |
| """Encode one temporal clip to moments, spatially tiled when enabled.""" | |
| if not self.use_tiling: | |
| return self.quant_conv(self.encoder(x)) | |
| y_starts, y_lens, y_overlaps = self._split_tiles(x.shape[-2]) | |
| x_starts, x_lens, x_overlaps = self._split_tiles(x.shape[-1]) | |
| rows = [] | |
| for i_pos, i_len in zip(y_starts, y_lens): | |
| row = [] | |
| for j_pos, j_len in zip(x_starts, x_lens): | |
| tile = x[..., i_pos : i_pos + i_len, j_pos : j_pos + j_len] | |
| row.append(self.quant_conv(self.encoder(tile))) | |
| rows.append(row) | |
| ratio = self.spatial_compression | |
| return self._stitch_tiles( | |
| rows, [o // ratio for o in y_overlaps], [o // ratio for o in x_overlaps] | |
| ) | |
| def _decode_clip(self, z): | |
| """Decode one temporal clip; tiles are laid out in pixel space and | |
| mapped back onto the latent grid, blended in pixel space.""" | |
| if not self.use_tiling: | |
| return self.decoder(self.post_quant_conv(z)) | |
| ratio = self.spatial_compression | |
| y_starts, y_lens, y_overlaps = self._split_tiles(z.shape[-2] * ratio) | |
| x_starts, x_lens, x_overlaps = self._split_tiles(z.shape[-1] * ratio) | |
| rows = [] | |
| for i_pos, i_len in zip(y_starts, y_lens): | |
| row = [] | |
| for j_pos, j_len in zip(x_starts, x_lens): | |
| tile = z[ | |
| ..., | |
| i_pos // ratio : (i_pos + i_len) // ratio, | |
| j_pos // ratio : (j_pos + j_len) // ratio, | |
| ] | |
| row.append(self.decoder(self.post_quant_conv(tile))) | |
| rows.append(row) | |
| return self._stitch_tiles(rows, y_overlaps, x_overlaps) | |
| # -- temporal chunking --------------------------------------------------- | |
| def _encode_video(self, x): | |
| if x.shape[2] % self.clip_length != 0: | |
| pad = x[:, :, -1:].repeat(1, 1, (-x.shape[2]) % self.clip_length, 1, 1) | |
| x = torch.cat([x, pad], dim=2) | |
| moments = torch.cat( | |
| [ | |
| self._encode_clip( | |
| x[:, :, i * self.clip_length : (i + 1) * self.clip_length] | |
| ) | |
| for i in range(x.shape[2] // self.clip_length) | |
| ], | |
| dim=2, | |
| ) | |
| if self.token_drop > 0: | |
| moments = moments[:, :, : -self.token_drop] | |
| return moments | |
| def _decode_video(self, z): | |
| tcs = self.tokens_chunk_size | |
| ratio_t = self.temporal_compression | |
| chunk_frames = tcs * ratio_t | |
| split_count = 2 if self.token_drop > 0 else 1 | |
| num_tokens = z.shape[2] + self.token_drop | |
| pad_tokens = (-num_tokens) % tcs | |
| num_chunks = (num_tokens + pad_tokens) // tcs - (split_count - 1) | |
| if num_chunks < 1: | |
| # too few tokens for one chunk (e.g. the 2-latent minimum clip) | |
| pad_tokens += tcs | |
| num_chunks += 1 | |
| if pad_tokens > 0: | |
| z = torch.cat([z, z[:, :, -1:].repeat(1, 1, pad_tokens, 1, 1)], dim=2) | |
| decoded = [] | |
| overlap = None | |
| for i in range(num_chunks): | |
| start = i * tcs | |
| clip = self._decode_clip(z[:, :, start : start + tcs + self.token_overlap]) | |
| for j in range(split_count): | |
| part = clip[:, :, j * chunk_frames : (j + 1) * chunk_frames] | |
| part = part[:, :, self.frame_pre_padding :] | |
| if j == 0: | |
| if overlap is not None: | |
| part = self._blend(overlap, part, self.frame_overlap, dim=-3) | |
| decoded.append(part) | |
| else: | |
| overlap = part | |
| if overlap is not None: | |
| decoded.append(overlap) | |
| dec = torch.cat(decoded, dim=2) | |
| if pad_tokens > 0: | |
| # repeated latent frames produced trailing pixel frames that were | |
| # never requested; a chunk's last token only covers | |
| # clip_length % ratio_t frames, the others cover ratio_t | |
| intra_tail = self.clip_length % ratio_t | |
| before_pad = z.shape[2] - pad_tokens | |
| pad_frames = sum( | |
| intra_tail if intra_tail and (before_pad + k) % tcs == 0 else ratio_t | |
| for k in range(pad_tokens) | |
| ) | |
| dec = dec[:, :, :-pad_frames] | |
| return dec | |
| # -- public interface ---------------------------------------------------- | |
| def encode(self, pixels, sample=True, generator=None, fp16_round=False): | |
| """pixels (B, 3, T, H, W) in [-1, 1], T == 17n + 5, or T == 1 (single | |
| keyframe: spatial encode only, no temporal chunking). Returns | |
| normalized latents (B, latent_channels, t, h, w): posterior sampled | |
| (or the mean when sample=False), optionally rounded through fp16 | |
| BEFORE the (z - latents_mean) / latents_std normalization | |
| (fp16_round=True is the released first-frame-conditioning recipe).""" | |
| x = pixels | |
| if x.ndim == 4: | |
| x = x.unsqueeze(2) | |
| x = (x.float() + 1.0) * 0.5 | |
| x = (x - self.pixel_mean) / self.pixel_std | |
| x = x.to(self.dtype) | |
| if x.shape[2] == 1: | |
| moments = self._encode_clip(x)[:, :, -1:] | |
| else: | |
| moments = self._encode_video(x) | |
| mean, logvar = moments.float().chunk(2, dim=1) | |
| if sample: | |
| std = torch.exp(0.5 * logvar.clamp(-30.0, 20.0)) | |
| noise = torch.randn( | |
| mean.shape, | |
| generator=generator, | |
| device=generator.device if generator is not None else mean.device, | |
| dtype=mean.dtype, | |
| ).to(mean.device) | |
| z = mean + std * noise | |
| else: | |
| z = mean | |
| if fp16_round: | |
| z = z.to(torch.float16).float() | |
| shape = (1, -1, 1, 1, 1) | |
| return (z - self.latents_mean.view(shape)) / self.latents_std.view(shape) | |
| def decode(self, latents, autocast_fp16=True): | |
| """Normalized latents (B, latent_channels, t, h, w) -> pixels | |
| (B, 3, T, H, W) in [-1, 1]. Runs under fp16 autocast over the fp32 | |
| weights on cuda (the released recipe) unless autocast_fp16=False.""" | |
| shape = (1, -1, 1, 1, 1) | |
| z = latents.float() * self.latents_std.view(shape) + self.latents_mean.view( | |
| shape | |
| ) | |
| ctx = ( | |
| torch.autocast(device_type="cuda", dtype=torch.float16) | |
| if autocast_fp16 and z.is_cuda | |
| else nullcontext() | |
| ) | |
| with ctx: | |
| if z.shape[2] == 1: | |
| # a lone temporal token is OOD for the chunk-trained ViT | |
| # decoder (visibly patchy output, ~16 dB vs ~35 dB roundtrip). | |
| # Decode it as the first latent of a 2-latent clip instead: | |
| # the first latent of a chunk covers exactly pixel frame 0 in | |
| # the (1, 4, 4, 4, 4) grouping, so that frame is a clean, | |
| # in-distribution single-frame decode. | |
| dec = self._decode_video(torch.cat([z, z], dim=2))[:, :, :1] | |
| else: | |
| dec = self._decode_video(z) | |
| # out-of-place so gradients can flow through decode (pixel-space losses) | |
| dec = dec.float() * self.pixel_std + self.pixel_mean | |
| return dec.clamp(0.0, 1.0) * 2.0 - 1.0 | |
Xet Storage Details
- Size:
- 31.2 kB
- Xet hash:
- 0b3a0f65d1063dfca0f1597b3e78c5ab94eb7f2f6a72b809a19163b497375fc8
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.