Buckets:
| """MiniMax-H3 diffusion transformer (33B), weight-compatible with the original | |
| ``MiniMaxAI/MiniMax-H3`` checkpoint keys (``video_patch_proj``, ``blocks.N.*``, | |
| ``final_layer.*``, ...). | |
| MiniMax-H3 runs one stack of blocks over a single packed 1-D sequence holding | |
| ``[text | keyframe-condition video rows | audio rows | target video rows]``. | |
| Full self-attention, no cross-attention, no per-modality weights: modality- | |
| specific behavior comes only from the two input patch projections, the per-row | |
| AdaLN modality tag (0 = video, 1 = text, 2 = audio) and the two output heads. | |
| Deviations from the reference implementations (both are pure generalizations — | |
| with batch size 1 and no padding the math is identical): | |
| - every structural input (``position_ids``, ``token_tags``, ``row_timesteps``) | |
| carries a batch axis, so items with different prompt lengths can share a | |
| training batch (pad rows are tagged ``-1`` and masked out of attention) | |
| - timesteps are passed per row and deduplicated internally instead of the | |
| caller supplying ``timestep`` + ``timestep_indices`` | |
| Mixed precision follows the shipped checkpoint: ``video_patch_proj``, | |
| ``audio_patch_proj``, ``time_embedder``, ``final_layer.video_out/audio_out`` | |
| and ``rope.inv_freq`` are float32 islands, everything else runs bf16. Inputs | |
| are aligned to each projection's own parameter dtype at the call site. | |
| Timesteps are consumed unscaled in [0, 1] with t = 1 - sigma (t=1 means clean), | |
| and both heads predict the data-ward velocity ``clean - noise``. | |
| """ | |
| import math | |
| from dataclasses import dataclass | |
| from typing import Optional, Tuple | |
| from toolkit.models.v2._mixin import OstrisModelMixin | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from torch.utils.checkpoint import checkpoint | |
| MODALITY_NUM = 3 # 0 = video, 1 = text, 2 = audio; -1 marks padding rows | |
| class MiniMaxH3TransformerParams: | |
| hidden_size: int = 5376 | |
| num_layers: int = 50 | |
| token_refiner_num_layers: int = 2 | |
| num_attention_heads: int = 56 | |
| attention_head_dim: int = 128 # heads * head_dim = 7168 > hidden_size | |
| ffn_hidden_size: int = 14336 | |
| latents_dim: int = 24 | |
| audio_latents_dim: int = 32 | |
| patch_size: Tuple[int, int, int] = (1, 2, 2) | |
| text_dim: int = 5120 | |
| timestep_input_dim: int = 256 | |
| time_embed_hidden_size: int = 5376 | |
| time_embed_dim: int = 2688 | |
| rope_inv_freq_len: int = 16 | |
| rope_theta: float = 10000.0 | |
| norm_eps: float = 1e-5 | |
| qk_norm_eps: float = 1e-5 | |
| final_norm_eps: float = 1e-5 | |
| # VSA-trained checkpoints (FastVideo FastH3) carry a per-token gate for | |
| # the coarse compression branch: blocks.N.attn.to_gate_compress | |
| gate_compress: bool = False | |
| # "pruned" checkpoints (e.g. Comfy-Org *_pruned_*) replace the timestep | |
| # MLP with a small lookup table: ``adaln_t_table`` of shape | |
| # (adaln_t_table_size, time_embed_dim) sampled by linear interpolation at | |
| # t * (size - 1), consumed by the AdaLN projections WITHOUT the SiLU. | |
| # These checkpoints also shrink time_embed_dim (8 in the released files) | |
| # and add a bias to the block AdaLN linears, which the original weights | |
| # lack (the final layer's AdaLN carries a bias in both variants). | |
| adaln_t_table_size: Optional[int] = None | |
| def adaln_apply_silu(self) -> bool: | |
| return self.adaln_t_table_size is None | |
| def adaln_bias(self) -> bool: | |
| return self.adaln_t_table_size is not None | |
| class MiniMaxH3Rope(nn.Module): | |
| """3-axis rotary embedding over the packed (t, h, w) coordinates. | |
| One shared ``inv_freq`` of 16 frequencies per axis; the three 16-angle | |
| blocks concatenate to 48 and duplicate to 96, so the first 96 of the 128 | |
| head channels rotate (rotate-half convention) and the last 32 pass through. | |
| """ | |
| def __init__(self, inv_freq_len: int = 16, theta: float = 10000.0): | |
| super().__init__() | |
| dim = 2 * inv_freq_len | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) | |
| # present in the checkpoint, so persistent | |
| self.register_buffer("inv_freq", inv_freq, persistent=True) | |
| def forward(self, position_ids: torch.Tensor): | |
| """position_ids (B, S, 3) -> cos, sin each (B, S, 96), float32.""" | |
| # compute on the input's device: the frequency buffer may be left | |
| # CPU-resident by layer offloading / low_vram loads | |
| position_ids = position_ids.to(dtype=torch.float32) | |
| inv_freq = self.inv_freq.to(position_ids.device) | |
| freqs = position_ids.unsqueeze(-1) * inv_freq.view(1, 1, 1, -1) | |
| # (B, S, 3, 16) -> (B, S, 48) in (t, h, w) axis order -> duplicate to 96 | |
| freqs = freqs.flatten(2, 3) | |
| freqs = torch.cat([freqs, freqs], dim=-1) | |
| return freqs.cos(), freqs.sin() | |
| def apply_rotary_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): | |
| """x (B, S, H, D); cos/sin (B, S, rot) rotate the leading ``rot`` channels.""" | |
| rot = cos.shape[-1] | |
| x_rot, x_pass = x[..., :rot], x[..., rot:] | |
| cos = cos.to(x.dtype).unsqueeze(2) | |
| sin = sin.to(x.dtype).unsqueeze(2) | |
| x1, x2 = x_rot.chunk(2, dim=-1) | |
| rotated = torch.cat([-x2, x1], dim=-1) | |
| return torch.cat([x_rot * cos + rotated * sin, x_pass], dim=-1) | |
| class MiniMaxH3TimeEmbedder(nn.Module): | |
| """Sinusoidal embedding (cos before sin, unscaled t in [0, 1]) -> MLP. | |
| A float32 island in the checkpoint; the output stays float32 so every | |
| block's AdaLN applies its SiLU at full precision before casting down. | |
| """ | |
| def __init__(self, freq_dim: int, hidden: int, out_dim: int): | |
| super().__init__() | |
| self.freq_dim = freq_dim | |
| self.proj_in = nn.Linear(freq_dim, hidden, bias=True) | |
| self.proj_out = nn.Linear(hidden, out_dim, bias=True) | |
| def forward(self, t: torch.Tensor) -> torch.Tensor: | |
| half = self.freq_dim // 2 | |
| freqs = torch.exp( | |
| -math.log(10000.0) | |
| * torch.arange(half, dtype=torch.float32, device=t.device) | |
| / half | |
| ) | |
| args = t.to(torch.float32)[:, None] * freqs[None] | |
| emb = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) | |
| emb = emb.to(self.proj_in.weight.dtype) | |
| return self.proj_out(F.silu(self.proj_in(emb))) | |
| class MiniMaxH3Attention(nn.Module): | |
| """Fused-QKV self-attention with per-head RMSNorm on q/k and partial RoPE.""" | |
| def __init__( | |
| self, | |
| hidden: int, | |
| heads: int, | |
| head_dim: int, | |
| qk_norm_eps: float, | |
| gate_compress: bool = False, | |
| ): | |
| super().__init__() | |
| self.heads = heads | |
| self.head_dim = head_dim | |
| inner = heads * head_dim | |
| self.qkv_proj = nn.Linear(hidden, inner * 3, bias=False) | |
| self.q_norm = nn.RMSNorm(head_dim, eps=qk_norm_eps) | |
| self.k_norm = nn.RMSNorm(head_dim, eps=qk_norm_eps) | |
| self.out_proj = nn.Linear(inner, hidden, bias=False) | |
| self.to_gate_compress = ( | |
| nn.Linear(hidden, inner, bias=False) if gate_compress else None | |
| ) | |
| def forward( | |
| self, | |
| x: torch.Tensor, # (B, S, hidden) | |
| rotary_emb=None, # (cos, sin) each (B, S, rot) or None | |
| attn_mask: Optional[torch.Tensor] = None, # (B, 1, 1, S) bool, True = attend | |
| vsa=None, # H3VSAContext or None (dense) | |
| ) -> torch.Tensor: | |
| b, s, _ = x.shape | |
| q, k, v = self.qkv_proj(x).chunk(3, dim=-1) | |
| q = q.view(b, s, self.heads, self.head_dim) | |
| k = k.view(b, s, self.heads, self.head_dim) | |
| v = v.view(b, s, self.heads, self.head_dim) | |
| q = self.q_norm(q) | |
| k = self.k_norm(k) | |
| if rotary_emb is not None: | |
| q = apply_rotary_emb(q, *rotary_emb) | |
| k = apply_rotary_emb(k, *rotary_emb) | |
| if vsa is not None and self.to_gate_compress is not None: | |
| from .vsa import vsa_attention | |
| gate = self.to_gate_compress(x).view(b, s, self.heads, self.head_dim) | |
| out = vsa_attention(q, k, v, gate, vsa) | |
| return self.out_proj(out.reshape(b, s, -1)) | |
| q = q.transpose(1, 2) | |
| k = k.transpose(1, 2) | |
| v = v.transpose(1, 2) | |
| out = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask) | |
| out = out.transpose(1, 2).reshape(b, s, -1) | |
| return self.out_proj(out) | |
| class MiniMaxH3Mlp(nn.Module): | |
| """SwiGLU: fc1 packs [gate | up]; silu(gate) * up -> fc2. No biases.""" | |
| def __init__(self, hidden: int, ffn: int): | |
| super().__init__() | |
| self.fc1 = nn.Linear(hidden, ffn * 2, bias=False) | |
| self.fc2 = nn.Linear(ffn, hidden, bias=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| gate, up = self.fc1(x).chunk(2, dim=-1) | |
| return self.fc2(F.silu(gate) * up) | |
| class MiniMaxH3AdalnProj(nn.Module): | |
| """Timestep embedding -> per-(timestep, modality) modulation parameters. | |
| (M, time_embed_dim) -> ``expand`` tensors of (M * modalities, hidden); row | |
| layout ``[t0_mod0, t0_mod1, t0_mod2, t1_mod0, ...]``, addressed by | |
| ``timestep_index * MODALITY_NUM + tag``. The whole projection computes in | |
| float32 (pruned checkpoints store these linears fp16, whose 65504 ceiling | |
| overflows here); the weights stay at their stored dtype and are only | |
| upcast for the matmul, so whole-model casts can't undo this. Outputs are | |
| float32; the use sites cast them to the block dtype. | |
| """ | |
| def __init__( | |
| self, | |
| t_dim: int, | |
| hidden: int, | |
| expand: int, | |
| modalities: int, | |
| apply_silu: bool = True, | |
| bias: bool = True, | |
| ): | |
| super().__init__() | |
| self.expand = expand | |
| self.modalities = modalities | |
| self.hidden = hidden | |
| self.apply_silu = apply_silu | |
| self.linear = nn.Linear(t_dim, expand * hidden * modalities, bias=bias) | |
| def forward(self, temb: torch.Tensor): | |
| if self.apply_silu: | |
| temb = F.silu(temb) | |
| # follow temb's device: the raw params may be CPU-resident under layer | |
| # offloading / low_vram, and F.linear bypasses the paging hooks | |
| weight = self.linear.weight.to(device=temb.device, dtype=torch.float32) | |
| bias = self.linear.bias | |
| if bias is not None: | |
| bias = bias.to(device=temb.device, dtype=torch.float32) | |
| x = F.linear(temb.float(), weight, bias) | |
| x = x.view(x.shape[0] * self.modalities, self.expand * self.hidden) | |
| return x.chunk(self.expand, dim=-1) | |
| class MiniMaxH3RefinerBlock(nn.Module): | |
| """Plain pre-norm block over the projected text stream. No AdaLN, no RoPE.""" | |
| def __init__(self, p: MiniMaxH3TransformerParams): | |
| super().__init__() | |
| self.norm1 = nn.RMSNorm(p.hidden_size, eps=p.norm_eps) | |
| self.norm2 = nn.RMSNorm(p.hidden_size, eps=p.norm_eps) | |
| self.attn = MiniMaxH3Attention( | |
| p.hidden_size, p.num_attention_heads, p.attention_head_dim, p.qk_norm_eps | |
| ) | |
| self.mlp = MiniMaxH3Mlp(p.hidden_size, p.ffn_hidden_size) | |
| def forward(self, x: torch.Tensor, attn_mask=None) -> torch.Tensor: | |
| x = x + self.attn(self.norm1(x), attn_mask=attn_mask) | |
| x = x + self.mlp(self.norm2(x)) | |
| return x | |
| class MiniMaxH3TokenRefiner(nn.Module): | |
| def __init__(self, p: MiniMaxH3TransformerParams): | |
| super().__init__() | |
| self.blocks = nn.ModuleList( | |
| [MiniMaxH3RefinerBlock(p) for _ in range(p.token_refiner_num_layers)] | |
| ) | |
| self.final_norm = nn.RMSNorm(p.hidden_size, eps=p.final_norm_eps) | |
| self.gradient_checkpointing = False | |
| def forward(self, x: torch.Tensor, attn_mask=None) -> torch.Tensor: | |
| for block in self.blocks: | |
| if torch.is_grad_enabled() and self.gradient_checkpointing: | |
| x = checkpoint(block, x, attn_mask, use_reentrant=False) | |
| else: | |
| x = block(x, attn_mask) | |
| return self.final_norm(x) | |
| class MiniMaxH3Block(nn.Module): | |
| """Pre-norm attention + SwiGLU MLP, each modulated by AdaLN parameters | |
| gathered per row from the (timestep, modality) table.""" | |
| def __init__(self, p: MiniMaxH3TransformerParams): | |
| super().__init__() | |
| self.norm1 = nn.RMSNorm(p.hidden_size, eps=p.norm_eps) | |
| self.norm2 = nn.RMSNorm(p.hidden_size, eps=p.norm_eps) | |
| self.attn = MiniMaxH3Attention( | |
| p.hidden_size, | |
| p.num_attention_heads, | |
| p.attention_head_dim, | |
| p.qk_norm_eps, | |
| gate_compress=p.gate_compress, | |
| ) | |
| self.mlp = MiniMaxH3Mlp(p.hidden_size, p.ffn_hidden_size) | |
| self.adaln_proj = MiniMaxH3AdalnProj( | |
| p.time_embed_dim, | |
| p.hidden_size, | |
| expand=6, | |
| modalities=MODALITY_NUM, | |
| apply_silu=p.adaln_apply_silu, | |
| bias=p.adaln_bias, | |
| ) | |
| def forward( | |
| self, | |
| x: torch.Tensor, # (B, S, hidden) | |
| temb: torch.Tensor, # (M, time_embed_dim) float32 | |
| adaln_indices: torch.Tensor, # (B, S) long into the (M * 3) table | |
| rotary_emb, # (cos, sin) | |
| attn_mask: Optional[torch.Tensor] = None, | |
| vsa=None, # H3VSAContext or None (dense) | |
| ) -> torch.Tensor: | |
| shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( | |
| self.adaln_proj(temb) | |
| ) | |
| dt = x.dtype # pruned checkpoints store the adaln projections fp16 | |
| h = self.norm1(x) * (1.0 + scale_msa[adaln_indices].to(dt)) + shift_msa[ | |
| adaln_indices | |
| ].to(dt) | |
| x = x + gate_msa[adaln_indices].to(dt) * self.attn( | |
| h, rotary_emb, attn_mask, vsa | |
| ) | |
| h = self.norm2(x) * (1.0 + scale_mlp[adaln_indices].to(dt)) + shift_mlp[ | |
| adaln_indices | |
| ].to(dt) | |
| x = x + gate_mlp[adaln_indices].to(dt) * self.mlp(h) | |
| return x | |
| class MiniMaxH3FinalLayer(nn.Module): | |
| """Shared shift/scale-modulated RMSNorm + the two per-modality heads. | |
| Both heads run over every row (matching the reference); the caller selects | |
| each modality's rows from the results. Heads are float32 islands. | |
| """ | |
| def __init__(self, p: MiniMaxH3TransformerParams): | |
| super().__init__() | |
| video_patch_dim = ( | |
| p.latents_dim * p.patch_size[0] * p.patch_size[1] * p.patch_size[2] | |
| ) | |
| self.norm = nn.RMSNorm(p.hidden_size, eps=p.final_norm_eps) | |
| self.adaln_proj = MiniMaxH3AdalnProj( | |
| p.time_embed_dim, | |
| p.hidden_size, | |
| expand=2, | |
| modalities=1, | |
| apply_silu=p.adaln_apply_silu, | |
| bias=True, | |
| ) | |
| self.video_out = nn.Linear(p.hidden_size, video_patch_dim, bias=True) | |
| self.audio_out = nn.Linear(p.hidden_size, p.audio_latents_dim, bias=True) | |
| def forward( | |
| self, | |
| x: torch.Tensor, # (B, S, hidden) | |
| temb: torch.Tensor, # (M, time_embed_dim) | |
| timestep_indices: torch.Tensor, # (B, S) long into the M rows | |
| ): | |
| shift, scale = self.adaln_proj(temb) | |
| dt = x.dtype | |
| h = self.norm(x) * (1.0 + scale[timestep_indices].to(dt)) + shift[ | |
| timestep_indices | |
| ].to(dt) | |
| h = h.to(self.video_out.weight.dtype) | |
| return self.video_out(h), self.audio_out(h) | |
| class MiniMaxH3Transformer(nn.Module, OstrisModelMixin): | |
| # comfy checkpoints carry a deliberate bf16/fp16/fp32 mix | |
| aitk_cast_on_load = False | |
| def aitk_config_from_state_dict(cls, state_dict): | |
| params = MiniMaxH3TransformerParams() | |
| table = state_dict.get("adaln_t_table", None) | |
| if table is not None: | |
| # pruned checkpoint: factored timestep table instead of the MLP | |
| params.adaln_t_table_size = table.shape[0] | |
| params.time_embed_dim = table.shape[1] | |
| params.gate_compress = "blocks.0.attn.to_gate_compress.weight" in state_dict | |
| return params | |
| def aitk_from_config(cls, config): | |
| from accelerate import init_empty_weights | |
| with init_empty_weights(include_buffers=False): | |
| return cls(config) | |
| def get_transformer_block_names(cls): | |
| return ["blocks"] | |
| def __init__(self, params: Optional[MiniMaxH3TransformerParams] = None): | |
| super().__init__() | |
| if params is None: | |
| params = MiniMaxH3TransformerParams() | |
| self.params = params | |
| p = params | |
| video_patch_dim = ( | |
| p.latents_dim * p.patch_size[0] * p.patch_size[1] * p.patch_size[2] | |
| ) | |
| self.video_patch_proj = nn.Linear(video_patch_dim, p.hidden_size, bias=True) | |
| self.audio_patch_proj = nn.Linear(p.audio_latents_dim, p.hidden_size, bias=True) | |
| self.condition_proj = nn.Linear(p.text_dim, p.hidden_size, bias=True) | |
| if p.adaln_t_table_size is not None: | |
| # pruned checkpoints: the timestep MLP is replaced by a lookup | |
| # table sampled with linear interpolation at t * (size - 1) | |
| self.time_embedder = None | |
| self.register_buffer( | |
| "adaln_t_table", | |
| torch.zeros(p.adaln_t_table_size, p.time_embed_dim), | |
| persistent=True, | |
| ) | |
| else: | |
| self.time_embedder = MiniMaxH3TimeEmbedder( | |
| p.timestep_input_dim, p.time_embed_hidden_size, p.time_embed_dim | |
| ) | |
| self.rope = MiniMaxH3Rope(p.rope_inv_freq_len, p.rope_theta) | |
| self.token_refiner = MiniMaxH3TokenRefiner(p) | |
| self.blocks = nn.ModuleList([MiniMaxH3Block(p) for _ in range(p.num_layers)]) | |
| self.final_layer = MiniMaxH3FinalLayer(p) | |
| self.gradient_checkpointing = False | |
| # None = dense attention. Set by the FastH3 model wrapper; only takes | |
| # effect on gate_compress checkpoints when the caller passes the grid. | |
| self.vsa_sparsity: Optional[float] = None | |
| # float32 islands of the shipped checkpoint; used by the loader to keep | |
| # these keys at full precision when the rest is cast to bf16 | |
| FP32_KEY_PREFIXES = ( | |
| "video_patch_proj", | |
| "audio_patch_proj", | |
| "time_embedder", | |
| "final_layer.video_out", | |
| "final_layer.audio_out", | |
| "rope", | |
| ) | |
| def enable_gradient_checkpointing(self, enable: bool = True): | |
| self.gradient_checkpointing = enable | |
| self.token_refiner.gradient_checkpointing = enable | |
| def disable_gradient_checkpointing(self): | |
| self.enable_gradient_checkpointing(False) | |
| def device(self): | |
| return self.condition_proj.weight.device | |
| def dtype(self): | |
| # working dtype of the block stack (norm weights are never quantized) | |
| return self.token_refiner.final_norm.weight.dtype | |
| def _time_embedding(self, t: torch.Tensor) -> torch.Tensor: | |
| """(M,) float32 in [0, 1] -> (M, time_embed_dim) float32.""" | |
| if self.time_embedder is not None: | |
| return self.time_embedder(t) | |
| table = self.adaln_t_table.float() | |
| pos = t.clamp(0.0, 1.0) * (table.shape[0] - 1) | |
| # the table may stay CPU-resident under offloading while timesteps | |
| # arrive on cuda: interpolate on the table's device, return on t's | |
| pos = pos.to(table.device) | |
| lo = pos.floor().long() | |
| hi = (lo + 1).clamp(max=table.shape[0] - 1) | |
| frac = (pos - lo.float()).unsqueeze(1) | |
| return (table[lo] * (1.0 - frac) + table[hi] * frac).to(t.device) | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, # (B, Nv, 96) patchified video rows (cond + target) | |
| audio_hidden_states: torch.Tensor, # (B, Na, 32) audio rows | |
| encoder_hidden_states: torch.Tensor, # (B, L, 5120) text conditioning | |
| row_timesteps: torch.Tensor, # (B, S) float in [0, 1], t = 1 - sigma | |
| token_tags: torch.Tensor, # (B, S) long: 0 video, 1 text, 2 audio, -1 pad | |
| position_ids: torch.Tensor, # (B, S, 3) float (t, h, w) rotary coords | |
| video_indices: torch.Tensor, # (Nv,) long positions of video rows in the pack | |
| audio_indices: torch.Tensor, # (Na,) long | |
| text_indices: torch.Tensor, # (L,) long | |
| vsa_video_grid: Optional[ | |
| Tuple[int, int, int] | |
| ] = None, # target-video token grid (t, h, w) | |
| ): | |
| """Returns (video_out (B, Nv, 96), audio_out (B, Na, 32)) — the | |
| data-ward velocity ``clean - noise`` for every row, in input order. | |
| Conditioning rows come back unmasked; discarding them is the caller's | |
| job.""" | |
| batch_size, seq_len = token_tags.shape | |
| rotary_emb = self.rope(position_ids) | |
| video_embeds = self.video_patch_proj( | |
| hidden_states.to(self.video_patch_proj.weight.dtype) | |
| ) | |
| audio_embeds = self.audio_patch_proj( | |
| audio_hidden_states.to(self.audio_patch_proj.weight.dtype) | |
| ) | |
| text_embeds = self.condition_proj(encoder_hidden_states.to(self.dtype)) | |
| # pad rows never act as attention keys; as queries they see everything | |
| # (their outputs are discarded), which keeps SDPA rows finite | |
| attn_mask = None | |
| text_attn_mask = None | |
| is_pad = token_tags < 0 | |
| if bool(is_pad.any()): | |
| live = ~is_pad | |
| attn_mask = live[:, None, None, :] | |
| text_attn_mask = live[:, text_indices][:, None, None, :] | |
| text_embeds = self.token_refiner(text_embeds, text_attn_mask) | |
| x = text_embeds.new_zeros((batch_size, seq_len, text_embeds.shape[-1])) | |
| x = x.index_copy(1, text_indices, text_embeds) | |
| x = x.index_copy(1, video_indices, video_embeds.to(x.dtype)) | |
| x = x.index_copy(1, audio_indices, audio_embeds.to(x.dtype)) | |
| # one timestep embedding per distinct noise level across the batch | |
| unique_t, inverse = torch.unique( | |
| row_timesteps.to(torch.float32), sorted=True, return_inverse=True | |
| ) | |
| temb = self._time_embedding(unique_t) | |
| adaln_indices = inverse * MODALITY_NUM + token_tags.clamp(min=0) | |
| vsa_ctx = None | |
| if ( | |
| self.vsa_sparsity is not None | |
| and self.params.gate_compress | |
| and vsa_video_grid is not None | |
| ): | |
| from .vsa import build_vsa_context, vsa_is_available | |
| # no triton -> warn once and run every block dense instead | |
| vsa_ctx = ( | |
| None | |
| if not vsa_is_available() | |
| else build_vsa_context( | |
| seq_len=seq_len, | |
| num_text_rows=int(text_indices.shape[0]), | |
| video_grid=tuple(int(g) for g in vsa_video_grid), | |
| token_tags=token_tags, | |
| sparsity=self.vsa_sparsity, | |
| device=x.device, | |
| ) | |
| ) | |
| for block in self.blocks: | |
| if torch.is_grad_enabled() and self.gradient_checkpointing: | |
| x = checkpoint( | |
| block, | |
| x, | |
| temb, | |
| adaln_indices, | |
| rotary_emb, | |
| attn_mask, | |
| vsa_ctx, | |
| use_reentrant=False, | |
| ) | |
| else: | |
| x = block(x, temb, adaln_indices, rotary_emb, attn_mask, vsa_ctx) | |
| video_all, audio_all = self.final_layer(x, temb, inverse) | |
| video_out = video_all.index_select(1, video_indices) | |
| audio_out = audio_all.index_select(1, audio_indices) | |
| return video_out, audio_out | |
Xet Storage Details
- Size:
- 23.5 kB
- Xet hash:
- b12b8153df3ba4b0e4b72df102e7c6b2019df93a699e1e40db5e33a26d03f5d4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.