invoker-render / h3vae /base_module.py
multimodalart's picture
multimodalart HF Staff
Invoker-Render demo
10cdaf1 verified
Raw History Blame Contribute Delete
10 kB
# SPDX-License-Identifier: Apache-2.0
# Transformer building blocks for the MiniMax H3 visual VAE ViT decoder.
import math
from typing import Optional
import torch
import torch.nn as nn
from diffusers.utils import logging
from diffusers.utils.torch_utils import maybe_allow_in_graph
# ===== JARVIS PATCH BEGIN(sglang custom kernels unavailable; upstream eager fallbacks below are used) =====
# from sglang.kernels.ops.activation.activation import (
# silu_and_mul_with_activation_rounding,
# )
# from sglang.kernels.ops.diffusion.triton.scale_shift import (
# try_fused_scaled_residual_add_exact,
# )
# ===== JARVIS PATCH END =====
from .attention import Attention
from .vit_utils import _env_flag, _vit_torch_compile_kwargs
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
def _vit_norm_input(module, hidden_states):
if _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"):
return hidden_states.float()
return hidden_states.to(getattr(module.weight, "dtype", hidden_states.dtype))
def _scaled_residual_add(residual, x, scale):
# ===== JARVIS PATCH BEGIN(sglang triton fused scaled-residual-add unavailable) =====
# fused = try_fused_scaled_residual_add_exact(residual, x, scale)
fused = None
# ===== JARVIS PATCH END =====
return residual + x * scale if fused is None else fused
class FeedForward(nn.Module):
def __init__(
self,
dim: int,
dim_out: Optional[int] = None,
mult: int = 4,
activation_fn: str = "silu",
bias: bool = True,
use_gated: bool = True,
glu_balanced: bool = False,
):
super().__init__()
ratio = 2 / 3 if (use_gated and glu_balanced) else 1
inner_dim = round(dim * mult * ratio)
dim_out = dim_out if dim_out is not None else dim
self.use_gated = use_gated
if use_gated:
self.w1 = nn.Linear(dim, inner_dim * 2, bias=bias)
else:
self.w1 = nn.Linear(dim, inner_dim, bias=bias)
if activation_fn == "silu":
self.act_fn = nn.SiLU()
elif activation_fn == "gelu":
self.act_fn = nn.GELU()
elif activation_fn == "gelu-approximate":
self.act_fn = nn.GELU(approximate="tanh")
else:
raise ValueError(f"Unsupported activation function: {activation_fn}")
self.w2 = nn.Linear(inner_dim, dim_out, bias=bias)
self._compile_forward_enabled = _env_flag(
"MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE", "0"
)
self._compile_forward_fatal = _env_flag(
"MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE_FATAL", "0"
)
self._compiled_forward = None
def _forward_impl(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states = self.w1(hidden_states)
if self.use_gated:
if (
# ===== JARVIS PATCH BEGIN(sglang silu_and_mul_with_activation_rounding unavailable) =====
False
and
# ===== JARVIS PATCH END =====
isinstance(self.act_fn, nn.SiLU)
and hidden_states.is_cuda
and hidden_states.dtype in (torch.float16, torch.bfloat16)
and hidden_states.is_contiguous()
and hidden_states.shape[-1] % 32 == 0
):
hidden_states = silu_and_mul_with_activation_rounding(hidden_states)
else:
gate, hidden_states = hidden_states.chunk(2, dim=-1)
hidden_states = self.act_fn(gate).mul_(hidden_states)
else:
hidden_states = self.act_fn(hidden_states)
hidden_states = self.w2(hidden_states)
return hidden_states
def _get_forward_impl(self):
if not self._compile_forward_enabled:
return self._forward_impl
if self._compiled_forward is not None:
return self._compiled_forward
if not hasattr(torch, "compile"):
message = (
"torch.compile is unavailable; falling back to eager ViT FeedForward"
)
if self._compile_forward_fatal:
raise RuntimeError(message)
logger.warning(f"[ViTFeedForward] {message}")
self._compile_forward_enabled = False
return self._forward_impl
kwargs = _vit_torch_compile_kwargs(
"MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE"
)
try:
self._compiled_forward = torch.compile(self._forward_impl, **kwargs)
logger.info(f"[ViTFeedForward] torch.compile enabled kwargs={kwargs}")
except Exception as exc:
if self._compile_forward_fatal:
raise
logger.warning(
f"[ViTFeedForward] torch.compile setup failed: {type(exc).__name__}: {exc}; "
"falling back to eager"
)
self._compile_forward_enabled = False
self._compiled_forward = None
return self._forward_impl
return self._compiled_forward
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
forward_impl = self._get_forward_impl()
try:
return forward_impl(hidden_states)
except Exception as exc:
if (
self._compile_forward_enabled
and self._compiled_forward is not None
and forward_impl is self._compiled_forward
and not self._compile_forward_fatal
):
logger.warning(
f"[ViTFeedForward] compiled forward failed: {type(exc).__name__}: {exc}; "
"disabling compile and retrying eager"
)
self._compile_forward_enabled = False
self._compiled_forward = None
return self._forward_impl(hidden_states)
raise
class RotaryEmbeddingND(nn.Module):
def __init__(self, dim, rotary_base=10000, n_dim=3, use_angle=False):
super().__init__()
self.dim = dim
self.n_dim = n_dim
if dim % (2 * n_dim) != 0:
raise ValueError(
f"head_dim {dim} must be divisible by 2 * n_dim {2 * n_dim}"
)
if use_angle:
self.angle_scale = 2.0 * math.pi
else:
self.angle_scale = 1.0
inv_freq = 1 / rotary_base ** torch.arange(
0, 1, 2 * n_dim / dim, dtype=torch.float32
)
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, img_ids):
B, N, D = img_ids.shape
if D != self.n_dim:
raise ValueError(f"Expected {self.n_dim} dimensions, got {D}")
with torch.autocast("cuda", enabled=False):
angles = (
self.angle_scale
* img_ids[:, :, :, None]
* self.inv_freq.to(img_ids.device)[None, None, None, :]
)
angles = angles.flatten(2, 3)
angles = angles.tile(2)
angles = angles.unsqueeze(2)
cos = torch.cos(angles)
sin = torch.sin(angles)
return cos.to(dtype=img_ids.dtype), sin.to(dtype=img_ids.dtype)
@maybe_allow_in_graph
class TransformerBlock(nn.Module):
def __init__(
self,
heads: int,
dim_head: int,
embed_dim: Optional[int] = None,
ffn_glu_balanced: bool = False,
norm_type: str = "layer_norm",
norm_affine: bool = True,
qk_norm_type: str = "rms_norm",
qk_norm_affine: bool = False,
ffn_activation_fn: str = "silu",
ffn_use_gated: bool = True,
use_scale: bool = True,
bias: bool = True,
eps: float = 1e-5,
**kwargs,
):
super().__init__()
dim = embed_dim if embed_dim is not None else dim_head * heads
self.use_scale = use_scale
if norm_type == "layer_norm":
norm_class = nn.LayerNorm
elif norm_type == "rms_norm":
norm_class = nn.RMSNorm
else:
raise ValueError(f"unknown norm_type {norm_type}")
self.norm1 = norm_class(
dim,
elementwise_affine=norm_affine,
eps=eps,
)
self.attn = Attention(
heads=heads,
dim_head=dim_head,
embed_dim=dim,
qk_norm_type=qk_norm_type,
qk_norm_affine=qk_norm_affine,
bias=bias,
eps=eps,
**kwargs,
)
if use_scale:
self.scale1 = nn.Parameter(torch.zeros(dim))
self.norm2 = norm_class(
dim,
elementwise_affine=norm_affine,
eps=eps,
)
self.ff = FeedForward(
dim=dim,
activation_fn=ffn_activation_fn,
bias=bias,
use_gated=ffn_use_gated,
glu_balanced=ffn_glu_balanced,
)
if use_scale:
self.scale2 = nn.Parameter(torch.zeros(dim))
def forward(
self,
hidden_states: torch.FloatTensor,
rotary_pos_emb: Optional[torch.FloatTensor] = None,
pack_info: dict = {},
):
norm_hidden_states = self.norm1(_vit_norm_input(self.norm1, hidden_states)).to(
hidden_states.dtype
)
attn_output = self.attn(norm_hidden_states, rotary_pos_emb, pack_info)
if self.use_scale:
hidden_states = _scaled_residual_add(
hidden_states, attn_output, self.scale1
)
else:
hidden_states = hidden_states + attn_output
norm_hidden_states = self.norm2(_vit_norm_input(self.norm2, hidden_states)).to(
hidden_states.dtype
)
ff_output = self.ff(norm_hidden_states)
if self.use_scale:
hidden_states = _scaled_residual_add(hidden_states, ff_output, self.scale2)
else:
hidden_states = hidden_states + ff_output
return hidden_states