Spaces:
Running on Zero
Running on Zero
Download h3vae/base_module.py from hugging-apps/invoker-render: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/h3vae/base_module.py
- Command line
-
hf download hf://spaces/hugging-apps/invoker-render/h3vae/base_module.py
-
curl -L -o base_module.py https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/h3vae/base_module.py
10 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # Transformer building blocks for the MiniMax H3 visual VAE ViT decoder. | |
| import math | |
| from typing import Optional | |
| import torch | |
| import torch.nn as nn | |
| from diffusers.utils import logging | |
| from diffusers.utils.torch_utils import maybe_allow_in_graph | |
| # ===== JARVIS PATCH BEGIN(sglang custom kernels unavailable; upstream eager fallbacks below are used) ===== | |
| # from sglang.kernels.ops.activation.activation import ( | |
| # silu_and_mul_with_activation_rounding, | |
| # ) | |
| # from sglang.kernels.ops.diffusion.triton.scale_shift import ( | |
| # try_fused_scaled_residual_add_exact, | |
| # ) | |
| # ===== JARVIS PATCH END ===== | |
| from .attention import Attention | |
| from .vit_utils import _env_flag, _vit_torch_compile_kwargs | |
| logger = logging.get_logger(__name__) # pylint: disable=invalid-name | |
| def _vit_norm_input(module, hidden_states): | |
| if _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"): | |
| return hidden_states.float() | |
| return hidden_states.to(getattr(module.weight, "dtype", hidden_states.dtype)) | |
| def _scaled_residual_add(residual, x, scale): | |
| # ===== JARVIS PATCH BEGIN(sglang triton fused scaled-residual-add unavailable) ===== | |
| # fused = try_fused_scaled_residual_add_exact(residual, x, scale) | |
| fused = None | |
| # ===== JARVIS PATCH END ===== | |
| return residual + x * scale if fused is None else fused | |
| class FeedForward(nn.Module): | |
| def __init__( | |
| self, | |
| dim: int, | |
| dim_out: Optional[int] = None, | |
| mult: int = 4, | |
| activation_fn: str = "silu", | |
| bias: bool = True, | |
| use_gated: bool = True, | |
| glu_balanced: bool = False, | |
| ): | |
| super().__init__() | |
| ratio = 2 / 3 if (use_gated and glu_balanced) else 1 | |
| inner_dim = round(dim * mult * ratio) | |
| dim_out = dim_out if dim_out is not None else dim | |
| self.use_gated = use_gated | |
| if use_gated: | |
| self.w1 = nn.Linear(dim, inner_dim * 2, bias=bias) | |
| else: | |
| self.w1 = nn.Linear(dim, inner_dim, bias=bias) | |
| if activation_fn == "silu": | |
| self.act_fn = nn.SiLU() | |
| elif activation_fn == "gelu": | |
| self.act_fn = nn.GELU() | |
| elif activation_fn == "gelu-approximate": | |
| self.act_fn = nn.GELU(approximate="tanh") | |
| else: | |
| raise ValueError(f"Unsupported activation function: {activation_fn}") | |
| self.w2 = nn.Linear(inner_dim, dim_out, bias=bias) | |
| self._compile_forward_enabled = _env_flag( | |
| "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE", "0" | |
| ) | |
| self._compile_forward_fatal = _env_flag( | |
| "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE_FATAL", "0" | |
| ) | |
| self._compiled_forward = None | |
| def _forward_impl(self, hidden_states: torch.Tensor) -> torch.Tensor: | |
| hidden_states = self.w1(hidden_states) | |
| if self.use_gated: | |
| if ( | |
| # ===== JARVIS PATCH BEGIN(sglang silu_and_mul_with_activation_rounding unavailable) ===== | |
| False | |
| and | |
| # ===== JARVIS PATCH END ===== | |
| isinstance(self.act_fn, nn.SiLU) | |
| and hidden_states.is_cuda | |
| and hidden_states.dtype in (torch.float16, torch.bfloat16) | |
| and hidden_states.is_contiguous() | |
| and hidden_states.shape[-1] % 32 == 0 | |
| ): | |
| hidden_states = silu_and_mul_with_activation_rounding(hidden_states) | |
| else: | |
| gate, hidden_states = hidden_states.chunk(2, dim=-1) | |
| hidden_states = self.act_fn(gate).mul_(hidden_states) | |
| else: | |
| hidden_states = self.act_fn(hidden_states) | |
| hidden_states = self.w2(hidden_states) | |
| return hidden_states | |
| def _get_forward_impl(self): | |
| if not self._compile_forward_enabled: | |
| return self._forward_impl | |
| if self._compiled_forward is not None: | |
| return self._compiled_forward | |
| if not hasattr(torch, "compile"): | |
| message = ( | |
| "torch.compile is unavailable; falling back to eager ViT FeedForward" | |
| ) | |
| if self._compile_forward_fatal: | |
| raise RuntimeError(message) | |
| logger.warning(f"[ViTFeedForward] {message}") | |
| self._compile_forward_enabled = False | |
| return self._forward_impl | |
| kwargs = _vit_torch_compile_kwargs( | |
| "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE" | |
| ) | |
| try: | |
| self._compiled_forward = torch.compile(self._forward_impl, **kwargs) | |
| logger.info(f"[ViTFeedForward] torch.compile enabled kwargs={kwargs}") | |
| except Exception as exc: | |
| if self._compile_forward_fatal: | |
| raise | |
| logger.warning( | |
| f"[ViTFeedForward] torch.compile setup failed: {type(exc).__name__}: {exc}; " | |
| "falling back to eager" | |
| ) | |
| self._compile_forward_enabled = False | |
| self._compiled_forward = None | |
| return self._forward_impl | |
| return self._compiled_forward | |
| def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: | |
| forward_impl = self._get_forward_impl() | |
| try: | |
| return forward_impl(hidden_states) | |
| except Exception as exc: | |
| if ( | |
| self._compile_forward_enabled | |
| and self._compiled_forward is not None | |
| and forward_impl is self._compiled_forward | |
| and not self._compile_forward_fatal | |
| ): | |
| logger.warning( | |
| f"[ViTFeedForward] compiled forward failed: {type(exc).__name__}: {exc}; " | |
| "disabling compile and retrying eager" | |
| ) | |
| self._compile_forward_enabled = False | |
| self._compiled_forward = None | |
| return self._forward_impl(hidden_states) | |
| raise | |
| class RotaryEmbeddingND(nn.Module): | |
| def __init__(self, dim, rotary_base=10000, n_dim=3, use_angle=False): | |
| super().__init__() | |
| self.dim = dim | |
| self.n_dim = n_dim | |
| if dim % (2 * n_dim) != 0: | |
| raise ValueError( | |
| f"head_dim {dim} must be divisible by 2 * n_dim {2 * n_dim}" | |
| ) | |
| if use_angle: | |
| self.angle_scale = 2.0 * math.pi | |
| else: | |
| self.angle_scale = 1.0 | |
| inv_freq = 1 / rotary_base ** torch.arange( | |
| 0, 1, 2 * n_dim / dim, dtype=torch.float32 | |
| ) | |
| self.register_buffer("inv_freq", inv_freq, persistent=False) | |
| def forward(self, img_ids): | |
| B, N, D = img_ids.shape | |
| if D != self.n_dim: | |
| raise ValueError(f"Expected {self.n_dim} dimensions, got {D}") | |
| with torch.autocast("cuda", enabled=False): | |
| angles = ( | |
| self.angle_scale | |
| * img_ids[:, :, :, None] | |
| * self.inv_freq.to(img_ids.device)[None, None, None, :] | |
| ) | |
| angles = angles.flatten(2, 3) | |
| angles = angles.tile(2) | |
| angles = angles.unsqueeze(2) | |
| cos = torch.cos(angles) | |
| sin = torch.sin(angles) | |
| return cos.to(dtype=img_ids.dtype), sin.to(dtype=img_ids.dtype) | |
| class TransformerBlock(nn.Module): | |
| def __init__( | |
| self, | |
| heads: int, | |
| dim_head: int, | |
| embed_dim: Optional[int] = None, | |
| ffn_glu_balanced: bool = False, | |
| norm_type: str = "layer_norm", | |
| norm_affine: bool = True, | |
| qk_norm_type: str = "rms_norm", | |
| qk_norm_affine: bool = False, | |
| ffn_activation_fn: str = "silu", | |
| ffn_use_gated: bool = True, | |
| use_scale: bool = True, | |
| bias: bool = True, | |
| eps: float = 1e-5, | |
| **kwargs, | |
| ): | |
| super().__init__() | |
| dim = embed_dim if embed_dim is not None else dim_head * heads | |
| self.use_scale = use_scale | |
| if norm_type == "layer_norm": | |
| norm_class = nn.LayerNorm | |
| elif norm_type == "rms_norm": | |
| norm_class = nn.RMSNorm | |
| else: | |
| raise ValueError(f"unknown norm_type {norm_type}") | |
| self.norm1 = norm_class( | |
| dim, | |
| elementwise_affine=norm_affine, | |
| eps=eps, | |
| ) | |
| self.attn = Attention( | |
| heads=heads, | |
| dim_head=dim_head, | |
| embed_dim=dim, | |
| qk_norm_type=qk_norm_type, | |
| qk_norm_affine=qk_norm_affine, | |
| bias=bias, | |
| eps=eps, | |
| **kwargs, | |
| ) | |
| if use_scale: | |
| self.scale1 = nn.Parameter(torch.zeros(dim)) | |
| self.norm2 = norm_class( | |
| dim, | |
| elementwise_affine=norm_affine, | |
| eps=eps, | |
| ) | |
| self.ff = FeedForward( | |
| dim=dim, | |
| activation_fn=ffn_activation_fn, | |
| bias=bias, | |
| use_gated=ffn_use_gated, | |
| glu_balanced=ffn_glu_balanced, | |
| ) | |
| if use_scale: | |
| self.scale2 = nn.Parameter(torch.zeros(dim)) | |
| def forward( | |
| self, | |
| hidden_states: torch.FloatTensor, | |
| rotary_pos_emb: Optional[torch.FloatTensor] = None, | |
| pack_info: dict = {}, | |
| ): | |
| norm_hidden_states = self.norm1(_vit_norm_input(self.norm1, hidden_states)).to( | |
| hidden_states.dtype | |
| ) | |
| attn_output = self.attn(norm_hidden_states, rotary_pos_emb, pack_info) | |
| if self.use_scale: | |
| hidden_states = _scaled_residual_add( | |
| hidden_states, attn_output, self.scale1 | |
| ) | |
| else: | |
| hidden_states = hidden_states + attn_output | |
| norm_hidden_states = self.norm2(_vit_norm_input(self.norm2, hidden_states)).to( | |
| hidden_states.dtype | |
| ) | |
| ff_output = self.ff(norm_hidden_states) | |
| if self.use_scale: | |
| hidden_states = _scaled_residual_add(hidden_states, ff_output, self.scale2) | |
| else: | |
| hidden_states = hidden_states + ff_output | |
| return hidden_states | |