"""Public CUDA normalization with the model's residual precision.""" from __future__ import annotations import torch def _bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """Recover BF16 operands for an FP32 sum of two BF16 residuals.""" if (value.is_cuda and value.dtype == torch.float32 and value.is_contiguous() and not torch.is_grad_enabled() and not torch.compiler.is_compiling()): from .kernels import bf16_pair_sum return bf16_pair_sum(value) limit = torch.finfo(torch.bfloat16).max high = value.clamp(-limit, limit).to(torch.bfloat16) low = (value - high.float()).to(torch.bfloat16) # Equal signed zeros keep the sign when the fused operation adds them. low = torch.where(value == 0, high, low) return high, low def rms_norm(value: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: """Normalize CUDA BF16 states or unrounded FP32 residual sums.""" from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm shape = value.shape flat = value.reshape(-1, shape[-1]).contiguous() if value.dtype == torch.bfloat16: return gemma_rmsnorm(flat, weight, eps).reshape(shape) if value.dtype != torch.float32: raise ValueError("Expected BF16 states or FP32 residual sums") high, low = _bf16_pair_sum(flat) gemma_fused_add_rmsnorm(low, high, weight, eps) return low.reshape(shape)