"""RMSNorm with fp32 statistics and parameter-free operations.""" from __future__ import annotations import torch from torch import nn class RMSNorm(nn.Module): def __init__(self, hidden_size: int, eps: float = 1e-6): super().__init__() self.weight = nn.Parameter(torch.ones(hidden_size)) self.eps = eps def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: dtype = hidden_states.dtype x = hidden_states.float() x = x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) return (self.weight.float() * x).to(dtype)