MyModel-sft / norm.py
HappyBulingling's picture
Upload MyModel checkpoint
72f5707 verified
Raw History Blame Contribute Delete
602 Bytes
"""RMSNorm with fp32 statistics and parameter-free operations."""
from __future__ import annotations
import torch
from torch import nn
class RMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.eps = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
dtype = hidden_states.dtype
x = hidden_states.float()
x = x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
return (self.weight.float() * x).to(dtype)