File size: 557 Bytes
ad63418 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | import torch
import torch.nn as nn
import math
class LayerNorm(nn.Module):
def __init__(self, d_model, eps=1e-8):
super().__init__()
self.eps = eps
self.gamma = nn.Parameter(torch.ones(d_model))
self.beta = nn.Parameter(torch.zeros(d_model))
def forward(self, x):
mean = torch.mean(x, dim=-1, keepdim=True)
var = torch.var(x, dim=-1, keepdim=True, unbiased=False)
x_norm = (x - mean) / torch.sqrt(var + self.eps)
final_norm = self.gamma * x_norm + self.beta
return final_norm |