anlp-a2-models / src /part2 /norm.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
557 Bytes
import torch
import torch.nn as nn
import math
class LayerNorm(nn.Module):
def __init__(self, d_model, eps=1e-8):
super().__init__()
self.eps = eps
self.gamma = nn.Parameter(torch.ones(d_model))
self.beta = nn.Parameter(torch.zeros(d_model))
def forward(self, x):
mean = torch.mean(x, dim=-1, keepdim=True)
var = torch.var(x, dim=-1, keepdim=True, unbiased=False)
x_norm = (x - mean) / torch.sqrt(var + self.eps)
final_norm = self.gamma * x_norm + self.beta
return final_norm