Download src/part2/norm.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 557 Bytes
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/norm.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part2/norm.py
-
curl -L -o norm.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/norm.py
557 Bytes
| import torch | |
| import torch.nn as nn | |
| import math | |
| class LayerNorm(nn.Module): | |
| def __init__(self, d_model, eps=1e-8): | |
| super().__init__() | |
| self.eps = eps | |
| self.gamma = nn.Parameter(torch.ones(d_model)) | |
| self.beta = nn.Parameter(torch.zeros(d_model)) | |
| def forward(self, x): | |
| mean = torch.mean(x, dim=-1, keepdim=True) | |
| var = torch.var(x, dim=-1, keepdim=True, unbiased=False) | |
| x_norm = (x - mean) / torch.sqrt(var + self.eps) | |
| final_norm = self.gamma * x_norm + self.beta | |
| return final_norm |