Download src/part2/positional.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 959 Bytes
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/positional.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part2/positional.py
-
curl -L -o positional.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/positional.py
959 Bytes
| import torch | |
| import torch.nn as nn | |
| import math | |
| class RoPE(nn.Module): | |
| def __init__(self, d_k, max_len=10000): | |
| super().__init__() | |
| self.d_k = d_k | |
| intermediate_term = torch.arange(0, d_k, 2) * (-math.log(10000.0) / d_k) | |
| theta = torch.exp(intermediate_term) | |
| self.register_buffer('theta', theta) | |
| positions = torch.arange(max_len).unsqueeze(1) | |
| angles = positions * theta | |
| self.register_buffer('cos', torch.cos(angles)) | |
| self.register_buffer('sin', torch.sin(angles)) | |
| def forward(self, x): | |
| seq_len = x.shape[2] | |
| cos = self.cos[:seq_len, :] | |
| sin = self.sin[:seq_len, :] | |
| x_even = x[..., 0::2] | |
| x_odd = x[..., 1::2] | |
| x_rotated_even = x_even * cos - x_odd * sin | |
| x_rotated_odd = x_even * sin + x_odd * cos | |
| x_out = torch.stack([x_rotated_even, x_rotated_odd], dim=-1) | |
| x_out = x_out.flatten(-2) | |
| return x_out | |