anlp-a2-models / src /part2 /positional.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
959 Bytes
import torch
import torch.nn as nn
import math
class RoPE(nn.Module):
def __init__(self, d_k, max_len=10000):
super().__init__()
self.d_k = d_k
intermediate_term = torch.arange(0, d_k, 2) * (-math.log(10000.0) / d_k)
theta = torch.exp(intermediate_term)
self.register_buffer('theta', theta)
positions = torch.arange(max_len).unsqueeze(1)
angles = positions * theta
self.register_buffer('cos', torch.cos(angles))
self.register_buffer('sin', torch.sin(angles))
def forward(self, x):
seq_len = x.shape[2]
cos = self.cos[:seq_len, :]
sin = self.sin[:seq_len, :]
x_even = x[..., 0::2]
x_odd = x[..., 1::2]
x_rotated_even = x_even * cos - x_odd * sin
x_rotated_odd = x_even * sin + x_odd * cos
x_out = torch.stack([x_rotated_even, x_rotated_odd], dim=-1)
x_out = x_out.flatten(-2)
return x_out