import torch import torch.nn as nn import math class RoPE(nn.Module): def __init__(self, d_k, max_len=10000): super().__init__() self.d_k = d_k intermediate_term = torch.arange(0, d_k, 2) * (-math.log(10000.0) / d_k) theta = torch.exp(intermediate_term) self.register_buffer('theta', theta) positions = torch.arange(max_len).unsqueeze(1) angles = positions * theta self.register_buffer('cos', torch.cos(angles)) self.register_buffer('sin', torch.sin(angles)) def forward(self, x): seq_len = x.shape[2] cos = self.cos[:seq_len, :] sin = self.sin[:seq_len, :] x_even = x[..., 0::2] x_odd = x[..., 1::2] x_rotated_even = x_even * cos - x_odd * sin x_rotated_odd = x_even * sin + x_odd * cos x_out = torch.stack([x_rotated_even, x_rotated_odd], dim=-1) x_out = x_out.flatten(-2) return x_out