File size: 959 Bytes
ad63418
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import torch
import torch.nn as nn
import math

class RoPE(nn.Module):
    def __init__(self, d_k, max_len=10000):
        super().__init__()
        self.d_k = d_k


        intermediate_term = torch.arange(0, d_k, 2) * (-math.log(10000.0) / d_k)
        theta = torch.exp(intermediate_term)

        self.register_buffer('theta', theta)

        positions = torch.arange(max_len).unsqueeze(1)
        angles = positions * theta

        self.register_buffer('cos', torch.cos(angles))
        self.register_buffer('sin', torch.sin(angles))

    def forward(self, x):
        seq_len = x.shape[2]
        cos = self.cos[:seq_len, :]
        sin = self.sin[:seq_len, :]

        x_even = x[..., 0::2]
        x_odd = x[..., 1::2]

        x_rotated_even = x_even * cos - x_odd * sin
        x_rotated_odd = x_even * sin + x_odd * cos

        x_out = torch.stack([x_rotated_even, x_rotated_odd], dim=-1)
        x_out = x_out.flatten(-2)

        return x_out