MyModel-pretrain / rope.py
HappyBulingling's picture
Upload MyModel checkpoint
9b53d42 verified
Raw History Blame Contribute Delete
1.15 kB
"""Rotary position embeddings."""
from __future__ import annotations
import torch
def precompute_rope(
head_dim: int,
max_position_embeddings: int,
rope_theta: float,
) -> tuple[torch.Tensor, torch.Tensor]:
if head_dim % 2 != 0:
raise ValueError("head_dim must be even for RoPE")
inv_freq = 1.0 / (
rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)
)
positions = torch.arange(max_position_embeddings, dtype=torch.float32)
freqs = torch.outer(positions, inv_freq)
emb = torch.cat((freqs, freqs), dim=-1)
return emb.cos(), emb.sin()
def rotate_half(x: torch.Tensor) -> torch.Tensor:
half = x.shape[-1] // 2
return torch.cat((-x[..., half:], x[..., :half]), dim=-1)
def apply_rotary_pos_emb(
q: torch.Tensor,
k: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
cos = cos.to(q.device).unsqueeze(0).unsqueeze(2)
sin = sin.to(q.device).unsqueeze(0).unsqueeze(2)
q_embed = (q * cos) + (rotate_half(q) * sin)
k_embed = (k * cos) + (rotate_half(k) * sin)
return q_embed, k_embed