"""Rotary position embeddings.""" from __future__ import annotations import torch def precompute_rope( head_dim: int, max_position_embeddings: int, rope_theta: float, ) -> tuple[torch.Tensor, torch.Tensor]: if head_dim % 2 != 0: raise ValueError("head_dim must be even for RoPE") inv_freq = 1.0 / ( rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim) ) positions = torch.arange(max_position_embeddings, dtype=torch.float32) freqs = torch.outer(positions, inv_freq) emb = torch.cat((freqs, freqs), dim=-1) return emb.cos(), emb.sin() def rotate_half(x: torch.Tensor) -> torch.Tensor: half = x.shape[-1] // 2 return torch.cat((-x[..., half:], x[..., :half]), dim=-1) def apply_rotary_pos_emb( q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: cos = cos.to(q.device).unsqueeze(0).unsqueeze(2) sin = sin.to(q.device).unsqueeze(0).unsqueeze(2) q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed