# SPDX-License-Identifier: Apache-2.0 """Small shared torch ops of the reference (fp32 math, matching the HF / RDT definitions).""" from __future__ import annotations import torch def rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: """Qwen2RMSNorm and RDT RMSNorm (identical in fp32): weight * x * rsqrt(mean(x^2) + eps), stats in fp32.""" xf = x.float() y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) return weight * y.to(x.dtype) def rotate_half(x: torch.Tensor) -> torch.Tensor: x1 = x[..., : x.shape[-1] // 2] x2 = x[..., x.shape[-1] // 2:] return torch.cat((-x2, x1), dim=-1) def repeat_kv_heads(x: torch.Tensor, n_rep: int, dim: int) -> torch.Tensor: """GQA expansion: query head h reads kv head h // n_rep (HF repeat_kv and RDT repeat_kv agree).""" return x if n_rep == 1 else torch.repeat_interleave(x, n_rep, dim=dim) def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)