File size: 1,065 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | # SPDX-License-Identifier: Apache-2.0
"""Small shared torch ops of the reference (fp32 math, matching the HF / RDT definitions)."""
from __future__ import annotations
import torch
def rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
"""Qwen2RMSNorm and RDT RMSNorm (identical in fp32): weight * x * rsqrt(mean(x^2) + eps), stats in fp32."""
xf = x.float()
y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
return weight * y.to(x.dtype)
def rotate_half(x: torch.Tensor) -> torch.Tensor:
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2:]
return torch.cat((-x2, x1), dim=-1)
def repeat_kv_heads(x: torch.Tensor, n_rep: int, dim: int) -> torch.Tensor:
"""GQA expansion: query head h reads kv head h // n_rep (HF repeat_kv and RDT repeat_kv agree)."""
return x if n_rep == 1 else torch.repeat_interleave(x, n_rep, dim=dim)
def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
|