File size: 1,065 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# SPDX-License-Identifier: Apache-2.0
"""Small shared torch ops of the reference (fp32 math, matching the HF / RDT definitions)."""
from __future__ import annotations

import torch


def rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
    """Qwen2RMSNorm and RDT RMSNorm (identical in fp32): weight * x * rsqrt(mean(x^2) + eps), stats in fp32."""
    xf = x.float()
    y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
    return weight * y.to(x.dtype)


def rotate_half(x: torch.Tensor) -> torch.Tensor:
    x1 = x[..., : x.shape[-1] // 2]
    x2 = x[..., x.shape[-1] // 2:]
    return torch.cat((-x2, x1), dim=-1)


def repeat_kv_heads(x: torch.Tensor, n_rep: int, dim: int) -> torch.Tensor:
    """GQA expansion: query head h reads kv head h // n_rep (HF repeat_kv and RDT repeat_kv agree)."""
    return x if n_rep == 1 else torch.repeat_interleave(x, n_rep, dim=dim)


def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
    return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)