changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
1.07 kB
# SPDX-License-Identifier: Apache-2.0
"""Small shared torch ops of the reference (fp32 math, matching the HF / RDT definitions)."""
from __future__ import annotations
import torch
def rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
"""Qwen2RMSNorm and RDT RMSNorm (identical in fp32): weight * x * rsqrt(mean(x^2) + eps), stats in fp32."""
xf = x.float()
y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
return weight * y.to(x.dtype)
def rotate_half(x: torch.Tensor) -> torch.Tensor:
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2:]
return torch.cat((-x2, x1), dim=-1)
def repeat_kv_heads(x: torch.Tensor, n_rep: int, dim: int) -> torch.Tensor:
"""GQA expansion: query head h reads kv head h // n_rep (HF repeat_kv and RDT repeat_kv agree)."""
return x if n_rep == 1 else torch.repeat_interleave(x, n_rep, dim=dim)
def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)