Download code/tt_rdt2/reference/ops.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 1.07 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/ops.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/ops.py
-
curl -L -o ops.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/ops.py
1.07 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Small shared torch ops of the reference (fp32 math, matching the HF / RDT definitions).""" | |
| from __future__ import annotations | |
| import torch | |
| def rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: | |
| """Qwen2RMSNorm and RDT RMSNorm (identical in fp32): weight * x * rsqrt(mean(x^2) + eps), stats in fp32.""" | |
| xf = x.float() | |
| y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) | |
| return weight * y.to(x.dtype) | |
| def rotate_half(x: torch.Tensor) -> torch.Tensor: | |
| x1 = x[..., : x.shape[-1] // 2] | |
| x2 = x[..., x.shape[-1] // 2:] | |
| return torch.cat((-x2, x1), dim=-1) | |
| def repeat_kv_heads(x: torch.Tensor, n_rep: int, dim: int) -> torch.Tensor: | |
| """GQA expansion: query head h reads kv head h // n_rep (HF repeat_kv and RDT repeat_kv agree).""" | |
| return x if n_rep == 1 else torch.repeat_interleave(x, n_rep, dim=dim) | |
| def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: | |
| return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) | |