File size: 1,442 Bytes
524fd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
"""Public CUDA normalization with the model's residual precision."""

from __future__ import annotations

import torch


def _bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
    """Recover BF16 operands for an FP32 sum of two BF16 residuals."""
    if (value.is_cuda and value.dtype == torch.float32 and value.is_contiguous()
            and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
        from .kernels import bf16_pair_sum
        return bf16_pair_sum(value)
    limit = torch.finfo(torch.bfloat16).max
    high = value.clamp(-limit, limit).to(torch.bfloat16)
    low = (value - high.float()).to(torch.bfloat16)
    # Equal signed zeros keep the sign when the fused operation adds them.
    low = torch.where(value == 0, high, low)
    return high, low


def rms_norm(value: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
    """Normalize CUDA BF16 states or unrounded FP32 residual sums."""
    from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm

    shape = value.shape
    flat = value.reshape(-1, shape[-1]).contiguous()
    if value.dtype == torch.bfloat16:
        return gemma_rmsnorm(flat, weight, eps).reshape(shape)
    if value.dtype != torch.float32:
        raise ValueError("Expected BF16 states or FP32 residual sums")
    high, low = _bf16_pair_sum(flat)
    gemma_fused_add_rmsnorm(low, high, weight, eps)
    return low.reshape(shape)