File size: 5,338 Bytes
2a81da6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
"""FlexAttention-only attention kernel.



One backend, one code path: no device probing, no backend dispatch, no try/except fallbacks.

Every attention layer calls `fused_attention(q, k, v, block_mask)`; what each layer may attend to

(causal / sliding window / same-document) lives entirely in the BlockMask, which GPT.forward builds

once per forward and shares across layers.



Requires CUDA + torch >= 2.5 (`create_block_mask(..., _compile=True)`). Attention-weight dropout is not

supported (DenseMHA refuses dropout > 0 at construction).

"""
import functools
import inspect

import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention

print(
    f"[kernel] import: cuda_avail={torch.cuda.is_available()} "
    f"cuda_ver={torch.version.cuda} file={__file__}",
    flush=True,
)


_flex = None


def _is_compiled() -> bool:
    return _flex is not None and _flex is not flex_attention


def _get_flex():
    global _flex
    if _flex is None:
        _flex = (
            torch.compile(flex_attention, dynamic=False)
            if torch.cuda.is_available()
            else flex_attention
        )
    return _flex


# Computed once at import: Dynamo never sees a call to an lru_cache-wrapped function.
try:
    _SUPPORTS_KERNEL_OPTIONS = "kernel_options" in inspect.signature(flex_attention).parameters
except (TypeError, ValueError):
    _SUPPORTS_KERNEL_OPTIONS = False


def _flex_opts(h, t, d):
    # Pure function of the shape, no cache: plain Python branches trace cleanly under Dynamo.
    if d <= 32:
        return {"BLOCK_M": 128, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2}
    if d <= 64:
        return {"BLOCK_M": 64, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2}
    return {"BLOCK_M": 128, "BLOCK_N": 128, "num_warps": 8, "num_stages": 2}


def fused_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, block_mask) -> torch.Tensor:
    """q: (B, Hq, T, D); k, v: (B, Hkv, T, D). GQA is handled natively (Hq may be a multiple of Hkv)."""
    kwargs = {
        "block_mask": block_mask,
        "enable_gqa": q.shape[1] != k.shape[1],
    }
    if _SUPPORTS_KERNEL_OPTIONS:
        kwargs["kernel_options"] = _flex_opts(q.shape[1], q.shape[2], q.shape[3])
    return _get_flex()(q, k, v, **kwargs)


# ---------------------------------------------------------------------------
# BlockMasks. One uniform mask_mod for every layer: `window = window_size or seq_len`, and since
# q - kv <= seq_len - 1 < seq_len, "no window" is the same expression as "window = seq_len".
# Both builders are compiler-disabled (BlockMask construction is not traced) and are called once per
# forward from GPT.forward -- that is the only graph boundary in the attention path.
# ---------------------------------------------------------------------------

@functools.lru_cache(maxsize=32)
def _causal_mask(seq_len: int, window_size, device_key: str):
    window = window_size or seq_len

    def mask_mod(b, h, q_idx, kv_idx):
        return (kv_idx <= q_idx) & (q_idx - kv_idx < window)

    return create_block_mask(mask_mod, B=None, H=None, Q_LEN=seq_len, KV_LEN=seq_len, device=device_key)


@torch.compiler.disable
def causal_masks(seq_len: int, windows, device) -> dict:
    """{window_size: BlockMask} for plain causal attention; cached per (seq_len, window, device)."""
    return {w: _causal_mask(seq_len, w, str(device)) for w in windows}


@torch.compiler.disable
def doc_masks(doc_ids: torch.Tensor, windows) -> dict:
    """{window_size: BlockMask} for causal + same-document attention. doc_ids: (B, T) int, per batch.

    On PyTorch 2.5+, _compile=True fuses mask_mod so the (B, T, T) boolean grid is never materialised."""
    b_, t_ = doc_ids.shape

    def build(window_size):
        window = window_size or t_

        def mask_mod(b, h, q_idx, kv_idx):
            return (doc_ids[b, q_idx] == doc_ids[b, kv_idx]) & (kv_idx <= q_idx) & (q_idx - kv_idx < window)

        mask_kwargs = dict(B=b_, H=None, Q_LEN=t_, KV_LEN=t_, device=doc_ids.device)
        version = torch.__version__.split("+", 1)[0].split(".")
        try:
            supports_compile = (int(version[0]), int(version[1])) >= (2, 5)
        except (IndexError, ValueError):
            supports_compile = False
        if supports_compile:
            mask_kwargs["_compile"] = True
        return create_block_mask(mask_mod, **mask_kwargs)

    return {w: build(w) for w in windows}


# ---------------------------------------------------------------------------
# Telemetry counters (model.py records its Liger/Triton success/fallback events here).
# ---------------------------------------------------------------------------

_backend_stats: dict[str, int] = {}


def _record_backend(name: str) -> None:
    # Inside a compiled region this body runs only at trace time, so counting here under-reports (verified:
    # 5 compiled calls -> 3 counts). Record nothing while compiling rather than report wrong numbers.
    if torch.compiler.is_compiling():
        return
    _backend_stats[name] = _backend_stats.get(name, 0) + 1


def get_backend_stats() -> dict[str, int]:
    return dict(_backend_stats)


def reset_backend_stats() -> None:
    _backend_stats.clear()