File size: 5,338 Bytes
2a81da6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | """FlexAttention-only attention kernel.
One backend, one code path: no device probing, no backend dispatch, no try/except fallbacks.
Every attention layer calls `fused_attention(q, k, v, block_mask)`; what each layer may attend to
(causal / sliding window / same-document) lives entirely in the BlockMask, which GPT.forward builds
once per forward and shares across layers.
Requires CUDA + torch >= 2.5 (`create_block_mask(..., _compile=True)`). Attention-weight dropout is not
supported (DenseMHA refuses dropout > 0 at construction).
"""
import functools
import inspect
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
print(
f"[kernel] import: cuda_avail={torch.cuda.is_available()} "
f"cuda_ver={torch.version.cuda} file={__file__}",
flush=True,
)
_flex = None
def _is_compiled() -> bool:
return _flex is not None and _flex is not flex_attention
def _get_flex():
global _flex
if _flex is None:
_flex = (
torch.compile(flex_attention, dynamic=False)
if torch.cuda.is_available()
else flex_attention
)
return _flex
# Computed once at import: Dynamo never sees a call to an lru_cache-wrapped function.
try:
_SUPPORTS_KERNEL_OPTIONS = "kernel_options" in inspect.signature(flex_attention).parameters
except (TypeError, ValueError):
_SUPPORTS_KERNEL_OPTIONS = False
def _flex_opts(h, t, d):
# Pure function of the shape, no cache: plain Python branches trace cleanly under Dynamo.
if d <= 32:
return {"BLOCK_M": 128, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2}
if d <= 64:
return {"BLOCK_M": 64, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2}
return {"BLOCK_M": 128, "BLOCK_N": 128, "num_warps": 8, "num_stages": 2}
def fused_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, block_mask) -> torch.Tensor:
"""q: (B, Hq, T, D); k, v: (B, Hkv, T, D). GQA is handled natively (Hq may be a multiple of Hkv)."""
kwargs = {
"block_mask": block_mask,
"enable_gqa": q.shape[1] != k.shape[1],
}
if _SUPPORTS_KERNEL_OPTIONS:
kwargs["kernel_options"] = _flex_opts(q.shape[1], q.shape[2], q.shape[3])
return _get_flex()(q, k, v, **kwargs)
# ---------------------------------------------------------------------------
# BlockMasks. One uniform mask_mod for every layer: `window = window_size or seq_len`, and since
# q - kv <= seq_len - 1 < seq_len, "no window" is the same expression as "window = seq_len".
# Both builders are compiler-disabled (BlockMask construction is not traced) and are called once per
# forward from GPT.forward -- that is the only graph boundary in the attention path.
# ---------------------------------------------------------------------------
@functools.lru_cache(maxsize=32)
def _causal_mask(seq_len: int, window_size, device_key: str):
window = window_size or seq_len
def mask_mod(b, h, q_idx, kv_idx):
return (kv_idx <= q_idx) & (q_idx - kv_idx < window)
return create_block_mask(mask_mod, B=None, H=None, Q_LEN=seq_len, KV_LEN=seq_len, device=device_key)
@torch.compiler.disable
def causal_masks(seq_len: int, windows, device) -> dict:
"""{window_size: BlockMask} for plain causal attention; cached per (seq_len, window, device)."""
return {w: _causal_mask(seq_len, w, str(device)) for w in windows}
@torch.compiler.disable
def doc_masks(doc_ids: torch.Tensor, windows) -> dict:
"""{window_size: BlockMask} for causal + same-document attention. doc_ids: (B, T) int, per batch.
On PyTorch 2.5+, _compile=True fuses mask_mod so the (B, T, T) boolean grid is never materialised."""
b_, t_ = doc_ids.shape
def build(window_size):
window = window_size or t_
def mask_mod(b, h, q_idx, kv_idx):
return (doc_ids[b, q_idx] == doc_ids[b, kv_idx]) & (kv_idx <= q_idx) & (q_idx - kv_idx < window)
mask_kwargs = dict(B=b_, H=None, Q_LEN=t_, KV_LEN=t_, device=doc_ids.device)
version = torch.__version__.split("+", 1)[0].split(".")
try:
supports_compile = (int(version[0]), int(version[1])) >= (2, 5)
except (IndexError, ValueError):
supports_compile = False
if supports_compile:
mask_kwargs["_compile"] = True
return create_block_mask(mask_mod, **mask_kwargs)
return {w: build(w) for w in windows}
# ---------------------------------------------------------------------------
# Telemetry counters (model.py records its Liger/Triton success/fallback events here).
# ---------------------------------------------------------------------------
_backend_stats: dict[str, int] = {}
def _record_backend(name: str) -> None:
# Inside a compiled region this body runs only at trace time, so counting here under-reports (verified:
# 5 compiled calls -> 3 counts). Record nothing while compiling rather than report wrong numbers.
if torch.compiler.is_compiling():
return
_backend_stats[name] = _backend_stats.get(name, 0) + 1
def get_backend_stats() -> dict[str, int]:
return dict(_backend_stats)
def reset_backend_stats() -> None:
_backend_stats.clear() |