pragya-ds / kernel.py
ArushBuilds's picture
Upload Pragya model files
2a81da6 verified
Raw History Blame Contribute Delete
5.34 kB
"""FlexAttention-only attention kernel.
One backend, one code path: no device probing, no backend dispatch, no try/except fallbacks.
Every attention layer calls `fused_attention(q, k, v, block_mask)`; what each layer may attend to
(causal / sliding window / same-document) lives entirely in the BlockMask, which GPT.forward builds
once per forward and shares across layers.
Requires CUDA + torch >= 2.5 (`create_block_mask(..., _compile=True)`). Attention-weight dropout is not
supported (DenseMHA refuses dropout > 0 at construction).
"""
import functools
import inspect
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
print(
f"[kernel] import: cuda_avail={torch.cuda.is_available()} "
f"cuda_ver={torch.version.cuda} file={__file__}",
flush=True,
)
_flex = None
def _is_compiled() -> bool:
return _flex is not None and _flex is not flex_attention
def _get_flex():
global _flex
if _flex is None:
_flex = (
torch.compile(flex_attention, dynamic=False)
if torch.cuda.is_available()
else flex_attention
)
return _flex
# Computed once at import: Dynamo never sees a call to an lru_cache-wrapped function.
try:
_SUPPORTS_KERNEL_OPTIONS = "kernel_options" in inspect.signature(flex_attention).parameters
except (TypeError, ValueError):
_SUPPORTS_KERNEL_OPTIONS = False
def _flex_opts(h, t, d):
# Pure function of the shape, no cache: plain Python branches trace cleanly under Dynamo.
if d <= 32:
return {"BLOCK_M": 128, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2}
if d <= 64:
return {"BLOCK_M": 64, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2}
return {"BLOCK_M": 128, "BLOCK_N": 128, "num_warps": 8, "num_stages": 2}
def fused_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, block_mask) -> torch.Tensor:
"""q: (B, Hq, T, D); k, v: (B, Hkv, T, D). GQA is handled natively (Hq may be a multiple of Hkv)."""
kwargs = {
"block_mask": block_mask,
"enable_gqa": q.shape[1] != k.shape[1],
}
if _SUPPORTS_KERNEL_OPTIONS:
kwargs["kernel_options"] = _flex_opts(q.shape[1], q.shape[2], q.shape[3])
return _get_flex()(q, k, v, **kwargs)
# ---------------------------------------------------------------------------
# BlockMasks. One uniform mask_mod for every layer: `window = window_size or seq_len`, and since
# q - kv <= seq_len - 1 < seq_len, "no window" is the same expression as "window = seq_len".
# Both builders are compiler-disabled (BlockMask construction is not traced) and are called once per
# forward from GPT.forward -- that is the only graph boundary in the attention path.
# ---------------------------------------------------------------------------
@functools.lru_cache(maxsize=32)
def _causal_mask(seq_len: int, window_size, device_key: str):
window = window_size or seq_len
def mask_mod(b, h, q_idx, kv_idx):
return (kv_idx <= q_idx) & (q_idx - kv_idx < window)
return create_block_mask(mask_mod, B=None, H=None, Q_LEN=seq_len, KV_LEN=seq_len, device=device_key)
@torch.compiler.disable
def causal_masks(seq_len: int, windows, device) -> dict:
"""{window_size: BlockMask} for plain causal attention; cached per (seq_len, window, device)."""
return {w: _causal_mask(seq_len, w, str(device)) for w in windows}
@torch.compiler.disable
def doc_masks(doc_ids: torch.Tensor, windows) -> dict:
"""{window_size: BlockMask} for causal + same-document attention. doc_ids: (B, T) int, per batch.
On PyTorch 2.5+, _compile=True fuses mask_mod so the (B, T, T) boolean grid is never materialised."""
b_, t_ = doc_ids.shape
def build(window_size):
window = window_size or t_
def mask_mod(b, h, q_idx, kv_idx):
return (doc_ids[b, q_idx] == doc_ids[b, kv_idx]) & (kv_idx <= q_idx) & (q_idx - kv_idx < window)
mask_kwargs = dict(B=b_, H=None, Q_LEN=t_, KV_LEN=t_, device=doc_ids.device)
version = torch.__version__.split("+", 1)[0].split(".")
try:
supports_compile = (int(version[0]), int(version[1])) >= (2, 5)
except (IndexError, ValueError):
supports_compile = False
if supports_compile:
mask_kwargs["_compile"] = True
return create_block_mask(mask_mod, **mask_kwargs)
return {w: build(w) for w in windows}
# ---------------------------------------------------------------------------
# Telemetry counters (model.py records its Liger/Triton success/fallback events here).
# ---------------------------------------------------------------------------
_backend_stats: dict[str, int] = {}
def _record_backend(name: str) -> None:
# Inside a compiled region this body runs only at trace time, so counting here under-reports (verified:
# 5 compiled calls -> 3 counts). Record nothing while compiling rather than report wrong numbers.
if torch.compiler.is_compiling():
return
_backend_stats[name] = _backend_stats.get(name, 0) + 1
def get_backend_stats() -> dict[str, int]:
return dict(_backend_stats)
def reset_backend_stats() -> None:
_backend_stats.clear()