Download kernel.py from RuVI-AI/pragya-ds: direct link, hf CLI and curl.
- Browser
- Download file 5.34 kB
-
https://huggingface.co/RuVI-AI/pragya-ds/resolve/main/kernel.py
- Command line
-
hf download hf://RuVI-AI/pragya-ds/kernel.py
-
curl -L -o kernel.py https://huggingface.co/RuVI-AI/pragya-ds/resolve/main/kernel.py
5.34 kB
| """FlexAttention-only attention kernel. | |
| One backend, one code path: no device probing, no backend dispatch, no try/except fallbacks. | |
| Every attention layer calls `fused_attention(q, k, v, block_mask)`; what each layer may attend to | |
| (causal / sliding window / same-document) lives entirely in the BlockMask, which GPT.forward builds | |
| once per forward and shares across layers. | |
| Requires CUDA + torch >= 2.5 (`create_block_mask(..., _compile=True)`). Attention-weight dropout is not | |
| supported (DenseMHA refuses dropout > 0 at construction). | |
| """ | |
| import functools | |
| import inspect | |
| import torch | |
| from torch.nn.attention.flex_attention import create_block_mask, flex_attention | |
| print( | |
| f"[kernel] import: cuda_avail={torch.cuda.is_available()} " | |
| f"cuda_ver={torch.version.cuda} file={__file__}", | |
| flush=True, | |
| ) | |
| _flex = None | |
| def _is_compiled() -> bool: | |
| return _flex is not None and _flex is not flex_attention | |
| def _get_flex(): | |
| global _flex | |
| if _flex is None: | |
| _flex = ( | |
| torch.compile(flex_attention, dynamic=False) | |
| if torch.cuda.is_available() | |
| else flex_attention | |
| ) | |
| return _flex | |
| # Computed once at import: Dynamo never sees a call to an lru_cache-wrapped function. | |
| try: | |
| _SUPPORTS_KERNEL_OPTIONS = "kernel_options" in inspect.signature(flex_attention).parameters | |
| except (TypeError, ValueError): | |
| _SUPPORTS_KERNEL_OPTIONS = False | |
| def _flex_opts(h, t, d): | |
| # Pure function of the shape, no cache: plain Python branches trace cleanly under Dynamo. | |
| if d <= 32: | |
| return {"BLOCK_M": 128, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2} | |
| if d <= 64: | |
| return {"BLOCK_M": 64, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2} | |
| return {"BLOCK_M": 128, "BLOCK_N": 128, "num_warps": 8, "num_stages": 2} | |
| def fused_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, block_mask) -> torch.Tensor: | |
| """q: (B, Hq, T, D); k, v: (B, Hkv, T, D). GQA is handled natively (Hq may be a multiple of Hkv).""" | |
| kwargs = { | |
| "block_mask": block_mask, | |
| "enable_gqa": q.shape[1] != k.shape[1], | |
| } | |
| if _SUPPORTS_KERNEL_OPTIONS: | |
| kwargs["kernel_options"] = _flex_opts(q.shape[1], q.shape[2], q.shape[3]) | |
| return _get_flex()(q, k, v, **kwargs) | |
| # --------------------------------------------------------------------------- | |
| # BlockMasks. One uniform mask_mod for every layer: `window = window_size or seq_len`, and since | |
| # q - kv <= seq_len - 1 < seq_len, "no window" is the same expression as "window = seq_len". | |
| # Both builders are compiler-disabled (BlockMask construction is not traced) and are called once per | |
| # forward from GPT.forward -- that is the only graph boundary in the attention path. | |
| # --------------------------------------------------------------------------- | |
| def _causal_mask(seq_len: int, window_size, device_key: str): | |
| window = window_size or seq_len | |
| def mask_mod(b, h, q_idx, kv_idx): | |
| return (kv_idx <= q_idx) & (q_idx - kv_idx < window) | |
| return create_block_mask(mask_mod, B=None, H=None, Q_LEN=seq_len, KV_LEN=seq_len, device=device_key) | |
| def causal_masks(seq_len: int, windows, device) -> dict: | |
| """{window_size: BlockMask} for plain causal attention; cached per (seq_len, window, device).""" | |
| return {w: _causal_mask(seq_len, w, str(device)) for w in windows} | |
| def doc_masks(doc_ids: torch.Tensor, windows) -> dict: | |
| """{window_size: BlockMask} for causal + same-document attention. doc_ids: (B, T) int, per batch. | |
| On PyTorch 2.5+, _compile=True fuses mask_mod so the (B, T, T) boolean grid is never materialised.""" | |
| b_, t_ = doc_ids.shape | |
| def build(window_size): | |
| window = window_size or t_ | |
| def mask_mod(b, h, q_idx, kv_idx): | |
| return (doc_ids[b, q_idx] == doc_ids[b, kv_idx]) & (kv_idx <= q_idx) & (q_idx - kv_idx < window) | |
| mask_kwargs = dict(B=b_, H=None, Q_LEN=t_, KV_LEN=t_, device=doc_ids.device) | |
| version = torch.__version__.split("+", 1)[0].split(".") | |
| try: | |
| supports_compile = (int(version[0]), int(version[1])) >= (2, 5) | |
| except (IndexError, ValueError): | |
| supports_compile = False | |
| if supports_compile: | |
| mask_kwargs["_compile"] = True | |
| return create_block_mask(mask_mod, **mask_kwargs) | |
| return {w: build(w) for w in windows} | |
| # --------------------------------------------------------------------------- | |
| # Telemetry counters (model.py records its Liger/Triton success/fallback events here). | |
| # --------------------------------------------------------------------------- | |
| _backend_stats: dict[str, int] = {} | |
| def _record_backend(name: str) -> None: | |
| # Inside a compiled region this body runs only at trace time, so counting here under-reports (verified: | |
| # 5 compiled calls -> 3 counts). Record nothing while compiling rather than report wrong numbers. | |
| if torch.compiler.is_compiling(): | |
| return | |
| _backend_stats[name] = _backend_stats.get(name, 0) + 1 | |
| def get_backend_stats() -> dict[str, int]: | |
| return dict(_backend_stats) | |
| def reset_backend_stats() -> None: | |
| _backend_stats.clear() |