"""FlexAttention-only attention kernel. One backend, one code path: no device probing, no backend dispatch, no try/except fallbacks. Every attention layer calls `fused_attention(q, k, v, block_mask)`; what each layer may attend to (causal / sliding window / same-document) lives entirely in the BlockMask, which GPT.forward builds once per forward and shares across layers. Requires CUDA + torch >= 2.5 (`create_block_mask(..., _compile=True)`). Attention-weight dropout is not supported (DenseMHA refuses dropout > 0 at construction). """ import functools import inspect import torch from torch.nn.attention.flex_attention import create_block_mask, flex_attention print( f"[kernel] import: cuda_avail={torch.cuda.is_available()} " f"cuda_ver={torch.version.cuda} file={__file__}", flush=True, ) _flex = None def _is_compiled() -> bool: return _flex is not None and _flex is not flex_attention def _get_flex(): global _flex if _flex is None: _flex = ( torch.compile(flex_attention, dynamic=False) if torch.cuda.is_available() else flex_attention ) return _flex # Computed once at import: Dynamo never sees a call to an lru_cache-wrapped function. try: _SUPPORTS_KERNEL_OPTIONS = "kernel_options" in inspect.signature(flex_attention).parameters except (TypeError, ValueError): _SUPPORTS_KERNEL_OPTIONS = False def _flex_opts(h, t, d): # Pure function of the shape, no cache: plain Python branches trace cleanly under Dynamo. if d <= 32: return {"BLOCK_M": 128, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2} if d <= 64: return {"BLOCK_M": 64, "BLOCK_N": 64, "num_warps": 4, "num_stages": 2} return {"BLOCK_M": 128, "BLOCK_N": 128, "num_warps": 8, "num_stages": 2} def fused_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, block_mask) -> torch.Tensor: """q: (B, Hq, T, D); k, v: (B, Hkv, T, D). GQA is handled natively (Hq may be a multiple of Hkv).""" kwargs = { "block_mask": block_mask, "enable_gqa": q.shape[1] != k.shape[1], } if _SUPPORTS_KERNEL_OPTIONS: kwargs["kernel_options"] = _flex_opts(q.shape[1], q.shape[2], q.shape[3]) return _get_flex()(q, k, v, **kwargs) # --------------------------------------------------------------------------- # BlockMasks. One uniform mask_mod for every layer: `window = window_size or seq_len`, and since # q - kv <= seq_len - 1 < seq_len, "no window" is the same expression as "window = seq_len". # Both builders are compiler-disabled (BlockMask construction is not traced) and are called once per # forward from GPT.forward -- that is the only graph boundary in the attention path. # --------------------------------------------------------------------------- @functools.lru_cache(maxsize=32) def _causal_mask(seq_len: int, window_size, device_key: str): window = window_size or seq_len def mask_mod(b, h, q_idx, kv_idx): return (kv_idx <= q_idx) & (q_idx - kv_idx < window) return create_block_mask(mask_mod, B=None, H=None, Q_LEN=seq_len, KV_LEN=seq_len, device=device_key) @torch.compiler.disable def causal_masks(seq_len: int, windows, device) -> dict: """{window_size: BlockMask} for plain causal attention; cached per (seq_len, window, device).""" return {w: _causal_mask(seq_len, w, str(device)) for w in windows} @torch.compiler.disable def doc_masks(doc_ids: torch.Tensor, windows) -> dict: """{window_size: BlockMask} for causal + same-document attention. doc_ids: (B, T) int, per batch. On PyTorch 2.5+, _compile=True fuses mask_mod so the (B, T, T) boolean grid is never materialised.""" b_, t_ = doc_ids.shape def build(window_size): window = window_size or t_ def mask_mod(b, h, q_idx, kv_idx): return (doc_ids[b, q_idx] == doc_ids[b, kv_idx]) & (kv_idx <= q_idx) & (q_idx - kv_idx < window) mask_kwargs = dict(B=b_, H=None, Q_LEN=t_, KV_LEN=t_, device=doc_ids.device) version = torch.__version__.split("+", 1)[0].split(".") try: supports_compile = (int(version[0]), int(version[1])) >= (2, 5) except (IndexError, ValueError): supports_compile = False if supports_compile: mask_kwargs["_compile"] = True return create_block_mask(mask_mod, **mask_kwargs) return {w: build(w) for w in windows} # --------------------------------------------------------------------------- # Telemetry counters (model.py records its Liger/Triton success/fallback events here). # --------------------------------------------------------------------------- _backend_stats: dict[str, int] = {} def _record_backend(name: str) -> None: # Inside a compiled region this body runs only at trace time, so counting here under-reports (verified: # 5 compiled calls -> 3 counts). Record nothing while compiling rather than report wrong numbers. if torch.compiler.is_compiling(): return _backend_stats[name] = _backend_stats.get(name, 0) + 1 def get_backend_stats() -> dict[str, int]: return dict(_backend_stats) def reset_backend_stats() -> None: _backend_stats.clear()