clairic / clairic_runtime /attention_metadata.py
areneau's picture
Update Clairic Transformers runtime
524fd9d verified
Raw History Blame Contribute Delete
7.61 kB
"""Reuse pinned FA3 unpadding metadata within one owned decode forward."""
from __future__ import annotations
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
import threading
from typing import Any
import torch
from transformers.integrations.flash_attention import flash_attention_forward
from transformers.modeling_flash_attention_utils import _get_unpad_data, _index_first_axis
_PINNED_FA3 = "kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0"
_VARLEN_KEYS = frozenset({"cu_seq_lens_q", "cu_seq_lens_k", "max_length_q", "max_length_k"})
class _Entry:
def __init__(self) -> None:
self.key: tuple[Any, ...] | None = None
self.mask: torch.Tensor | None = None
self.indices: torch.Tensor | None = None
self.cu_k: torch.Tensor | None = None
self.cu_q: torch.Tensor | None = None
self.maximum = 0
self.bytes = 0
def clear(self) -> None:
self.key, self.mask, self.indices, self.cu_k, self.cu_q = None, None, None, None, None
self.maximum, self.bytes = 0, 0
class DecodeMetadata:
def __init__(self, max_bytes: int = 8 * 1024**2) -> None:
if max_bytes <= 0:
raise ValueError("Expected a positive metadata bound")
self.max_bytes = max_bytes
self._lock = threading.Lock()
self._current: ContextVar[_Entry | None] = ContextVar("clairic_decode_metadata", default=None)
self._active: _Entry | None = None
self._closed = False
self.scopes = 0
self.materializations = 0
self.hits = 0
self.peak_bytes = 0
@contextmanager
def scope(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Iterator[None]:
token = self._current.set(None)
acquired = False
entry = None
try:
inputs = kwargs.get("input_ids", args[0] if args else None)
if inputs is None:
inputs = kwargs.get("inputs_embeds")
eligible = (not self._closed and not torch.is_grad_enabled()
and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling()
and isinstance(inputs, torch.Tensor) and inputs.is_cuda
and inputs.ndim in (2, 3) and inputs.shape[1] == 1)
if eligible and self._lock.acquire(blocking=False):
acquired = True
if not self._closed:
entry = _Entry()
self._active = entry
self._current.set(entry)
self.scopes += 1
yield
finally:
if entry is not None:
entry.clear()
self._active = None
self._current.reset(token)
if acquired:
self._lock.release()
def _metadata(self, mask: torch.Tensor) -> _Entry | None:
entry = self._current.get()
if self._closed or entry is None:
return None
batch = mask.shape[0]
if mask.numel() * 8 + 2 * (batch + 1) * 4 > self.max_bytes:
entry.clear()
return None
try:
version = mask._version
except RuntimeError:
version = None
key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device,
mask.dtype, tuple(mask.shape), tuple(mask.stride()),
torch.cuda.current_stream(mask.device).cuda_stream)
if entry.mask is mask and entry.key == key:
self.hits += 1
return entry
entry.clear()
indices, cu_k, maximum = _get_unpad_data(mask)
cu_q = torch.arange(batch + 1, dtype=torch.int32, device=mask.device)
retained = sum(value.numel() * value.element_size() for value in (indices, cu_k, cu_q))
if retained > self.max_bytes:
return None
entry.key, entry.mask = key, mask
entry.indices, entry.cu_k, entry.cu_q = indices, cu_k, cu_q
entry.maximum, entry.bytes = int(maximum.item()), retained
self.materializations += 1
self.peak_bytes = max(self.peak_bytes, retained)
return entry
def attend(self, interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]],
module: Any, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
mask: torch.Tensor | None, *, dropout: float, scaling: float,
**kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
eligible = (not self._closed and self._current.get() is not None
and not torch.is_grad_enabled() and torch.is_inference_mode_enabled()
and not torch.compiler.is_compiling() and interface is flash_attention_forward
and module.config._attn_implementation == _PINNED_FA3
and dropout == 0.0
and getattr(module.config, "sliding_window", None) is None
and kwargs.get("sliding_window") is None and not (_VARLEN_KEYS & kwargs.keys())
and mask is not None and mask.is_cuda and mask.ndim == 2
and mask.dtype == torch.bool and mask.numel() > 0
and all(item.is_cuda and item.dtype == torch.bfloat16 and item.ndim == 4
and item.device == mask.device and item.stride(-1) == 1
for item in (query, key, value)))
if eligible:
assert mask is not None
positions = kwargs.get("position_ids")
eligible = (query.shape[0] == key.shape[0] == value.shape[0] == mask.shape[0]
and query.shape[2] == 1 and key.shape[2] == value.shape[2] == mask.shape[1]
and mask.shape[1] > 1
and query.shape[1] in (16, 24) and key.shape[1] == value.shape[1] == 4
and query.shape[3] == key.shape[3] == value.shape[3] == 256
and (positions is None or (isinstance(positions, torch.Tensor)
and positions.ndim == 2 and tuple(positions.shape) == (query.shape[0], 1))))
entry = self._metadata(mask) if eligible and mask is not None else None
if entry is None:
active = self._current.get()
if active is not None:
active.clear()
return interface(module, query, key, value, mask, dropout=dropout, scaling=scaling, **kwargs)
assert entry.indices is not None and entry.cu_q is not None and entry.cu_k is not None
packed_key = _index_first_axis(key.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
packed_value = _index_first_axis(value.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
return interface(module, query, packed_key, packed_value, None, dropout=dropout, scaling=scaling,
cu_seq_lens_q=entry.cu_q, cu_seq_lens_k=entry.cu_k,
max_length_q=1, max_length_k=entry.maximum, **kwargs)
def snapshot(self) -> dict[str, int | bool]:
return {"scopes": self.scopes, "materializations": self.materializations, "hits": self.hits,
"peak_bytes": self.peak_bytes, "retained_bytes": self._active.bytes if self._active else 0,
"closed": self._closed}
def close(self) -> None:
self._closed = True
with self._lock:
if self._active is not None:
self._active.clear()
self._active = None