Buckets:
| """Autograd glue for the vendored FastVideo Triton block-sparse kernels. | |
| Adapted from FastVideo's ``fastvideo_kernel/block_sparse_attn.py`` (Apache-2.0) | |
| with the compiled sm90/sm100a backends stripped: only the pure-Triton fwd+bwd | |
| pair remains, JIT-compiled on first call. Custom ops live in their own | |
| ``aitk_h3_vsa::`` namespace so a real fastvideo-kernel install cannot clash. | |
| Public entry: ``block_sparse_attn(q, k, v, block_map, variable_block_sizes)`` | |
| with q/k/v ``[B, H, S_pad, D]`` (S_pad a multiple of 64), block_map a bool | |
| ``[B, H, n_tiles, n_tiles]``, variable_block_sizes int32 ``[n_tiles]`` live | |
| rows per tile (live rows at the front of each 64-token tile). | |
| """ | |
| from typing import Tuple | |
| import torch | |
| def _as_int32_contig(t: torch.Tensor, name: str) -> torch.Tensor: | |
| if not t.is_cuda: | |
| raise RuntimeError(f"{name} must be a CUDA tensor, got device={t.device}") | |
| if t.dtype != torch.int32: | |
| t = t.to(torch.int32) | |
| if not t.is_contiguous(): | |
| t = t.contiguous() | |
| return t | |
| def _block_sparse_attn_triton( | |
| q: torch.Tensor, | |
| k: torch.Tensor, | |
| v: torch.Tensor, | |
| q2k_idx: torch.Tensor, | |
| q2k_num: torch.Tensor, | |
| variable_block_sizes: torch.Tensor, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| from .block_sparse_attn_triton import triton_block_sparse_attn_forward | |
| o, M = triton_block_sparse_attn_forward( | |
| q.contiguous(), | |
| k.contiguous(), | |
| v.contiguous(), | |
| q2k_idx, | |
| q2k_num, | |
| variable_block_sizes, | |
| ) | |
| return o, M | |
| def _block_sparse_attn_triton_fake(q, k, v, q2k_idx, q2k_num, variable_block_sizes): | |
| o = torch.empty_like(q) | |
| M = torch.empty( | |
| (q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32 | |
| ) | |
| return o, M | |
| def _block_sparse_attn_backward_triton( | |
| grad_output: torch.Tensor, | |
| q: torch.Tensor, | |
| k: torch.Tensor, | |
| v: torch.Tensor, | |
| o: torch.Tensor, | |
| M: torch.Tensor, | |
| q2k_idx: torch.Tensor, | |
| q2k_num: torch.Tensor, | |
| variable_block_sizes: torch.Tensor, | |
| ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| from .block_sparse_attn_triton import triton_block_sparse_attn_backward | |
| from .index import invert_indices | |
| num_kv_blocks = int(variable_block_sizes.numel()) | |
| k2q_idx, k2q_num = invert_indices(q2k_idx, q2k_num, num_kv_blocks=num_kv_blocks) | |
| # q/k/v are saved from the user-facing inputs and may be non-contiguous; | |
| # o/M are kernel outputs so are already contiguous. | |
| dq, dk, dv = triton_block_sparse_attn_backward( | |
| grad_output.contiguous(), | |
| q.contiguous(), | |
| k.contiguous(), | |
| v.contiguous(), | |
| o, | |
| M, | |
| q2k_idx, | |
| q2k_num, | |
| k2q_idx, | |
| k2q_num, | |
| variable_block_sizes, | |
| ) | |
| return dq, dk, dv | |
| def _block_sparse_attn_backward_triton_fake( | |
| grad_output, q, k, v, o, M, q2k_idx, q2k_num, variable_block_sizes | |
| ): | |
| return torch.empty_like(q), torch.empty_like(k), torch.empty_like(v) | |
| def _setup_context(ctx, inputs, output): | |
| q, k, v, q2k_idx, q2k_num, variable_block_sizes = inputs | |
| o, M = output | |
| ctx.save_for_backward(q, k, v, o, M, q2k_idx, q2k_num, variable_block_sizes) | |
| def _backward(ctx, grad_o, grad_M): | |
| q, k, v, o, M, q2k_idx, q2k_num, variable_block_sizes = ctx.saved_tensors | |
| dq, dk, dv = _block_sparse_attn_backward_triton( | |
| grad_o, q, k, v, o, M, q2k_idx, q2k_num, variable_block_sizes | |
| ) | |
| return dq, dk, dv, None, None, None | |
| _block_sparse_attn_triton.register_autograd(_backward, setup_context=_setup_context) | |
| def block_sparse_attn( | |
| q: torch.Tensor, | |
| k: torch.Tensor, | |
| v: torch.Tensor, | |
| block_map: torch.Tensor, | |
| variable_block_sizes: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """Block-sparse attention with autograd from a bool tile map. Returns o.""" | |
| from .index import map_to_index | |
| q2k_idx, q2k_num = map_to_index(block_map) | |
| q2k_idx = _as_int32_contig(q2k_idx, "q2k_idx") | |
| q2k_num = _as_int32_contig(q2k_num, "q2k_num") | |
| variable_block_sizes = _as_int32_contig( | |
| variable_block_sizes, "variable_block_sizes" | |
| ) | |
| o, _ = _block_sparse_attn_triton(q, k, v, q2k_idx, q2k_num, variable_block_sizes) | |
| return o | |
Xet Storage Details
- Size:
- 4.6 kB
- Xet hash:
- 655be3643e50a71caf7431b64f8a8063c053676c61f75995a0d9ee86699a2744
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.