wan/modules/sparse_attention.py

import warnings
import math

import torch

from .kernel import (
    _attention,
    get_sm_80_120_block_map,
    get_sm_90_100_block_map,
)

try:
    from flash_attn.cute import flash_attn_func as flash_attn_func_v4
    FLASH_ATTN_4_AVAILABLE = True
except ModuleNotFoundError:
    FLASH_ATTN_4_AVAILABLE = False


__all__ = [
    "DEVICE_SM",
    "calculate_chunk_sparsities",
    "sparse_attention",
]


def _get_device_sm(device=None):
    if not torch.cuda.is_available():
        return None
    major, minor = torch.cuda.get_device_capability(device)
    return major * 10 + minor


DEVICE_SM = _get_device_sm()


def _dense_attention(q, k, v, softmax_scale=None):
    q_t = q.transpose(1, 2)
    k_t = k.transpose(1, 2)
    v_t = v.transpose(1, 2)
    out = torch.nn.functional.scaled_dot_product_attention(
        q_t,
        k_t,
        v_t,
        scale=softmax_scale,
    )
    return out.transpose(1, 2).contiguous()


def calculate_chunk_sparsities(num_output_frames, num_frame_per_block, local_attn_size=21, sparse_config=None):
    sparse_config = sparse_config or {}
    target_sparsity = sparse_config.get("sparsity", None)
    base_sparsity = sparse_config.get("sparsity_base", target_sparsity)
    if target_sparsity is None:
        return []

    target_sparsity = float(target_sparsity)
    base_sparsity = float(base_sparsity)
    chunk_frame_counts = range(
        2 * num_frame_per_block,
        num_output_frames + 1,
        num_frame_per_block,
    )
    kv_lengths = [
        frame_count if local_attn_size == -1 else min(frame_count, local_attn_size)
        for frame_count in chunk_frame_counts
    ]
    alphas = [1 / math.sqrt(frame_count) for frame_count in chunk_frame_counts]

    target_flops = sum((1 - target_sparsity) * kv_length for kv_length in kv_lengths)
    base_flops = sum((1 - base_sparsity) * kv_length for kv_length in kv_lengths)
    alpha_weighted_flops = sum(
        alpha * kv_length
        for alpha, kv_length in zip(alphas, kv_lengths)
    )
    if alpha_weighted_flops == 0:
        return [base_sparsity] * len(alphas)

    beta = (target_flops - base_flops) / alpha_weighted_flops
    return  [0.0] + [
        base_sparsity - alpha * beta
        for alpha in alphas
    ]


def sparse_attention(
    q,
    k,
    v,
    sparsity_list=None,
    chunk_id=None,
    BLKQ=None,
    BLKK=None,
    frame_seq=1536,
    keep_frames=6,
    keep_sink=0,
    keep_near=0,
    softmax_scale=None,
):
    """Sparse attention dispatcher for BLHD tensors.

    SM90/SM100 use FA4 block sparse tensors. SM80/SM120 use the local Triton
    sparse kernel in kernel.py.
    """
    topk_ratio = 1.0 - float(sparsity_list[chunk_id])

    if DEVICE_SM in (90, 100):
        if not FLASH_ATTN_4_AVAILABLE:
            warnings.warn("FA4 is not available; falling back to dense attention.")
            return _dense_attention(q, k, v, softmax_scale=softmax_scale)

        BLKQ = 128 if BLKQ is None else BLKQ
        BLKK = 128 if BLKK is None else BLKK
        sparse_kwargs = get_sm_90_100_block_map(
            q,
            k,
            topk_ratio=topk_ratio,
            BLKQ=BLKQ,
            BLKK=BLKK,
            frame_seq=frame_seq,
            keep_frames=keep_frames,
            keep_sink=keep_sink,
            keep_near=keep_near,
        )
        return flash_attn_func_v4(q, k, v, softmax_scale=softmax_scale, **sparse_kwargs)[0]

    if DEVICE_SM in (80, 120):
        BLKQ = 64 if BLKQ is None else BLKQ
        BLKK = 64 if BLKK is None else BLKK
        sparse_map, lut, topk = get_sm_80_120_block_map(
            q,
            k,
            topk_ratio=topk_ratio,
            BLKQ=BLKQ,
            BLKK=BLKK,
            frame_seq=frame_seq,
            keep_frames=keep_frames,
            keep_sink=keep_sink,
            keep_near=keep_near,
        )
        return _attention.apply(q, k, v, sparse_map, lut, topk, BLKQ, BLKK, softmax_scale)

    warnings.warn(f"Unsupported sparse attention SM{DEVICE_SM}; falling back to dense attention.")
    return _dense_attention(q, k, v, softmax_scale=softmax_scale)