agent.pic.editor / qwenimage /qwen_fa3_processor.py
Onise's picture
Update qwenimage/qwen_fa3_processor.py
c3b3e31 verified
Raw History Blame Contribute Delete
7.45 kB
"""
Paired with a good language model. Thanks!
ZeroGPU note:
FlashAttention-3 (kernels-community/vllm-flash-attn3) has no binary for
Blackwell (sm_120), which is what ZeroGPU currently attaches. Loading the
kernel at import time under CUDA *emulation* reports a fake capability, so
FA3 gets enabled and then crashes on the real GPU with:
CUDA error: no kernel image is available for execution on the device
Kernel init is therefore deferred until a real CUDA device is present, and
sm_120+ always falls back to PyTorch SDPA.
"""
import torch
from typing import Optional, Tuple
from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
import torch.nn.functional as F
_flash_attn_func = None
_kernels_err = None
_fa3_inited = False
def _ensure_fa3() -> None:
global _flash_attn_func, _kernels_err, _fa3_inited
if _fa3_inited:
return
_fa3_inited = True
if not torch.cuda.is_available():
_kernels_err = RuntimeError("CUDA not available; using SDPA fallback.")
return
try:
cap = torch.cuda.get_device_capability()
except Exception as e:
_kernels_err = e
return
# Blackwell (sm_120+) is not supported by the vllm-flash-attn3 kernel binary.
if cap >= (12, 0):
_kernels_err = RuntimeError(
f"GPU compute capability sm_{cap[0]}{cap[1]} (Blackwell+) is not supported "
"by the kernels-community/vllm-flash-attn3 binary; using SDPA fallback."
)
return
try:
from kernels import get_kernel
_k = get_kernel("kernels-community/vllm-flash-attn3")
_flash_attn_func = _k.flash_attn_func
except Exception as e:
_flash_attn_func = None
_kernels_err = e
def _fa3_available() -> bool:
_ensure_fa3()
return _flash_attn_func is not None
def _sdpa_fallback(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
# q/k/v: (B, S, H, D_h) → SDPA expects (B, H, S, D_h)
q = q.transpose(1, 2)
k = k.transpose(1, 2)
v = v.transpose(1, 2)
out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
return out.transpose(1, 2) # back to (B, S, H, D_h)
@torch.library.custom_op("flash::flash_attn_func", mutates_args=())
def flash_attn_func(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
) -> torch.Tensor:
_ensure_fa3()
if _flash_attn_func is None:
return _sdpa_fallback(q, k, v)
outputs, lse = _flash_attn_func(q, k, v, causal=causal)
return outputs
@flash_attn_func.register_fake
def _(q, k, v, **kwargs):
return torch.empty_like(q).contiguous()
class QwenDoubleStreamAttnProcessorFA3:
"""
FA3-based attention processor for Qwen double-stream architecture.
Computes joint attention over concatenated [text, image] streams using vLLM FlashAttention-3
accessed via Hugging Face `kernels`, with an automatic SDPA fallback.
Notes / limitations:
- General attention masks are not supported here (FA3 path). `is_causal=False` and no arbitrary mask.
- Optional windowed attention / sink tokens / softcap can be plumbed through if you use those features.
- Expects an available `apply_rotary_emb_qwen` in scope (same as your non-FA3 processor).
"""
_attention_backend = "fa3" # for parity with your other processors, not used internally
def __init__(self):
_ensure_fa3()
if not _fa3_available():
print(f"[QwenDoubleStreamAttnProcessorFA3] FA3 unavailable, using SDPA fallback. Reason: {_kernels_err}")
@torch.no_grad()
def __call__(
self,
attn, # Attention module with to_q/to_k/to_v/add_*_proj, norms, to_out, to_add_out, and .heads
hidden_states: torch.FloatTensor, # (B, S_img, D_model) image stream
encoder_hidden_states: torch.FloatTensor = None, # (B, S_txt, D_model) text stream
encoder_hidden_states_mask: torch.FloatTensor = None, # unused in FA3 path
attention_mask: Optional[torch.FloatTensor] = None, # unused in FA3 path
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # (img_freqs, txt_freqs)
) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
if encoder_hidden_states is None:
raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states (text stream).")
if attention_mask is not None:
# FA3 kernel path here does not consume arbitrary masks; fail fast to avoid silent correctness issues.
raise NotImplementedError("attention_mask is not supported in this FA3 implementation.")
B, S_img, _ = hidden_states.shape
S_txt = encoder_hidden_states.shape[1]
# ---- QKV projections (image/sample stream) ----
img_q = attn.to_q(hidden_states) # (B, S_img, D)
img_k = attn.to_k(hidden_states)
img_v = attn.to_v(hidden_states)
# ---- QKV projections (text/context stream) ----
txt_q = attn.add_q_proj(encoder_hidden_states) # (B, S_txt, D)
txt_k = attn.add_k_proj(encoder_hidden_states)
txt_v = attn.add_v_proj(encoder_hidden_states)
# ---- Reshape to (B, S, H, D_h) ----
H = attn.heads
img_q = img_q.unflatten(-1, (H, -1))
img_k = img_k.unflatten(-1, (H, -1))
img_v = img_v.unflatten(-1, (H, -1))
txt_q = txt_q.unflatten(-1, (H, -1))
txt_k = txt_k.unflatten(-1, (H, -1))
txt_v = txt_v.unflatten(-1, (H, -1))
# ---- Q/K normalization (per your module contract) ----
if getattr(attn, "norm_q", None) is not None:
img_q = attn.norm_q(img_q)
if getattr(attn, "norm_k", None) is not None:
img_k = attn.norm_k(img_k)
if getattr(attn, "norm_added_q", None) is not None:
txt_q = attn.norm_added_q(txt_q)
if getattr(attn, "norm_added_k", None) is not None:
txt_k = attn.norm_added_k(txt_k)
# ---- RoPE (Qwen variant) ----
if image_rotary_emb is not None:
img_freqs, txt_freqs = image_rotary_emb
# expects tensors shaped (B, S, H, D_h)
img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
# ---- Joint attention over [text, image] along sequence axis ----
# Shapes: (B, S_total, H, D_h)
q = torch.cat([txt_q, img_q], dim=1)
k = torch.cat([txt_k, img_k], dim=1)
v = torch.cat([txt_v, img_v], dim=1)
if _fa3_available():
out = flash_attn_func(q, k, v, causal=False) # out: (B, S_total, H, D_h)
else:
out = _sdpa_fallback(q, k, v) # out: (B, S_total, H, D_h)
# ---- Back to (B, S, D_model) ----
out = out.flatten(2, 3).to(q.dtype)
# Split back to text / image segments
txt_attn_out = out[:, :S_txt, :]
img_attn_out = out[:, S_txt:, :]
# ---- Output projections ----
img_attn_out = attn.to_out[0](img_attn_out)
if len(attn.to_out) > 1:
img_attn_out = attn.to_out[1](img_attn_out) # dropout if present
txt_attn_out = attn.to_add_out(txt_attn_out)
return img_attn_out, txt_attn_out