Onise commited on
Commit
4dd73bb
·
verified ·
1 Parent(s): cfdab6c

Update qwenimage/qwen_fa3_processor.py

Browse files
Files changed (1) hide show
  1. qwenimage/qwen_fa3_processor.py +157 -156
qwenimage/qwen_fa3_processor.py CHANGED
@@ -1,157 +1,158 @@
1
- """
2
- Paired with a good language model. Thanks!
3
- """
4
-
5
- import torch
6
- from typing import Optional, Tuple
7
- from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
8
-
9
- import torch.nn.functional as F
10
-
11
- _flash_attn_func = None
12
- _kernels_err = None
13
-
14
- try:
15
- from kernels import get_kernel
16
- # Blackwell (sm_120+) is not yet supported by the vllm-flash-attn3 kernel binary
17
- _cap = torch.cuda.get_device_capability() if torch.cuda.is_available() else (0, 0)
18
- if _cap >= (12, 0):
19
- _kernels_err = RuntimeError(
20
- f"GPU compute capability sm_{_cap[0]}{_cap[1]} (Blackwell+) is not supported "
21
- "by the kernels-community/vllm-flash-attn3 binary; using SDPA fallback."
22
- )
23
- else:
24
- _k = get_kernel("kernels-community/vllm-flash-attn3")
25
- _flash_attn_func = _k.flash_attn_func
26
- except Exception as e:
27
- _flash_attn_func = None
28
- _kernels_err = e
29
-
30
-
31
- def _fa3_available() -> bool:
32
- return _flash_attn_func is not None
33
-
34
-
35
- def _sdpa_fallback(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
36
- # q/k/v: (B, S, H, D_h) → SDPA expects (B, H, S, D_h)
37
- q = q.transpose(1, 2)
38
- k = k.transpose(1, 2)
39
- v = v.transpose(1, 2)
40
- out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
41
- return out.transpose(1, 2) # back to (B, S, H, D_h)
42
-
43
-
44
- @torch.library.custom_op("flash::flash_attn_func", mutates_args=())
45
- def flash_attn_func(
46
- q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
47
- ) -> torch.Tensor:
48
- outputs, lse = _flash_attn_func(q, k, v, causal=causal)
49
- return outputs
50
-
51
- @flash_attn_func.register_fake
52
- def _(q, k, v, **kwargs):
53
- return torch.empty_like(q).contiguous()
54
-
55
-
56
- class QwenDoubleStreamAttnProcessorFA3:
57
- """
58
- FA3-based attention processor for Qwen double-stream architecture.
59
- Computes joint attention over concatenated [text, image] streams using vLLM FlashAttention-3
60
- accessed via Hugging Face `kernels`.
61
-
62
- Notes / limitations:
63
- - General attention masks are not supported here (FA3 path). `is_causal=False` and no arbitrary mask.
64
- - Optional windowed attention / sink tokens / softcap can be plumbed through if you use those features.
65
- - Expects an available `apply_rotary_emb_qwen` in scope (same as your non-FA3 processor).
66
- """
67
-
68
- _attention_backend = "fa3" # for parity with your other processors, not used internally
69
-
70
- def __init__(self):
71
- if not _fa3_available():
72
- print(f"[QwenDoubleStreamAttnProcessorFA3] FA3 unavailable, using SDPA fallback. Reason: {_kernels_err}")
73
-
74
- @torch.no_grad()
75
- def __call__(
76
- self,
77
- attn, # Attention module with to_q/to_k/to_v/add_*_proj, norms, to_out, to_add_out, and .heads
78
- hidden_states: torch.FloatTensor, # (B, S_img, D_model) image stream
79
- encoder_hidden_states: torch.FloatTensor = None, # (B, S_txt, D_model) text stream
80
- encoder_hidden_states_mask: torch.FloatTensor = None, # unused in FA3 path
81
- attention_mask: Optional[torch.FloatTensor] = None, # unused in FA3 path
82
- image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # (img_freqs, txt_freqs)
83
- ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
84
- if encoder_hidden_states is None:
85
- raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states (text stream).")
86
- if attention_mask is not None:
87
- # FA3 kernel path here does not consume arbitrary masks; fail fast to avoid silent correctness issues.
88
- raise NotImplementedError("attention_mask is not supported in this FA3 implementation.")
89
-
90
- B, S_img, _ = hidden_states.shape
91
- S_txt = encoder_hidden_states.shape[1]
92
-
93
- # ---- QKV projections (image/sample stream) ----
94
- img_q = attn.to_q(hidden_states) # (B, S_img, D)
95
- img_k = attn.to_k(hidden_states)
96
- img_v = attn.to_v(hidden_states)
97
-
98
- # ---- QKV projections (text/context stream) ----
99
- txt_q = attn.add_q_proj(encoder_hidden_states) # (B, S_txt, D)
100
- txt_k = attn.add_k_proj(encoder_hidden_states)
101
- txt_v = attn.add_v_proj(encoder_hidden_states)
102
-
103
- # ---- Reshape to (B, S, H, D_h) ----
104
- H = attn.heads
105
- img_q = img_q.unflatten(-1, (H, -1))
106
- img_k = img_k.unflatten(-1, (H, -1))
107
- img_v = img_v.unflatten(-1, (H, -1))
108
-
109
- txt_q = txt_q.unflatten(-1, (H, -1))
110
- txt_k = txt_k.unflatten(-1, (H, -1))
111
- txt_v = txt_v.unflatten(-1, (H, -1))
112
-
113
- # ---- Q/K normalization (per your module contract) ----
114
- if getattr(attn, "norm_q", None) is not None:
115
- img_q = attn.norm_q(img_q)
116
- if getattr(attn, "norm_k", None) is not None:
117
- img_k = attn.norm_k(img_k)
118
- if getattr(attn, "norm_added_q", None) is not None:
119
- txt_q = attn.norm_added_q(txt_q)
120
- if getattr(attn, "norm_added_k", None) is not None:
121
- txt_k = attn.norm_added_k(txt_k)
122
-
123
- # ---- RoPE (Qwen variant) ----
124
- if image_rotary_emb is not None:
125
- img_freqs, txt_freqs = image_rotary_emb
126
- # expects tensors shaped (B, S, H, D_h)
127
- img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
128
- img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
129
- txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
130
- txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
131
-
132
- # ---- Joint attention over [text, image] along sequence axis ----
133
- # Shapes: (B, S_total, H, D_h)
134
- q = torch.cat([txt_q, img_q], dim=1)
135
- k = torch.cat([txt_k, img_k], dim=1)
136
- v = torch.cat([txt_v, img_v], dim=1)
137
-
138
- if _fa3_available():
139
- out = flash_attn_func(q, k, v, causal=False) # out: (B, S_total, H, D_h)
140
- else:
141
- out = _sdpa_fallback(q, k, v) # out: (B, S_total, H, D_h)
142
-
143
- # ---- Back to (B, S, D_model) ----
144
- out = out.flatten(2, 3).to(q.dtype)
145
-
146
- # Split back to text / image segments
147
- txt_attn_out = out[:, :S_txt, :]
148
- img_attn_out = out[:, S_txt:, :]
149
-
150
- # ---- Output projections ----
151
- img_attn_out = attn.to_out[0](img_attn_out)
152
- if len(attn.to_out) > 1:
153
- img_attn_out = attn.to_out[1](img_attn_out) # dropout if present
154
-
155
- txt_attn_out = attn.to_add_out(txt_attn_out)
156
-
 
157
  return img_attn_out, txt_attn_out
 
1
+ """
2
+ Paired with a good language model. Thanks!
3
+ """
4
+
5
+ import torch
6
+ from typing import Optional, Tuple
7
+ from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
8
+
9
+ import torch.nn.functional as F
10
+
11
+ _flash_attn_func = None
12
+ _kernels_err = None
13
+
14
+ try:
15
+ from kernels import get_kernel
16
+ # Blackwell (sm_120+) is not yet supported by the vllm-flash-attn3 kernel binary
17
+ _cap = torch.cuda.get_device_capability() if torch.cuda.is_available() else (0, 0)
18
+ if _cap >= (12, 0):
19
+ _kernels_err = RuntimeError(
20
+ f"GPU compute capability sm_{_cap[0]}{_cap[1]} (Blackwell+) is not supported "
21
+ "by the kernels-community/vllm-flash-attn3 binary; using SDPA fallback."
22
+ )
23
+ else:
24
+ _k = get_kernel("kernels-community/vllm-flash-attn3")
25
+ _flash_attn_func = _k.flash_attn_func
26
+ except Exception as e:
27
+ _flash_attn_func = None
28
+ _kernels_err = e
29
+
30
+
31
+ def _fa3_available() -> bool:
32
+ return _flash_attn_func is not None
33
+
34
+
35
+ def _sdpa_fallback(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
36
+ # q/k/v: (B, S, H, D_h) → SDPA expects (B, H, S, D_h)
37
+ q = q.transpose(1, 2)
38
+ k = k.transpose(1, 2)
39
+ v = v.transpose(1, 2)
40
+ out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
41
+ return out.transpose(1, 2) # back to (B, S, H, D_h)
42
+
43
+
44
+ @torch.library.custom_op("flash::flash_attn_func", mutates_args=())
45
+ def flash_attn_func(
46
+ q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
47
+ ) -> torch.Tensor:
48
+ outputs, lse = _flash_attn_func(q, k, v, causal=causal)
49
+ return outputs
50
+
51
+
52
+ @flash_attn_func.register_fake
53
+ def _(q, k, v, **kwargs):
54
+ return torch.empty_like(q).contiguous()
55
+
56
+
57
+ class QwenDoubleStreamAttnProcessorFA3:
58
+ """
59
+ FA3-based attention processor for Qwen double-stream architecture.
60
+ Computes joint attention over concatenated [text, image] streams using vLLM FlashAttention-3
61
+ accessed via Hugging Face `kernels`.
62
+
63
+ Notes / limitations:
64
+ - General attention masks are not supported here (FA3 path). `is_causal=False` and no arbitrary mask.
65
+ - Optional windowed attention / sink tokens / softcap can be plumbed through if you use those features.
66
+ - Expects an available `apply_rotary_emb_qwen` in scope (same as your non-FA3 processor).
67
+ """
68
+
69
+ _attention_backend = "fa3" # for parity with your other processors, not used internally
70
+
71
+ def __init__(self):
72
+ if not _fa3_available():
73
+ print(f"[QwenDoubleStreamAttnProcessorFA3] FA3 unavailable, using SDPA fallback. Reason: {_kernels_err}")
74
+
75
+ @torch.no_grad()
76
+ def __call__(
77
+ self,
78
+ attn, # Attention module with to_q/to_k/to_v/add_*_proj, norms, to_out, to_add_out, and .heads
79
+ hidden_states: torch.FloatTensor, # (B, S_img, D_model) image stream
80
+ encoder_hidden_states: torch.FloatTensor = None, # (B, S_txt, D_model) text stream
81
+ encoder_hidden_states_mask: torch.FloatTensor = None, # unused in FA3 path
82
+ attention_mask: Optional[torch.FloatTensor] = None, # unused in FA3 path
83
+ image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # (img_freqs, txt_freqs)
84
+ ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
85
+ if encoder_hidden_states is None:
86
+ raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states (text stream).")
87
+ if attention_mask is not None:
88
+ # FA3 kernel path here does not consume arbitrary masks; fail fast to avoid silent correctness issues.
89
+ raise NotImplementedError("attention_mask is not supported in this FA3 implementation.")
90
+
91
+ B, S_img, _ = hidden_states.shape
92
+ S_txt = encoder_hidden_states.shape[1]
93
+
94
+ # ---- QKV projections (image/sample stream) ----
95
+ img_q = attn.to_q(hidden_states) # (B, S_img, D)
96
+ img_k = attn.to_k(hidden_states)
97
+ img_v = attn.to_v(hidden_states)
98
+
99
+ # ---- QKV projections (text/context stream) ----
100
+ txt_q = attn.add_q_proj(encoder_hidden_states) # (B, S_txt, D)
101
+ txt_k = attn.add_k_proj(encoder_hidden_states)
102
+ txt_v = attn.add_v_proj(encoder_hidden_states)
103
+
104
+ # ---- Reshape to (B, S, H, D_h) ----
105
+ H = attn.heads
106
+ img_q = img_q.unflatten(-1, (H, -1))
107
+ img_k = img_k.unflatten(-1, (H, -1))
108
+ img_v = img_v.unflatten(-1, (H, -1))
109
+
110
+ txt_q = txt_q.unflatten(-1, (H, -1))
111
+ txt_k = txt_k.unflatten(-1, (H, -1))
112
+ txt_v = txt_v.unflatten(-1, (H, -1))
113
+
114
+ # ---- Q/K normalization (per your module contract) ----
115
+ if getattr(attn, "norm_q", None) is not None:
116
+ img_q = attn.norm_q(img_q)
117
+ if getattr(attn, "norm_k", None) is not None:
118
+ img_k = attn.norm_k(img_k)
119
+ if getattr(attn, "norm_added_q", None) is not None:
120
+ txt_q = attn.norm_added_q(txt_q)
121
+ if getattr(attn, "norm_added_k", None) is not None:
122
+ txt_k = attn.norm_added_k(txt_k)
123
+
124
+ # ---- RoPE (Qwen variant) ----
125
+ if image_rotary_emb is not None:
126
+ img_freqs, txt_freqs = image_rotary_emb
127
+ # expects tensors shaped (B, S, H, D_h)
128
+ img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
129
+ img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
130
+ txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
131
+ txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
132
+
133
+ # ---- Joint attention over [text, image] along sequence axis ----
134
+ # Shapes: (B, S_total, H, D_h)
135
+ q = torch.cat([txt_q, img_q], dim=1)
136
+ k = torch.cat([txt_k, img_k], dim=1)
137
+ v = torch.cat([txt_v, img_v], dim=1)
138
+
139
+ if _fa3_available():
140
+ out = flash_attn_func(q, k, v, causal=False) # out: (B, S_total, H, D_h)
141
+ else:
142
+ out = _sdpa_fallback(q, k, v) # out: (B, S_total, H, D_h)
143
+
144
+ # ---- Back to (B, S, D_model) ----
145
+ out = out.flatten(2, 3).to(q.dtype)
146
+
147
+ # Split back to text / image segments
148
+ txt_attn_out = out[:, :S_txt, :]
149
+ img_attn_out = out[:, S_txt:, :]
150
+
151
+ # ---- Output projections ----
152
+ img_attn_out = attn.to_out[0](img_attn_out)
153
+ if len(attn.to_out) > 1:
154
+ img_attn_out = attn.to_out[1](img_attn_out) # dropout if present
155
+
156
+ txt_attn_out = attn.to_add_out(txt_attn_out)
157
+
158
  return img_attn_out, txt_attn_out