File size: 3,743 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | # SPDX-License-Identifier: Apache-2.0
"""Qwen2.5-VL vision tower (32 blocks, window attention except blocks 7/15/23/31, 2x2 patch merger), fp32 torch.
Structure mirrors the TT port's VISION program:
host: pixel_values (N=1512, 1176) in raster-merge order, static geometry (window permutation, cu_seqlens, cos/sin)
device: patch_embed (= Linear 1176 -> 1280, the Conv3d with kernel == stride), permute rows to window order,
32 x [x += attn(rms(x)); x += mlp(rms(x))], merger (rms, 2x2 concat -> 5120, Linear, GELU, Linear -> 3584),
inverse window permutation -> (378, 3584) image embeddings in raster order.
"""
from __future__ import annotations
from typing import Dict, Optional
import torch
import torch.nn.functional as F
from .config import CONFIG
from .ops import rms_norm, rotate_half
V = CONFIG.vision
PFX = "visual."
class VisionTower:
def __init__(self, sd: Dict[str, torch.Tensor], dtype: torch.dtype = torch.float32):
"""sd: the 'visual.*' tensors of the RDT2-VQ checkpoint (bf16); upcast to ``dtype`` here."""
self.dtype = dtype
self.w = {k[len(PFX):]: v.to(dtype) for k, v in sd.items() if k.startswith(PFX)}
self.patch_w = self.w["patch_embed.proj.weight"].reshape(V.hidden, V.in_dim) # Conv3d -> Linear
def block(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, cu) -> torch.Tensor:
w = self.w
p = f"blocks.{i}."
n = x.shape[0]
h = rms_norm(x, w[p + "norm1.weight"], V.norm_eps)
qkv = F.linear(h, w[p + "attn.qkv.weight"], w[p + "attn.qkv.bias"])
q, k, v = qkv.reshape(n, 3, V.heads, V.head_dim).permute(1, 0, 2, 3).unbind(0) # (n, H, 80)
c, s = cos.unsqueeze(-2), sin.unsqueeze(-2)
q = q * c + rotate_half(q) * s
k = k * c + rotate_half(k) * s
out = torch.empty_like(q)
for a, b in zip(cu[:-1], cu[1:]): # block-diagonal attention == HF's masked attention over cu_seqlens
qs, ks, vs = (t[a:b].transpose(0, 1) for t in (q, k, v)) # (H, len, 80)
out[a:b] = F.scaled_dot_product_attention(qs, ks, vs).transpose(0, 1)
x = x + F.linear(out.reshape(n, -1), w[p + "attn.proj.weight"], w[p + "attn.proj.bias"])
h = rms_norm(x, w[p + "norm2.weight"], V.norm_eps)
g = F.linear(h, w[p + "mlp.gate_proj.weight"], w[p + "mlp.gate_proj.bias"])
u = F.linear(h, w[p + "mlp.up_proj.weight"], w[p + "mlp.up_proj.bias"])
return x + F.linear(F.silu(g) * u, w[p + "mlp.down_proj.weight"], w[p + "mlp.down_proj.bias"])
def merger(self, x: torch.Tensor) -> torch.Tensor:
w = self.w
x = rms_norm(x, w["merger.ln_q.weight"], V.norm_eps).reshape(-1, V.hidden * V.merge ** 2)
x = F.gelu(F.linear(x, w["merger.mlp.0.weight"], w["merger.mlp.0.bias"]))
return F.linear(x, w["merger.mlp.2.weight"], w["merger.mlp.2.bias"])
def forward(self, pixel_values: torch.Tensor, geom: Dict[str, object],
taps: Optional[Dict[str, torch.Tensor]] = None) -> torch.Tensor:
x = F.linear(pixel_values.to(self.dtype), self.patch_w) # (N, 1280) raster-merge order
if taps is not None:
taps["vis.patch_embed"] = x
x = x[geom["patch_perm"]] # window order
cos, sin = geom["cos"].to(self.dtype), geom["sin"].to(self.dtype)
for i in range(V.depth):
cu = geom["cu_full"] if i in V.fullatt_blocks else geom["cu_window"]
x = self.block(i, x, cos, sin, cu)
if taps is not None:
taps[f"vis.block{i:02d}"] = x
x = self.merger(x) # (N/4, 3584) window order
if taps is not None:
taps["vis.merger_windowed"] = x
return x[geom["reverse"]]
|