# SPDX-License-Identifier: Apache-2.0 """Qwen2.5-VL vision tower (32 blocks, window attention except blocks 7/15/23/31, 2x2 patch merger), fp32 torch. Structure mirrors the TT port's VISION program: host: pixel_values (N=1512, 1176) in raster-merge order, static geometry (window permutation, cu_seqlens, cos/sin) device: patch_embed (= Linear 1176 -> 1280, the Conv3d with kernel == stride), permute rows to window order, 32 x [x += attn(rms(x)); x += mlp(rms(x))], merger (rms, 2x2 concat -> 5120, Linear, GELU, Linear -> 3584), inverse window permutation -> (378, 3584) image embeddings in raster order. """ from __future__ import annotations from typing import Dict, Optional import torch import torch.nn.functional as F from .config import CONFIG from .ops import rms_norm, rotate_half V = CONFIG.vision PFX = "visual." class VisionTower: def __init__(self, sd: Dict[str, torch.Tensor], dtype: torch.dtype = torch.float32): """sd: the 'visual.*' tensors of the RDT2-VQ checkpoint (bf16); upcast to ``dtype`` here.""" self.dtype = dtype self.w = {k[len(PFX):]: v.to(dtype) for k, v in sd.items() if k.startswith(PFX)} self.patch_w = self.w["patch_embed.proj.weight"].reshape(V.hidden, V.in_dim) # Conv3d -> Linear def block(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, cu) -> torch.Tensor: w = self.w p = f"blocks.{i}." n = x.shape[0] h = rms_norm(x, w[p + "norm1.weight"], V.norm_eps) qkv = F.linear(h, w[p + "attn.qkv.weight"], w[p + "attn.qkv.bias"]) q, k, v = qkv.reshape(n, 3, V.heads, V.head_dim).permute(1, 0, 2, 3).unbind(0) # (n, H, 80) c, s = cos.unsqueeze(-2), sin.unsqueeze(-2) q = q * c + rotate_half(q) * s k = k * c + rotate_half(k) * s out = torch.empty_like(q) for a, b in zip(cu[:-1], cu[1:]): # block-diagonal attention == HF's masked attention over cu_seqlens qs, ks, vs = (t[a:b].transpose(0, 1) for t in (q, k, v)) # (H, len, 80) out[a:b] = F.scaled_dot_product_attention(qs, ks, vs).transpose(0, 1) x = x + F.linear(out.reshape(n, -1), w[p + "attn.proj.weight"], w[p + "attn.proj.bias"]) h = rms_norm(x, w[p + "norm2.weight"], V.norm_eps) g = F.linear(h, w[p + "mlp.gate_proj.weight"], w[p + "mlp.gate_proj.bias"]) u = F.linear(h, w[p + "mlp.up_proj.weight"], w[p + "mlp.up_proj.bias"]) return x + F.linear(F.silu(g) * u, w[p + "mlp.down_proj.weight"], w[p + "mlp.down_proj.bias"]) def merger(self, x: torch.Tensor) -> torch.Tensor: w = self.w x = rms_norm(x, w["merger.ln_q.weight"], V.norm_eps).reshape(-1, V.hidden * V.merge ** 2) x = F.gelu(F.linear(x, w["merger.mlp.0.weight"], w["merger.mlp.0.bias"])) return F.linear(x, w["merger.mlp.2.weight"], w["merger.mlp.2.bias"]) def forward(self, pixel_values: torch.Tensor, geom: Dict[str, object], taps: Optional[Dict[str, torch.Tensor]] = None) -> torch.Tensor: x = F.linear(pixel_values.to(self.dtype), self.patch_w) # (N, 1280) raster-merge order if taps is not None: taps["vis.patch_embed"] = x x = x[geom["patch_perm"]] # window order cos, sin = geom["cos"].to(self.dtype), geom["sin"].to(self.dtype) for i in range(V.depth): cu = geom["cu_full"] if i in V.fullatt_blocks else geom["cu_window"] x = self.block(i, x, cos, sin, cu) if taps is not None: taps[f"vis.block{i:02d}"] = x x = self.merger(x) # (N/4, 3584) window order if taps is not None: taps["vis.merger_windowed"] = x return x[geom["reverse"]]