File size: 3,743 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# SPDX-License-Identifier: Apache-2.0
"""Qwen2.5-VL vision tower (32 blocks, window attention except blocks 7/15/23/31, 2x2 patch merger), fp32 torch.

Structure mirrors the TT port's VISION program:
  host:   pixel_values (N=1512, 1176) in raster-merge order, static geometry (window permutation, cu_seqlens, cos/sin)
  device: patch_embed (= Linear 1176 -> 1280, the Conv3d with kernel == stride), permute rows to window order,
          32 x [x += attn(rms(x)); x += mlp(rms(x))], merger (rms, 2x2 concat -> 5120, Linear, GELU, Linear -> 3584),
          inverse window permutation -> (378, 3584) image embeddings in raster order.
"""
from __future__ import annotations

from typing import Dict, Optional

import torch
import torch.nn.functional as F

from .config import CONFIG
from .ops import rms_norm, rotate_half

V = CONFIG.vision
PFX = "visual."


class VisionTower:
    def __init__(self, sd: Dict[str, torch.Tensor], dtype: torch.dtype = torch.float32):
        """sd: the 'visual.*' tensors of the RDT2-VQ checkpoint (bf16); upcast to ``dtype`` here."""
        self.dtype = dtype
        self.w = {k[len(PFX):]: v.to(dtype) for k, v in sd.items() if k.startswith(PFX)}
        self.patch_w = self.w["patch_embed.proj.weight"].reshape(V.hidden, V.in_dim)  # Conv3d -> Linear

    def block(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, cu) -> torch.Tensor:
        w = self.w
        p = f"blocks.{i}."
        n = x.shape[0]
        h = rms_norm(x, w[p + "norm1.weight"], V.norm_eps)
        qkv = F.linear(h, w[p + "attn.qkv.weight"], w[p + "attn.qkv.bias"])
        q, k, v = qkv.reshape(n, 3, V.heads, V.head_dim).permute(1, 0, 2, 3).unbind(0)  # (n, H, 80)
        c, s = cos.unsqueeze(-2), sin.unsqueeze(-2)
        q = q * c + rotate_half(q) * s
        k = k * c + rotate_half(k) * s
        out = torch.empty_like(q)
        for a, b in zip(cu[:-1], cu[1:]):  # block-diagonal attention == HF's masked attention over cu_seqlens
            qs, ks, vs = (t[a:b].transpose(0, 1) for t in (q, k, v))  # (H, len, 80)
            out[a:b] = F.scaled_dot_product_attention(qs, ks, vs).transpose(0, 1)
        x = x + F.linear(out.reshape(n, -1), w[p + "attn.proj.weight"], w[p + "attn.proj.bias"])
        h = rms_norm(x, w[p + "norm2.weight"], V.norm_eps)
        g = F.linear(h, w[p + "mlp.gate_proj.weight"], w[p + "mlp.gate_proj.bias"])
        u = F.linear(h, w[p + "mlp.up_proj.weight"], w[p + "mlp.up_proj.bias"])
        return x + F.linear(F.silu(g) * u, w[p + "mlp.down_proj.weight"], w[p + "mlp.down_proj.bias"])

    def merger(self, x: torch.Tensor) -> torch.Tensor:
        w = self.w
        x = rms_norm(x, w["merger.ln_q.weight"], V.norm_eps).reshape(-1, V.hidden * V.merge ** 2)
        x = F.gelu(F.linear(x, w["merger.mlp.0.weight"], w["merger.mlp.0.bias"]))
        return F.linear(x, w["merger.mlp.2.weight"], w["merger.mlp.2.bias"])

    def forward(self, pixel_values: torch.Tensor, geom: Dict[str, object],
                taps: Optional[Dict[str, torch.Tensor]] = None) -> torch.Tensor:
        x = F.linear(pixel_values.to(self.dtype), self.patch_w)  # (N, 1280) raster-merge order
        if taps is not None:
            taps["vis.patch_embed"] = x
        x = x[geom["patch_perm"]]  # window order
        cos, sin = geom["cos"].to(self.dtype), geom["sin"].to(self.dtype)
        for i in range(V.depth):
            cu = geom["cu_full"] if i in V.fullatt_blocks else geom["cu_window"]
            x = self.block(i, x, cos, sin, cu)
            if taps is not None:
                taps[f"vis.block{i:02d}"] = x
        x = self.merger(x)  # (N/4, 3584) window order
        if taps is not None:
            taps["vis.merger_windowed"] = x
        return x[geom["reverse"]]