Download code/tt_rdt2/reference/vision.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 3.74 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/vision.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/vision.py
-
curl -L -o vision.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/vision.py
3.74 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Qwen2.5-VL vision tower (32 blocks, window attention except blocks 7/15/23/31, 2x2 patch merger), fp32 torch. | |
| Structure mirrors the TT port's VISION program: | |
| host: pixel_values (N=1512, 1176) in raster-merge order, static geometry (window permutation, cu_seqlens, cos/sin) | |
| device: patch_embed (= Linear 1176 -> 1280, the Conv3d with kernel == stride), permute rows to window order, | |
| 32 x [x += attn(rms(x)); x += mlp(rms(x))], merger (rms, 2x2 concat -> 5120, Linear, GELU, Linear -> 3584), | |
| inverse window permutation -> (378, 3584) image embeddings in raster order. | |
| """ | |
| from __future__ import annotations | |
| from typing import Dict, Optional | |
| import torch | |
| import torch.nn.functional as F | |
| from .config import CONFIG | |
| from .ops import rms_norm, rotate_half | |
| V = CONFIG.vision | |
| PFX = "visual." | |
| class VisionTower: | |
| def __init__(self, sd: Dict[str, torch.Tensor], dtype: torch.dtype = torch.float32): | |
| """sd: the 'visual.*' tensors of the RDT2-VQ checkpoint (bf16); upcast to ``dtype`` here.""" | |
| self.dtype = dtype | |
| self.w = {k[len(PFX):]: v.to(dtype) for k, v in sd.items() if k.startswith(PFX)} | |
| self.patch_w = self.w["patch_embed.proj.weight"].reshape(V.hidden, V.in_dim) # Conv3d -> Linear | |
| def block(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, cu) -> torch.Tensor: | |
| w = self.w | |
| p = f"blocks.{i}." | |
| n = x.shape[0] | |
| h = rms_norm(x, w[p + "norm1.weight"], V.norm_eps) | |
| qkv = F.linear(h, w[p + "attn.qkv.weight"], w[p + "attn.qkv.bias"]) | |
| q, k, v = qkv.reshape(n, 3, V.heads, V.head_dim).permute(1, 0, 2, 3).unbind(0) # (n, H, 80) | |
| c, s = cos.unsqueeze(-2), sin.unsqueeze(-2) | |
| q = q * c + rotate_half(q) * s | |
| k = k * c + rotate_half(k) * s | |
| out = torch.empty_like(q) | |
| for a, b in zip(cu[:-1], cu[1:]): # block-diagonal attention == HF's masked attention over cu_seqlens | |
| qs, ks, vs = (t[a:b].transpose(0, 1) for t in (q, k, v)) # (H, len, 80) | |
| out[a:b] = F.scaled_dot_product_attention(qs, ks, vs).transpose(0, 1) | |
| x = x + F.linear(out.reshape(n, -1), w[p + "attn.proj.weight"], w[p + "attn.proj.bias"]) | |
| h = rms_norm(x, w[p + "norm2.weight"], V.norm_eps) | |
| g = F.linear(h, w[p + "mlp.gate_proj.weight"], w[p + "mlp.gate_proj.bias"]) | |
| u = F.linear(h, w[p + "mlp.up_proj.weight"], w[p + "mlp.up_proj.bias"]) | |
| return x + F.linear(F.silu(g) * u, w[p + "mlp.down_proj.weight"], w[p + "mlp.down_proj.bias"]) | |
| def merger(self, x: torch.Tensor) -> torch.Tensor: | |
| w = self.w | |
| x = rms_norm(x, w["merger.ln_q.weight"], V.norm_eps).reshape(-1, V.hidden * V.merge ** 2) | |
| x = F.gelu(F.linear(x, w["merger.mlp.0.weight"], w["merger.mlp.0.bias"])) | |
| return F.linear(x, w["merger.mlp.2.weight"], w["merger.mlp.2.bias"]) | |
| def forward(self, pixel_values: torch.Tensor, geom: Dict[str, object], | |
| taps: Optional[Dict[str, torch.Tensor]] = None) -> torch.Tensor: | |
| x = F.linear(pixel_values.to(self.dtype), self.patch_w) # (N, 1280) raster-merge order | |
| if taps is not None: | |
| taps["vis.patch_embed"] = x | |
| x = x[geom["patch_perm"]] # window order | |
| cos, sin = geom["cos"].to(self.dtype), geom["sin"].to(self.dtype) | |
| for i in range(V.depth): | |
| cu = geom["cu_full"] if i in V.fullatt_blocks else geom["cu_window"] | |
| x = self.block(i, x, cos, sin, cu) | |
| if taps is not None: | |
| taps[f"vis.block{i:02d}"] = x | |
| x = self.merger(x) # (N/4, 3584) window order | |
| if taps is not None: | |
| taps["vis.merger_windowed"] = x | |
| return x[geom["reverse"]] | |