Download code/tt_rdt2/reference/vlm.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 5.77 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/vlm.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/vlm.py
-
curl -L -o vlm.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/vlm.py
5.77 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Qwen2.5-VL-7B language model prefill of RDT2 (fp32 torch), producing the K/V the RDT expert cross-attends to. | |
| RDT2-FM reads ``past_key_values[i]`` for i in selected_layers = 0..13 of one prefill (use_cache=True). The cached K is | |
| **post-mRoPE** (HF caches after ``apply_multimodal_rotary_pos_emb``) and includes the k_proj bias; V includes the | |
| v_proj bias; layout (1, 4 kv heads, L, 128). Because the LM is causal and the expert only reads layer-i K/V, layers | |
| 0..12 run in full and layer 13 only needs input RMSNorm + k/v projections + RoPE on K: layers 13 (q/o/MLP) .. 27, the | |
| final norm and the lm_head never influence RDT2-FM (the official code computes them and discards them). | |
| ``PrefixLM(num_layers=28, kv_layers=range(28), full_layers=28)`` is the extension point of the later RDT2-VQ profile | |
| (all 28 layers + final norm + lm_head for autoregressive action-token decoding); ``logits_last`` covers it. | |
| """ | |
| from __future__ import annotations | |
| from typing import Dict, Iterable, Optional, Sequence | |
| import torch | |
| import torch.nn.functional as F | |
| from .config import CONFIG, IMAGE_TOKEN_ID | |
| from .ops import repeat_kv_heads, rms_norm, rotate_half | |
| LM = CONFIG.lm | |
| class PrefixLM: | |
| def __init__(self, index, kv_layers: Sequence[int] = LM.selected_layers, full_layers: Optional[int] = None, | |
| dtype: torch.dtype = torch.float32, with_head: bool = False): | |
| """index: weights.SafetensorsIndex of RDT2-VQ. Loads embed_tokens + the layers needed (bf16 kept, upcast per use). | |
| full_layers: number of complete decoder layers to run (default max(kv_layers), i.e. 13 for RDT2-FM); | |
| the layer at index ``full_layers`` (if it is in kv_layers) only gets its K/V computed. | |
| """ | |
| self.dtype = dtype | |
| self.kv_layers = list(kv_layers) | |
| self.full_layers = max(self.kv_layers) if full_layers is None else full_layers | |
| need = ["model.embed_tokens.weight"] | |
| n_load = max(self.full_layers, max(self.kv_layers) + 1) | |
| for i in range(n_load): | |
| need += [k for k in index.keys() if k.startswith(f"model.layers.{i}.")] | |
| if with_head: | |
| need += ["model.norm.weight", "lm_head.weight"] | |
| self.w = index.load(need) | |
| def _w(self, name: str) -> torch.Tensor: | |
| return self.w[name].to(self.dtype) | |
| def embed(self, input_ids: torch.Tensor, image_embeds: torch.Tensor) -> torch.Tensor: | |
| x = self.w["model.embed_tokens.weight"][input_ids[0]].to(self.dtype) # (L, 3584) | |
| m = input_ids[0] == IMAGE_TOKEN_ID | |
| if int(m.sum()) != image_embeds.shape[0]: | |
| raise ValueError("image tokens / features mismatch") | |
| x[m] = image_embeds.to(self.dtype) | |
| return x[None] | |
| def qkv(self, i: int, h: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, need_q: bool = True): | |
| p = f"model.layers.{i}.self_attn." | |
| L = h.shape[1] | |
| k = F.linear(h, self._w(p + "k_proj.weight"), self._w(p + "k_proj.bias")).view(1, L, LM.kv_heads, LM.head_dim) | |
| v = F.linear(h, self._w(p + "v_proj.weight"), self._w(p + "v_proj.bias")).view(1, L, LM.kv_heads, LM.head_dim) | |
| k = k.transpose(1, 2) | |
| v = v.transpose(1, 2) | |
| c, s = cos[None, None], sin[None, None] # (1, 1, L, 128) | |
| k = k * c + rotate_half(k) * s | |
| q = None | |
| if need_q: | |
| q = F.linear(h, self._w(p + "q_proj.weight"), self._w(p + "q_proj.bias")).view(1, L, LM.heads, LM.head_dim) | |
| q = q.transpose(1, 2) | |
| q = q * c + rotate_half(q) * s | |
| return q, k, v | |
| def layer(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): | |
| p = f"model.layers.{i}." | |
| h = rms_norm(x, self._w(p + "input_layernorm.weight"), LM.norm_eps) | |
| q, k, v = self.qkv(i, h, cos, sin) | |
| rep = LM.heads // LM.kv_heads | |
| a = F.scaled_dot_product_attention(q, repeat_kv_heads(k, rep, 1), repeat_kv_heads(v, rep, 1), is_causal=True) | |
| a = a.transpose(1, 2).reshape(1, x.shape[1], LM.heads * LM.head_dim) | |
| x = x + F.linear(a, self._w(p + "self_attn.o_proj.weight")) | |
| h = rms_norm(x, self._w(p + "post_attention_layernorm.weight"), LM.norm_eps) | |
| g = F.linear(h, self._w(p + "mlp.gate_proj.weight")) | |
| u = F.linear(h, self._w(p + "mlp.up_proj.weight")) | |
| x = x + F.linear(F.silu(g) * u, self._w(p + "mlp.down_proj.weight")) | |
| return x, k, v | |
| def forward(self, inputs_embeds: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, | |
| taps: Optional[Dict[str, torch.Tensor]] = None, logits_last: bool = False): | |
| """inputs_embeds (1, L, 3584); cos/sin (L, 128) mRoPE tables. Returns K, V (n_kv, 1, 4, L, 128) [, logits].""" | |
| x = inputs_embeds.to(self.dtype) | |
| cos, sin = cos.to(self.dtype), sin.to(self.dtype) | |
| ks, vs = {}, {} | |
| for i in range(self.full_layers): | |
| x, k, v = self.layer(i, x, cos, sin) | |
| ks[i], vs[i] = k, v | |
| if taps is not None: | |
| taps[f"llm.layer{i:02d}.out"] = x | |
| for i in self.kv_layers: | |
| if i not in ks: # K/V-only layer (layer 13 for RDT2-FM) | |
| if i != self.full_layers: | |
| raise ValueError("K/V-only layers must directly follow the full layers") | |
| h = rms_norm(x, self._w(f"model.layers.{i}.input_layernorm.weight"), LM.norm_eps) | |
| _, ks[i], vs[i] = self.qkv(i, h, cos, sin, need_q=False) | |
| K = torch.stack([ks[i] for i in self.kv_layers], 0) | |
| Vv = torch.stack([vs[i] for i in self.kv_layers], 0) | |
| if not logits_last: | |
| return K, Vv | |
| h = rms_norm(x[:, -1:], self._w("model.norm.weight"), LM.norm_eps) | |
| return K, Vv, F.linear(h, self._w("lm_head.weight"))[:, 0] | |