changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
5.77 kB
# SPDX-License-Identifier: Apache-2.0
"""Qwen2.5-VL-7B language model prefill of RDT2 (fp32 torch), producing the K/V the RDT expert cross-attends to.
RDT2-FM reads ``past_key_values[i]`` for i in selected_layers = 0..13 of one prefill (use_cache=True). The cached K is
**post-mRoPE** (HF caches after ``apply_multimodal_rotary_pos_emb``) and includes the k_proj bias; V includes the
v_proj bias; layout (1, 4 kv heads, L, 128). Because the LM is causal and the expert only reads layer-i K/V, layers
0..12 run in full and layer 13 only needs input RMSNorm + k/v projections + RoPE on K: layers 13 (q/o/MLP) .. 27, the
final norm and the lm_head never influence RDT2-FM (the official code computes them and discards them).
``PrefixLM(num_layers=28, kv_layers=range(28), full_layers=28)`` is the extension point of the later RDT2-VQ profile
(all 28 layers + final norm + lm_head for autoregressive action-token decoding); ``logits_last`` covers it.
"""
from __future__ import annotations
from typing import Dict, Iterable, Optional, Sequence
import torch
import torch.nn.functional as F
from .config import CONFIG, IMAGE_TOKEN_ID
from .ops import repeat_kv_heads, rms_norm, rotate_half
LM = CONFIG.lm
class PrefixLM:
def __init__(self, index, kv_layers: Sequence[int] = LM.selected_layers, full_layers: Optional[int] = None,
dtype: torch.dtype = torch.float32, with_head: bool = False):
"""index: weights.SafetensorsIndex of RDT2-VQ. Loads embed_tokens + the layers needed (bf16 kept, upcast per use).
full_layers: number of complete decoder layers to run (default max(kv_layers), i.e. 13 for RDT2-FM);
the layer at index ``full_layers`` (if it is in kv_layers) only gets its K/V computed.
"""
self.dtype = dtype
self.kv_layers = list(kv_layers)
self.full_layers = max(self.kv_layers) if full_layers is None else full_layers
need = ["model.embed_tokens.weight"]
n_load = max(self.full_layers, max(self.kv_layers) + 1)
for i in range(n_load):
need += [k for k in index.keys() if k.startswith(f"model.layers.{i}.")]
if with_head:
need += ["model.norm.weight", "lm_head.weight"]
self.w = index.load(need)
def _w(self, name: str) -> torch.Tensor:
return self.w[name].to(self.dtype)
def embed(self, input_ids: torch.Tensor, image_embeds: torch.Tensor) -> torch.Tensor:
x = self.w["model.embed_tokens.weight"][input_ids[0]].to(self.dtype) # (L, 3584)
m = input_ids[0] == IMAGE_TOKEN_ID
if int(m.sum()) != image_embeds.shape[0]:
raise ValueError("image tokens / features mismatch")
x[m] = image_embeds.to(self.dtype)
return x[None]
def qkv(self, i: int, h: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, need_q: bool = True):
p = f"model.layers.{i}.self_attn."
L = h.shape[1]
k = F.linear(h, self._w(p + "k_proj.weight"), self._w(p + "k_proj.bias")).view(1, L, LM.kv_heads, LM.head_dim)
v = F.linear(h, self._w(p + "v_proj.weight"), self._w(p + "v_proj.bias")).view(1, L, LM.kv_heads, LM.head_dim)
k = k.transpose(1, 2)
v = v.transpose(1, 2)
c, s = cos[None, None], sin[None, None] # (1, 1, L, 128)
k = k * c + rotate_half(k) * s
q = None
if need_q:
q = F.linear(h, self._w(p + "q_proj.weight"), self._w(p + "q_proj.bias")).view(1, L, LM.heads, LM.head_dim)
q = q.transpose(1, 2)
q = q * c + rotate_half(q) * s
return q, k, v
def layer(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor):
p = f"model.layers.{i}."
h = rms_norm(x, self._w(p + "input_layernorm.weight"), LM.norm_eps)
q, k, v = self.qkv(i, h, cos, sin)
rep = LM.heads // LM.kv_heads
a = F.scaled_dot_product_attention(q, repeat_kv_heads(k, rep, 1), repeat_kv_heads(v, rep, 1), is_causal=True)
a = a.transpose(1, 2).reshape(1, x.shape[1], LM.heads * LM.head_dim)
x = x + F.linear(a, self._w(p + "self_attn.o_proj.weight"))
h = rms_norm(x, self._w(p + "post_attention_layernorm.weight"), LM.norm_eps)
g = F.linear(h, self._w(p + "mlp.gate_proj.weight"))
u = F.linear(h, self._w(p + "mlp.up_proj.weight"))
x = x + F.linear(F.silu(g) * u, self._w(p + "mlp.down_proj.weight"))
return x, k, v
def forward(self, inputs_embeds: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor,
taps: Optional[Dict[str, torch.Tensor]] = None, logits_last: bool = False):
"""inputs_embeds (1, L, 3584); cos/sin (L, 128) mRoPE tables. Returns K, V (n_kv, 1, 4, L, 128) [, logits]."""
x = inputs_embeds.to(self.dtype)
cos, sin = cos.to(self.dtype), sin.to(self.dtype)
ks, vs = {}, {}
for i in range(self.full_layers):
x, k, v = self.layer(i, x, cos, sin)
ks[i], vs[i] = k, v
if taps is not None:
taps[f"llm.layer{i:02d}.out"] = x
for i in self.kv_layers:
if i not in ks: # K/V-only layer (layer 13 for RDT2-FM)
if i != self.full_layers:
raise ValueError("K/V-only layers must directly follow the full layers")
h = rms_norm(x, self._w(f"model.layers.{i}.input_layernorm.weight"), LM.norm_eps)
_, ks[i], vs[i] = self.qkv(i, h, cos, sin, need_q=False)
K = torch.stack([ks[i] for i in self.kv_layers], 0)
Vv = torch.stack([vs[i] for i in self.kv_layers], 0)
if not logits_last:
return K, Vv
h = rms_norm(x[:, -1:], self._w("model.norm.weight"), LM.norm_eps)
return K, Vv, F.linear(h, self._w("lm_head.weight"))[:, 0]