# SPDX-License-Identifier: Apache-2.0 """Qwen2.5-VL-7B language model prefill of RDT2 (fp32 torch), producing the K/V the RDT expert cross-attends to. RDT2-FM reads ``past_key_values[i]`` for i in selected_layers = 0..13 of one prefill (use_cache=True). The cached K is **post-mRoPE** (HF caches after ``apply_multimodal_rotary_pos_emb``) and includes the k_proj bias; V includes the v_proj bias; layout (1, 4 kv heads, L, 128). Because the LM is causal and the expert only reads layer-i K/V, layers 0..12 run in full and layer 13 only needs input RMSNorm + k/v projections + RoPE on K: layers 13 (q/o/MLP) .. 27, the final norm and the lm_head never influence RDT2-FM (the official code computes them and discards them). ``PrefixLM(num_layers=28, kv_layers=range(28), full_layers=28)`` is the extension point of the later RDT2-VQ profile (all 28 layers + final norm + lm_head for autoregressive action-token decoding); ``logits_last`` covers it. """ from __future__ import annotations from typing import Dict, Iterable, Optional, Sequence import torch import torch.nn.functional as F from .config import CONFIG, IMAGE_TOKEN_ID from .ops import repeat_kv_heads, rms_norm, rotate_half LM = CONFIG.lm class PrefixLM: def __init__(self, index, kv_layers: Sequence[int] = LM.selected_layers, full_layers: Optional[int] = None, dtype: torch.dtype = torch.float32, with_head: bool = False): """index: weights.SafetensorsIndex of RDT2-VQ. Loads embed_tokens + the layers needed (bf16 kept, upcast per use). full_layers: number of complete decoder layers to run (default max(kv_layers), i.e. 13 for RDT2-FM); the layer at index ``full_layers`` (if it is in kv_layers) only gets its K/V computed. """ self.dtype = dtype self.kv_layers = list(kv_layers) self.full_layers = max(self.kv_layers) if full_layers is None else full_layers need = ["model.embed_tokens.weight"] n_load = max(self.full_layers, max(self.kv_layers) + 1) for i in range(n_load): need += [k for k in index.keys() if k.startswith(f"model.layers.{i}.")] if with_head: need += ["model.norm.weight", "lm_head.weight"] self.w = index.load(need) def _w(self, name: str) -> torch.Tensor: return self.w[name].to(self.dtype) def embed(self, input_ids: torch.Tensor, image_embeds: torch.Tensor) -> torch.Tensor: x = self.w["model.embed_tokens.weight"][input_ids[0]].to(self.dtype) # (L, 3584) m = input_ids[0] == IMAGE_TOKEN_ID if int(m.sum()) != image_embeds.shape[0]: raise ValueError("image tokens / features mismatch") x[m] = image_embeds.to(self.dtype) return x[None] def qkv(self, i: int, h: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, need_q: bool = True): p = f"model.layers.{i}.self_attn." L = h.shape[1] k = F.linear(h, self._w(p + "k_proj.weight"), self._w(p + "k_proj.bias")).view(1, L, LM.kv_heads, LM.head_dim) v = F.linear(h, self._w(p + "v_proj.weight"), self._w(p + "v_proj.bias")).view(1, L, LM.kv_heads, LM.head_dim) k = k.transpose(1, 2) v = v.transpose(1, 2) c, s = cos[None, None], sin[None, None] # (1, 1, L, 128) k = k * c + rotate_half(k) * s q = None if need_q: q = F.linear(h, self._w(p + "q_proj.weight"), self._w(p + "q_proj.bias")).view(1, L, LM.heads, LM.head_dim) q = q.transpose(1, 2) q = q * c + rotate_half(q) * s return q, k, v def layer(self, i: int, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): p = f"model.layers.{i}." h = rms_norm(x, self._w(p + "input_layernorm.weight"), LM.norm_eps) q, k, v = self.qkv(i, h, cos, sin) rep = LM.heads // LM.kv_heads a = F.scaled_dot_product_attention(q, repeat_kv_heads(k, rep, 1), repeat_kv_heads(v, rep, 1), is_causal=True) a = a.transpose(1, 2).reshape(1, x.shape[1], LM.heads * LM.head_dim) x = x + F.linear(a, self._w(p + "self_attn.o_proj.weight")) h = rms_norm(x, self._w(p + "post_attention_layernorm.weight"), LM.norm_eps) g = F.linear(h, self._w(p + "mlp.gate_proj.weight")) u = F.linear(h, self._w(p + "mlp.up_proj.weight")) x = x + F.linear(F.silu(g) * u, self._w(p + "mlp.down_proj.weight")) return x, k, v def forward(self, inputs_embeds: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, taps: Optional[Dict[str, torch.Tensor]] = None, logits_last: bool = False): """inputs_embeds (1, L, 3584); cos/sin (L, 128) mRoPE tables. Returns K, V (n_kv, 1, 4, L, 128) [, logits].""" x = inputs_embeds.to(self.dtype) cos, sin = cos.to(self.dtype), sin.to(self.dtype) ks, vs = {}, {} for i in range(self.full_layers): x, k, v = self.layer(i, x, cos, sin) ks[i], vs[i] = k, v if taps is not None: taps[f"llm.layer{i:02d}.out"] = x for i in self.kv_layers: if i not in ks: # K/V-only layer (layer 13 for RDT2-FM) if i != self.full_layers: raise ValueError("K/V-only layers must directly follow the full layers") h = rms_norm(x, self._w(f"model.layers.{i}.input_layernorm.weight"), LM.norm_eps) _, ks[i], vs[i] = self.qkv(i, h, cos, sin, need_q=False) K = torch.stack([ks[i] for i in self.kv_layers], 0) Vv = torch.stack([vs[i] for i in self.kv_layers], 0) if not logits_last: return K, Vv h = rms_norm(x[:, -1:], self._w("model.norm.weight"), LM.norm_eps) return K, Vv, F.linear(h, self._w("lm_head.weight"))[:, 0]