# SPDX-License-Identifier: Apache-2.0 """RDT2-VQ host reference (torch, no transformers, no official code): what the official README path (``thu-ml/RDT2 utils.batch_predict_action``) computes around the Qwen2.5-VL-7B forward. cameras [left, right] --concat--> (384, 768, 3) --JPEG q95 round trip (README / deploy: apply_jpeg_compression=True) --> slow Qwen2VLImageProcessor (PIL bicubic resize to 392 x 756, x * (1/255) in fp64 -> fp32, (x - mean) / std fp32) --> pixel_values; prompt = the FM prompt (same chat template + "<|im_start|>assistant\\n<|quad_start|>") --> Qwen2.5-VL-7B generate, max_new_tokens 29: greedy on the repetition-penalized logits (penalty 1.05 over every token of the prompt and of the output so far; the checkpoint's do_sample=True at temperature 1e-6 is greedy up to exact ties), eos 151645 / 151643 --> action ids = the 27 generated ids before the first <|quad_end|> (only if it is at index 27, else 27 zeros) --> action tokens = clamp(151643 - (id + 1), 0, 1023) --> MultiVQVAE.decode (fp32) --> (24, 20) normalized --> UMI unnormalize (x - offset) / scale. :class:`RVQDecoder` re-implements ``MultiVQVAE.decode`` from the pinned ``model.safetensors`` (decoder + codebook + out_proj tensors only; the encoders are never read). """ from __future__ import annotations import io import json import os from functools import lru_cache from typing import Dict, Iterable, List, Optional, Sequence, Tuple import numpy as np import torch import torch.nn.functional as F from .config import CONFIG, TOKENS RVQ_REPO = "robotics-diffusion-transformer/RVQActionTokenizer" RVQ_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec" RVQ_FILES = ("config.json", "model.safetensors") QUAD_START_ID = TOKENS["<|quad_start|>"] # 151650 (last prompt token) QUAD_END_ID = TOKENS["<|quad_end|>"] # 151651 (ends the action ids) EOS_IDS = (TOKENS["<|im_end|>"], TOKENS["<|endoftext|>"]) # generation_config.json eos_token_id TOKENIZER_VOCAB = 151643 # processor.tokenizer.vocab_size (base BPE vocab, no added tokens) NUM_EMBEDDINGS = 1024 # RVQ codebook size (config.json num_embeddings) VALID_ACTION_LEN = 27 # pos 6 x 3 + rot 2 x 3 + grip 1 x 3 (vae.*_id_len) MAX_NEW_TOKENS = VALID_ACTION_LEN + 2 # batch_predict_action: max_new_tokens=valid_action_id_length + 2 REPETITION_PENALTY = 1.05 # RDT2-VQ generation_config.json JPEG_QUALITY = 95 # cv2.imencode('.jpg') default (IMWRITE_JPEG_QUALITY) # tokens the action needs: 27 action ids + the <|quad_end|> that validates them (the 29th token, normally <|im_end|>, # never changes the action): 1 from the prefill + 27 decode steps NEEDED_TOKENS = VALID_ACTION_LEN + 1 # --------------------------------------------------------------------------------------------------------------- # pre-processing (README path: JPEG round trip + the slow image processor) # --------------------------------------------------------------------------------------------------------------- def jpeg_roundtrip(image: np.ndarray, quality: int = JPEG_QUALITY) -> np.ndarray: """(H, W, 3) uint8 RGB -> the RGB image PIL decodes from its baseline JPEG (quality 95, 4:2:0, libjpeg-turbo). The official ``cv2.imencode('.jpg', rgb[..., ::-1])`` + ``PIL.Image.open`` gives the same pixels (both encoders are libjpeg-turbo with the IJG tables scaled to quality 95, 4:2:0 subsampling, no optimization; host test ``test_vq_jpeg_matches_golden`` compares to the official run's decoded image).""" from PIL import Image buf = io.BytesIO() Image.fromarray(np.ascontiguousarray(image)).save(buf, format="JPEG", quality=int(quality)) buf.seek(0) return np.asarray(Image.open(buf).convert("RGB"), dtype=np.uint8).copy() def slow_normalized(image: np.ndarray, size_hw: Tuple[int, int]) -> np.ndarray: """The slow Qwen2VLImageProcessor on one RGB uint8 image: PIL BICUBIC resize to ``size_hw`` (uint8), then ``rescale`` (x.astype(float64) * (1/255) -> float32) and ``normalize`` ((x - mean) / std in float32). Returns (H, W, 3) float32.""" from PIL import Image P = CONFIG.proc h, w = size_hw im = Image.fromarray(np.ascontiguousarray(image)) if (im.height, im.width) != (h, w): im = im.resize((w, h), resample=Image.BICUBIC, reducing_gap=None) x = np.asarray(im, dtype=np.uint8) x = (x.astype(np.float64) * P.rescale_factor).astype(np.float32) mean = np.array(P.image_mean, dtype=np.float32) std = np.array(P.image_std, dtype=np.float32) return (x - mean) / std def vq_pixel_values(images: Sequence[np.ndarray], jpeg: bool = True) -> Tuple[torch.Tensor, np.ndarray]: """cameras (left, right) -> (pixel_values (N, 1176) fp32 in the processor's raster-merge order, the image fed to the processor (H, W, 3) uint8).""" from .preprocess import concat_cameras, smart_resize V = CONFIG.vision img = concat_cameras(images) if jpeg: img = jpeg_roundtrip(img) rh, rw = smart_resize(img.shape[0], img.shape[1], factor=V.patch * V.merge) x = torch.from_numpy(slow_normalized(img, (rh, rw))).permute(2, 0, 1)[None] # (1, 3, H, W) x = x.repeat(V.temporal_patch, 1, 1, 1) gh, gw = rh // V.patch, rw // V.patch x = x.view(1, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch) x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8) return x.reshape(gh * gw, 3 * V.temporal_patch * V.patch * V.patch).contiguous(), img # --------------------------------------------------------------------------------------------------------------- # generation semantics # --------------------------------------------------------------------------------------------------------------- def penalize(logits: torch.Tensor, seen: Iterable[int], penalty: float = REPETITION_PENALTY) -> torch.Tensor: """HF RepetitionPenaltyLogitsProcessor on one row: every token id in ``seen`` gets logit / p (> 0) or * p (< 0).""" out = logits.detach().float().clone() idx = torch.tensor(sorted(set(int(t) for t in seen)), dtype=torch.long) if idx.numel(): s = out[idx] out[idx] = torch.where(s < 0, s * penalty, s / penalty) return out def penalized_argmax(logits: torch.Tensor, seen: Iterable[int], penalty: float = REPETITION_PENALTY) -> int: return int(torch.argmax(penalize(logits, seen, penalty))) def penalized_argmax_topk(values: Sequence[float], indices: Sequence[int], seen: set, penalty: float = REPETITION_PENALTY) -> Optional[int]: """Exact penalized argmax from the top-k raw logits (descending): the penalty only lowers logits, so the best penalized candidate is the answer whenever it is >= the k-th raw value (no token outside the top-k can beat it). Returns None when the top-k does not decide (the caller then reads the full row).""" best, best_v = None, None for v, i in zip(values, indices): v = float(v) if int(i) in seen: v = v * penalty if v < 0 else v / penalty if best_v is None or v > best_v: best, best_v = int(i), v if best_v is None or best_v < float(values[-1]): return None return best def extract_action_ids(generated: Sequence[int]) -> Tuple[torch.Tensor, bool]: """batch_predict_action's extraction for one example: the ids before the first <|quad_end|> if exactly 27 of them, else 27 zeros (valid False).""" g = list(int(t) for t in generated) if QUAD_END_ID in g and g.index(QUAD_END_ID) == VALID_ACTION_LEN: return torch.tensor(g[:VALID_ACTION_LEN], dtype=torch.long), True return torch.zeros(VALID_ACTION_LEN, dtype=torch.long), False def action_tokens_from_ids(action_ids: torch.Tensor) -> torch.Tensor: """vocab_size - (id + 1), clamped to the codebook [0, 1023].""" return torch.clamp(TOKENIZER_VOCAB - (action_ids.long() + 1), min=0, max=NUM_EMBEDDINGS - 1) def is_finished(generated: Sequence[int]) -> bool: """The action is decided once <|quad_end|> or an eos is generated, or once 28 tokens exist (the 29th never changes the action ids).""" g = list(generated) return len(g) >= NEEDED_TOKENS or any(t in EOS_IDS or t == QUAD_END_ID for t in g) # --------------------------------------------------------------------------------------------------------------- # RVQ action-token decoder (MultiVQVAE.decode, fp32) # --------------------------------------------------------------------------------------------------------------- GROUPS = (("pos", 6), ("rot", 2), ("grip", 1)) # (name, n_codebooks); 3 latent time steps each def load_rvq_state(root: str) -> Tuple[Dict[str, torch.Tensor], Dict]: """Decoder + quantizer tensors of the pinned RVQActionTokenizer ``model.safetensors`` (fp32), and config.json.""" from safetensors import safe_open with open(os.path.join(root, "config.json")) as f: cfg = json.load(f) sd = {} with safe_open(os.path.join(root, "model.safetensors"), framework="pt", device="cpu") as f: for k in f.keys(): if ".decoder." in k or (".vq.quantizers." in k and (".codebook." in k or ".out_proj." in k)): sd[k] = f.get_tensor(k).float() return sd, cfg class RVQDecoder: """``MultiVQVAE.decode`` (thu-ml/RDT2 vqvae/models) in torch fp32: per group (pos 6 dims / rot 12 / grip 2), z = sum_i out_proj_i(codebook_i[id_i]) over its residual codebooks for 3 latent steps, then the CNN decoder (GroupNorm(1) + SiLU + Conv1d k3, 3 up stages with ConvTranspose1d(k3, s2, p1, op1): 3 -> 6 -> 12 -> 24 steps).""" def __init__(self, root: str): self.sd, self.cfg = load_rvq_state(root) assert self.cfg["num_embeddings"] == NUM_EMBEDDINGS and self.cfg["action_horizon"] == 24 assert tuple(self.cfg["n_codebooks"][g] for g, _ in GROUPS) == tuple(n for _, n in GROUPS) def _gn(self, x, p): return F.group_norm(x, 1, self.sd[p + ".weight"], self.sd[p + ".bias"], eps=1e-6) def _conv(self, x, p, pad=1): return F.conv1d(x, self.sd[p + ".weight"], self.sd[p + ".bias"], padding=pad) def _block(self, x, p): h = self._conv(F.silu(self._gn(x, p + ".norm1")), p + ".conv1") h = self._conv(F.silu(self._gn(h, p + ".norm2")), p + ".conv2") sc = x if (p + ".nin_shortcut.weight") not in self.sd else self._conv(x, p + ".nin_shortcut", pad=0) return sc + h def _up(self, x, p): return F.conv_transpose1d(x, self.sd[p + ".conv_transpose.weight"], self.sd[p + ".conv_transpose.bias"], stride=2, padding=1, output_padding=1) def latent(self, group: str, ids: torch.Tensor) -> torch.Tensor: """ids (B, 3 * n) -> z (B, 256, 3).""" n = dict(GROUPS)[group] B = ids.shape[0] idx = ids.reshape(B, -1, n).reshape(-1, n) z = 0.0 for i in range(n): q = f"{group}_vqvae.vq.quantizers.{i}." e = self.sd[q + "codebook.weight"][idx[:, i]] z = z + F.linear(e, self.sd[q + "out_proj.weight"], self.sd[q + "out_proj.bias"]) return z.reshape(B, -1, z.shape[-1]).permute(0, 2, 1) def decode_group(self, group: str, ids: torch.Tensor) -> torch.Tensor: """ids (B, 3 * n) -> (B, 24, D).""" p = f"{group}_vqvae.decoder." z = self.latent(group, ids) h = self._conv(F.silu(self._gn(z, p + "norm_in")), p + "conv_in") for s in ("up_1", "up_2", "up_3"): h = self._block(self._up(self._block(h, p + s + ".block_1"), p + s + ".upsample"), p + s + ".block_2") return self._conv(h, p + "conv_out").permute(0, 2, 1) @torch.no_grad() def decode(self, tokens: torch.Tensor) -> torch.Tensor: """action tokens (B, 27) in [0, 1023] (order pos | rot | grip) -> normalized action (B, 24, 20) fp32.""" tokens = tokens.long().reshape(-1, VALID_ACTION_LEN) B = tokens.shape[0] pos = self.decode_group("pos", tokens[:, :18]) rot = self.decode_group("rot", tokens[:, 18:24]) grip = self.decode_group("grip", tokens[:, 24:27]) x = torch.zeros(B, 24, 20) x[..., 0:3], x[..., 10:13] = pos[..., :3], pos[..., 3:] x[..., 3:9], x[..., 13:19] = rot[..., :6], rot[..., 6:] x[..., 9:10], x[..., 19:20] = grip[..., :1], grip[..., 1:] return x class RVQDecoderMM(RVQDecoder): """The same decoder as :class:`RVQDecoder` with every convolution as one matmul on (time, channel) rows (the serving host path, ~10x faster than torch conv1d at these 3..24-step lengths): Conv1d(k3) = [x_{t-1} | x_t | x_{t+1}] @ W (3 Cin x Cout), ConvTranspose1d(k3, s2, p1, op1): out[2i] = x_i W_1, out[2i + 1] = x_i W_2 + x_{i+1} W_0. fp32; equal to the conv1d reference up to summation order (host test).""" def __init__(self, root: str): super().__init__(root) self.W: Dict[str, object] = {} for k, v in self.sd.items(): if k.endswith(".weight") and v.dim() == 3: if "conv_transpose" in k: self.W[k] = (v[:, :, 1].contiguous(), torch.cat([v[:, :, 2], v[:, :, 0]], 0).contiguous()) elif v.shape[2] == 3: self.W[k] = torch.cat([v[:, :, 0].t(), v[:, :, 1].t(), v[:, :, 2].t()], 0).contiguous() else: self.W[k] = v[:, :, 0].t().contiguous() def _gn_t(self, x, p): # x (T, C) return F.silu(F.group_norm(x.t()[None], 1, self.sd[p + ".weight"], self.sd[p + ".bias"], eps=1e-6)[0].t()) def _conv_t(self, x, p, k=3): if k == 3: z = x.new_zeros(1, x.shape[1]) x = torch.cat([torch.cat([z, x[:-1]]), x, torch.cat([x[1:], z])], 1) return torch.addmm(self.sd[p + ".bias"], x, self.W[p + ".weight"]) def _block_t(self, x, p): h = self._conv_t(self._gn_t(x, p + ".norm1"), p + ".conv1") h = self._conv_t(self._gn_t(h, p + ".norm2"), p + ".conv2") sc = self._conv_t(x, p + ".nin_shortcut", 1) if (p + ".nin_shortcut.weight") in self.sd else x return sc + h def _up_t(self, x, p): we, wo = self.W[p + ".conv_transpose.weight"] b = self.sd[p + ".conv_transpose.bias"] e = torch.addmm(b, x, we) o = torch.addmm(b, torch.cat([x, torch.cat([x[1:], x.new_zeros(1, x.shape[1])])], 1), wo) return torch.stack([e, o], 1).reshape(-1, e.shape[1]) @torch.no_grad() def decode_group(self, group: str, ids: torch.Tensor) -> torch.Tensor: """ids (B, 3 * n) -> (B, 24, D).""" p = f"{group}_vqvae.decoder." outs = [] for z in self.latent(group, ids): # (256, 3) per batch row h = self._conv_t(self._gn_t(z.t().contiguous(), p + "norm_in"), p + "conv_in") for s in ("up_1", "up_2", "up_3"): h = self._block_t(self._up_t(self._block_t(h, p + s + ".block_1"), p + s + ".upsample"), p + s + ".block_2") outs.append(self._conv_t(h, p + "conv_out")) return torch.stack(outs, 0) GROUP_SLICES = {"pos": slice(0, 18), "rot": slice(18, 24), "grip": slice(24, 27)} def assemble(pos: torch.Tensor, rot: torch.Tensor, grip: torch.Tensor) -> torch.Tensor: """(B, 24, 6 / 12 / 2) group outputs -> (B, 24, 20) (MultiVQVAE apply_act_dim).""" x = torch.zeros(pos.shape[0], 24, 20) x[..., 0:3], x[..., 10:13] = pos[..., :3], pos[..., 3:] x[..., 3:9], x[..., 13:19] = rot[..., :6], rot[..., 6:] x[..., 9:10], x[..., 19:20] = grip[..., :1], grip[..., 1:] return x @lru_cache(maxsize=2) def rvq_decoder(root: str) -> RVQDecoder: return RVQDecoder(root) @lru_cache(maxsize=2) def rvq_decoder_mm(root: str) -> RVQDecoderMM: return RVQDecoderMM(root)