changh95's picture
tt-model push rdt2-base-p150 (container)
c869f1f verified
Raw History Blame Contribute Delete
15.9 kB
# SPDX-License-Identifier: Apache-2.0
"""RDT2-VQ host reference (torch, no transformers, no official code): what the official README path
(``thu-ml/RDT2 utils.batch_predict_action``) computes around the Qwen2.5-VL-7B forward.
cameras [left, right] --concat--> (384, 768, 3) --JPEG q95 round trip (README / deploy: apply_jpeg_compression=True)
--> slow Qwen2VLImageProcessor (PIL bicubic resize to 392 x 756, x * (1/255) in fp64 -> fp32, (x - mean) / std fp32)
--> pixel_values; prompt = the FM prompt (same chat template + "<|im_start|>assistant\\n<|quad_start|>")
--> Qwen2.5-VL-7B generate, max_new_tokens 29: greedy on the repetition-penalized logits (penalty 1.05 over every
token of the prompt and of the output so far; the checkpoint's do_sample=True at temperature 1e-6 is greedy up to
exact ties), eos 151645 / 151643
--> action ids = the 27 generated ids before the first <|quad_end|> (only if it is at index 27, else 27 zeros)
--> action tokens = clamp(151643 - (id + 1), 0, 1023) --> MultiVQVAE.decode (fp32) --> (24, 20) normalized
--> UMI unnormalize (x - offset) / scale.
:class:`RVQDecoder` re-implements ``MultiVQVAE.decode`` from the pinned ``model.safetensors`` (decoder + codebook +
out_proj tensors only; the encoders are never read).
"""
from __future__ import annotations
import io
import json
import os
from functools import lru_cache
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from .config import CONFIG, TOKENS
RVQ_REPO = "robotics-diffusion-transformer/RVQActionTokenizer"
RVQ_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec"
RVQ_FILES = ("config.json", "model.safetensors")
QUAD_START_ID = TOKENS["<|quad_start|>"] # 151650 (last prompt token)
QUAD_END_ID = TOKENS["<|quad_end|>"] # 151651 (ends the action ids)
EOS_IDS = (TOKENS["<|im_end|>"], TOKENS["<|endoftext|>"]) # generation_config.json eos_token_id
TOKENIZER_VOCAB = 151643 # processor.tokenizer.vocab_size (base BPE vocab, no added tokens)
NUM_EMBEDDINGS = 1024 # RVQ codebook size (config.json num_embeddings)
VALID_ACTION_LEN = 27 # pos 6 x 3 + rot 2 x 3 + grip 1 x 3 (vae.*_id_len)
MAX_NEW_TOKENS = VALID_ACTION_LEN + 2 # batch_predict_action: max_new_tokens=valid_action_id_length + 2
REPETITION_PENALTY = 1.05 # RDT2-VQ generation_config.json
JPEG_QUALITY = 95 # cv2.imencode('.jpg') default (IMWRITE_JPEG_QUALITY)
# tokens the action needs: 27 action ids + the <|quad_end|> that validates them (the 29th token, normally <|im_end|>,
# never changes the action): 1 from the prefill + 27 decode steps
NEEDED_TOKENS = VALID_ACTION_LEN + 1
# ---------------------------------------------------------------------------------------------------------------
# pre-processing (README path: JPEG round trip + the slow image processor)
# ---------------------------------------------------------------------------------------------------------------
def jpeg_roundtrip(image: np.ndarray, quality: int = JPEG_QUALITY) -> np.ndarray:
"""(H, W, 3) uint8 RGB -> the RGB image PIL decodes from its baseline JPEG (quality 95, 4:2:0, libjpeg-turbo).
The official ``cv2.imencode('.jpg', rgb[..., ::-1])`` + ``PIL.Image.open`` gives the same pixels (both encoders
are libjpeg-turbo with the IJG tables scaled to quality 95, 4:2:0 subsampling, no optimization; host test
``test_vq_jpeg_matches_golden`` compares to the official run's decoded image)."""
from PIL import Image
buf = io.BytesIO()
Image.fromarray(np.ascontiguousarray(image)).save(buf, format="JPEG", quality=int(quality))
buf.seek(0)
return np.asarray(Image.open(buf).convert("RGB"), dtype=np.uint8).copy()
def slow_normalized(image: np.ndarray, size_hw: Tuple[int, int]) -> np.ndarray:
"""The slow Qwen2VLImageProcessor on one RGB uint8 image: PIL BICUBIC resize to ``size_hw`` (uint8), then
``rescale`` (x.astype(float64) * (1/255) -> float32) and ``normalize`` ((x - mean) / std in float32).
Returns (H, W, 3) float32."""
from PIL import Image
P = CONFIG.proc
h, w = size_hw
im = Image.fromarray(np.ascontiguousarray(image))
if (im.height, im.width) != (h, w):
im = im.resize((w, h), resample=Image.BICUBIC, reducing_gap=None)
x = np.asarray(im, dtype=np.uint8)
x = (x.astype(np.float64) * P.rescale_factor).astype(np.float32)
mean = np.array(P.image_mean, dtype=np.float32)
std = np.array(P.image_std, dtype=np.float32)
return (x - mean) / std
def vq_pixel_values(images: Sequence[np.ndarray], jpeg: bool = True) -> Tuple[torch.Tensor, np.ndarray]:
"""cameras (left, right) -> (pixel_values (N, 1176) fp32 in the processor's raster-merge order, the image fed to
the processor (H, W, 3) uint8)."""
from .preprocess import concat_cameras, smart_resize
V = CONFIG.vision
img = concat_cameras(images)
if jpeg:
img = jpeg_roundtrip(img)
rh, rw = smart_resize(img.shape[0], img.shape[1], factor=V.patch * V.merge)
x = torch.from_numpy(slow_normalized(img, (rh, rw))).permute(2, 0, 1)[None] # (1, 3, H, W)
x = x.repeat(V.temporal_patch, 1, 1, 1)
gh, gw = rh // V.patch, rw // V.patch
x = x.view(1, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch)
x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
return x.reshape(gh * gw, 3 * V.temporal_patch * V.patch * V.patch).contiguous(), img
# ---------------------------------------------------------------------------------------------------------------
# generation semantics
# ---------------------------------------------------------------------------------------------------------------
def penalize(logits: torch.Tensor, seen: Iterable[int], penalty: float = REPETITION_PENALTY) -> torch.Tensor:
"""HF RepetitionPenaltyLogitsProcessor on one row: every token id in ``seen`` gets logit / p (> 0) or * p (< 0)."""
out = logits.detach().float().clone()
idx = torch.tensor(sorted(set(int(t) for t in seen)), dtype=torch.long)
if idx.numel():
s = out[idx]
out[idx] = torch.where(s < 0, s * penalty, s / penalty)
return out
def penalized_argmax(logits: torch.Tensor, seen: Iterable[int], penalty: float = REPETITION_PENALTY) -> int:
return int(torch.argmax(penalize(logits, seen, penalty)))
def penalized_argmax_topk(values: Sequence[float], indices: Sequence[int], seen: set,
penalty: float = REPETITION_PENALTY) -> Optional[int]:
"""Exact penalized argmax from the top-k raw logits (descending): the penalty only lowers logits, so the
best penalized candidate is the answer whenever it is >= the k-th raw value (no token outside the top-k can beat
it). Returns None when the top-k does not decide (the caller then reads the full row)."""
best, best_v = None, None
for v, i in zip(values, indices):
v = float(v)
if int(i) in seen:
v = v * penalty if v < 0 else v / penalty
if best_v is None or v > best_v:
best, best_v = int(i), v
if best_v is None or best_v < float(values[-1]):
return None
return best
def extract_action_ids(generated: Sequence[int]) -> Tuple[torch.Tensor, bool]:
"""batch_predict_action's extraction for one example: the ids before the first <|quad_end|> if exactly 27 of
them, else 27 zeros (valid False)."""
g = list(int(t) for t in generated)
if QUAD_END_ID in g and g.index(QUAD_END_ID) == VALID_ACTION_LEN:
return torch.tensor(g[:VALID_ACTION_LEN], dtype=torch.long), True
return torch.zeros(VALID_ACTION_LEN, dtype=torch.long), False
def action_tokens_from_ids(action_ids: torch.Tensor) -> torch.Tensor:
"""vocab_size - (id + 1), clamped to the codebook [0, 1023]."""
return torch.clamp(TOKENIZER_VOCAB - (action_ids.long() + 1), min=0, max=NUM_EMBEDDINGS - 1)
def is_finished(generated: Sequence[int]) -> bool:
"""The action is decided once <|quad_end|> or an eos is generated, or once 28 tokens exist (the 29th never
changes the action ids)."""
g = list(generated)
return len(g) >= NEEDED_TOKENS or any(t in EOS_IDS or t == QUAD_END_ID for t in g)
# ---------------------------------------------------------------------------------------------------------------
# RVQ action-token decoder (MultiVQVAE.decode, fp32)
# ---------------------------------------------------------------------------------------------------------------
GROUPS = (("pos", 6), ("rot", 2), ("grip", 1)) # (name, n_codebooks); 3 latent time steps each
def load_rvq_state(root: str) -> Tuple[Dict[str, torch.Tensor], Dict]:
"""Decoder + quantizer tensors of the pinned RVQActionTokenizer ``model.safetensors`` (fp32), and config.json."""
from safetensors import safe_open
with open(os.path.join(root, "config.json")) as f:
cfg = json.load(f)
sd = {}
with safe_open(os.path.join(root, "model.safetensors"), framework="pt", device="cpu") as f:
for k in f.keys():
if ".decoder." in k or (".vq.quantizers." in k and (".codebook." in k or ".out_proj." in k)):
sd[k] = f.get_tensor(k).float()
return sd, cfg
class RVQDecoder:
"""``MultiVQVAE.decode`` (thu-ml/RDT2 vqvae/models) in torch fp32: per group (pos 6 dims / rot 12 / grip 2),
z = sum_i out_proj_i(codebook_i[id_i]) over its residual codebooks for 3 latent steps, then the CNN decoder
(GroupNorm(1) + SiLU + Conv1d k3, 3 up stages with ConvTranspose1d(k3, s2, p1, op1): 3 -> 6 -> 12 -> 24 steps)."""
def __init__(self, root: str):
self.sd, self.cfg = load_rvq_state(root)
assert self.cfg["num_embeddings"] == NUM_EMBEDDINGS and self.cfg["action_horizon"] == 24
assert tuple(self.cfg["n_codebooks"][g] for g, _ in GROUPS) == tuple(n for _, n in GROUPS)
def _gn(self, x, p):
return F.group_norm(x, 1, self.sd[p + ".weight"], self.sd[p + ".bias"], eps=1e-6)
def _conv(self, x, p, pad=1):
return F.conv1d(x, self.sd[p + ".weight"], self.sd[p + ".bias"], padding=pad)
def _block(self, x, p):
h = self._conv(F.silu(self._gn(x, p + ".norm1")), p + ".conv1")
h = self._conv(F.silu(self._gn(h, p + ".norm2")), p + ".conv2")
sc = x if (p + ".nin_shortcut.weight") not in self.sd else self._conv(x, p + ".nin_shortcut", pad=0)
return sc + h
def _up(self, x, p):
return F.conv_transpose1d(x, self.sd[p + ".conv_transpose.weight"], self.sd[p + ".conv_transpose.bias"],
stride=2, padding=1, output_padding=1)
def latent(self, group: str, ids: torch.Tensor) -> torch.Tensor:
"""ids (B, 3 * n) -> z (B, 256, 3)."""
n = dict(GROUPS)[group]
B = ids.shape[0]
idx = ids.reshape(B, -1, n).reshape(-1, n)
z = 0.0
for i in range(n):
q = f"{group}_vqvae.vq.quantizers.{i}."
e = self.sd[q + "codebook.weight"][idx[:, i]]
z = z + F.linear(e, self.sd[q + "out_proj.weight"], self.sd[q + "out_proj.bias"])
return z.reshape(B, -1, z.shape[-1]).permute(0, 2, 1)
def decode_group(self, group: str, ids: torch.Tensor) -> torch.Tensor:
"""ids (B, 3 * n) -> (B, 24, D)."""
p = f"{group}_vqvae.decoder."
z = self.latent(group, ids)
h = self._conv(F.silu(self._gn(z, p + "norm_in")), p + "conv_in")
for s in ("up_1", "up_2", "up_3"):
h = self._block(self._up(self._block(h, p + s + ".block_1"), p + s + ".upsample"), p + s + ".block_2")
return self._conv(h, p + "conv_out").permute(0, 2, 1)
@torch.no_grad()
def decode(self, tokens: torch.Tensor) -> torch.Tensor:
"""action tokens (B, 27) in [0, 1023] (order pos | rot | grip) -> normalized action (B, 24, 20) fp32."""
tokens = tokens.long().reshape(-1, VALID_ACTION_LEN)
B = tokens.shape[0]
pos = self.decode_group("pos", tokens[:, :18])
rot = self.decode_group("rot", tokens[:, 18:24])
grip = self.decode_group("grip", tokens[:, 24:27])
x = torch.zeros(B, 24, 20)
x[..., 0:3], x[..., 10:13] = pos[..., :3], pos[..., 3:]
x[..., 3:9], x[..., 13:19] = rot[..., :6], rot[..., 6:]
x[..., 9:10], x[..., 19:20] = grip[..., :1], grip[..., 1:]
return x
class RVQDecoderMM(RVQDecoder):
"""The same decoder as :class:`RVQDecoder` with every convolution as one matmul on (time, channel) rows (the
serving host path, ~10x faster than torch conv1d at these 3..24-step lengths): Conv1d(k3) = [x_{t-1} | x_t |
x_{t+1}] @ W (3 Cin x Cout), ConvTranspose1d(k3, s2, p1, op1): out[2i] = x_i W_1, out[2i + 1] = x_i W_2 +
x_{i+1} W_0. fp32; equal to the conv1d reference up to summation order (host test)."""
def __init__(self, root: str):
super().__init__(root)
self.W: Dict[str, object] = {}
for k, v in self.sd.items():
if k.endswith(".weight") and v.dim() == 3:
if "conv_transpose" in k:
self.W[k] = (v[:, :, 1].contiguous(), torch.cat([v[:, :, 2], v[:, :, 0]], 0).contiguous())
elif v.shape[2] == 3:
self.W[k] = torch.cat([v[:, :, 0].t(), v[:, :, 1].t(), v[:, :, 2].t()], 0).contiguous()
else:
self.W[k] = v[:, :, 0].t().contiguous()
def _gn_t(self, x, p): # x (T, C)
return F.silu(F.group_norm(x.t()[None], 1, self.sd[p + ".weight"], self.sd[p + ".bias"], eps=1e-6)[0].t())
def _conv_t(self, x, p, k=3):
if k == 3:
z = x.new_zeros(1, x.shape[1])
x = torch.cat([torch.cat([z, x[:-1]]), x, torch.cat([x[1:], z])], 1)
return torch.addmm(self.sd[p + ".bias"], x, self.W[p + ".weight"])
def _block_t(self, x, p):
h = self._conv_t(self._gn_t(x, p + ".norm1"), p + ".conv1")
h = self._conv_t(self._gn_t(h, p + ".norm2"), p + ".conv2")
sc = self._conv_t(x, p + ".nin_shortcut", 1) if (p + ".nin_shortcut.weight") in self.sd else x
return sc + h
def _up_t(self, x, p):
we, wo = self.W[p + ".conv_transpose.weight"]
b = self.sd[p + ".conv_transpose.bias"]
e = torch.addmm(b, x, we)
o = torch.addmm(b, torch.cat([x, torch.cat([x[1:], x.new_zeros(1, x.shape[1])])], 1), wo)
return torch.stack([e, o], 1).reshape(-1, e.shape[1])
@torch.no_grad()
def decode_group(self, group: str, ids: torch.Tensor) -> torch.Tensor:
"""ids (B, 3 * n) -> (B, 24, D)."""
p = f"{group}_vqvae.decoder."
outs = []
for z in self.latent(group, ids): # (256, 3) per batch row
h = self._conv_t(self._gn_t(z.t().contiguous(), p + "norm_in"), p + "conv_in")
for s in ("up_1", "up_2", "up_3"):
h = self._block_t(self._up_t(self._block_t(h, p + s + ".block_1"), p + s + ".upsample"),
p + s + ".block_2")
outs.append(self._conv_t(h, p + "conv_out"))
return torch.stack(outs, 0)
GROUP_SLICES = {"pos": slice(0, 18), "rot": slice(18, 24), "grip": slice(24, 27)}
def assemble(pos: torch.Tensor, rot: torch.Tensor, grip: torch.Tensor) -> torch.Tensor:
"""(B, 24, 6 / 12 / 2) group outputs -> (B, 24, 20) (MultiVQVAE apply_act_dim)."""
x = torch.zeros(pos.shape[0], 24, 20)
x[..., 0:3], x[..., 10:13] = pos[..., :3], pos[..., 3:]
x[..., 3:9], x[..., 13:19] = rot[..., :6], rot[..., 6:]
x[..., 9:10], x[..., 19:20] = grip[..., :1], grip[..., 1:]
return x
@lru_cache(maxsize=2)
def rvq_decoder(root: str) -> RVQDecoder:
return RVQDecoder(root)
@lru_cache(maxsize=2)
def rvq_decoder_mm(root: str) -> RVQDecoderMM:
return RVQDecoderMM(root)