Download code/tt_rdt2/reference/vq.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 15.9 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/vq.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/vq.py
-
curl -L -o vq.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/vq.py
15.9 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """RDT2-VQ host reference (torch, no transformers, no official code): what the official README path | |
| (``thu-ml/RDT2 utils.batch_predict_action``) computes around the Qwen2.5-VL-7B forward. | |
| cameras [left, right] --concat--> (384, 768, 3) --JPEG q95 round trip (README / deploy: apply_jpeg_compression=True) | |
| --> slow Qwen2VLImageProcessor (PIL bicubic resize to 392 x 756, x * (1/255) in fp64 -> fp32, (x - mean) / std fp32) | |
| --> pixel_values; prompt = the FM prompt (same chat template + "<|im_start|>assistant\\n<|quad_start|>") | |
| --> Qwen2.5-VL-7B generate, max_new_tokens 29: greedy on the repetition-penalized logits (penalty 1.05 over every | |
| token of the prompt and of the output so far; the checkpoint's do_sample=True at temperature 1e-6 is greedy up to | |
| exact ties), eos 151645 / 151643 | |
| --> action ids = the 27 generated ids before the first <|quad_end|> (only if it is at index 27, else 27 zeros) | |
| --> action tokens = clamp(151643 - (id + 1), 0, 1023) --> MultiVQVAE.decode (fp32) --> (24, 20) normalized | |
| --> UMI unnormalize (x - offset) / scale. | |
| :class:`RVQDecoder` re-implements ``MultiVQVAE.decode`` from the pinned ``model.safetensors`` (decoder + codebook + | |
| out_proj tensors only; the encoders are never read). | |
| """ | |
| from __future__ import annotations | |
| import io | |
| import json | |
| import os | |
| from functools import lru_cache | |
| from typing import Dict, Iterable, List, Optional, Sequence, Tuple | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from .config import CONFIG, TOKENS | |
| RVQ_REPO = "robotics-diffusion-transformer/RVQActionTokenizer" | |
| RVQ_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec" | |
| RVQ_FILES = ("config.json", "model.safetensors") | |
| QUAD_START_ID = TOKENS["<|quad_start|>"] # 151650 (last prompt token) | |
| QUAD_END_ID = TOKENS["<|quad_end|>"] # 151651 (ends the action ids) | |
| EOS_IDS = (TOKENS["<|im_end|>"], TOKENS["<|endoftext|>"]) # generation_config.json eos_token_id | |
| TOKENIZER_VOCAB = 151643 # processor.tokenizer.vocab_size (base BPE vocab, no added tokens) | |
| NUM_EMBEDDINGS = 1024 # RVQ codebook size (config.json num_embeddings) | |
| VALID_ACTION_LEN = 27 # pos 6 x 3 + rot 2 x 3 + grip 1 x 3 (vae.*_id_len) | |
| MAX_NEW_TOKENS = VALID_ACTION_LEN + 2 # batch_predict_action: max_new_tokens=valid_action_id_length + 2 | |
| REPETITION_PENALTY = 1.05 # RDT2-VQ generation_config.json | |
| JPEG_QUALITY = 95 # cv2.imencode('.jpg') default (IMWRITE_JPEG_QUALITY) | |
| # tokens the action needs: 27 action ids + the <|quad_end|> that validates them (the 29th token, normally <|im_end|>, | |
| # never changes the action): 1 from the prefill + 27 decode steps | |
| NEEDED_TOKENS = VALID_ACTION_LEN + 1 | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # pre-processing (README path: JPEG round trip + the slow image processor) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def jpeg_roundtrip(image: np.ndarray, quality: int = JPEG_QUALITY) -> np.ndarray: | |
| """(H, W, 3) uint8 RGB -> the RGB image PIL decodes from its baseline JPEG (quality 95, 4:2:0, libjpeg-turbo). | |
| The official ``cv2.imencode('.jpg', rgb[..., ::-1])`` + ``PIL.Image.open`` gives the same pixels (both encoders | |
| are libjpeg-turbo with the IJG tables scaled to quality 95, 4:2:0 subsampling, no optimization; host test | |
| ``test_vq_jpeg_matches_golden`` compares to the official run's decoded image).""" | |
| from PIL import Image | |
| buf = io.BytesIO() | |
| Image.fromarray(np.ascontiguousarray(image)).save(buf, format="JPEG", quality=int(quality)) | |
| buf.seek(0) | |
| return np.asarray(Image.open(buf).convert("RGB"), dtype=np.uint8).copy() | |
| def slow_normalized(image: np.ndarray, size_hw: Tuple[int, int]) -> np.ndarray: | |
| """The slow Qwen2VLImageProcessor on one RGB uint8 image: PIL BICUBIC resize to ``size_hw`` (uint8), then | |
| ``rescale`` (x.astype(float64) * (1/255) -> float32) and ``normalize`` ((x - mean) / std in float32). | |
| Returns (H, W, 3) float32.""" | |
| from PIL import Image | |
| P = CONFIG.proc | |
| h, w = size_hw | |
| im = Image.fromarray(np.ascontiguousarray(image)) | |
| if (im.height, im.width) != (h, w): | |
| im = im.resize((w, h), resample=Image.BICUBIC, reducing_gap=None) | |
| x = np.asarray(im, dtype=np.uint8) | |
| x = (x.astype(np.float64) * P.rescale_factor).astype(np.float32) | |
| mean = np.array(P.image_mean, dtype=np.float32) | |
| std = np.array(P.image_std, dtype=np.float32) | |
| return (x - mean) / std | |
| def vq_pixel_values(images: Sequence[np.ndarray], jpeg: bool = True) -> Tuple[torch.Tensor, np.ndarray]: | |
| """cameras (left, right) -> (pixel_values (N, 1176) fp32 in the processor's raster-merge order, the image fed to | |
| the processor (H, W, 3) uint8).""" | |
| from .preprocess import concat_cameras, smart_resize | |
| V = CONFIG.vision | |
| img = concat_cameras(images) | |
| if jpeg: | |
| img = jpeg_roundtrip(img) | |
| rh, rw = smart_resize(img.shape[0], img.shape[1], factor=V.patch * V.merge) | |
| x = torch.from_numpy(slow_normalized(img, (rh, rw))).permute(2, 0, 1)[None] # (1, 3, H, W) | |
| x = x.repeat(V.temporal_patch, 1, 1, 1) | |
| gh, gw = rh // V.patch, rw // V.patch | |
| x = x.view(1, V.temporal_patch, 3, gh // V.merge, V.merge, V.patch, gw // V.merge, V.merge, V.patch) | |
| x = x.permute(0, 3, 6, 4, 7, 2, 1, 5, 8) | |
| return x.reshape(gh * gw, 3 * V.temporal_patch * V.patch * V.patch).contiguous(), img | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # generation semantics | |
| # --------------------------------------------------------------------------------------------------------------- | |
| def penalize(logits: torch.Tensor, seen: Iterable[int], penalty: float = REPETITION_PENALTY) -> torch.Tensor: | |
| """HF RepetitionPenaltyLogitsProcessor on one row: every token id in ``seen`` gets logit / p (> 0) or * p (< 0).""" | |
| out = logits.detach().float().clone() | |
| idx = torch.tensor(sorted(set(int(t) for t in seen)), dtype=torch.long) | |
| if idx.numel(): | |
| s = out[idx] | |
| out[idx] = torch.where(s < 0, s * penalty, s / penalty) | |
| return out | |
| def penalized_argmax(logits: torch.Tensor, seen: Iterable[int], penalty: float = REPETITION_PENALTY) -> int: | |
| return int(torch.argmax(penalize(logits, seen, penalty))) | |
| def penalized_argmax_topk(values: Sequence[float], indices: Sequence[int], seen: set, | |
| penalty: float = REPETITION_PENALTY) -> Optional[int]: | |
| """Exact penalized argmax from the top-k raw logits (descending): the penalty only lowers logits, so the | |
| best penalized candidate is the answer whenever it is >= the k-th raw value (no token outside the top-k can beat | |
| it). Returns None when the top-k does not decide (the caller then reads the full row).""" | |
| best, best_v = None, None | |
| for v, i in zip(values, indices): | |
| v = float(v) | |
| if int(i) in seen: | |
| v = v * penalty if v < 0 else v / penalty | |
| if best_v is None or v > best_v: | |
| best, best_v = int(i), v | |
| if best_v is None or best_v < float(values[-1]): | |
| return None | |
| return best | |
| def extract_action_ids(generated: Sequence[int]) -> Tuple[torch.Tensor, bool]: | |
| """batch_predict_action's extraction for one example: the ids before the first <|quad_end|> if exactly 27 of | |
| them, else 27 zeros (valid False).""" | |
| g = list(int(t) for t in generated) | |
| if QUAD_END_ID in g and g.index(QUAD_END_ID) == VALID_ACTION_LEN: | |
| return torch.tensor(g[:VALID_ACTION_LEN], dtype=torch.long), True | |
| return torch.zeros(VALID_ACTION_LEN, dtype=torch.long), False | |
| def action_tokens_from_ids(action_ids: torch.Tensor) -> torch.Tensor: | |
| """vocab_size - (id + 1), clamped to the codebook [0, 1023].""" | |
| return torch.clamp(TOKENIZER_VOCAB - (action_ids.long() + 1), min=0, max=NUM_EMBEDDINGS - 1) | |
| def is_finished(generated: Sequence[int]) -> bool: | |
| """The action is decided once <|quad_end|> or an eos is generated, or once 28 tokens exist (the 29th never | |
| changes the action ids).""" | |
| g = list(generated) | |
| return len(g) >= NEEDED_TOKENS or any(t in EOS_IDS or t == QUAD_END_ID for t in g) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # RVQ action-token decoder (MultiVQVAE.decode, fp32) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| GROUPS = (("pos", 6), ("rot", 2), ("grip", 1)) # (name, n_codebooks); 3 latent time steps each | |
| def load_rvq_state(root: str) -> Tuple[Dict[str, torch.Tensor], Dict]: | |
| """Decoder + quantizer tensors of the pinned RVQActionTokenizer ``model.safetensors`` (fp32), and config.json.""" | |
| from safetensors import safe_open | |
| with open(os.path.join(root, "config.json")) as f: | |
| cfg = json.load(f) | |
| sd = {} | |
| with safe_open(os.path.join(root, "model.safetensors"), framework="pt", device="cpu") as f: | |
| for k in f.keys(): | |
| if ".decoder." in k or (".vq.quantizers." in k and (".codebook." in k or ".out_proj." in k)): | |
| sd[k] = f.get_tensor(k).float() | |
| return sd, cfg | |
| class RVQDecoder: | |
| """``MultiVQVAE.decode`` (thu-ml/RDT2 vqvae/models) in torch fp32: per group (pos 6 dims / rot 12 / grip 2), | |
| z = sum_i out_proj_i(codebook_i[id_i]) over its residual codebooks for 3 latent steps, then the CNN decoder | |
| (GroupNorm(1) + SiLU + Conv1d k3, 3 up stages with ConvTranspose1d(k3, s2, p1, op1): 3 -> 6 -> 12 -> 24 steps).""" | |
| def __init__(self, root: str): | |
| self.sd, self.cfg = load_rvq_state(root) | |
| assert self.cfg["num_embeddings"] == NUM_EMBEDDINGS and self.cfg["action_horizon"] == 24 | |
| assert tuple(self.cfg["n_codebooks"][g] for g, _ in GROUPS) == tuple(n for _, n in GROUPS) | |
| def _gn(self, x, p): | |
| return F.group_norm(x, 1, self.sd[p + ".weight"], self.sd[p + ".bias"], eps=1e-6) | |
| def _conv(self, x, p, pad=1): | |
| return F.conv1d(x, self.sd[p + ".weight"], self.sd[p + ".bias"], padding=pad) | |
| def _block(self, x, p): | |
| h = self._conv(F.silu(self._gn(x, p + ".norm1")), p + ".conv1") | |
| h = self._conv(F.silu(self._gn(h, p + ".norm2")), p + ".conv2") | |
| sc = x if (p + ".nin_shortcut.weight") not in self.sd else self._conv(x, p + ".nin_shortcut", pad=0) | |
| return sc + h | |
| def _up(self, x, p): | |
| return F.conv_transpose1d(x, self.sd[p + ".conv_transpose.weight"], self.sd[p + ".conv_transpose.bias"], | |
| stride=2, padding=1, output_padding=1) | |
| def latent(self, group: str, ids: torch.Tensor) -> torch.Tensor: | |
| """ids (B, 3 * n) -> z (B, 256, 3).""" | |
| n = dict(GROUPS)[group] | |
| B = ids.shape[0] | |
| idx = ids.reshape(B, -1, n).reshape(-1, n) | |
| z = 0.0 | |
| for i in range(n): | |
| q = f"{group}_vqvae.vq.quantizers.{i}." | |
| e = self.sd[q + "codebook.weight"][idx[:, i]] | |
| z = z + F.linear(e, self.sd[q + "out_proj.weight"], self.sd[q + "out_proj.bias"]) | |
| return z.reshape(B, -1, z.shape[-1]).permute(0, 2, 1) | |
| def decode_group(self, group: str, ids: torch.Tensor) -> torch.Tensor: | |
| """ids (B, 3 * n) -> (B, 24, D).""" | |
| p = f"{group}_vqvae.decoder." | |
| z = self.latent(group, ids) | |
| h = self._conv(F.silu(self._gn(z, p + "norm_in")), p + "conv_in") | |
| for s in ("up_1", "up_2", "up_3"): | |
| h = self._block(self._up(self._block(h, p + s + ".block_1"), p + s + ".upsample"), p + s + ".block_2") | |
| return self._conv(h, p + "conv_out").permute(0, 2, 1) | |
| def decode(self, tokens: torch.Tensor) -> torch.Tensor: | |
| """action tokens (B, 27) in [0, 1023] (order pos | rot | grip) -> normalized action (B, 24, 20) fp32.""" | |
| tokens = tokens.long().reshape(-1, VALID_ACTION_LEN) | |
| B = tokens.shape[0] | |
| pos = self.decode_group("pos", tokens[:, :18]) | |
| rot = self.decode_group("rot", tokens[:, 18:24]) | |
| grip = self.decode_group("grip", tokens[:, 24:27]) | |
| x = torch.zeros(B, 24, 20) | |
| x[..., 0:3], x[..., 10:13] = pos[..., :3], pos[..., 3:] | |
| x[..., 3:9], x[..., 13:19] = rot[..., :6], rot[..., 6:] | |
| x[..., 9:10], x[..., 19:20] = grip[..., :1], grip[..., 1:] | |
| return x | |
| class RVQDecoderMM(RVQDecoder): | |
| """The same decoder as :class:`RVQDecoder` with every convolution as one matmul on (time, channel) rows (the | |
| serving host path, ~10x faster than torch conv1d at these 3..24-step lengths): Conv1d(k3) = [x_{t-1} | x_t | | |
| x_{t+1}] @ W (3 Cin x Cout), ConvTranspose1d(k3, s2, p1, op1): out[2i] = x_i W_1, out[2i + 1] = x_i W_2 + | |
| x_{i+1} W_0. fp32; equal to the conv1d reference up to summation order (host test).""" | |
| def __init__(self, root: str): | |
| super().__init__(root) | |
| self.W: Dict[str, object] = {} | |
| for k, v in self.sd.items(): | |
| if k.endswith(".weight") and v.dim() == 3: | |
| if "conv_transpose" in k: | |
| self.W[k] = (v[:, :, 1].contiguous(), torch.cat([v[:, :, 2], v[:, :, 0]], 0).contiguous()) | |
| elif v.shape[2] == 3: | |
| self.W[k] = torch.cat([v[:, :, 0].t(), v[:, :, 1].t(), v[:, :, 2].t()], 0).contiguous() | |
| else: | |
| self.W[k] = v[:, :, 0].t().contiguous() | |
| def _gn_t(self, x, p): # x (T, C) | |
| return F.silu(F.group_norm(x.t()[None], 1, self.sd[p + ".weight"], self.sd[p + ".bias"], eps=1e-6)[0].t()) | |
| def _conv_t(self, x, p, k=3): | |
| if k == 3: | |
| z = x.new_zeros(1, x.shape[1]) | |
| x = torch.cat([torch.cat([z, x[:-1]]), x, torch.cat([x[1:], z])], 1) | |
| return torch.addmm(self.sd[p + ".bias"], x, self.W[p + ".weight"]) | |
| def _block_t(self, x, p): | |
| h = self._conv_t(self._gn_t(x, p + ".norm1"), p + ".conv1") | |
| h = self._conv_t(self._gn_t(h, p + ".norm2"), p + ".conv2") | |
| sc = self._conv_t(x, p + ".nin_shortcut", 1) if (p + ".nin_shortcut.weight") in self.sd else x | |
| return sc + h | |
| def _up_t(self, x, p): | |
| we, wo = self.W[p + ".conv_transpose.weight"] | |
| b = self.sd[p + ".conv_transpose.bias"] | |
| e = torch.addmm(b, x, we) | |
| o = torch.addmm(b, torch.cat([x, torch.cat([x[1:], x.new_zeros(1, x.shape[1])])], 1), wo) | |
| return torch.stack([e, o], 1).reshape(-1, e.shape[1]) | |
| def decode_group(self, group: str, ids: torch.Tensor) -> torch.Tensor: | |
| """ids (B, 3 * n) -> (B, 24, D).""" | |
| p = f"{group}_vqvae.decoder." | |
| outs = [] | |
| for z in self.latent(group, ids): # (256, 3) per batch row | |
| h = self._conv_t(self._gn_t(z.t().contiguous(), p + "norm_in"), p + "conv_in") | |
| for s in ("up_1", "up_2", "up_3"): | |
| h = self._block_t(self._up_t(self._block_t(h, p + s + ".block_1"), p + s + ".upsample"), | |
| p + s + ".block_2") | |
| outs.append(self._conv_t(h, p + "conv_out")) | |
| return torch.stack(outs, 0) | |
| GROUP_SLICES = {"pos": slice(0, 18), "rot": slice(18, 24), "grip": slice(24, 27)} | |
| def assemble(pos: torch.Tensor, rot: torch.Tensor, grip: torch.Tensor) -> torch.Tensor: | |
| """(B, 24, 6 / 12 / 2) group outputs -> (B, 24, 20) (MultiVQVAE apply_act_dim).""" | |
| x = torch.zeros(pos.shape[0], 24, 20) | |
| x[..., 0:3], x[..., 10:13] = pos[..., :3], pos[..., 3:] | |
| x[..., 3:9], x[..., 13:19] = rot[..., :6], rot[..., 6:] | |
| x[..., 9:10], x[..., 19:20] = grip[..., :1], grip[..., 1:] | |
| return x | |
| def rvq_decoder(root: str) -> RVQDecoder: | |
| return RVQDecoder(root) | |
| def rvq_decoder_mm(root: str) -> RVQDecoderMM: | |
| return RVQDecoderMM(root) | |