# SPDX-License-Identifier: Apache-2.0 """Static configuration of RDT2-FM (robotics-diffusion-transformer/RDT2-FM on the RDT2-VQ Qwen2.5-VL-7B backbone). Every number here was read from the pinned files (research/rdt2/SPEC.md section 1 cites them): RDT2-VQ config.json @ ec7e7f3a, RDT2-FM config.json @ f803808e, thu-ml/RDT2 configs/rdt/post_train.yaml @ 0797b4c6, Qwen/Qwen2.5-VL-7B-Instruct preprocessor_config.json / tokenizer @ cc594898. """ from __future__ import annotations import os from dataclasses import dataclass, field from pathlib import Path from typing import Optional, Tuple # --------------------------------------------------------------------------------------------------------------- # Pins # --------------------------------------------------------------------------------------------------------------- VLM_REPO = "robotics-diffusion-transformer/RDT2-VQ" VLM_REVISION = "ec7e7f3aa4259224481c3cbd5816cc3f35354bec" EXPERT_REPO = "robotics-diffusion-transformer/RDT2-FM" EXPERT_REVISION = "f803808e3f79c77bba8bb4f3326e83d04e60783a" NORMALIZER_REPO = "robotics-diffusion-transformer/RVQActionTokenizer" NORMALIZER_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec" NORMALIZER_FILE = "umi_normalizer_wo_downsample_indentity_rot.pt" PROCESSOR_REPO = "Qwen/Qwen2.5-VL-7B-Instruct" PROCESSOR_REVISION = "cc594898137f460bfe9f0759e9844b3ce807cfb5" OFFICIAL_CODE = ("github.com/thu-ml/RDT2", "0797b4c65e588e088d41602685e00dc2bc95852f") def _dev_workspace() -> Optional[Path]: """The development workspace when this file sits in its checkout (``/bundles/rdt2-base-p150/code/tt_rdt2/reference/config.py``), else None (the container image, a pip install or a download of the published package).""" p = Path(__file__).resolve().parents if len(p) > 5 and p[2].name == "code" and p[4].name == "bundles" and (p[5] / "assets").is_dir(): return p[5] return None DEV_WORKSPACE = _dev_workspace() # Local copies of the pinned weights and the official golden runs, used by the development tests and scripts only: # RDT2_ASSETS_DIR, else /assets/rdt2 in the development checkout. Without either, the defaults below name # no existing directory: the API resolves the pinned files from the Hugging Face cache and the golden tests skip. _WS_ASSETS = os.environ.get("RDT2_ASSETS_DIR") or (str(DEV_WORKSPACE / "assets" / "rdt2") if DEV_WORKSPACE else "") DEFAULT_DIRS = { "vlm": os.environ.get("RDT2_VLM_DIR", f"{_WS_ASSETS}/hf_RDT2-VQ_ec7e7f3a"), "expert": os.environ.get("RDT2_EXPERT_DIR", f"{_WS_ASSETS}/hf_RDT2-FM_f803808e"), "normalizer": os.environ.get("RDT2_NORMALIZER_DIR", f"{_WS_ASSETS}/hf_RVQActionTokenizer_a6a614ae_normalizer_only"), "processor": os.environ.get("RDT2_PROCESSOR_DIR", f"{_WS_ASSETS}/hf_Qwen2.5-VL-7B-Instruct_cc594898_processor_only"), "golden": os.environ.get("RDT2_GOLDEN_DIR", f"{_WS_ASSETS}/golden"), } # --------------------------------------------------------------------------------------------------------------- # Vision tower (Qwen2.5-VL ViT) # --------------------------------------------------------------------------------------------------------------- @dataclass(frozen=True) class VisionConfig: depth: int = 32 hidden: int = 1280 heads: int = 16 head_dim: int = 80 mlp: int = 3420 patch: int = 14 temporal_patch: int = 2 merge: int = 2 window: int = 112 # pixels -> 112 / 14 / 2 = 4 merged tokens per window side (8x8 = 64 patches) fullatt_blocks: Tuple[int, ...] = (7, 15, 23, 31) out_hidden: int = 3584 rope_theta: float = 10000.0 norm_eps: float = 1e-6 in_dim: int = 3 * 2 * 14 * 14 # 1176 = C x T x 14 x 14 per patch row of pixel_values # --------------------------------------------------------------------------------------------------------------- # Language model (Qwen2.5-VL-7B decoder; RDT2-FM uses layers 0..13 only) # --------------------------------------------------------------------------------------------------------------- @dataclass(frozen=True) class LMConfig: hidden: int = 3584 layers: int = 28 heads: int = 28 kv_heads: int = 4 head_dim: int = 128 mlp: int = 18944 vocab: int = 152064 rope_theta: float = 1_000_000.0 mrope_section: Tuple[int, int, int] = (16, 24, 24) norm_eps: float = 1e-6 selected_layers: Tuple[int, ...] = tuple(range(14)) # post_train.yaml model.selected_layers # --------------------------------------------------------------------------------------------------------------- # Action expert (RDT, flow matching) # --------------------------------------------------------------------------------------------------------------- @dataclass(frozen=True) class ExpertConfig: hidden: int = 1024 depth: int = 14 heads: int = 8 kv_heads: int = 4 head_dim: int = 128 ffn: int = 2816 # llama rule: int(2 * 4096 / 3) = 2730 -> multiple_of 256 -> 2816 final_mlp: int = 4096 registers: int = 4 horizon: int = 24 action_dim: int = 20 state_dim: int = 20 norm_eps: float = 1e-5 freq_dim: int = 256 max_period: float = 10000.0 num_steps: int = 5 @property def tokens(self) -> int: return self.horizon + self.registers # 28 # --------------------------------------------------------------------------------------------------------------- # Pre-processing / prompt # --------------------------------------------------------------------------------------------------------------- @dataclass(frozen=True) class ProcessorConfig: image_mean: Tuple[float, float, float] = (0.48145466, 0.4578275, 0.40821073) image_std: Tuple[float, float, float] = (0.26862954, 0.26130258, 0.27577711) rescale_factor: float = 0.00392156862745098 min_pixels: int = 3136 max_pixels: int = 12845056 camera_hw: Tuple[int, int] = (384, 384) camera_names: Tuple[str, ...] = ("left_stereo", "right_stereo") # concatenated left -> right (post_train.yaml) system_prompt: str = "You are a helpful assistant." suffix: str = "<|im_start|>assistant\n<|quad_start|>" TOKENS = { "<|endoftext|>": 151643, "<|im_start|>": 151644, "<|im_end|>": 151645, "<|quad_start|>": 151650, "<|quad_end|>": 151651, "<|vision_start|>": 151652, "<|vision_end|>": 151653, "<|image_pad|>": 151655, } IMAGE_TOKEN_ID = TOKENS["<|image_pad|>"] VISION_START_ID = TOKENS["<|vision_start|>"] PAD_TOKEN_ID = TOKENS["<|endoftext|>"] # Qwen2.5-VL tokenizer pad token @dataclass(frozen=True) class RDT2Config: vision: VisionConfig = field(default_factory=VisionConfig) lm: LMConfig = field(default_factory=LMConfig) expert: ExpertConfig = field(default_factory=ExpertConfig) proc: ProcessorConfig = field(default_factory=ProcessorConfig) CONFIG = RDT2Config()