File size: 6,901 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | # SPDX-License-Identifier: Apache-2.0
"""Static configuration of RDT2-FM (robotics-diffusion-transformer/RDT2-FM on the RDT2-VQ Qwen2.5-VL-7B backbone).
Every number here was read from the pinned files (research/rdt2/SPEC.md section 1 cites them):
RDT2-VQ config.json @ ec7e7f3a, RDT2-FM config.json @ f803808e, thu-ml/RDT2 configs/rdt/post_train.yaml @ 0797b4c6,
Qwen/Qwen2.5-VL-7B-Instruct preprocessor_config.json / tokenizer @ cc594898.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Tuple
# ---------------------------------------------------------------------------------------------------------------
# Pins
# ---------------------------------------------------------------------------------------------------------------
VLM_REPO = "robotics-diffusion-transformer/RDT2-VQ"
VLM_REVISION = "ec7e7f3aa4259224481c3cbd5816cc3f35354bec"
EXPERT_REPO = "robotics-diffusion-transformer/RDT2-FM"
EXPERT_REVISION = "f803808e3f79c77bba8bb4f3326e83d04e60783a"
NORMALIZER_REPO = "robotics-diffusion-transformer/RVQActionTokenizer"
NORMALIZER_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec"
NORMALIZER_FILE = "umi_normalizer_wo_downsample_indentity_rot.pt"
PROCESSOR_REPO = "Qwen/Qwen2.5-VL-7B-Instruct"
PROCESSOR_REVISION = "cc594898137f460bfe9f0759e9844b3ce807cfb5"
OFFICIAL_CODE = ("github.com/thu-ml/RDT2", "0797b4c65e588e088d41602685e00dc2bc95852f")
def _dev_workspace() -> Optional[Path]:
"""The development workspace when this file sits in its checkout
(``<workspace>/bundles/rdt2-base-p150/code/tt_rdt2/reference/config.py``), else None (the container image, a pip
install or a download of the published package)."""
p = Path(__file__).resolve().parents
if len(p) > 5 and p[2].name == "code" and p[4].name == "bundles" and (p[5] / "assets").is_dir():
return p[5]
return None
DEV_WORKSPACE = _dev_workspace()
# Local copies of the pinned weights and the official golden runs, used by the development tests and scripts only:
# RDT2_ASSETS_DIR, else <workspace>/assets/rdt2 in the development checkout. Without either, the defaults below name
# no existing directory: the API resolves the pinned files from the Hugging Face cache and the golden tests skip.
_WS_ASSETS = os.environ.get("RDT2_ASSETS_DIR") or (str(DEV_WORKSPACE / "assets" / "rdt2") if DEV_WORKSPACE
else "<RDT2_ASSETS_DIR unset>")
DEFAULT_DIRS = {
"vlm": os.environ.get("RDT2_VLM_DIR", f"{_WS_ASSETS}/hf_RDT2-VQ_ec7e7f3a"),
"expert": os.environ.get("RDT2_EXPERT_DIR", f"{_WS_ASSETS}/hf_RDT2-FM_f803808e"),
"normalizer": os.environ.get("RDT2_NORMALIZER_DIR", f"{_WS_ASSETS}/hf_RVQActionTokenizer_a6a614ae_normalizer_only"),
"processor": os.environ.get("RDT2_PROCESSOR_DIR", f"{_WS_ASSETS}/hf_Qwen2.5-VL-7B-Instruct_cc594898_processor_only"),
"golden": os.environ.get("RDT2_GOLDEN_DIR", f"{_WS_ASSETS}/golden"),
}
# ---------------------------------------------------------------------------------------------------------------
# Vision tower (Qwen2.5-VL ViT)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class VisionConfig:
depth: int = 32
hidden: int = 1280
heads: int = 16
head_dim: int = 80
mlp: int = 3420
patch: int = 14
temporal_patch: int = 2
merge: int = 2
window: int = 112 # pixels -> 112 / 14 / 2 = 4 merged tokens per window side (8x8 = 64 patches)
fullatt_blocks: Tuple[int, ...] = (7, 15, 23, 31)
out_hidden: int = 3584
rope_theta: float = 10000.0
norm_eps: float = 1e-6
in_dim: int = 3 * 2 * 14 * 14 # 1176 = C x T x 14 x 14 per patch row of pixel_values
# ---------------------------------------------------------------------------------------------------------------
# Language model (Qwen2.5-VL-7B decoder; RDT2-FM uses layers 0..13 only)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class LMConfig:
hidden: int = 3584
layers: int = 28
heads: int = 28
kv_heads: int = 4
head_dim: int = 128
mlp: int = 18944
vocab: int = 152064
rope_theta: float = 1_000_000.0
mrope_section: Tuple[int, int, int] = (16, 24, 24)
norm_eps: float = 1e-6
selected_layers: Tuple[int, ...] = tuple(range(14)) # post_train.yaml model.selected_layers
# ---------------------------------------------------------------------------------------------------------------
# Action expert (RDT, flow matching)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class ExpertConfig:
hidden: int = 1024
depth: int = 14
heads: int = 8
kv_heads: int = 4
head_dim: int = 128
ffn: int = 2816 # llama rule: int(2 * 4096 / 3) = 2730 -> multiple_of 256 -> 2816
final_mlp: int = 4096
registers: int = 4
horizon: int = 24
action_dim: int = 20
state_dim: int = 20
norm_eps: float = 1e-5
freq_dim: int = 256
max_period: float = 10000.0
num_steps: int = 5
@property
def tokens(self) -> int:
return self.horizon + self.registers # 28
# ---------------------------------------------------------------------------------------------------------------
# Pre-processing / prompt
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class ProcessorConfig:
image_mean: Tuple[float, float, float] = (0.48145466, 0.4578275, 0.40821073)
image_std: Tuple[float, float, float] = (0.26862954, 0.26130258, 0.27577711)
rescale_factor: float = 0.00392156862745098
min_pixels: int = 3136
max_pixels: int = 12845056
camera_hw: Tuple[int, int] = (384, 384)
camera_names: Tuple[str, ...] = ("left_stereo", "right_stereo") # concatenated left -> right (post_train.yaml)
system_prompt: str = "You are a helpful assistant."
suffix: str = "<|im_start|>assistant\n<|quad_start|>"
TOKENS = {
"<|endoftext|>": 151643,
"<|im_start|>": 151644,
"<|im_end|>": 151645,
"<|quad_start|>": 151650,
"<|quad_end|>": 151651,
"<|vision_start|>": 151652,
"<|vision_end|>": 151653,
"<|image_pad|>": 151655,
}
IMAGE_TOKEN_ID = TOKENS["<|image_pad|>"]
VISION_START_ID = TOKENS["<|vision_start|>"]
PAD_TOKEN_ID = TOKENS["<|endoftext|>"] # Qwen2.5-VL tokenizer pad token
@dataclass(frozen=True)
class RDT2Config:
vision: VisionConfig = field(default_factory=VisionConfig)
lm: LMConfig = field(default_factory=LMConfig)
expert: ExpertConfig = field(default_factory=ExpertConfig)
proc: ProcessorConfig = field(default_factory=ProcessorConfig)
CONFIG = RDT2Config()
|