changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
6.9 kB
# SPDX-License-Identifier: Apache-2.0
"""Static configuration of RDT2-FM (robotics-diffusion-transformer/RDT2-FM on the RDT2-VQ Qwen2.5-VL-7B backbone).
Every number here was read from the pinned files (research/rdt2/SPEC.md section 1 cites them):
RDT2-VQ config.json @ ec7e7f3a, RDT2-FM config.json @ f803808e, thu-ml/RDT2 configs/rdt/post_train.yaml @ 0797b4c6,
Qwen/Qwen2.5-VL-7B-Instruct preprocessor_config.json / tokenizer @ cc594898.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Tuple
# ---------------------------------------------------------------------------------------------------------------
# Pins
# ---------------------------------------------------------------------------------------------------------------
VLM_REPO = "robotics-diffusion-transformer/RDT2-VQ"
VLM_REVISION = "ec7e7f3aa4259224481c3cbd5816cc3f35354bec"
EXPERT_REPO = "robotics-diffusion-transformer/RDT2-FM"
EXPERT_REVISION = "f803808e3f79c77bba8bb4f3326e83d04e60783a"
NORMALIZER_REPO = "robotics-diffusion-transformer/RVQActionTokenizer"
NORMALIZER_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec"
NORMALIZER_FILE = "umi_normalizer_wo_downsample_indentity_rot.pt"
PROCESSOR_REPO = "Qwen/Qwen2.5-VL-7B-Instruct"
PROCESSOR_REVISION = "cc594898137f460bfe9f0759e9844b3ce807cfb5"
OFFICIAL_CODE = ("github.com/thu-ml/RDT2", "0797b4c65e588e088d41602685e00dc2bc95852f")
def _dev_workspace() -> Optional[Path]:
"""The development workspace when this file sits in its checkout
(``<workspace>/bundles/rdt2-base-p150/code/tt_rdt2/reference/config.py``), else None (the container image, a pip
install or a download of the published package)."""
p = Path(__file__).resolve().parents
if len(p) > 5 and p[2].name == "code" and p[4].name == "bundles" and (p[5] / "assets").is_dir():
return p[5]
return None
DEV_WORKSPACE = _dev_workspace()
# Local copies of the pinned weights and the official golden runs, used by the development tests and scripts only:
# RDT2_ASSETS_DIR, else <workspace>/assets/rdt2 in the development checkout. Without either, the defaults below name
# no existing directory: the API resolves the pinned files from the Hugging Face cache and the golden tests skip.
_WS_ASSETS = os.environ.get("RDT2_ASSETS_DIR") or (str(DEV_WORKSPACE / "assets" / "rdt2") if DEV_WORKSPACE
else "<RDT2_ASSETS_DIR unset>")
DEFAULT_DIRS = {
"vlm": os.environ.get("RDT2_VLM_DIR", f"{_WS_ASSETS}/hf_RDT2-VQ_ec7e7f3a"),
"expert": os.environ.get("RDT2_EXPERT_DIR", f"{_WS_ASSETS}/hf_RDT2-FM_f803808e"),
"normalizer": os.environ.get("RDT2_NORMALIZER_DIR", f"{_WS_ASSETS}/hf_RVQActionTokenizer_a6a614ae_normalizer_only"),
"processor": os.environ.get("RDT2_PROCESSOR_DIR", f"{_WS_ASSETS}/hf_Qwen2.5-VL-7B-Instruct_cc594898_processor_only"),
"golden": os.environ.get("RDT2_GOLDEN_DIR", f"{_WS_ASSETS}/golden"),
}
# ---------------------------------------------------------------------------------------------------------------
# Vision tower (Qwen2.5-VL ViT)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class VisionConfig:
depth: int = 32
hidden: int = 1280
heads: int = 16
head_dim: int = 80
mlp: int = 3420
patch: int = 14
temporal_patch: int = 2
merge: int = 2
window: int = 112 # pixels -> 112 / 14 / 2 = 4 merged tokens per window side (8x8 = 64 patches)
fullatt_blocks: Tuple[int, ...] = (7, 15, 23, 31)
out_hidden: int = 3584
rope_theta: float = 10000.0
norm_eps: float = 1e-6
in_dim: int = 3 * 2 * 14 * 14 # 1176 = C x T x 14 x 14 per patch row of pixel_values
# ---------------------------------------------------------------------------------------------------------------
# Language model (Qwen2.5-VL-7B decoder; RDT2-FM uses layers 0..13 only)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class LMConfig:
hidden: int = 3584
layers: int = 28
heads: int = 28
kv_heads: int = 4
head_dim: int = 128
mlp: int = 18944
vocab: int = 152064
rope_theta: float = 1_000_000.0
mrope_section: Tuple[int, int, int] = (16, 24, 24)
norm_eps: float = 1e-6
selected_layers: Tuple[int, ...] = tuple(range(14)) # post_train.yaml model.selected_layers
# ---------------------------------------------------------------------------------------------------------------
# Action expert (RDT, flow matching)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class ExpertConfig:
hidden: int = 1024
depth: int = 14
heads: int = 8
kv_heads: int = 4
head_dim: int = 128
ffn: int = 2816 # llama rule: int(2 * 4096 / 3) = 2730 -> multiple_of 256 -> 2816
final_mlp: int = 4096
registers: int = 4
horizon: int = 24
action_dim: int = 20
state_dim: int = 20
norm_eps: float = 1e-5
freq_dim: int = 256
max_period: float = 10000.0
num_steps: int = 5
@property
def tokens(self) -> int:
return self.horizon + self.registers # 28
# ---------------------------------------------------------------------------------------------------------------
# Pre-processing / prompt
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class ProcessorConfig:
image_mean: Tuple[float, float, float] = (0.48145466, 0.4578275, 0.40821073)
image_std: Tuple[float, float, float] = (0.26862954, 0.26130258, 0.27577711)
rescale_factor: float = 0.00392156862745098
min_pixels: int = 3136
max_pixels: int = 12845056
camera_hw: Tuple[int, int] = (384, 384)
camera_names: Tuple[str, ...] = ("left_stereo", "right_stereo") # concatenated left -> right (post_train.yaml)
system_prompt: str = "You are a helpful assistant."
suffix: str = "<|im_start|>assistant\n<|quad_start|>"
TOKENS = {
"<|endoftext|>": 151643,
"<|im_start|>": 151644,
"<|im_end|>": 151645,
"<|quad_start|>": 151650,
"<|quad_end|>": 151651,
"<|vision_start|>": 151652,
"<|vision_end|>": 151653,
"<|image_pad|>": 151655,
}
IMAGE_TOKEN_ID = TOKENS["<|image_pad|>"]
VISION_START_ID = TOKENS["<|vision_start|>"]
PAD_TOKEN_ID = TOKENS["<|endoftext|>"] # Qwen2.5-VL tokenizer pad token
@dataclass(frozen=True)
class RDT2Config:
vision: VisionConfig = field(default_factory=VisionConfig)
lm: LMConfig = field(default_factory=LMConfig)
expert: ExpertConfig = field(default_factory=ExpertConfig)
proc: ProcessorConfig = field(default_factory=ProcessorConfig)
CONFIG = RDT2Config()