File size: 6,901 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
# SPDX-License-Identifier: Apache-2.0
"""Static configuration of RDT2-FM (robotics-diffusion-transformer/RDT2-FM on the RDT2-VQ Qwen2.5-VL-7B backbone).

Every number here was read from the pinned files (research/rdt2/SPEC.md section 1 cites them):
RDT2-VQ config.json @ ec7e7f3a, RDT2-FM config.json @ f803808e, thu-ml/RDT2 configs/rdt/post_train.yaml @ 0797b4c6,
Qwen/Qwen2.5-VL-7B-Instruct preprocessor_config.json / tokenizer @ cc594898.
"""
from __future__ import annotations

import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Tuple

# ---------------------------------------------------------------------------------------------------------------
# Pins
# ---------------------------------------------------------------------------------------------------------------
VLM_REPO = "robotics-diffusion-transformer/RDT2-VQ"
VLM_REVISION = "ec7e7f3aa4259224481c3cbd5816cc3f35354bec"
EXPERT_REPO = "robotics-diffusion-transformer/RDT2-FM"
EXPERT_REVISION = "f803808e3f79c77bba8bb4f3326e83d04e60783a"
NORMALIZER_REPO = "robotics-diffusion-transformer/RVQActionTokenizer"
NORMALIZER_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec"
NORMALIZER_FILE = "umi_normalizer_wo_downsample_indentity_rot.pt"
PROCESSOR_REPO = "Qwen/Qwen2.5-VL-7B-Instruct"
PROCESSOR_REVISION = "cc594898137f460bfe9f0759e9844b3ce807cfb5"
OFFICIAL_CODE = ("github.com/thu-ml/RDT2", "0797b4c65e588e088d41602685e00dc2bc95852f")


def _dev_workspace() -> Optional[Path]:
    """The development workspace when this file sits in its checkout
    (``<workspace>/bundles/rdt2-base-p150/code/tt_rdt2/reference/config.py``), else None (the container image, a pip
    install or a download of the published package)."""
    p = Path(__file__).resolve().parents
    if len(p) > 5 and p[2].name == "code" and p[4].name == "bundles" and (p[5] / "assets").is_dir():
        return p[5]
    return None


DEV_WORKSPACE = _dev_workspace()
# Local copies of the pinned weights and the official golden runs, used by the development tests and scripts only:
# RDT2_ASSETS_DIR, else <workspace>/assets/rdt2 in the development checkout. Without either, the defaults below name
# no existing directory: the API resolves the pinned files from the Hugging Face cache and the golden tests skip.
_WS_ASSETS = os.environ.get("RDT2_ASSETS_DIR") or (str(DEV_WORKSPACE / "assets" / "rdt2") if DEV_WORKSPACE
                                                   else "<RDT2_ASSETS_DIR unset>")
DEFAULT_DIRS = {
    "vlm": os.environ.get("RDT2_VLM_DIR", f"{_WS_ASSETS}/hf_RDT2-VQ_ec7e7f3a"),
    "expert": os.environ.get("RDT2_EXPERT_DIR", f"{_WS_ASSETS}/hf_RDT2-FM_f803808e"),
    "normalizer": os.environ.get("RDT2_NORMALIZER_DIR", f"{_WS_ASSETS}/hf_RVQActionTokenizer_a6a614ae_normalizer_only"),
    "processor": os.environ.get("RDT2_PROCESSOR_DIR", f"{_WS_ASSETS}/hf_Qwen2.5-VL-7B-Instruct_cc594898_processor_only"),
    "golden": os.environ.get("RDT2_GOLDEN_DIR", f"{_WS_ASSETS}/golden"),
}


# ---------------------------------------------------------------------------------------------------------------
# Vision tower (Qwen2.5-VL ViT)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class VisionConfig:
    depth: int = 32
    hidden: int = 1280
    heads: int = 16
    head_dim: int = 80
    mlp: int = 3420
    patch: int = 14
    temporal_patch: int = 2
    merge: int = 2
    window: int = 112  # pixels -> 112 / 14 / 2 = 4 merged tokens per window side (8x8 = 64 patches)
    fullatt_blocks: Tuple[int, ...] = (7, 15, 23, 31)
    out_hidden: int = 3584
    rope_theta: float = 10000.0
    norm_eps: float = 1e-6
    in_dim: int = 3 * 2 * 14 * 14  # 1176 = C x T x 14 x 14 per patch row of pixel_values


# ---------------------------------------------------------------------------------------------------------------
# Language model (Qwen2.5-VL-7B decoder; RDT2-FM uses layers 0..13 only)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class LMConfig:
    hidden: int = 3584
    layers: int = 28
    heads: int = 28
    kv_heads: int = 4
    head_dim: int = 128
    mlp: int = 18944
    vocab: int = 152064
    rope_theta: float = 1_000_000.0
    mrope_section: Tuple[int, int, int] = (16, 24, 24)
    norm_eps: float = 1e-6
    selected_layers: Tuple[int, ...] = tuple(range(14))  # post_train.yaml model.selected_layers


# ---------------------------------------------------------------------------------------------------------------
# Action expert (RDT, flow matching)
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class ExpertConfig:
    hidden: int = 1024
    depth: int = 14
    heads: int = 8
    kv_heads: int = 4
    head_dim: int = 128
    ffn: int = 2816  # llama rule: int(2 * 4096 / 3) = 2730 -> multiple_of 256 -> 2816
    final_mlp: int = 4096
    registers: int = 4
    horizon: int = 24
    action_dim: int = 20
    state_dim: int = 20
    norm_eps: float = 1e-5
    freq_dim: int = 256
    max_period: float = 10000.0
    num_steps: int = 5

    @property
    def tokens(self) -> int:
        return self.horizon + self.registers  # 28


# ---------------------------------------------------------------------------------------------------------------
# Pre-processing / prompt
# ---------------------------------------------------------------------------------------------------------------
@dataclass(frozen=True)
class ProcessorConfig:
    image_mean: Tuple[float, float, float] = (0.48145466, 0.4578275, 0.40821073)
    image_std: Tuple[float, float, float] = (0.26862954, 0.26130258, 0.27577711)
    rescale_factor: float = 0.00392156862745098
    min_pixels: int = 3136
    max_pixels: int = 12845056
    camera_hw: Tuple[int, int] = (384, 384)
    camera_names: Tuple[str, ...] = ("left_stereo", "right_stereo")  # concatenated left -> right (post_train.yaml)
    system_prompt: str = "You are a helpful assistant."
    suffix: str = "<|im_start|>assistant\n<|quad_start|>"


TOKENS = {
    "<|endoftext|>": 151643,
    "<|im_start|>": 151644,
    "<|im_end|>": 151645,
    "<|quad_start|>": 151650,
    "<|quad_end|>": 151651,
    "<|vision_start|>": 151652,
    "<|vision_end|>": 151653,
    "<|image_pad|>": 151655,
}
IMAGE_TOKEN_ID = TOKENS["<|image_pad|>"]
VISION_START_ID = TOKENS["<|vision_start|>"]
PAD_TOKEN_ID = TOKENS["<|endoftext|>"]  # Qwen2.5-VL tokenizer pad token


@dataclass(frozen=True)
class RDT2Config:
    vision: VisionConfig = field(default_factory=VisionConfig)
    lm: LMConfig = field(default_factory=LMConfig)
    expert: ExpertConfig = field(default_factory=ExpertConfig)
    proc: ProcessorConfig = field(default_factory=ProcessorConfig)


CONFIG = RDT2Config()