Download code/tt_rdt2/reference/config.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 6.9 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/config.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/config.py
-
curl -L -o config.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/config.py
6.9 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Static configuration of RDT2-FM (robotics-diffusion-transformer/RDT2-FM on the RDT2-VQ Qwen2.5-VL-7B backbone). | |
| Every number here was read from the pinned files (research/rdt2/SPEC.md section 1 cites them): | |
| RDT2-VQ config.json @ ec7e7f3a, RDT2-FM config.json @ f803808e, thu-ml/RDT2 configs/rdt/post_train.yaml @ 0797b4c6, | |
| Qwen/Qwen2.5-VL-7B-Instruct preprocessor_config.json / tokenizer @ cc594898. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Optional, Tuple | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Pins | |
| # --------------------------------------------------------------------------------------------------------------- | |
| VLM_REPO = "robotics-diffusion-transformer/RDT2-VQ" | |
| VLM_REVISION = "ec7e7f3aa4259224481c3cbd5816cc3f35354bec" | |
| EXPERT_REPO = "robotics-diffusion-transformer/RDT2-FM" | |
| EXPERT_REVISION = "f803808e3f79c77bba8bb4f3326e83d04e60783a" | |
| NORMALIZER_REPO = "robotics-diffusion-transformer/RVQActionTokenizer" | |
| NORMALIZER_REVISION = "a6a614ae049cc3b9fa6bd4f5fadaeee3375d4dec" | |
| NORMALIZER_FILE = "umi_normalizer_wo_downsample_indentity_rot.pt" | |
| PROCESSOR_REPO = "Qwen/Qwen2.5-VL-7B-Instruct" | |
| PROCESSOR_REVISION = "cc594898137f460bfe9f0759e9844b3ce807cfb5" | |
| OFFICIAL_CODE = ("github.com/thu-ml/RDT2", "0797b4c65e588e088d41602685e00dc2bc95852f") | |
| def _dev_workspace() -> Optional[Path]: | |
| """The development workspace when this file sits in its checkout | |
| (``<workspace>/bundles/rdt2-base-p150/code/tt_rdt2/reference/config.py``), else None (the container image, a pip | |
| install or a download of the published package).""" | |
| p = Path(__file__).resolve().parents | |
| if len(p) > 5 and p[2].name == "code" and p[4].name == "bundles" and (p[5] / "assets").is_dir(): | |
| return p[5] | |
| return None | |
| DEV_WORKSPACE = _dev_workspace() | |
| # Local copies of the pinned weights and the official golden runs, used by the development tests and scripts only: | |
| # RDT2_ASSETS_DIR, else <workspace>/assets/rdt2 in the development checkout. Without either, the defaults below name | |
| # no existing directory: the API resolves the pinned files from the Hugging Face cache and the golden tests skip. | |
| _WS_ASSETS = os.environ.get("RDT2_ASSETS_DIR") or (str(DEV_WORKSPACE / "assets" / "rdt2") if DEV_WORKSPACE | |
| else "<RDT2_ASSETS_DIR unset>") | |
| DEFAULT_DIRS = { | |
| "vlm": os.environ.get("RDT2_VLM_DIR", f"{_WS_ASSETS}/hf_RDT2-VQ_ec7e7f3a"), | |
| "expert": os.environ.get("RDT2_EXPERT_DIR", f"{_WS_ASSETS}/hf_RDT2-FM_f803808e"), | |
| "normalizer": os.environ.get("RDT2_NORMALIZER_DIR", f"{_WS_ASSETS}/hf_RVQActionTokenizer_a6a614ae_normalizer_only"), | |
| "processor": os.environ.get("RDT2_PROCESSOR_DIR", f"{_WS_ASSETS}/hf_Qwen2.5-VL-7B-Instruct_cc594898_processor_only"), | |
| "golden": os.environ.get("RDT2_GOLDEN_DIR", f"{_WS_ASSETS}/golden"), | |
| } | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Vision tower (Qwen2.5-VL ViT) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| class VisionConfig: | |
| depth: int = 32 | |
| hidden: int = 1280 | |
| heads: int = 16 | |
| head_dim: int = 80 | |
| mlp: int = 3420 | |
| patch: int = 14 | |
| temporal_patch: int = 2 | |
| merge: int = 2 | |
| window: int = 112 # pixels -> 112 / 14 / 2 = 4 merged tokens per window side (8x8 = 64 patches) | |
| fullatt_blocks: Tuple[int, ...] = (7, 15, 23, 31) | |
| out_hidden: int = 3584 | |
| rope_theta: float = 10000.0 | |
| norm_eps: float = 1e-6 | |
| in_dim: int = 3 * 2 * 14 * 14 # 1176 = C x T x 14 x 14 per patch row of pixel_values | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Language model (Qwen2.5-VL-7B decoder; RDT2-FM uses layers 0..13 only) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| class LMConfig: | |
| hidden: int = 3584 | |
| layers: int = 28 | |
| heads: int = 28 | |
| kv_heads: int = 4 | |
| head_dim: int = 128 | |
| mlp: int = 18944 | |
| vocab: int = 152064 | |
| rope_theta: float = 1_000_000.0 | |
| mrope_section: Tuple[int, int, int] = (16, 24, 24) | |
| norm_eps: float = 1e-6 | |
| selected_layers: Tuple[int, ...] = tuple(range(14)) # post_train.yaml model.selected_layers | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Action expert (RDT, flow matching) | |
| # --------------------------------------------------------------------------------------------------------------- | |
| class ExpertConfig: | |
| hidden: int = 1024 | |
| depth: int = 14 | |
| heads: int = 8 | |
| kv_heads: int = 4 | |
| head_dim: int = 128 | |
| ffn: int = 2816 # llama rule: int(2 * 4096 / 3) = 2730 -> multiple_of 256 -> 2816 | |
| final_mlp: int = 4096 | |
| registers: int = 4 | |
| horizon: int = 24 | |
| action_dim: int = 20 | |
| state_dim: int = 20 | |
| norm_eps: float = 1e-5 | |
| freq_dim: int = 256 | |
| max_period: float = 10000.0 | |
| num_steps: int = 5 | |
| def tokens(self) -> int: | |
| return self.horizon + self.registers # 28 | |
| # --------------------------------------------------------------------------------------------------------------- | |
| # Pre-processing / prompt | |
| # --------------------------------------------------------------------------------------------------------------- | |
| class ProcessorConfig: | |
| image_mean: Tuple[float, float, float] = (0.48145466, 0.4578275, 0.40821073) | |
| image_std: Tuple[float, float, float] = (0.26862954, 0.26130258, 0.27577711) | |
| rescale_factor: float = 0.00392156862745098 | |
| min_pixels: int = 3136 | |
| max_pixels: int = 12845056 | |
| camera_hw: Tuple[int, int] = (384, 384) | |
| camera_names: Tuple[str, ...] = ("left_stereo", "right_stereo") # concatenated left -> right (post_train.yaml) | |
| system_prompt: str = "You are a helpful assistant." | |
| suffix: str = "<|im_start|>assistant\n<|quad_start|>" | |
| TOKENS = { | |
| "<|endoftext|>": 151643, | |
| "<|im_start|>": 151644, | |
| "<|im_end|>": 151645, | |
| "<|quad_start|>": 151650, | |
| "<|quad_end|>": 151651, | |
| "<|vision_start|>": 151652, | |
| "<|vision_end|>": 151653, | |
| "<|image_pad|>": 151655, | |
| } | |
| IMAGE_TOKEN_ID = TOKENS["<|image_pad|>"] | |
| VISION_START_ID = TOKENS["<|vision_start|>"] | |
| PAD_TOKEN_ID = TOKENS["<|endoftext|>"] # Qwen2.5-VL tokenizer pad token | |
| class RDT2Config: | |
| vision: VisionConfig = field(default_factory=VisionConfig) | |
| lm: LMConfig = field(default_factory=LMConfig) | |
| expert: ExpertConfig = field(default_factory=ExpertConfig) | |
| proc: ProcessorConfig = field(default_factory=ProcessorConfig) | |
| CONFIG = RDT2Config() | |