File size: 8,321 Bytes
c869f1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | # SPDX-License-Identifier: Apache-2.0
"""Camera count and flow-matching step count of a served RDT2-FM configuration (round 4, stage Config).
Both are fixed when the model is built (``RDT2_NUM_CAMERAS`` 1..4, default 2; ``RDT2_NUM_STEPS`` 1..16, default 5,
read once at start, or the Python API's ``num_cameras`` / ``num_steps``); per request only the prompt bucket changes.
Camera count -> static geometry, derived exactly as the official preprocessing does it
(thu-ml/RDT2 ``models/rdt_inferencer.py``: ``np.concatenate([obs[cam] for cam in camera_names], axis=1)``, then the
Qwen2.5-VL processor's ``smart_resize(factor=28)``): N cameras of 384 x 384 are one 384 x 384N image, resized to
the grid below; the ViT sees ``gh x gw`` patches (window attention over 4 x 4 merged-token windows), the LM
``gh x gw / 4`` image tokens.
cameras concatenated resized patch grid patches image tokens prompt fixed buckets (instruction tokens)
1 384 x 384 392 x 392 28 x 28 784 196 218 256 / 288 / 320 (38 / 70 / 102)
2 384 x 768 392 x 756 28 x 54 1512 378 400 416 / 448 / 512 (16 / 48 / 112)
3 384 x 1152 392 x 1148 28 x 82 2296 574 596 640 / 672 / 736 (44 / 76 / 140)
4 384 x 1536 392 x 1540 28 x 110 3080 770 792 832 / 864 / 928 (40 / 72 / 136)
Buckets: ``round_up(fixed + c, 32)`` for the instruction capacities c = 16 / 48 / 112 of the 2-camera package
(fixed = 15 head tokens + the image tokens + 7 tail tokens), so 2 cameras keep 416 / 448 / 512 exactly. One exception:
1 camera's largest bucket is 320 (not 352) because the EXPERT megakernel splits the keys of a bucket into 4 chunks of
at most 4 tiles per head and every chunk must start below the shortest prompt (15 + 196 keys always valid): 352 keys
(11 tiles) cannot be split that way, 320 can.
Camera names (the order of the concatenation, left to right): 2 cameras = the checkpoint's ``left_stereo``,
``right_stereo`` (configs/rdt/post_train.yaml); other counts ``left_stereo`` (1), then ``right_stereo``, ``camera_2``,
``camera_3``. The shipped checkpoints were trained with 2 cameras only: other counts run the same weights on a
different image (the official code accepts any ``camera_names``), validated against the official inferencer with
the same configuration, not against any training data.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from functools import lru_cache
from typing import Mapping, Optional, Tuple
import torch
from .config import CONFIG
V = CONFIG.vision
NUM_CAMERAS_RANGE = (1, 4)
NUM_STEPS_RANGE = (1, 16)
DEFAULT_NUM_CAMERAS = len(CONFIG.proc.camera_names) # 2
DEFAULT_NUM_STEPS = CONFIG.expert.num_steps # 5
PROMPT_HEAD_TOKENS = 15 # system turn + user header + <|vision_start|>
PROMPT_TAIL_TOKENS = 7 # <|vision_end|> ... <|im_end|>\n<|im_start|>assistant\n<|quad_start|> (without instruction)
INSTRUCTION_CAPS = (16, 48, 112)
CAMERA_HW = (384, 384) # the official per-camera image (deploy / dataset)
EXTRA_CAMERA_NAMES = ("camera_2", "camera_3")
# Bucket exceptions (see the module docstring): the EXPERT megakernel's cross-attention chunk rule
BUCKET_OVERRIDES = {1: (256, 288, 320)}
def _round_up(x: int, m: int) -> int:
return -(-int(x) // m) * m
def camera_names(n: int) -> Tuple[str, ...]:
official = tuple(CONFIG.proc.camera_names)
return (official + EXTRA_CAMERA_NAMES)[:n]
@dataclass(frozen=True)
class Geometry:
"""Static image / prompt geometry of one camera count (every number derived from the official preprocessing)."""
num_cameras: int
camera_names: Tuple[str, ...]
camera_hw: Tuple[int, int]
image_hw: Tuple[int, int] # smart-resized concatenation (rows, cols)
grid: Tuple[int, int] # patch grid (gh, gw)
n_patch: int # ViT rows (patches)
n_rows: int # padded ViT rows on the device (multiple of 128: the merged rows are whole tiles)
n_img: int # image tokens (merged patches)
n_merged: int # padded merged rows (n_rows / 4)
prompt_fixed_tokens: int # every token of the prompt except the instruction
buckets: Tuple[int, ...] # resident prompt buckets
@property
def grid_thw(self) -> torch.Tensor:
return torch.tensor([[1, self.grid[0], self.grid[1]]])
@property
def image_start(self) -> int:
return PROMPT_HEAD_TOKENS
@property
def min_valid_keys(self) -> int:
"""Keys valid in every prompt (head tokens + image tokens): the EXPERT megakernel's chunk bound."""
return PROMPT_HEAD_TOKENS + self.n_img
@property
def is_default(self) -> bool:
return self.num_cameras == DEFAULT_NUM_CAMERAS
def max_instruction_tokens(self) -> dict:
return {int(L): int(L) - self.prompt_fixed_tokens for L in self.buckets}
def as_dict(self) -> dict:
return {"num_cameras": self.num_cameras, "camera_names": list(self.camera_names),
"camera_hw": list(self.camera_hw), "concatenated_hw": [self.camera_hw[0],
self.camera_hw[1] * self.num_cameras],
"resized_hw": list(self.image_hw), "patch_grid": list(self.grid), "patches": self.n_patch,
"image_tokens": self.n_img, "prompt_fixed_tokens": self.prompt_fixed_tokens,
"prompt_buckets": list(self.buckets),
"max_instruction_tokens": {str(k): v for k, v in self.max_instruction_tokens().items()}}
def check_num_cameras(n) -> int:
if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit():
raise ValueError(f"num_cameras must be an integer in {list(range(NUM_CAMERAS_RANGE[0], NUM_CAMERAS_RANGE[1] + 1))}"
f", got {n!r}")
n = int(str(n).strip())
if not NUM_CAMERAS_RANGE[0] <= n <= NUM_CAMERAS_RANGE[1]:
raise ValueError(f"num_cameras must be in {NUM_CAMERAS_RANGE[0]}..{NUM_CAMERAS_RANGE[1]}, got {n}")
return n
def check_num_steps(n) -> int:
if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit():
raise ValueError(f"num_steps must be an integer in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n!r}")
n = int(str(n).strip())
if not NUM_STEPS_RANGE[0] <= n <= NUM_STEPS_RANGE[1]:
raise ValueError(f"num_steps must be in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n}")
return n
@lru_cache(maxsize=8)
def geometry(num_cameras: int = DEFAULT_NUM_CAMERAS) -> Geometry:
from .preprocess import smart_resize
n = check_num_cameras(num_cameras)
ch, cw = CAMERA_HW
rh, rw = smart_resize(ch, cw * n, factor=V.patch * V.merge)
gh, gw = rh // V.patch, rw // V.patch
n_patch = gh * gw
n_rows = _round_up(n_patch, 128)
n_img = n_patch // (V.merge ** 2)
fixed = PROMPT_HEAD_TOKENS + n_img + PROMPT_TAIL_TOKENS
buckets = BUCKET_OVERRIDES.get(n) or tuple(_round_up(fixed + c, 32) for c in INSTRUCTION_CAPS)
return Geometry(num_cameras=n, camera_names=camera_names(n), camera_hw=CAMERA_HW, image_hw=(rh, rw),
grid=(gh, gw), n_patch=n_patch, n_rows=n_rows, n_img=n_img, n_merged=n_rows // 4,
prompt_fixed_tokens=fixed, buckets=tuple(buckets))
GEOM2 = geometry(DEFAULT_NUM_CAMERAS) # the checkpoint's configuration (and the VQ profile's)
def config_from_env(env: Optional[Mapping[str, str]] = None) -> Tuple[int, int]:
"""(num_cameras, num_steps) from RDT2_NUM_CAMERAS / RDT2_NUM_STEPS (unset or empty = the defaults 2 / 5).
Raises ValueError for an unsupported value (never silently replaced)."""
env = os.environ if env is None else env
nc = (env.get("RDT2_NUM_CAMERAS") or "").strip() or DEFAULT_NUM_CAMERAS
ns = (env.get("RDT2_NUM_STEPS") or "").strip() or DEFAULT_NUM_STEPS
try:
nc = check_num_cameras(nc)
except ValueError as e:
raise ValueError(f"RDT2_NUM_CAMERAS: {e}") from None
try:
ns = check_num_steps(ns)
except ValueError as e:
raise ValueError(f"RDT2_NUM_STEPS: {e}") from None
return nc, ns
|