# SPDX-License-Identifier: Apache-2.0 """Camera count and flow-matching step count of a served RDT2-FM configuration (round 4, stage Config). Both are fixed when the model is built (``RDT2_NUM_CAMERAS`` 1..4, default 2; ``RDT2_NUM_STEPS`` 1..16, default 5, read once at start, or the Python API's ``num_cameras`` / ``num_steps``); per request only the prompt bucket changes. Camera count -> static geometry, derived exactly as the official preprocessing does it (thu-ml/RDT2 ``models/rdt_inferencer.py``: ``np.concatenate([obs[cam] for cam in camera_names], axis=1)``, then the Qwen2.5-VL processor's ``smart_resize(factor=28)``): N cameras of 384 x 384 are one 384 x 384N image, resized to the grid below; the ViT sees ``gh x gw`` patches (window attention over 4 x 4 merged-token windows), the LM ``gh x gw / 4`` image tokens. cameras concatenated resized patch grid patches image tokens prompt fixed buckets (instruction tokens) 1 384 x 384 392 x 392 28 x 28 784 196 218 256 / 288 / 320 (38 / 70 / 102) 2 384 x 768 392 x 756 28 x 54 1512 378 400 416 / 448 / 512 (16 / 48 / 112) 3 384 x 1152 392 x 1148 28 x 82 2296 574 596 640 / 672 / 736 (44 / 76 / 140) 4 384 x 1536 392 x 1540 28 x 110 3080 770 792 832 / 864 / 928 (40 / 72 / 136) Buckets: ``round_up(fixed + c, 32)`` for the instruction capacities c = 16 / 48 / 112 of the 2-camera package (fixed = 15 head tokens + the image tokens + 7 tail tokens), so 2 cameras keep 416 / 448 / 512 exactly. One exception: 1 camera's largest bucket is 320 (not 352) because the EXPERT megakernel splits the keys of a bucket into 4 chunks of at most 4 tiles per head and every chunk must start below the shortest prompt (15 + 196 keys always valid): 352 keys (11 tiles) cannot be split that way, 320 can. Camera names (the order of the concatenation, left to right): 2 cameras = the checkpoint's ``left_stereo``, ``right_stereo`` (configs/rdt/post_train.yaml); other counts ``left_stereo`` (1), then ``right_stereo``, ``camera_2``, ``camera_3``. The shipped checkpoints were trained with 2 cameras only: other counts run the same weights on a different image (the official code accepts any ``camera_names``), validated against the official inferencer with the same configuration, not against any training data. """ from __future__ import annotations import os from dataclasses import dataclass from functools import lru_cache from typing import Mapping, Optional, Tuple import torch from .config import CONFIG V = CONFIG.vision NUM_CAMERAS_RANGE = (1, 4) NUM_STEPS_RANGE = (1, 16) DEFAULT_NUM_CAMERAS = len(CONFIG.proc.camera_names) # 2 DEFAULT_NUM_STEPS = CONFIG.expert.num_steps # 5 PROMPT_HEAD_TOKENS = 15 # system turn + user header + <|vision_start|> PROMPT_TAIL_TOKENS = 7 # <|vision_end|> ... <|im_end|>\n<|im_start|>assistant\n<|quad_start|> (without instruction) INSTRUCTION_CAPS = (16, 48, 112) CAMERA_HW = (384, 384) # the official per-camera image (deploy / dataset) EXTRA_CAMERA_NAMES = ("camera_2", "camera_3") # Bucket exceptions (see the module docstring): the EXPERT megakernel's cross-attention chunk rule BUCKET_OVERRIDES = {1: (256, 288, 320)} def _round_up(x: int, m: int) -> int: return -(-int(x) // m) * m def camera_names(n: int) -> Tuple[str, ...]: official = tuple(CONFIG.proc.camera_names) return (official + EXTRA_CAMERA_NAMES)[:n] @dataclass(frozen=True) class Geometry: """Static image / prompt geometry of one camera count (every number derived from the official preprocessing).""" num_cameras: int camera_names: Tuple[str, ...] camera_hw: Tuple[int, int] image_hw: Tuple[int, int] # smart-resized concatenation (rows, cols) grid: Tuple[int, int] # patch grid (gh, gw) n_patch: int # ViT rows (patches) n_rows: int # padded ViT rows on the device (multiple of 128: the merged rows are whole tiles) n_img: int # image tokens (merged patches) n_merged: int # padded merged rows (n_rows / 4) prompt_fixed_tokens: int # every token of the prompt except the instruction buckets: Tuple[int, ...] # resident prompt buckets @property def grid_thw(self) -> torch.Tensor: return torch.tensor([[1, self.grid[0], self.grid[1]]]) @property def image_start(self) -> int: return PROMPT_HEAD_TOKENS @property def min_valid_keys(self) -> int: """Keys valid in every prompt (head tokens + image tokens): the EXPERT megakernel's chunk bound.""" return PROMPT_HEAD_TOKENS + self.n_img @property def is_default(self) -> bool: return self.num_cameras == DEFAULT_NUM_CAMERAS def max_instruction_tokens(self) -> dict: return {int(L): int(L) - self.prompt_fixed_tokens for L in self.buckets} def as_dict(self) -> dict: return {"num_cameras": self.num_cameras, "camera_names": list(self.camera_names), "camera_hw": list(self.camera_hw), "concatenated_hw": [self.camera_hw[0], self.camera_hw[1] * self.num_cameras], "resized_hw": list(self.image_hw), "patch_grid": list(self.grid), "patches": self.n_patch, "image_tokens": self.n_img, "prompt_fixed_tokens": self.prompt_fixed_tokens, "prompt_buckets": list(self.buckets), "max_instruction_tokens": {str(k): v for k, v in self.max_instruction_tokens().items()}} def check_num_cameras(n) -> int: if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit(): raise ValueError(f"num_cameras must be an integer in {list(range(NUM_CAMERAS_RANGE[0], NUM_CAMERAS_RANGE[1] + 1))}" f", got {n!r}") n = int(str(n).strip()) if not NUM_CAMERAS_RANGE[0] <= n <= NUM_CAMERAS_RANGE[1]: raise ValueError(f"num_cameras must be in {NUM_CAMERAS_RANGE[0]}..{NUM_CAMERAS_RANGE[1]}, got {n}") return n def check_num_steps(n) -> int: if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit(): raise ValueError(f"num_steps must be an integer in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n!r}") n = int(str(n).strip()) if not NUM_STEPS_RANGE[0] <= n <= NUM_STEPS_RANGE[1]: raise ValueError(f"num_steps must be in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n}") return n @lru_cache(maxsize=8) def geometry(num_cameras: int = DEFAULT_NUM_CAMERAS) -> Geometry: from .preprocess import smart_resize n = check_num_cameras(num_cameras) ch, cw = CAMERA_HW rh, rw = smart_resize(ch, cw * n, factor=V.patch * V.merge) gh, gw = rh // V.patch, rw // V.patch n_patch = gh * gw n_rows = _round_up(n_patch, 128) n_img = n_patch // (V.merge ** 2) fixed = PROMPT_HEAD_TOKENS + n_img + PROMPT_TAIL_TOKENS buckets = BUCKET_OVERRIDES.get(n) or tuple(_round_up(fixed + c, 32) for c in INSTRUCTION_CAPS) return Geometry(num_cameras=n, camera_names=camera_names(n), camera_hw=CAMERA_HW, image_hw=(rh, rw), grid=(gh, gw), n_patch=n_patch, n_rows=n_rows, n_img=n_img, n_merged=n_rows // 4, prompt_fixed_tokens=fixed, buckets=tuple(buckets)) GEOM2 = geometry(DEFAULT_NUM_CAMERAS) # the checkpoint's configuration (and the VQ profile's) def config_from_env(env: Optional[Mapping[str, str]] = None) -> Tuple[int, int]: """(num_cameras, num_steps) from RDT2_NUM_CAMERAS / RDT2_NUM_STEPS (unset or empty = the defaults 2 / 5). Raises ValueError for an unsupported value (never silently replaced).""" env = os.environ if env is None else env nc = (env.get("RDT2_NUM_CAMERAS") or "").strip() or DEFAULT_NUM_CAMERAS ns = (env.get("RDT2_NUM_STEPS") or "").strip() or DEFAULT_NUM_STEPS try: nc = check_num_cameras(nc) except ValueError as e: raise ValueError(f"RDT2_NUM_CAMERAS: {e}") from None try: ns = check_num_steps(ns) except ValueError as e: raise ValueError(f"RDT2_NUM_STEPS: {e}") from None return nc, ns