changh95's picture
tt-model push rdt2-base-p150 (container)
c869f1f verified
Raw History Blame Contribute Delete
8.32 kB
# SPDX-License-Identifier: Apache-2.0
"""Camera count and flow-matching step count of a served RDT2-FM configuration (round 4, stage Config).
Both are fixed when the model is built (``RDT2_NUM_CAMERAS`` 1..4, default 2; ``RDT2_NUM_STEPS`` 1..16, default 5,
read once at start, or the Python API's ``num_cameras`` / ``num_steps``); per request only the prompt bucket changes.
Camera count -> static geometry, derived exactly as the official preprocessing does it
(thu-ml/RDT2 ``models/rdt_inferencer.py``: ``np.concatenate([obs[cam] for cam in camera_names], axis=1)``, then the
Qwen2.5-VL processor's ``smart_resize(factor=28)``): N cameras of 384 x 384 are one 384 x 384N image, resized to
the grid below; the ViT sees ``gh x gw`` patches (window attention over 4 x 4 merged-token windows), the LM
``gh x gw / 4`` image tokens.
cameras concatenated resized patch grid patches image tokens prompt fixed buckets (instruction tokens)
1 384 x 384 392 x 392 28 x 28 784 196 218 256 / 288 / 320 (38 / 70 / 102)
2 384 x 768 392 x 756 28 x 54 1512 378 400 416 / 448 / 512 (16 / 48 / 112)
3 384 x 1152 392 x 1148 28 x 82 2296 574 596 640 / 672 / 736 (44 / 76 / 140)
4 384 x 1536 392 x 1540 28 x 110 3080 770 792 832 / 864 / 928 (40 / 72 / 136)
Buckets: ``round_up(fixed + c, 32)`` for the instruction capacities c = 16 / 48 / 112 of the 2-camera package
(fixed = 15 head tokens + the image tokens + 7 tail tokens), so 2 cameras keep 416 / 448 / 512 exactly. One exception:
1 camera's largest bucket is 320 (not 352) because the EXPERT megakernel splits the keys of a bucket into 4 chunks of
at most 4 tiles per head and every chunk must start below the shortest prompt (15 + 196 keys always valid): 352 keys
(11 tiles) cannot be split that way, 320 can.
Camera names (the order of the concatenation, left to right): 2 cameras = the checkpoint's ``left_stereo``,
``right_stereo`` (configs/rdt/post_train.yaml); other counts ``left_stereo`` (1), then ``right_stereo``, ``camera_2``,
``camera_3``. The shipped checkpoints were trained with 2 cameras only: other counts run the same weights on a
different image (the official code accepts any ``camera_names``), validated against the official inferencer with
the same configuration, not against any training data.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from functools import lru_cache
from typing import Mapping, Optional, Tuple
import torch
from .config import CONFIG
V = CONFIG.vision
NUM_CAMERAS_RANGE = (1, 4)
NUM_STEPS_RANGE = (1, 16)
DEFAULT_NUM_CAMERAS = len(CONFIG.proc.camera_names) # 2
DEFAULT_NUM_STEPS = CONFIG.expert.num_steps # 5
PROMPT_HEAD_TOKENS = 15 # system turn + user header + <|vision_start|>
PROMPT_TAIL_TOKENS = 7 # <|vision_end|> ... <|im_end|>\n<|im_start|>assistant\n<|quad_start|> (without instruction)
INSTRUCTION_CAPS = (16, 48, 112)
CAMERA_HW = (384, 384) # the official per-camera image (deploy / dataset)
EXTRA_CAMERA_NAMES = ("camera_2", "camera_3")
# Bucket exceptions (see the module docstring): the EXPERT megakernel's cross-attention chunk rule
BUCKET_OVERRIDES = {1: (256, 288, 320)}
def _round_up(x: int, m: int) -> int:
return -(-int(x) // m) * m
def camera_names(n: int) -> Tuple[str, ...]:
official = tuple(CONFIG.proc.camera_names)
return (official + EXTRA_CAMERA_NAMES)[:n]
@dataclass(frozen=True)
class Geometry:
"""Static image / prompt geometry of one camera count (every number derived from the official preprocessing)."""
num_cameras: int
camera_names: Tuple[str, ...]
camera_hw: Tuple[int, int]
image_hw: Tuple[int, int] # smart-resized concatenation (rows, cols)
grid: Tuple[int, int] # patch grid (gh, gw)
n_patch: int # ViT rows (patches)
n_rows: int # padded ViT rows on the device (multiple of 128: the merged rows are whole tiles)
n_img: int # image tokens (merged patches)
n_merged: int # padded merged rows (n_rows / 4)
prompt_fixed_tokens: int # every token of the prompt except the instruction
buckets: Tuple[int, ...] # resident prompt buckets
@property
def grid_thw(self) -> torch.Tensor:
return torch.tensor([[1, self.grid[0], self.grid[1]]])
@property
def image_start(self) -> int:
return PROMPT_HEAD_TOKENS
@property
def min_valid_keys(self) -> int:
"""Keys valid in every prompt (head tokens + image tokens): the EXPERT megakernel's chunk bound."""
return PROMPT_HEAD_TOKENS + self.n_img
@property
def is_default(self) -> bool:
return self.num_cameras == DEFAULT_NUM_CAMERAS
def max_instruction_tokens(self) -> dict:
return {int(L): int(L) - self.prompt_fixed_tokens for L in self.buckets}
def as_dict(self) -> dict:
return {"num_cameras": self.num_cameras, "camera_names": list(self.camera_names),
"camera_hw": list(self.camera_hw), "concatenated_hw": [self.camera_hw[0],
self.camera_hw[1] * self.num_cameras],
"resized_hw": list(self.image_hw), "patch_grid": list(self.grid), "patches": self.n_patch,
"image_tokens": self.n_img, "prompt_fixed_tokens": self.prompt_fixed_tokens,
"prompt_buckets": list(self.buckets),
"max_instruction_tokens": {str(k): v for k, v in self.max_instruction_tokens().items()}}
def check_num_cameras(n) -> int:
if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit():
raise ValueError(f"num_cameras must be an integer in {list(range(NUM_CAMERAS_RANGE[0], NUM_CAMERAS_RANGE[1] + 1))}"
f", got {n!r}")
n = int(str(n).strip())
if not NUM_CAMERAS_RANGE[0] <= n <= NUM_CAMERAS_RANGE[1]:
raise ValueError(f"num_cameras must be in {NUM_CAMERAS_RANGE[0]}..{NUM_CAMERAS_RANGE[1]}, got {n}")
return n
def check_num_steps(n) -> int:
if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit():
raise ValueError(f"num_steps must be an integer in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n!r}")
n = int(str(n).strip())
if not NUM_STEPS_RANGE[0] <= n <= NUM_STEPS_RANGE[1]:
raise ValueError(f"num_steps must be in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n}")
return n
@lru_cache(maxsize=8)
def geometry(num_cameras: int = DEFAULT_NUM_CAMERAS) -> Geometry:
from .preprocess import smart_resize
n = check_num_cameras(num_cameras)
ch, cw = CAMERA_HW
rh, rw = smart_resize(ch, cw * n, factor=V.patch * V.merge)
gh, gw = rh // V.patch, rw // V.patch
n_patch = gh * gw
n_rows = _round_up(n_patch, 128)
n_img = n_patch // (V.merge ** 2)
fixed = PROMPT_HEAD_TOKENS + n_img + PROMPT_TAIL_TOKENS
buckets = BUCKET_OVERRIDES.get(n) or tuple(_round_up(fixed + c, 32) for c in INSTRUCTION_CAPS)
return Geometry(num_cameras=n, camera_names=camera_names(n), camera_hw=CAMERA_HW, image_hw=(rh, rw),
grid=(gh, gw), n_patch=n_patch, n_rows=n_rows, n_img=n_img, n_merged=n_rows // 4,
prompt_fixed_tokens=fixed, buckets=tuple(buckets))
GEOM2 = geometry(DEFAULT_NUM_CAMERAS) # the checkpoint's configuration (and the VQ profile's)
def config_from_env(env: Optional[Mapping[str, str]] = None) -> Tuple[int, int]:
"""(num_cameras, num_steps) from RDT2_NUM_CAMERAS / RDT2_NUM_STEPS (unset or empty = the defaults 2 / 5).
Raises ValueError for an unsupported value (never silently replaced)."""
env = os.environ if env is None else env
nc = (env.get("RDT2_NUM_CAMERAS") or "").strip() or DEFAULT_NUM_CAMERAS
ns = (env.get("RDT2_NUM_STEPS") or "").strip() or DEFAULT_NUM_STEPS
try:
nc = check_num_cameras(nc)
except ValueError as e:
raise ValueError(f"RDT2_NUM_CAMERAS: {e}") from None
try:
ns = check_num_steps(ns)
except ValueError as e:
raise ValueError(f"RDT2_NUM_STEPS: {e}") from None
return nc, ns