File size: 8,321 Bytes
c869f1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
# SPDX-License-Identifier: Apache-2.0
"""Camera count and flow-matching step count of a served RDT2-FM configuration (round 4, stage Config).

Both are fixed when the model is built (``RDT2_NUM_CAMERAS`` 1..4, default 2; ``RDT2_NUM_STEPS`` 1..16, default 5,
read once at start, or the Python API's ``num_cameras`` / ``num_steps``); per request only the prompt bucket changes.

Camera count -> static geometry, derived exactly as the official preprocessing does it
(thu-ml/RDT2 ``models/rdt_inferencer.py``: ``np.concatenate([obs[cam] for cam in camera_names], axis=1)``, then the
Qwen2.5-VL processor's ``smart_resize(factor=28)``): N cameras of 384 x 384 are one 384 x 384N image, resized to
the grid below; the ViT sees ``gh x gw`` patches (window attention over 4 x 4 merged-token windows), the LM
``gh x gw / 4`` image tokens.

    cameras  concatenated  resized      patch grid  patches  image tokens  prompt fixed  buckets (instruction tokens)
    1        384 x  384    392 x  392   28 x  28      784      196          218           256 / 288 / 320 (38 / 70 / 102)
    2        384 x  768    392 x  756   28 x  54     1512      378          400           416 / 448 / 512 (16 / 48 / 112)
    3        384 x 1152    392 x 1148   28 x  82     2296      574          596           640 / 672 / 736 (44 / 76 / 140)
    4        384 x 1536    392 x 1540   28 x 110     3080      770          792           832 / 864 / 928 (40 / 72 / 136)

Buckets: ``round_up(fixed + c, 32)`` for the instruction capacities c = 16 / 48 / 112 of the 2-camera package
(fixed = 15 head tokens + the image tokens + 7 tail tokens), so 2 cameras keep 416 / 448 / 512 exactly. One exception:
1 camera's largest bucket is 320 (not 352) because the EXPERT megakernel splits the keys of a bucket into 4 chunks of
at most 4 tiles per head and every chunk must start below the shortest prompt (15 + 196 keys always valid): 352 keys
(11 tiles) cannot be split that way, 320 can.

Camera names (the order of the concatenation, left to right): 2 cameras = the checkpoint's ``left_stereo``,
``right_stereo`` (configs/rdt/post_train.yaml); other counts ``left_stereo`` (1), then ``right_stereo``, ``camera_2``,
``camera_3``. The shipped checkpoints were trained with 2 cameras only: other counts run the same weights on a
different image (the official code accepts any ``camera_names``), validated against the official inferencer with
the same configuration, not against any training data.
"""
from __future__ import annotations

import os
from dataclasses import dataclass
from functools import lru_cache
from typing import Mapping, Optional, Tuple

import torch

from .config import CONFIG

V = CONFIG.vision
NUM_CAMERAS_RANGE = (1, 4)
NUM_STEPS_RANGE = (1, 16)
DEFAULT_NUM_CAMERAS = len(CONFIG.proc.camera_names)  # 2
DEFAULT_NUM_STEPS = CONFIG.expert.num_steps          # 5
PROMPT_HEAD_TOKENS = 15   # system turn + user header + <|vision_start|>
PROMPT_TAIL_TOKENS = 7    # <|vision_end|> ... <|im_end|>\n<|im_start|>assistant\n<|quad_start|> (without instruction)
INSTRUCTION_CAPS = (16, 48, 112)
CAMERA_HW = (384, 384)    # the official per-camera image (deploy / dataset)
EXTRA_CAMERA_NAMES = ("camera_2", "camera_3")
# Bucket exceptions (see the module docstring): the EXPERT megakernel's cross-attention chunk rule
BUCKET_OVERRIDES = {1: (256, 288, 320)}


def _round_up(x: int, m: int) -> int:
    return -(-int(x) // m) * m


def camera_names(n: int) -> Tuple[str, ...]:
    official = tuple(CONFIG.proc.camera_names)
    return (official + EXTRA_CAMERA_NAMES)[:n]


@dataclass(frozen=True)
class Geometry:
    """Static image / prompt geometry of one camera count (every number derived from the official preprocessing)."""

    num_cameras: int
    camera_names: Tuple[str, ...]
    camera_hw: Tuple[int, int]
    image_hw: Tuple[int, int]      # smart-resized concatenation (rows, cols)
    grid: Tuple[int, int]          # patch grid (gh, gw)
    n_patch: int                   # ViT rows (patches)
    n_rows: int                    # padded ViT rows on the device (multiple of 128: the merged rows are whole tiles)
    n_img: int                     # image tokens (merged patches)
    n_merged: int                  # padded merged rows (n_rows / 4)
    prompt_fixed_tokens: int       # every token of the prompt except the instruction
    buckets: Tuple[int, ...]       # resident prompt buckets

    @property
    def grid_thw(self) -> torch.Tensor:
        return torch.tensor([[1, self.grid[0], self.grid[1]]])

    @property
    def image_start(self) -> int:
        return PROMPT_HEAD_TOKENS

    @property
    def min_valid_keys(self) -> int:
        """Keys valid in every prompt (head tokens + image tokens): the EXPERT megakernel's chunk bound."""
        return PROMPT_HEAD_TOKENS + self.n_img

    @property
    def is_default(self) -> bool:
        return self.num_cameras == DEFAULT_NUM_CAMERAS

    def max_instruction_tokens(self) -> dict:
        return {int(L): int(L) - self.prompt_fixed_tokens for L in self.buckets}

    def as_dict(self) -> dict:
        return {"num_cameras": self.num_cameras, "camera_names": list(self.camera_names),
                "camera_hw": list(self.camera_hw), "concatenated_hw": [self.camera_hw[0],
                                                                      self.camera_hw[1] * self.num_cameras],
                "resized_hw": list(self.image_hw), "patch_grid": list(self.grid), "patches": self.n_patch,
                "image_tokens": self.n_img, "prompt_fixed_tokens": self.prompt_fixed_tokens,
                "prompt_buckets": list(self.buckets),
                "max_instruction_tokens": {str(k): v for k, v in self.max_instruction_tokens().items()}}


def check_num_cameras(n) -> int:
    if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit():
        raise ValueError(f"num_cameras must be an integer in {list(range(NUM_CAMERAS_RANGE[0], NUM_CAMERAS_RANGE[1] + 1))}"
                         f", got {n!r}")
    n = int(str(n).strip())
    if not NUM_CAMERAS_RANGE[0] <= n <= NUM_CAMERAS_RANGE[1]:
        raise ValueError(f"num_cameras must be in {NUM_CAMERAS_RANGE[0]}..{NUM_CAMERAS_RANGE[1]}, got {n}")
    return n


def check_num_steps(n) -> int:
    if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit():
        raise ValueError(f"num_steps must be an integer in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n!r}")
    n = int(str(n).strip())
    if not NUM_STEPS_RANGE[0] <= n <= NUM_STEPS_RANGE[1]:
        raise ValueError(f"num_steps must be in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n}")
    return n


@lru_cache(maxsize=8)
def geometry(num_cameras: int = DEFAULT_NUM_CAMERAS) -> Geometry:
    from .preprocess import smart_resize

    n = check_num_cameras(num_cameras)
    ch, cw = CAMERA_HW
    rh, rw = smart_resize(ch, cw * n, factor=V.patch * V.merge)
    gh, gw = rh // V.patch, rw // V.patch
    n_patch = gh * gw
    n_rows = _round_up(n_patch, 128)
    n_img = n_patch // (V.merge ** 2)
    fixed = PROMPT_HEAD_TOKENS + n_img + PROMPT_TAIL_TOKENS
    buckets = BUCKET_OVERRIDES.get(n) or tuple(_round_up(fixed + c, 32) for c in INSTRUCTION_CAPS)
    return Geometry(num_cameras=n, camera_names=camera_names(n), camera_hw=CAMERA_HW, image_hw=(rh, rw),
                    grid=(gh, gw), n_patch=n_patch, n_rows=n_rows, n_img=n_img, n_merged=n_rows // 4,
                    prompt_fixed_tokens=fixed, buckets=tuple(buckets))


GEOM2 = geometry(DEFAULT_NUM_CAMERAS)  # the checkpoint's configuration (and the VQ profile's)


def config_from_env(env: Optional[Mapping[str, str]] = None) -> Tuple[int, int]:
    """(num_cameras, num_steps) from RDT2_NUM_CAMERAS / RDT2_NUM_STEPS (unset or empty = the defaults 2 / 5).
    Raises ValueError for an unsupported value (never silently replaced)."""
    env = os.environ if env is None else env
    nc = (env.get("RDT2_NUM_CAMERAS") or "").strip() or DEFAULT_NUM_CAMERAS
    ns = (env.get("RDT2_NUM_STEPS") or "").strip() or DEFAULT_NUM_STEPS
    try:
        nc = check_num_cameras(nc)
    except ValueError as e:
        raise ValueError(f"RDT2_NUM_CAMERAS: {e}") from None
    try:
        ns = check_num_steps(ns)
    except ValueError as e:
        raise ValueError(f"RDT2_NUM_STEPS: {e}") from None
    return nc, ns