Download code/tt_rdt2/reference/geometry.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 8.32 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/geometry.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/reference/geometry.py
-
curl -L -o geometry.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/reference/geometry.py
8.32 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Camera count and flow-matching step count of a served RDT2-FM configuration (round 4, stage Config). | |
| Both are fixed when the model is built (``RDT2_NUM_CAMERAS`` 1..4, default 2; ``RDT2_NUM_STEPS`` 1..16, default 5, | |
| read once at start, or the Python API's ``num_cameras`` / ``num_steps``); per request only the prompt bucket changes. | |
| Camera count -> static geometry, derived exactly as the official preprocessing does it | |
| (thu-ml/RDT2 ``models/rdt_inferencer.py``: ``np.concatenate([obs[cam] for cam in camera_names], axis=1)``, then the | |
| Qwen2.5-VL processor's ``smart_resize(factor=28)``): N cameras of 384 x 384 are one 384 x 384N image, resized to | |
| the grid below; the ViT sees ``gh x gw`` patches (window attention over 4 x 4 merged-token windows), the LM | |
| ``gh x gw / 4`` image tokens. | |
| cameras concatenated resized patch grid patches image tokens prompt fixed buckets (instruction tokens) | |
| 1 384 x 384 392 x 392 28 x 28 784 196 218 256 / 288 / 320 (38 / 70 / 102) | |
| 2 384 x 768 392 x 756 28 x 54 1512 378 400 416 / 448 / 512 (16 / 48 / 112) | |
| 3 384 x 1152 392 x 1148 28 x 82 2296 574 596 640 / 672 / 736 (44 / 76 / 140) | |
| 4 384 x 1536 392 x 1540 28 x 110 3080 770 792 832 / 864 / 928 (40 / 72 / 136) | |
| Buckets: ``round_up(fixed + c, 32)`` for the instruction capacities c = 16 / 48 / 112 of the 2-camera package | |
| (fixed = 15 head tokens + the image tokens + 7 tail tokens), so 2 cameras keep 416 / 448 / 512 exactly. One exception: | |
| 1 camera's largest bucket is 320 (not 352) because the EXPERT megakernel splits the keys of a bucket into 4 chunks of | |
| at most 4 tiles per head and every chunk must start below the shortest prompt (15 + 196 keys always valid): 352 keys | |
| (11 tiles) cannot be split that way, 320 can. | |
| Camera names (the order of the concatenation, left to right): 2 cameras = the checkpoint's ``left_stereo``, | |
| ``right_stereo`` (configs/rdt/post_train.yaml); other counts ``left_stereo`` (1), then ``right_stereo``, ``camera_2``, | |
| ``camera_3``. The shipped checkpoints were trained with 2 cameras only: other counts run the same weights on a | |
| different image (the official code accepts any ``camera_names``), validated against the official inferencer with | |
| the same configuration, not against any training data. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| from dataclasses import dataclass | |
| from functools import lru_cache | |
| from typing import Mapping, Optional, Tuple | |
| import torch | |
| from .config import CONFIG | |
| V = CONFIG.vision | |
| NUM_CAMERAS_RANGE = (1, 4) | |
| NUM_STEPS_RANGE = (1, 16) | |
| DEFAULT_NUM_CAMERAS = len(CONFIG.proc.camera_names) # 2 | |
| DEFAULT_NUM_STEPS = CONFIG.expert.num_steps # 5 | |
| PROMPT_HEAD_TOKENS = 15 # system turn + user header + <|vision_start|> | |
| PROMPT_TAIL_TOKENS = 7 # <|vision_end|> ... <|im_end|>\n<|im_start|>assistant\n<|quad_start|> (without instruction) | |
| INSTRUCTION_CAPS = (16, 48, 112) | |
| CAMERA_HW = (384, 384) # the official per-camera image (deploy / dataset) | |
| EXTRA_CAMERA_NAMES = ("camera_2", "camera_3") | |
| # Bucket exceptions (see the module docstring): the EXPERT megakernel's cross-attention chunk rule | |
| BUCKET_OVERRIDES = {1: (256, 288, 320)} | |
| def _round_up(x: int, m: int) -> int: | |
| return -(-int(x) // m) * m | |
| def camera_names(n: int) -> Tuple[str, ...]: | |
| official = tuple(CONFIG.proc.camera_names) | |
| return (official + EXTRA_CAMERA_NAMES)[:n] | |
| class Geometry: | |
| """Static image / prompt geometry of one camera count (every number derived from the official preprocessing).""" | |
| num_cameras: int | |
| camera_names: Tuple[str, ...] | |
| camera_hw: Tuple[int, int] | |
| image_hw: Tuple[int, int] # smart-resized concatenation (rows, cols) | |
| grid: Tuple[int, int] # patch grid (gh, gw) | |
| n_patch: int # ViT rows (patches) | |
| n_rows: int # padded ViT rows on the device (multiple of 128: the merged rows are whole tiles) | |
| n_img: int # image tokens (merged patches) | |
| n_merged: int # padded merged rows (n_rows / 4) | |
| prompt_fixed_tokens: int # every token of the prompt except the instruction | |
| buckets: Tuple[int, ...] # resident prompt buckets | |
| def grid_thw(self) -> torch.Tensor: | |
| return torch.tensor([[1, self.grid[0], self.grid[1]]]) | |
| def image_start(self) -> int: | |
| return PROMPT_HEAD_TOKENS | |
| def min_valid_keys(self) -> int: | |
| """Keys valid in every prompt (head tokens + image tokens): the EXPERT megakernel's chunk bound.""" | |
| return PROMPT_HEAD_TOKENS + self.n_img | |
| def is_default(self) -> bool: | |
| return self.num_cameras == DEFAULT_NUM_CAMERAS | |
| def max_instruction_tokens(self) -> dict: | |
| return {int(L): int(L) - self.prompt_fixed_tokens for L in self.buckets} | |
| def as_dict(self) -> dict: | |
| return {"num_cameras": self.num_cameras, "camera_names": list(self.camera_names), | |
| "camera_hw": list(self.camera_hw), "concatenated_hw": [self.camera_hw[0], | |
| self.camera_hw[1] * self.num_cameras], | |
| "resized_hw": list(self.image_hw), "patch_grid": list(self.grid), "patches": self.n_patch, | |
| "image_tokens": self.n_img, "prompt_fixed_tokens": self.prompt_fixed_tokens, | |
| "prompt_buckets": list(self.buckets), | |
| "max_instruction_tokens": {str(k): v for k, v in self.max_instruction_tokens().items()}} | |
| def check_num_cameras(n) -> int: | |
| if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit(): | |
| raise ValueError(f"num_cameras must be an integer in {list(range(NUM_CAMERAS_RANGE[0], NUM_CAMERAS_RANGE[1] + 1))}" | |
| f", got {n!r}") | |
| n = int(str(n).strip()) | |
| if not NUM_CAMERAS_RANGE[0] <= n <= NUM_CAMERAS_RANGE[1]: | |
| raise ValueError(f"num_cameras must be in {NUM_CAMERAS_RANGE[0]}..{NUM_CAMERAS_RANGE[1]}, got {n}") | |
| return n | |
| def check_num_steps(n) -> int: | |
| if isinstance(n, bool) or not isinstance(n, (int, str)) or not str(n).strip().isdigit(): | |
| raise ValueError(f"num_steps must be an integer in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n!r}") | |
| n = int(str(n).strip()) | |
| if not NUM_STEPS_RANGE[0] <= n <= NUM_STEPS_RANGE[1]: | |
| raise ValueError(f"num_steps must be in {NUM_STEPS_RANGE[0]}..{NUM_STEPS_RANGE[1]}, got {n}") | |
| return n | |
| def geometry(num_cameras: int = DEFAULT_NUM_CAMERAS) -> Geometry: | |
| from .preprocess import smart_resize | |
| n = check_num_cameras(num_cameras) | |
| ch, cw = CAMERA_HW | |
| rh, rw = smart_resize(ch, cw * n, factor=V.patch * V.merge) | |
| gh, gw = rh // V.patch, rw // V.patch | |
| n_patch = gh * gw | |
| n_rows = _round_up(n_patch, 128) | |
| n_img = n_patch // (V.merge ** 2) | |
| fixed = PROMPT_HEAD_TOKENS + n_img + PROMPT_TAIL_TOKENS | |
| buckets = BUCKET_OVERRIDES.get(n) or tuple(_round_up(fixed + c, 32) for c in INSTRUCTION_CAPS) | |
| return Geometry(num_cameras=n, camera_names=camera_names(n), camera_hw=CAMERA_HW, image_hw=(rh, rw), | |
| grid=(gh, gw), n_patch=n_patch, n_rows=n_rows, n_img=n_img, n_merged=n_rows // 4, | |
| prompt_fixed_tokens=fixed, buckets=tuple(buckets)) | |
| GEOM2 = geometry(DEFAULT_NUM_CAMERAS) # the checkpoint's configuration (and the VQ profile's) | |
| def config_from_env(env: Optional[Mapping[str, str]] = None) -> Tuple[int, int]: | |
| """(num_cameras, num_steps) from RDT2_NUM_CAMERAS / RDT2_NUM_STEPS (unset or empty = the defaults 2 / 5). | |
| Raises ValueError for an unsupported value (never silently replaced).""" | |
| env = os.environ if env is None else env | |
| nc = (env.get("RDT2_NUM_CAMERAS") or "").strip() or DEFAULT_NUM_CAMERAS | |
| ns = (env.get("RDT2_NUM_STEPS") or "").strip() or DEFAULT_NUM_STEPS | |
| try: | |
| nc = check_num_cameras(nc) | |
| except ValueError as e: | |
| raise ValueError(f"RDT2_NUM_CAMERAS: {e}") from None | |
| try: | |
| ns = check_num_steps(ns) | |
| except ValueError as e: | |
| raise ValueError(f"RDT2_NUM_STEPS: {e}") from None | |
| return nc, ns | |