File size: 34,975 Bytes
9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c c869f1f 9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 | # SPDX-License-Identifier: Apache-2.0
"""Python API of rdt2-base-p150: RDT2-FM (Qwen2.5-VL-7B prefix of RDT2-VQ + the RDT2-FM flow-matching action expert)
on one Tenstorrent Blackhole p150, batch 1.
from tt_rdt2 import RDT2FM, load_sample
with RDT2FM.from_pretrained() as model: # weights resolved first, then the device, then warm-up
out = model(**load_sample()) # images (left_stereo, right_stereo), instruction, state
print(out.actions.shape, out.timing_ms) # (24, 20) float32, unnormalized (metres / 6-D rotation)
What runs: the device path the gates validated (``tt.model.RDT2FMTT``: VISION | PREFIX | EXPERT, one Metal trace per
prompt bucket, every bucket resident and warm) behind the exact host pre-processing of the official RDT2 inferencer
(``reference.preprocess`` / ``reference.host_fold``: camera concat, uint8 bicubic-antialias resize, CLIP
normalization, Qwen chat prompt) and the official UMI un-normalization. ``model(...)`` validates the request first
(:class:`InputError` for anything a client got wrong), then calls ``RDT2FMTT.__call__`` under a lock.
Configuration (load time, not per request; round 4 stage Config: ``num_cameras`` 1..4 and ``num_steps`` 1..16 of
:meth:`RDT2FM.from_pretrained`, else RDT2_NUM_CAMERAS / RDT2_NUM_STEPS; reference/geometry.py derives the image
geometry and the buckets of a camera count). The defaults, the published configuration: 2 cameras in the checkpoint's order
(``left_stereo``, ``right_stereo``), concatenated side by side (each 384 x 384 in the official setup; any pair whose
concatenation resizes to the same 392 x 756 grid is accepted and resized exactly as the official processor does),
5 Euler flow-matching steps, prompt buckets 416 / 448 / 512 tokens (instructions of up to 16 / 48 / 112 tokens), an
action chunk of 24 x 20. The default engines are ``vision=mk, prefix=fused, expert=mk``; ``prefix=mk`` is experimental
(known hang) and refused unless ``RDT2_EXPERIMENTAL_PREFIX_MK=1``.
Weights (resolved BEFORE the device opens; pinned revisions; local snapshots work offline):
* VLM ``robotics-diffusion-transformer/RDT2-VQ`` (vision tower + LM layers 0..13; only the 2 shards that hold them),
* action expert ``robotics-diffusion-transformer/RDT2-FM`` (``pytorch_model.bin``, ``torch.load(weights_only=True)``),
* action normalizer ``robotics-diffusion-transformer/RVQActionTokenizer`` (one file),
* tokenizer ``Qwen/Qwen2.5-VL-7B-Instruct`` (``tokenizer.json`` only, no weights).
Resolution per component: ``weights_dirs[role]`` > ``$RDT2_<ROLE>_DIR`` > (VLM only) a local ``model_id`` directory >
``hf_hub_download(repo, file, revision=<pin>)`` into the Hugging Face cache (a cached file at a pinned commit is used
without any network access; on a network failure the cache is retried with ``local_files_only=True``).
"""
from __future__ import annotations
import atexit
import logging
import math
import os
import re
import threading
import time
import weakref
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
import numpy as np
from . import __version__
from .reference.config import (CONFIG, EXPERT_REPO, EXPERT_REVISION, NORMALIZER_FILE, NORMALIZER_REPO,
NORMALIZER_REVISION, OFFICIAL_CODE, PROCESSOR_REPO, PROCESSOR_REVISION, VLM_REPO,
VLM_REVISION)
from .reference.geometry import GEOM2, Geometry, check_num_cameras, check_num_steps, geometry
from .reference.geometry import config_from_env as geometry_from_env
__all__ = ["RDT2FM", "ActionChunk", "InputError", "load_sample", "resolve_weights", "vlm_needed_files",
"SAMPLE_PATH", "PROMPT_BUCKETS", "CAMERA_ORDER", "NUM_STEPS", "CHUNK_SHAPE"]
log = logging.getLogger("tt_rdt2")
E = CONFIG.expert
CAMERA_ORDER: Tuple[str, ...] = tuple(CONFIG.proc.camera_names) # ("left_stereo", "right_stereo")
NUM_CAMERAS = len(CAMERA_ORDER) # 2 (fixed)
NUM_STEPS = E.num_steps # 5 Euler steps (fixed)
PROMPT_BUCKETS: Tuple[int, ...] = (416, 448, 512) # all resident, one trace each
CHUNK_SHAPE: Tuple[int, int] = (E.horizon, E.action_dim) # (24, 20)
STATE_DIM = E.state_dim # 20
DEFAULT_SEED = 0 # the fixed seeded noise (golden case seed)
SEED_MAX = 2 ** 63 - 1
ACTION_RATE_HZ = 30
# prompt = 15 head tokens (system turn, user header, <|vision_start|>) + 378 image tokens + the instruction + 7 tail
# tokens (<|vision_end|>, <|im_end|>, "\n", <|im_start|>, "assistant", "\n", <|quad_start|>)
PROMPT_FIXED_TOKENS = 400
IMAGE_GRID = (392, 756) # the concatenated cameras must smart-resize to this (grid 28 x 54, 378 tokens)
MAX_INSTRUCTION_CHARS = 4096 # refused before tokenization (the largest bucket holds 112 tokens anyway)
MAX_IMAGE_SIDE = 8192 # header check before a full decode
SPECIAL_TOKEN = re.compile(r"<\|[^<>|]*\|>")
EXPERIMENTAL_PREFIX_MK_ENV = "RDT2_EXPERIMENTAL_PREFIX_MK"
SAMPLE_PATH = Path(__file__).resolve().parent / "samples" / "ur5e_bimanual_example_0.npz"
ACTION_LAYOUT = {
"right_arm": {"position_m": [0, 3], "rotation_6d": [3, 9], "gripper_width_m": [9, 10]},
"left_arm": {"position_m": [10, 13], "rotation_6d": [13, 19], "gripper_width_m": [19, 20]},
}
ACTION_UNITS = {
"position": "metres, end-effector position relative to the current end-effector pose",
"rotation": "6-D rotation representation (unitless), relative to the current end-effector pose",
"gripper": "metres (gripper width)",
}
# component -> (repo, pinned revision, env override, license); the VLM file list is computed from its index
COMPONENTS: Dict[str, Dict[str, Any]] = {
"vlm": dict(repo=VLM_REPO, revision=VLM_REVISION, env="RDT2_VLM_DIR", license="apache-2.0",
files=("config.json", "model.safetensors.index.json"),
use="Qwen2.5-VL-7B vision tower and language-model layers 0..13 (K/V of layers 0..13)"),
"expert": dict(repo=EXPERT_REPO, revision=EXPERT_REVISION, env="RDT2_EXPERT_DIR", license="apache-2.0",
files=("config.json", "pytorch_model.bin"), use="RDT2-FM flow-matching action expert"),
"normalizer": dict(repo=NORMALIZER_REPO, revision=NORMALIZER_REVISION, env="RDT2_NORMALIZER_DIR",
license="apache-2.0", files=(NORMALIZER_FILE,), use="UMI action normalizer (one file)"),
"processor": dict(repo=PROCESSOR_REPO, revision=PROCESSOR_REVISION, env="RDT2_PROCESSOR_DIR",
license="apache-2.0", files=("tokenizer.json",), use="Qwen2.5-VL tokenizer (no weights)"),
}
VLM_PREFIXES = ("visual.", "model.embed_tokens.") + tuple(f"model.layers.{i}." for i in CONFIG.lm.selected_layers)
class InputError(ValueError):
"""A request the model cannot serve as given (HTTP 400): the message says why."""
# ------------------------------------------------------------------------------------------------- weights
def vlm_needed_files(index_path: Union[str, os.PathLike]) -> List[str]:
"""The VLM shards RDT2-FM reads (vision tower, embedding, LM layers 0..13), from the safetensors index."""
import json
with open(index_path) as f:
wm = json.load(f)["weight_map"]
return sorted({fn for k, fn in wm.items() if k.startswith(VLM_PREFIXES)})
def _hub_file(repo: str, filename: str, revision: str, local_files_only: bool) -> str:
from huggingface_hub import hf_hub_download
try:
return hf_hub_download(repo, filename, revision=revision, local_files_only=local_files_only)
except Exception as e: # noqa: BLE001 -- offline / network failure: the cache may still hold the pinned file
if local_files_only:
raise
try:
return hf_hub_download(repo, filename, revision=revision, local_files_only=True)
except Exception: # noqa: BLE001
raise RuntimeError(f"cannot resolve {repo}@{revision[:12]}/{filename}: {type(e).__name__}: {e}") from e
def resolve_weights(model_id: Optional[str] = None, revision: Optional[str] = None, *,
weights_dirs: Optional[Mapping[str, str]] = None, local_files_only: bool = False,
env: Optional[Mapping[str, str]] = None, log_fn=None) -> Dict[str, Dict[str, Any]]:
"""Resolve every component to a local directory (no device). Returns ``{role: {repo, revision, path, files,
source, license, use}}``. ``model_id`` / ``revision`` replace the VLM repo / revision (default: the pins)."""
env = os.environ if env is None else env
weights_dirs = dict(weights_dirs or {})
say = log_fn or log.info
out: Dict[str, Dict[str, Any]] = {}
for role, c in COMPONENTS.items():
repo, rev = c["repo"], c["revision"]
if role == "vlm":
repo = model_id or repo
rev = revision or (VLM_REVISION if repo == VLM_REPO else None) or "main"
local = weights_dirs.get(role) or env.get(c["env"]) or None
if role == "vlm" and not local and model_id and os.path.isdir(model_id):
local = model_id
files = list(c["files"])
if local:
path = Path(local)
if role == "vlm":
files += vlm_needed_files(path / "model.safetensors.index.json")
missing = [f for f in files if not (path / f).is_file()]
if missing:
raise FileNotFoundError(f"{role} weights directory {path} lacks {missing}")
source = "local"
else:
first = _hub_file(repo, files[0], rev, local_files_only)
path = Path(first).parent
if role == "vlm":
idx = _hub_file(repo, "model.safetensors.index.json", rev, local_files_only)
files += vlm_needed_files(idx)
for f in files[1:]:
_hub_file(repo, f, rev, local_files_only)
source = "hf-cache"
out[role] = dict(repo=repo, revision=rev, path=str(path), files=files, source=source, license=c["license"],
use=c["use"])
say(f"Loading weights: {role} {repo}@{str(rev)[:12]} ({source}: {path})")
return out
# ------------------------------------------------------------------------------------------------- inputs
def _decode_image(x: Any, i: int) -> np.ndarray:
"""One camera image -> (H, W, 3) uint8 RGB. ndarray uint8 HxWx3 as is; PIL image / encoded bytes / path via PIL
``convert("RGB")`` (what the official inferencer applies to its camera arrays)."""
name = CAMERA_ORDER[i] if i < len(CAMERA_ORDER) else f"#{i}"
if isinstance(x, np.ndarray):
if x.dtype != np.uint8 or x.ndim != 3 or x.shape[2] != 3:
raise InputError(f"image {name}: expected an HxWx3 uint8 array, got {x.dtype} {tuple(x.shape)}")
return np.ascontiguousarray(x)
if hasattr(x, "numpy") and hasattr(x, "dtype") and not isinstance(x, (bytes, bytearray)): # torch tensor
return _decode_image(np.asarray(x.numpy() if hasattr(x, "numpy") else x), i)
from PIL import Image, UnidentifiedImageError
try:
if isinstance(x, Image.Image):
im = x
elif isinstance(x, (bytes, bytearray, memoryview)):
import io
im = Image.open(io.BytesIO(bytes(x)))
elif isinstance(x, (str, os.PathLike)):
im = Image.open(os.fspath(x))
else:
raise InputError(f"image {name}: unsupported type {type(x).__name__} (array, PIL image, bytes or path)")
w, h = im.size
if max(w, h) > MAX_IMAGE_SIDE:
raise InputError(f"image {name}: {w}x{h} is too large (max side {MAX_IMAGE_SIDE})")
arr = np.asarray(im.convert("RGB"), dtype=np.uint8)
except InputError:
raise
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as e:
raise InputError(f"image {name}: cannot decode ({type(e).__name__}: {e}); send PNG or JPEG") from None
return np.ascontiguousarray(arr)
def check_geometry(images: Sequence[np.ndarray], geom: Optional[Geometry] = None) -> None:
"""The concatenated cameras (side by side) must smart-resize to 392 x 756 (the one static ViT geometry of 2
cameras; ``geom.image_hw`` for another camera count)."""
from .reference.preprocess import smart_resize
geom = geom or GEOM2
grid = tuple(geom.image_hw)
hs = {int(im.shape[0]) for im in images}
if len(hs) != 1:
raise InputError(f"the cameras must have the same height (they are concatenated side by side), got "
f"{[tuple(im.shape[:2]) for im in images]}")
h, w = hs.pop(), sum(int(im.shape[1]) for im in images)
try:
rh, rw = smart_resize(h, w, factor=CONFIG.vision.patch * CONFIG.vision.merge)
except ValueError as e:
raise InputError(f"unsupported image geometry {h}x{w}: {e}") from None
if (rh, rw) != grid:
n = geom.num_cameras
raise InputError(f"unsupported image geometry: the cameras concatenated side by side are {h}x{w}, which "
f"resizes to {rh}x{rw}; this configuration ({n} camera{'s' if n > 1 else ''}) serves only "
f"geometries that resize to {grid[0]}x{grid[1]} (e.g. {n} x 384x384)")
def decode_images(images: Any, geom: Optional[Geometry] = None) -> List[np.ndarray]:
"""Exactly the configured cameras, in the configured order: a sequence ``[left_stereo, right_stereo]`` or a
mapping ``{"left_stereo": ..., "right_stereo": ...}`` (2 cameras; ``geom.camera_names`` for another count)."""
geom = geom or GEOM2
order, ncam = tuple(geom.camera_names), geom.num_cameras
if isinstance(images, Mapping):
unknown = sorted(set(images) - set(order))
missing = [c for c in order if c not in images]
if unknown or missing:
raise InputError(f"cameras must be exactly {list(order)} (missing {missing}, unknown {unknown})")
images = [images[c] for c in order]
if isinstance(images, (str, bytes, np.ndarray)) or not isinstance(images, Sequence):
raise InputError(f"images must be a list of {ncam} camera images {list(order)}")
if len(images) != ncam:
raise InputError(f"expected exactly {ncam} images in the order {list(order)}, got "
f"{len(images)} (this server is configured for {ncam} camera{'s' if ncam > 1 else ''}, "
"RDT2_NUM_CAMERAS)")
if sum(isinstance(x, (bytes, bytearray, memoryview, str, os.PathLike)) for x in images) > 1:
# encoded cameras decode in parallel (PIL releases the GIL while decoding: 2 x 3.7 -> 4.1 ms for 2 PNGs);
# the same function per image, so the result and the error of the first bad image are unchanged
out = list(_decode_pool().map(_decode_image, images, range(len(images))))
else:
out = [_decode_image(x, i) for i, x in enumerate(images)]
check_geometry(out, geom)
return out
_POOL = None
def _decode_pool():
global _POOL
if _POOL is None:
from concurrent.futures import ThreadPoolExecutor
_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="rdt2-decode") # up to 4 cameras
return _POOL
def check_instruction(instruction: Any) -> str:
if not isinstance(instruction, str):
raise InputError(f"instruction must be a string, got {type(instruction).__name__}")
if not instruction.strip():
raise InputError("instruction is empty")
if len(instruction) > MAX_INSTRUCTION_CHARS:
raise InputError(f"instruction of {len(instruction)} characters is too long (the largest prompt bucket holds "
f"{max(PROMPT_BUCKETS) - PROMPT_FIXED_TOKENS} tokens)")
m = SPECIAL_TOKEN.search(instruction)
if m:
raise InputError(f"instruction contains the special-token text {m.group(0)!r}; special tokens are not "
"allowed inside the instruction (they would change the chat-template prompt)")
return instruction
def check_state(state: Any) -> np.ndarray:
"""Raw 20-d proprio state (not normalized; the official FM deployment sends zeros)."""
if state is None:
return np.zeros(STATE_DIM, np.float32)
try:
a = np.asarray(state, dtype=np.float64).reshape(-1)
except (TypeError, ValueError):
raise InputError(f"state must be {STATE_DIM} numbers") from None
if a.size != STATE_DIM:
raise InputError(f"state must have exactly {STATE_DIM} values, got {a.size}")
if not np.all(np.isfinite(a)):
raise InputError("state contains a non-finite value")
return a.astype(np.float32)
def check_seed(seed: Any) -> int:
if seed is None:
return DEFAULT_SEED
if isinstance(seed, bool) or not isinstance(seed, (int, np.integer)):
raise InputError(f"seed must be an integer, got {type(seed).__name__}")
if not 0 <= int(seed) <= SEED_MAX:
raise InputError(f"seed must be in [0, 2**63 - 1], got {seed}")
return int(seed)
def seeded_noise(seed: int):
"""The initial flow-matching noise of a request: ``torch.randn((1, 24, 20), generator=manual_seed(seed))`` fp32
(the golden driver's injected noise; seed 0 is the shipped sample's reference)."""
import torch
return torch.randn((1,) + CHUNK_SHAPE, generator=torch.Generator().manual_seed(int(seed)))
def load_sample(path: Optional[Union[str, os.PathLike]] = None) -> Dict[str, Any]:
"""The shipped Apache-2.0 sample (BimanualUR5eExample sample 0) as call kwargs: ``images`` (left_stereo,
right_stereo), ``instruction``, ``state`` (zeros)."""
s = np.load(path or SAMPLE_PATH, allow_pickle=False)
return {"images": [s["left_stereo"], s["right_stereo"]], "instruction": str(s["instruction"]),
"state": np.asarray(s["state"], np.float32)}
# ------------------------------------------------------------------------------------------------- output
@dataclass
class ActionChunk:
"""One action chunk. ``actions`` (24, 20) float32 = ``RDTInferencer.step``'s output: UMI-unnormalized, relative to
the current end-effector poses, right arm in columns 0..9 and left arm in 10..19 (``ACTION_LAYOUT``)."""
actions: np.ndarray
actions_normalized: np.ndarray
prompt_bucket: int
num_tokens: int
seed: int
num_steps: int = NUM_STEPS
timing_ms: Dict[str, float] = field(default_factory=dict)
@property
def shape(self) -> Tuple[int, int]:
return tuple(self.actions.shape)
def to_dict(self, include_normalized: bool = False) -> Dict[str, Any]:
d: Dict[str, Any] = {
"actions": self.actions.astype(np.float32).tolist(),
"shape": list(self.actions.shape),
"normalized": False,
"units": dict(ACTION_UNITS),
"action_layout": ACTION_LAYOUT,
"chunk": {"horizon": CHUNK_SHAPE[0], "action_dim": CHUNK_SHAPE[1], "rate_hz": ACTION_RATE_HZ},
"prompt_bucket": int(self.prompt_bucket),
"num_tokens": int(self.num_tokens),
"num_steps": int(self.num_steps),
"seed": int(self.seed),
"timing_ms": {k: round(float(v), 3) for k, v in self.timing_ms.items()},
}
if include_normalized:
d["actions_normalized"] = self.actions_normalized.astype(np.float32).tolist()
return d
# ------------------------------------------------------------------------------------------------- the model
_LIVE: "weakref.WeakSet[RDT2FM]" = weakref.WeakSet()
@atexit.register
def _close_all() -> None: # close at interpreter exit (traces released, device closed)
for m in list(_LIVE):
try:
m.close()
except Exception: # noqa: BLE001
pass
class RDT2FM:
"""RDT2-FM on one p150. Build with :meth:`from_pretrained`; call ``model(images, instruction, state=None,
seed=None, prompt_bucket=None, num_steps=None)`` -> :class:`ActionChunk`. Thread-safe (one lock), batch 1."""
MODEL_NAME = "rdt2-base-p150"
DEFAULT_REPO = VLM_REPO
DEFAULT_REVISION = VLM_REVISION
CAMERA_ORDER = CAMERA_ORDER
NUM_CAMERAS = NUM_CAMERAS
NUM_STEPS = NUM_STEPS
PROMPT_BUCKETS = PROMPT_BUCKETS
CHUNK_SHAPE = CHUNK_SHAPE
DEFAULT_SEED = DEFAULT_SEED
def __init__(self, tt_model, device, *, weights: Dict[str, Dict[str, Any]], owns_device: bool,
device_id: Optional[int], precision: str, log_fn=None):
self._m = tt_model
self.device = device
self.weights = weights
self._owns_device = owns_device
self.device_id = device_id
self.precision = precision
self._log = log_fn or log.info
self._lock = threading.RLock()
self._closed = False
self.warmup_ms: Dict[int, float] = {}
self.capture_s: Optional[float] = None
self.processor_dir = weights["processor"]["path"]
# the configuration of this object (stage Config): fixed at construction, per request only the bucket
self.geom: Geometry = getattr(tt_model, "geom", None) or GEOM2
self.CAMERA_ORDER = tuple(self.geom.camera_names)
self.NUM_CAMERAS = self.geom.num_cameras
self.NUM_STEPS = int(getattr(tt_model, "num_steps", NUM_STEPS))
self.PROMPT_BUCKETS = tuple(getattr(tt_model, "buckets", None) or self.geom.buckets)
self.prompt_fixed_tokens = self.geom.prompt_fixed_tokens
if owns_device: # closed at interpreter exit; a caller-owned device is the caller's to release
_LIVE.add(self)
# -------------------------------------------------------------------------------------------- construction
@classmethod
def from_pretrained(cls, model_id: Optional[str] = None, revision: Optional[str] = None, *, device_id: int = 0,
device=None, dispatch: str = "eth", engine=None, precision: str = "baseline",
weights_dirs: Optional[Mapping[str, str]] = None, local_files_only: bool = False,
warmup: bool = True, allow_fallback: bool = False, log_fn=None,
num_cameras: Optional[int] = None, num_steps: Optional[int] = None) -> "RDT2FM":
"""Resolve the pinned weights (no device), open the chip (ETH dispatch, 12 x 10 grid unless
``dispatch="worker"``), load every program with all prompt buckets resident and, with ``warmup`` (default),
capture one trace per bucket and run one call per bucket: the first real call is as fast as later ones.
``device``: a caller-owned ttnn device (``close()`` leaves it open). ``engine``: per-program override
(``"vision=ttnn"`` etc.; default ``vision=mk, prefix=fused, expert=mk``; ``prefix=mk`` needs
``RDT2_EXPERIMENTAL_PREFIX_MK=1``). ``precision``: the gated preset ``baseline`` (others are development
presets of ``tt/common.py``). ``num_cameras`` (1..4) / ``num_steps`` (1..16): the fixed configuration of the
object; None reads ``RDT2_NUM_CAMERAS`` / ``RDT2_NUM_STEPS`` (unset: 2 cameras, 5 steps)."""
from .tt.model import RDT2FMTT, TRACE_REGION_SIZE, check_engine_allowed, resolve_engine
from .ttaw.device import close_device, open_device
say = log_fn or log.info
env_cams, env_steps = geometry_from_env()
try:
ncam = env_cams if num_cameras is None else check_num_cameras(num_cameras)
nsteps = env_steps if num_steps is None else check_num_steps(num_steps)
except ValueError as e:
raise InputError(str(e)) from None
geom = geometry(ncam)
eng = resolve_engine(engine)
check_engine_allowed(eng) # before the weights and the device: the experimental engine is refused early
weights = resolve_weights(model_id, revision, weights_dirs=weights_dirs, local_files_only=local_files_only,
log_fn=say)
owns = device is None
if owns:
say(f"Opening device {device_id} ({dispatch} dispatch) ...")
device = open_device(int(device_id), dispatch=dispatch, num_command_queues=1,
trace_region_size=TRACE_REGION_SIZE, allow_fallback=allow_fallback)
try:
g = device.compute_with_storage_grid_size()
say(f"Device open: grid {g.x}x{g.y}; loading the programs (engines {eng}, precision {precision}) ...")
say(f"Configuration: {ncam} camera(s) {list(geom.camera_names)}, {nsteps} Euler steps, prompt buckets "
f"{list(geom.buckets)}")
tt = RDT2FMTT(device, buckets=geom.buckets, precision=precision, num_steps=nsteps, num_cameras=ncam,
vlm_dir=weights["vlm"]["path"], expert_dir=weights["expert"]["path"],
normalizer_dir=weights["normalizer"]["path"],
processor_dir=weights["processor"]["path"], engine=eng,
log=lambda s: say(s))
model = cls(tt, device, weights=weights, owns_device=owns, device_id=int(device_id) if owns else None,
precision=precision if isinstance(precision, str) else "custom", log_fn=say)
except BaseException:
if owns:
close_device(device)
raise
if warmup:
try:
model.warmup()
except BaseException:
model.close()
raise
return model
def warmup(self) -> Dict[int, float]:
"""Capture one trace per prompt bucket (kernel JIT on the first boot), then one call per bucket. Idempotent."""
with self._lock:
self._check_open()
if all(t is not None for t in self._m.traces.values()) and self.warmup_ms:
return dict(self.warmup_ms)
self._log(f"Warming up: capturing trace of prompt buckets {list(self.PROMPT_BUCKETS)} ...")
t0 = time.perf_counter()
self._m.capture()
self.capture_s = time.perf_counter() - t0
dummy = [np.zeros((384, 384, 3), np.uint8)] * self.NUM_CAMERAS
for L in self.PROMPT_BUCKETS:
self._call(dummy, "warm up", np.zeros(STATE_DIM, np.float32), DEFAULT_SEED, L)
t1 = time.perf_counter()
self._call(dummy, "warm up", np.zeros(STATE_DIM, np.float32), DEFAULT_SEED, L)
self.warmup_ms[L] = (time.perf_counter() - t1) * 1e3
self._log(f"Warmup complete in {time.perf_counter() - t0:.1f} s (one call per bucket, ms: "
+ ", ".join(f"{L}: {v:.1f}" for L, v in self.warmup_ms.items()) + ")")
return dict(self.warmup_ms)
# -------------------------------------------------------------------------------------------- calls
def tokens_for(self, instruction: str) -> int:
"""Prompt length of an instruction (400 fixed tokens with 2 cameras + the instruction's own)."""
from .reference.preprocess import tokenize
return int(tokenize(instruction, self.geom.n_img, self.processor_dir).shape[1])
def validate(self, images: Any, instruction: Any, state: Any = None, seed: Any = None,
prompt_bucket: Any = None, num_steps: Any = None) -> Dict[str, Any]:
"""Decode and check one request (host only, no lock): raises :class:`InputError` with the reason."""
S, B, F = self.NUM_STEPS, self.PROMPT_BUCKETS, self.prompt_fixed_tokens
if num_steps is not None and (isinstance(num_steps, bool) or num_steps != S):
raise InputError(f"num_steps must be {S} (this server runs a fixed {S}-step Euler flow-matching schedule, "
f"set at start with RDT2_NUM_STEPS), got {num_steps!r}")
imgs = decode_images(images, self.geom)
instr = check_instruction(instruction)
st = check_state(state)
sd = check_seed(seed)
n = self.tokens_for(instr)
if n > max(B):
raise InputError(f"prompt of {n} tokens ({n - F} instruction tokens) exceeds the largest "
f"prompt bucket {max(B)} (at most {max(B) - F}"
" instruction tokens); instructions are never truncated")
if prompt_bucket is None:
L = min(b for b in B if b >= n)
else:
if isinstance(prompt_bucket, bool) or not isinstance(prompt_bucket, (int, np.integer)) \
or int(prompt_bucket) not in B:
raise InputError(f"prompt_bucket must be one of {list(B)}, got {prompt_bucket!r}")
L = int(prompt_bucket)
if n > L:
raise InputError(f"prompt of {n} tokens does not fit prompt_bucket {L}")
return {"images": imgs, "instruction": instr, "state": st, "seed": sd, "bucket": L, "n_tokens": n}
def _call(self, images, instruction, state, seed, bucket):
out = self._m(images, instruction, state=state, noise=seeded_noise(seed), bucket=bucket)
return out, dict(self._m.timings)
def __call__(self, images: Any, instruction: Optional[str] = None, state: Any = None, seed: Optional[int] = None,
prompt_bucket: Optional[int] = None, num_steps: Optional[int] = None, *,
prompt: Optional[str] = None) -> ActionChunk:
t0 = time.perf_counter()
if instruction is None:
instruction = prompt
elif prompt is not None:
raise InputError("give either instruction or prompt, not both")
if instruction is None:
raise InputError("instruction is required")
req = self.validate(images, instruction, state, seed, prompt_bucket, num_steps)
t1 = time.perf_counter()
with self._lock:
self._check_open()
out, tm = self._call(req["images"], req["instruction"], req["state"], req["seed"], req["bucket"])
t2 = time.perf_counter()
if int(out["bucket"]) != req["bucket"] or int(out["n_tokens"]) != req["n_tokens"]:
raise RuntimeError(f"internal: bucket / token count mismatch {out['bucket']}/{out['n_tokens']} vs "
f"{req['bucket']}/{req['n_tokens']}")
pre = (t1 - t0) * 1e3 + tm.get("host_pre_ms", 0.0)
timing = {"preprocess": pre, "inference": (t2 - t1) * 1e3 - tm.get("host_pre_ms", 0.0),
"total": (time.perf_counter() - t0) * 1e3}
return ActionChunk(actions=out["action"].detach().float().numpy().astype(np.float32).reshape(CHUNK_SHAPE),
actions_normalized=out["nsample"].detach().float().numpy().astype(np.float32)
.reshape(CHUNK_SHAPE),
prompt_bucket=req["bucket"], num_tokens=req["n_tokens"], seed=req["seed"],
num_steps=self.NUM_STEPS, timing_ms=timing)
# -------------------------------------------------------------------------------------------- lifetime / info
def _check_open(self) -> None:
if self._closed:
raise RuntimeError("the model is closed")
@property
def closed(self) -> bool:
return self._closed
def close(self) -> None:
"""Release the traces and (if this object opened it) close the device. Idempotent."""
with self._lock:
if self._closed:
return
self._closed = True
try:
self._m.release()
finally:
if self._owns_device and self.device is not None:
from .ttaw.device import close_device
close_device(self.device)
self.device = None
_LIVE.discard(self)
def __enter__(self) -> "RDT2FM":
return self
def __exit__(self, *exc) -> None:
self.close()
@property
def engine(self) -> Dict[str, str]:
return dict(self._m.engine)
@property
def device_info(self) -> Optional[Dict[str, Any]]:
if self.device is None:
return None
from .ttaw.device import describe_device
return describe_device(self.device, self.device_id)
@property
def info(self) -> Dict[str, Any]:
w = self.weights
return {
"model": self.MODEL_NAME,
"version": __version__,
"weights": {"repo": w["vlm"]["repo"], "revision": w["vlm"]["revision"],
"components": {r: {k: v[k] for k in ("repo", "revision", "files", "source", "license", "use")}
for r, v in w.items()}},
"engine": self.engine,
"precision": self.precision,
"device": self.device_info,
"cameras": list(self.CAMERA_ORDER),
"num_cameras": self.NUM_CAMERAS,
"image_geometry": {"layout": "cameras concatenated side by side, " + " | ".join(self.CAMERA_ORDER),
"resized_to": list(self.geom.image_hw), "image_tokens": self.geom.n_img,
"patch_grid": list(self.geom.grid),
"accepted": f"any {self.NUM_CAMERAS} equal-height RGB image(s) whose concatenation "
f"resizes to {self.geom.image_hw[0]}x{self.geom.image_hw[1]} (e.g. "
f"{self.NUM_CAMERAS} x 384x384)"},
"num_steps": self.NUM_STEPS,
"configuration": {"num_cameras": self.NUM_CAMERAS, "num_steps": self.NUM_STEPS,
"prompt_buckets": list(self.PROMPT_BUCKETS), "fixed_at": "start (RDT2_NUM_CAMERAS, "
"RDT2_NUM_STEPS or the Python API's num_cameras / num_steps); per request only the "
"prompt bucket", "default": self.NUM_CAMERAS == 2 and self.NUM_STEPS == 5,
"engine_fallbacks": dict(getattr(self._m, "engine_fallbacks", {}) or {})},
"chunk_shape": list(CHUNK_SHAPE),
"action_rate_hz": ACTION_RATE_HZ,
"action_layout": ACTION_LAYOUT,
"units": dict(ACTION_UNITS),
"normalized": False,
"state_dim": STATE_DIM,
"default_seed": DEFAULT_SEED,
"prompt_buckets": list(self.PROMPT_BUCKETS),
"max_instruction_tokens": {str(L): L - self.prompt_fixed_tokens for L in self.PROMPT_BUCKETS},
"warmup_ms": {str(k): round(v, 2) for k, v in self.warmup_ms.items()},
"capture_s": None if self.capture_s is None else round(self.capture_s, 2),
"load_s": round(float(self._m.timings.get("load_s", 0.0)), 2),
"official_code": {"repo": OFFICIAL_CODE[0], "commit": OFFICIAL_CODE[1]},
}
|