# SPDX-License-Identifier: Apache-2.0 """Python API of rdt2-base-p150: RDT2-FM (Qwen2.5-VL-7B prefix of RDT2-VQ + the RDT2-FM flow-matching action expert) on one Tenstorrent Blackhole p150, batch 1. from tt_rdt2 import RDT2FM, load_sample with RDT2FM.from_pretrained() as model: # weights resolved first, then the device, then warm-up out = model(**load_sample()) # images (left_stereo, right_stereo), instruction, state print(out.actions.shape, out.timing_ms) # (24, 20) float32, unnormalized (metres / 6-D rotation) What runs: the device path the gates validated (``tt.model.RDT2FMTT``: VISION | PREFIX | EXPERT, one Metal trace per prompt bucket, every bucket resident and warm) behind the exact host pre-processing of the official RDT2 inferencer (``reference.preprocess`` / ``reference.host_fold``: camera concat, uint8 bicubic-antialias resize, CLIP normalization, Qwen chat prompt) and the official UMI un-normalization. ``model(...)`` validates the request first (:class:`InputError` for anything a client got wrong), then calls ``RDT2FMTT.__call__`` under a lock. Configuration (load time, not per request; round 4 stage Config: ``num_cameras`` 1..4 and ``num_steps`` 1..16 of :meth:`RDT2FM.from_pretrained`, else RDT2_NUM_CAMERAS / RDT2_NUM_STEPS; reference/geometry.py derives the image geometry and the buckets of a camera count). The defaults, the published configuration: 2 cameras in the checkpoint's order (``left_stereo``, ``right_stereo``), concatenated side by side (each 384 x 384 in the official setup; any pair whose concatenation resizes to the same 392 x 756 grid is accepted and resized exactly as the official processor does), 5 Euler flow-matching steps, prompt buckets 416 / 448 / 512 tokens (instructions of up to 16 / 48 / 112 tokens), an action chunk of 24 x 20. The default engines are ``vision=mk, prefix=fused, expert=mk``; ``prefix=mk`` is experimental (known hang) and refused unless ``RDT2_EXPERIMENTAL_PREFIX_MK=1``. Weights (resolved BEFORE the device opens; pinned revisions; local snapshots work offline): * VLM ``robotics-diffusion-transformer/RDT2-VQ`` (vision tower + LM layers 0..13; only the 2 shards that hold them), * action expert ``robotics-diffusion-transformer/RDT2-FM`` (``pytorch_model.bin``, ``torch.load(weights_only=True)``), * action normalizer ``robotics-diffusion-transformer/RVQActionTokenizer`` (one file), * tokenizer ``Qwen/Qwen2.5-VL-7B-Instruct`` (``tokenizer.json`` only, no weights). Resolution per component: ``weights_dirs[role]`` > ``$RDT2__DIR`` > (VLM only) a local ``model_id`` directory > ``hf_hub_download(repo, file, revision=)`` into the Hugging Face cache (a cached file at a pinned commit is used without any network access; on a network failure the cache is retried with ``local_files_only=True``). """ from __future__ import annotations import atexit import logging import math import os import re import threading import time import weakref from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union import numpy as np from . import __version__ from .reference.config import (CONFIG, EXPERT_REPO, EXPERT_REVISION, NORMALIZER_FILE, NORMALIZER_REPO, NORMALIZER_REVISION, OFFICIAL_CODE, PROCESSOR_REPO, PROCESSOR_REVISION, VLM_REPO, VLM_REVISION) from .reference.geometry import GEOM2, Geometry, check_num_cameras, check_num_steps, geometry from .reference.geometry import config_from_env as geometry_from_env __all__ = ["RDT2FM", "ActionChunk", "InputError", "load_sample", "resolve_weights", "vlm_needed_files", "SAMPLE_PATH", "PROMPT_BUCKETS", "CAMERA_ORDER", "NUM_STEPS", "CHUNK_SHAPE"] log = logging.getLogger("tt_rdt2") E = CONFIG.expert CAMERA_ORDER: Tuple[str, ...] = tuple(CONFIG.proc.camera_names) # ("left_stereo", "right_stereo") NUM_CAMERAS = len(CAMERA_ORDER) # 2 (fixed) NUM_STEPS = E.num_steps # 5 Euler steps (fixed) PROMPT_BUCKETS: Tuple[int, ...] = (416, 448, 512) # all resident, one trace each CHUNK_SHAPE: Tuple[int, int] = (E.horizon, E.action_dim) # (24, 20) STATE_DIM = E.state_dim # 20 DEFAULT_SEED = 0 # the fixed seeded noise (golden case seed) SEED_MAX = 2 ** 63 - 1 ACTION_RATE_HZ = 30 # prompt = 15 head tokens (system turn, user header, <|vision_start|>) + 378 image tokens + the instruction + 7 tail # tokens (<|vision_end|>, <|im_end|>, "\n", <|im_start|>, "assistant", "\n", <|quad_start|>) PROMPT_FIXED_TOKENS = 400 IMAGE_GRID = (392, 756) # the concatenated cameras must smart-resize to this (grid 28 x 54, 378 tokens) MAX_INSTRUCTION_CHARS = 4096 # refused before tokenization (the largest bucket holds 112 tokens anyway) MAX_IMAGE_SIDE = 8192 # header check before a full decode SPECIAL_TOKEN = re.compile(r"<\|[^<>|]*\|>") EXPERIMENTAL_PREFIX_MK_ENV = "RDT2_EXPERIMENTAL_PREFIX_MK" SAMPLE_PATH = Path(__file__).resolve().parent / "samples" / "ur5e_bimanual_example_0.npz" ACTION_LAYOUT = { "right_arm": {"position_m": [0, 3], "rotation_6d": [3, 9], "gripper_width_m": [9, 10]}, "left_arm": {"position_m": [10, 13], "rotation_6d": [13, 19], "gripper_width_m": [19, 20]}, } ACTION_UNITS = { "position": "metres, end-effector position relative to the current end-effector pose", "rotation": "6-D rotation representation (unitless), relative to the current end-effector pose", "gripper": "metres (gripper width)", } # component -> (repo, pinned revision, env override, license); the VLM file list is computed from its index COMPONENTS: Dict[str, Dict[str, Any]] = { "vlm": dict(repo=VLM_REPO, revision=VLM_REVISION, env="RDT2_VLM_DIR", license="apache-2.0", files=("config.json", "model.safetensors.index.json"), use="Qwen2.5-VL-7B vision tower and language-model layers 0..13 (K/V of layers 0..13)"), "expert": dict(repo=EXPERT_REPO, revision=EXPERT_REVISION, env="RDT2_EXPERT_DIR", license="apache-2.0", files=("config.json", "pytorch_model.bin"), use="RDT2-FM flow-matching action expert"), "normalizer": dict(repo=NORMALIZER_REPO, revision=NORMALIZER_REVISION, env="RDT2_NORMALIZER_DIR", license="apache-2.0", files=(NORMALIZER_FILE,), use="UMI action normalizer (one file)"), "processor": dict(repo=PROCESSOR_REPO, revision=PROCESSOR_REVISION, env="RDT2_PROCESSOR_DIR", license="apache-2.0", files=("tokenizer.json",), use="Qwen2.5-VL tokenizer (no weights)"), } VLM_PREFIXES = ("visual.", "model.embed_tokens.") + tuple(f"model.layers.{i}." for i in CONFIG.lm.selected_layers) class InputError(ValueError): """A request the model cannot serve as given (HTTP 400): the message says why.""" # ------------------------------------------------------------------------------------------------- weights def vlm_needed_files(index_path: Union[str, os.PathLike]) -> List[str]: """The VLM shards RDT2-FM reads (vision tower, embedding, LM layers 0..13), from the safetensors index.""" import json with open(index_path) as f: wm = json.load(f)["weight_map"] return sorted({fn for k, fn in wm.items() if k.startswith(VLM_PREFIXES)}) def _hub_file(repo: str, filename: str, revision: str, local_files_only: bool) -> str: from huggingface_hub import hf_hub_download try: return hf_hub_download(repo, filename, revision=revision, local_files_only=local_files_only) except Exception as e: # noqa: BLE001 -- offline / network failure: the cache may still hold the pinned file if local_files_only: raise try: return hf_hub_download(repo, filename, revision=revision, local_files_only=True) except Exception: # noqa: BLE001 raise RuntimeError(f"cannot resolve {repo}@{revision[:12]}/{filename}: {type(e).__name__}: {e}") from e def resolve_weights(model_id: Optional[str] = None, revision: Optional[str] = None, *, weights_dirs: Optional[Mapping[str, str]] = None, local_files_only: bool = False, env: Optional[Mapping[str, str]] = None, log_fn=None) -> Dict[str, Dict[str, Any]]: """Resolve every component to a local directory (no device). Returns ``{role: {repo, revision, path, files, source, license, use}}``. ``model_id`` / ``revision`` replace the VLM repo / revision (default: the pins).""" env = os.environ if env is None else env weights_dirs = dict(weights_dirs or {}) say = log_fn or log.info out: Dict[str, Dict[str, Any]] = {} for role, c in COMPONENTS.items(): repo, rev = c["repo"], c["revision"] if role == "vlm": repo = model_id or repo rev = revision or (VLM_REVISION if repo == VLM_REPO else None) or "main" local = weights_dirs.get(role) or env.get(c["env"]) or None if role == "vlm" and not local and model_id and os.path.isdir(model_id): local = model_id files = list(c["files"]) if local: path = Path(local) if role == "vlm": files += vlm_needed_files(path / "model.safetensors.index.json") missing = [f for f in files if not (path / f).is_file()] if missing: raise FileNotFoundError(f"{role} weights directory {path} lacks {missing}") source = "local" else: first = _hub_file(repo, files[0], rev, local_files_only) path = Path(first).parent if role == "vlm": idx = _hub_file(repo, "model.safetensors.index.json", rev, local_files_only) files += vlm_needed_files(idx) for f in files[1:]: _hub_file(repo, f, rev, local_files_only) source = "hf-cache" out[role] = dict(repo=repo, revision=rev, path=str(path), files=files, source=source, license=c["license"], use=c["use"]) say(f"Loading weights: {role} {repo}@{str(rev)[:12]} ({source}: {path})") return out # ------------------------------------------------------------------------------------------------- inputs def _decode_image(x: Any, i: int) -> np.ndarray: """One camera image -> (H, W, 3) uint8 RGB. ndarray uint8 HxWx3 as is; PIL image / encoded bytes / path via PIL ``convert("RGB")`` (what the official inferencer applies to its camera arrays).""" name = CAMERA_ORDER[i] if i < len(CAMERA_ORDER) else f"#{i}" if isinstance(x, np.ndarray): if x.dtype != np.uint8 or x.ndim != 3 or x.shape[2] != 3: raise InputError(f"image {name}: expected an HxWx3 uint8 array, got {x.dtype} {tuple(x.shape)}") return np.ascontiguousarray(x) if hasattr(x, "numpy") and hasattr(x, "dtype") and not isinstance(x, (bytes, bytearray)): # torch tensor return _decode_image(np.asarray(x.numpy() if hasattr(x, "numpy") else x), i) from PIL import Image, UnidentifiedImageError try: if isinstance(x, Image.Image): im = x elif isinstance(x, (bytes, bytearray, memoryview)): import io im = Image.open(io.BytesIO(bytes(x))) elif isinstance(x, (str, os.PathLike)): im = Image.open(os.fspath(x)) else: raise InputError(f"image {name}: unsupported type {type(x).__name__} (array, PIL image, bytes or path)") w, h = im.size if max(w, h) > MAX_IMAGE_SIDE: raise InputError(f"image {name}: {w}x{h} is too large (max side {MAX_IMAGE_SIDE})") arr = np.asarray(im.convert("RGB"), dtype=np.uint8) except InputError: raise except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as e: raise InputError(f"image {name}: cannot decode ({type(e).__name__}: {e}); send PNG or JPEG") from None return np.ascontiguousarray(arr) def check_geometry(images: Sequence[np.ndarray], geom: Optional[Geometry] = None) -> None: """The concatenated cameras (side by side) must smart-resize to 392 x 756 (the one static ViT geometry of 2 cameras; ``geom.image_hw`` for another camera count).""" from .reference.preprocess import smart_resize geom = geom or GEOM2 grid = tuple(geom.image_hw) hs = {int(im.shape[0]) for im in images} if len(hs) != 1: raise InputError(f"the cameras must have the same height (they are concatenated side by side), got " f"{[tuple(im.shape[:2]) for im in images]}") h, w = hs.pop(), sum(int(im.shape[1]) for im in images) try: rh, rw = smart_resize(h, w, factor=CONFIG.vision.patch * CONFIG.vision.merge) except ValueError as e: raise InputError(f"unsupported image geometry {h}x{w}: {e}") from None if (rh, rw) != grid: n = geom.num_cameras raise InputError(f"unsupported image geometry: the cameras concatenated side by side are {h}x{w}, which " f"resizes to {rh}x{rw}; this configuration ({n} camera{'s' if n > 1 else ''}) serves only " f"geometries that resize to {grid[0]}x{grid[1]} (e.g. {n} x 384x384)") def decode_images(images: Any, geom: Optional[Geometry] = None) -> List[np.ndarray]: """Exactly the configured cameras, in the configured order: a sequence ``[left_stereo, right_stereo]`` or a mapping ``{"left_stereo": ..., "right_stereo": ...}`` (2 cameras; ``geom.camera_names`` for another count).""" geom = geom or GEOM2 order, ncam = tuple(geom.camera_names), geom.num_cameras if isinstance(images, Mapping): unknown = sorted(set(images) - set(order)) missing = [c for c in order if c not in images] if unknown or missing: raise InputError(f"cameras must be exactly {list(order)} (missing {missing}, unknown {unknown})") images = [images[c] for c in order] if isinstance(images, (str, bytes, np.ndarray)) or not isinstance(images, Sequence): raise InputError(f"images must be a list of {ncam} camera images {list(order)}") if len(images) != ncam: raise InputError(f"expected exactly {ncam} images in the order {list(order)}, got " f"{len(images)} (this server is configured for {ncam} camera{'s' if ncam > 1 else ''}, " "RDT2_NUM_CAMERAS)") if sum(isinstance(x, (bytes, bytearray, memoryview, str, os.PathLike)) for x in images) > 1: # encoded cameras decode in parallel (PIL releases the GIL while decoding: 2 x 3.7 -> 4.1 ms for 2 PNGs); # the same function per image, so the result and the error of the first bad image are unchanged out = list(_decode_pool().map(_decode_image, images, range(len(images)))) else: out = [_decode_image(x, i) for i, x in enumerate(images)] check_geometry(out, geom) return out _POOL = None def _decode_pool(): global _POOL if _POOL is None: from concurrent.futures import ThreadPoolExecutor _POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="rdt2-decode") # up to 4 cameras return _POOL def check_instruction(instruction: Any) -> str: if not isinstance(instruction, str): raise InputError(f"instruction must be a string, got {type(instruction).__name__}") if not instruction.strip(): raise InputError("instruction is empty") if len(instruction) > MAX_INSTRUCTION_CHARS: raise InputError(f"instruction of {len(instruction)} characters is too long (the largest prompt bucket holds " f"{max(PROMPT_BUCKETS) - PROMPT_FIXED_TOKENS} tokens)") m = SPECIAL_TOKEN.search(instruction) if m: raise InputError(f"instruction contains the special-token text {m.group(0)!r}; special tokens are not " "allowed inside the instruction (they would change the chat-template prompt)") return instruction def check_state(state: Any) -> np.ndarray: """Raw 20-d proprio state (not normalized; the official FM deployment sends zeros).""" if state is None: return np.zeros(STATE_DIM, np.float32) try: a = np.asarray(state, dtype=np.float64).reshape(-1) except (TypeError, ValueError): raise InputError(f"state must be {STATE_DIM} numbers") from None if a.size != STATE_DIM: raise InputError(f"state must have exactly {STATE_DIM} values, got {a.size}") if not np.all(np.isfinite(a)): raise InputError("state contains a non-finite value") return a.astype(np.float32) def check_seed(seed: Any) -> int: if seed is None: return DEFAULT_SEED if isinstance(seed, bool) or not isinstance(seed, (int, np.integer)): raise InputError(f"seed must be an integer, got {type(seed).__name__}") if not 0 <= int(seed) <= SEED_MAX: raise InputError(f"seed must be in [0, 2**63 - 1], got {seed}") return int(seed) def seeded_noise(seed: int): """The initial flow-matching noise of a request: ``torch.randn((1, 24, 20), generator=manual_seed(seed))`` fp32 (the golden driver's injected noise; seed 0 is the shipped sample's reference).""" import torch return torch.randn((1,) + CHUNK_SHAPE, generator=torch.Generator().manual_seed(int(seed))) def load_sample(path: Optional[Union[str, os.PathLike]] = None) -> Dict[str, Any]: """The shipped Apache-2.0 sample (BimanualUR5eExample sample 0) as call kwargs: ``images`` (left_stereo, right_stereo), ``instruction``, ``state`` (zeros).""" s = np.load(path or SAMPLE_PATH, allow_pickle=False) return {"images": [s["left_stereo"], s["right_stereo"]], "instruction": str(s["instruction"]), "state": np.asarray(s["state"], np.float32)} # ------------------------------------------------------------------------------------------------- output @dataclass class ActionChunk: """One action chunk. ``actions`` (24, 20) float32 = ``RDTInferencer.step``'s output: UMI-unnormalized, relative to the current end-effector poses, right arm in columns 0..9 and left arm in 10..19 (``ACTION_LAYOUT``).""" actions: np.ndarray actions_normalized: np.ndarray prompt_bucket: int num_tokens: int seed: int num_steps: int = NUM_STEPS timing_ms: Dict[str, float] = field(default_factory=dict) @property def shape(self) -> Tuple[int, int]: return tuple(self.actions.shape) def to_dict(self, include_normalized: bool = False) -> Dict[str, Any]: d: Dict[str, Any] = { "actions": self.actions.astype(np.float32).tolist(), "shape": list(self.actions.shape), "normalized": False, "units": dict(ACTION_UNITS), "action_layout": ACTION_LAYOUT, "chunk": {"horizon": CHUNK_SHAPE[0], "action_dim": CHUNK_SHAPE[1], "rate_hz": ACTION_RATE_HZ}, "prompt_bucket": int(self.prompt_bucket), "num_tokens": int(self.num_tokens), "num_steps": int(self.num_steps), "seed": int(self.seed), "timing_ms": {k: round(float(v), 3) for k, v in self.timing_ms.items()}, } if include_normalized: d["actions_normalized"] = self.actions_normalized.astype(np.float32).tolist() return d # ------------------------------------------------------------------------------------------------- the model _LIVE: "weakref.WeakSet[RDT2FM]" = weakref.WeakSet() @atexit.register def _close_all() -> None: # close at interpreter exit (traces released, device closed) for m in list(_LIVE): try: m.close() except Exception: # noqa: BLE001 pass class RDT2FM: """RDT2-FM on one p150. Build with :meth:`from_pretrained`; call ``model(images, instruction, state=None, seed=None, prompt_bucket=None, num_steps=None)`` -> :class:`ActionChunk`. Thread-safe (one lock), batch 1.""" MODEL_NAME = "rdt2-base-p150" DEFAULT_REPO = VLM_REPO DEFAULT_REVISION = VLM_REVISION CAMERA_ORDER = CAMERA_ORDER NUM_CAMERAS = NUM_CAMERAS NUM_STEPS = NUM_STEPS PROMPT_BUCKETS = PROMPT_BUCKETS CHUNK_SHAPE = CHUNK_SHAPE DEFAULT_SEED = DEFAULT_SEED def __init__(self, tt_model, device, *, weights: Dict[str, Dict[str, Any]], owns_device: bool, device_id: Optional[int], precision: str, log_fn=None): self._m = tt_model self.device = device self.weights = weights self._owns_device = owns_device self.device_id = device_id self.precision = precision self._log = log_fn or log.info self._lock = threading.RLock() self._closed = False self.warmup_ms: Dict[int, float] = {} self.capture_s: Optional[float] = None self.processor_dir = weights["processor"]["path"] # the configuration of this object (stage Config): fixed at construction, per request only the bucket self.geom: Geometry = getattr(tt_model, "geom", None) or GEOM2 self.CAMERA_ORDER = tuple(self.geom.camera_names) self.NUM_CAMERAS = self.geom.num_cameras self.NUM_STEPS = int(getattr(tt_model, "num_steps", NUM_STEPS)) self.PROMPT_BUCKETS = tuple(getattr(tt_model, "buckets", None) or self.geom.buckets) self.prompt_fixed_tokens = self.geom.prompt_fixed_tokens if owns_device: # closed at interpreter exit; a caller-owned device is the caller's to release _LIVE.add(self) # -------------------------------------------------------------------------------------------- construction @classmethod def from_pretrained(cls, model_id: Optional[str] = None, revision: Optional[str] = None, *, device_id: int = 0, device=None, dispatch: str = "eth", engine=None, precision: str = "baseline", weights_dirs: Optional[Mapping[str, str]] = None, local_files_only: bool = False, warmup: bool = True, allow_fallback: bool = False, log_fn=None, num_cameras: Optional[int] = None, num_steps: Optional[int] = None) -> "RDT2FM": """Resolve the pinned weights (no device), open the chip (ETH dispatch, 12 x 10 grid unless ``dispatch="worker"``), load every program with all prompt buckets resident and, with ``warmup`` (default), capture one trace per bucket and run one call per bucket: the first real call is as fast as later ones. ``device``: a caller-owned ttnn device (``close()`` leaves it open). ``engine``: per-program override (``"vision=ttnn"`` etc.; default ``vision=mk, prefix=fused, expert=mk``; ``prefix=mk`` needs ``RDT2_EXPERIMENTAL_PREFIX_MK=1``). ``precision``: the gated preset ``baseline`` (others are development presets of ``tt/common.py``). ``num_cameras`` (1..4) / ``num_steps`` (1..16): the fixed configuration of the object; None reads ``RDT2_NUM_CAMERAS`` / ``RDT2_NUM_STEPS`` (unset: 2 cameras, 5 steps).""" from .tt.model import RDT2FMTT, TRACE_REGION_SIZE, check_engine_allowed, resolve_engine from .ttaw.device import close_device, open_device say = log_fn or log.info env_cams, env_steps = geometry_from_env() try: ncam = env_cams if num_cameras is None else check_num_cameras(num_cameras) nsteps = env_steps if num_steps is None else check_num_steps(num_steps) except ValueError as e: raise InputError(str(e)) from None geom = geometry(ncam) eng = resolve_engine(engine) check_engine_allowed(eng) # before the weights and the device: the experimental engine is refused early weights = resolve_weights(model_id, revision, weights_dirs=weights_dirs, local_files_only=local_files_only, log_fn=say) owns = device is None if owns: say(f"Opening device {device_id} ({dispatch} dispatch) ...") device = open_device(int(device_id), dispatch=dispatch, num_command_queues=1, trace_region_size=TRACE_REGION_SIZE, allow_fallback=allow_fallback) try: g = device.compute_with_storage_grid_size() say(f"Device open: grid {g.x}x{g.y}; loading the programs (engines {eng}, precision {precision}) ...") say(f"Configuration: {ncam} camera(s) {list(geom.camera_names)}, {nsteps} Euler steps, prompt buckets " f"{list(geom.buckets)}") tt = RDT2FMTT(device, buckets=geom.buckets, precision=precision, num_steps=nsteps, num_cameras=ncam, vlm_dir=weights["vlm"]["path"], expert_dir=weights["expert"]["path"], normalizer_dir=weights["normalizer"]["path"], processor_dir=weights["processor"]["path"], engine=eng, log=lambda s: say(s)) model = cls(tt, device, weights=weights, owns_device=owns, device_id=int(device_id) if owns else None, precision=precision if isinstance(precision, str) else "custom", log_fn=say) except BaseException: if owns: close_device(device) raise if warmup: try: model.warmup() except BaseException: model.close() raise return model def warmup(self) -> Dict[int, float]: """Capture one trace per prompt bucket (kernel JIT on the first boot), then one call per bucket. Idempotent.""" with self._lock: self._check_open() if all(t is not None for t in self._m.traces.values()) and self.warmup_ms: return dict(self.warmup_ms) self._log(f"Warming up: capturing trace of prompt buckets {list(self.PROMPT_BUCKETS)} ...") t0 = time.perf_counter() self._m.capture() self.capture_s = time.perf_counter() - t0 dummy = [np.zeros((384, 384, 3), np.uint8)] * self.NUM_CAMERAS for L in self.PROMPT_BUCKETS: self._call(dummy, "warm up", np.zeros(STATE_DIM, np.float32), DEFAULT_SEED, L) t1 = time.perf_counter() self._call(dummy, "warm up", np.zeros(STATE_DIM, np.float32), DEFAULT_SEED, L) self.warmup_ms[L] = (time.perf_counter() - t1) * 1e3 self._log(f"Warmup complete in {time.perf_counter() - t0:.1f} s (one call per bucket, ms: " + ", ".join(f"{L}: {v:.1f}" for L, v in self.warmup_ms.items()) + ")") return dict(self.warmup_ms) # -------------------------------------------------------------------------------------------- calls def tokens_for(self, instruction: str) -> int: """Prompt length of an instruction (400 fixed tokens with 2 cameras + the instruction's own).""" from .reference.preprocess import tokenize return int(tokenize(instruction, self.geom.n_img, self.processor_dir).shape[1]) def validate(self, images: Any, instruction: Any, state: Any = None, seed: Any = None, prompt_bucket: Any = None, num_steps: Any = None) -> Dict[str, Any]: """Decode and check one request (host only, no lock): raises :class:`InputError` with the reason.""" S, B, F = self.NUM_STEPS, self.PROMPT_BUCKETS, self.prompt_fixed_tokens if num_steps is not None and (isinstance(num_steps, bool) or num_steps != S): raise InputError(f"num_steps must be {S} (this server runs a fixed {S}-step Euler flow-matching schedule, " f"set at start with RDT2_NUM_STEPS), got {num_steps!r}") imgs = decode_images(images, self.geom) instr = check_instruction(instruction) st = check_state(state) sd = check_seed(seed) n = self.tokens_for(instr) if n > max(B): raise InputError(f"prompt of {n} tokens ({n - F} instruction tokens) exceeds the largest " f"prompt bucket {max(B)} (at most {max(B) - F}" " instruction tokens); instructions are never truncated") if prompt_bucket is None: L = min(b for b in B if b >= n) else: if isinstance(prompt_bucket, bool) or not isinstance(prompt_bucket, (int, np.integer)) \ or int(prompt_bucket) not in B: raise InputError(f"prompt_bucket must be one of {list(B)}, got {prompt_bucket!r}") L = int(prompt_bucket) if n > L: raise InputError(f"prompt of {n} tokens does not fit prompt_bucket {L}") return {"images": imgs, "instruction": instr, "state": st, "seed": sd, "bucket": L, "n_tokens": n} def _call(self, images, instruction, state, seed, bucket): out = self._m(images, instruction, state=state, noise=seeded_noise(seed), bucket=bucket) return out, dict(self._m.timings) def __call__(self, images: Any, instruction: Optional[str] = None, state: Any = None, seed: Optional[int] = None, prompt_bucket: Optional[int] = None, num_steps: Optional[int] = None, *, prompt: Optional[str] = None) -> ActionChunk: t0 = time.perf_counter() if instruction is None: instruction = prompt elif prompt is not None: raise InputError("give either instruction or prompt, not both") if instruction is None: raise InputError("instruction is required") req = self.validate(images, instruction, state, seed, prompt_bucket, num_steps) t1 = time.perf_counter() with self._lock: self._check_open() out, tm = self._call(req["images"], req["instruction"], req["state"], req["seed"], req["bucket"]) t2 = time.perf_counter() if int(out["bucket"]) != req["bucket"] or int(out["n_tokens"]) != req["n_tokens"]: raise RuntimeError(f"internal: bucket / token count mismatch {out['bucket']}/{out['n_tokens']} vs " f"{req['bucket']}/{req['n_tokens']}") pre = (t1 - t0) * 1e3 + tm.get("host_pre_ms", 0.0) timing = {"preprocess": pre, "inference": (t2 - t1) * 1e3 - tm.get("host_pre_ms", 0.0), "total": (time.perf_counter() - t0) * 1e3} return ActionChunk(actions=out["action"].detach().float().numpy().astype(np.float32).reshape(CHUNK_SHAPE), actions_normalized=out["nsample"].detach().float().numpy().astype(np.float32) .reshape(CHUNK_SHAPE), prompt_bucket=req["bucket"], num_tokens=req["n_tokens"], seed=req["seed"], num_steps=self.NUM_STEPS, timing_ms=timing) # -------------------------------------------------------------------------------------------- lifetime / info def _check_open(self) -> None: if self._closed: raise RuntimeError("the model is closed") @property def closed(self) -> bool: return self._closed def close(self) -> None: """Release the traces and (if this object opened it) close the device. Idempotent.""" with self._lock: if self._closed: return self._closed = True try: self._m.release() finally: if self._owns_device and self.device is not None: from .ttaw.device import close_device close_device(self.device) self.device = None _LIVE.discard(self) def __enter__(self) -> "RDT2FM": return self def __exit__(self, *exc) -> None: self.close() @property def engine(self) -> Dict[str, str]: return dict(self._m.engine) @property def device_info(self) -> Optional[Dict[str, Any]]: if self.device is None: return None from .ttaw.device import describe_device return describe_device(self.device, self.device_id) @property def info(self) -> Dict[str, Any]: w = self.weights return { "model": self.MODEL_NAME, "version": __version__, "weights": {"repo": w["vlm"]["repo"], "revision": w["vlm"]["revision"], "components": {r: {k: v[k] for k in ("repo", "revision", "files", "source", "license", "use")} for r, v in w.items()}}, "engine": self.engine, "precision": self.precision, "device": self.device_info, "cameras": list(self.CAMERA_ORDER), "num_cameras": self.NUM_CAMERAS, "image_geometry": {"layout": "cameras concatenated side by side, " + " | ".join(self.CAMERA_ORDER), "resized_to": list(self.geom.image_hw), "image_tokens": self.geom.n_img, "patch_grid": list(self.geom.grid), "accepted": f"any {self.NUM_CAMERAS} equal-height RGB image(s) whose concatenation " f"resizes to {self.geom.image_hw[0]}x{self.geom.image_hw[1]} (e.g. " f"{self.NUM_CAMERAS} x 384x384)"}, "num_steps": self.NUM_STEPS, "configuration": {"num_cameras": self.NUM_CAMERAS, "num_steps": self.NUM_STEPS, "prompt_buckets": list(self.PROMPT_BUCKETS), "fixed_at": "start (RDT2_NUM_CAMERAS, " "RDT2_NUM_STEPS or the Python API's num_cameras / num_steps); per request only the " "prompt bucket", "default": self.NUM_CAMERAS == 2 and self.NUM_STEPS == 5, "engine_fallbacks": dict(getattr(self._m, "engine_fallbacks", {}) or {})}, "chunk_shape": list(CHUNK_SHAPE), "action_rate_hz": ACTION_RATE_HZ, "action_layout": ACTION_LAYOUT, "units": dict(ACTION_UNITS), "normalized": False, "state_dim": STATE_DIM, "default_seed": DEFAULT_SEED, "prompt_buckets": list(self.PROMPT_BUCKETS), "max_instruction_tokens": {str(L): L - self.prompt_fixed_tokens for L in self.PROMPT_BUCKETS}, "warmup_ms": {str(k): round(v, 2) for k, v in self.warmup_ms.items()}, "capture_s": None if self.capture_s is None else round(self.capture_s, 2), "load_s": round(float(self._m.timings.get("load_s", 0.0)), 2), "official_code": {"repo": OFFICIAL_CODE[0], "commit": OFFICIAL_CODE[1]}, }