Download code/tt_rdt2/api.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 35 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/api.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/api.py
-
curl -L -o api.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/api.py
35 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Python API of rdt2-base-p150: RDT2-FM (Qwen2.5-VL-7B prefix of RDT2-VQ + the RDT2-FM flow-matching action expert) | |
| on one Tenstorrent Blackhole p150, batch 1. | |
| from tt_rdt2 import RDT2FM, load_sample | |
| with RDT2FM.from_pretrained() as model: # weights resolved first, then the device, then warm-up | |
| out = model(**load_sample()) # images (left_stereo, right_stereo), instruction, state | |
| print(out.actions.shape, out.timing_ms) # (24, 20) float32, unnormalized (metres / 6-D rotation) | |
| What runs: the device path the gates validated (``tt.model.RDT2FMTT``: VISION | PREFIX | EXPERT, one Metal trace per | |
| prompt bucket, every bucket resident and warm) behind the exact host pre-processing of the official RDT2 inferencer | |
| (``reference.preprocess`` / ``reference.host_fold``: camera concat, uint8 bicubic-antialias resize, CLIP | |
| normalization, Qwen chat prompt) and the official UMI un-normalization. ``model(...)`` validates the request first | |
| (:class:`InputError` for anything a client got wrong), then calls ``RDT2FMTT.__call__`` under a lock. | |
| Configuration (load time, not per request; round 4 stage Config: ``num_cameras`` 1..4 and ``num_steps`` 1..16 of | |
| :meth:`RDT2FM.from_pretrained`, else RDT2_NUM_CAMERAS / RDT2_NUM_STEPS; reference/geometry.py derives the image | |
| geometry and the buckets of a camera count). The defaults, the published configuration: 2 cameras in the checkpoint's order | |
| (``left_stereo``, ``right_stereo``), concatenated side by side (each 384 x 384 in the official setup; any pair whose | |
| concatenation resizes to the same 392 x 756 grid is accepted and resized exactly as the official processor does), | |
| 5 Euler flow-matching steps, prompt buckets 416 / 448 / 512 tokens (instructions of up to 16 / 48 / 112 tokens), an | |
| action chunk of 24 x 20. The default engines are ``vision=mk, prefix=fused, expert=mk``; ``prefix=mk`` is experimental | |
| (known hang) and refused unless ``RDT2_EXPERIMENTAL_PREFIX_MK=1``. | |
| Weights (resolved BEFORE the device opens; pinned revisions; local snapshots work offline): | |
| * VLM ``robotics-diffusion-transformer/RDT2-VQ`` (vision tower + LM layers 0..13; only the 2 shards that hold them), | |
| * action expert ``robotics-diffusion-transformer/RDT2-FM`` (``pytorch_model.bin``, ``torch.load(weights_only=True)``), | |
| * action normalizer ``robotics-diffusion-transformer/RVQActionTokenizer`` (one file), | |
| * tokenizer ``Qwen/Qwen2.5-VL-7B-Instruct`` (``tokenizer.json`` only, no weights). | |
| Resolution per component: ``weights_dirs[role]`` > ``$RDT2_<ROLE>_DIR`` > (VLM only) a local ``model_id`` directory > | |
| ``hf_hub_download(repo, file, revision=<pin>)`` into the Hugging Face cache (a cached file at a pinned commit is used | |
| without any network access; on a network failure the cache is retried with ``local_files_only=True``). | |
| """ | |
| from __future__ import annotations | |
| import atexit | |
| import logging | |
| import math | |
| import os | |
| import re | |
| import threading | |
| import time | |
| import weakref | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union | |
| import numpy as np | |
| from . import __version__ | |
| from .reference.config import (CONFIG, EXPERT_REPO, EXPERT_REVISION, NORMALIZER_FILE, NORMALIZER_REPO, | |
| NORMALIZER_REVISION, OFFICIAL_CODE, PROCESSOR_REPO, PROCESSOR_REVISION, VLM_REPO, | |
| VLM_REVISION) | |
| from .reference.geometry import GEOM2, Geometry, check_num_cameras, check_num_steps, geometry | |
| from .reference.geometry import config_from_env as geometry_from_env | |
| __all__ = ["RDT2FM", "ActionChunk", "InputError", "load_sample", "resolve_weights", "vlm_needed_files", | |
| "SAMPLE_PATH", "PROMPT_BUCKETS", "CAMERA_ORDER", "NUM_STEPS", "CHUNK_SHAPE"] | |
| log = logging.getLogger("tt_rdt2") | |
| E = CONFIG.expert | |
| CAMERA_ORDER: Tuple[str, ...] = tuple(CONFIG.proc.camera_names) # ("left_stereo", "right_stereo") | |
| NUM_CAMERAS = len(CAMERA_ORDER) # 2 (fixed) | |
| NUM_STEPS = E.num_steps # 5 Euler steps (fixed) | |
| PROMPT_BUCKETS: Tuple[int, ...] = (416, 448, 512) # all resident, one trace each | |
| CHUNK_SHAPE: Tuple[int, int] = (E.horizon, E.action_dim) # (24, 20) | |
| STATE_DIM = E.state_dim # 20 | |
| DEFAULT_SEED = 0 # the fixed seeded noise (golden case seed) | |
| SEED_MAX = 2 ** 63 - 1 | |
| ACTION_RATE_HZ = 30 | |
| # prompt = 15 head tokens (system turn, user header, <|vision_start|>) + 378 image tokens + the instruction + 7 tail | |
| # tokens (<|vision_end|>, <|im_end|>, "\n", <|im_start|>, "assistant", "\n", <|quad_start|>) | |
| PROMPT_FIXED_TOKENS = 400 | |
| IMAGE_GRID = (392, 756) # the concatenated cameras must smart-resize to this (grid 28 x 54, 378 tokens) | |
| MAX_INSTRUCTION_CHARS = 4096 # refused before tokenization (the largest bucket holds 112 tokens anyway) | |
| MAX_IMAGE_SIDE = 8192 # header check before a full decode | |
| SPECIAL_TOKEN = re.compile(r"<\|[^<>|]*\|>") | |
| EXPERIMENTAL_PREFIX_MK_ENV = "RDT2_EXPERIMENTAL_PREFIX_MK" | |
| SAMPLE_PATH = Path(__file__).resolve().parent / "samples" / "ur5e_bimanual_example_0.npz" | |
| ACTION_LAYOUT = { | |
| "right_arm": {"position_m": [0, 3], "rotation_6d": [3, 9], "gripper_width_m": [9, 10]}, | |
| "left_arm": {"position_m": [10, 13], "rotation_6d": [13, 19], "gripper_width_m": [19, 20]}, | |
| } | |
| ACTION_UNITS = { | |
| "position": "metres, end-effector position relative to the current end-effector pose", | |
| "rotation": "6-D rotation representation (unitless), relative to the current end-effector pose", | |
| "gripper": "metres (gripper width)", | |
| } | |
| # component -> (repo, pinned revision, env override, license); the VLM file list is computed from its index | |
| COMPONENTS: Dict[str, Dict[str, Any]] = { | |
| "vlm": dict(repo=VLM_REPO, revision=VLM_REVISION, env="RDT2_VLM_DIR", license="apache-2.0", | |
| files=("config.json", "model.safetensors.index.json"), | |
| use="Qwen2.5-VL-7B vision tower and language-model layers 0..13 (K/V of layers 0..13)"), | |
| "expert": dict(repo=EXPERT_REPO, revision=EXPERT_REVISION, env="RDT2_EXPERT_DIR", license="apache-2.0", | |
| files=("config.json", "pytorch_model.bin"), use="RDT2-FM flow-matching action expert"), | |
| "normalizer": dict(repo=NORMALIZER_REPO, revision=NORMALIZER_REVISION, env="RDT2_NORMALIZER_DIR", | |
| license="apache-2.0", files=(NORMALIZER_FILE,), use="UMI action normalizer (one file)"), | |
| "processor": dict(repo=PROCESSOR_REPO, revision=PROCESSOR_REVISION, env="RDT2_PROCESSOR_DIR", | |
| license="apache-2.0", files=("tokenizer.json",), use="Qwen2.5-VL tokenizer (no weights)"), | |
| } | |
| VLM_PREFIXES = ("visual.", "model.embed_tokens.") + tuple(f"model.layers.{i}." for i in CONFIG.lm.selected_layers) | |
| class InputError(ValueError): | |
| """A request the model cannot serve as given (HTTP 400): the message says why.""" | |
| # ------------------------------------------------------------------------------------------------- weights | |
| def vlm_needed_files(index_path: Union[str, os.PathLike]) -> List[str]: | |
| """The VLM shards RDT2-FM reads (vision tower, embedding, LM layers 0..13), from the safetensors index.""" | |
| import json | |
| with open(index_path) as f: | |
| wm = json.load(f)["weight_map"] | |
| return sorted({fn for k, fn in wm.items() if k.startswith(VLM_PREFIXES)}) | |
| def _hub_file(repo: str, filename: str, revision: str, local_files_only: bool) -> str: | |
| from huggingface_hub import hf_hub_download | |
| try: | |
| return hf_hub_download(repo, filename, revision=revision, local_files_only=local_files_only) | |
| except Exception as e: # noqa: BLE001 -- offline / network failure: the cache may still hold the pinned file | |
| if local_files_only: | |
| raise | |
| try: | |
| return hf_hub_download(repo, filename, revision=revision, local_files_only=True) | |
| except Exception: # noqa: BLE001 | |
| raise RuntimeError(f"cannot resolve {repo}@{revision[:12]}/{filename}: {type(e).__name__}: {e}") from e | |
| def resolve_weights(model_id: Optional[str] = None, revision: Optional[str] = None, *, | |
| weights_dirs: Optional[Mapping[str, str]] = None, local_files_only: bool = False, | |
| env: Optional[Mapping[str, str]] = None, log_fn=None) -> Dict[str, Dict[str, Any]]: | |
| """Resolve every component to a local directory (no device). Returns ``{role: {repo, revision, path, files, | |
| source, license, use}}``. ``model_id`` / ``revision`` replace the VLM repo / revision (default: the pins).""" | |
| env = os.environ if env is None else env | |
| weights_dirs = dict(weights_dirs or {}) | |
| say = log_fn or log.info | |
| out: Dict[str, Dict[str, Any]] = {} | |
| for role, c in COMPONENTS.items(): | |
| repo, rev = c["repo"], c["revision"] | |
| if role == "vlm": | |
| repo = model_id or repo | |
| rev = revision or (VLM_REVISION if repo == VLM_REPO else None) or "main" | |
| local = weights_dirs.get(role) or env.get(c["env"]) or None | |
| if role == "vlm" and not local and model_id and os.path.isdir(model_id): | |
| local = model_id | |
| files = list(c["files"]) | |
| if local: | |
| path = Path(local) | |
| if role == "vlm": | |
| files += vlm_needed_files(path / "model.safetensors.index.json") | |
| missing = [f for f in files if not (path / f).is_file()] | |
| if missing: | |
| raise FileNotFoundError(f"{role} weights directory {path} lacks {missing}") | |
| source = "local" | |
| else: | |
| first = _hub_file(repo, files[0], rev, local_files_only) | |
| path = Path(first).parent | |
| if role == "vlm": | |
| idx = _hub_file(repo, "model.safetensors.index.json", rev, local_files_only) | |
| files += vlm_needed_files(idx) | |
| for f in files[1:]: | |
| _hub_file(repo, f, rev, local_files_only) | |
| source = "hf-cache" | |
| out[role] = dict(repo=repo, revision=rev, path=str(path), files=files, source=source, license=c["license"], | |
| use=c["use"]) | |
| say(f"Loading weights: {role} {repo}@{str(rev)[:12]} ({source}: {path})") | |
| return out | |
| # ------------------------------------------------------------------------------------------------- inputs | |
| def _decode_image(x: Any, i: int) -> np.ndarray: | |
| """One camera image -> (H, W, 3) uint8 RGB. ndarray uint8 HxWx3 as is; PIL image / encoded bytes / path via PIL | |
| ``convert("RGB")`` (what the official inferencer applies to its camera arrays).""" | |
| name = CAMERA_ORDER[i] if i < len(CAMERA_ORDER) else f"#{i}" | |
| if isinstance(x, np.ndarray): | |
| if x.dtype != np.uint8 or x.ndim != 3 or x.shape[2] != 3: | |
| raise InputError(f"image {name}: expected an HxWx3 uint8 array, got {x.dtype} {tuple(x.shape)}") | |
| return np.ascontiguousarray(x) | |
| if hasattr(x, "numpy") and hasattr(x, "dtype") and not isinstance(x, (bytes, bytearray)): # torch tensor | |
| return _decode_image(np.asarray(x.numpy() if hasattr(x, "numpy") else x), i) | |
| from PIL import Image, UnidentifiedImageError | |
| try: | |
| if isinstance(x, Image.Image): | |
| im = x | |
| elif isinstance(x, (bytes, bytearray, memoryview)): | |
| import io | |
| im = Image.open(io.BytesIO(bytes(x))) | |
| elif isinstance(x, (str, os.PathLike)): | |
| im = Image.open(os.fspath(x)) | |
| else: | |
| raise InputError(f"image {name}: unsupported type {type(x).__name__} (array, PIL image, bytes or path)") | |
| w, h = im.size | |
| if max(w, h) > MAX_IMAGE_SIDE: | |
| raise InputError(f"image {name}: {w}x{h} is too large (max side {MAX_IMAGE_SIDE})") | |
| arr = np.asarray(im.convert("RGB"), dtype=np.uint8) | |
| except InputError: | |
| raise | |
| except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as e: | |
| raise InputError(f"image {name}: cannot decode ({type(e).__name__}: {e}); send PNG or JPEG") from None | |
| return np.ascontiguousarray(arr) | |
| def check_geometry(images: Sequence[np.ndarray], geom: Optional[Geometry] = None) -> None: | |
| """The concatenated cameras (side by side) must smart-resize to 392 x 756 (the one static ViT geometry of 2 | |
| cameras; ``geom.image_hw`` for another camera count).""" | |
| from .reference.preprocess import smart_resize | |
| geom = geom or GEOM2 | |
| grid = tuple(geom.image_hw) | |
| hs = {int(im.shape[0]) for im in images} | |
| if len(hs) != 1: | |
| raise InputError(f"the cameras must have the same height (they are concatenated side by side), got " | |
| f"{[tuple(im.shape[:2]) for im in images]}") | |
| h, w = hs.pop(), sum(int(im.shape[1]) for im in images) | |
| try: | |
| rh, rw = smart_resize(h, w, factor=CONFIG.vision.patch * CONFIG.vision.merge) | |
| except ValueError as e: | |
| raise InputError(f"unsupported image geometry {h}x{w}: {e}") from None | |
| if (rh, rw) != grid: | |
| n = geom.num_cameras | |
| raise InputError(f"unsupported image geometry: the cameras concatenated side by side are {h}x{w}, which " | |
| f"resizes to {rh}x{rw}; this configuration ({n} camera{'s' if n > 1 else ''}) serves only " | |
| f"geometries that resize to {grid[0]}x{grid[1]} (e.g. {n} x 384x384)") | |
| def decode_images(images: Any, geom: Optional[Geometry] = None) -> List[np.ndarray]: | |
| """Exactly the configured cameras, in the configured order: a sequence ``[left_stereo, right_stereo]`` or a | |
| mapping ``{"left_stereo": ..., "right_stereo": ...}`` (2 cameras; ``geom.camera_names`` for another count).""" | |
| geom = geom or GEOM2 | |
| order, ncam = tuple(geom.camera_names), geom.num_cameras | |
| if isinstance(images, Mapping): | |
| unknown = sorted(set(images) - set(order)) | |
| missing = [c for c in order if c not in images] | |
| if unknown or missing: | |
| raise InputError(f"cameras must be exactly {list(order)} (missing {missing}, unknown {unknown})") | |
| images = [images[c] for c in order] | |
| if isinstance(images, (str, bytes, np.ndarray)) or not isinstance(images, Sequence): | |
| raise InputError(f"images must be a list of {ncam} camera images {list(order)}") | |
| if len(images) != ncam: | |
| raise InputError(f"expected exactly {ncam} images in the order {list(order)}, got " | |
| f"{len(images)} (this server is configured for {ncam} camera{'s' if ncam > 1 else ''}, " | |
| "RDT2_NUM_CAMERAS)") | |
| if sum(isinstance(x, (bytes, bytearray, memoryview, str, os.PathLike)) for x in images) > 1: | |
| # encoded cameras decode in parallel (PIL releases the GIL while decoding: 2 x 3.7 -> 4.1 ms for 2 PNGs); | |
| # the same function per image, so the result and the error of the first bad image are unchanged | |
| out = list(_decode_pool().map(_decode_image, images, range(len(images)))) | |
| else: | |
| out = [_decode_image(x, i) for i, x in enumerate(images)] | |
| check_geometry(out, geom) | |
| return out | |
| _POOL = None | |
| def _decode_pool(): | |
| global _POOL | |
| if _POOL is None: | |
| from concurrent.futures import ThreadPoolExecutor | |
| _POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="rdt2-decode") # up to 4 cameras | |
| return _POOL | |
| def check_instruction(instruction: Any) -> str: | |
| if not isinstance(instruction, str): | |
| raise InputError(f"instruction must be a string, got {type(instruction).__name__}") | |
| if not instruction.strip(): | |
| raise InputError("instruction is empty") | |
| if len(instruction) > MAX_INSTRUCTION_CHARS: | |
| raise InputError(f"instruction of {len(instruction)} characters is too long (the largest prompt bucket holds " | |
| f"{max(PROMPT_BUCKETS) - PROMPT_FIXED_TOKENS} tokens)") | |
| m = SPECIAL_TOKEN.search(instruction) | |
| if m: | |
| raise InputError(f"instruction contains the special-token text {m.group(0)!r}; special tokens are not " | |
| "allowed inside the instruction (they would change the chat-template prompt)") | |
| return instruction | |
| def check_state(state: Any) -> np.ndarray: | |
| """Raw 20-d proprio state (not normalized; the official FM deployment sends zeros).""" | |
| if state is None: | |
| return np.zeros(STATE_DIM, np.float32) | |
| try: | |
| a = np.asarray(state, dtype=np.float64).reshape(-1) | |
| except (TypeError, ValueError): | |
| raise InputError(f"state must be {STATE_DIM} numbers") from None | |
| if a.size != STATE_DIM: | |
| raise InputError(f"state must have exactly {STATE_DIM} values, got {a.size}") | |
| if not np.all(np.isfinite(a)): | |
| raise InputError("state contains a non-finite value") | |
| return a.astype(np.float32) | |
| def check_seed(seed: Any) -> int: | |
| if seed is None: | |
| return DEFAULT_SEED | |
| if isinstance(seed, bool) or not isinstance(seed, (int, np.integer)): | |
| raise InputError(f"seed must be an integer, got {type(seed).__name__}") | |
| if not 0 <= int(seed) <= SEED_MAX: | |
| raise InputError(f"seed must be in [0, 2**63 - 1], got {seed}") | |
| return int(seed) | |
| def seeded_noise(seed: int): | |
| """The initial flow-matching noise of a request: ``torch.randn((1, 24, 20), generator=manual_seed(seed))`` fp32 | |
| (the golden driver's injected noise; seed 0 is the shipped sample's reference).""" | |
| import torch | |
| return torch.randn((1,) + CHUNK_SHAPE, generator=torch.Generator().manual_seed(int(seed))) | |
| def load_sample(path: Optional[Union[str, os.PathLike]] = None) -> Dict[str, Any]: | |
| """The shipped Apache-2.0 sample (BimanualUR5eExample sample 0) as call kwargs: ``images`` (left_stereo, | |
| right_stereo), ``instruction``, ``state`` (zeros).""" | |
| s = np.load(path or SAMPLE_PATH, allow_pickle=False) | |
| return {"images": [s["left_stereo"], s["right_stereo"]], "instruction": str(s["instruction"]), | |
| "state": np.asarray(s["state"], np.float32)} | |
| # ------------------------------------------------------------------------------------------------- output | |
| class ActionChunk: | |
| """One action chunk. ``actions`` (24, 20) float32 = ``RDTInferencer.step``'s output: UMI-unnormalized, relative to | |
| the current end-effector poses, right arm in columns 0..9 and left arm in 10..19 (``ACTION_LAYOUT``).""" | |
| actions: np.ndarray | |
| actions_normalized: np.ndarray | |
| prompt_bucket: int | |
| num_tokens: int | |
| seed: int | |
| num_steps: int = NUM_STEPS | |
| timing_ms: Dict[str, float] = field(default_factory=dict) | |
| def shape(self) -> Tuple[int, int]: | |
| return tuple(self.actions.shape) | |
| def to_dict(self, include_normalized: bool = False) -> Dict[str, Any]: | |
| d: Dict[str, Any] = { | |
| "actions": self.actions.astype(np.float32).tolist(), | |
| "shape": list(self.actions.shape), | |
| "normalized": False, | |
| "units": dict(ACTION_UNITS), | |
| "action_layout": ACTION_LAYOUT, | |
| "chunk": {"horizon": CHUNK_SHAPE[0], "action_dim": CHUNK_SHAPE[1], "rate_hz": ACTION_RATE_HZ}, | |
| "prompt_bucket": int(self.prompt_bucket), | |
| "num_tokens": int(self.num_tokens), | |
| "num_steps": int(self.num_steps), | |
| "seed": int(self.seed), | |
| "timing_ms": {k: round(float(v), 3) for k, v in self.timing_ms.items()}, | |
| } | |
| if include_normalized: | |
| d["actions_normalized"] = self.actions_normalized.astype(np.float32).tolist() | |
| return d | |
| # ------------------------------------------------------------------------------------------------- the model | |
| _LIVE: "weakref.WeakSet[RDT2FM]" = weakref.WeakSet() | |
| def _close_all() -> None: # close at interpreter exit (traces released, device closed) | |
| for m in list(_LIVE): | |
| try: | |
| m.close() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| class RDT2FM: | |
| """RDT2-FM on one p150. Build with :meth:`from_pretrained`; call ``model(images, instruction, state=None, | |
| seed=None, prompt_bucket=None, num_steps=None)`` -> :class:`ActionChunk`. Thread-safe (one lock), batch 1.""" | |
| MODEL_NAME = "rdt2-base-p150" | |
| DEFAULT_REPO = VLM_REPO | |
| DEFAULT_REVISION = VLM_REVISION | |
| CAMERA_ORDER = CAMERA_ORDER | |
| NUM_CAMERAS = NUM_CAMERAS | |
| NUM_STEPS = NUM_STEPS | |
| PROMPT_BUCKETS = PROMPT_BUCKETS | |
| CHUNK_SHAPE = CHUNK_SHAPE | |
| DEFAULT_SEED = DEFAULT_SEED | |
| def __init__(self, tt_model, device, *, weights: Dict[str, Dict[str, Any]], owns_device: bool, | |
| device_id: Optional[int], precision: str, log_fn=None): | |
| self._m = tt_model | |
| self.device = device | |
| self.weights = weights | |
| self._owns_device = owns_device | |
| self.device_id = device_id | |
| self.precision = precision | |
| self._log = log_fn or log.info | |
| self._lock = threading.RLock() | |
| self._closed = False | |
| self.warmup_ms: Dict[int, float] = {} | |
| self.capture_s: Optional[float] = None | |
| self.processor_dir = weights["processor"]["path"] | |
| # the configuration of this object (stage Config): fixed at construction, per request only the bucket | |
| self.geom: Geometry = getattr(tt_model, "geom", None) or GEOM2 | |
| self.CAMERA_ORDER = tuple(self.geom.camera_names) | |
| self.NUM_CAMERAS = self.geom.num_cameras | |
| self.NUM_STEPS = int(getattr(tt_model, "num_steps", NUM_STEPS)) | |
| self.PROMPT_BUCKETS = tuple(getattr(tt_model, "buckets", None) or self.geom.buckets) | |
| self.prompt_fixed_tokens = self.geom.prompt_fixed_tokens | |
| if owns_device: # closed at interpreter exit; a caller-owned device is the caller's to release | |
| _LIVE.add(self) | |
| # -------------------------------------------------------------------------------------------- construction | |
| def from_pretrained(cls, model_id: Optional[str] = None, revision: Optional[str] = None, *, device_id: int = 0, | |
| device=None, dispatch: str = "eth", engine=None, precision: str = "baseline", | |
| weights_dirs: Optional[Mapping[str, str]] = None, local_files_only: bool = False, | |
| warmup: bool = True, allow_fallback: bool = False, log_fn=None, | |
| num_cameras: Optional[int] = None, num_steps: Optional[int] = None) -> "RDT2FM": | |
| """Resolve the pinned weights (no device), open the chip (ETH dispatch, 12 x 10 grid unless | |
| ``dispatch="worker"``), load every program with all prompt buckets resident and, with ``warmup`` (default), | |
| capture one trace per bucket and run one call per bucket: the first real call is as fast as later ones. | |
| ``device``: a caller-owned ttnn device (``close()`` leaves it open). ``engine``: per-program override | |
| (``"vision=ttnn"`` etc.; default ``vision=mk, prefix=fused, expert=mk``; ``prefix=mk`` needs | |
| ``RDT2_EXPERIMENTAL_PREFIX_MK=1``). ``precision``: the gated preset ``baseline`` (others are development | |
| presets of ``tt/common.py``). ``num_cameras`` (1..4) / ``num_steps`` (1..16): the fixed configuration of the | |
| object; None reads ``RDT2_NUM_CAMERAS`` / ``RDT2_NUM_STEPS`` (unset: 2 cameras, 5 steps).""" | |
| from .tt.model import RDT2FMTT, TRACE_REGION_SIZE, check_engine_allowed, resolve_engine | |
| from .ttaw.device import close_device, open_device | |
| say = log_fn or log.info | |
| env_cams, env_steps = geometry_from_env() | |
| try: | |
| ncam = env_cams if num_cameras is None else check_num_cameras(num_cameras) | |
| nsteps = env_steps if num_steps is None else check_num_steps(num_steps) | |
| except ValueError as e: | |
| raise InputError(str(e)) from None | |
| geom = geometry(ncam) | |
| eng = resolve_engine(engine) | |
| check_engine_allowed(eng) # before the weights and the device: the experimental engine is refused early | |
| weights = resolve_weights(model_id, revision, weights_dirs=weights_dirs, local_files_only=local_files_only, | |
| log_fn=say) | |
| owns = device is None | |
| if owns: | |
| say(f"Opening device {device_id} ({dispatch} dispatch) ...") | |
| device = open_device(int(device_id), dispatch=dispatch, num_command_queues=1, | |
| trace_region_size=TRACE_REGION_SIZE, allow_fallback=allow_fallback) | |
| try: | |
| g = device.compute_with_storage_grid_size() | |
| say(f"Device open: grid {g.x}x{g.y}; loading the programs (engines {eng}, precision {precision}) ...") | |
| say(f"Configuration: {ncam} camera(s) {list(geom.camera_names)}, {nsteps} Euler steps, prompt buckets " | |
| f"{list(geom.buckets)}") | |
| tt = RDT2FMTT(device, buckets=geom.buckets, precision=precision, num_steps=nsteps, num_cameras=ncam, | |
| vlm_dir=weights["vlm"]["path"], expert_dir=weights["expert"]["path"], | |
| normalizer_dir=weights["normalizer"]["path"], | |
| processor_dir=weights["processor"]["path"], engine=eng, | |
| log=lambda s: say(s)) | |
| model = cls(tt, device, weights=weights, owns_device=owns, device_id=int(device_id) if owns else None, | |
| precision=precision if isinstance(precision, str) else "custom", log_fn=say) | |
| except BaseException: | |
| if owns: | |
| close_device(device) | |
| raise | |
| if warmup: | |
| try: | |
| model.warmup() | |
| except BaseException: | |
| model.close() | |
| raise | |
| return model | |
| def warmup(self) -> Dict[int, float]: | |
| """Capture one trace per prompt bucket (kernel JIT on the first boot), then one call per bucket. Idempotent.""" | |
| with self._lock: | |
| self._check_open() | |
| if all(t is not None for t in self._m.traces.values()) and self.warmup_ms: | |
| return dict(self.warmup_ms) | |
| self._log(f"Warming up: capturing trace of prompt buckets {list(self.PROMPT_BUCKETS)} ...") | |
| t0 = time.perf_counter() | |
| self._m.capture() | |
| self.capture_s = time.perf_counter() - t0 | |
| dummy = [np.zeros((384, 384, 3), np.uint8)] * self.NUM_CAMERAS | |
| for L in self.PROMPT_BUCKETS: | |
| self._call(dummy, "warm up", np.zeros(STATE_DIM, np.float32), DEFAULT_SEED, L) | |
| t1 = time.perf_counter() | |
| self._call(dummy, "warm up", np.zeros(STATE_DIM, np.float32), DEFAULT_SEED, L) | |
| self.warmup_ms[L] = (time.perf_counter() - t1) * 1e3 | |
| self._log(f"Warmup complete in {time.perf_counter() - t0:.1f} s (one call per bucket, ms: " | |
| + ", ".join(f"{L}: {v:.1f}" for L, v in self.warmup_ms.items()) + ")") | |
| return dict(self.warmup_ms) | |
| # -------------------------------------------------------------------------------------------- calls | |
| def tokens_for(self, instruction: str) -> int: | |
| """Prompt length of an instruction (400 fixed tokens with 2 cameras + the instruction's own).""" | |
| from .reference.preprocess import tokenize | |
| return int(tokenize(instruction, self.geom.n_img, self.processor_dir).shape[1]) | |
| def validate(self, images: Any, instruction: Any, state: Any = None, seed: Any = None, | |
| prompt_bucket: Any = None, num_steps: Any = None) -> Dict[str, Any]: | |
| """Decode and check one request (host only, no lock): raises :class:`InputError` with the reason.""" | |
| S, B, F = self.NUM_STEPS, self.PROMPT_BUCKETS, self.prompt_fixed_tokens | |
| if num_steps is not None and (isinstance(num_steps, bool) or num_steps != S): | |
| raise InputError(f"num_steps must be {S} (this server runs a fixed {S}-step Euler flow-matching schedule, " | |
| f"set at start with RDT2_NUM_STEPS), got {num_steps!r}") | |
| imgs = decode_images(images, self.geom) | |
| instr = check_instruction(instruction) | |
| st = check_state(state) | |
| sd = check_seed(seed) | |
| n = self.tokens_for(instr) | |
| if n > max(B): | |
| raise InputError(f"prompt of {n} tokens ({n - F} instruction tokens) exceeds the largest " | |
| f"prompt bucket {max(B)} (at most {max(B) - F}" | |
| " instruction tokens); instructions are never truncated") | |
| if prompt_bucket is None: | |
| L = min(b for b in B if b >= n) | |
| else: | |
| if isinstance(prompt_bucket, bool) or not isinstance(prompt_bucket, (int, np.integer)) \ | |
| or int(prompt_bucket) not in B: | |
| raise InputError(f"prompt_bucket must be one of {list(B)}, got {prompt_bucket!r}") | |
| L = int(prompt_bucket) | |
| if n > L: | |
| raise InputError(f"prompt of {n} tokens does not fit prompt_bucket {L}") | |
| return {"images": imgs, "instruction": instr, "state": st, "seed": sd, "bucket": L, "n_tokens": n} | |
| def _call(self, images, instruction, state, seed, bucket): | |
| out = self._m(images, instruction, state=state, noise=seeded_noise(seed), bucket=bucket) | |
| return out, dict(self._m.timings) | |
| def __call__(self, images: Any, instruction: Optional[str] = None, state: Any = None, seed: Optional[int] = None, | |
| prompt_bucket: Optional[int] = None, num_steps: Optional[int] = None, *, | |
| prompt: Optional[str] = None) -> ActionChunk: | |
| t0 = time.perf_counter() | |
| if instruction is None: | |
| instruction = prompt | |
| elif prompt is not None: | |
| raise InputError("give either instruction or prompt, not both") | |
| if instruction is None: | |
| raise InputError("instruction is required") | |
| req = self.validate(images, instruction, state, seed, prompt_bucket, num_steps) | |
| t1 = time.perf_counter() | |
| with self._lock: | |
| self._check_open() | |
| out, tm = self._call(req["images"], req["instruction"], req["state"], req["seed"], req["bucket"]) | |
| t2 = time.perf_counter() | |
| if int(out["bucket"]) != req["bucket"] or int(out["n_tokens"]) != req["n_tokens"]: | |
| raise RuntimeError(f"internal: bucket / token count mismatch {out['bucket']}/{out['n_tokens']} vs " | |
| f"{req['bucket']}/{req['n_tokens']}") | |
| pre = (t1 - t0) * 1e3 + tm.get("host_pre_ms", 0.0) | |
| timing = {"preprocess": pre, "inference": (t2 - t1) * 1e3 - tm.get("host_pre_ms", 0.0), | |
| "total": (time.perf_counter() - t0) * 1e3} | |
| return ActionChunk(actions=out["action"].detach().float().numpy().astype(np.float32).reshape(CHUNK_SHAPE), | |
| actions_normalized=out["nsample"].detach().float().numpy().astype(np.float32) | |
| .reshape(CHUNK_SHAPE), | |
| prompt_bucket=req["bucket"], num_tokens=req["n_tokens"], seed=req["seed"], | |
| num_steps=self.NUM_STEPS, timing_ms=timing) | |
| # -------------------------------------------------------------------------------------------- lifetime / info | |
| def _check_open(self) -> None: | |
| if self._closed: | |
| raise RuntimeError("the model is closed") | |
| def closed(self) -> bool: | |
| return self._closed | |
| def close(self) -> None: | |
| """Release the traces and (if this object opened it) close the device. Idempotent.""" | |
| with self._lock: | |
| if self._closed: | |
| return | |
| self._closed = True | |
| try: | |
| self._m.release() | |
| finally: | |
| if self._owns_device and self.device is not None: | |
| from .ttaw.device import close_device | |
| close_device(self.device) | |
| self.device = None | |
| _LIVE.discard(self) | |
| def __enter__(self) -> "RDT2FM": | |
| return self | |
| def __exit__(self, *exc) -> None: | |
| self.close() | |
| def engine(self) -> Dict[str, str]: | |
| return dict(self._m.engine) | |
| def device_info(self) -> Optional[Dict[str, Any]]: | |
| if self.device is None: | |
| return None | |
| from .ttaw.device import describe_device | |
| return describe_device(self.device, self.device_id) | |
| def info(self) -> Dict[str, Any]: | |
| w = self.weights | |
| return { | |
| "model": self.MODEL_NAME, | |
| "version": __version__, | |
| "weights": {"repo": w["vlm"]["repo"], "revision": w["vlm"]["revision"], | |
| "components": {r: {k: v[k] for k in ("repo", "revision", "files", "source", "license", "use")} | |
| for r, v in w.items()}}, | |
| "engine": self.engine, | |
| "precision": self.precision, | |
| "device": self.device_info, | |
| "cameras": list(self.CAMERA_ORDER), | |
| "num_cameras": self.NUM_CAMERAS, | |
| "image_geometry": {"layout": "cameras concatenated side by side, " + " | ".join(self.CAMERA_ORDER), | |
| "resized_to": list(self.geom.image_hw), "image_tokens": self.geom.n_img, | |
| "patch_grid": list(self.geom.grid), | |
| "accepted": f"any {self.NUM_CAMERAS} equal-height RGB image(s) whose concatenation " | |
| f"resizes to {self.geom.image_hw[0]}x{self.geom.image_hw[1]} (e.g. " | |
| f"{self.NUM_CAMERAS} x 384x384)"}, | |
| "num_steps": self.NUM_STEPS, | |
| "configuration": {"num_cameras": self.NUM_CAMERAS, "num_steps": self.NUM_STEPS, | |
| "prompt_buckets": list(self.PROMPT_BUCKETS), "fixed_at": "start (RDT2_NUM_CAMERAS, " | |
| "RDT2_NUM_STEPS or the Python API's num_cameras / num_steps); per request only the " | |
| "prompt bucket", "default": self.NUM_CAMERAS == 2 and self.NUM_STEPS == 5, | |
| "engine_fallbacks": dict(getattr(self._m, "engine_fallbacks", {}) or {})}, | |
| "chunk_shape": list(CHUNK_SHAPE), | |
| "action_rate_hz": ACTION_RATE_HZ, | |
| "action_layout": ACTION_LAYOUT, | |
| "units": dict(ACTION_UNITS), | |
| "normalized": False, | |
| "state_dim": STATE_DIM, | |
| "default_seed": DEFAULT_SEED, | |
| "prompt_buckets": list(self.PROMPT_BUCKETS), | |
| "max_instruction_tokens": {str(L): L - self.prompt_fixed_tokens for L in self.PROMPT_BUCKETS}, | |
| "warmup_ms": {str(k): round(v, 2) for k, v in self.warmup_ms.items()}, | |
| "capture_s": None if self.capture_s is None else round(self.capture_s, 2), | |
| "load_s": round(float(self._m.timings.get("load_s", 0.0)), 2), | |
| "official_code": {"repo": OFFICIAL_CODE[0], "commit": OFFICIAL_CODE[1]}, | |
| } | |