# SPDX-License-Identifier: Apache-2.0 """HTTP server of rdt2-base-p150 (FastAPI; tt-model kind ``tt-dit-server``: ``python -m uvicorn --host 0.0.0.0 --port 20000 --lifespan on tt_rdt2.server.app:app``). Routes: ``GET /health`` (and ``/v1/health``; always 200, ``ok`` only after warm-up), ``GET /info``, ``GET /v1/models`` (an OpenAI-shaped stub so tt-model's ready card does not 404: NOT a chat API), ``POST /predict``. The contract is in SERVING.md ("Request and response contract"). Everything device-touching happens in the lifespan (weights -> device -> programs -> one trace per prompt bucket -> one warm call per bucket), so uvicorn logs ``Application startup complete`` only when every bucket is warm. Shutdown (SIGTERM, tt-model's 120 s budget) closes the model under the lock. Importing this module has no side effects beyond importing fastapi / pydantic (no device, no download, no environment read). ``POST /predict`` decodes the request (base64 PNG / JPEG -> RGB uint8) and calls the Python API (:class:`tt_rdt2.api.RDT2FM`), which runs the same host pre-processing and device path as the device gates; one lock serializes the requests (batch 1, one chip). Environment (read once, in the lifespan; the serve profile pins the fixed configuration): HF_MODEL, TT_MODEL_WEIGHTS_REVISION (then TT_WEIGHTS_REVISION) the VLM repo / revision (default: the pins) RDT2_VLM_DIR, RDT2_EXPERT_DIR, RDT2_NORMALIZER_DIR, RDT2_PROCESSOR_DIR local weight directories (optional) TT_MESH_SHAPE (only 1x1), TT_DEVICE_ID (0), RDT2_DISPATCH (eth | worker; default eth) RDT2_ENGINE ("" = vision=mk,prefix=fused,expert=mk; prefix=mk is refused unless RDT2_EXPERIMENTAL_PREFIX_MK=1) RDT2_PRECISION (only "baseline"): any other value is refused at start RDT2_NUM_CAMERAS (1..4, default 2), RDT2_NUM_STEPS (1..16, default 5): the fixed configuration of the served model (reference/geometry.py; SERVING.md "Configuration"); RDT2_PROMPT_BUCKETS: unset or exactly the camera count's buckets (416,448,512 for 2 cameras); anything else is refused at start. RDT2_MODEL=vq serves 2 cameras only and has no flow-matching steps (RDT2_NUM_STEPS must be unset or 5) RDT2_MODEL (fm | vq; default fm): fm = RDT2-FM (flow matching, the package default); vq = RDT2-VQ (the full Qwen2.5-VL-7B emits 27 RVQ action tokens, tt_rdt2.vq; same /predict request, the response adds "tokens") RDT2_MAX_BODY_MB (default 32; a larger Content-Length is answered 400, see BODY_DRAIN_FACTOR), RDT2_TORCH_THREADS (host pre-processing threads; default torch's) Tests replace the model: ``app.state.rdt2.model_factory = model>`` before ``TestClient(app)``. """ from __future__ import annotations import base64 import binascii import logging import os import threading import time from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Mapping, Optional from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse from pydantic import BaseModel, ConfigDict from .. import __version__ from ..ttaw.server.app import parse_mesh_shape # noqa: F401 (re-exported: the image's verify: line imports it) MODEL_NAME = "rdt2-base-p150" ENV = "RDT2" HARDWARE = "Tenstorrent Blackhole p150 (single chip) via tt-nn" TASK = "vision-language-action: 2 camera images + an instruction (+ a 20-d proprio state) -> a 24 x 20 action chunk" # PRECISION: "baseline" is the only preset gated on the device and stress-gated ("fast" fails the long-prompt gate) FIXED = {"PRECISION": "baseline"} # stage Config: RDT2_NUM_CAMERAS / RDT2_NUM_STEPS are configurable (reference/geometry.py); the buckets follow the # camera count (RDT2_PROMPT_BUCKETS may only restate them) # A body above the limit is answered 400. Bodies up to this many times the limit are read and discarded first, so a # client that sends the whole body before reading the answer receives the 400 instead of a connection reset; # larger ones are refused without reading (the connection is closed). BODY_DRAIN_FACTOR = 4 # RDT2_MODEL: the served variant (read once in the lifespan). "vq" = tt_rdt2.vq (round 4, SERVING.md "RDT2-VQ") MODEL_VARIANTS = ("fm", "vq") LICENSE = {"package": "apache-2.0", "weights": "apache-2.0 (RDT2-VQ, RDT2-FM, RVQActionTokenizer, " "Qwen2.5-VL-7B-Instruct tokenizer)"} log = logging.getLogger("tt_rdt2.server") class PredictRequest(BaseModel): """``POST /predict`` body. Unknown fields are a 422 (a misspelled option is never silently ignored).""" model_config = ConfigDict(extra="forbid") images: List[str] # base64 PNG / JPEG, exactly [left_stereo, right_stereo] instruction: Optional[str] = None # the task in natural language (alias: prompt) prompt: Optional[str] = None state: Optional[List[float]] = None # raw 20-d proprio state; default zeros (the official FM deployment) seed: Optional[int] = None # flow-matching noise seed; default 0 (the fixed seeded noise) prompt_bucket: Optional[int] = None # 416 / 448 / 512; default the smallest that holds the prompt num_steps: Optional[int] = None # must be 5 class InputError(ValueError): pass def config_from_env(env: Optional[Mapping[str, str]] = None) -> Dict[str, Any]: """The server configuration (read once, in the lifespan). Raises RuntimeError for a refused setting.""" env = os.environ if env is None else env for k, want in FIXED.items(): got = (env.get(f"{ENV}_{k}") or want).replace(" ", "") if got != want: raise RuntimeError(f"{ENV}_{k}={got!r} is not supported: this package is built for {ENV}_{k}={want} only " "(fixed configuration; see SERVING.md)") from ..reference.geometry import DEFAULT_NUM_STEPS, geometry from ..reference.geometry import config_from_env as geometry_from_env try: num_cameras, num_steps = geometry_from_env(env) except ValueError as e: raise RuntimeError(f"{e} (see SERVING.md, Configuration)") from None buckets = ",".join(str(b) for b in geometry(num_cameras).buckets) got = (env.get(f"{ENV}_PROMPT_BUCKETS") or buckets).replace(" ", "") if got != buckets: raise RuntimeError(f"{ENV}_PROMPT_BUCKETS={got!r} is not supported: with {num_cameras} camera(s) the " f"resident buckets are {buckets} (derived from the camera count; see SERVING.md)") dispatch = (env.get(f"{ENV}_DISPATCH") or "eth").strip().lower() if dispatch not in ("eth", "worker"): raise RuntimeError(f"{ENV}_DISPATCH={dispatch!r}: expected eth or worker") variant = (env.get(f"{ENV}_MODEL") or "fm").strip().lower() if variant not in MODEL_VARIANTS: raise RuntimeError(f"{ENV}_MODEL={variant!r}: expected one of {list(MODEL_VARIANTS)}") if variant == "vq" and (num_cameras != 2 or num_steps != DEFAULT_NUM_STEPS): raise RuntimeError(f"{ENV}_MODEL=vq serves the checkpoint's 2 cameras and has no flow-matching steps: " f"{ENV}_NUM_CAMERAS must be unset or 2 and {ENV}_NUM_STEPS unset or 5 (got " f"{num_cameras} / {num_steps})") mesh = parse_mesh_shape(env.get("TT_MESH_SHAPE")) if mesh != (1, 1): raise RuntimeError(f"TT_MESH_SHAPE={mesh} is a multi-chip mesh; {MODEL_NAME} runs on exactly one Blackhole " "chip (mesh_device P150, shape 1x1)") return { "model_variant": variant, "model_id": env.get("HF_MODEL") or None, "revision": env.get("TT_MODEL_WEIGHTS_REVISION") or env.get("TT_WEIGHTS_REVISION") or None, "device_id": int(env.get("TT_DEVICE_ID") or 0), "dispatch": dispatch, "engine": (env.get(f"{ENV}_ENGINE") or "").strip() or None, "precision": FIXED["PRECISION"], "num_cameras": num_cameras, "num_steps": num_steps, "max_body_bytes": int(float(env.get(f"{ENV}_MAX_BODY_MB") or 32) * (1 << 20)), "torch_threads": env.get(f"{ENV}_TORCH_THREADS") or None, } def default_model_factory(cfg: Dict[str, Any]): if cfg.get("model_variant") == "vq": from ..vq.api import RDT2VQ return RDT2VQ.from_pretrained(cfg["model_id"], cfg["revision"], device_id=cfg["device_id"], dispatch=cfg["dispatch"], engine=cfg["engine"], precision=cfg["precision"]) from ..api import RDT2FM return RDT2FM.from_pretrained(cfg["model_id"], cfg["revision"], device_id=cfg["device_id"], dispatch=cfg["dispatch"], engine=cfg["engine"], precision=cfg["precision"], num_cameras=cfg.get("num_cameras", 2), num_steps=cfg.get("num_steps", 5)) @dataclass class ServerState: model_factory: Callable[[Dict[str, Any]], Any] = default_model_factory ready: bool = False error: Optional[str] = None model: Any = None config: Dict[str, Any] = field(default_factory=dict) boot_ms: Optional[float] = None lock: Any = field(default_factory=threading.Lock) # one chip, batch 1: device calls are serialized predict: Optional[Callable[[Any], Dict[str, Any]]] = None def status(self) -> str: return "ok" if self.ready else ("error" if self.error else "starting") def device_desc(self) -> Optional[Dict[str, Any]]: return getattr(self.model, "device_info", None) if self.model is not None else None def decode_b64(s: str, i: int) -> bytes: if not isinstance(s, str) or not s: raise InputError(f"images[{i}] is empty") if s.startswith("data:"): # tolerate a data URL s = s.split(",", 1)[-1] try: return base64.b64decode(s, validate=False) except (binascii.Error, ValueError) as e: raise InputError(f"images[{i}] is not valid base64: {e}") from None def create_app() -> FastAPI: state = ServerState() @asynccontextmanager async def lifespan(_app: FastAPI): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") state.ready, state.error = False, None t0 = time.perf_counter() try: cfg = config_from_env() state.config = cfg if cfg["torch_threads"]: import torch torch.set_num_threads(int(cfg["torch_threads"])) state.model = state.model_factory(cfg) # Loading weights / Opening device / Warming up / Warmup complete except Exception as e: # startup failures exit uvicorn non-zero: no silent CPU fallback state.error = f"{type(e).__name__}: {e}" log.exception("startup failed") raise state.boot_ms = (time.perf_counter() - t0) * 1e3 state.ready = True log.info("%s ready in %.1f s (%s)", MODEL_NAME, state.boot_ms / 1e3, state.device_desc()) try: yield finally: state.ready = False model, state.model = state.model, None if model is not None: log.info("Closing the model and the device") with state.lock: model.close() app = FastAPI(title=MODEL_NAME, version=__version__, lifespan=lifespan, description=f"RDT2-FM ({TASK}) on one Tenstorrent Blackhole p150. Not an OpenAI-compatible API.") app.state.rdt2 = state @app.middleware("http") async def _body_limit(request: Request, call_next): limit = (state.config or {}).get("max_body_bytes") length = request.headers.get("content-length") if limit and length and length.isdigit() and int(length) > limit: if int(length) <= BODY_DRAIN_FACTOR * limit: async for _ in request.stream(): # read and discard: nothing is kept pass return JSONResponse(status_code=400, headers={"Connection": "close"}, content={"detail": f"request body of {length} bytes exceeds " f"{ENV}_MAX_BODY_MB ({limit} bytes)"}) return await call_next(request) @app.get("/") def root() -> dict: return {"model": MODEL_NAME, "routes": ["/health", "/v1/health", "/info", "/v1/models", "/predict"], "docs": "/docs"} @app.get("/health") def health() -> dict: """Always 200: ``ok`` after warm-up, ``starting`` before, ``error`` if the boot failed.""" return {"status": state.status(), "model": MODEL_NAME, "device": state.device_desc(), "error": state.error} @app.get("/v1/health") def v1_health() -> dict: return health() @app.get("/v1/models") def v1_models() -> dict: """OpenAI-shaped stub so tt-model's ready card / probes do not 404. This is NOT a chat API.""" from ..reference.config import VLM_REPO return {"object": "list", "data": [{"id": (state.config or {}).get("model_id") or VLM_REPO, "object": "model", "owned_by": "changh95"}]} @app.get("/info") def info() -> dict: m = state.model body: Dict[str, Any] = {"model": MODEL_NAME, "version": __version__, "task": TASK, "status": state.status(), "hardware": HARDWARE, "device": state.device_desc(), "mesh_shape": os.environ.get("TT_MESH_SHAPE", "1x1"), "license": LICENSE, "boot_ms": None if state.boot_ms is None else round(state.boot_ms, 1)} if m is not None: minfo = dict(getattr(m, "info", {}) or {}) minfo.pop("device", None) body.update(minfo) body["limits"] = {"batch_size": 1, "max_body_bytes": (state.config or {}).get("max_body_bytes"), "images": "exactly 2 base64 PNG / JPEG, [left_stereo, right_stereo]", "instruction": "non-empty, no special-token text such as <|im_end|>, at most " "112 tokens (the 512 bucket)", "state": "optional, 20 finite floats (raw, not normalized); default zeros", "seed": "optional integer in [0, 2**63 - 1]; default 0", "num_steps": "optional, must be 5", "prompt_bucket": "optional, 416 / 448 / 512 and must hold the prompt"} body["fixed_configuration"] = {f"{ENV}_{k}": v for k, v in FIXED.items()} cams = list(getattr(m, "CAMERA_ORDER", None) or ("left_stereo", "right_stereo")) steps = getattr(m, "NUM_STEPS", 5) bks = list(getattr(m, "PROMPT_BUCKETS", None) or (416, 448, 512)) fixed_tok = getattr(m, "prompt_fixed_tokens", 400) body["fixed_configuration"].update({f"{ENV}_NUM_CAMERAS": str(len(cams)), f"{ENV}_NUM_STEPS": str(steps), f"{ENV}_PROMPT_BUCKETS": ",".join(str(b) for b in bks)}) body["limits"].update({ "images": f"exactly {len(cams)} base64 PNG / JPEG, [{', '.join(cams)}]", "instruction": "non-empty, no special-token text such as <|im_end|>, at most " f"{max(bks) - fixed_tok} tokens (the {max(bks)} bucket)", "num_steps": f"optional, must be {steps}", "prompt_bucket": f"optional, {' / '.join(str(b) for b in bks)} and must hold the prompt"}) body["experimental"] = {"prefix=mk": "refused unless RDT2_EXPERIMENTAL_PREFIX_MK=1 (known hang at bucket 512)"} variant = (state.config or {}).get("model_variant", "fm") body["model_variant"] = variant body["task"] = (f"vision-language-action: {len(cams)} camera images + an instruction (+ a 20-d proprio state) " "-> a 24 x 20 action chunk") if variant == "vq": # the VQ profile runs no flow-matching expert and takes no proprio state body["task"] = ("vision-language-action (RDT2-VQ): 2 camera images + an instruction -> 27 RVQ action " "tokens, decoded to a 24 x 20 action chunk") body["fixed_configuration"].pop(f"{ENV}_NUM_STEPS", None) body["limits"]["num_steps"] = "optional, flow-matching only: omit it (or 5, ignored)" body["limits"]["state"] = "optional, validated but ignored (the VQ path takes no proprio state)" body["limits"]["seed"] = "optional, validated but ignored (greedy decoding)" body["not_included"] = "the RDT2-FM flow-matching head (serve it with RDT2_MODEL=fm, the default)" else: body["not_included"] = ("RDT2-VQ autoregressive action tokens: served by the same package with " "RDT2_MODEL=vq (SERVING.md)") return body @app.post("/predict") def predict(req: PredictRequest) -> dict: """One observation -> one action chunk. Sync handler: uvicorn runs it in a worker thread; the lock serializes the device.""" if not state.ready: raise HTTPException(status_code=503, detail="model is still starting" if not state.error else f"model failed to start: {state.error}") from ..api import InputError as ApiInputError model = state.model t0 = time.perf_counter() try: if req.instruction is not None and req.prompt is not None: raise InputError("give either instruction or prompt, not both") instruction = req.instruction if req.instruction is not None else req.prompt if instruction is None: raise InputError("instruction (or prompt) is required") raw = [decode_b64(s, i) for i, s in enumerate(req.images)] t_dec = (time.perf_counter() - t0) * 1e3 with state.lock: out = model(raw, instruction, state=req.state, seed=req.seed, prompt_bucket=req.prompt_bucket, num_steps=req.num_steps) except (InputError, ApiInputError) as e: raise HTTPException(status_code=400, detail=str(e)) from None except Exception as e: # noqa: BLE001 -- a device / runtime failure log.exception("inference failed") raise HTTPException(status_code=500, detail=f"inference failed: {type(e).__name__}: {e}") from None body = out.to_dict() timing = dict(body.get("timing_ms") or {}) timing["preprocess"] = round(timing.get("preprocess", 0.0) + t_dec, 3) timing["total"] = round((time.perf_counter() - t0) * 1e3, 3) body["timing_ms"] = timing body["model"] = MODEL_NAME return body state.predict = predict return app app = create_app()