changh95's picture
tt-model push rdt2-base-p150 (container)
c869f1f verified
Raw History Blame Contribute Delete
18.9 kB
# SPDX-License-Identifier: Apache-2.0
"""HTTP server of rdt2-base-p150 (FastAPI; tt-model kind ``tt-dit-server``:
``python -m uvicorn --host 0.0.0.0 --port 20000 --lifespan on tt_rdt2.server.app:app``).
Routes: ``GET /health`` (and ``/v1/health``; always 200, ``ok`` only after warm-up), ``GET /info``, ``GET /v1/models``
(an OpenAI-shaped stub so tt-model's ready card does not 404: NOT a chat API), ``POST /predict``. The contract is in
SERVING.md ("Request and response contract"). Everything device-touching happens in the lifespan (weights -> device ->
programs -> one trace per prompt bucket -> one warm call per bucket), so uvicorn logs ``Application startup complete``
only when every bucket is warm. Shutdown (SIGTERM, tt-model's 120 s budget) closes the model under the lock. Importing
this module has no side effects beyond importing fastapi / pydantic (no device, no download, no environment read).
``POST /predict`` decodes the request (base64 PNG / JPEG -> RGB uint8) and calls the Python API
(:class:`tt_rdt2.api.RDT2FM`), which runs the same host pre-processing and device path as the device gates; one lock
serializes the requests (batch 1, one chip).
Environment (read once, in the lifespan; the serve profile pins the fixed configuration):
HF_MODEL, TT_MODEL_WEIGHTS_REVISION (then TT_WEIGHTS_REVISION) the VLM repo / revision (default: the pins)
RDT2_VLM_DIR, RDT2_EXPERT_DIR, RDT2_NORMALIZER_DIR, RDT2_PROCESSOR_DIR local weight directories (optional)
TT_MESH_SHAPE (only 1x1), TT_DEVICE_ID (0), RDT2_DISPATCH (eth | worker; default eth)
RDT2_ENGINE ("" = vision=mk,prefix=fused,expert=mk; prefix=mk is refused unless RDT2_EXPERIMENTAL_PREFIX_MK=1)
RDT2_PRECISION (only "baseline"): any other value is refused at start
RDT2_NUM_CAMERAS (1..4, default 2), RDT2_NUM_STEPS (1..16, default 5): the fixed configuration of the served
model (reference/geometry.py; SERVING.md "Configuration"); RDT2_PROMPT_BUCKETS: unset or exactly the camera
count's buckets (416,448,512 for 2 cameras); anything else is refused at start. RDT2_MODEL=vq serves 2 cameras
only and has no flow-matching steps (RDT2_NUM_STEPS must be unset or 5)
RDT2_MODEL (fm | vq; default fm): fm = RDT2-FM (flow matching, the package default); vq = RDT2-VQ (the full
Qwen2.5-VL-7B emits 27 RVQ action tokens, tt_rdt2.vq; same /predict request, the response adds "tokens")
RDT2_MAX_BODY_MB (default 32; a larger Content-Length is answered 400, see BODY_DRAIN_FACTOR),
RDT2_TORCH_THREADS (host pre-processing threads; default torch's)
Tests replace the model: ``app.state.rdt2.model_factory = <callable(cfg) -> model>`` before ``TestClient(app)``.
"""
from __future__ import annotations
import base64
import binascii
import logging
import os
import threading
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Mapping, Optional
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict
from .. import __version__
from ..ttaw.server.app import parse_mesh_shape # noqa: F401 (re-exported: the image's verify: line imports it)
MODEL_NAME = "rdt2-base-p150"
ENV = "RDT2"
HARDWARE = "Tenstorrent Blackhole p150 (single chip) via tt-nn"
TASK = "vision-language-action: 2 camera images + an instruction (+ a 20-d proprio state) -> a 24 x 20 action chunk"
# PRECISION: "baseline" is the only preset gated on the device and stress-gated ("fast" fails the long-prompt gate)
FIXED = {"PRECISION": "baseline"}
# stage Config: RDT2_NUM_CAMERAS / RDT2_NUM_STEPS are configurable (reference/geometry.py); the buckets follow the
# camera count (RDT2_PROMPT_BUCKETS may only restate them)
# A body above the limit is answered 400. Bodies up to this many times the limit are read and discarded first, so a
# client that sends the whole body before reading the answer receives the 400 instead of a connection reset;
# larger ones are refused without reading (the connection is closed).
BODY_DRAIN_FACTOR = 4
# RDT2_MODEL: the served variant (read once in the lifespan). "vq" = tt_rdt2.vq (round 4, SERVING.md "RDT2-VQ")
MODEL_VARIANTS = ("fm", "vq")
LICENSE = {"package": "apache-2.0", "weights": "apache-2.0 (RDT2-VQ, RDT2-FM, RVQActionTokenizer, "
"Qwen2.5-VL-7B-Instruct tokenizer)"}
log = logging.getLogger("tt_rdt2.server")
class PredictRequest(BaseModel):
"""``POST /predict`` body. Unknown fields are a 422 (a misspelled option is never silently ignored)."""
model_config = ConfigDict(extra="forbid")
images: List[str] # base64 PNG / JPEG, exactly [left_stereo, right_stereo]
instruction: Optional[str] = None # the task in natural language (alias: prompt)
prompt: Optional[str] = None
state: Optional[List[float]] = None # raw 20-d proprio state; default zeros (the official FM deployment)
seed: Optional[int] = None # flow-matching noise seed; default 0 (the fixed seeded noise)
prompt_bucket: Optional[int] = None # 416 / 448 / 512; default the smallest that holds the prompt
num_steps: Optional[int] = None # must be 5
class InputError(ValueError):
pass
def config_from_env(env: Optional[Mapping[str, str]] = None) -> Dict[str, Any]:
"""The server configuration (read once, in the lifespan). Raises RuntimeError for a refused setting."""
env = os.environ if env is None else env
for k, want in FIXED.items():
got = (env.get(f"{ENV}_{k}") or want).replace(" ", "")
if got != want:
raise RuntimeError(f"{ENV}_{k}={got!r} is not supported: this package is built for {ENV}_{k}={want} only "
"(fixed configuration; see SERVING.md)")
from ..reference.geometry import DEFAULT_NUM_STEPS, geometry
from ..reference.geometry import config_from_env as geometry_from_env
try:
num_cameras, num_steps = geometry_from_env(env)
except ValueError as e:
raise RuntimeError(f"{e} (see SERVING.md, Configuration)") from None
buckets = ",".join(str(b) for b in geometry(num_cameras).buckets)
got = (env.get(f"{ENV}_PROMPT_BUCKETS") or buckets).replace(" ", "")
if got != buckets:
raise RuntimeError(f"{ENV}_PROMPT_BUCKETS={got!r} is not supported: with {num_cameras} camera(s) the "
f"resident buckets are {buckets} (derived from the camera count; see SERVING.md)")
dispatch = (env.get(f"{ENV}_DISPATCH") or "eth").strip().lower()
if dispatch not in ("eth", "worker"):
raise RuntimeError(f"{ENV}_DISPATCH={dispatch!r}: expected eth or worker")
variant = (env.get(f"{ENV}_MODEL") or "fm").strip().lower()
if variant not in MODEL_VARIANTS:
raise RuntimeError(f"{ENV}_MODEL={variant!r}: expected one of {list(MODEL_VARIANTS)}")
if variant == "vq" and (num_cameras != 2 or num_steps != DEFAULT_NUM_STEPS):
raise RuntimeError(f"{ENV}_MODEL=vq serves the checkpoint's 2 cameras and has no flow-matching steps: "
f"{ENV}_NUM_CAMERAS must be unset or 2 and {ENV}_NUM_STEPS unset or 5 (got "
f"{num_cameras} / {num_steps})")
mesh = parse_mesh_shape(env.get("TT_MESH_SHAPE"))
if mesh != (1, 1):
raise RuntimeError(f"TT_MESH_SHAPE={mesh} is a multi-chip mesh; {MODEL_NAME} runs on exactly one Blackhole "
"chip (mesh_device P150, shape 1x1)")
return {
"model_variant": variant,
"model_id": env.get("HF_MODEL") or None,
"revision": env.get("TT_MODEL_WEIGHTS_REVISION") or env.get("TT_WEIGHTS_REVISION") or None,
"device_id": int(env.get("TT_DEVICE_ID") or 0),
"dispatch": dispatch,
"engine": (env.get(f"{ENV}_ENGINE") or "").strip() or None,
"precision": FIXED["PRECISION"],
"num_cameras": num_cameras,
"num_steps": num_steps,
"max_body_bytes": int(float(env.get(f"{ENV}_MAX_BODY_MB") or 32) * (1 << 20)),
"torch_threads": env.get(f"{ENV}_TORCH_THREADS") or None,
}
def default_model_factory(cfg: Dict[str, Any]):
if cfg.get("model_variant") == "vq":
from ..vq.api import RDT2VQ
return RDT2VQ.from_pretrained(cfg["model_id"], cfg["revision"], device_id=cfg["device_id"],
dispatch=cfg["dispatch"], engine=cfg["engine"], precision=cfg["precision"])
from ..api import RDT2FM
return RDT2FM.from_pretrained(cfg["model_id"], cfg["revision"], device_id=cfg["device_id"],
dispatch=cfg["dispatch"], engine=cfg["engine"], precision=cfg["precision"],
num_cameras=cfg.get("num_cameras", 2), num_steps=cfg.get("num_steps", 5))
@dataclass
class ServerState:
model_factory: Callable[[Dict[str, Any]], Any] = default_model_factory
ready: bool = False
error: Optional[str] = None
model: Any = None
config: Dict[str, Any] = field(default_factory=dict)
boot_ms: Optional[float] = None
lock: Any = field(default_factory=threading.Lock) # one chip, batch 1: device calls are serialized
predict: Optional[Callable[[Any], Dict[str, Any]]] = None
def status(self) -> str:
return "ok" if self.ready else ("error" if self.error else "starting")
def device_desc(self) -> Optional[Dict[str, Any]]:
return getattr(self.model, "device_info", None) if self.model is not None else None
def decode_b64(s: str, i: int) -> bytes:
if not isinstance(s, str) or not s:
raise InputError(f"images[{i}] is empty")
if s.startswith("data:"): # tolerate a data URL
s = s.split(",", 1)[-1]
try:
return base64.b64decode(s, validate=False)
except (binascii.Error, ValueError) as e:
raise InputError(f"images[{i}] is not valid base64: {e}") from None
def create_app() -> FastAPI:
state = ServerState()
@asynccontextmanager
async def lifespan(_app: FastAPI):
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
state.ready, state.error = False, None
t0 = time.perf_counter()
try:
cfg = config_from_env()
state.config = cfg
if cfg["torch_threads"]:
import torch
torch.set_num_threads(int(cfg["torch_threads"]))
state.model = state.model_factory(cfg) # Loading weights / Opening device / Warming up / Warmup complete
except Exception as e: # startup failures exit uvicorn non-zero: no silent CPU fallback
state.error = f"{type(e).__name__}: {e}"
log.exception("startup failed")
raise
state.boot_ms = (time.perf_counter() - t0) * 1e3
state.ready = True
log.info("%s ready in %.1f s (%s)", MODEL_NAME, state.boot_ms / 1e3, state.device_desc())
try:
yield
finally:
state.ready = False
model, state.model = state.model, None
if model is not None:
log.info("Closing the model and the device")
with state.lock:
model.close()
app = FastAPI(title=MODEL_NAME, version=__version__, lifespan=lifespan,
description=f"RDT2-FM ({TASK}) on one Tenstorrent Blackhole p150. Not an OpenAI-compatible API.")
app.state.rdt2 = state
@app.middleware("http")
async def _body_limit(request: Request, call_next):
limit = (state.config or {}).get("max_body_bytes")
length = request.headers.get("content-length")
if limit and length and length.isdigit() and int(length) > limit:
if int(length) <= BODY_DRAIN_FACTOR * limit:
async for _ in request.stream(): # read and discard: nothing is kept
pass
return JSONResponse(status_code=400, headers={"Connection": "close"},
content={"detail": f"request body of {length} bytes exceeds "
f"{ENV}_MAX_BODY_MB ({limit} bytes)"})
return await call_next(request)
@app.get("/")
def root() -> dict:
return {"model": MODEL_NAME, "routes": ["/health", "/v1/health", "/info", "/v1/models", "/predict"],
"docs": "/docs"}
@app.get("/health")
def health() -> dict:
"""Always 200: ``ok`` after warm-up, ``starting`` before, ``error`` if the boot failed."""
return {"status": state.status(), "model": MODEL_NAME, "device": state.device_desc(), "error": state.error}
@app.get("/v1/health")
def v1_health() -> dict:
return health()
@app.get("/v1/models")
def v1_models() -> dict:
"""OpenAI-shaped stub so tt-model's ready card / probes do not 404. This is NOT a chat API."""
from ..reference.config import VLM_REPO
return {"object": "list", "data": [{"id": (state.config or {}).get("model_id") or VLM_REPO,
"object": "model", "owned_by": "changh95"}]}
@app.get("/info")
def info() -> dict:
m = state.model
body: Dict[str, Any] = {"model": MODEL_NAME, "version": __version__, "task": TASK, "status": state.status(),
"hardware": HARDWARE, "device": state.device_desc(),
"mesh_shape": os.environ.get("TT_MESH_SHAPE", "1x1"), "license": LICENSE,
"boot_ms": None if state.boot_ms is None else round(state.boot_ms, 1)}
if m is not None:
minfo = dict(getattr(m, "info", {}) or {})
minfo.pop("device", None)
body.update(minfo)
body["limits"] = {"batch_size": 1, "max_body_bytes": (state.config or {}).get("max_body_bytes"),
"images": "exactly 2 base64 PNG / JPEG, [left_stereo, right_stereo]",
"instruction": "non-empty, no special-token text such as <|im_end|>, at most "
"112 tokens (the 512 bucket)",
"state": "optional, 20 finite floats (raw, not normalized); default zeros",
"seed": "optional integer in [0, 2**63 - 1]; default 0",
"num_steps": "optional, must be 5",
"prompt_bucket": "optional, 416 / 448 / 512 and must hold the prompt"}
body["fixed_configuration"] = {f"{ENV}_{k}": v for k, v in FIXED.items()}
cams = list(getattr(m, "CAMERA_ORDER", None) or ("left_stereo", "right_stereo"))
steps = getattr(m, "NUM_STEPS", 5)
bks = list(getattr(m, "PROMPT_BUCKETS", None) or (416, 448, 512))
fixed_tok = getattr(m, "prompt_fixed_tokens", 400)
body["fixed_configuration"].update({f"{ENV}_NUM_CAMERAS": str(len(cams)), f"{ENV}_NUM_STEPS": str(steps),
f"{ENV}_PROMPT_BUCKETS": ",".join(str(b) for b in bks)})
body["limits"].update({
"images": f"exactly {len(cams)} base64 PNG / JPEG, [{', '.join(cams)}]",
"instruction": "non-empty, no special-token text such as <|im_end|>, at most "
f"{max(bks) - fixed_tok} tokens (the {max(bks)} bucket)",
"num_steps": f"optional, must be {steps}",
"prompt_bucket": f"optional, {' / '.join(str(b) for b in bks)} and must hold the prompt"})
body["experimental"] = {"prefix=mk": "refused unless RDT2_EXPERIMENTAL_PREFIX_MK=1 (known hang at bucket 512)"}
variant = (state.config or {}).get("model_variant", "fm")
body["model_variant"] = variant
body["task"] = (f"vision-language-action: {len(cams)} camera images + an instruction (+ a 20-d proprio state) "
"-> a 24 x 20 action chunk")
if variant == "vq":
# the VQ profile runs no flow-matching expert and takes no proprio state
body["task"] = ("vision-language-action (RDT2-VQ): 2 camera images + an instruction -> 27 RVQ action "
"tokens, decoded to a 24 x 20 action chunk")
body["fixed_configuration"].pop(f"{ENV}_NUM_STEPS", None)
body["limits"]["num_steps"] = "optional, flow-matching only: omit it (or 5, ignored)"
body["limits"]["state"] = "optional, validated but ignored (the VQ path takes no proprio state)"
body["limits"]["seed"] = "optional, validated but ignored (greedy decoding)"
body["not_included"] = "the RDT2-FM flow-matching head (serve it with RDT2_MODEL=fm, the default)"
else:
body["not_included"] = ("RDT2-VQ autoregressive action tokens: served by the same package with "
"RDT2_MODEL=vq (SERVING.md)")
return body
@app.post("/predict")
def predict(req: PredictRequest) -> dict:
"""One observation -> one action chunk. Sync handler: uvicorn runs it in a worker thread; the lock serializes
the device."""
if not state.ready:
raise HTTPException(status_code=503, detail="model is still starting" if not state.error
else f"model failed to start: {state.error}")
from ..api import InputError as ApiInputError
model = state.model
t0 = time.perf_counter()
try:
if req.instruction is not None and req.prompt is not None:
raise InputError("give either instruction or prompt, not both")
instruction = req.instruction if req.instruction is not None else req.prompt
if instruction is None:
raise InputError("instruction (or prompt) is required")
raw = [decode_b64(s, i) for i, s in enumerate(req.images)]
t_dec = (time.perf_counter() - t0) * 1e3
with state.lock:
out = model(raw, instruction, state=req.state, seed=req.seed, prompt_bucket=req.prompt_bucket,
num_steps=req.num_steps)
except (InputError, ApiInputError) as e:
raise HTTPException(status_code=400, detail=str(e)) from None
except Exception as e: # noqa: BLE001 -- a device / runtime failure
log.exception("inference failed")
raise HTTPException(status_code=500, detail=f"inference failed: {type(e).__name__}: {e}") from None
body = out.to_dict()
timing = dict(body.get("timing_ms") or {})
timing["preprocess"] = round(timing.get("preprocess", 0.0) + t_dec, 3)
timing["total"] = round((time.perf_counter() - t0) * 1e3, 3)
body["timing_ms"] = timing
body["model"] = MODEL_NAME
return body
state.predict = predict
return app
app = create_app()