Download code/tt_rdt2/server/app.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 18.9 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/server/app.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/server/app.py
-
curl -L -o app.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/server/app.py
18.9 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """HTTP server of rdt2-base-p150 (FastAPI; tt-model kind ``tt-dit-server``: | |
| ``python -m uvicorn --host 0.0.0.0 --port 20000 --lifespan on tt_rdt2.server.app:app``). | |
| Routes: ``GET /health`` (and ``/v1/health``; always 200, ``ok`` only after warm-up), ``GET /info``, ``GET /v1/models`` | |
| (an OpenAI-shaped stub so tt-model's ready card does not 404: NOT a chat API), ``POST /predict``. The contract is in | |
| SERVING.md ("Request and response contract"). Everything device-touching happens in the lifespan (weights -> device -> | |
| programs -> one trace per prompt bucket -> one warm call per bucket), so uvicorn logs ``Application startup complete`` | |
| only when every bucket is warm. Shutdown (SIGTERM, tt-model's 120 s budget) closes the model under the lock. Importing | |
| this module has no side effects beyond importing fastapi / pydantic (no device, no download, no environment read). | |
| ``POST /predict`` decodes the request (base64 PNG / JPEG -> RGB uint8) and calls the Python API | |
| (:class:`tt_rdt2.api.RDT2FM`), which runs the same host pre-processing and device path as the device gates; one lock | |
| serializes the requests (batch 1, one chip). | |
| Environment (read once, in the lifespan; the serve profile pins the fixed configuration): | |
| HF_MODEL, TT_MODEL_WEIGHTS_REVISION (then TT_WEIGHTS_REVISION) the VLM repo / revision (default: the pins) | |
| RDT2_VLM_DIR, RDT2_EXPERT_DIR, RDT2_NORMALIZER_DIR, RDT2_PROCESSOR_DIR local weight directories (optional) | |
| TT_MESH_SHAPE (only 1x1), TT_DEVICE_ID (0), RDT2_DISPATCH (eth | worker; default eth) | |
| RDT2_ENGINE ("" = vision=mk,prefix=fused,expert=mk; prefix=mk is refused unless RDT2_EXPERIMENTAL_PREFIX_MK=1) | |
| RDT2_PRECISION (only "baseline"): any other value is refused at start | |
| RDT2_NUM_CAMERAS (1..4, default 2), RDT2_NUM_STEPS (1..16, default 5): the fixed configuration of the served | |
| model (reference/geometry.py; SERVING.md "Configuration"); RDT2_PROMPT_BUCKETS: unset or exactly the camera | |
| count's buckets (416,448,512 for 2 cameras); anything else is refused at start. RDT2_MODEL=vq serves 2 cameras | |
| only and has no flow-matching steps (RDT2_NUM_STEPS must be unset or 5) | |
| RDT2_MODEL (fm | vq; default fm): fm = RDT2-FM (flow matching, the package default); vq = RDT2-VQ (the full | |
| Qwen2.5-VL-7B emits 27 RVQ action tokens, tt_rdt2.vq; same /predict request, the response adds "tokens") | |
| RDT2_MAX_BODY_MB (default 32; a larger Content-Length is answered 400, see BODY_DRAIN_FACTOR), | |
| RDT2_TORCH_THREADS (host pre-processing threads; default torch's) | |
| Tests replace the model: ``app.state.rdt2.model_factory = <callable(cfg) -> model>`` before ``TestClient(app)``. | |
| """ | |
| from __future__ import annotations | |
| import base64 | |
| import binascii | |
| import logging | |
| import os | |
| import threading | |
| import time | |
| from contextlib import asynccontextmanager | |
| from dataclasses import dataclass, field | |
| from typing import Any, Callable, Dict, List, Mapping, Optional | |
| from fastapi import FastAPI, HTTPException, Request | |
| from fastapi.responses import JSONResponse | |
| from pydantic import BaseModel, ConfigDict | |
| from .. import __version__ | |
| from ..ttaw.server.app import parse_mesh_shape # noqa: F401 (re-exported: the image's verify: line imports it) | |
| MODEL_NAME = "rdt2-base-p150" | |
| ENV = "RDT2" | |
| HARDWARE = "Tenstorrent Blackhole p150 (single chip) via tt-nn" | |
| TASK = "vision-language-action: 2 camera images + an instruction (+ a 20-d proprio state) -> a 24 x 20 action chunk" | |
| # PRECISION: "baseline" is the only preset gated on the device and stress-gated ("fast" fails the long-prompt gate) | |
| FIXED = {"PRECISION": "baseline"} | |
| # stage Config: RDT2_NUM_CAMERAS / RDT2_NUM_STEPS are configurable (reference/geometry.py); the buckets follow the | |
| # camera count (RDT2_PROMPT_BUCKETS may only restate them) | |
| # A body above the limit is answered 400. Bodies up to this many times the limit are read and discarded first, so a | |
| # client that sends the whole body before reading the answer receives the 400 instead of a connection reset; | |
| # larger ones are refused without reading (the connection is closed). | |
| BODY_DRAIN_FACTOR = 4 | |
| # RDT2_MODEL: the served variant (read once in the lifespan). "vq" = tt_rdt2.vq (round 4, SERVING.md "RDT2-VQ") | |
| MODEL_VARIANTS = ("fm", "vq") | |
| LICENSE = {"package": "apache-2.0", "weights": "apache-2.0 (RDT2-VQ, RDT2-FM, RVQActionTokenizer, " | |
| "Qwen2.5-VL-7B-Instruct tokenizer)"} | |
| log = logging.getLogger("tt_rdt2.server") | |
| class PredictRequest(BaseModel): | |
| """``POST /predict`` body. Unknown fields are a 422 (a misspelled option is never silently ignored).""" | |
| model_config = ConfigDict(extra="forbid") | |
| images: List[str] # base64 PNG / JPEG, exactly [left_stereo, right_stereo] | |
| instruction: Optional[str] = None # the task in natural language (alias: prompt) | |
| prompt: Optional[str] = None | |
| state: Optional[List[float]] = None # raw 20-d proprio state; default zeros (the official FM deployment) | |
| seed: Optional[int] = None # flow-matching noise seed; default 0 (the fixed seeded noise) | |
| prompt_bucket: Optional[int] = None # 416 / 448 / 512; default the smallest that holds the prompt | |
| num_steps: Optional[int] = None # must be 5 | |
| class InputError(ValueError): | |
| pass | |
| def config_from_env(env: Optional[Mapping[str, str]] = None) -> Dict[str, Any]: | |
| """The server configuration (read once, in the lifespan). Raises RuntimeError for a refused setting.""" | |
| env = os.environ if env is None else env | |
| for k, want in FIXED.items(): | |
| got = (env.get(f"{ENV}_{k}") or want).replace(" ", "") | |
| if got != want: | |
| raise RuntimeError(f"{ENV}_{k}={got!r} is not supported: this package is built for {ENV}_{k}={want} only " | |
| "(fixed configuration; see SERVING.md)") | |
| from ..reference.geometry import DEFAULT_NUM_STEPS, geometry | |
| from ..reference.geometry import config_from_env as geometry_from_env | |
| try: | |
| num_cameras, num_steps = geometry_from_env(env) | |
| except ValueError as e: | |
| raise RuntimeError(f"{e} (see SERVING.md, Configuration)") from None | |
| buckets = ",".join(str(b) for b in geometry(num_cameras).buckets) | |
| got = (env.get(f"{ENV}_PROMPT_BUCKETS") or buckets).replace(" ", "") | |
| if got != buckets: | |
| raise RuntimeError(f"{ENV}_PROMPT_BUCKETS={got!r} is not supported: with {num_cameras} camera(s) the " | |
| f"resident buckets are {buckets} (derived from the camera count; see SERVING.md)") | |
| dispatch = (env.get(f"{ENV}_DISPATCH") or "eth").strip().lower() | |
| if dispatch not in ("eth", "worker"): | |
| raise RuntimeError(f"{ENV}_DISPATCH={dispatch!r}: expected eth or worker") | |
| variant = (env.get(f"{ENV}_MODEL") or "fm").strip().lower() | |
| if variant not in MODEL_VARIANTS: | |
| raise RuntimeError(f"{ENV}_MODEL={variant!r}: expected one of {list(MODEL_VARIANTS)}") | |
| if variant == "vq" and (num_cameras != 2 or num_steps != DEFAULT_NUM_STEPS): | |
| raise RuntimeError(f"{ENV}_MODEL=vq serves the checkpoint's 2 cameras and has no flow-matching steps: " | |
| f"{ENV}_NUM_CAMERAS must be unset or 2 and {ENV}_NUM_STEPS unset or 5 (got " | |
| f"{num_cameras} / {num_steps})") | |
| mesh = parse_mesh_shape(env.get("TT_MESH_SHAPE")) | |
| if mesh != (1, 1): | |
| raise RuntimeError(f"TT_MESH_SHAPE={mesh} is a multi-chip mesh; {MODEL_NAME} runs on exactly one Blackhole " | |
| "chip (mesh_device P150, shape 1x1)") | |
| return { | |
| "model_variant": variant, | |
| "model_id": env.get("HF_MODEL") or None, | |
| "revision": env.get("TT_MODEL_WEIGHTS_REVISION") or env.get("TT_WEIGHTS_REVISION") or None, | |
| "device_id": int(env.get("TT_DEVICE_ID") or 0), | |
| "dispatch": dispatch, | |
| "engine": (env.get(f"{ENV}_ENGINE") or "").strip() or None, | |
| "precision": FIXED["PRECISION"], | |
| "num_cameras": num_cameras, | |
| "num_steps": num_steps, | |
| "max_body_bytes": int(float(env.get(f"{ENV}_MAX_BODY_MB") or 32) * (1 << 20)), | |
| "torch_threads": env.get(f"{ENV}_TORCH_THREADS") or None, | |
| } | |
| def default_model_factory(cfg: Dict[str, Any]): | |
| if cfg.get("model_variant") == "vq": | |
| from ..vq.api import RDT2VQ | |
| return RDT2VQ.from_pretrained(cfg["model_id"], cfg["revision"], device_id=cfg["device_id"], | |
| dispatch=cfg["dispatch"], engine=cfg["engine"], precision=cfg["precision"]) | |
| from ..api import RDT2FM | |
| return RDT2FM.from_pretrained(cfg["model_id"], cfg["revision"], device_id=cfg["device_id"], | |
| dispatch=cfg["dispatch"], engine=cfg["engine"], precision=cfg["precision"], | |
| num_cameras=cfg.get("num_cameras", 2), num_steps=cfg.get("num_steps", 5)) | |
| class ServerState: | |
| model_factory: Callable[[Dict[str, Any]], Any] = default_model_factory | |
| ready: bool = False | |
| error: Optional[str] = None | |
| model: Any = None | |
| config: Dict[str, Any] = field(default_factory=dict) | |
| boot_ms: Optional[float] = None | |
| lock: Any = field(default_factory=threading.Lock) # one chip, batch 1: device calls are serialized | |
| predict: Optional[Callable[[Any], Dict[str, Any]]] = None | |
| def status(self) -> str: | |
| return "ok" if self.ready else ("error" if self.error else "starting") | |
| def device_desc(self) -> Optional[Dict[str, Any]]: | |
| return getattr(self.model, "device_info", None) if self.model is not None else None | |
| def decode_b64(s: str, i: int) -> bytes: | |
| if not isinstance(s, str) or not s: | |
| raise InputError(f"images[{i}] is empty") | |
| if s.startswith("data:"): # tolerate a data URL | |
| s = s.split(",", 1)[-1] | |
| try: | |
| return base64.b64decode(s, validate=False) | |
| except (binascii.Error, ValueError) as e: | |
| raise InputError(f"images[{i}] is not valid base64: {e}") from None | |
| def create_app() -> FastAPI: | |
| state = ServerState() | |
| async def lifespan(_app: FastAPI): | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") | |
| state.ready, state.error = False, None | |
| t0 = time.perf_counter() | |
| try: | |
| cfg = config_from_env() | |
| state.config = cfg | |
| if cfg["torch_threads"]: | |
| import torch | |
| torch.set_num_threads(int(cfg["torch_threads"])) | |
| state.model = state.model_factory(cfg) # Loading weights / Opening device / Warming up / Warmup complete | |
| except Exception as e: # startup failures exit uvicorn non-zero: no silent CPU fallback | |
| state.error = f"{type(e).__name__}: {e}" | |
| log.exception("startup failed") | |
| raise | |
| state.boot_ms = (time.perf_counter() - t0) * 1e3 | |
| state.ready = True | |
| log.info("%s ready in %.1f s (%s)", MODEL_NAME, state.boot_ms / 1e3, state.device_desc()) | |
| try: | |
| yield | |
| finally: | |
| state.ready = False | |
| model, state.model = state.model, None | |
| if model is not None: | |
| log.info("Closing the model and the device") | |
| with state.lock: | |
| model.close() | |
| app = FastAPI(title=MODEL_NAME, version=__version__, lifespan=lifespan, | |
| description=f"RDT2-FM ({TASK}) on one Tenstorrent Blackhole p150. Not an OpenAI-compatible API.") | |
| app.state.rdt2 = state | |
| async def _body_limit(request: Request, call_next): | |
| limit = (state.config or {}).get("max_body_bytes") | |
| length = request.headers.get("content-length") | |
| if limit and length and length.isdigit() and int(length) > limit: | |
| if int(length) <= BODY_DRAIN_FACTOR * limit: | |
| async for _ in request.stream(): # read and discard: nothing is kept | |
| pass | |
| return JSONResponse(status_code=400, headers={"Connection": "close"}, | |
| content={"detail": f"request body of {length} bytes exceeds " | |
| f"{ENV}_MAX_BODY_MB ({limit} bytes)"}) | |
| return await call_next(request) | |
| def root() -> dict: | |
| return {"model": MODEL_NAME, "routes": ["/health", "/v1/health", "/info", "/v1/models", "/predict"], | |
| "docs": "/docs"} | |
| def health() -> dict: | |
| """Always 200: ``ok`` after warm-up, ``starting`` before, ``error`` if the boot failed.""" | |
| return {"status": state.status(), "model": MODEL_NAME, "device": state.device_desc(), "error": state.error} | |
| def v1_health() -> dict: | |
| return health() | |
| def v1_models() -> dict: | |
| """OpenAI-shaped stub so tt-model's ready card / probes do not 404. This is NOT a chat API.""" | |
| from ..reference.config import VLM_REPO | |
| return {"object": "list", "data": [{"id": (state.config or {}).get("model_id") or VLM_REPO, | |
| "object": "model", "owned_by": "changh95"}]} | |
| def info() -> dict: | |
| m = state.model | |
| body: Dict[str, Any] = {"model": MODEL_NAME, "version": __version__, "task": TASK, "status": state.status(), | |
| "hardware": HARDWARE, "device": state.device_desc(), | |
| "mesh_shape": os.environ.get("TT_MESH_SHAPE", "1x1"), "license": LICENSE, | |
| "boot_ms": None if state.boot_ms is None else round(state.boot_ms, 1)} | |
| if m is not None: | |
| minfo = dict(getattr(m, "info", {}) or {}) | |
| minfo.pop("device", None) | |
| body.update(minfo) | |
| body["limits"] = {"batch_size": 1, "max_body_bytes": (state.config or {}).get("max_body_bytes"), | |
| "images": "exactly 2 base64 PNG / JPEG, [left_stereo, right_stereo]", | |
| "instruction": "non-empty, no special-token text such as <|im_end|>, at most " | |
| "112 tokens (the 512 bucket)", | |
| "state": "optional, 20 finite floats (raw, not normalized); default zeros", | |
| "seed": "optional integer in [0, 2**63 - 1]; default 0", | |
| "num_steps": "optional, must be 5", | |
| "prompt_bucket": "optional, 416 / 448 / 512 and must hold the prompt"} | |
| body["fixed_configuration"] = {f"{ENV}_{k}": v for k, v in FIXED.items()} | |
| cams = list(getattr(m, "CAMERA_ORDER", None) or ("left_stereo", "right_stereo")) | |
| steps = getattr(m, "NUM_STEPS", 5) | |
| bks = list(getattr(m, "PROMPT_BUCKETS", None) or (416, 448, 512)) | |
| fixed_tok = getattr(m, "prompt_fixed_tokens", 400) | |
| body["fixed_configuration"].update({f"{ENV}_NUM_CAMERAS": str(len(cams)), f"{ENV}_NUM_STEPS": str(steps), | |
| f"{ENV}_PROMPT_BUCKETS": ",".join(str(b) for b in bks)}) | |
| body["limits"].update({ | |
| "images": f"exactly {len(cams)} base64 PNG / JPEG, [{', '.join(cams)}]", | |
| "instruction": "non-empty, no special-token text such as <|im_end|>, at most " | |
| f"{max(bks) - fixed_tok} tokens (the {max(bks)} bucket)", | |
| "num_steps": f"optional, must be {steps}", | |
| "prompt_bucket": f"optional, {' / '.join(str(b) for b in bks)} and must hold the prompt"}) | |
| body["experimental"] = {"prefix=mk": "refused unless RDT2_EXPERIMENTAL_PREFIX_MK=1 (known hang at bucket 512)"} | |
| variant = (state.config or {}).get("model_variant", "fm") | |
| body["model_variant"] = variant | |
| body["task"] = (f"vision-language-action: {len(cams)} camera images + an instruction (+ a 20-d proprio state) " | |
| "-> a 24 x 20 action chunk") | |
| if variant == "vq": | |
| # the VQ profile runs no flow-matching expert and takes no proprio state | |
| body["task"] = ("vision-language-action (RDT2-VQ): 2 camera images + an instruction -> 27 RVQ action " | |
| "tokens, decoded to a 24 x 20 action chunk") | |
| body["fixed_configuration"].pop(f"{ENV}_NUM_STEPS", None) | |
| body["limits"]["num_steps"] = "optional, flow-matching only: omit it (or 5, ignored)" | |
| body["limits"]["state"] = "optional, validated but ignored (the VQ path takes no proprio state)" | |
| body["limits"]["seed"] = "optional, validated but ignored (greedy decoding)" | |
| body["not_included"] = "the RDT2-FM flow-matching head (serve it with RDT2_MODEL=fm, the default)" | |
| else: | |
| body["not_included"] = ("RDT2-VQ autoregressive action tokens: served by the same package with " | |
| "RDT2_MODEL=vq (SERVING.md)") | |
| return body | |
| def predict(req: PredictRequest) -> dict: | |
| """One observation -> one action chunk. Sync handler: uvicorn runs it in a worker thread; the lock serializes | |
| the device.""" | |
| if not state.ready: | |
| raise HTTPException(status_code=503, detail="model is still starting" if not state.error | |
| else f"model failed to start: {state.error}") | |
| from ..api import InputError as ApiInputError | |
| model = state.model | |
| t0 = time.perf_counter() | |
| try: | |
| if req.instruction is not None and req.prompt is not None: | |
| raise InputError("give either instruction or prompt, not both") | |
| instruction = req.instruction if req.instruction is not None else req.prompt | |
| if instruction is None: | |
| raise InputError("instruction (or prompt) is required") | |
| raw = [decode_b64(s, i) for i, s in enumerate(req.images)] | |
| t_dec = (time.perf_counter() - t0) * 1e3 | |
| with state.lock: | |
| out = model(raw, instruction, state=req.state, seed=req.seed, prompt_bucket=req.prompt_bucket, | |
| num_steps=req.num_steps) | |
| except (InputError, ApiInputError) as e: | |
| raise HTTPException(status_code=400, detail=str(e)) from None | |
| except Exception as e: # noqa: BLE001 -- a device / runtime failure | |
| log.exception("inference failed") | |
| raise HTTPException(status_code=500, detail=f"inference failed: {type(e).__name__}: {e}") from None | |
| body = out.to_dict() | |
| timing = dict(body.get("timing_ms") or {}) | |
| timing["preprocess"] = round(timing.get("preprocess", 0.0) + t_dec, 3) | |
| timing["total"] = round((time.perf_counter() - t0) * 1e3, 3) | |
| body["timing_ms"] = timing | |
| body["model"] = MODEL_NAME | |
| return body | |
| state.predict = predict | |
| return app | |
| app = create_app() | |