# SPDX-License-Identifier: Apache-2.0 """The real HTTP server in-process for device tests: ``tt_rdt2.server.app`` under uvicorn on a loopback port, in a thread, with the default model factory (``RDT2FM.from_pretrained`` from the server's environment) -- exactly what ``tt-model serve`` runs, minus the container (the server and the device in the main thread, the client in a worker thread). Weights: the ``RDT2_*_DIR`` variables when set, else the development workspace's pinned snapshots when they exist (never a download), else the Hugging Face cache.""" from __future__ import annotations import json import os import socket import threading import time import urllib.request from typing import Any, Callable, Dict, Optional from tt_rdt2.reference.config import DEFAULT_DIRS WEIGHT_ENV = {"vlm": "RDT2_VLM_DIR", "expert": "RDT2_EXPERT_DIR", "normalizer": "RDT2_NORMALIZER_DIR", "processor": "RDT2_PROCESSOR_DIR"} def local_weight_env() -> Dict[str, str]: env = {} for role, var in WEIGHT_ENV.items(): if not os.environ.get(var) and os.path.isdir(DEFAULT_DIRS[role]): env[var] = DEFAULT_DIRS[role] return env def free_port() -> int: s = socket.socket() s.bind(("127.0.0.1", 0)) p = s.getsockname()[1] s.close() return p def get_json(url: str, timeout: float = 30.0) -> Dict[str, Any]: with urllib.request.urlopen(url, timeout=timeout) as r: return json.loads(r.read().decode()) class Served: """What a workload sees: ``base`` URL, the ``app``, the live ``model`` (the server's), ``boot_s``.""" def __init__(self, app, base: str, boot_s: float): self.app, self.base, self.boot_s = app, base, boot_s @property def model(self): return self.app.state.rdt2.model def serve_and_run(workload: Callable[[Served], Any], env: Optional[Dict[str, str]] = None, ready_timeout_s: float = 1500.0) -> Any: """Run the server in THIS (the main) thread -- the lifespan opens and closes the device here, as under ``uvicorn`` in the container -- and ``workload(served)`` in a worker thread once every bucket is warm. Returns the workload's result (re-raises its exception). The server stops (model closed, traces released, device closed) when the workload returns.""" import uvicorn from tt_rdt2.server import app as app_mod env = dict(local_weight_env(), **(env or {})) saved = {k: os.environ.get(k) for k in env} os.environ.update(env) app = app_mod.create_app() port = free_port() server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="info", lifespan="on", timeout_keep_alive=30)) box: Dict[str, Any] = {} t0 = time.time() def client(): try: while True: st = app.state.rdt2 if st.ready and server.started: break if st.error or server.should_exit: raise RuntimeError(f"server failed to start: {st.error}") if time.time() - t0 > ready_timeout_s: raise TimeoutError(f"server not ready after {ready_timeout_s} s") time.sleep(1.0) box["result"] = workload(Served(app, f"http://127.0.0.1:{port}", time.time() - t0)) except BaseException as e: # noqa: BLE001 -- handed to the main thread box["error"] = e finally: server.should_exit = True th = threading.Thread(target=client, name="rdt2-workload", daemon=True) th.start() try: server.run() # returns after should_exit (lifespan shutdown closes the model) or a failed start-up finally: for k, v in saved.items(): if v is None: os.environ.pop(k, None) else: os.environ[k] = v th.join(timeout=60) if "error" in box: raise box["error"] if app.state.rdt2.error and "result" not in box: raise RuntimeError(f"server failed to start: {app.state.rdt2.error}") return box.get("result")