rdt2-base-p150 / code /tt_rdt2 /tests /serving.py
changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
4.12 kB
# SPDX-License-Identifier: Apache-2.0
"""The real HTTP server in-process for device tests: ``tt_rdt2.server.app`` under uvicorn on a loopback port, in a
thread, with the default model factory (``RDT2FM.from_pretrained`` from the server's environment) -- exactly what
``tt-model serve`` runs, minus the container (the server and the device in the main thread, the client in a worker thread). Weights: the ``RDT2_*_DIR`` variables when set, else the development
workspace's pinned snapshots when they exist (never a download), else the Hugging Face cache."""
from __future__ import annotations
import json
import os
import socket
import threading
import time
import urllib.request
from typing import Any, Callable, Dict, Optional
from tt_rdt2.reference.config import DEFAULT_DIRS
WEIGHT_ENV = {"vlm": "RDT2_VLM_DIR", "expert": "RDT2_EXPERT_DIR", "normalizer": "RDT2_NORMALIZER_DIR",
"processor": "RDT2_PROCESSOR_DIR"}
def local_weight_env() -> Dict[str, str]:
env = {}
for role, var in WEIGHT_ENV.items():
if not os.environ.get(var) and os.path.isdir(DEFAULT_DIRS[role]):
env[var] = DEFAULT_DIRS[role]
return env
def free_port() -> int:
s = socket.socket()
s.bind(("127.0.0.1", 0))
p = s.getsockname()[1]
s.close()
return p
def get_json(url: str, timeout: float = 30.0) -> Dict[str, Any]:
with urllib.request.urlopen(url, timeout=timeout) as r:
return json.loads(r.read().decode())
class Served:
"""What a workload sees: ``base`` URL, the ``app``, the live ``model`` (the server's), ``boot_s``."""
def __init__(self, app, base: str, boot_s: float):
self.app, self.base, self.boot_s = app, base, boot_s
@property
def model(self):
return self.app.state.rdt2.model
def serve_and_run(workload: Callable[[Served], Any], env: Optional[Dict[str, str]] = None,
ready_timeout_s: float = 1500.0) -> Any:
"""Run the server in THIS (the main) thread -- the lifespan opens and closes the device here, as under
``uvicorn`` in the container -- and ``workload(served)`` in a worker thread once every bucket is warm. Returns the
workload's result (re-raises its exception). The server stops (model closed, traces released, device closed)
when the workload returns."""
import uvicorn
from tt_rdt2.server import app as app_mod
env = dict(local_weight_env(), **(env or {}))
saved = {k: os.environ.get(k) for k in env}
os.environ.update(env)
app = app_mod.create_app()
port = free_port()
server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="info", lifespan="on",
timeout_keep_alive=30))
box: Dict[str, Any] = {}
t0 = time.time()
def client():
try:
while True:
st = app.state.rdt2
if st.ready and server.started:
break
if st.error or server.should_exit:
raise RuntimeError(f"server failed to start: {st.error}")
if time.time() - t0 > ready_timeout_s:
raise TimeoutError(f"server not ready after {ready_timeout_s} s")
time.sleep(1.0)
box["result"] = workload(Served(app, f"http://127.0.0.1:{port}", time.time() - t0))
except BaseException as e: # noqa: BLE001 -- handed to the main thread
box["error"] = e
finally:
server.should_exit = True
th = threading.Thread(target=client, name="rdt2-workload", daemon=True)
th.start()
try:
server.run() # returns after should_exit (lifespan shutdown closes the model) or a failed start-up
finally:
for k, v in saved.items():
if v is None:
os.environ.pop(k, None)
else:
os.environ[k] = v
th.join(timeout=60)
if "error" in box:
raise box["error"]
if app.state.rdt2.error and "result" not in box:
raise RuntimeError(f"server failed to start: {app.state.rdt2.error}")
return box.get("result")