Download code/tt_rdt2/tests/serving.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 4.12 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/serving.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/tests/serving.py
-
curl -L -o serving.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/serving.py
4.12 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """The real HTTP server in-process for device tests: ``tt_rdt2.server.app`` under uvicorn on a loopback port, in a | |
| thread, with the default model factory (``RDT2FM.from_pretrained`` from the server's environment) -- exactly what | |
| ``tt-model serve`` runs, minus the container (the server and the device in the main thread, the client in a worker thread). Weights: the ``RDT2_*_DIR`` variables when set, else the development | |
| workspace's pinned snapshots when they exist (never a download), else the Hugging Face cache.""" | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import socket | |
| import threading | |
| import time | |
| import urllib.request | |
| from typing import Any, Callable, Dict, Optional | |
| from tt_rdt2.reference.config import DEFAULT_DIRS | |
| WEIGHT_ENV = {"vlm": "RDT2_VLM_DIR", "expert": "RDT2_EXPERT_DIR", "normalizer": "RDT2_NORMALIZER_DIR", | |
| "processor": "RDT2_PROCESSOR_DIR"} | |
| def local_weight_env() -> Dict[str, str]: | |
| env = {} | |
| for role, var in WEIGHT_ENV.items(): | |
| if not os.environ.get(var) and os.path.isdir(DEFAULT_DIRS[role]): | |
| env[var] = DEFAULT_DIRS[role] | |
| return env | |
| def free_port() -> int: | |
| s = socket.socket() | |
| s.bind(("127.0.0.1", 0)) | |
| p = s.getsockname()[1] | |
| s.close() | |
| return p | |
| def get_json(url: str, timeout: float = 30.0) -> Dict[str, Any]: | |
| with urllib.request.urlopen(url, timeout=timeout) as r: | |
| return json.loads(r.read().decode()) | |
| class Served: | |
| """What a workload sees: ``base`` URL, the ``app``, the live ``model`` (the server's), ``boot_s``.""" | |
| def __init__(self, app, base: str, boot_s: float): | |
| self.app, self.base, self.boot_s = app, base, boot_s | |
| def model(self): | |
| return self.app.state.rdt2.model | |
| def serve_and_run(workload: Callable[[Served], Any], env: Optional[Dict[str, str]] = None, | |
| ready_timeout_s: float = 1500.0) -> Any: | |
| """Run the server in THIS (the main) thread -- the lifespan opens and closes the device here, as under | |
| ``uvicorn`` in the container -- and ``workload(served)`` in a worker thread once every bucket is warm. Returns the | |
| workload's result (re-raises its exception). The server stops (model closed, traces released, device closed) | |
| when the workload returns.""" | |
| import uvicorn | |
| from tt_rdt2.server import app as app_mod | |
| env = dict(local_weight_env(), **(env or {})) | |
| saved = {k: os.environ.get(k) for k in env} | |
| os.environ.update(env) | |
| app = app_mod.create_app() | |
| port = free_port() | |
| server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="info", lifespan="on", | |
| timeout_keep_alive=30)) | |
| box: Dict[str, Any] = {} | |
| t0 = time.time() | |
| def client(): | |
| try: | |
| while True: | |
| st = app.state.rdt2 | |
| if st.ready and server.started: | |
| break | |
| if st.error or server.should_exit: | |
| raise RuntimeError(f"server failed to start: {st.error}") | |
| if time.time() - t0 > ready_timeout_s: | |
| raise TimeoutError(f"server not ready after {ready_timeout_s} s") | |
| time.sleep(1.0) | |
| box["result"] = workload(Served(app, f"http://127.0.0.1:{port}", time.time() - t0)) | |
| except BaseException as e: # noqa: BLE001 -- handed to the main thread | |
| box["error"] = e | |
| finally: | |
| server.should_exit = True | |
| th = threading.Thread(target=client, name="rdt2-workload", daemon=True) | |
| th.start() | |
| try: | |
| server.run() # returns after should_exit (lifespan shutdown closes the model) or a failed start-up | |
| finally: | |
| for k, v in saved.items(): | |
| if v is None: | |
| os.environ.pop(k, None) | |
| else: | |
| os.environ[k] = v | |
| th.join(timeout=60) | |
| if "error" in box: | |
| raise box["error"] | |
| if app.state.rdt2.error and "result" not in box: | |
| raise RuntimeError(f"server failed to start: {app.state.rdt2.error}") | |
| return box.get("result") | |