Download code/tt_rdt2/tests/test_server_host.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/test_server_host.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/tests/test_server_host.py
-
curl -L -o test_server_host.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/test_server_host.py
12.4 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Host tests of the HTTP server (no device): the real app on the stub model (the real API class over a host-only | |
| stand-in of the device model) -- routes, the /predict contract, 400 / 422 / 503 / 500, start-up refusals, served == | |
| Python API, and the stdlib smoke test passing over real HTTP against the stub server.""" | |
| from __future__ import annotations | |
| import base64 | |
| import json | |
| import socket | |
| import threading | |
| import time | |
| import urllib.error | |
| import urllib.request | |
| import pytest | |
| from tt_rdt2.server import app as app_mod | |
| from tt_rdt2.server import client as rdt2_client | |
| from tt_rdt2.tests.stubs import make_model, need_tokenizer, sample_reference | |
| fastapi = pytest.importorskip("fastapi") | |
| from fastapi.testclient import TestClient # noqa: E402 | |
| def _client(factory): | |
| a = app_mod.create_app() | |
| a.state.rdt2.model_factory = factory | |
| return a, TestClient(a) | |
| def served(): | |
| need_tokenizer() | |
| m = make_model() | |
| a, c = _client(lambda cfg: m) | |
| with c: | |
| yield a, c, m | |
| def test_routes(served): | |
| a, c, m = served | |
| h = c.get("/health").json() | |
| assert h["status"] == "ok" and h["device"]["dispatch"] == "eth" and c.get("/v1/health").json() == h | |
| assert c.get("/v1/models").json()["data"][0]["id"] == "robotics-diffusion-transformer/RDT2-VQ" | |
| info = c.get("/info").json() | |
| for k in ("model", "weights", "engine", "device", "prompt_buckets", "cameras", "num_steps", "chunk_shape", "limits", | |
| "warmup_ms", "license", "fixed_configuration", "max_instruction_tokens", "action_layout", "units"): | |
| assert k in info, k | |
| assert info["engine"] == {"vision": "mk", "prefix": "fused", "expert": "mk"} | |
| assert info["prompt_buckets"] == [416, 448, 512] and info["num_steps"] == 5 and info["chunk_shape"] == [24, 20] | |
| assert info["cameras"] == ["left_stereo", "right_stereo"] and info["precision"] == "baseline" | |
| assert info["weights"]["revision"] == "ec7e7f3aa4259224481c3cbd5816cc3f35354bec" | |
| assert set(info["weights"]["components"]) == {"vlm", "expert", "normalizer", "processor"} | |
| assert set(info["warmup_ms"]) == {"416", "448", "512"} | |
| assert c.get("/").json()["routes"] | |
| def test_predict_sample_and_served_equals_api(served): | |
| a, c, m = served | |
| req = rdt2_client.sample_request() | |
| r = c.post("/predict", json=req) | |
| assert r.status_code == 200, r.text | |
| body = r.json() | |
| for k in ("actions", "shape", "normalized", "units", "chunk", "prompt_bucket", "num_tokens", "num_steps", "seed", | |
| "timing_ms", "model"): | |
| assert k in body, k | |
| assert body["prompt_bucket"] == 416 and body["num_tokens"] == 409 and body["seed"] == 0 | |
| assert set(body["timing_ms"]) == {"preprocess", "inference", "total"} | |
| raw = [base64.b64decode(s) for s in req["images"]] | |
| api_out = m(raw, req["instruction"], state=req["state"], seed=req["seed"]).to_dict() | |
| assert api_out["actions"] == body["actions"] | |
| # the stub returns the stored CPU-reference chunk for the sample (as float32): equal up to float32 rounding | |
| ref = sample_reference()["actions"] | |
| assert body["shape"] == [24, 20] and len(body["actions"]) == 24 and all(len(r) == 20 for r in body["actions"]) | |
| assert max(abs(x - y) for r, q in zip(body["actions"], ref) for x, y in zip(r, q)) <= 1e-6 | |
| # identical requests -> identical outputs; another seed -> another chunk (on a non-sample instruction) | |
| r2 = dict(req, instruction="wipe the table") | |
| b1, b2 = c.post("/predict", json=r2).json(), c.post("/predict", json=r2).json() | |
| assert b1["actions"] == b2["actions"] | |
| b3 = c.post("/predict", json=dict(r2, seed=7)).json() | |
| assert b3["actions"] != b1["actions"] and b3["seed"] == 7 | |
| assert c.post("/predict", json=dict(r2, prompt="wipe the table", instruction=None)).json()["actions"] == b1["actions"] | |
| def test_bad_requests(served, change, code, needle): | |
| a, c, m = served | |
| req = rdt2_client.sample_request() | |
| req.update(change) | |
| if req.get("instruction") is None: | |
| req.pop("instruction") | |
| if req["images"] == "ONE": | |
| req["images"] = req["images"][:1] if isinstance(req["images"], list) else rdt2_client.sample_request()["images"][:1] | |
| elif req["images"] == "THREE": | |
| req["images"] = rdt2_client.sample_request()["images"] * 2 | |
| req["images"] = req["images"][:3] | |
| elif req["images"] == "BAD64": | |
| req["images"] = ["bm90IGFuIGltYWdl", rdt2_client.sample_request()["images"][1]] | |
| r = c.post("/predict", json=req) | |
| assert r.status_code == code, r.text | |
| assert needle in r.text | |
| def test_503_before_startup_and_500_on_device_failure(): | |
| need_tokenizer() | |
| a, c = _client(lambda cfg: make_model()) | |
| r = c.post("/predict", json=rdt2_client.sample_request()) # no lifespan: never started | |
| assert r.status_code == 503 and c.get("/health").json()["status"] == "starting" | |
| m = make_model(fail=RuntimeError("device timeout in execute_trace")) | |
| a, c = _client(lambda cfg: m) | |
| with c: | |
| r = c.post("/predict", json=rdt2_client.sample_request()) | |
| assert r.status_code == 500 and "RuntimeError: device timeout in execute_trace" in r.json()["detail"] | |
| assert m.closed # shutdown closes the model | |
| def test_body_limit(served, monkeypatch): | |
| monkeypatch.setenv("RDT2_MAX_BODY_MB", "0.01") | |
| need_tokenizer() | |
| a, c = _client(lambda cfg: make_model()) | |
| with c: | |
| r = c.post("/predict", json=rdt2_client.sample_request()) | |
| assert r.status_code == 400 and "RDT2_MAX_BODY_MB" in r.text | |
| def test_config_refusals(env, needle): | |
| with pytest.raises(RuntimeError, match=needle): | |
| app_mod.config_from_env(env) | |
| cfg = app_mod.config_from_env({"RDT2_NUM_STEPS": "5", "RDT2_PROMPT_BUCKETS": "416, 448, 512", | |
| "TT_MODEL_WEIGHTS_REVISION": "abc", "RDT2_DISPATCH": "WORKER"}) | |
| assert cfg["revision"] == "abc" and cfg["dispatch"] == "worker" and cfg["precision"] == "baseline" | |
| assert (cfg["num_cameras"], cfg["num_steps"]) == (2, 5) | |
| def test_config_cameras_steps(cams, steps, buckets): | |
| """Stage Config: RDT2_NUM_CAMERAS 1..4 / RDT2_NUM_STEPS 1..16 are accepted (read once at start); the buckets | |
| follow the camera count and RDT2_PROMPT_BUCKETS may restate them.""" | |
| env = {"RDT2_NUM_CAMERAS": cams, "RDT2_NUM_STEPS": steps} | |
| cfg = app_mod.config_from_env(env) | |
| assert (cfg["num_cameras"], cfg["num_steps"]) == (int(cams or 2), int(steps or 5)) | |
| assert app_mod.config_from_env(dict(env, RDT2_PROMPT_BUCKETS=buckets.replace(",", ", ")))["num_cameras"] == int(cams or 2) | |
| def test_startup_failure_is_reported(): | |
| def boom(cfg): | |
| raise ValueError("RDT2_EXPERIMENTAL_PREFIX_MK refused") | |
| a, c = _client(boom) | |
| with pytest.raises(ValueError): | |
| with c: | |
| pass | |
| assert a.state.rdt2.status() == "error" and "refused" in a.state.rdt2.error | |
| def _free_port(): | |
| s = socket.socket() | |
| s.bind(("127.0.0.1", 0)) | |
| p = s.getsockname()[1] | |
| s.close() | |
| return p | |
| def _serve_stub(app, port): | |
| import uvicorn | |
| server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning", lifespan="on")) | |
| th = threading.Thread(target=server.run, daemon=True) | |
| th.start() | |
| base = f"http://127.0.0.1:{port}" | |
| t_end = time.time() + 60 | |
| while True: | |
| try: | |
| with urllib.request.urlopen(base + "/health", timeout=5) as r: | |
| if json.loads(r.read())["status"] == "ok": | |
| break | |
| except OSError: | |
| pass | |
| if time.time() > t_end: | |
| raise TimeoutError("stub server did not start") | |
| time.sleep(0.1) | |
| return server, th, base | |
| def test_body_limit_over_http(monkeypatch): | |
| """A body above RDT2_MAX_BODY_MB that the client sends in full before reading the answer (urllib) gets the 400 with | |
| the reason, not a connection reset; the server keeps serving.""" | |
| pytest.importorskip("uvicorn") | |
| need_tokenizer() | |
| monkeypatch.setenv("RDT2_MAX_BODY_MB", "1") | |
| m = make_model() | |
| a = app_mod.create_app() | |
| a.state.rdt2.model_factory = lambda cfg: m | |
| server, th, base = _serve_stub(a, _free_port()) | |
| try: | |
| sample = json.dumps(rdt2_client.sample_request()).encode() | |
| for target in (2 << 20, (app_mod.BODY_DRAIN_FACTOR << 20) - 1024): # above the limit, within the drain bound | |
| body = json.dumps(dict(rdt2_client.sample_request(), instruction="x" * (target - len(sample)))).encode() | |
| assert (1 << 20) < len(body) <= app_mod.BODY_DRAIN_FACTOR << 20 | |
| req = urllib.request.Request(base + "/predict", data=body, headers={"Content-Type": "application/json"}) | |
| with pytest.raises(urllib.error.HTTPError) as e: | |
| urllib.request.urlopen(req, timeout=60) | |
| assert e.value.code == 400 and "RDT2_MAX_BODY_MB" in e.value.read().decode() | |
| req = urllib.request.Request(base + "/predict", data=json.dumps(rdt2_client.sample_request()).encode(), | |
| headers={"Content-Type": "application/json"}) | |
| with urllib.request.urlopen(req, timeout=60) as r: | |
| assert r.status == 200 and json.loads(r.read())["shape"] == [24, 20] | |
| finally: | |
| server.should_exit = True | |
| th.join(timeout=30) | |
| def test_smoke_over_http_against_stub(tmp_path): | |
| """The shipped stdlib smoke test passes against the real app over loopback HTTP (stub model).""" | |
| uvicorn = pytest.importorskip("uvicorn") | |
| need_tokenizer() | |
| m = make_model() | |
| a = app_mod.create_app() | |
| a.state.rdt2.model_factory = lambda cfg: m | |
| port = _free_port() | |
| server = uvicorn.Server(uvicorn.Config(a, host="127.0.0.1", port=port, log_level="warning", lifespan="on")) | |
| th = threading.Thread(target=server.run, daemon=True) | |
| th.start() | |
| try: | |
| from tt_rdt2.server import smoke_test | |
| rc = smoke_test.main(["--url", f"http://127.0.0.1:{port}", "--wait", "30", "--out", str(tmp_path / "s.json")]) | |
| assert rc == 0 | |
| assert json.loads((tmp_path / "s.json").read_text())["prompt_bucket"] == 416 | |
| # a perturbed chunk fails the reference gates | |
| ref = sample_reference() | |
| bad = [[x + (0.02 if j == 0 else 0.0) for j, x in enumerate(r)] for r in ref["actions"]] | |
| assert smoke_test.judge(bad, ref)[1] and not smoke_test.judge(ref["actions"], ref)[1] | |
| finally: | |
| server.should_exit = True | |
| th.join(timeout=30) | |