Download code/tt_rdt2/tests/test_api_host.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/test_api_host.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/tests/test_api_host.py
-
curl -L -o test_api_host.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/test_api_host.py
12.7 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Host tests of the Python API (no device; fake ttnn in the workspace): request validation and refusals, the | |
| experimental-engine gate, weight resolution, and request decoding == the reference pre-processing the device gates | |
| use (bit for bit, against the official golden where it is present).""" | |
| from __future__ import annotations | |
| import io | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pytest | |
| import torch | |
| from tt_rdt2 import api | |
| from tt_rdt2.api import InputError | |
| from tt_rdt2.tests.stubs import make_model, need_tokenizer, sample_reference | |
| BUNDLE = Path(__file__).resolve().parents[3] | |
| def _png(a: np.ndarray) -> bytes: | |
| from PIL import Image | |
| b = io.BytesIO() | |
| Image.fromarray(a).save(b, format="PNG") | |
| return b.getvalue() | |
| def model(): | |
| need_tokenizer() | |
| return make_model() | |
| def sample(): | |
| return api.load_sample() | |
| def test_import_has_no_side_effects(): | |
| code = ("import sys, tt_rdt2; assert tt_rdt2.__version__; " | |
| "bad = [m for m in ('torch', 'ttnn', 'numpy', 'huggingface_hub') if m in sys.modules]; " | |
| "assert not bad, bad; tt_rdt2.RDT2FM; assert 'torch' not in sys.modules or True") | |
| r = subprocess.run([sys.executable, "-c", code], cwd=str(BUNDLE / "code"), capture_output=True, text=True, | |
| env=dict(os.environ, PYTHONPATH=str(BUNDLE / "code"))) | |
| assert r.returncode == 0, r.stderr[-800:] | |
| def test_experimental_prefix_mk_is_refused(monkeypatch): | |
| from tt_rdt2.tt.model import RDT2FMTT, check_engine_allowed, resolve_engine | |
| monkeypatch.delenv("RDT2_EXPERIMENTAL_PREFIX_MK", raising=False) | |
| assert resolve_engine(None if "RDT2_ENGINE" not in os.environ else "") == {"vision": "mk", "prefix": "fused", | |
| "expert": "mk"} | |
| check_engine_allowed(resolve_engine("")) # the defaults pass | |
| with pytest.raises(ValueError, match="hangs the device"): | |
| check_engine_allowed(resolve_engine("prefix=mk")) | |
| with pytest.raises(ValueError, match="RDT2_EXPERIMENTAL_PREFIX_MK=1"): | |
| RDT2FMTT(None, engine="prefix=mk") # refused before anything touches the device | |
| monkeypatch.setenv("RDT2_ENGINE", "prefix=mk") | |
| with pytest.raises(ValueError, match="experimental"): | |
| RDT2FMTT(None) | |
| def no_weights(*a, **k): | |
| raise AssertionError("weights resolved before the engine check") | |
| monkeypatch.setattr(api, "resolve_weights", no_weights) | |
| with pytest.raises(ValueError, match="experimental"): | |
| api.RDT2FM.from_pretrained(engine="prefix=mk") | |
| check_engine_allowed(resolve_engine("prefix=mk"), env={"RDT2_EXPERIMENTAL_PREFIX_MK": "1"}) # explicit opt-in | |
| def test_sample_reference_is_the_official_request(sample): | |
| ref = sample_reference() | |
| assert ref["request"]["instruction"] == sample["instruction"] and ref["request"]["seed"] == 0 | |
| assert np.array_equal(np.asarray(ref["request"]["state"], np.float32), sample["state"]) | |
| from PIL import Image | |
| for cam, a in zip(api.CAMERA_ORDER, sample["images"]): | |
| png = api.SAMPLE_PATH.parent / f"ur5e_bimanual_example_0_{cam}.png" | |
| assert np.array_equal(np.asarray(Image.open(png).convert("RGB")), a) | |
| assert np.asarray(ref["actions"]).shape == (24, 20) and set(ref["limits"]) == {"one_minus_pcc", "pos_m", "rot", | |
| "grip_m"} | |
| sc, of = np.asarray(ref["normalizer"]["scale"]), np.asarray(ref["normalizer"]["offset"]) | |
| assert np.allclose((np.asarray(ref["actions_normalized"]) - of) / sc, ref["actions"], atol=1e-5) | |
| def test_seeded_noise_matches_golden_driver(): | |
| n0 = api.seeded_noise(0) | |
| assert n0.shape == (1, 24, 20) and n0.dtype == torch.float32 | |
| assert torch.equal(n0, torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0))) | |
| assert not torch.equal(n0, api.seeded_noise(1)) | |
| def test_request_decoding_equals_reference_preprocessing(model, sample, golden_or_none): | |
| """PNG over the API == the arrays the gates feed: same pixels, ids, mask, noise, state; and (golden present) | |
| the pixel rows equal the official processor's pixel_values and the ids the official input_ids, bit for bit.""" | |
| from tt_rdt2.tt.vision import host_pixels, host_pixels_from_pixel_values | |
| tt = model._m | |
| out = model([_png(a) for a in sample["images"]], sample["instruction"]) | |
| png_host = tt.calls[-1] | |
| model(sample["images"], sample["instruction"], state=sample["state"], seed=0) | |
| arr_host = tt.calls[-1] | |
| for k in ("pix", "ids", "mask", "noise", "state"): | |
| assert torch.equal(png_host[k], arr_host[k]), k | |
| assert int(png_host["n_tokens"]) == 409 and png_host["bucket"] == 416 == out.prompt_bucket | |
| assert torch.equal(png_host["pix"], host_pixels(sample["images"])) | |
| if golden_or_none is not None: | |
| d = golden_or_none.load("real_ur5e__s0", with_vlm=False) | |
| assert torch.equal(png_host["pix"], host_pixels_from_pixel_values(d["in.pixel_values"])) | |
| ids = d["in.input_ids"].reshape(-1) | |
| got = png_host["ids"].reshape(-1)[:ids.numel()].to(torch.int64) | |
| img = ids == 151655 | |
| assert torch.equal(got[~img], ids[~img]) | |
| assert torch.equal(png_host["noise"][0, 0, :24, :20], d["in.noise"].float().reshape(24, 20)) | |
| def golden_or_none(): | |
| from tt_rdt2.reference.config import DEFAULT_DIRS | |
| from tt_rdt2.reference.golden import GoldenSet | |
| if not os.path.exists(os.path.join(DEFAULT_DIRS["golden"], "fp32", "manifest.json")): | |
| return None | |
| return GoldenSet("fp32") | |
| def test_outputs_and_timing(model, sample): | |
| out = model(**sample) | |
| d = out.to_dict() | |
| assert out.actions.shape == (24, 20) and out.actions.dtype == np.float32 | |
| assert d["shape"] == [24, 20] and d["normalized"] is False and d["num_steps"] == 5 and d["seed"] == 0 | |
| assert set(d["timing_ms"]) == {"preprocess", "inference", "total"} | |
| assert d["chunk"] == {"horizon": 24, "action_dim": 20, "rate_hz": 30} | |
| assert np.allclose(d["actions"], sample_reference()["actions"]) | |
| json.dumps(d) | |
| def test_bucket_selection(model, sample, words, bucket): | |
| instr = " ".join(["move"] * words) | |
| n = model.tokens_for(instr) | |
| out = model(sample["images"], instr) | |
| assert out.prompt_bucket == bucket == min(b for b in api.PROMPT_BUCKETS if b >= n) and out.num_tokens == n | |
| assert model(sample["images"], instr, prompt_bucket=512).prompt_bucket == 512 | |
| def test_geometry_accepted_and_refused(model, sample): | |
| model([np.zeros((380, 380, 3), np.uint8)] * 2, "pick up the cup") # same 392 x 756 grid: resized | |
| for imgs, msg in [([np.zeros((400, 400, 3), np.uint8)] * 2, "unsupported image geometry"), | |
| ([np.zeros((384, 384, 3), np.uint8), np.zeros((380, 384, 3), np.uint8)], "same height"), | |
| ([np.zeros((384, 384, 3), np.float32)] * 2, "uint8"), | |
| ([np.zeros((384, 384), np.uint8)] * 2, "HxWx3"), | |
| ([b"not an image", b"x"], "cannot decode")]: | |
| with pytest.raises(InputError, match=msg): | |
| model(imgs, "pick up the cup") | |
| def test_refusals(model, sample, kw, msg): | |
| req = dict(images=list(sample["images"]), instruction="pick up the cup") | |
| req.update(kw) | |
| if req["images"] == "n_cams_1": | |
| req["images"] = sample["images"][:1] | |
| elif req["images"] == "n_cams_3": | |
| req["images"] = list(sample["images"]) * 2 | |
| req["images"] = req["images"][:3] | |
| with pytest.raises(InputError, match=msg): | |
| model(**req) | |
| def test_camera_mapping(model, sample): | |
| l, r = sample["images"] | |
| a = model({"right_stereo": r, "left_stereo": l}, sample["instruction"]).actions | |
| b = model([l, r], sample["instruction"]).actions | |
| assert np.array_equal(a, b) | |
| with pytest.raises(InputError, match="cameras must be exactly"): | |
| model({"left": l, "right_stereo": r}, "x") | |
| def test_close_is_idempotent(): | |
| need_tokenizer() | |
| m = make_model() | |
| with m: | |
| pass | |
| assert m.closed and m._m.released | |
| m.close() | |
| with pytest.raises(RuntimeError, match="closed"): | |
| m(*[[np.zeros((384, 384, 3), np.uint8)] * 2], "pick up the cup") | |
| def test_vlm_needed_files(tmp_path): | |
| wm = {"visual.blocks.0.attn.qkv.weight": "a.safetensors", "model.embed_tokens.weight": "a.safetensors", | |
| "model.layers.13.self_attn.k_proj.weight": "b.safetensors", "model.layers.14.mlp.up_proj.weight": "c.safetensors", | |
| "model.layers.1.mlp.up_proj.weight": "a.safetensors", "lm_head.weight": "d.safetensors", | |
| "model.norm.weight": "d.safetensors"} | |
| (tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": wm})) | |
| assert api.vlm_needed_files(tmp_path / "model.safetensors.index.json") == ["a.safetensors", "b.safetensors"] | |
| from tt_rdt2.reference.config import DEFAULT_DIRS | |
| real = Path(DEFAULT_DIRS["vlm"]) / "model.safetensors.index.json" | |
| if real.is_file(): # the pinned RDT2-VQ: shards 1-2 only, and tt-model.yaml's allow_patterns cover them | |
| need = api.vlm_needed_files(real) | |
| assert need == ["model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"] | |
| y = (BUNDLE / "tt-model.yaml").read_text() if (BUNDLE / "tt-model.yaml").is_file() else "" | |
| if y: | |
| for f in need + ["model.safetensors.index.json", "config.json"]: | |
| assert f in y, f | |
| def test_resolve_weights_local_and_hub(tmp_path, monkeypatch): | |
| # local directories (env) win; a missing file is an error that names it | |
| dirs = {} | |
| for role, c in api.COMPONENTS.items(): | |
| d = tmp_path / role | |
| d.mkdir() | |
| for f in c["files"]: | |
| (d / f).write_text("x") | |
| dirs[role] = str(d) | |
| (tmp_path / "vlm" / "model.safetensors.index.json").write_text(json.dumps( | |
| {"weight_map": {"visual.x": "s1.safetensors", "lm_head.weight": "s9.safetensors"}})) | |
| (tmp_path / "vlm" / "s1.safetensors").write_text("x") | |
| env = {api.COMPONENTS[r]["env"]: p for r, p in dirs.items()} | |
| w = api.resolve_weights(env=env, log_fn=lambda s: None) | |
| assert w["vlm"]["source"] == "local" and w["vlm"]["files"][-1] == "s1.safetensors" | |
| os.remove(tmp_path / "expert" / "pytorch_model.bin") | |
| with pytest.raises(FileNotFoundError, match="pytorch_model.bin"): | |
| api.resolve_weights(env=env, log_fn=lambda s: None) | |
| # the hub path: pinned revisions; a network failure retries the cache with local_files_only=True | |
| calls = [] | |
| snap = tmp_path / "snap" | |
| snap.mkdir() | |
| (snap / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {"visual.x": "s1.safetensors"}})) | |
| def fake_dl(repo, filename, revision=None, local_files_only=False): | |
| calls.append((repo, filename, revision, local_files_only)) | |
| if not local_files_only: | |
| raise OSError("offline") | |
| return str(snap / filename) | |
| import huggingface_hub | |
| monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_dl) | |
| w = api.resolve_weights(env={}, log_fn=lambda s: None) | |
| assert {r: v["revision"] for r, v in w.items()} == {r: c["revision"] for r, c in api.COMPONENTS.items()} | |
| assert all(c[2] == api.COMPONENTS[r]["revision"] for r in api.COMPONENTS for c in calls | |
| if c[0] == api.COMPONENTS[r]["repo"]) | |
| assert ("robotics-diffusion-transformer/RDT2-VQ", "s1.safetensors", api.VLM_REVISION, True) in calls | |
| assert w["vlm"]["source"] == "hf-cache" and w["vlm"]["path"] == str(snap) | |