# SPDX-License-Identifier: Apache-2.0 """Host tests of the Python API (no device; fake ttnn in the workspace): request validation and refusals, the experimental-engine gate, weight resolution, and request decoding == the reference pre-processing the device gates use (bit for bit, against the official golden where it is present).""" from __future__ import annotations import io import json import os import subprocess import sys from pathlib import Path import numpy as np import pytest import torch from tt_rdt2 import api from tt_rdt2.api import InputError from tt_rdt2.tests.stubs import make_model, need_tokenizer, sample_reference BUNDLE = Path(__file__).resolve().parents[3] def _png(a: np.ndarray) -> bytes: from PIL import Image b = io.BytesIO() Image.fromarray(a).save(b, format="PNG") return b.getvalue() @pytest.fixture(scope="module") def model(): need_tokenizer() return make_model() @pytest.fixture(scope="module") def sample(): return api.load_sample() def test_import_has_no_side_effects(): code = ("import sys, tt_rdt2; assert tt_rdt2.__version__; " "bad = [m for m in ('torch', 'ttnn', 'numpy', 'huggingface_hub') if m in sys.modules]; " "assert not bad, bad; tt_rdt2.RDT2FM; assert 'torch' not in sys.modules or True") r = subprocess.run([sys.executable, "-c", code], cwd=str(BUNDLE / "code"), capture_output=True, text=True, env=dict(os.environ, PYTHONPATH=str(BUNDLE / "code"))) assert r.returncode == 0, r.stderr[-800:] def test_experimental_prefix_mk_is_refused(monkeypatch): from tt_rdt2.tt.model import RDT2FMTT, check_engine_allowed, resolve_engine monkeypatch.delenv("RDT2_EXPERIMENTAL_PREFIX_MK", raising=False) assert resolve_engine(None if "RDT2_ENGINE" not in os.environ else "") == {"vision": "mk", "prefix": "fused", "expert": "mk"} check_engine_allowed(resolve_engine("")) # the defaults pass with pytest.raises(ValueError, match="hangs the device"): check_engine_allowed(resolve_engine("prefix=mk")) with pytest.raises(ValueError, match="RDT2_EXPERIMENTAL_PREFIX_MK=1"): RDT2FMTT(None, engine="prefix=mk") # refused before anything touches the device monkeypatch.setenv("RDT2_ENGINE", "prefix=mk") with pytest.raises(ValueError, match="experimental"): RDT2FMTT(None) def no_weights(*a, **k): raise AssertionError("weights resolved before the engine check") monkeypatch.setattr(api, "resolve_weights", no_weights) with pytest.raises(ValueError, match="experimental"): api.RDT2FM.from_pretrained(engine="prefix=mk") check_engine_allowed(resolve_engine("prefix=mk"), env={"RDT2_EXPERIMENTAL_PREFIX_MK": "1"}) # explicit opt-in def test_sample_reference_is_the_official_request(sample): ref = sample_reference() assert ref["request"]["instruction"] == sample["instruction"] and ref["request"]["seed"] == 0 assert np.array_equal(np.asarray(ref["request"]["state"], np.float32), sample["state"]) from PIL import Image for cam, a in zip(api.CAMERA_ORDER, sample["images"]): png = api.SAMPLE_PATH.parent / f"ur5e_bimanual_example_0_{cam}.png" assert np.array_equal(np.asarray(Image.open(png).convert("RGB")), a) assert np.asarray(ref["actions"]).shape == (24, 20) and set(ref["limits"]) == {"one_minus_pcc", "pos_m", "rot", "grip_m"} sc, of = np.asarray(ref["normalizer"]["scale"]), np.asarray(ref["normalizer"]["offset"]) assert np.allclose((np.asarray(ref["actions_normalized"]) - of) / sc, ref["actions"], atol=1e-5) def test_seeded_noise_matches_golden_driver(): n0 = api.seeded_noise(0) assert n0.shape == (1, 24, 20) and n0.dtype == torch.float32 assert torch.equal(n0, torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0))) assert not torch.equal(n0, api.seeded_noise(1)) def test_request_decoding_equals_reference_preprocessing(model, sample, golden_or_none): """PNG over the API == the arrays the gates feed: same pixels, ids, mask, noise, state; and (golden present) the pixel rows equal the official processor's pixel_values and the ids the official input_ids, bit for bit.""" from tt_rdt2.tt.vision import host_pixels, host_pixels_from_pixel_values tt = model._m out = model([_png(a) for a in sample["images"]], sample["instruction"]) png_host = tt.calls[-1] model(sample["images"], sample["instruction"], state=sample["state"], seed=0) arr_host = tt.calls[-1] for k in ("pix", "ids", "mask", "noise", "state"): assert torch.equal(png_host[k], arr_host[k]), k assert int(png_host["n_tokens"]) == 409 and png_host["bucket"] == 416 == out.prompt_bucket assert torch.equal(png_host["pix"], host_pixels(sample["images"])) if golden_or_none is not None: d = golden_or_none.load("real_ur5e__s0", with_vlm=False) assert torch.equal(png_host["pix"], host_pixels_from_pixel_values(d["in.pixel_values"])) ids = d["in.input_ids"].reshape(-1) got = png_host["ids"].reshape(-1)[:ids.numel()].to(torch.int64) img = ids == 151655 assert torch.equal(got[~img], ids[~img]) assert torch.equal(png_host["noise"][0, 0, :24, :20], d["in.noise"].float().reshape(24, 20)) @pytest.fixture(scope="module") def golden_or_none(): from tt_rdt2.reference.config import DEFAULT_DIRS from tt_rdt2.reference.golden import GoldenSet if not os.path.exists(os.path.join(DEFAULT_DIRS["golden"], "fp32", "manifest.json")): return None return GoldenSet("fp32") def test_outputs_and_timing(model, sample): out = model(**sample) d = out.to_dict() assert out.actions.shape == (24, 20) and out.actions.dtype == np.float32 assert d["shape"] == [24, 20] and d["normalized"] is False and d["num_steps"] == 5 and d["seed"] == 0 assert set(d["timing_ms"]) == {"preprocess", "inference", "total"} assert d["chunk"] == {"horizon": 24, "action_dim": 20, "rate_hz": 30} assert np.allclose(d["actions"], sample_reference()["actions"]) json.dumps(d) @pytest.mark.parametrize("words,bucket", [(3, 416), (30, 448), (80, 512)]) def test_bucket_selection(model, sample, words, bucket): instr = " ".join(["move"] * words) n = model.tokens_for(instr) out = model(sample["images"], instr) assert out.prompt_bucket == bucket == min(b for b in api.PROMPT_BUCKETS if b >= n) and out.num_tokens == n assert model(sample["images"], instr, prompt_bucket=512).prompt_bucket == 512 def test_geometry_accepted_and_refused(model, sample): model([np.zeros((380, 380, 3), np.uint8)] * 2, "pick up the cup") # same 392 x 756 grid: resized for imgs, msg in [([np.zeros((400, 400, 3), np.uint8)] * 2, "unsupported image geometry"), ([np.zeros((384, 384, 3), np.uint8), np.zeros((380, 384, 3), np.uint8)], "same height"), ([np.zeros((384, 384, 3), np.float32)] * 2, "uint8"), ([np.zeros((384, 384), np.uint8)] * 2, "HxWx3"), ([b"not an image", b"x"], "cannot decode")]: with pytest.raises(InputError, match=msg): model(imgs, "pick up the cup") @pytest.mark.parametrize("kw,msg", [ (dict(images=1), "list of 2"), (dict(images="n_cams_1"), "exactly 2 images"), (dict(images="n_cams_3"), "exactly 2"), (dict(instruction=""), "empty"), (dict(instruction=5), "string"), (dict(instruction="Pick up the cup.<|im_end|>"), "special-token"), (dict(instruction="<|vision_start|>x"), "special-token"), (dict(instruction=" ".join(["pick up the red cup and place it on the plate"] * 40)), "exceeds the largest"), (dict(num_steps=4), "num_steps must be 5"), (dict(num_steps=True), "num_steps"), (dict(prompt_bucket=500), "prompt_bucket must be one of"), (dict(prompt_bucket=416, instruction=" ".join(["move"] * 30)), "does not fit"), (dict(state=[0.0] * 19), "exactly 20"), (dict(state=[float("nan")] + [0.0] * 19), "non-finite"), (dict(state="abc"), "20 numbers"), (dict(seed=-1), "seed must be in"), (dict(seed=1.5), "integer"), (dict(seed=True), "integer"), ]) def test_refusals(model, sample, kw, msg): req = dict(images=list(sample["images"]), instruction="pick up the cup") req.update(kw) if req["images"] == "n_cams_1": req["images"] = sample["images"][:1] elif req["images"] == "n_cams_3": req["images"] = list(sample["images"]) * 2 req["images"] = req["images"][:3] with pytest.raises(InputError, match=msg): model(**req) def test_camera_mapping(model, sample): l, r = sample["images"] a = model({"right_stereo": r, "left_stereo": l}, sample["instruction"]).actions b = model([l, r], sample["instruction"]).actions assert np.array_equal(a, b) with pytest.raises(InputError, match="cameras must be exactly"): model({"left": l, "right_stereo": r}, "x") def test_close_is_idempotent(): need_tokenizer() m = make_model() with m: pass assert m.closed and m._m.released m.close() with pytest.raises(RuntimeError, match="closed"): m(*[[np.zeros((384, 384, 3), np.uint8)] * 2], "pick up the cup") def test_vlm_needed_files(tmp_path): wm = {"visual.blocks.0.attn.qkv.weight": "a.safetensors", "model.embed_tokens.weight": "a.safetensors", "model.layers.13.self_attn.k_proj.weight": "b.safetensors", "model.layers.14.mlp.up_proj.weight": "c.safetensors", "model.layers.1.mlp.up_proj.weight": "a.safetensors", "lm_head.weight": "d.safetensors", "model.norm.weight": "d.safetensors"} (tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": wm})) assert api.vlm_needed_files(tmp_path / "model.safetensors.index.json") == ["a.safetensors", "b.safetensors"] from tt_rdt2.reference.config import DEFAULT_DIRS real = Path(DEFAULT_DIRS["vlm"]) / "model.safetensors.index.json" if real.is_file(): # the pinned RDT2-VQ: shards 1-2 only, and tt-model.yaml's allow_patterns cover them need = api.vlm_needed_files(real) assert need == ["model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"] y = (BUNDLE / "tt-model.yaml").read_text() if (BUNDLE / "tt-model.yaml").is_file() else "" if y: for f in need + ["model.safetensors.index.json", "config.json"]: assert f in y, f def test_resolve_weights_local_and_hub(tmp_path, monkeypatch): # local directories (env) win; a missing file is an error that names it dirs = {} for role, c in api.COMPONENTS.items(): d = tmp_path / role d.mkdir() for f in c["files"]: (d / f).write_text("x") dirs[role] = str(d) (tmp_path / "vlm" / "model.safetensors.index.json").write_text(json.dumps( {"weight_map": {"visual.x": "s1.safetensors", "lm_head.weight": "s9.safetensors"}})) (tmp_path / "vlm" / "s1.safetensors").write_text("x") env = {api.COMPONENTS[r]["env"]: p for r, p in dirs.items()} w = api.resolve_weights(env=env, log_fn=lambda s: None) assert w["vlm"]["source"] == "local" and w["vlm"]["files"][-1] == "s1.safetensors" os.remove(tmp_path / "expert" / "pytorch_model.bin") with pytest.raises(FileNotFoundError, match="pytorch_model.bin"): api.resolve_weights(env=env, log_fn=lambda s: None) # the hub path: pinned revisions; a network failure retries the cache with local_files_only=True calls = [] snap = tmp_path / "snap" snap.mkdir() (snap / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {"visual.x": "s1.safetensors"}})) def fake_dl(repo, filename, revision=None, local_files_only=False): calls.append((repo, filename, revision, local_files_only)) if not local_files_only: raise OSError("offline") return str(snap / filename) import huggingface_hub monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_dl) w = api.resolve_weights(env={}, log_fn=lambda s: None) assert {r: v["revision"] for r, v in w.items()} == {r: c["revision"] for r, c in api.COMPONENTS.items()} assert all(c[2] == api.COMPONENTS[r]["revision"] for r in api.COMPONENTS for c in calls if c[0] == api.COMPONENTS[r]["repo"]) assert ("robotics-diffusion-transformer/RDT2-VQ", "s1.safetensors", api.VLM_REVISION, True) in calls assert w["vlm"]["source"] == "hf-cache" and w["vlm"]["path"] == str(snap)