rdt2-base-p150 / code /tt_rdt2 /tests /test_api_host.py
changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
12.7 kB
# SPDX-License-Identifier: Apache-2.0
"""Host tests of the Python API (no device; fake ttnn in the workspace): request validation and refusals, the
experimental-engine gate, weight resolution, and request decoding == the reference pre-processing the device gates
use (bit for bit, against the official golden where it is present)."""
from __future__ import annotations
import io
import json
import os
import subprocess
import sys
from pathlib import Path
import numpy as np
import pytest
import torch
from tt_rdt2 import api
from tt_rdt2.api import InputError
from tt_rdt2.tests.stubs import make_model, need_tokenizer, sample_reference
BUNDLE = Path(__file__).resolve().parents[3]
def _png(a: np.ndarray) -> bytes:
from PIL import Image
b = io.BytesIO()
Image.fromarray(a).save(b, format="PNG")
return b.getvalue()
@pytest.fixture(scope="module")
def model():
need_tokenizer()
return make_model()
@pytest.fixture(scope="module")
def sample():
return api.load_sample()
def test_import_has_no_side_effects():
code = ("import sys, tt_rdt2; assert tt_rdt2.__version__; "
"bad = [m for m in ('torch', 'ttnn', 'numpy', 'huggingface_hub') if m in sys.modules]; "
"assert not bad, bad; tt_rdt2.RDT2FM; assert 'torch' not in sys.modules or True")
r = subprocess.run([sys.executable, "-c", code], cwd=str(BUNDLE / "code"), capture_output=True, text=True,
env=dict(os.environ, PYTHONPATH=str(BUNDLE / "code")))
assert r.returncode == 0, r.stderr[-800:]
def test_experimental_prefix_mk_is_refused(monkeypatch):
from tt_rdt2.tt.model import RDT2FMTT, check_engine_allowed, resolve_engine
monkeypatch.delenv("RDT2_EXPERIMENTAL_PREFIX_MK", raising=False)
assert resolve_engine(None if "RDT2_ENGINE" not in os.environ else "") == {"vision": "mk", "prefix": "fused",
"expert": "mk"}
check_engine_allowed(resolve_engine("")) # the defaults pass
with pytest.raises(ValueError, match="hangs the device"):
check_engine_allowed(resolve_engine("prefix=mk"))
with pytest.raises(ValueError, match="RDT2_EXPERIMENTAL_PREFIX_MK=1"):
RDT2FMTT(None, engine="prefix=mk") # refused before anything touches the device
monkeypatch.setenv("RDT2_ENGINE", "prefix=mk")
with pytest.raises(ValueError, match="experimental"):
RDT2FMTT(None)
def no_weights(*a, **k):
raise AssertionError("weights resolved before the engine check")
monkeypatch.setattr(api, "resolve_weights", no_weights)
with pytest.raises(ValueError, match="experimental"):
api.RDT2FM.from_pretrained(engine="prefix=mk")
check_engine_allowed(resolve_engine("prefix=mk"), env={"RDT2_EXPERIMENTAL_PREFIX_MK": "1"}) # explicit opt-in
def test_sample_reference_is_the_official_request(sample):
ref = sample_reference()
assert ref["request"]["instruction"] == sample["instruction"] and ref["request"]["seed"] == 0
assert np.array_equal(np.asarray(ref["request"]["state"], np.float32), sample["state"])
from PIL import Image
for cam, a in zip(api.CAMERA_ORDER, sample["images"]):
png = api.SAMPLE_PATH.parent / f"ur5e_bimanual_example_0_{cam}.png"
assert np.array_equal(np.asarray(Image.open(png).convert("RGB")), a)
assert np.asarray(ref["actions"]).shape == (24, 20) and set(ref["limits"]) == {"one_minus_pcc", "pos_m", "rot",
"grip_m"}
sc, of = np.asarray(ref["normalizer"]["scale"]), np.asarray(ref["normalizer"]["offset"])
assert np.allclose((np.asarray(ref["actions_normalized"]) - of) / sc, ref["actions"], atol=1e-5)
def test_seeded_noise_matches_golden_driver():
n0 = api.seeded_noise(0)
assert n0.shape == (1, 24, 20) and n0.dtype == torch.float32
assert torch.equal(n0, torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0)))
assert not torch.equal(n0, api.seeded_noise(1))
def test_request_decoding_equals_reference_preprocessing(model, sample, golden_or_none):
"""PNG over the API == the arrays the gates feed: same pixels, ids, mask, noise, state; and (golden present)
the pixel rows equal the official processor's pixel_values and the ids the official input_ids, bit for bit."""
from tt_rdt2.tt.vision import host_pixels, host_pixels_from_pixel_values
tt = model._m
out = model([_png(a) for a in sample["images"]], sample["instruction"])
png_host = tt.calls[-1]
model(sample["images"], sample["instruction"], state=sample["state"], seed=0)
arr_host = tt.calls[-1]
for k in ("pix", "ids", "mask", "noise", "state"):
assert torch.equal(png_host[k], arr_host[k]), k
assert int(png_host["n_tokens"]) == 409 and png_host["bucket"] == 416 == out.prompt_bucket
assert torch.equal(png_host["pix"], host_pixels(sample["images"]))
if golden_or_none is not None:
d = golden_or_none.load("real_ur5e__s0", with_vlm=False)
assert torch.equal(png_host["pix"], host_pixels_from_pixel_values(d["in.pixel_values"]))
ids = d["in.input_ids"].reshape(-1)
got = png_host["ids"].reshape(-1)[:ids.numel()].to(torch.int64)
img = ids == 151655
assert torch.equal(got[~img], ids[~img])
assert torch.equal(png_host["noise"][0, 0, :24, :20], d["in.noise"].float().reshape(24, 20))
@pytest.fixture(scope="module")
def golden_or_none():
from tt_rdt2.reference.config import DEFAULT_DIRS
from tt_rdt2.reference.golden import GoldenSet
if not os.path.exists(os.path.join(DEFAULT_DIRS["golden"], "fp32", "manifest.json")):
return None
return GoldenSet("fp32")
def test_outputs_and_timing(model, sample):
out = model(**sample)
d = out.to_dict()
assert out.actions.shape == (24, 20) and out.actions.dtype == np.float32
assert d["shape"] == [24, 20] and d["normalized"] is False and d["num_steps"] == 5 and d["seed"] == 0
assert set(d["timing_ms"]) == {"preprocess", "inference", "total"}
assert d["chunk"] == {"horizon": 24, "action_dim": 20, "rate_hz": 30}
assert np.allclose(d["actions"], sample_reference()["actions"])
json.dumps(d)
@pytest.mark.parametrize("words,bucket", [(3, 416), (30, 448), (80, 512)])
def test_bucket_selection(model, sample, words, bucket):
instr = " ".join(["move"] * words)
n = model.tokens_for(instr)
out = model(sample["images"], instr)
assert out.prompt_bucket == bucket == min(b for b in api.PROMPT_BUCKETS if b >= n) and out.num_tokens == n
assert model(sample["images"], instr, prompt_bucket=512).prompt_bucket == 512
def test_geometry_accepted_and_refused(model, sample):
model([np.zeros((380, 380, 3), np.uint8)] * 2, "pick up the cup") # same 392 x 756 grid: resized
for imgs, msg in [([np.zeros((400, 400, 3), np.uint8)] * 2, "unsupported image geometry"),
([np.zeros((384, 384, 3), np.uint8), np.zeros((380, 384, 3), np.uint8)], "same height"),
([np.zeros((384, 384, 3), np.float32)] * 2, "uint8"),
([np.zeros((384, 384), np.uint8)] * 2, "HxWx3"),
([b"not an image", b"x"], "cannot decode")]:
with pytest.raises(InputError, match=msg):
model(imgs, "pick up the cup")
@pytest.mark.parametrize("kw,msg", [
(dict(images=1), "list of 2"), (dict(images="n_cams_1"), "exactly 2 images"), (dict(images="n_cams_3"), "exactly 2"),
(dict(instruction=""), "empty"), (dict(instruction=5), "string"),
(dict(instruction="Pick up the cup.<|im_end|>"), "special-token"),
(dict(instruction="<|vision_start|>x"), "special-token"),
(dict(instruction=" ".join(["pick up the red cup and place it on the plate"] * 40)), "exceeds the largest"),
(dict(num_steps=4), "num_steps must be 5"), (dict(num_steps=True), "num_steps"),
(dict(prompt_bucket=500), "prompt_bucket must be one of"), (dict(prompt_bucket=416, instruction=" ".join(["move"] * 30)), "does not fit"),
(dict(state=[0.0] * 19), "exactly 20"), (dict(state=[float("nan")] + [0.0] * 19), "non-finite"),
(dict(state="abc"), "20 numbers"),
(dict(seed=-1), "seed must be in"), (dict(seed=1.5), "integer"), (dict(seed=True), "integer"),
])
def test_refusals(model, sample, kw, msg):
req = dict(images=list(sample["images"]), instruction="pick up the cup")
req.update(kw)
if req["images"] == "n_cams_1":
req["images"] = sample["images"][:1]
elif req["images"] == "n_cams_3":
req["images"] = list(sample["images"]) * 2
req["images"] = req["images"][:3]
with pytest.raises(InputError, match=msg):
model(**req)
def test_camera_mapping(model, sample):
l, r = sample["images"]
a = model({"right_stereo": r, "left_stereo": l}, sample["instruction"]).actions
b = model([l, r], sample["instruction"]).actions
assert np.array_equal(a, b)
with pytest.raises(InputError, match="cameras must be exactly"):
model({"left": l, "right_stereo": r}, "x")
def test_close_is_idempotent():
need_tokenizer()
m = make_model()
with m:
pass
assert m.closed and m._m.released
m.close()
with pytest.raises(RuntimeError, match="closed"):
m(*[[np.zeros((384, 384, 3), np.uint8)] * 2], "pick up the cup")
def test_vlm_needed_files(tmp_path):
wm = {"visual.blocks.0.attn.qkv.weight": "a.safetensors", "model.embed_tokens.weight": "a.safetensors",
"model.layers.13.self_attn.k_proj.weight": "b.safetensors", "model.layers.14.mlp.up_proj.weight": "c.safetensors",
"model.layers.1.mlp.up_proj.weight": "a.safetensors", "lm_head.weight": "d.safetensors",
"model.norm.weight": "d.safetensors"}
(tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": wm}))
assert api.vlm_needed_files(tmp_path / "model.safetensors.index.json") == ["a.safetensors", "b.safetensors"]
from tt_rdt2.reference.config import DEFAULT_DIRS
real = Path(DEFAULT_DIRS["vlm"]) / "model.safetensors.index.json"
if real.is_file(): # the pinned RDT2-VQ: shards 1-2 only, and tt-model.yaml's allow_patterns cover them
need = api.vlm_needed_files(real)
assert need == ["model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"]
y = (BUNDLE / "tt-model.yaml").read_text() if (BUNDLE / "tt-model.yaml").is_file() else ""
if y:
for f in need + ["model.safetensors.index.json", "config.json"]:
assert f in y, f
def test_resolve_weights_local_and_hub(tmp_path, monkeypatch):
# local directories (env) win; a missing file is an error that names it
dirs = {}
for role, c in api.COMPONENTS.items():
d = tmp_path / role
d.mkdir()
for f in c["files"]:
(d / f).write_text("x")
dirs[role] = str(d)
(tmp_path / "vlm" / "model.safetensors.index.json").write_text(json.dumps(
{"weight_map": {"visual.x": "s1.safetensors", "lm_head.weight": "s9.safetensors"}}))
(tmp_path / "vlm" / "s1.safetensors").write_text("x")
env = {api.COMPONENTS[r]["env"]: p for r, p in dirs.items()}
w = api.resolve_weights(env=env, log_fn=lambda s: None)
assert w["vlm"]["source"] == "local" and w["vlm"]["files"][-1] == "s1.safetensors"
os.remove(tmp_path / "expert" / "pytorch_model.bin")
with pytest.raises(FileNotFoundError, match="pytorch_model.bin"):
api.resolve_weights(env=env, log_fn=lambda s: None)
# the hub path: pinned revisions; a network failure retries the cache with local_files_only=True
calls = []
snap = tmp_path / "snap"
snap.mkdir()
(snap / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {"visual.x": "s1.safetensors"}}))
def fake_dl(repo, filename, revision=None, local_files_only=False):
calls.append((repo, filename, revision, local_files_only))
if not local_files_only:
raise OSError("offline")
return str(snap / filename)
import huggingface_hub
monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_dl)
w = api.resolve_weights(env={}, log_fn=lambda s: None)
assert {r: v["revision"] for r, v in w.items()} == {r: c["revision"] for r, c in api.COMPONENTS.items()}
assert all(c[2] == api.COMPONENTS[r]["revision"] for r in api.COMPONENTS for c in calls
if c[0] == api.COMPONENTS[r]["repo"])
assert ("robotics-diffusion-transformer/RDT2-VQ", "s1.safetensors", api.VLM_REVISION, True) in calls
assert w["vlm"]["source"] == "hf-cache" and w["vlm"]["path"] == str(snap)