File size: 12,676 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
# SPDX-License-Identifier: Apache-2.0
"""Host tests of the Python API (no device; fake ttnn in the workspace): request validation and refusals, the
experimental-engine gate, weight resolution, and request decoding == the reference pre-processing the device gates
use (bit for bit, against the official golden where it is present)."""
from __future__ import annotations

import io
import json
import os
import subprocess
import sys
from pathlib import Path

import numpy as np
import pytest
import torch

from tt_rdt2 import api
from tt_rdt2.api import InputError
from tt_rdt2.tests.stubs import make_model, need_tokenizer, sample_reference

BUNDLE = Path(__file__).resolve().parents[3]


def _png(a: np.ndarray) -> bytes:
    from PIL import Image

    b = io.BytesIO()
    Image.fromarray(a).save(b, format="PNG")
    return b.getvalue()


@pytest.fixture(scope="module")
def model():
    need_tokenizer()
    return make_model()


@pytest.fixture(scope="module")
def sample():
    return api.load_sample()


def test_import_has_no_side_effects():
    code = ("import sys, tt_rdt2; assert tt_rdt2.__version__; "
            "bad = [m for m in ('torch', 'ttnn', 'numpy', 'huggingface_hub') if m in sys.modules]; "
            "assert not bad, bad; tt_rdt2.RDT2FM; assert 'torch' not in sys.modules or True")
    r = subprocess.run([sys.executable, "-c", code], cwd=str(BUNDLE / "code"), capture_output=True, text=True,
                       env=dict(os.environ, PYTHONPATH=str(BUNDLE / "code")))
    assert r.returncode == 0, r.stderr[-800:]


def test_experimental_prefix_mk_is_refused(monkeypatch):
    from tt_rdt2.tt.model import RDT2FMTT, check_engine_allowed, resolve_engine

    monkeypatch.delenv("RDT2_EXPERIMENTAL_PREFIX_MK", raising=False)
    assert resolve_engine(None if "RDT2_ENGINE" not in os.environ else "") == {"vision": "mk", "prefix": "fused",
                                                                                "expert": "mk"}
    check_engine_allowed(resolve_engine(""))  # the defaults pass
    with pytest.raises(ValueError, match="hangs the device"):
        check_engine_allowed(resolve_engine("prefix=mk"))
    with pytest.raises(ValueError, match="RDT2_EXPERIMENTAL_PREFIX_MK=1"):
        RDT2FMTT(None, engine="prefix=mk")  # refused before anything touches the device
    monkeypatch.setenv("RDT2_ENGINE", "prefix=mk")
    with pytest.raises(ValueError, match="experimental"):
        RDT2FMTT(None)

    def no_weights(*a, **k):
        raise AssertionError("weights resolved before the engine check")

    monkeypatch.setattr(api, "resolve_weights", no_weights)
    with pytest.raises(ValueError, match="experimental"):
        api.RDT2FM.from_pretrained(engine="prefix=mk")
    check_engine_allowed(resolve_engine("prefix=mk"), env={"RDT2_EXPERIMENTAL_PREFIX_MK": "1"})  # explicit opt-in


def test_sample_reference_is_the_official_request(sample):
    ref = sample_reference()
    assert ref["request"]["instruction"] == sample["instruction"] and ref["request"]["seed"] == 0
    assert np.array_equal(np.asarray(ref["request"]["state"], np.float32), sample["state"])
    from PIL import Image

    for cam, a in zip(api.CAMERA_ORDER, sample["images"]):
        png = api.SAMPLE_PATH.parent / f"ur5e_bimanual_example_0_{cam}.png"
        assert np.array_equal(np.asarray(Image.open(png).convert("RGB")), a)
    assert np.asarray(ref["actions"]).shape == (24, 20) and set(ref["limits"]) == {"one_minus_pcc", "pos_m", "rot",
                                                                                     "grip_m"}
    sc, of = np.asarray(ref["normalizer"]["scale"]), np.asarray(ref["normalizer"]["offset"])
    assert np.allclose((np.asarray(ref["actions_normalized"]) - of) / sc, ref["actions"], atol=1e-5)


def test_seeded_noise_matches_golden_driver():
    n0 = api.seeded_noise(0)
    assert n0.shape == (1, 24, 20) and n0.dtype == torch.float32
    assert torch.equal(n0, torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0)))
    assert not torch.equal(n0, api.seeded_noise(1))


def test_request_decoding_equals_reference_preprocessing(model, sample, golden_or_none):
    """PNG over the API == the arrays the gates feed: same pixels, ids, mask, noise, state; and (golden present)
    the pixel rows equal the official processor's pixel_values and the ids the official input_ids, bit for bit."""
    from tt_rdt2.tt.vision import host_pixels, host_pixels_from_pixel_values

    tt = model._m
    out = model([_png(a) for a in sample["images"]], sample["instruction"])
    png_host = tt.calls[-1]
    model(sample["images"], sample["instruction"], state=sample["state"], seed=0)
    arr_host = tt.calls[-1]
    for k in ("pix", "ids", "mask", "noise", "state"):
        assert torch.equal(png_host[k], arr_host[k]), k
    assert int(png_host["n_tokens"]) == 409 and png_host["bucket"] == 416 == out.prompt_bucket
    assert torch.equal(png_host["pix"], host_pixels(sample["images"]))
    if golden_or_none is not None:
        d = golden_or_none.load("real_ur5e__s0", with_vlm=False)
        assert torch.equal(png_host["pix"], host_pixels_from_pixel_values(d["in.pixel_values"]))
        ids = d["in.input_ids"].reshape(-1)
        got = png_host["ids"].reshape(-1)[:ids.numel()].to(torch.int64)
        img = ids == 151655
        assert torch.equal(got[~img], ids[~img])
        assert torch.equal(png_host["noise"][0, 0, :24, :20], d["in.noise"].float().reshape(24, 20))


@pytest.fixture(scope="module")
def golden_or_none():
    from tt_rdt2.reference.config import DEFAULT_DIRS
    from tt_rdt2.reference.golden import GoldenSet

    if not os.path.exists(os.path.join(DEFAULT_DIRS["golden"], "fp32", "manifest.json")):
        return None
    return GoldenSet("fp32")


def test_outputs_and_timing(model, sample):
    out = model(**sample)
    d = out.to_dict()
    assert out.actions.shape == (24, 20) and out.actions.dtype == np.float32
    assert d["shape"] == [24, 20] and d["normalized"] is False and d["num_steps"] == 5 and d["seed"] == 0
    assert set(d["timing_ms"]) == {"preprocess", "inference", "total"}
    assert d["chunk"] == {"horizon": 24, "action_dim": 20, "rate_hz": 30}
    assert np.allclose(d["actions"], sample_reference()["actions"])
    json.dumps(d)


@pytest.mark.parametrize("words,bucket", [(3, 416), (30, 448), (80, 512)])
def test_bucket_selection(model, sample, words, bucket):
    instr = " ".join(["move"] * words)
    n = model.tokens_for(instr)
    out = model(sample["images"], instr)
    assert out.prompt_bucket == bucket == min(b for b in api.PROMPT_BUCKETS if b >= n) and out.num_tokens == n
    assert model(sample["images"], instr, prompt_bucket=512).prompt_bucket == 512


def test_geometry_accepted_and_refused(model, sample):
    model([np.zeros((380, 380, 3), np.uint8)] * 2, "pick up the cup")  # same 392 x 756 grid: resized
    for imgs, msg in [([np.zeros((400, 400, 3), np.uint8)] * 2, "unsupported image geometry"),
                      ([np.zeros((384, 384, 3), np.uint8), np.zeros((380, 384, 3), np.uint8)], "same height"),
                      ([np.zeros((384, 384, 3), np.float32)] * 2, "uint8"),
                      ([np.zeros((384, 384), np.uint8)] * 2, "HxWx3"),
                      ([b"not an image", b"x"], "cannot decode")]:
        with pytest.raises(InputError, match=msg):
            model(imgs, "pick up the cup")


@pytest.mark.parametrize("kw,msg", [
    (dict(images=1), "list of 2"), (dict(images="n_cams_1"), "exactly 2 images"), (dict(images="n_cams_3"), "exactly 2"),
    (dict(instruction=""), "empty"), (dict(instruction=5), "string"),
    (dict(instruction="Pick up the cup.<|im_end|>"), "special-token"),
    (dict(instruction="<|vision_start|>x"), "special-token"),
    (dict(instruction=" ".join(["pick up the red cup and place it on the plate"] * 40)), "exceeds the largest"),
    (dict(num_steps=4), "num_steps must be 5"), (dict(num_steps=True), "num_steps"),
    (dict(prompt_bucket=500), "prompt_bucket must be one of"), (dict(prompt_bucket=416, instruction=" ".join(["move"] * 30)), "does not fit"),
    (dict(state=[0.0] * 19), "exactly 20"), (dict(state=[float("nan")] + [0.0] * 19), "non-finite"),
    (dict(state="abc"), "20 numbers"),
    (dict(seed=-1), "seed must be in"), (dict(seed=1.5), "integer"), (dict(seed=True), "integer"),
])
def test_refusals(model, sample, kw, msg):
    req = dict(images=list(sample["images"]), instruction="pick up the cup")
    req.update(kw)
    if req["images"] == "n_cams_1":
        req["images"] = sample["images"][:1]
    elif req["images"] == "n_cams_3":
        req["images"] = list(sample["images"]) * 2
        req["images"] = req["images"][:3]
    with pytest.raises(InputError, match=msg):
        model(**req)


def test_camera_mapping(model, sample):
    l, r = sample["images"]
    a = model({"right_stereo": r, "left_stereo": l}, sample["instruction"]).actions
    b = model([l, r], sample["instruction"]).actions
    assert np.array_equal(a, b)
    with pytest.raises(InputError, match="cameras must be exactly"):
        model({"left": l, "right_stereo": r}, "x")


def test_close_is_idempotent():
    need_tokenizer()
    m = make_model()
    with m:
        pass
    assert m.closed and m._m.released
    m.close()
    with pytest.raises(RuntimeError, match="closed"):
        m(*[[np.zeros((384, 384, 3), np.uint8)] * 2], "pick up the cup")


def test_vlm_needed_files(tmp_path):
    wm = {"visual.blocks.0.attn.qkv.weight": "a.safetensors", "model.embed_tokens.weight": "a.safetensors",
          "model.layers.13.self_attn.k_proj.weight": "b.safetensors", "model.layers.14.mlp.up_proj.weight": "c.safetensors",
          "model.layers.1.mlp.up_proj.weight": "a.safetensors", "lm_head.weight": "d.safetensors",
          "model.norm.weight": "d.safetensors"}
    (tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": wm}))
    assert api.vlm_needed_files(tmp_path / "model.safetensors.index.json") == ["a.safetensors", "b.safetensors"]
    from tt_rdt2.reference.config import DEFAULT_DIRS

    real = Path(DEFAULT_DIRS["vlm"]) / "model.safetensors.index.json"
    if real.is_file():  # the pinned RDT2-VQ: shards 1-2 only, and tt-model.yaml's allow_patterns cover them
        need = api.vlm_needed_files(real)
        assert need == ["model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"]
        y = (BUNDLE / "tt-model.yaml").read_text() if (BUNDLE / "tt-model.yaml").is_file() else ""
        if y:
            for f in need + ["model.safetensors.index.json", "config.json"]:
                assert f in y, f


def test_resolve_weights_local_and_hub(tmp_path, monkeypatch):
    # local directories (env) win; a missing file is an error that names it
    dirs = {}
    for role, c in api.COMPONENTS.items():
        d = tmp_path / role
        d.mkdir()
        for f in c["files"]:
            (d / f).write_text("x")
        dirs[role] = str(d)
    (tmp_path / "vlm" / "model.safetensors.index.json").write_text(json.dumps(
        {"weight_map": {"visual.x": "s1.safetensors", "lm_head.weight": "s9.safetensors"}}))
    (tmp_path / "vlm" / "s1.safetensors").write_text("x")
    env = {api.COMPONENTS[r]["env"]: p for r, p in dirs.items()}
    w = api.resolve_weights(env=env, log_fn=lambda s: None)
    assert w["vlm"]["source"] == "local" and w["vlm"]["files"][-1] == "s1.safetensors"
    os.remove(tmp_path / "expert" / "pytorch_model.bin")
    with pytest.raises(FileNotFoundError, match="pytorch_model.bin"):
        api.resolve_weights(env=env, log_fn=lambda s: None)
    # the hub path: pinned revisions; a network failure retries the cache with local_files_only=True
    calls = []
    snap = tmp_path / "snap"
    snap.mkdir()
    (snap / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {"visual.x": "s1.safetensors"}}))

    def fake_dl(repo, filename, revision=None, local_files_only=False):
        calls.append((repo, filename, revision, local_files_only))
        if not local_files_only:
            raise OSError("offline")
        return str(snap / filename)

    import huggingface_hub

    monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_dl)
    w = api.resolve_weights(env={}, log_fn=lambda s: None)
    assert {r: v["revision"] for r, v in w.items()} == {r: c["revision"] for r, c in api.COMPONENTS.items()}
    assert all(c[2] == api.COMPONENTS[r]["revision"] for r in api.COMPONENTS for c in calls
               if c[0] == api.COMPONENTS[r]["repo"])
    assert ("robotics-diffusion-transformer/RDT2-VQ", "s1.safetensors", api.VLM_REVISION, True) in calls
    assert w["vlm"]["source"] == "hf-cache" and w["vlm"]["path"] == str(snap)