Download code/tt_rdt2/tests/test_reference_host.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 14.5 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/test_reference_host.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/tests/test_reference_host.py
-
curl -L -o test_reference_host.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tests/test_reference_host.py
14.5 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Host tests of the RDT2-FM torch reference against the official golden (no device). | |
| Gate: fp32 PCC >= 0.99999 per module and end to end (research/rdt2/SPEC.md section 9). Every module is fed the golden | |
| input of that module (isolated), and the end-to-end test feeds only the raw request (cameras, instruction, state, | |
| noise). Negative controls check that the comparisons can fail. | |
| """ | |
| from __future__ import annotations | |
| import numpy as np | |
| import pytest | |
| import torch | |
| from tt_rdt2.reference.config import CONFIG, IMAGE_TOKEN_ID | |
| from tt_rdt2.reference.golden import compare, pcc | |
| from tt_rdt2.reference.preprocess import ( | |
| concat_cameras, mrope_cos_sin, preprocess, smart_resize, vision_geometry, vision_window_index) | |
| E = CONFIG.expert | |
| GATE = 0.99999 | |
| # ------------------------------------------------------------------------------------------------ static geometry | |
| def test_geometry_constants(): | |
| assert smart_resize(384, 768) == (392, 756) | |
| grid = torch.tensor([[1, 28, 54]]) | |
| geom = vision_geometry(grid) | |
| assert geom["n"] == 1512 | |
| idx, cu = vision_window_index(grid) | |
| assert idx.numel() == 378 and sorted(idx.tolist()) == list(range(378)) | |
| sizes = np.diff(cu).tolist() | |
| # 14 x 27 merged grid in 4x4 windows: 4 x 7 = 28 windows of 64 / 48 / 32 / 24 patches | |
| assert len(sizes) == 28 and sorted(set(sizes)) == [24, 32, 48, 64] and sum(sizes) == 1512 | |
| assert geom["cu_full"] == [0, 1512] | |
| assert geom["cos"].shape == (1512, 80) | |
| assert torch.equal(geom["patch_perm"].sort().values, torch.arange(1512)) | |
| def test_prompt_layout(dirs): | |
| import os | |
| if not os.path.exists(os.path.join(dirs["processor"], "tokenizer.json")): | |
| pytest.skip("processor files missing") | |
| imgs = [np.zeros((384, 384, 3), np.uint8)] * 2 | |
| inp = preprocess(imgs, "Pick up the apple.") | |
| ids = inp["input_ids"][0] | |
| assert ids.numel() == 405 and int((ids == IMAGE_TOKEN_ID).sum()) == 378 | |
| first = int((ids == IMAGE_TOKEN_ID).nonzero()[0]) | |
| assert first == 15 # 14 system/user tokens + <|vision_start|> | |
| assert ids[-1].item() == 151650 # <|quad_start|> | |
| pos = inp["position_ids"][:, 0] | |
| assert pos[:, :15].tolist() == [list(range(15))] * 3 | |
| assert pos[1, 15:15 + 378].max() == 15 + 13 and pos[2, 15:15 + 378].max() == 15 + 26 | |
| assert pos[:, 393].tolist() == [42, 42, 42] # text resumes at max(image position) + 1 | |
| cos, sin = mrope_cos_sin(inp["position_ids"]) | |
| assert cos.shape == (405, 128) | |
| def test_camera_order_matters(): | |
| a = np.zeros((4, 4, 3), np.uint8) | |
| b = np.full((4, 4, 3), 255, np.uint8) | |
| img = concat_cameras([a, b]) | |
| assert img[:, :4].max() == 0 and img[:, 4:].min() == 255 | |
| # ------------------------------------------------------------------------------------------------ preprocessing | |
| def test_preprocess_matches_golden(golden): | |
| for case in golden.cases: | |
| g = golden.load(case) | |
| inp = preprocess([g["in.left"].numpy(), g["in.right"].numpy()], golden.info(case)["prompt"]) | |
| assert torch.equal(inp["input_ids"], g["in.input_ids"]), case | |
| assert torch.equal(inp["pixel_values"], g["in.pixel_values"]), case # bit-exact (same uint8 AA kernel) | |
| assert torch.equal(inp["position_ids"], g["llm.position_ids"]), case | |
| assert bool(g["cond.mask"].all()) # batch 1: the official mask is all True | |
| # ------------------------------------------------------------------------------------------------ expert | |
| def test_adaln_decomposition_equals_direct(expert): | |
| from tt_rdt2.reference.expert import euler_timesteps | |
| torch.manual_seed(0) | |
| s = expert.state_cond(0.1 * torch.randn(20)) | |
| ts = euler_timesteps() | |
| tparts = expert.precompute_time_part(ts) | |
| sb, sf = expert.precompute_state_part(s) | |
| for k, t in enumerate(ts): | |
| direct_b, direct_f = expert.ada(torch.cat([expert.time_emb(t), s], 1)) | |
| for i in range(E.depth): | |
| assert (tparts[k][0][i] + sb[i] - direct_b[i]).abs().max() < 1e-5 | |
| assert (tparts[k][1] + sf - direct_f).abs().max() < 1e-5 | |
| def test_expert_matches_golden(golden, expert): | |
| from tt_rdt2.reference.expert import euler_timesteps, unnormalize_action | |
| ts = euler_timesteps() | |
| for case in golden.cases: | |
| g = golden.load(case) | |
| for k in range(E.num_steps): | |
| assert torch.equal(ts[k], g[f"exp.step{k}.t"]), (case, k) | |
| s_ad = expert.mlp3("state_adaptor.", g["in.state"].reshape(1, 1, -1)) | |
| assert pcc(s_ad, g["exp.state_emb"]) > GATE | |
| taps = {} | |
| a = expert.sample(g["in.noise"], g["in.state"], g["cond.k"], g["cond.v"], g["cond.mask"], taps) | |
| for k in range(E.num_steps): | |
| m = compare(taps[f"exp.step{k}.velocity"], g[f"exp.step{k}.velocity"]) | |
| assert m["pcc"] > GATE, (case, k, m) | |
| m = compare(a, g["exp.nsample"]) | |
| assert m["pcc"] > GATE and m["max_abs"] < 1e-4, (case, m) | |
| act = unnormalize_action(a, expert.norm)[0] | |
| m = compare(act, g["out.action"]) | |
| assert m["pcc"] > GATE and m["max_abs"] < 1e-4, (case, m) | |
| def test_expert_negative_controls(golden, expert): | |
| """The expert comparison must fail for wrong conditioning / timesteps / noise.""" | |
| from tt_rdt2.reference.expert import euler_timesteps | |
| g = golden.load("synth_grad__p0__s0") | |
| other = golden.load("synth_noise__p0__s0") # same prompt (same L), different images | |
| ref = g["exp.nsample"] | |
| bad_kv = expert.sample(g["in.noise"], g["in.state"], other["cond.k"], other["cond.v"], g["cond.mask"]) | |
| assert pcc(bad_kv, ref) < 0.9999 | |
| bad_noise = expert.sample(torch.randn(1, 24, 20, generator=torch.Generator().manual_seed(99)), g["in.state"], | |
| g["cond.k"], g["cond.v"], g["cond.mask"]) | |
| assert pcc(bad_noise, ref) < 0.9999 | |
| # K without the RoPE rotation is NOT equivalent (the expert attends to post-mRoPE keys) | |
| taps = {} | |
| K = g["cond.k"].clone() | |
| K[..., :64], K[..., 64:] = K[..., 64:].clone(), K[..., :64].clone() # scramble the rotary halves | |
| bad_k = expert.sample(g["in.noise"], g["in.state"], K, g["cond.v"], g["cond.mask"], taps) | |
| assert pcc(bad_k, ref) < 0.99999 | |
| # masking all-but-the-image keys changes the answer: the text tokens matter | |
| m = g["cond.mask"].clone() | |
| m[:, 393:] = False | |
| assert pcc(expert.sample(g["in.noise"], g["in.state"], g["cond.k"], g["cond.v"], m), ref) < 0.99999 | |
| # a shifted time grid (t = 0.2 .. 1.0) is detected | |
| ts = euler_timesteps() | |
| shifted = [t + 0.2 for t in ts] | |
| tparts = expert.precompute_time_part(shifted) | |
| assert pcc(tparts[0][1], expert.precompute_time_part(ts)[0][1]) < 0.9999 | |
| # ------------------------------------------------------------------------------------------------ VLM (slow) | |
| def test_vision_and_prefix_match_golden(golden, reference): | |
| case = golden.cases[0] # the case with every ViT block / LM layer tapped | |
| g = golden.load(case) | |
| taps = {} | |
| img = reference.vision.forward(g["in.pixel_values"], vision_geometry(g["in.image_grid_thw"]), taps) | |
| for k in [k for k in g if k.startswith("vis.block")] + ["vis.patch_embed", "vis.merger_windowed"]: | |
| m = compare(taps[k], g[k]) | |
| assert m["pcc"] > GATE, (k, m) | |
| assert compare(img, g["vis.image_embeds"])["pcc"] > GATE | |
| cos, sin = mrope_cos_sin(g["llm.position_ids"]) | |
| x = reference.lm.embed(g["in.input_ids"], g["vis.image_embeds"]) | |
| assert compare(x, g["llm.inputs_embeds"])["max_abs"] == 0.0 | |
| taps = {} | |
| K, V = reference.lm.forward(g["llm.inputs_embeds"], cos, sin, taps) | |
| lk = [k for k in g if k.startswith("llm.layer") and int(k[9:11]) < 13] # layer 13: K/V only (not needed) | |
| assert len(lk) == 13 | |
| for k in lk: | |
| assert compare(taps[k], g[k])["pcc"] > GATE, k | |
| for li in range(14): | |
| assert compare(K[li], g["cond.k"][li])["pcc"] > GATE, li | |
| assert compare(V[li], g["cond.v"][li])["pcc"] > GATE, li | |
| def test_end_to_end_matches_golden(golden, reference): | |
| for case in [c for c in golden.cases if c.endswith("__s0")]: | |
| g = golden.load(case) | |
| out = reference([g["in.left"].numpy(), g["in.right"].numpy()], golden.info(case)["prompt"], | |
| g["in.state"].numpy(), g["in.noise"]) | |
| for name, got, ref in (("K", out["K"], g["cond.k"]), ("V", out["V"], g["cond.v"]), | |
| ("nsample", out["nsample"], g["exp.nsample"]), ("action", out["action"], g["out.action"])): | |
| m = compare(got, ref) | |
| assert m["pcc"] > GATE, (case, name, m) | |
| # ------------------------------------------------------------------------------------------------ bf16 noise floor | |
| def test_official_bf16_vs_fp32_floor(golden, golden_bf16): | |
| """Documents the official bf16-vs-fp32 drift (the floor TT gates are judged against); only sanity-gated.""" | |
| for case in golden_bf16.cases: | |
| b, f = golden_bf16.load(case), golden.load(case) | |
| assert torch.equal(b["in.input_ids"], f["in.input_ids"]) | |
| m = compare(b["out.action"], f["out.action"]) | |
| assert m["pcc"] > 0.9, (case, m) | |
| # ------------------------------------------------------------------------------------------------ host folds (port) | |
| def test_fold_patch_embed_and_window_order(golden, dirs): | |
| import torch.nn.functional as F | |
| from tt_rdt2.reference.host_fold import fold_patch_embed, window_order_pixels | |
| from tt_rdt2.reference.weights import load_vlm_index | |
| conv = load_vlm_index().get("visual.patch_embed.proj.weight").float() | |
| w_fold, b_fold = fold_patch_embed(conv) | |
| for case in golden.cases[:1] + [c for c in golden.cases if c.startswith("real")]: | |
| g = golden.load(case) | |
| raw, grid = window_order_pixels([g["in.left"].numpy(), g["in.right"].numpy()]) | |
| assert raw.shape == (1512, 588) and float(raw.max()) <= 255 and torch.equal(raw, raw.round()) # exact in bf16 | |
| got = F.linear(raw, w_fold, b_fold) | |
| ref = F.linear(g["in.pixel_values"], conv.reshape(1280, -1))[vision_geometry(grid)["patch_perm"]] | |
| m = compare(got, ref) | |
| assert m["pcc"] > GATE and m["max_abs"] < 1e-3 * m["ref_absmax"], m | |
| def test_deinterleave_wkv(expert): | |
| from tt_rdt2.reference.host_fold import deinterleave_wkv | |
| w = expert.w["model.blocks.3.attn.wkv.weight"] | |
| wk, wv = deinterleave_wkv(w) | |
| x = torch.randn(28, 1024, generator=torch.Generator().manual_seed(1)) | |
| k, v = torch.nn.functional.linear(x, w).view(28, 4, 128, 2).unbind(-1) | |
| assert torch.equal(k.reshape(28, -1), torch.nn.functional.linear(x, wk)) | |
| assert torch.equal(v.reshape(28, -1), torch.nn.functional.linear(x, wv)) | |
| def test_lm_scatter_index(golden): | |
| from tt_rdt2.reference.host_fold import lm_input_scatter_index | |
| g = golden.load(golden.cases[0]) | |
| idx = lm_input_scatter_index(g["in.image_grid_thw"], 15) | |
| x = torch.zeros_like(g["llm.inputs_embeds"][0]) | |
| x[idx] = g["vis.merger_windowed"] | |
| assert torch.equal(x[15:393], g["vis.image_embeds"]) | |
| assert torch.equal(x[15:393], g["llm.inputs_embeds"][0, 15:393]) | |
| def test_shipped_sample_is_the_golden_real_case(golden): | |
| import os | |
| p = os.path.join(os.path.dirname(os.path.dirname(__file__)), "samples", "ur5e_bimanual_example_0.npz") | |
| s = np.load(p, allow_pickle=False) | |
| g = golden.load("real_ur5e__s0") | |
| assert np.array_equal(s["left_stereo"], g["in.left"].numpy()) | |
| assert np.array_equal(s["right_stereo"], g["in.right"].numpy()) | |
| assert str(s["instruction"]) == golden.info("real_ur5e__s0")["prompt"] | |
| def test_right_padding_bucket_equivalence(golden, reference): | |
| """Static-shape plan proof: right-padding the prompt to a bucket leaves the valid K/V and the action unchanged | |
| (causal prefix + key mask in the expert); unmasked pads must change the action (negative control).""" | |
| from tt_rdt2.reference.preprocess import bucket_inputs, choose_bucket | |
| case = "synth_grad__p0__s0" | |
| g = golden.load(case) | |
| inp = preprocess([g["in.left"].numpy(), g["in.right"].numpy()], golden.info(case)["prompt"]) | |
| L = inp["input_ids"].shape[1] | |
| assert choose_bucket(L) == 416 | |
| b = bucket_inputs(inp, 416) | |
| x = reference.lm.embed(b["input_ids"], g["vis.image_embeds"]) | |
| cos, sin = mrope_cos_sin(b["position_ids"]) | |
| K, V = reference.lm.forward(x, cos, sin) | |
| assert K.shape[-2] == 416 | |
| for name, got, ref in (("K", K[..., :L, :], g["cond.k"]), ("V", V[..., :L, :], g["cond.v"])): | |
| m = compare(got, ref) | |
| assert m["pcc"] > GATE and m["max_abs"] < 1e-3, (name, m) | |
| mask = b["attention_mask"].to(torch.bool) | |
| a = reference.expert.sample(g["in.noise"], g["in.state"], K, V, mask) | |
| m = compare(a, g["exp.nsample"]) | |
| assert m["pcc"] > GATE and m["max_abs"] < 1e-4, m | |
| a_bad = reference.expert.sample(g["in.noise"], g["in.state"], K, V, torch.ones_like(mask)) | |
| m_bad = compare(a_bad, g["exp.nsample"]) | |
| assert m_bad["max_abs"] > 100 * max(m["max_abs"], 1e-6), (m_bad, m) | |
| def test_expert_matches_ext_golden(golden_ext, expert): | |
| """Ext cases (large state, 448 / 512-bucket prompts): the fp32 reference expert on the official K/V reproduces the | |
| official nsample, and the large state moves the official action (the state VALUE is load-bearing here).""" | |
| for case in golden_ext.cases: | |
| g = golden_ext.load(case) | |
| n = g["cond.k"].shape[3] | |
| ns = expert.sample(g["in.noise"], g["in.state"], g["cond.k"], g["cond.v"], torch.ones(1, n, dtype=torch.bool)) | |
| assert pcc(ns, g["exp.nsample"]) >= 0.99999, case | |
| big, zero = golden_ext.load("long_bigstate"), golden_ext.load("long_zerostate") | |
| assert pcc(big["exp.nsample"], zero["exp.nsample"]) < 0.99 | |
| def test_preprocess_ext_ids(golden_ext): | |
| """Our tokenizer path equals the official input_ids on the ext prompts (436 and 492 tokens).""" | |
| from tt_rdt2.reference.preprocess import tokenize | |
| for case in golden_ext.cases: | |
| g = golden_ext.load(case) | |
| assert torch.equal(tokenize(golden_ext.info(case)["prompt"], 378), g["in.input_ids"]), case | |