rdt2-base-p150 / code /tt_rdt2 /tests /test_reference_host.py
changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
14.5 kB
# SPDX-License-Identifier: Apache-2.0
"""Host tests of the RDT2-FM torch reference against the official golden (no device).
Gate: fp32 PCC >= 0.99999 per module and end to end (research/rdt2/SPEC.md section 9). Every module is fed the golden
input of that module (isolated), and the end-to-end test feeds only the raw request (cameras, instruction, state,
noise). Negative controls check that the comparisons can fail.
"""
from __future__ import annotations
import numpy as np
import pytest
import torch
from tt_rdt2.reference.config import CONFIG, IMAGE_TOKEN_ID
from tt_rdt2.reference.golden import compare, pcc
from tt_rdt2.reference.preprocess import (
concat_cameras, mrope_cos_sin, preprocess, smart_resize, vision_geometry, vision_window_index)
E = CONFIG.expert
GATE = 0.99999
# ------------------------------------------------------------------------------------------------ static geometry
def test_geometry_constants():
assert smart_resize(384, 768) == (392, 756)
grid = torch.tensor([[1, 28, 54]])
geom = vision_geometry(grid)
assert geom["n"] == 1512
idx, cu = vision_window_index(grid)
assert idx.numel() == 378 and sorted(idx.tolist()) == list(range(378))
sizes = np.diff(cu).tolist()
# 14 x 27 merged grid in 4x4 windows: 4 x 7 = 28 windows of 64 / 48 / 32 / 24 patches
assert len(sizes) == 28 and sorted(set(sizes)) == [24, 32, 48, 64] and sum(sizes) == 1512
assert geom["cu_full"] == [0, 1512]
assert geom["cos"].shape == (1512, 80)
assert torch.equal(geom["patch_perm"].sort().values, torch.arange(1512))
def test_prompt_layout(dirs):
import os
if not os.path.exists(os.path.join(dirs["processor"], "tokenizer.json")):
pytest.skip("processor files missing")
imgs = [np.zeros((384, 384, 3), np.uint8)] * 2
inp = preprocess(imgs, "Pick up the apple.")
ids = inp["input_ids"][0]
assert ids.numel() == 405 and int((ids == IMAGE_TOKEN_ID).sum()) == 378
first = int((ids == IMAGE_TOKEN_ID).nonzero()[0])
assert first == 15 # 14 system/user tokens + <|vision_start|>
assert ids[-1].item() == 151650 # <|quad_start|>
pos = inp["position_ids"][:, 0]
assert pos[:, :15].tolist() == [list(range(15))] * 3
assert pos[1, 15:15 + 378].max() == 15 + 13 and pos[2, 15:15 + 378].max() == 15 + 26
assert pos[:, 393].tolist() == [42, 42, 42] # text resumes at max(image position) + 1
cos, sin = mrope_cos_sin(inp["position_ids"])
assert cos.shape == (405, 128)
def test_camera_order_matters():
a = np.zeros((4, 4, 3), np.uint8)
b = np.full((4, 4, 3), 255, np.uint8)
img = concat_cameras([a, b])
assert img[:, :4].max() == 0 and img[:, 4:].min() == 255
# ------------------------------------------------------------------------------------------------ preprocessing
@pytest.mark.golden
def test_preprocess_matches_golden(golden):
for case in golden.cases:
g = golden.load(case)
inp = preprocess([g["in.left"].numpy(), g["in.right"].numpy()], golden.info(case)["prompt"])
assert torch.equal(inp["input_ids"], g["in.input_ids"]), case
assert torch.equal(inp["pixel_values"], g["in.pixel_values"]), case # bit-exact (same uint8 AA kernel)
assert torch.equal(inp["position_ids"], g["llm.position_ids"]), case
assert bool(g["cond.mask"].all()) # batch 1: the official mask is all True
# ------------------------------------------------------------------------------------------------ expert
@pytest.mark.golden
@pytest.mark.weights
def test_adaln_decomposition_equals_direct(expert):
from tt_rdt2.reference.expert import euler_timesteps
torch.manual_seed(0)
s = expert.state_cond(0.1 * torch.randn(20))
ts = euler_timesteps()
tparts = expert.precompute_time_part(ts)
sb, sf = expert.precompute_state_part(s)
for k, t in enumerate(ts):
direct_b, direct_f = expert.ada(torch.cat([expert.time_emb(t), s], 1))
for i in range(E.depth):
assert (tparts[k][0][i] + sb[i] - direct_b[i]).abs().max() < 1e-5
assert (tparts[k][1] + sf - direct_f).abs().max() < 1e-5
@pytest.mark.golden
@pytest.mark.weights
def test_expert_matches_golden(golden, expert):
from tt_rdt2.reference.expert import euler_timesteps, unnormalize_action
ts = euler_timesteps()
for case in golden.cases:
g = golden.load(case)
for k in range(E.num_steps):
assert torch.equal(ts[k], g[f"exp.step{k}.t"]), (case, k)
s_ad = expert.mlp3("state_adaptor.", g["in.state"].reshape(1, 1, -1))
assert pcc(s_ad, g["exp.state_emb"]) > GATE
taps = {}
a = expert.sample(g["in.noise"], g["in.state"], g["cond.k"], g["cond.v"], g["cond.mask"], taps)
for k in range(E.num_steps):
m = compare(taps[f"exp.step{k}.velocity"], g[f"exp.step{k}.velocity"])
assert m["pcc"] > GATE, (case, k, m)
m = compare(a, g["exp.nsample"])
assert m["pcc"] > GATE and m["max_abs"] < 1e-4, (case, m)
act = unnormalize_action(a, expert.norm)[0]
m = compare(act, g["out.action"])
assert m["pcc"] > GATE and m["max_abs"] < 1e-4, (case, m)
@pytest.mark.golden
@pytest.mark.weights
def test_expert_negative_controls(golden, expert):
"""The expert comparison must fail for wrong conditioning / timesteps / noise."""
from tt_rdt2.reference.expert import euler_timesteps
g = golden.load("synth_grad__p0__s0")
other = golden.load("synth_noise__p0__s0") # same prompt (same L), different images
ref = g["exp.nsample"]
bad_kv = expert.sample(g["in.noise"], g["in.state"], other["cond.k"], other["cond.v"], g["cond.mask"])
assert pcc(bad_kv, ref) < 0.9999
bad_noise = expert.sample(torch.randn(1, 24, 20, generator=torch.Generator().manual_seed(99)), g["in.state"],
g["cond.k"], g["cond.v"], g["cond.mask"])
assert pcc(bad_noise, ref) < 0.9999
# K without the RoPE rotation is NOT equivalent (the expert attends to post-mRoPE keys)
taps = {}
K = g["cond.k"].clone()
K[..., :64], K[..., 64:] = K[..., 64:].clone(), K[..., :64].clone() # scramble the rotary halves
bad_k = expert.sample(g["in.noise"], g["in.state"], K, g["cond.v"], g["cond.mask"], taps)
assert pcc(bad_k, ref) < 0.99999
# masking all-but-the-image keys changes the answer: the text tokens matter
m = g["cond.mask"].clone()
m[:, 393:] = False
assert pcc(expert.sample(g["in.noise"], g["in.state"], g["cond.k"], g["cond.v"], m), ref) < 0.99999
# a shifted time grid (t = 0.2 .. 1.0) is detected
ts = euler_timesteps()
shifted = [t + 0.2 for t in ts]
tparts = expert.precompute_time_part(shifted)
assert pcc(tparts[0][1], expert.precompute_time_part(ts)[0][1]) < 0.9999
# ------------------------------------------------------------------------------------------------ VLM (slow)
@pytest.mark.golden
@pytest.mark.weights
@pytest.mark.slow
def test_vision_and_prefix_match_golden(golden, reference):
case = golden.cases[0] # the case with every ViT block / LM layer tapped
g = golden.load(case)
taps = {}
img = reference.vision.forward(g["in.pixel_values"], vision_geometry(g["in.image_grid_thw"]), taps)
for k in [k for k in g if k.startswith("vis.block")] + ["vis.patch_embed", "vis.merger_windowed"]:
m = compare(taps[k], g[k])
assert m["pcc"] > GATE, (k, m)
assert compare(img, g["vis.image_embeds"])["pcc"] > GATE
cos, sin = mrope_cos_sin(g["llm.position_ids"])
x = reference.lm.embed(g["in.input_ids"], g["vis.image_embeds"])
assert compare(x, g["llm.inputs_embeds"])["max_abs"] == 0.0
taps = {}
K, V = reference.lm.forward(g["llm.inputs_embeds"], cos, sin, taps)
lk = [k for k in g if k.startswith("llm.layer") and int(k[9:11]) < 13] # layer 13: K/V only (not needed)
assert len(lk) == 13
for k in lk:
assert compare(taps[k], g[k])["pcc"] > GATE, k
for li in range(14):
assert compare(K[li], g["cond.k"][li])["pcc"] > GATE, li
assert compare(V[li], g["cond.v"][li])["pcc"] > GATE, li
@pytest.mark.golden
@pytest.mark.weights
@pytest.mark.slow
def test_end_to_end_matches_golden(golden, reference):
for case in [c for c in golden.cases if c.endswith("__s0")]:
g = golden.load(case)
out = reference([g["in.left"].numpy(), g["in.right"].numpy()], golden.info(case)["prompt"],
g["in.state"].numpy(), g["in.noise"])
for name, got, ref in (("K", out["K"], g["cond.k"]), ("V", out["V"], g["cond.v"]),
("nsample", out["nsample"], g["exp.nsample"]), ("action", out["action"], g["out.action"])):
m = compare(got, ref)
assert m["pcc"] > GATE, (case, name, m)
# ------------------------------------------------------------------------------------------------ bf16 noise floor
@pytest.mark.golden
def test_official_bf16_vs_fp32_floor(golden, golden_bf16):
"""Documents the official bf16-vs-fp32 drift (the floor TT gates are judged against); only sanity-gated."""
for case in golden_bf16.cases:
b, f = golden_bf16.load(case), golden.load(case)
assert torch.equal(b["in.input_ids"], f["in.input_ids"])
m = compare(b["out.action"], f["out.action"])
assert m["pcc"] > 0.9, (case, m)
# ------------------------------------------------------------------------------------------------ host folds (port)
@pytest.mark.golden
@pytest.mark.weights
def test_fold_patch_embed_and_window_order(golden, dirs):
import torch.nn.functional as F
from tt_rdt2.reference.host_fold import fold_patch_embed, window_order_pixels
from tt_rdt2.reference.weights import load_vlm_index
conv = load_vlm_index().get("visual.patch_embed.proj.weight").float()
w_fold, b_fold = fold_patch_embed(conv)
for case in golden.cases[:1] + [c for c in golden.cases if c.startswith("real")]:
g = golden.load(case)
raw, grid = window_order_pixels([g["in.left"].numpy(), g["in.right"].numpy()])
assert raw.shape == (1512, 588) and float(raw.max()) <= 255 and torch.equal(raw, raw.round()) # exact in bf16
got = F.linear(raw, w_fold, b_fold)
ref = F.linear(g["in.pixel_values"], conv.reshape(1280, -1))[vision_geometry(grid)["patch_perm"]]
m = compare(got, ref)
assert m["pcc"] > GATE and m["max_abs"] < 1e-3 * m["ref_absmax"], m
@pytest.mark.weights
def test_deinterleave_wkv(expert):
from tt_rdt2.reference.host_fold import deinterleave_wkv
w = expert.w["model.blocks.3.attn.wkv.weight"]
wk, wv = deinterleave_wkv(w)
x = torch.randn(28, 1024, generator=torch.Generator().manual_seed(1))
k, v = torch.nn.functional.linear(x, w).view(28, 4, 128, 2).unbind(-1)
assert torch.equal(k.reshape(28, -1), torch.nn.functional.linear(x, wk))
assert torch.equal(v.reshape(28, -1), torch.nn.functional.linear(x, wv))
@pytest.mark.golden
def test_lm_scatter_index(golden):
from tt_rdt2.reference.host_fold import lm_input_scatter_index
g = golden.load(golden.cases[0])
idx = lm_input_scatter_index(g["in.image_grid_thw"], 15)
x = torch.zeros_like(g["llm.inputs_embeds"][0])
x[idx] = g["vis.merger_windowed"]
assert torch.equal(x[15:393], g["vis.image_embeds"])
assert torch.equal(x[15:393], g["llm.inputs_embeds"][0, 15:393])
@pytest.mark.golden
def test_shipped_sample_is_the_golden_real_case(golden):
import os
p = os.path.join(os.path.dirname(os.path.dirname(__file__)), "samples", "ur5e_bimanual_example_0.npz")
s = np.load(p, allow_pickle=False)
g = golden.load("real_ur5e__s0")
assert np.array_equal(s["left_stereo"], g["in.left"].numpy())
assert np.array_equal(s["right_stereo"], g["in.right"].numpy())
assert str(s["instruction"]) == golden.info("real_ur5e__s0")["prompt"]
@pytest.mark.golden
@pytest.mark.weights
@pytest.mark.slow
def test_right_padding_bucket_equivalence(golden, reference):
"""Static-shape plan proof: right-padding the prompt to a bucket leaves the valid K/V and the action unchanged
(causal prefix + key mask in the expert); unmasked pads must change the action (negative control)."""
from tt_rdt2.reference.preprocess import bucket_inputs, choose_bucket
case = "synth_grad__p0__s0"
g = golden.load(case)
inp = preprocess([g["in.left"].numpy(), g["in.right"].numpy()], golden.info(case)["prompt"])
L = inp["input_ids"].shape[1]
assert choose_bucket(L) == 416
b = bucket_inputs(inp, 416)
x = reference.lm.embed(b["input_ids"], g["vis.image_embeds"])
cos, sin = mrope_cos_sin(b["position_ids"])
K, V = reference.lm.forward(x, cos, sin)
assert K.shape[-2] == 416
for name, got, ref in (("K", K[..., :L, :], g["cond.k"]), ("V", V[..., :L, :], g["cond.v"])):
m = compare(got, ref)
assert m["pcc"] > GATE and m["max_abs"] < 1e-3, (name, m)
mask = b["attention_mask"].to(torch.bool)
a = reference.expert.sample(g["in.noise"], g["in.state"], K, V, mask)
m = compare(a, g["exp.nsample"])
assert m["pcc"] > GATE and m["max_abs"] < 1e-4, m
a_bad = reference.expert.sample(g["in.noise"], g["in.state"], K, V, torch.ones_like(mask))
m_bad = compare(a_bad, g["exp.nsample"])
assert m_bad["max_abs"] > 100 * max(m["max_abs"], 1e-6), (m_bad, m)
@pytest.mark.golden
@pytest.mark.weights
def test_expert_matches_ext_golden(golden_ext, expert):
"""Ext cases (large state, 448 / 512-bucket prompts): the fp32 reference expert on the official K/V reproduces the
official nsample, and the large state moves the official action (the state VALUE is load-bearing here)."""
for case in golden_ext.cases:
g = golden_ext.load(case)
n = g["cond.k"].shape[3]
ns = expert.sample(g["in.noise"], g["in.state"], g["cond.k"], g["cond.v"], torch.ones(1, n, dtype=torch.bool))
assert pcc(ns, g["exp.nsample"]) >= 0.99999, case
big, zero = golden_ext.load("long_bigstate"), golden_ext.load("long_zerostate")
assert pcc(big["exp.nsample"], zero["exp.nsample"]) < 0.99
@pytest.mark.golden
def test_preprocess_ext_ids(golden_ext):
"""Our tokenizer path equals the official input_ids on the ext prompts (436 and 492 tokens)."""
from tt_rdt2.reference.preprocess import tokenize
for case in golden_ext.cases:
g = golden_ext.load(case)
assert torch.equal(tokenize(golden_ext.info(case)["prompt"], 378), g["in.input_ids"]), case