File size: 11,660 Bytes
b0b832a
 
 
 
 
 
1f59843
 
b0b832a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
799f51f
 
b0b832a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1f59843
 
b0b832a
 
 
 
 
 
 
1f59843
 
b0b832a
 
1f59843
b0b832a
 
1f59843
 
 
b0b832a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
"""CPU-only checks before any GPU time (no training, no VLM weights loaded).

Usage: <psi0 python> verify_cpu.py [--task b6|basket|a3|a7|a18] [--samples 3] [--no-header] [--no-data]
  1. hb_args -> tyro -> DynamicLaunchConfig (same parse as scripts/train.py) and its HB validators;
     dims / keys / cameras / body dims must match the task's robot layout (G1 A76/S44/29, H2 A75/S78/31)
  2. step plans (20 epochs, save at 10) for every prepared task
  3. synthetic repack/field/model transform: no proprio delay, joint bias redrawn per sample + noise within 3 sigma
     (aligned with GR00T A20, 2026-11-12), untouched non-joint dims, no-aug output identical to the native
     Psi0ModelTransform, HB v2 aug active
  4. real view through LeRobotDatasetWrapper (+ timestamp sync check) -> shapes / ranges / cameras / episode_index
  5. action header built by the native Psi0Model ctor (dummy VLM) + re-dimensioned load of the real
     action_header.safetensors: newly initialized keys must equal hb_patches.expected_new(A, S, 30);
     CPU forward -> (1, 30, A)
"""
import argparse
import json
import math
import os
import sys
import tempfile
import time
from pathlib import Path
from types import SimpleNamespace

for _k in ("RANK", "WORLD_SIZE", "LOCAL_RANK", "MASTER_ADDR", "MASTER_PORT", "LOCAL_WORLD_SIZE", "GROUP_RANK"):
    os.environ.pop(_k, None)  # DLC node shells export these; psi overwatch would think it is distributed
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ.setdefault("HF_HUB_OFFLINE", "1")
H = Path(__file__).resolve().parent
G = Path(os.environ.get("G", H.parent))
PSI = G / "HumanWAM/third_party/Psi0"
sys.path[:0] = [str(H), str(PSI / "src")]
os.environ.setdefault("HB_AUG_DIR", str(G / "hb_groot"))
os.environ.setdefault("HF_DATASETS_CACHE", f"/tmp/hb_psi0_hfds_{os.getuid()}")

import numpy as np  # noqa: E402
import torch  # noqa: E402
from dotenv import load_dotenv  # noqa: E402

load_dotenv(PSI / ".env")
FAIL = []
EXPECT = {"g1": (76, 44, 29, "observation.images.ego_view"), "h2": (75, 78, 31, "observation.images.head"),
          "h2p": (82, 78, 31, "observation.images.head")}  # 2026-11-14: a14 v2 action82 (+ planner)


def check(cond, msg):
    print(("  ok   " if cond else "  FAIL ") + msg, flush=True)
    if not cond:
        FAIL.append(msg)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--task", default="b6")
    ap.add_argument("--samples", type=int, default=3)
    ap.add_argument("--no-header", action="store_true")
    ap.add_argument("--no-data", action="store_true")
    a = ap.parse_args()

    import tyro
    import hb_args
    import hb_patches
    import hb_psi0_config as C
    from psi.config import transform as pt
    sys.modules["psi.config.train.hb_psi0_config"] = C

    print("[1] config")
    check(C.A.VERSION == C.HB_AUG_VERSION, f"hb_aug_v2 imported from {C._AUG_DIR} ({C.A.VERSION})")
    r = hb_args.receipt(a.task)
    robot = r.get("robot")
    check(robot in EXPECT, f"receipt robot '{robot}' (rebuild old views with prepare_data.py --force)")
    if robot not in EXPECT:
        return 1
    AD, SD, NB, CAM0 = EXPECT[robot]
    SK, AK = r["state_key"], r["action_key"]
    out = tempfile.mkdtemp(prefix="hb_psi0_verify_")
    args = hb_args.build_args(a.task, 16, 100, 50, 3, out, "verify", 2, resume=False)
    cfg = tyro.cli(C.DynamicLaunchConfig, config=(tyro.conf.ConsolidateSubcommandArgs,), args=args)
    m, rp, fd = cfg.model, cfg.data.transform.repack, cfg.data.transform.field
    check((m.action_dim, m.odim, m.action_chunk_size) == (AD, SD, 30), f"model A/S/H = {m.action_dim}/{m.odim}/{m.action_chunk_size}")
    check(len(fd.action_min) == AD and len(fd.state_min) == SD, f"stats widths {AD}/{SD} (no padding)")
    check(rp.hb_body_dims == NB, f"joint-perturbed dims {rp.hb_body_dims}: {r['perturbed_joint_names']}")
    check(rp.image_keys == r["image_keys"] and rp.image_keys[0] == CAM0, f"cameras {rp.image_keys} (cam0 = {CAM0})")
    mt = cfg.data.transform.model
    check(not mt.img_aug and not mt.view_aug and mt.hb_aug, "native img/view aug off, HB v2 aug on")
    check(fd.state_noise_std == 0 and rp.state_temporal_jitter == 0, "native state noise/jitter off")
    check(m.num_blocks == 12 and m.qk_norm == "rms_norm" and m.state_null_token and m.combined_temb,
          "native SONIC v1.1 header recipe flags")
    s32 = torch.tensor([1.0 / AD] * AD, dtype=torch.float32).sum().item()
    print(f"  info fp32 sum(1/{AD} x {AD}) on CPU = {s32!r} (hb_patches handles != 1.0 on the GPU)")

    print("[2] step plans (global batch = per_gpu x 32)")
    for t in hb_args.TASKS:
        try:
            fr = hb_args.receipt(t)["frames"]
        except SystemExit:
            print(f"  {t}: no view yet")
            continue
        print(f"  {t}: frames={fr} " + " ".join(f"B{b}:{hb_args.plan(fr, b)['steps']}/{hb_args.plan(fr, b)['save_steps']}"
                                               for b in (32, 24, 16, 12, 8)))

    print("[3] synthetic transforms")
    from transformers import AutoProcessor
    post = hb_args.POST
    proc = AutoProcessor.from_pretrained(str(post)) if post.is_dir() else None
    check(proc is not None, f"Qwen3-VL processor from init dir {post}")
    rng = np.random.default_rng(0)
    smin, smax = np.array(fd.state_min), np.array(fd.state_max)
    amin, amax = np.array(fd.action_min), np.array(fd.action_max)
    win = np.stack([(smin + smax) / 2 + 0.01 * f for f in range(3)]).astype(np.float32)  # frames t-2,t-1,t (distinct)
    act = (amin + (amax - amin) * rng.random((30, AD))).astype(np.float32)
    imgs = {k: torch.rand(3, 480, 640) for k in rp.image_keys}
    base = {SK: torch.from_numpy(win), AK: torch.from_numpy(act), "task": "Verify Task",
            "episode_index": torch.tensor(5), **imgs}
    o = rp(dict(base), no_aug=True)
    check(np.array_equal(o["states"], win[-1:]) and np.array_equal(o["actions"], act), "no_aug: state = s_t, actions untouched")
    nb = slice(NB, SD)
    sig3 = 3 * math.radians(C.NOISE_DEG)
    # 2026-11-12 user: aligned with GR00T A20 (hb_aug_v2.perturb_joints) -> no proprio delay, bias redrawn per sample
    delays, worst, ds = [0, 0, 0], 0.0, []
    for _ in range(3000):
        o = rp(dict(base))
        k = [kk for kk in range(3) if np.array_equal(o["states"][0, nb], win[2 - kk, nb])]
        assert len(k) == 1, "non-joint dims changed"
        delays[k[0]] += 1
        d = o["states"][0, :NB] - win[2 - k[0], :NB]
        worst = max(worst, float(np.abs(d).max()))
        if len(ds) < 200:
            ds.append(d)
        assert np.array_equal(o["actions"], act)
    p = (delays[1] + delays[2]) / 3000
    check(p == 0.0, f"delay rate {p:.3f} (aligned spec: no proprio delay)")
    bound = math.radians(C.BIAS_DEG) + sig3
    check(0 < worst <= bound + 1e-5, f"joint perturbation max {math.degrees(worst):.3f} deg <= {math.degrees(bound):.2f}")
    D = np.stack(ds)
    spread = float(np.abs(D - D.mean(0)).max())
    check(spread > math.radians(1.0), f"joint offset redrawn per sample (spread {math.degrees(spread):.2f} deg; aligned per-sample bias)")
    if proc is not None:
        native = pt.Psi0ModelTransform(resize=mt.resize, center_crop=mt.center_crop)
        d0 = fd(rp(dict(base), no_aug=True), no_aug=True)
        x_hb = mt(dict(d0), vlm_processor=proc, no_aug=True)
        x_nat = native(dict(d0), vlm_processor=proc, no_aug=True)
        check(torch.equal(x_hb["pixel_values"], x_nat["pixel_values"]) and torch.equal(x_hb["input_ids"], x_nat["input_ids"]),
              "no_aug output identical to native Psi0ModelTransform")
        x_aug = mt(dict(d0), vlm_processor=proc)
        check(x_aug["pixel_values"].shape == x_nat["pixel_values"].shape and not torch.equal(x_aug["pixel_values"], x_nat["pixel_values"]),
              "HB v2 aug changes pixels, same token layout")
        check(int(x_nat["image_grid_thw"].shape[0]) == len(rp.image_keys), f"{len(rp.image_keys)} images per sample to Qwen3-VL")
        boosts = {}
        x = torch.rand(1, len(rp.image_keys), 3, 270, 480)
        for _ in range(150):
            b = C.A.augment(x, C.hb_rng())[1]["boost"]
            boosts[b] = boosts.get(b, 0) + 1
        check(len(boosts) >= 12, f"1.5x boost spread over {len(boosts)}/15 effects")

    if not a.no_data and proc is not None:
        print(f"[4] real view {hb_args.DATA / a.task}")
        t0 = time.time()
        ds = cfg.data(split="train", transform_kwargs={"vlm_processor": proc})
        check(len(ds) == r["frames"], f"len {len(ds)} == receipt frames {r['frames']} (built in {time.time() - t0:.1f}s)")
        tasks_lc = {x.lower() for x in r["tasks"]}
        for i in np.linspace(0, len(ds) - 1, a.samples).astype(int):
            raw = ds.raw_dataset[int(i)]
            check(tuple(raw[SK].shape) == (3, SD) and tuple(raw[AK].shape) == (30, AD),
                  f"raw[{i}] windows {tuple(raw[SK].shape)} {tuple(raw[AK].shape)}")
            check("episode_index" in raw, f"raw[{i}] carries episode_index={int(np.asarray(raw.get('episode_index', -1)))}")
            check(all(tuple(raw[k].shape) == (3, 480, 640) for k in rp.image_keys), f"raw[{i}] frames 3x480x640")
            t1 = time.time()
            s = ds[int(i)]
            st, ac = np.asarray(s["states"]), np.asarray(s["actions"])
            check(st.shape == (1, SD) and ac.shape == (30, AD) and np.abs(st).max() <= 1 and np.abs(ac).max() <= 1
                  and bool(np.all(s["actions_mask"])), f"sample[{i}] states {st.shape} actions {ac.shape} in [-1,1] ({time.time() - t1:.2f}s)")
            check(s["instruction"] in tasks_lc and int(s["image_grid_thw"].shape[0]) == len(rp.image_keys),
                  f"sample[{i}] instruction '{s['instruction']}' / {int(s['image_grid_thw'].shape[0])} images")

    if not a.no_header:
        print("[5] action header re-dimensioning (CPU, ~6 GB RAM)")
        from psi.models.psi0 import Psi0Model
        from psi.trainers.finetune import FinetuneTrainer
        model = Psi0Model(model_cfg=m, vlm_model=torch.nn.Linear(1, 1))
        head = model.action_header
        fake = SimpleNamespace(model=SimpleNamespace(action_header=head))
        expected = hb_patches.expected_new(m.action_dim, m.odim, m.action_chunk_size)
        rep = hb_patches.load_action_header_redim(
            head, str(post), lambda sd: FinetuneTrainer._reconcile_action_header_state_dict(fake, sd), expected)
        new = {e["key"] for e in rep["newly_initialized"]}
        for e in rep["newly_initialized"]:
            print(f"  new  {e['key']}: {e['model_shape']} (ckpt {e['checkpoint_shape']})")
        check(new == expected and len(new) == 6, f"newly initialized == expected 6 keys; loaded {rep['loaded_keys']}/{rep['model_keys']}")
        check(set(rep["checkpoint_keys_not_loaded"]) <= new, f"checkpoint keys not loaded: {rep['checkpoint_keys_not_loaded']}")
        head.eval()
        with torch.no_grad():
            y = head(hidden_states=None, timestep=torch.tensor([500.0]), pooled_projections=torch.randn(1, 768),
                     joint_attention_kwargs=dict(action_hidden_embeds=torch.randn(1, 30, AD), views=torch.randn(1, 12, 8, 2048),
                                                 obs=torch.rand(1, 1, SD) * 2 - 1, traj2ds=None),
                     vlm_attn_mask=torch.ones(1, 8)).action
        check(tuple(y.shape) == (1, 30, AD) and bool(torch.isfinite(y).all()), f"header forward -> {tuple(y.shape)}")
        Path(out, "hb_init_report.json").write_text(json.dumps(rep, indent=2))

    print("VERIFY_OK" if not FAIL else f"VERIFY_FAIL ({len(FAIL)}): " + "; ".join(FAIL))
    return 1 if FAIL else 0


if __name__ == "__main__":
    sys.exit(main())