"""CPU-only checks before any GPU time (no training, no VLM weights loaded). Usage: verify_cpu.py [--task b6|basket|a3|a7|a18] [--samples 3] [--no-header] [--no-data] 1. hb_args -> tyro -> DynamicLaunchConfig (same parse as scripts/train.py) and its HB validators; dims / keys / cameras / body dims must match the task's robot layout (G1 A76/S44/29, H2 A75/S78/31) 2. step plans (20 epochs, save at 10) for every prepared task 3. synthetic repack/field/model transform: no proprio delay, joint bias redrawn per sample + noise within 3 sigma (aligned with GR00T A20, 2026-11-12), untouched non-joint dims, no-aug output identical to the native Psi0ModelTransform, HB v2 aug active 4. real view through LeRobotDatasetWrapper (+ timestamp sync check) -> shapes / ranges / cameras / episode_index 5. action header built by the native Psi0Model ctor (dummy VLM) + re-dimensioned load of the real action_header.safetensors: newly initialized keys must equal hb_patches.expected_new(A, S, 30); CPU forward -> (1, 30, A) """ import argparse import json import math import os import sys import tempfile import time from pathlib import Path from types import SimpleNamespace for _k in ("RANK", "WORLD_SIZE", "LOCAL_RANK", "MASTER_ADDR", "MASTER_PORT", "LOCAL_WORLD_SIZE", "GROUP_RANK"): os.environ.pop(_k, None) # DLC node shells export these; psi overwatch would think it is distributed os.environ["CUDA_VISIBLE_DEVICES"] = "" os.environ.setdefault("HF_HUB_OFFLINE", "1") H = Path(__file__).resolve().parent G = Path(os.environ.get("G", H.parent)) PSI = G / "HumanWAM/third_party/Psi0" sys.path[:0] = [str(H), str(PSI / "src")] os.environ.setdefault("HB_AUG_DIR", str(G / "hb_groot")) os.environ.setdefault("HF_DATASETS_CACHE", f"/tmp/hb_psi0_hfds_{os.getuid()}") import numpy as np # noqa: E402 import torch # noqa: E402 from dotenv import load_dotenv # noqa: E402 load_dotenv(PSI / ".env") FAIL = [] EXPECT = {"g1": (76, 44, 29, "observation.images.ego_view"), "h2": (75, 78, 31, "observation.images.head"), "h2p": (82, 78, 31, "observation.images.head")} # 2026-11-14: a14 v2 action82 (+ planner) def check(cond, msg): print((" ok " if cond else " FAIL ") + msg, flush=True) if not cond: FAIL.append(msg) def main(): ap = argparse.ArgumentParser() ap.add_argument("--task", default="b6") ap.add_argument("--samples", type=int, default=3) ap.add_argument("--no-header", action="store_true") ap.add_argument("--no-data", action="store_true") a = ap.parse_args() import tyro import hb_args import hb_patches import hb_psi0_config as C from psi.config import transform as pt sys.modules["psi.config.train.hb_psi0_config"] = C print("[1] config") check(C.A.VERSION == C.HB_AUG_VERSION, f"hb_aug_v2 imported from {C._AUG_DIR} ({C.A.VERSION})") r = hb_args.receipt(a.task) robot = r.get("robot") check(robot in EXPECT, f"receipt robot '{robot}' (rebuild old views with prepare_data.py --force)") if robot not in EXPECT: return 1 AD, SD, NB, CAM0 = EXPECT[robot] SK, AK = r["state_key"], r["action_key"] out = tempfile.mkdtemp(prefix="hb_psi0_verify_") args = hb_args.build_args(a.task, 16, 100, 50, 3, out, "verify", 2, resume=False) cfg = tyro.cli(C.DynamicLaunchConfig, config=(tyro.conf.ConsolidateSubcommandArgs,), args=args) m, rp, fd = cfg.model, cfg.data.transform.repack, cfg.data.transform.field check((m.action_dim, m.odim, m.action_chunk_size) == (AD, SD, 30), f"model A/S/H = {m.action_dim}/{m.odim}/{m.action_chunk_size}") check(len(fd.action_min) == AD and len(fd.state_min) == SD, f"stats widths {AD}/{SD} (no padding)") check(rp.hb_body_dims == NB, f"joint-perturbed dims {rp.hb_body_dims}: {r['perturbed_joint_names']}") check(rp.image_keys == r["image_keys"] and rp.image_keys[0] == CAM0, f"cameras {rp.image_keys} (cam0 = {CAM0})") mt = cfg.data.transform.model check(not mt.img_aug and not mt.view_aug and mt.hb_aug, "native img/view aug off, HB v2 aug on") check(fd.state_noise_std == 0 and rp.state_temporal_jitter == 0, "native state noise/jitter off") check(m.num_blocks == 12 and m.qk_norm == "rms_norm" and m.state_null_token and m.combined_temb, "native SONIC v1.1 header recipe flags") s32 = torch.tensor([1.0 / AD] * AD, dtype=torch.float32).sum().item() print(f" info fp32 sum(1/{AD} x {AD}) on CPU = {s32!r} (hb_patches handles != 1.0 on the GPU)") print("[2] step plans (global batch = per_gpu x 32)") for t in hb_args.TASKS: try: fr = hb_args.receipt(t)["frames"] except SystemExit: print(f" {t}: no view yet") continue print(f" {t}: frames={fr} " + " ".join(f"B{b}:{hb_args.plan(fr, b)['steps']}/{hb_args.plan(fr, b)['save_steps']}" for b in (32, 24, 16, 12, 8))) print("[3] synthetic transforms") from transformers import AutoProcessor post = hb_args.POST proc = AutoProcessor.from_pretrained(str(post)) if post.is_dir() else None check(proc is not None, f"Qwen3-VL processor from init dir {post}") rng = np.random.default_rng(0) smin, smax = np.array(fd.state_min), np.array(fd.state_max) amin, amax = np.array(fd.action_min), np.array(fd.action_max) win = np.stack([(smin + smax) / 2 + 0.01 * f for f in range(3)]).astype(np.float32) # frames t-2,t-1,t (distinct) act = (amin + (amax - amin) * rng.random((30, AD))).astype(np.float32) imgs = {k: torch.rand(3, 480, 640) for k in rp.image_keys} base = {SK: torch.from_numpy(win), AK: torch.from_numpy(act), "task": "Verify Task", "episode_index": torch.tensor(5), **imgs} o = rp(dict(base), no_aug=True) check(np.array_equal(o["states"], win[-1:]) and np.array_equal(o["actions"], act), "no_aug: state = s_t, actions untouched") nb = slice(NB, SD) sig3 = 3 * math.radians(C.NOISE_DEG) # 2026-11-12 user: aligned with GR00T A20 (hb_aug_v2.perturb_joints) -> no proprio delay, bias redrawn per sample delays, worst, ds = [0, 0, 0], 0.0, [] for _ in range(3000): o = rp(dict(base)) k = [kk for kk in range(3) if np.array_equal(o["states"][0, nb], win[2 - kk, nb])] assert len(k) == 1, "non-joint dims changed" delays[k[0]] += 1 d = o["states"][0, :NB] - win[2 - k[0], :NB] worst = max(worst, float(np.abs(d).max())) if len(ds) < 200: ds.append(d) assert np.array_equal(o["actions"], act) p = (delays[1] + delays[2]) / 3000 check(p == 0.0, f"delay rate {p:.3f} (aligned spec: no proprio delay)") bound = math.radians(C.BIAS_DEG) + sig3 check(0 < worst <= bound + 1e-5, f"joint perturbation max {math.degrees(worst):.3f} deg <= {math.degrees(bound):.2f}") D = np.stack(ds) spread = float(np.abs(D - D.mean(0)).max()) check(spread > math.radians(1.0), f"joint offset redrawn per sample (spread {math.degrees(spread):.2f} deg; aligned per-sample bias)") if proc is not None: native = pt.Psi0ModelTransform(resize=mt.resize, center_crop=mt.center_crop) d0 = fd(rp(dict(base), no_aug=True), no_aug=True) x_hb = mt(dict(d0), vlm_processor=proc, no_aug=True) x_nat = native(dict(d0), vlm_processor=proc, no_aug=True) check(torch.equal(x_hb["pixel_values"], x_nat["pixel_values"]) and torch.equal(x_hb["input_ids"], x_nat["input_ids"]), "no_aug output identical to native Psi0ModelTransform") x_aug = mt(dict(d0), vlm_processor=proc) check(x_aug["pixel_values"].shape == x_nat["pixel_values"].shape and not torch.equal(x_aug["pixel_values"], x_nat["pixel_values"]), "HB v2 aug changes pixels, same token layout") check(int(x_nat["image_grid_thw"].shape[0]) == len(rp.image_keys), f"{len(rp.image_keys)} images per sample to Qwen3-VL") boosts = {} x = torch.rand(1, len(rp.image_keys), 3, 270, 480) for _ in range(150): b = C.A.augment(x, C.hb_rng())[1]["boost"] boosts[b] = boosts.get(b, 0) + 1 check(len(boosts) >= 12, f"1.5x boost spread over {len(boosts)}/15 effects") if not a.no_data and proc is not None: print(f"[4] real view {hb_args.DATA / a.task}") t0 = time.time() ds = cfg.data(split="train", transform_kwargs={"vlm_processor": proc}) check(len(ds) == r["frames"], f"len {len(ds)} == receipt frames {r['frames']} (built in {time.time() - t0:.1f}s)") tasks_lc = {x.lower() for x in r["tasks"]} for i in np.linspace(0, len(ds) - 1, a.samples).astype(int): raw = ds.raw_dataset[int(i)] check(tuple(raw[SK].shape) == (3, SD) and tuple(raw[AK].shape) == (30, AD), f"raw[{i}] windows {tuple(raw[SK].shape)} {tuple(raw[AK].shape)}") check("episode_index" in raw, f"raw[{i}] carries episode_index={int(np.asarray(raw.get('episode_index', -1)))}") check(all(tuple(raw[k].shape) == (3, 480, 640) for k in rp.image_keys), f"raw[{i}] frames 3x480x640") t1 = time.time() s = ds[int(i)] st, ac = np.asarray(s["states"]), np.asarray(s["actions"]) check(st.shape == (1, SD) and ac.shape == (30, AD) and np.abs(st).max() <= 1 and np.abs(ac).max() <= 1 and bool(np.all(s["actions_mask"])), f"sample[{i}] states {st.shape} actions {ac.shape} in [-1,1] ({time.time() - t1:.2f}s)") check(s["instruction"] in tasks_lc and int(s["image_grid_thw"].shape[0]) == len(rp.image_keys), f"sample[{i}] instruction '{s['instruction']}' / {int(s['image_grid_thw'].shape[0])} images") if not a.no_header: print("[5] action header re-dimensioning (CPU, ~6 GB RAM)") from psi.models.psi0 import Psi0Model from psi.trainers.finetune import FinetuneTrainer model = Psi0Model(model_cfg=m, vlm_model=torch.nn.Linear(1, 1)) head = model.action_header fake = SimpleNamespace(model=SimpleNamespace(action_header=head)) expected = hb_patches.expected_new(m.action_dim, m.odim, m.action_chunk_size) rep = hb_patches.load_action_header_redim( head, str(post), lambda sd: FinetuneTrainer._reconcile_action_header_state_dict(fake, sd), expected) new = {e["key"] for e in rep["newly_initialized"]} for e in rep["newly_initialized"]: print(f" new {e['key']}: {e['model_shape']} (ckpt {e['checkpoint_shape']})") check(new == expected and len(new) == 6, f"newly initialized == expected 6 keys; loaded {rep['loaded_keys']}/{rep['model_keys']}") check(set(rep["checkpoint_keys_not_loaded"]) <= new, f"checkpoint keys not loaded: {rep['checkpoint_keys_not_loaded']}") head.eval() with torch.no_grad(): y = head(hidden_states=None, timestep=torch.tensor([500.0]), pooled_projections=torch.randn(1, 768), joint_attention_kwargs=dict(action_hidden_embeds=torch.randn(1, 30, AD), views=torch.randn(1, 12, 8, 2048), obs=torch.rand(1, 1, SD) * 2 - 1, traj2ds=None), vlm_attn_mask=torch.ones(1, 8)).action check(tuple(y.shape) == (1, 30, AD) and bool(torch.isfinite(y).all()), f"header forward -> {tuple(y.shape)}") Path(out, "hb_init_report.json").write_text(json.dumps(rep, indent=2)) print("VERIFY_OK" if not FAIL else f"VERIFY_FAIL ({len(FAIL)}): " + "; ".join(FAIL)) return 1 if FAIL else 0 if __name__ == "__main__": sys.exit(main())