BIue-Jay commited on
Commit
eef1b33
·
verified ·
1 Parent(s): 4e97ca9

b8: last two checkpoints + eval/deploy files + training records

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. b8-table-wipe/b512/step-002000/batch_benchmark/chosen.json +1 -0
  2. b8-table-wipe/b512/step-002000/batch_benchmark/summary.txt +2 -0
  3. b8-table-wipe/b512/step-002000/code/ad_fastwam.py +163 -0
  4. b8-table-wipe/b512/step-002000/code/fastwam_prep.sh +61 -0
  5. b8-table-wipe/b512/step-002000/code/fastwam_task.sh +48 -0
  6. b8-table-wipe/b512/step-002000/code/hb_aug_v2.py +163 -0
  7. b8-table-wipe/b512/step-002000/code/hb_eval_common.py +260 -0
  8. b8-table-wipe/b512/step-002000/code/hb_fastwam_aug.py +73 -0
  9. b8-table-wipe/b512/step-002000/code/hb_fastwam_convert.py +121 -0
  10. b8-table-wipe/b512/step-002000/code/hb_fastwam_train.py +104 -0
  11. b8-table-wipe/b512/step-002000/code/hb_inloop_core.py +249 -0
  12. b8-table-wipe/b512/step-002000/code/hb_inloop_fastwam.py +126 -0
  13. b8-table-wipe/b512/step-002000/code/test_fastwam_inputs.py +61 -0
  14. b8-table-wipe/b512/step-002000/complete.json +25 -0
  15. b8-table-wipe/b512/step-002000/dataset_meta/hb_fastwam_interface.json +14 -0
  16. b8-table-wipe/b512/step-002000/dataset_meta/info.json +538 -0
  17. b8-table-wipe/b512/step-002000/dataset_meta/tasks.jsonl +1 -0
  18. b8-table-wipe/b512/step-002000/dataset_stats.json +0 -0
  19. b8-table-wipe/b512/step-002000/logs/node-0.log +0 -0
  20. b8-table-wipe/b512/step-002000/logs/node-1.log +0 -0
  21. b8-table-wipe/b512/step-002000/logs/node-2.log +0 -0
  22. b8-table-wipe/b512/step-002000/logs/node-3.log +0 -0
  23. b8-table-wipe/b512/step-002000/loss_summary.json +23 -0
  24. b8-table-wipe/b512/step-002000/metrics.csv +201 -0
  25. b8-table-wipe/b512/step-002000/model.pt +3 -0
  26. b8-table-wipe/b512/step-002000/model_config.yaml +57 -0
  27. b8-table-wipe/b512/step-002000/protocol.json +72 -0
  28. b8-table-wipe/b512/step-002000/text_embeddings/5a20e43a27564ea8978edc7394a96e0c1295e82528d59dba0c4169f3bba9a630.t5_len128.wan22ti2v5b.pt +3 -0
  29. b8-table-wipe/b512/step-002000/training_config.yaml +259 -0
  30. b8-table-wipe/b512/step-004000/batch_benchmark/chosen.json +1 -0
  31. b8-table-wipe/b512/step-004000/batch_benchmark/summary.txt +2 -0
  32. b8-table-wipe/b512/step-004000/code/ad_fastwam.py +163 -0
  33. b8-table-wipe/b512/step-004000/code/fastwam_prep.sh +61 -0
  34. b8-table-wipe/b512/step-004000/code/fastwam_task.sh +48 -0
  35. b8-table-wipe/b512/step-004000/code/hb_aug_v2.py +163 -0
  36. b8-table-wipe/b512/step-004000/code/hb_eval_common.py +260 -0
  37. b8-table-wipe/b512/step-004000/code/hb_fastwam_aug.py +73 -0
  38. b8-table-wipe/b512/step-004000/code/hb_fastwam_convert.py +121 -0
  39. b8-table-wipe/b512/step-004000/code/hb_fastwam_train.py +104 -0
  40. b8-table-wipe/b512/step-004000/code/hb_inloop_core.py +249 -0
  41. b8-table-wipe/b512/step-004000/code/hb_inloop_fastwam.py +126 -0
  42. b8-table-wipe/b512/step-004000/code/test_fastwam_inputs.py +61 -0
  43. b8-table-wipe/b512/step-004000/complete.json +25 -0
  44. b8-table-wipe/b512/step-004000/dataset_meta/hb_fastwam_interface.json +14 -0
  45. b8-table-wipe/b512/step-004000/dataset_meta/info.json +538 -0
  46. b8-table-wipe/b512/step-004000/dataset_meta/tasks.jsonl +1 -0
  47. b8-table-wipe/b512/step-004000/dataset_stats.json +0 -0
  48. b8-table-wipe/b512/step-004000/logs/node-0.log +0 -0
  49. b8-table-wipe/b512/step-004000/logs/node-1.log +0 -0
  50. b8-table-wipe/b512/step-004000/logs/node-2.log +0 -0
b8-table-wipe/b512/step-002000/batch_benchmark/chosen.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"per_gpu_batch": 16, "global_batch": 512, "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/bench-20261116-r2/summary.txt"}
b8-table-wipe/b512/step-002000/batch_benchmark/summary.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ bs=16 global=512 rc=[0 0 0 0 ] oom=0 peak_mem_MiB=58421 speed=
2
+ bs=32 global=1024 rc=[1 1 1 1 ] oom=4 peak_mem_MiB=81123 speed=
b8-table-wipe/b512/step-002000/code/ad_fastwam.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM adapter (2026-11-16). Deployment path of FastWAM's own policy (experiments/robotwin/fastwam_policy/
2
+ deploy_policy.py, WorldActionRobotWinPolicy._infer_action_chunk: model.infer_action, the action-only path = first-frame
3
+ video KV cache + action-expert denoising, no future video) fed exactly like our training dataset (RobotVideoDataset +
4
+ FastWAMProcessor of the run's config.yaml, no HB augmentation):
5
+ * cameras in shape_meta order (g1 ego_view + chest, h2 head + chest): uint8 [1,C,H,W] -> processor val transforms
6
+ (ToTensor /255 + Resize 224x224), concatenated horizontally (224 x 448), dataset ResizeSmallestSide/CenterCrop
7
+ (no-op at 224 x 448), Normalize(0.5, 0.5) -> [-1, 1];
8
+ * state: processor.action_state_transform -> normalizer.forward (min/max of the run's
9
+ dataset_stats.json), the proprio of the first frame;
10
+ * text: the run's T5 cache for DEFAULT_PROMPT.format(task=instruction) with context[~mask] = 0 and mask = ones, as in
11
+ training (the text encoder is not loaded in training either; deploy_policy encodes the same prompt with T5 online);
12
+ * infer_action(action_horizon = num_frames - 1 = 16, num_inference_steps = config eval_num_inference_steps (10),
13
+ text_cfg_scale 1, seed 42 = native trainer eval), denormalised with normalizer.normalizers["action"][key].backward.
14
+ ckpt = .../fastwam/runs/<task>/out/checkpoints/weights/step_<N>.pt (config.yaml + dataset_stats.json in .../out).
15
+ Weights: model built from the config (Wan2.2 + ActionDiT init), then the checkpoint's mot + proprio_encoder are
16
+ loaded STRICTLY (FastWAM.load_checkpoint uses strict=False for mot; a missing key here is an error).
17
+ """
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+
24
+ import hb_eval_common as E
25
+
26
+ FW = E.G / "fastwam"
27
+
28
+
29
+ def setup_env():
30
+ os.environ.setdefault("FASTWAM_SRC", str(FW / "FastWAM"))
31
+ os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(FW / "models"))
32
+ os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "true")
33
+ os.environ.setdefault("HF_HUB_OFFLINE", "1")
34
+ src = str(Path(os.environ["FASTWAM_SRC"]) / "src")
35
+ if src not in sys.path:
36
+ sys.path.insert(0, src)
37
+
38
+
39
+ class Inputs:
40
+ """model-agnostic cache query -> FastWAM inputs (shared by the post-hoc adapter and the in-training eval)"""
41
+
42
+ def __init__(self, cfg, stats_path):
43
+ setup_env()
44
+ from hydra.utils import instantiate
45
+ from omegaconf import OmegaConf
46
+ from fastwam.datasets.dataset_utils import CenterCrop, Normalize, ResizeSmallestSideAspectPreserving
47
+ from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT
48
+ from fastwam.datasets.lerobot.utils.normalizer import load_dataset_stats_from_json
49
+ d = cfg.data.train
50
+ self.DEFAULT_PROMPT = DEFAULT_PROMPT
51
+ self.proc = instantiate(d.processor).eval()
52
+ self.proc.set_normalizer_from_stats(load_dataset_stats_from_json(str(stats_path)))
53
+ self.cams = [m["key"] for m in self.proc.shape_meta["images"]]
54
+ if d.concat_multi_camera != "horizontal":
55
+ raise ValueError(f"concat_multi_camera {d.concat_multi_camera} not supported")
56
+ vs = list(d.video_size)
57
+ self.resize = ResizeSmallestSideAspectPreserving(args={"img_w": vs[1], "img_h": vs[0]})
58
+ self.crop = CenterCrop(args={"img_w": vs[1], "img_h": vs[0]})
59
+ self.norm = Normalize(args={"mean": 0.5, "std": 0.5})
60
+ self.cache_dir = d.text_embedding_cache_dir
61
+ self.context_len = int(d.context_len)
62
+ self.horizon = int(d.num_frames) - 1
63
+ self.skey = self.proc.shape_meta["state"][0]["key"]
64
+ self.akey = self.proc.shape_meta["action"][0]["key"]
65
+ if len(self.proc.shape_meta["state"]) != 1 or len(self.proc.shape_meta["action"]) != 1:
66
+ raise ValueError("expected one merged state key and one merged action key")
67
+ self._ctx = {}
68
+ self.OmegaConf = OmegaConf
69
+
70
+ def image(self, images):
71
+ import torch
72
+ per = []
73
+ for c in self.cams:
74
+ x = torch.from_numpy(np.ascontiguousarray(images[c])).permute(2, 0, 1).unsqueeze(0) # [1,C,H,W] uint8
75
+ for tr in self.proc.val_transforms:
76
+ x = tr(x)
77
+ per.append(x)
78
+ v = torch.cat(per, dim=-1) # [1, C, 224, 448]
79
+ v = self.norm(self.crop(self.resize(v)))
80
+ return v # [1, 3, H, W] in [-1, 1], float32
81
+
82
+ def proprio(self, state):
83
+ import torch
84
+ b = {"state": {self.skey: torch.as_tensor(np.asarray(state, dtype=np.float32)).unsqueeze(0)}}
85
+ b = self.proc.action_state_transform(b)
86
+ b = self.proc.normalizer.forward(b) # = deploy_policy._normalize_state (one merged key: merger is identity)
87
+ return b["state"][self.skey] # [1, D]
88
+
89
+ def context(self, instruction):
90
+ import hashlib
91
+ import torch
92
+ prompt = self.DEFAULT_PROMPT.format(task=instruction)
93
+ if prompt not in self._ctx:
94
+ h = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
95
+ p = os.path.join(self.cache_dir, f"{h}.t5_len{self.context_len}.wan22ti2v5b.pt")
96
+ if not os.path.exists(p):
97
+ raise FileNotFoundError(f"no T5 cache for the instruction ({p}): {instruction[:80]!r}")
98
+ obj = torch.load(p, map_location="cpu")
99
+ ctx, mask = obj["context"], obj["mask"]
100
+ ctx = ctx.clone()
101
+ ctx[~mask.bool()] = 0.0
102
+ self._ctx[prompt] = (ctx, torch.ones_like(mask.bool()))
103
+ return self._ctx[prompt]
104
+
105
+ def denorm(self, action):
106
+ import torch
107
+ a = action
108
+ if a.ndim == 2:
109
+ a = a.unsqueeze(0)
110
+ n = self.proc.normalizer.normalizers["action"][self.akey]
111
+ return n.backward(a.to(dtype=torch.float32, device="cpu")).numpy()[0]
112
+
113
+
114
+ def infer(model, inp, obs, steps, seed=42):
115
+ import torch
116
+ img = inp.image(obs["images"]).to(device=model.device, dtype=model.torch_dtype)
117
+ ctx, mask = inp.context(obs["prompt"])
118
+ with torch.no_grad():
119
+ out = model.infer_action(prompt=None, input_image=img, action_horizon=inp.horizon, proprio=inp.proprio(obs["state"]),
120
+ context=ctx, context_mask=mask, text_cfg_scale=1.0, num_inference_steps=steps, seed=seed,
121
+ rand_device="cpu", tiled=False)
122
+ return inp.denorm(out["action"])
123
+
124
+
125
+ class Adapter:
126
+ def __init__(self, ckpt, task, split, meta, device="cuda:0", steps=None, seed="42"):
127
+ setup_env()
128
+ import torch
129
+ from hydra.utils import instantiate
130
+ from omegaconf import OmegaConf
131
+ self.torch = torch
132
+ ckpt = Path(ckpt)
133
+ out = ckpt.parent.parent.parent
134
+ self.cfg = OmegaConf.load(out / "config.yaml")
135
+ self.inp = Inputs(self.cfg, out / "dataset_stats.json")
136
+ miss = [c for c in self.inp.cams if c not in meta["cams"]]
137
+ if miss:
138
+ raise KeyError(f"cameras {miss} not in the cache {meta['cams']}")
139
+ self.device = device if torch.cuda.is_available() else "cpu"
140
+ mp = str(self.cfg.get("mixed_precision", "bf16")).lower()
141
+ dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[mp]
142
+ self.model = instantiate(self.cfg.model, model_dtype=dtype, device=self.device)
143
+ payload = torch.load(str(ckpt), map_location="cpu")
144
+ miss_k, unexp = self.model.mot.load_state_dict(payload["mot"], strict=False)
145
+ if miss_k or unexp:
146
+ raise RuntimeError(f"checkpoint mot keys: missing {list(miss_k)[:5]} unexpected {list(unexp)[:5]}")
147
+ if self.model.proprio_encoder is not None:
148
+ self.model.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
149
+ self.ckpt_step = payload.get("step")
150
+ del payload
151
+ self.model = self.model.to(self.device).eval()
152
+ self.native_steps = int(self.cfg.eval_num_inference_steps)
153
+ self.steps = int(steps or self.native_steps)
154
+ self.seed = int(seed)
155
+ self.ckpt = str(ckpt)
156
+
157
+ def info(self):
158
+ return {"model": "fastwam", "ckpt": self.ckpt, "ckpt_step": self.ckpt_step, "denoising_steps": self.steps,
159
+ "native_steps_in_config": self.native_steps, "cams": self.inp.cams, "chunk": self.inp.horizon,
160
+ "seed": self.seed, "path": "FastWAM.infer_action (action-only, deploy_policy)"}
161
+
162
+ def predict(self, obs):
163
+ return infer(self.model, self.inp, obs, self.steps, self.seed)
b8-table-wipe/b512/step-002000/code/fastwam_prep.sh ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # 2026-11-16: CPU-only preparation of FastWAM for the 12 Delta-flow tasks (no GPU is touched: training is running).
3
+ # Data copies on our interface, ActionDiT initializers per profile, one shared T5 text cache, augmented-sample checks.
4
+ P=/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z
5
+ G=$P/policies-20261109
6
+ F=$G/fastwam
7
+ E=$G/envs/fastwam/bin/python
8
+ exec >> $F/prep.log 2>&1
9
+ echo "$(date) start (v2: 12 tasks)"
10
+ until [ -f $G/envs/fastwam/.hb_ready ] || grep -q ENV_FAILED $F/setup.log; do sleep 30; done
11
+ [ -f $G/envs/fastwam/.hb_ready ] || { echo "ENV_FAILED"; exit 1; }
12
+ grep -q FW_DL_DONE $F/dl2.log || { echo "DOWNLOAD_FAILED"; exit 1; }
13
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true CUDA_VISIBLE_DEVICES="" HF_HUB_OFFLINE=1
14
+ export HB_AUG_DIR=$G/hb_groot
15
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT $(compgen -e | grep -E "^PET_")
16
+ TC=$F/text-cache/all
17
+ # task profile source [first N episodes]
18
+ cat > $F/tasks.txt <<EOF
19
+ a10v3 g1 $P/a10-trolley-push-v3-6e410c5/data
20
+ a3v4 g1 $P/a3-fridge-open-v4-6393d4a/data
21
+ a7v4 g1 $P/a7-cabinet-close-v4-90cc2b5/data
22
+ b7 g1 $P/b7-black-clothes-hamper-v1-1bb002a/data 100
23
+ b5 g1 $P/b5-chair-push-v1-b85b7e9/data
24
+ b8 g1 $P/b8-table-wipe-v1-3b2e618/data
25
+ a17cr2 h2 $P/a17-unscrew-bottle-cropped-v2-3520b70/data
26
+ basketv2 g1 $P/basket-trimv2-c4df4d4/data
27
+ c6 g1 $P/c6-bathroom-door-fetch-v1-3afba69/data
28
+ b6v6 g1 $P/b6-pedal-bin-v6-ad2f5bf/data
29
+ a14v2 h2p $P/a14-unscrew-bulb-v2-8411a6d/data
30
+ b1 g1 $P/b1-riceball-microwave-v1-36da298/data
31
+ EOF
32
+ DIRS=""
33
+ while read t prof src lim; do
34
+ D=$F/data/$t
35
+ if [ ! -f $D/meta/hb_fastwam_interface.json ]; then
36
+ $E $F/tools/hb_fastwam_convert.py $prof $src $D ${lim:+--episodes $lim} || { echo "CONVERT_FAILED $t"; continue; }
37
+ fi
38
+ DIRS="$DIRS,\"$D\""
39
+ done < $F/tasks.txt
40
+ DIRS="[${DIRS#,}]"
41
+ echo "$(date) CONVERTED $DIRS"
42
+ cd $F/FastWAM
43
+ for prof in g1 h2 h2p; do
44
+ AI=$F/action-init-$prof.pt
45
+ [ -f $AI ] && continue
46
+ HB_FW_PROFILE=$prof HB_MODEL_YAML=$F/model-$prof.yaml $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" || { echo "YAML_FAILED $prof"; continue; }
47
+ grep -E 'action_dim|proprio_dim' $F/model-$prof.yaml | tr '\n' ' '; echo
48
+ $E scripts/preprocess_action_dit_backbone.py --model-config $F/model-$prof.yaml --output $AI --device cpu --dtype bfloat16 && echo "$(date) INIT_OK $prof $(stat -c %s $AI)" || echo "INIT_FAILED $prof"
49
+ done
50
+ mkdir -p $TC
51
+ $E scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 model.redirect_common_files=false "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" && echo "$(date) TEXT_OK $(ls $TC | wc -l) files" || echo TEXT_FAILED
52
+ ok=1
53
+ for pair in a10v3:g1 a17cr2:h2 a14v2:h2p; do
54
+ t=${pair%%:*}; prof=${pair##*:}
55
+ HB_FW_PROFILE=$prof HB_DATA_SAMPLE=1 HB_SAMPLE_WORKDIR=/tmp/hbfw-sample-$t $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=[\"$F/data/$t\"]" "data.train.text_embedding_cache_dir=$TC" "model.action_dit_pretrained_path=$F/action-init-$prof.pt" > $F/sample-$t.log 2>&1
56
+ grep -E 'HB_FW_AUG|HBFW_|Error' $F/sample-$t.log | cut -c1-800
57
+ grep -q "HBFW_SAMPLE" $F/sample-$t.log || { ok=0; echo "SAMPLE_FAILED $t"; }
58
+ done
59
+ for t in $(cut -d' ' -f1 $F/tasks.txt); do [ -f $F/data/$t/meta/hb_fastwam_interface.json ] || { ok=0; echo "MISSING_DATA $t"; }; done
60
+ for prof in g1 h2 h2p; do [ -f $F/action-init-$prof.pt ] || { ok=0; echo "MISSING_INIT $prof"; }; done
61
+ [ $ok = 1 ] && touch $F/bench.ready && echo "$(date) BENCH_READY" || echo "$(date) NOT_READY"
b8-table-wipe/b512/step-002000/code/fastwam_task.sh ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Relay cmd item helper (2026-11-16 user): FastWAM on one benchmark task, 4 nodes x 8 GPUs, per-GPU batch from the
3
+ # benchmark ($G/fastwam/chosen.json: largest batch that fits), no gradient accumulation, 4000 steps (user default while
4
+ # the batch is unknown), native Gaussian FM recipe (lr 1e-4 cosine, ZeRO-1, bf16), HB augmentation v2 + proprio
5
+ # perturbation, our state/action interface. Checkpoints at 2000 and 4000. Skips (exit 3) unless the benchmark passed.
6
+ # fastwam_task.sh <task> <profile g1|h2|h2p> <port>
7
+ G=$(dirname "$(dirname "$(realpath "$0")")")
8
+ TASK=$1 PROF=$2 PORT=$3
9
+ F=$G/fastwam; E=$G/envs/fastwam/bin/python; R=$F/runs/$TASK
10
+ RK=${RANK:?}; M0=${MASTER_ADDR:?}
11
+ [ -f $F/fastwam.ok ] && [ -s $F/chosen.json ] || { echo "SKIP fastwam $TASK: benchmark not passed"; exit 3; }
12
+ [ -f $F/data/$TASK/meta/hb_fastwam_interface.json ] && [ -f $F/action-init-$PROF.pt ] || { echo "SKIP fastwam $TASK: data not prepared"; exit 3; }
13
+ BS=$(python3 -c "import json;print(json.load(open('$F/chosen.json'))['per_gpu_batch'])")
14
+ mkdir -p $R/logs
15
+ exec > >(tee -a "$R/logs/node-$RK.log") 2>&1
16
+ date -u; hostname
17
+ [ -f $R/complete.json ] && { echo "already complete"; exit 0; }
18
+ echo "FASTWAM task=$TASK profile=$PROF rank=$RK master=$M0:$PORT per_gpu=$BS global=$((BS * 32)) steps=4000"
19
+ if ! ldconfig -p | grep -q 'libavutil.so.56'; then
20
+ apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
21
+ libavcodec58 libavformat58 libavutil56 libswscale5 libswresample3 libavdevice58 libavfilter7 || true
22
+ fi
23
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT
24
+ unset $(compgen -e | grep -E '^PET_') NPROC_PER_NODE WORLD_SIZE_WITH_STANDBY TORCHELASTIC_RUN_ID
25
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true HF_HUB_OFFLINE=1 HB_AUG_DIR=$G/hb_groot HB_FW_PROFILE=$PROF
26
+ export NCCL_SOCKET_IFNAME=eth GLOO_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false WANDB_MODE=disabled
27
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
28
+ export PATH=$G/envs/fastwam/bin:$PATH # r2 (11-16 05:3x): r1 failed, accelerate called the missing deepspeed CLI (pdsh launcher)
29
+ for i in $(seq 36); do [ -z "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ] && break; sleep 5; done
30
+ cd $F/FastWAM
31
+ $E -m accelerate.commands.launch --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml --deepspeed_multinode_launcher standard \
32
+ --num_machines 4 --machine_rank $RK --main_process_ip $M0 --main_process_port $PORT --num_processes 32 \
33
+ $F/tools/hb_fastwam_train.py "output_dir=$R/out" "batch_size=$BS" max_steps=4000 log_every=20 save_every=2000 eval_every=0 \
34
+ num_workers=8 wandb.enabled=false gradient_accumulation_steps=1 "data.train.dataset_dirs=[\"$F/data/$TASK\"]" \
35
+ "data.train.text_embedding_cache_dir=$F/text-cache/all" "model.action_dit_pretrained_path=$F/action-init-$PROF.pt"
36
+ rc=$?
37
+ echo "$rc" > $R/logs/exit-node-$RK
38
+ if [ "$rc" = 0 ] && [ "$RK" = 0 ]; then
39
+ python3 - "$R/complete.json" "$TASK" "$PROF" "$BS" "$F/data/$TASK/meta/hb_fastwam_interface.json" <<'EOF'
40
+ import json, sys, time
41
+ out, task, prof, bs, iface = sys.argv[1:6]
42
+ d = dict(task=task, profile=prof, per_gpu_batch=int(bs), global_batch=int(bs) * 32, gradient_accumulation_steps=1,
43
+ max_steps=4000, save_every=2000, recipe="native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
44
+ interface=json.load(open(iface)), finished_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
45
+ json.dump(d, open(out, 'w'), indent=1); print('COMPLETE', d)
46
+ EOF
47
+ fi
48
+ echo "EXIT $rc"; exit $rc
b8-table-wipe/b512/step-002000/code/hb_aug_v2.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark data augmentation v2 (project CLAUDE.md), model-agnostic torch implementation.
2
+
3
+ augment(x, rng, obs_mask=None) takes x: float [T, V, 3, H, W] in [0, 1] (T frames, V cameras) and returns the
4
+ augmented tensor. Each call is one training sample: one of the 15 effects is chosen uniformly and its magnitude
5
+ is scaled by 1.5 around identity; the rest stay at x1. Colour is shared across cameras and time; geometry, light
6
+ patches and white balance are drawn per camera and fixed over time; sensor/optics effects (noise, gaussian blur,
7
+ JPEG, motion blur, sharpen, defocus) apply only to observation frames (obs_mask True).
8
+ Pixel-size parameters are defined at 224 px and scaled by H / 224.
9
+ perturb_joints(q, rng) applies the proprio spec: per-joint bias +-2 deg plus N(0, 0.25 deg) clipped at 3 sigma.
10
+ """
11
+ import io
12
+ import math
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from PIL import Image
18
+ from torchvision.transforms import InterpolationMode
19
+ from torchvision.transforms import functional as TF
20
+
21
+ EFFECTS = ("exposure", "contrast", "saturation", "hue", "gamma", "temperature", "rotation_crop", "noise",
22
+ "gaussian_blur", "jpeg", "motion_blur", "sharpen", "defocus", "light_shadow", "wb_channel")
23
+ CENTER = {"exposure": 0.0, "contrast": 1.0, "saturation": 1.0, "hue": 0.0, "gamma": 1.0, "temperature": 0.0}
24
+ RANGE = {"exposure": (-0.22, 0.22), "contrast": (0.88, 1.12), "saturation": (0.76, 1.24), "hue": (-0.015, 0.015),
25
+ "gamma": (0.89, 1.11), "temperature": (-0.066, 0.066)}
26
+ VERSION = "hb-aug-v2-20261109"
27
+
28
+
29
+ def sample(rng, views, height):
30
+ boost = EFFECTS[int(rng.integers(len(EFFECTS)))]
31
+
32
+ def k(name):
33
+ return 1.5 if boost == name else 1.0
34
+
35
+ px = height / 224.0
36
+ p = {"boost": boost}
37
+ for name, (a, b) in RANGE.items():
38
+ c = CENTER[name]
39
+ p[name] = float(rng.uniform(c + k(name) * (a - c), c + k(name) * (b - c)))
40
+ p["geometry"] = []
41
+ for _ in range(views):
42
+ angle = float(rng.uniform(-1.5 * k("rotation_crop"), 1.5 * k("rotation_crop")))
43
+ p["geometry"].append((angle, 1.0 + abs(angle) * 0.015 + float(rng.uniform(0, 0.02 * k("rotation_crop")))))
44
+ p["noise"] = 4.0 / 255.0 * k("noise") if rng.random() < 0.5 else 0.0
45
+ p["gaussian_blur"] = float(rng.uniform(0.25, 0.65)) * k("gaussian_blur") if rng.random() < 0.15 else 0.0
46
+ if rng.random() < 0.35:
47
+ lo = int(round(100 - k("jpeg") * 30))
48
+ hi = int(round(100 - k("jpeg") * 5))
49
+ p["jpeg"] = int(rng.integers(lo, hi + 1))
50
+ else:
51
+ p["jpeg"] = None
52
+ p["motion_blur"] = ((float(rng.uniform(3, 7)) * k("motion_blur") * px, float(rng.uniform(0, 180)))
53
+ if rng.random() < 0.15 else None)
54
+ p["sharpen"] = float(rng.uniform(0.2, 0.6)) * k("sharpen") if rng.random() < 0.15 else 0.0
55
+ p["defocus"] = float(rng.uniform(1, 2)) * k("defocus") * px if rng.random() < 0.10 else 0.0
56
+ if rng.random() < 0.30:
57
+ g = 0.30 * k("light_shadow")
58
+ p["light_shadow"] = [[(float(rng.uniform(0.1, 0.9)), float(rng.uniform(0.1, 0.9)),
59
+ float(rng.uniform(0.15, 0.35)), float(rng.uniform(1 - g, 1 + g)))
60
+ for _ in range(int(rng.integers(1, 3)))] for _ in range(views)]
61
+ else:
62
+ p["light_shadow"] = None
63
+ if rng.random() < 0.30:
64
+ g = 0.10 * k("wb_channel")
65
+ gains = []
66
+ for _ in range(views):
67
+ row = [1.0, 1.0, 1.0]
68
+ row[int(rng.integers(0, 3))] = float(rng.uniform(1 - g, 1 + g))
69
+ gains.append(row)
70
+ p["wb_channel"] = gains
71
+ else:
72
+ p["wb_channel"] = None
73
+ return p
74
+
75
+
76
+ def _conv(x, kernel):
77
+ n, c, h, w = x.shape
78
+ k = kernel.to(x)[None, None].repeat(c, 1, 1, 1)
79
+ pad = kernel.shape[-1] // 2
80
+ return F.conv2d(F.pad(x, (pad, pad, pad, pad), mode="reflect"), k, groups=c)
81
+
82
+
83
+ def motion_kernel(length, angle_deg):
84
+ size = max(3, int(round(length)) | 1)
85
+ k = torch.zeros(size, size)
86
+ c = size // 2
87
+ th = math.radians(angle_deg)
88
+ for s in np.linspace(-(length - 1) / 2, (length - 1) / 2, 8 * size):
89
+ k[int(round(c - s * math.sin(th))), int(round(c + s * math.cos(th)))] = 1.0
90
+ return k / k.sum()
91
+
92
+
93
+ def disk_kernel(r):
94
+ size = 2 * int(math.ceil(r)) + 1
95
+ c = size // 2
96
+ yy, xx = torch.meshgrid(torch.arange(size), torch.arange(size), indexing="ij")
97
+ k = (((yy - c) ** 2 + (xx - c) ** 2) <= (r + 0.5) ** 2).float()
98
+ return k / k.sum()
99
+
100
+
101
+ def apply(x, p, obs_mask=None, noise_seed=0):
102
+ t, v, c, h, w = x.shape
103
+ x = x.clone().clamp(0, 1)
104
+ for vi, (angle, scale) in enumerate(p["geometry"]):
105
+ x[:, vi] = TF.affine(x[:, vi], angle=angle, translate=[0, 0], scale=scale, shear=[0, 0],
106
+ interpolation=InterpolationMode.BILINEAR)
107
+ if p["light_shadow"] is not None:
108
+ yy, xx = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij")
109
+ for vi, blobs in enumerate(p["light_shadow"]):
110
+ m = torch.ones(h, w)
111
+ for cx, cy, s, g in blobs:
112
+ m = m + (g - 1.0) * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * s * s))
113
+ x[:, vi] = (x[:, vi] * m.clamp(min=0).to(x)).clamp(0, 1)
114
+ if p["wb_channel"] is not None:
115
+ for vi, g in enumerate(p["wb_channel"]):
116
+ x[:, vi] = (x[:, vi] * torch.tensor(g, dtype=x.dtype).view(1, 3, 1, 1)).clamp(0, 1)
117
+ y = x.reshape(t * v, c, h, w)
118
+ y = (y * float(2.0 ** p["exposure"])).clamp(0, 1)
119
+ d = p["temperature"]
120
+ y = (y * y.new_tensor([1.0 + d, 1.0, 1.0 - d]).view(1, 3, 1, 1)).clamp(0, 1)
121
+ y = ((y - 0.5) * p["contrast"] + 0.5).clamp(0, 1)
122
+ y = TF.adjust_saturation(y, p["saturation"])
123
+ y = TF.adjust_hue(y, p["hue"])
124
+ y = y.clamp(0, 1).pow(p["gamma"])
125
+ x = y.reshape(t, v, c, h, w)
126
+ mask = torch.ones(t, dtype=torch.bool) if obs_mask is None else torch.as_tensor(obs_mask, dtype=torch.bool)
127
+ if bool(mask.any()):
128
+ o = x[mask].reshape(-1, c, h, w)
129
+ if p["gaussian_blur"] > 0:
130
+ o = TF.gaussian_blur(o, [3, 3], [p["gaussian_blur"]] * 2)
131
+ if p["motion_blur"] is not None:
132
+ o = _conv(o, motion_kernel(*p["motion_blur"]))
133
+ if p["defocus"] > 0:
134
+ o = _conv(o, disk_kernel(p["defocus"]))
135
+ if p["sharpen"] > 0:
136
+ o = o + p["sharpen"] * (o - TF.gaussian_blur(o, [5, 5], [1.0, 1.0]))
137
+ o = o.clamp(0, 1)
138
+ if p["noise"] > 0:
139
+ gen = torch.Generator().manual_seed(int(noise_seed))
140
+ o = (o + torch.randn(o.shape, generator=gen, dtype=o.dtype) * p["noise"]).clamp(0, 1)
141
+ if p["jpeg"] is not None:
142
+ for i in range(o.shape[0]):
143
+ buf = io.BytesIO()
144
+ TF.to_pil_image(o[i]).save(buf, format="JPEG", quality=int(p["jpeg"]))
145
+ buf.seek(0)
146
+ with Image.open(buf) as im:
147
+ o[i] = TF.pil_to_tensor(im.convert("RGB")).to(o.dtype).div(255.0)
148
+ x[mask] = o.reshape(-1, v, c, h, w)
149
+ return x.clamp(0, 1)
150
+
151
+
152
+ def augment(x, rng, obs_mask=None):
153
+ p = sample(rng, x.shape[1], x.shape[-2])
154
+ return apply(x, p, obs_mask, noise_seed=int(rng.integers(0, 2 ** 62))), p
155
+
156
+
157
+ def perturb_joints(q, rng):
158
+ q = np.asarray(q, dtype=np.float64)
159
+ d = q.shape[-1]
160
+ bias = rng.uniform(-math.radians(2.0), math.radians(2.0), size=d)
161
+ sigma = math.radians(0.25)
162
+ noise = np.clip(rng.normal(0.0, sigma, size=q.shape), -3 * sigma, 3 * sigma)
163
+ return (q + bias + noise).astype(np.float32)
b8-table-wipe/b512/step-002000/code/hb_eval_common.py ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark open-loop action-reconstruction evaluation: shared task table, cache layout and metrics.
2
+
3
+ Protocol (identical for every model; mirrors Isaac-GR00T gr00t/eval/open_loop_eval.py):
4
+ * query frames t = 0, 16, 32, ... over the whole episode (non-overlapping 16-step chunks);
5
+ * the model sees the observation at t exactly as at deployment (cameras, raw benchmark state 44/78, instruction;
6
+ Delta Flow also its native 16-step past-action history), all augmentation off;
7
+ * the predicted chunk (first 16 steps, raw action units) is compared with the recorded actions t..t+15;
8
+ rows beyond the episode end are excluded (no clamped / padded targets);
9
+ * metrics: raw MSE / MAE, and nMSE = MSE after dividing every action dim by its std over the task's TRAINING
10
+ frames (floored, see STD_FLOOR_FRAC), overall, per action group and per horizon step.
11
+ Splits: "heldout" = recorded episodes that were never trained on (B6 100-148, trolley 100-102, B7 100-106);
12
+ "train10" = 10 fixed, evenly spaced training episodes (fit on seen data, NOT generalisation).
13
+ Only numpy here: imported by every model environment.
14
+ """
15
+ import json
16
+ import os
17
+ import time
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+
22
+ G = Path(os.environ.get("G", "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/"
23
+ "preflight-20261104T1648Z/policies-20261109"))
24
+ P = G.parent
25
+ ROOT = G / "eval-action-mse"
26
+ CACHE = Path(os.environ.get("HB_EVAL_CACHE", ROOT / "cache"))
27
+ HORIZON = 16
28
+ STRIDE = 16
29
+ N_TRAIN_EVAL = 10
30
+ STD_FLOOR_FRAC = 0.05 # per-dim std floor = 5 % of the median std of the action vector
31
+
32
+ # task -> (source LeRobot v2.1 root, number of TRAINING episodes = fixed first N (None = all recorded), layout)
33
+ TASKS = {
34
+ "b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100, "g1"),
35
+ "basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None, "g1"),
36
+ "trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100, "g1"),
37
+ "b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100, "g1"),
38
+ "b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None, "g1"),
39
+ "a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None, "g1"),
40
+ "a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None, "g1"),
41
+ "a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None, "g1"),
42
+ "c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None, "g1"),
43
+ "a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None, "g1"),
44
+ "a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None, "g1"),
45
+ "a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None, "h2"),
46
+ "a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None, "h2"),
47
+ "a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None, "h2"),
48
+ # 2026-11-15: newer datasets (GR00T in-training eval needs a cache for every task it trains on)
49
+ "a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None, "g1"),
50
+ "a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None, "g1"),
51
+ "b1": (P / "b1-riceball-microwave-v1-36da298/data", None, "g1"),
52
+ "b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None, "g1"),
53
+ "b8": (P / "b8-table-wipe-v1-3b2e618/data", None, "g1"),
54
+ "basketv2": (P / "basket-trimv2-c4df4d4/data", None, "g1"),
55
+ "a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None, "h2p"), # action 82 = hb75 + SONIC planner 7
56
+ # 2026-11-15 evening: v6/v4 datasets (a3v3, a7v3 retired, not registered)
57
+ "b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None, "g1"), # 542 ep, all used (B6 first-100 rule is v5t7 only)
58
+ "b5": (P / "b5-chair-push-v1-b85b7e9/data", None, "g1"),
59
+ "a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None, "g1"),
60
+ "a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None, "g1"),
61
+ # 2026-11-16: B5 standing start (HF b5-chair-push-standing-start-expert-v1 @913f5a16, 100 ep; FastWAM b5ss)
62
+ "b5ss": (P / "b5-chair-push-standing-start-v1-913f5a1/data", None, "g1"),
63
+ # 2026-11-17: B7 black clothes trimmed (HF b7-black-clothes-hamper @949da46, 107 ep; train = first 100, heldout 100-106)
64
+ "b7t": (P / "b7-black-clothes-hamper-trim-949da46/data", 100, "g1"),
65
+ # 2026-11-15 night: H2 cropped v2 (GR00T h32 relay item 71); head+chest only -> build_cache --cams head,chest
66
+ "a18cr2": (P / "a18-pour-water-cropped-v2-44cdd57/data", None, "h2"), # HF a18-pour-water-expert-cropped-v2 @44cdd571
67
+ "a17cr2": (P / "a17-unscrew-bottle-cropped-v2-3520b70/data", None, "h2"), # HF a17-unscrew-bottle-expert-cropped-v2 @3520b70f
68
+ }
69
+
70
+
71
+ def task_for_source(source):
72
+ """benchmark task whose recording root is <source> (groot-data/<view>/hb-dataset-receipt.json "source")"""
73
+ src = Path(source).resolve()
74
+ hits = [t for t, (root, _, _) in TASKS.items() if Path(root).resolve() == src]
75
+ return hits[0] if hits else None
76
+ G1_BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint",
77
+ "left_ankle_pitch_joint", "left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint",
78
+ "right_hip_yaw_joint", "right_knee_joint", "right_ankle_pitch_joint", "right_ankle_roll_joint",
79
+ "waist_yaw_joint", "waist_roll_joint", "waist_pitch_joint", "left_shoulder_pitch_joint",
80
+ "left_shoulder_roll_joint", "left_shoulder_yaw_joint", "left_elbow_joint", "left_wrist_roll_joint",
81
+ "left_wrist_pitch_joint", "left_wrist_yaw_joint", "right_shoulder_pitch_joint",
82
+ "right_shoulder_roll_joint", "right_shoulder_yaw_joint", "right_elbow_joint", "right_wrist_roll_joint",
83
+ "right_wrist_pitch_joint", "right_wrist_yaw_joint"]
84
+ LAYOUTS = {
85
+ # state: body29 | inspire L6 | inspire R6 | projected_gravity3 ; action: motion_token64 | inspire L6 | R6 (same row)
86
+ "g1": dict(state_dim=44, action_dim=76, cams=("ego_view", "chest"),
87
+ groups={"motion_token": (0, 64), "hands": (64, 76)}),
88
+ # state: observation.state75 (legs12 waist3 arms14 head2 hands22+22) | gravity3 ; action: body_target31 | hands44
89
+ "h2": dict(state_dim=78, action_dim=75, cams=("head", "chest", "wrist"),
90
+ groups={"body_target": (0, 31), "hands": (31, 75)}),
91
+ # 2026-11-15 user: A14 v2 is trained on action82 = body_target31 | hands44 | action.planner7 (SONIC planner
92
+ # command); all 82 dims are scored (planner reported as its own group)
93
+ "h2p": dict(state_dim=78, action_dim=82, cams=("head", "chest"),
94
+ groups={"body_target": (0, 31), "hands": (31, 75), "planner": (75, 82)}),
95
+ }
96
+
97
+
98
+ def jl(path):
99
+ return [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
100
+
101
+
102
+ def layout(task):
103
+ return LAYOUTS[TASKS[task][2]]
104
+
105
+
106
+ def split_episodes(task, split, n_recorded):
107
+ _, n_train, _ = TASKS[task]
108
+ n_train = n_recorded if n_train is None else n_train
109
+ if split == "heldout":
110
+ return list(range(n_train, n_recorded))
111
+ if split == "train10":
112
+ return sorted({int(round(x)) for x in np.linspace(0, n_train - 1, N_TRAIN_EVAL)})
113
+ if split.startswith("ep:"): # explicit list, e.g. ep:100,101
114
+ return [int(x) for x in split[3:].split(",")]
115
+ raise ValueError(f"unknown split {split!r}")
116
+
117
+
118
+ def split_dir(task, split):
119
+ return CACHE / task / split
120
+
121
+
122
+ def load_meta(task, split):
123
+ return json.loads((split_dir(task, split) / "meta.json").read_text())
124
+
125
+
126
+ def load_stats(task):
127
+ z = np.load(CACHE / task / "stats.npz")
128
+ return {k: z[k] for k in z.files}
129
+
130
+
131
+ def norm_std(stats):
132
+ std = stats["action_std"].astype(np.float64)
133
+ return np.maximum(std, STD_FLOOR_FRAC * float(np.median(std)))
134
+
135
+
136
+ def iter_episodes(task, split, max_episodes=None):
137
+ meta = load_meta(task, split)
138
+ eps = meta["episodes"][: max_episodes or None]
139
+ for ep in eps:
140
+ with np.load(split_dir(task, split) / f"ep_{ep:06d}.npz") as z:
141
+ yield ep, {k: z[k] for k in z.files}
142
+
143
+
144
+ class Accumulator:
145
+ """Sums of squared / absolute errors; everything is computed in float64 over valid (row, dim) entries."""
146
+
147
+ def __init__(self, std, groups, horizon=HORIZON):
148
+ self.std = np.asarray(std, dtype=np.float64)
149
+ self.groups = groups
150
+ D = len(self.std)
151
+ self.sse_raw = np.zeros(D)
152
+ self.sae_raw = np.zeros(D)
153
+ self.sse_n = np.zeros(D)
154
+ self.rows = 0
155
+ self.step_sse_n = np.zeros(horizon)
156
+ self.step_rows = np.zeros(horizon)
157
+ self.per_episode = {}
158
+
159
+ def add(self, ep, pred, gt, valid):
160
+ pred = np.asarray(pred, dtype=np.float64)
161
+ gt = np.asarray(gt, dtype=np.float64)
162
+ v = np.asarray(valid, dtype=bool)
163
+ if pred.shape != gt.shape:
164
+ raise ValueError(f"pred {pred.shape} vs gt {gt.shape}")
165
+ if not np.isfinite(pred[v]).all():
166
+ raise FloatingPointError("non-finite prediction")
167
+ e = (pred - gt)[v]
168
+ en = e / self.std
169
+ self.sse_raw += (e ** 2).sum(0)
170
+ self.sae_raw += np.abs(e).sum(0)
171
+ self.sse_n += (en ** 2).sum(0)
172
+ self.rows += int(v.sum())
173
+ k = np.nonzero(v)[0]
174
+ self.step_sse_n[k] += (en ** 2).sum(1)
175
+ self.step_rows[k] += 1
176
+ s = self.per_episode.setdefault(int(ep), [0.0, 0])
177
+ s[0] += float((en ** 2).sum())
178
+ s[1] += int(v.sum()) * len(self.std)
179
+
180
+ def result(self):
181
+ D = len(self.std)
182
+ n = max(self.rows, 1)
183
+ out = {"rows": self.rows, "mse_raw": float(self.sse_raw.sum() / (n * D)),
184
+ "mae_raw": float(self.sae_raw.sum() / (n * D)), "nmse": float(self.sse_n.sum() / (n * D)),
185
+ "groups": {}, "nmse_by_step": (self.step_sse_n / np.maximum(self.step_rows, 1) / D).tolist(),
186
+ "nmse_by_dim": (self.sse_n / n).tolist()}
187
+ for g, (a, b) in self.groups.items():
188
+ out["groups"][g] = {"mse_raw": float(self.sse_raw[a:b].sum() / (n * (b - a))),
189
+ "nmse": float(self.sse_n[a:b].sum() / (n * (b - a)))}
190
+ pe = {ep: s / max(c, 1) for ep, (s, c) in sorted(self.per_episode.items())}
191
+ out["nmse_per_episode"] = pe
192
+ vals = np.array(list(pe.values()))
193
+ if len(vals) > 1: # bootstrap-free spread: std error over episodes
194
+ out["nmse_episode_sem"] = float(vals.std(ddof=1) / np.sqrt(len(vals)))
195
+ return out
196
+
197
+
198
+ def queries(T):
199
+ return list(range(0, T, STRIDE))
200
+
201
+
202
+ def gt_chunk(action, t, horizon=HORIZON):
203
+ T = len(action)
204
+ idx = np.arange(t, t + horizon)
205
+ valid = idx < T
206
+ return action[np.minimum(idx, T - 1)], valid
207
+
208
+
209
+ def run(adapter, task, split, out_path, max_episodes=None, max_queries=None, save_pred=True, log_every=50):
210
+ """Evaluate one adapter on one cached split; writes <out_path> (json) and <out_path>.pred.npz."""
211
+ meta = load_meta(task, split)
212
+ lay = LAYOUTS[meta["layout"]]
213
+ stats = load_stats(task)
214
+ acc = Accumulator(norm_std(stats), lay["groups"])
215
+ preds, keys = [], []
216
+ t0 = time.time()
217
+ infer_s = 0.0
218
+ nq = 0
219
+ for ep, z in iter_episodes(task, split, max_episodes):
220
+ state, action = z["state"], z["action"]
221
+ qts = list(z["query_t"])
222
+ for qi, t in enumerate(qts):
223
+ obs = {"episode": ep, "t": int(t), "T": len(action), "state": state[t], "prompt": meta["prompts"][str(int(z["task_index"]))],
224
+ "images": {c: z[f"img_{c}"][qi] for c in meta["cams"]},
225
+ "past_action": action[max(0, t - HORIZON):t],
226
+ "_gt_check": action[t:t + HORIZON]} # only for adapters' input-pipeline consistency checks
227
+ s = time.time()
228
+ pred = np.asarray(adapter.predict(obs), dtype=np.float64)
229
+ infer_s += time.time() - s
230
+ if pred.ndim != 2 or pred.shape[0] < HORIZON or pred.shape[1] != lay["action_dim"]:
231
+ raise ValueError(f"adapter returned {pred.shape}, need (>={HORIZON}, {lay['action_dim']})")
232
+ pred = pred[:HORIZON]
233
+ gt, valid = gt_chunk(action, t)
234
+ acc.add(ep, pred, gt, valid)
235
+ if save_pred:
236
+ preds.append(pred.astype(np.float32))
237
+ keys.append((ep, int(t)))
238
+ nq += 1
239
+ if log_every and nq % log_every == 0:
240
+ print(f"[eval] {task}/{split} {nq} queries, running nMSE {acc.result()['nmse']:.4f}, "
241
+ f"{infer_s / nq:.3f} s/query", flush=True)
242
+ if max_queries and nq >= max_queries:
243
+ break
244
+ if max_queries and nq >= max_queries:
245
+ break
246
+ res = acc.result()
247
+ res.update(task=task, split=split, layout=meta["layout"], episodes=sorted(acc.per_episode), queries=nq,
248
+ horizon=HORIZON, stride=STRIDE, std_floor_frac=STD_FLOOR_FRAC, wall_s=round(time.time() - t0, 1),
249
+ infer_s_per_query=round(infer_s / max(nq, 1), 4), adapter=adapter.info(),
250
+ baselines=meta.get("baselines", {}), cache_created=meta.get("created"),
251
+ partial=bool(max_episodes or max_queries), finished_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
252
+ out_path = Path(out_path)
253
+ out_path.parent.mkdir(parents=True, exist_ok=True)
254
+ tmp = out_path.with_suffix(".tmp")
255
+ tmp.write_text(json.dumps(res, indent=1))
256
+ tmp.replace(out_path)
257
+ if save_pred and preds:
258
+ np.savez_compressed(str(out_path) + ".pred.npz", pred=np.stack(preds).astype(np.float16),
259
+ keys=np.asarray(keys, dtype=np.int64))
260
+ return res
b8-table-wipe/b512/step-002000/code/hb_fastwam_aug.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: Humanoid Benchmark augmentation v2 (project CLAUDE.md) inside the native FastWAM data pipeline.
2
+ - Visual: hb_aug_v2.augment on every training sample's frames before the two cameras are concatenated; frame 0 is the
3
+ current observation (sensor / optics effects only there), the later frames are the future video targets (colour,
4
+ per-camera geometry, light patches and white balance shared with frame 0, as the spec requires).
5
+ - Proprio: hb_aug_v2.perturb_joints on the raw body joints (G1 29, H2 31 = body 29 + head 2) before normalization;
6
+ hands and gravity untouched; supervision labels untouched. Normalization statistics are computed on the raw data.
7
+ FastWAM's own image augmentation is only a resize (no colour jitter / crop), so nothing native is stacked.
8
+ Installed in the main process before the dataloaders fork; HB_FW_JOINTS sets the number of perturbed joints."""
9
+ import os
10
+ import sys
11
+ import time
12
+
13
+ import numpy as np
14
+ import torch
15
+
16
+ sys.path.insert(0, os.environ.get("HB_AUG_DIR", ""))
17
+ import hb_aug_v2 as A # noqa: E402
18
+
19
+ NJ = int(os.environ.get("HB_FW_JOINTS", "29"))
20
+ _seen = {"img": False, "state": False}
21
+
22
+
23
+ def _rng(idx=0):
24
+ return np.random.default_rng([os.getpid(), int(idx) & 0xFFFFFFFF, time.time_ns() & 0xFFFFFFFF])
25
+
26
+
27
+ def _train(proc):
28
+ v = getattr(proc, "is_train", False)
29
+ return bool(v() if callable(v) else v)
30
+
31
+
32
+ def install():
33
+ from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset
34
+ from fastwam.datasets.lerobot.processors.fastwam_processor import FastWAMProcessor
35
+
36
+ pre0 = FastWAMProcessor.preprocess
37
+
38
+ def preprocess(self, data):
39
+ if _train(self) and isinstance(data.get("state"), dict) and "default" in data["state"]:
40
+ st = data["state"]["default"]
41
+ q = st[..., :NJ].detach().cpu().numpy()
42
+ st = st.clone()
43
+ st[..., :NJ] = torch.from_numpy(A.perturb_joints(q, _rng())).to(st.dtype)
44
+ data["state"]["default"] = st
45
+ if not _seen["state"]:
46
+ _seen["state"] = True
47
+ print(f"HB_FW_AUG proprio perturb_joints on state[..., :{NJ}] shape {tuple(st.shape)}", flush=True)
48
+ return pre0(self, data)
49
+
50
+ get0 = BaseLerobotDataset.__getitem__
51
+
52
+ def getitem(self, idx):
53
+ s = get0(self, idx)
54
+ if self.processor is not None and _train(self.processor) and "pixel_values" in s:
55
+ pv = s["pixel_values"]
56
+ five = pv.ndim == 5
57
+ x = (pv if five else pv[None]).permute(1, 0, 2, 3, 4).float() # [T, V, C, H, W]
58
+ hi = float(x.max())
59
+ scale = 255.0 if hi > 1.5 else 1.0
60
+ mask = torch.zeros(x.shape[0], dtype=torch.bool)
61
+ mask[0] = True
62
+ y, p = A.augment((x / scale).clamp(0, 1), _rng(idx), mask)
63
+ y = (y * scale).permute(1, 0, 2, 3, 4)
64
+ s["pixel_values"] = (y if five else y[0]).to(pv.dtype)
65
+ if not _seen["img"]:
66
+ _seen["img"] = True
67
+ print(f"HB_FW_AUG visual {A.VERSION} on pixel_values {tuple(pv.shape)} dtype {pv.dtype} max {hi:.3f} "
68
+ f"(T={x.shape[0]}, obs frame 0) boost={p['boost']}", flush=True)
69
+ return s
70
+
71
+ FastWAMProcessor.preprocess = preprocess
72
+ BaseLerobotDataset.__getitem__ = getitem
73
+ print(f"HB_FW_AUG installed: {A.VERSION}, joints {NJ}", flush=True)
b8-table-wipe/b512/step-002000/code/hb_fastwam_convert.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: LeRobot v2.1 copy of one of our datasets for FastWAM on OUR benchmark interface (CLAUDE.md section 4):
2
+ g1 : state44 = observation.state[0:29] (29 body joints) + measured Inspire 6 + 6 + projected gravity 3;
3
+ action76 = action.motion_token 64 + measured Inspire 6 + 6 (same-row labels, as GR00T / Psi-Zero / Delta Flow)
4
+ h2 : state78 = observation.state[0:75] + projected gravity 3; action75 = action.body_target 31 + action.hands 44
5
+ h2p: as h2 with action82 = action75 + action.planner 7 (A14 v2)
6
+ HumanWAM's fastwam_sonic_data.py expects a 43-D G1 state and takes state[0:22] + state[29:36]; on our 41-D state that
7
+ would put left-hand values into the right arm, so it is not used. Source files are never modified; media are symlinked.
8
+ --episodes N keeps the first N episodes (B7: first 100 of 107, as for every other model).
9
+ Usage: hb_fastwam_convert.py <profile> <source data dir> <fresh output dir> [--episodes N]"""
10
+ import copy, json, shutil, sys, tempfile
11
+ from pathlib import Path
12
+ import numpy as np
13
+ import pyarrow as pa
14
+ import pyarrow.parquet as pq
15
+
16
+ G1 = {"observation.state": 41, "action.motion_token": 64, "observation.inspire_left_hand_state": 6,
17
+ "observation.inspire_right_hand_state": 6, "observation.projected_gravity": 3}
18
+ H2 = {"observation.state": 75, "observation.projected_gravity": 3, "action.body_target": 31, "action.hands": 44}
19
+ H2P = dict(H2, **{"action.planner": 7})
20
+ SPEC = {"g1": (G1, ("ego_view", "chest")), "h2": (H2, ("head", "chest")), "h2p": (H2P, ("head", "chest"))}
21
+
22
+
23
+ def stats(v):
24
+ v = np.asarray(v, dtype=np.float64)
25
+ return {"min": v.min(0).tolist(), "max": v.max(0).tolist(), "mean": v.mean(0).tolist(), "std": v.std(0).tolist(), "count": [len(v)]}
26
+
27
+
28
+ def vec(table, key, values):
29
+ arr = pa.array(values.tolist(), type=pa.list_(pa.float32(), values.shape[1]))
30
+ i = table.schema.get_field_index(key)
31
+ return table.append_column(key, arr) if i < 0 else table.set_column(i, key, arr)
32
+
33
+
34
+ def map_rows(profile, c):
35
+ if profile == "g1":
36
+ L, R = c["observation.inspire_left_hand_state"], c["observation.inspire_right_hand_state"]
37
+ dev = float(np.abs(c["observation.state"][:, 29:41] - np.concatenate([L, R], 1)).max())
38
+ assert dev < 1e-4, f"observation.state[29:41] differs from the Inspire columns by {dev}"
39
+ return (np.concatenate([c["action.motion_token"], L, R], 1),
40
+ np.concatenate([c["observation.state"][:, :29], L, R, c["observation.projected_gravity"]], 1))
41
+ a = [c["action.body_target"], c["action.hands"]] + ([c["action.planner"]] if profile == "h2p" else [])
42
+ return np.concatenate(a, 1), np.concatenate([c["observation.state"], c["observation.projected_gravity"]], 1)
43
+
44
+
45
+ def main(profile, src, out, limit=None):
46
+ raw, cams = SPEC[profile]
47
+ src, out = Path(src).resolve(strict=True), Path(out).resolve()
48
+ if out.exists():
49
+ raise SystemExit(f"{out} exists")
50
+ info = json.loads((src / "meta/info.json").read_text())
51
+ assert info.get("codebase_version") == "v2.1", info.get("codebase_version")
52
+ for k, w in raw.items():
53
+ assert info["features"][k]["shape"] == [w], (k, info["features"][k]["shape"])
54
+ for c in cams:
55
+ assert info["features"]["observation.images." + c]["dtype"] in ("video", "image"), c
56
+ eps = [json.loads(l) for l in (src / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
57
+ assert [e["episode_index"] for e in eps] == list(range(len(eps)))
58
+ if limit is not None:
59
+ eps = eps[:limit]
60
+ old = {r["episode_index"]: r for r in (json.loads(l) for l in (src / "meta/episodes_stats.jsonl").read_text().splitlines() if l.strip())}
61
+ out.parent.mkdir(parents=True, exist_ok=True)
62
+ stg = Path(tempfile.mkdtemp(prefix=".hbfw-", dir=out.parent))
63
+ shutil.copytree(src / "meta", stg / "meta")
64
+ for n in ("videos", "images"):
65
+ if (src / n).exists():
66
+ (stg / n).symlink_to(src / n, target_is_directory=True)
67
+ new, total = [], 0
68
+ for e in eps:
69
+ i = e["episode_index"]
70
+ rel = Path(info["data_path"].format(episode_chunk=i // info["chunks_size"], episode_index=i))
71
+ t = pq.read_table(src / rel)
72
+ c = {k: np.asarray(t[k].to_pylist(), dtype=np.float32) for k in raw}
73
+ act, st = map_rows(profile, c)
74
+ assert len(act) == e["length"] and np.isfinite(act).all() and np.isfinite(st).all()
75
+ total += len(act)
76
+ t = vec(vec(t, "action", act), "observation.state", st)
77
+ md = dict(t.schema.metadata or {})
78
+ if b"huggingface" in md:
79
+ hf = json.loads(md[b"huggingface"])
80
+ for k, w in (("action", act.shape[1]), ("observation.state", st.shape[1])):
81
+ hf["info"]["features"][k] = {"feature": {"dtype": "float32", "_type": "Value"}, "length": w, "_type": "Sequence"}
82
+ md[b"huggingface"] = json.dumps(hf).encode()
83
+ t = t.replace_schema_metadata(md)
84
+ (stg / rel).parent.mkdir(parents=True, exist_ok=True)
85
+ pq.write_table(t, stg / rel)
86
+ r = copy.deepcopy(old[i])
87
+ r["stats"]["action"], r["stats"]["observation.state"] = stats(act), stats(st)
88
+ new.append(r)
89
+ if limit is None:
90
+ assert total == info["total_frames"], (total, info["total_frames"])
91
+ else:
92
+ info["total_episodes"], info["total_frames"] = len(eps), total
93
+ info["total_videos"] = len(eps) * len(cams) if "total_videos" in info else info.get("total_videos")
94
+ info["splits"] = {"train": f"0:{len(eps)}"}
95
+ (stg / "meta/episodes.jsonl").write_text("".join(json.dumps(e) + "\n" for e in eps))
96
+ info["features"]["action"] = {"dtype": "float32", "shape": [int(act.shape[1])], "names": None}
97
+ info["features"]["observation.state"] = {"dtype": "float32", "shape": [int(st.shape[1])], "names": None}
98
+ drop = [k for k, v in info["features"].items() if v["dtype"] in ("video", "image") and k not in ["observation.images." + c for c in cams]]
99
+ for k in drop: # 2-camera rule: unused cameras are not downloaded, FastWAM checks every listed video file
100
+ del info["features"][k]
101
+ for r in new:
102
+ r["stats"].pop(k, None)
103
+ if drop and info.get("total_videos"):
104
+ info["total_videos"] = len(eps) * len(cams)
105
+ (stg / "meta/info.json").write_text(json.dumps(info, indent=2) + "\n")
106
+ (stg / "meta/episodes_stats.jsonl").write_text("".join(json.dumps(r) + "\n" for r in new))
107
+ for s in (stg / "meta").glob("*stats*.json"):
108
+ s.unlink()
109
+ (stg / "meta/hb_fastwam_interface.json").write_text(json.dumps(dict(
110
+ profile=profile, source=str(src), cameras=list(cams), state_dim=int(st.shape[1]), action_dim=int(act.shape[1]),
111
+ episodes=len(eps), frames=total, fps=info["fps"], first_episodes_only=limit), indent=2) + "\n")
112
+ stg.rename(out)
113
+ print("HBFW_CONVERT_OK", profile, out, "episodes", len(eps), "frames", total, "state", st.shape[1], "action", act.shape[1], flush=True)
114
+
115
+
116
+ if __name__ == "__main__":
117
+ a = sys.argv[1:]
118
+ lim = None
119
+ if "--episodes" in a:
120
+ j = a.index("--episodes"); lim = int(a[j + 1]); del a[j:j + 2]
121
+ main(a[0], a[1], a[2], lim)
b8-table-wipe/b512/step-002000/code/hb_fastwam_train.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: native FastWAM Gaussian-FM training on our benchmark interface, with the HB augmentation v2.
2
+ Profiles (HB_FW_PROFILE): g1 = state44 / action76 / cameras ego_view + chest (HumanWAM fastwam_sonic recipe);
3
+ h2 = state78 / action75 / head + chest; h2p = state78 / action82 (+ planner 7) / head + chest. Fixed recipe of HumanWAM
4
+ fastwam_sonic_training.compose_sonic_config: 17 raw samples = 16 actions + 5 video frames (stride 4), 224 x 448 side by
5
+ side, min/max statistics, absolute labels, action scheduler shift 1.0, original Wan files.
6
+ Arguments: key=value Hydra overrides (JSON values parsed). Env: FASTWAM_SRC, DIFFSYNTH_MODEL_BASE_PATH, HB_AUG_DIR.
7
+ HB_MODEL_YAML=<path> writes the resolved model config; HB_COMPOSE_ONLY=1 prints the config; HB_DATA_SAMPLE=1 reads one
8
+ augmented training sample on CPU; HB_FW_AUG=0 disables the augmentation (benchmark only)."""
9
+ import json, os, sys
10
+ from pathlib import Path
11
+
12
+ ROOT = Path(os.environ["FASTWAM_SRC"]).resolve()
13
+ sys.path.insert(0, str(ROOT / "src"))
14
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
15
+ TASK = "libero_uncond_2cam224_1e-4"
16
+ PROFILES = {"g1": (("ego_view", "chest"), 44, 76, 29), "h2": (("head", "chest"), 78, 75, 31),
17
+ "h2p": (("head", "chest"), 78, 82, 31)}
18
+ CAMS, SDIM, ADIM, NJ = PROFILES[os.environ.get("HB_FW_PROFILE", "g1")]
19
+ os.environ.setdefault("HB_FW_JOINTS", str(NJ))
20
+ SHAPE_META = {
21
+ "images": [{"key": n, "raw_shape": [3, 224, 224], "shape": [3, 224, 224]} for n in CAMS],
22
+ "action": [{"key": "default", "raw_shape": ADIM, "shape": ADIM}],
23
+ "state": [{"key": "default", "raw_shape": SDIM, "shape": SDIM}],
24
+ }
25
+ FIXED = {
26
+ "data.train.shape_meta": SHAPE_META, "data.train.num_frames": 17, "data.train.global_sample_stride": 1,
27
+ "data.train.action_video_freq_ratio": 4, "data.train.video_size": [224, 448], "data.train.concat_multi_camera": "horizontal",
28
+ "data.train.processor.action_output_dim": ADIM, "data.train.processor.proprio_output_dim": SDIM,
29
+ "data.train.processor.delta_action_dim_mask": {"default": [False] * ADIM}, "data.train.processor.action_state_transforms": None,
30
+ "data.train.processor.use_stepwise_action_norm": False, "data.train.processor.norm_default_mode": "min/max",
31
+ "data.train.processor.norm_exception_mode": None, "model.redirect_common_files": False,
32
+ "model.action_scheduler.train_shift": 1.0, "model.action_scheduler.infer_shift": 1.0,
33
+ }
34
+
35
+
36
+ def build(overrides):
37
+ from hydra import compose, initialize_config_dir
38
+ from omegaconf import OmegaConf
39
+ from fastwam.utils.config_resolvers import register_default_resolvers
40
+ register_default_resolvers()
41
+ with initialize_config_dir(config_dir=str(ROOT / "configs"), version_base="1.3"):
42
+ cfg = compose(config_name="train", overrides=["task=" + TASK])
43
+ for k, v in {**FIXED, **overrides}.items():
44
+ OmegaConf.update(cfg, k, v, merge=False)
45
+ for k, v in FIXED.items():
46
+ a = OmegaConf.select(cfg, k)
47
+ if OmegaConf.is_config(a):
48
+ a = OmegaConf.to_container(a, resolve=True)
49
+ assert a == v, (k, a, v)
50
+ return cfg
51
+
52
+
53
+ def parse(argv):
54
+ out = {}
55
+ for a in argv:
56
+ k, v = a.split("=", 1)
57
+ try:
58
+ out[k] = json.loads(v)
59
+ except ValueError:
60
+ out[k] = v
61
+ return out
62
+
63
+
64
+ if __name__ == "__main__":
65
+ from omegaconf import OmegaConf
66
+ cfg = build(parse(sys.argv[1:]))
67
+ if os.environ.get("HB_MODEL_YAML"):
68
+ Path(os.environ["HB_MODEL_YAML"]).write_text(OmegaConf.to_yaml(cfg.model, resolve=True))
69
+ print("HBFW_MODEL_YAML", os.environ["HB_MODEL_YAML"], flush=True)
70
+ raise SystemExit(0)
71
+ if os.environ.get("HB_COMPOSE_ONLY"):
72
+ print(OmegaConf.to_yaml(cfg, resolve=True))
73
+ raise SystemExit(0)
74
+ import fastwam.runtime as native
75
+ assert Path(native.__file__).resolve() == ROOT / "src/fastwam/runtime.py", native.__file__
76
+ if os.environ.get("HB_FW_AUG", "1") != "0":
77
+ import hb_fastwam_aug
78
+ hb_fastwam_aug.install()
79
+ print("HBFW_RECIPE", json.dumps(dict(profile=os.environ.get("HB_FW_PROFILE", "g1"), cameras=CAMS, state_dim=SDIM,
80
+ action_dim=ADIM, perturbed_joints=int(os.environ["HB_FW_JOINTS"]),
81
+ aug=os.environ.get("HB_FW_AUG", "1") != "0",
82
+ batch_size=cfg.batch_size, max_steps=cfg.max_steps, lr=cfg.learning_rate)), flush=True)
83
+ if os.environ.get("HB_DATA_SAMPLE"): # CPU check: build the dataset and read augmented samples, no model
84
+ from omegaconf import open_dict
85
+ from fastwam.utils import misc
86
+ misc.register_work_dir(os.environ.get("HB_SAMPLE_WORKDIR", "/tmp/hbfw-sample"))
87
+ tr, _ = native.build_datasets(cfg.data)
88
+ s = tr[len(tr) // 2]
89
+
90
+ def desc(x):
91
+ if isinstance(x, dict):
92
+ return {k: desc(v) for k, v in x.items()}
93
+ if hasattr(x, "shape"):
94
+ return [tuple(x.shape), str(x.dtype), round(float(x.float().min()), 3), round(float(x.float().max()), 3)]
95
+ return type(x).__name__
96
+ print("HBFW_SAMPLE", len(tr), json.dumps(desc(s), default=str), flush=True)
97
+ raise SystemExit(0)
98
+ if os.environ.get("HB_INLOOP_EVERY", "250") != "0": # 2026-11-16: benchmark action nMSE every 250 steps
99
+ try:
100
+ import hb_inloop_fastwam
101
+ hb_inloop_fastwam.install()
102
+ except Exception as e: # noqa: BLE001 - never block training
103
+ print("HB_INLOOP fastwam install failed:", repr(e), flush=True)
104
+ native.run_training(cfg)
b8-table-wipe/b512/step-002000/code/hb_inloop_core.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared machinery of the in-training open-loop action MSE (Psi-Zero, Delta Flow; GR00T has its own tested copy in
2
+ hb_groot/hb_inloop_eval.py with the same protocol).
3
+
4
+ InLoop(model_name, task, split, out_jsonl, prepare, predict):
5
+ prepare(queries) -> optional per-model setup on this rank's queries (first eval only, inside the RNG fork)
6
+ predict(batch, seed) -> np.ndarray (B, >=16, >=Da) in raw action units; the first 16 steps and the first Da dims
7
+ (Da = benchmark action dim of the layout) are scored.
8
+ InLoop.evaluate(step, model): all train/eval modes of model.modules() and every RNG state (torch CPU + this CUDA device,
9
+ numpy, python) are restored afterwards; partial sums are all-reduced once (error flag included, so a failure on one
10
+ rank cannot leave the others waiting); rank 0 appends one json line per eval. Any failure disables later evals.
11
+ Protocol and metrics = hb_eval_common (post-hoc eval): queries every 16 frames, nMSE with the training-frame std.
12
+ """
13
+ import json
14
+ import os
15
+ import random
16
+ import sys
17
+ import time
18
+ import traceback
19
+ from pathlib import Path
20
+
21
+ import numpy as np
22
+ import torch
23
+
24
+ HERE = Path(__file__).resolve().parent
25
+ if str(HERE) not in sys.path:
26
+ sys.path.insert(0, str(HERE))
27
+ import hb_eval_common as E # noqa: E402
28
+
29
+ EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
30
+ BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
31
+ MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
32
+ SEED = 1234
33
+
34
+
35
+ def dist_info():
36
+ import torch.distributed as dist
37
+ if dist.is_available() and dist.is_initialized():
38
+ return dist.get_rank(), dist.get_world_size()
39
+ return int(os.environ.get("RANK", "0")), int(os.environ.get("WORLD_SIZE", "1"))
40
+
41
+
42
+ def pick_split(task):
43
+ split = os.environ.get("HB_INLOOP_SPLIT")
44
+ if not split:
45
+ split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
46
+ if not (E.split_dir(task, split) / "meta.json").exists():
47
+ raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
48
+ return split
49
+
50
+
51
+ class InLoop:
52
+ def __init__(self, model_name, task, split, out_jsonl, predict, prepare=None, cams=None, info=None):
53
+ self.model_name, self.task, self.split = model_name, task, split
54
+ self.out = Path(out_jsonl)
55
+ self.predict, self.prepare = predict, prepare
56
+ self.cams = cams
57
+ self.info = info or {}
58
+ self.disabled = False
59
+ self.ready = False
60
+ self.done_steps = set()
61
+
62
+ def say(self, *a):
63
+ if dist_info()[0] == 0:
64
+ print("HB_INLOOP", *a, flush=True)
65
+
66
+ def _setup(self):
67
+ rank, world = dist_info()
68
+ meta = E.load_meta(self.task, self.split)
69
+ self.meta, self.lay = meta, E.LAYOUTS[meta["layout"]]
70
+ self.std = E.norm_std(E.load_stats(self.task))
71
+ cams = self.cams or meta["cams"]
72
+ miss = [c for c in cams if c not in meta["cams"]]
73
+ if miss:
74
+ raise KeyError(f"cameras {miss} not in the eval cache {meta['cams']}")
75
+ allq = []
76
+ for ep in meta["episodes"]:
77
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
78
+ allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
79
+ if MAX_Q:
80
+ allq = allq[:MAX_Q]
81
+ self.n_total = len(allq)
82
+ mine = [q for i, q in enumerate(allq) if i % world == rank]
83
+ self.eps = list(meta["episodes"])
84
+ by_ep = {}
85
+ for ep, qi, t in mine:
86
+ by_ep.setdefault(ep, []).append((qi, t))
87
+ self.q = []
88
+ for ep, lst in by_ep.items():
89
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
90
+ state, action = z["state"], z["action"]
91
+ prompt = meta["prompts"][str(int(z["task_index"]))]
92
+ imgs = {c: z[f"img_{c}"] for c in cams}
93
+ for qi, t in lst:
94
+ gt, valid = E.gt_chunk(action, t)
95
+ self.q.append(dict(episode=int(ep), t=int(t), T=len(action), state=state[t].copy(), prompt=prompt,
96
+ images={c: imgs[c][qi].copy() for c in cams}, gt=gt, valid=valid,
97
+ past_action=action[max(0, t - E.HORIZON):t].copy()))
98
+ if self.prepare:
99
+ self.prepare(self.q)
100
+ self.ready = True
101
+ self.say(f"{self.model_name} ready: task {self.task} split {self.split} queries {self.n_total} "
102
+ f"(rank {rank}/{world}: {len(self.q)}), every {EVERY} steps, batch {BATCH}, cams {cams}")
103
+
104
+ def evaluate(self, step, model):
105
+ if self.disabled or step in self.done_steps:
106
+ return
107
+ self.done_steps.add(step)
108
+ rank, world = dist_info()
109
+ t0 = time.time()
110
+ err, msg = 0.0, ""
111
+ acc, nq, infer = None, 0, 0.0
112
+ modes = [(m, m.training) for m in model.modules()]
113
+ dev = torch.cuda.current_device() if torch.cuda.is_available() else None
114
+ np_state, py_state = np.random.get_state(), random.getstate()
115
+ try:
116
+ with torch.random.fork_rng(devices=[dev] if dev is not None else []):
117
+ if not self.ready:
118
+ self._setup()
119
+ D = self.lay["action_dim"]
120
+ model.eval()
121
+ acc = E.Accumulator(self.std, self.lay["groups"])
122
+ for bi in range(0, len(self.q), BATCH):
123
+ batch = self.q[bi:bi + BATCH]
124
+ seed = SEED + 1000 * rank + bi
125
+ torch.default_generator.manual_seed(seed)
126
+ if dev is not None:
127
+ torch.cuda.manual_seed(seed)
128
+ s = time.time()
129
+ with torch.no_grad():
130
+ pred = np.asarray(self.predict(batch, seed), dtype=np.float64)
131
+ infer += time.time() - s
132
+ if pred.ndim != 3 or pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] < D:
133
+ raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, >={D})")
134
+ for q, p in zip(batch, pred):
135
+ acc.add(q["episode"], p[:E.HORIZON, :D], q["gt"], q["valid"])
136
+ nq += 1
137
+ except Exception as e: # noqa: BLE001 - never stop training
138
+ err, msg = 1.0, f"{type(e).__name__}: {e}"
139
+ print(f"HB_INLOOP {self.model_name} rank {rank} step {step} FAILED: {msg}\n{traceback.format_exc()}",
140
+ file=sys.stderr, flush=True)
141
+ finally:
142
+ for m, tr in modes:
143
+ m.training = tr
144
+ np.random.set_state(np_state)
145
+ random.setstate(py_state)
146
+ D = self.lay["action_dim"] if self.ready else 0
147
+ H = E.HORIZON
148
+ eps = self.eps if self.ready else []
149
+ nE = len(eps)
150
+ vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
151
+ if acc is not None and not err:
152
+ o = 0
153
+ vec[o:o + D] = acc.sse_raw; o += D
154
+ vec[o:o + D] = acc.sae_raw; o += D
155
+ vec[o:o + D] = acc.sse_n; o += D
156
+ vec[o] = acc.rows; o += 1
157
+ vec[o:o + H] = acc.step_sse_n; o += H
158
+ vec[o:o + H] = acc.step_rows; o += H
159
+ for j, ep in enumerate(eps):
160
+ s_, c_ = acc.per_episode.get(int(ep), (0.0, 0))
161
+ vec[o + j], vec[o + nE + j] = s_, c_
162
+ o += 2 * nE
163
+ vec[o], vec[o + 1] = nq, infer
164
+ vec[-1] = err
165
+ tot = self._allreduce(vec)
166
+ if tot is None or tot[-1] > 0 or not self.ready:
167
+ self.disabled = True
168
+ nfail = None if tot is None else int(tot[-1])
169
+ self._write({"step": step, "error": msg or "failed on another rank / collective failed",
170
+ "ranks_failed": nfail})
171
+ self.say(f"{self.model_name} step {step}: eval failed ({nfail} rank(s)) -> in-training eval disabled: {msg}")
172
+ return
173
+ o = 0
174
+ acc = E.Accumulator(self.std, self.lay["groups"])
175
+ acc.sse_raw = tot[o:o + D]; o += D
176
+ acc.sae_raw = tot[o:o + D]; o += D
177
+ acc.sse_n = tot[o:o + D]; o += D
178
+ acc.rows = int(round(tot[o])); o += 1
179
+ acc.step_sse_n = tot[o:o + H]; o += H
180
+ acc.step_rows = tot[o:o + H]; o += H
181
+ acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
182
+ for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
183
+ o += 2 * nE
184
+ n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
185
+ res = acc.result()
186
+ res.pop("nmse_by_dim", None)
187
+ rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "model": self.model_name,
188
+ "task": self.task, "split": self.split, "queries": n_q, "ranks": world,
189
+ "wall_s": round(time.time() - t0, 1), "infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
190
+ "seed": SEED, **self.info, **res}
191
+ self._write(rec)
192
+ g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
193
+ self.say(f"{self.model_name} step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} "
194
+ f"queries, {rec['wall_s']} s")
195
+
196
+ @staticmethod
197
+ def _allreduce(vec):
198
+ import torch.distributed as dist
199
+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1:
200
+ return vec
201
+ try:
202
+ dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
203
+ n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
204
+ mx, mn = n.clone(), n.clone()
205
+ dist.all_reduce(mx, op=dist.ReduceOp.MAX)
206
+ dist.all_reduce(mn, op=dist.ReduceOp.MIN)
207
+ if mx.item() != mn.item():
208
+ return None
209
+ t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
210
+ dist.all_reduce(t)
211
+ return t.cpu().numpy()
212
+ except Exception as e: # noqa: BLE001
213
+ print(f"HB_INLOOP all_reduce failed: {e!r}", file=sys.stderr, flush=True)
214
+ return None
215
+
216
+ def _write(self, rec):
217
+ if dist_info()[0] != 0:
218
+ return
219
+ try:
220
+ self.out.parent.mkdir(parents=True, exist_ok=True)
221
+ with open(self.out, "a") as f:
222
+ f.write(json.dumps(rec) + "\n")
223
+ except Exception as e: # noqa: BLE001
224
+ print(f"HB_INLOOP write failed: {e!r}", flush=True)
225
+
226
+
227
+ class TempAttrs:
228
+ """set attributes on an object for the duration of a block, then restore (or delete) them"""
229
+ _MISSING = object()
230
+
231
+ def __init__(self, obj, **kw):
232
+ self.obj, self.kw, self.old = obj, kw, {}
233
+
234
+ def __enter__(self):
235
+ for k, v in self.kw.items():
236
+ self.old[k] = self.obj.__dict__.get(k, self._MISSING)
237
+ setattr(self.obj, k, v)
238
+ return self.obj
239
+
240
+ def __exit__(self, *exc):
241
+ for k, v in self.old.items():
242
+ if v is self._MISSING:
243
+ try:
244
+ delattr(self.obj, k)
245
+ except AttributeError:
246
+ pass
247
+ else:
248
+ setattr(self.obj, k, v)
249
+ return False
b8-table-wipe/b512/step-002000/code/hb_inloop_fastwam.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM in-training open-loop action MSE (2026-11-16 user: every 250 steps, like GR00T / Psi-Zero / Delta Flow).
2
+
3
+ install() is called by fastwam/tools/hb_fastwam_train.py right before native.run_training(cfg). It uses the trainer's
4
+ own eval slot: after Wan22Trainer.__init__ it sets eval_every = HB_INLOOP_EVERY (default 250) and a non-None
5
+ val_dataset marker, and replaces Wan22Trainer.evaluate (the native one generates a full video for one random val sample;
6
+ it is off in our runs, eval_every=0 on the command line). The native loop then calls evaluate() on every rank right after
7
+ the optimizer step of every 250th update, followed by accelerator.wait_for_everyone(); returning None logs nothing.
8
+ evaluate(): all ranks score their shard of the benchmark eval cache with the live model (accelerator.unwrap_model; ZeRO-1
9
+ keeps full bf16 weights on every rank) through EXACTLY the post-hoc adapter path (eval-action-mse/adapters/ad_fastwam.py
10
+ Inputs + infer: FastWAM.infer_action, the action-only deployment path; 10 denoising steps = config
11
+ eval_num_inference_steps; seed 42 per query; same cameras / proprio normalisation / T5 cache as training).
12
+ Results: <output_dir>/inloop-mse.jsonl (rank 0) + "HB_INLOOP" log lines. Training is untouched (module modes and every
13
+ RNG restored by hb_inloop_core, no gradients); any failure disables later evals on all ranks.
14
+ """
15
+ import json
16
+ import os
17
+ import sys
18
+ from pathlib import Path
19
+
20
+ G = Path(__file__).resolve().parents[2] # .../policies-20261109 (this file: $G/fastwam/tools/)
21
+ EVAL = G / "eval-action-mse"
22
+ for p in (str(EVAL), str(EVAL / "adapters")):
23
+ if p not in sys.path:
24
+ sys.path.insert(0, p)
25
+ import numpy as np # noqa: E402
26
+ import hb_eval_common as E # noqa: E402
27
+ import hb_inloop_core as C # noqa: E402
28
+ import ad_fastwam as A # noqa: E402
29
+
30
+
31
+ class _Marker:
32
+ """non-None val_dataset so the native loop reaches evaluate(); never indexed"""
33
+
34
+ def __len__(self):
35
+ return 1
36
+
37
+
38
+ class FwEval:
39
+ def __init__(self, trainer):
40
+ cfg = trainer.cfg
41
+ self.cfg = cfg
42
+ ddir = Path(list(cfg.data.train.dataset_dirs)[0])
43
+ iface = json.loads((ddir / "meta/hb_fastwam_interface.json").read_text())
44
+ task = E.task_for_source(iface["source"])
45
+ if task is None:
46
+ raise LookupError(f"no benchmark task for {iface['source']}")
47
+ split = C.pick_split(task)
48
+ self.steps = int(cfg.eval_num_inference_steps)
49
+ self.inp = None
50
+ self.model = None
51
+ out = Path(cfg.output_dir) / "inloop-mse.jsonl"
52
+ self.il = C.InLoop("fastwam", task, split, out, predict=self.predict, prepare=self.prepare,
53
+ info={"denoising_steps": self.steps, "seed_per_query": 42, "dataset": ddir.name,
54
+ "path": "FastWAM.infer_action (ad_fastwam)"})
55
+ self.il.cams = None
56
+
57
+ def prepare(self, queries):
58
+ self.inp = A.Inputs(self.cfg, Path(self.cfg.output_dir) / "dataset_stats.json")
59
+ miss = [c for c in self.inp.cams if c not in self.il.meta["cams"]]
60
+ if miss:
61
+ raise KeyError(f"cameras {miss} not in the eval cache {self.il.meta['cams']}")
62
+ for q in {q["prompt"] for q in queries}:
63
+ self.inp.context(q) # fail early if an instruction has no T5 cache
64
+
65
+ def predict(self, batch, seed):
66
+ return np.stack([np.asarray(A.infer(self.model, self.inp, q, self.steps, 42), dtype=np.float64) for q in batch])
67
+
68
+ def evaluate(self, step, model):
69
+ self.model = model
70
+ try:
71
+ self.il.evaluate(step, model)
72
+ finally:
73
+ self.model = None
74
+
75
+
76
+ class _Broken:
77
+ def __init__(self, err):
78
+ self.err = err
79
+ self.il = C.InLoop("fastwam", "b6", "train10", "/dev/null", predict=None, prepare=self._boom)
80
+
81
+ def _boom(self, _q):
82
+ raise RuntimeError(self.err)
83
+
84
+ def evaluate(self, step, model):
85
+ self.il.evaluate(step, model)
86
+
87
+
88
+ def install():
89
+ if C.EVERY <= 0:
90
+ return
91
+ from fastwam import trainer as T
92
+ WT = T.Wan22Trainer
93
+ orig_init = WT.__init__
94
+
95
+ def __init__(self, *a, **kw):
96
+ orig_init(self, *a, **kw)
97
+ # native eval off (eval_every 0 on our command line). NB runtime.build_datasets sets val_ds = train_ds when
98
+ # there is no data.val, so val_dataset is never None here (11-16 smoke: the old "val_dataset is None" test
99
+ # silently left the in-loop eval off).
100
+ if int(self.eval_every) == 0:
101
+ self.eval_every = C.EVERY
102
+ if self.val_dataset is None:
103
+ self.val_dataset = _Marker()
104
+ self._hb_inloop_on = True
105
+ else: # a native eval is configured: leave it alone
106
+ self._hb_inloop_on = False
107
+
108
+ def evaluate(self):
109
+ if not getattr(self, "_hb_inloop_on", False):
110
+ return orig_eval(self)
111
+ ev = getattr(self, "_hb_inloop", None)
112
+ if ev is None:
113
+ try:
114
+ ev = FwEval(self)
115
+ except Exception as e: # noqa: BLE001
116
+ ev = _Broken(f"{type(e).__name__}: {e}")
117
+ self._hb_inloop = ev
118
+ if not ev.il.disabled:
119
+ ev.evaluate(int(self.global_step), self.accelerator.unwrap_model(self.model))
120
+ return None
121
+
122
+ orig_eval = WT.evaluate
123
+ WT.__init__ = __init__
124
+ WT.evaluate = evaluate
125
+ if int(os.environ.get("RANK", "0")) == 0:
126
+ print(f"HB_INLOOP fastwam installed: every {C.EVERY} steps", flush=True)
b8-table-wipe/b512/step-002000/code/test_fastwam_inputs.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU check (no model): the FastWAM adapter's inputs built from the model-agnostic eval cache == the inputs our FastWAM
2
+ training dataset (RobotVideoDataset of the run's config, no HB augmentation) builds for the same frame.
3
+ python test_fastwam_inputs.py <task> <run out dir> [n_queries]"""
4
+ import json
5
+ import os
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+
11
+ HERE = Path(__file__).resolve().parent
12
+ sys.path.insert(0, str(HERE))
13
+ sys.path.insert(0, str(HERE / "adapters"))
14
+ import hb_eval_common as E # noqa: E402
15
+ import ad_fastwam as A # noqa: E402
16
+
17
+ task, out = sys.argv[1], Path(sys.argv[2])
18
+ nq = int(sys.argv[3]) if len(sys.argv) > 3 else 3
19
+ A.setup_env()
20
+ import torch # noqa: E402
21
+ from omegaconf import OmegaConf # noqa: E402
22
+ import fastwam.runtime as native # noqa: E402
23
+ from fastwam.utils import misc # noqa: E402
24
+
25
+ cfg = OmegaConf.load(out / "config.yaml")
26
+ inp = A.Inputs(cfg, out / "dataset_stats.json")
27
+ misc.register_work_dir("/tmp/hbfw-inputs-check")
28
+ OmegaConf.update(cfg, "data.train.pretrained_norm_stats", str(out / "dataset_stats.json"), merge=False, force_add=True)
29
+ tr, _ = native.build_datasets(cfg.data)
30
+ ddir = Path(cfg.data.train.dataset_dirs[0])
31
+ lens = [json.loads(l)["length"] for l in (ddir / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
32
+ meta = E.load_meta(task, "train10")
33
+ split = "train10"
34
+ worst = {}
35
+ done = 0
36
+ for ep, z in E.iter_episodes(task, split, None):
37
+ base = int(sum(lens[:ep]))
38
+ for qi, t in enumerate(list(z["query_t"])[1:1 + nq], start=1):
39
+ t = int(t)
40
+ s = tr[base + t]
41
+ obs = {"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]}, "state": z["state"][t],
42
+ "prompt": meta["prompts"][str(int(z["task_index"]))]}
43
+ img = inp.image(obs["images"])[0]
44
+ d_img = float((img - s["video"][:, 0].float()).abs().max())
45
+ pr = inp.proprio(obs["state"])[0]
46
+ d_pr = float((pr - s["proprio"][0].float()).abs().max())
47
+ ctx, mask = inp.context(obs["prompt"])
48
+ d_ctx = float((ctx.float() - s["context"].float()).abs().max())
49
+ d_mask = int((mask != s["context_mask"].bool()).sum())
50
+ gt = z["action"][t:t + 16]
51
+ a = inp.denorm(s["action"][:16])
52
+ d_act = float(np.abs(a[:len(gt)] - gt).max())
53
+ row = dict(ep=ep, t=t, idx=base + t, img=d_img, proprio=d_pr, ctx=d_ctx, mask=d_mask, act_denorm_vs_cache=d_act,
54
+ prompt_same=s["prompt"] == inp.DEFAULT_PROMPT.format(task=obs["prompt"]))
55
+ print("CHECK", json.dumps(row), flush=True)
56
+ for k in ("img", "proprio", "ctx", "act_denorm_vs_cache"):
57
+ worst[k] = max(worst.get(k, 0.0), row[k])
58
+ done += 1
59
+ if done >= 2 * nq:
60
+ break
61
+ print("WORST", json.dumps(worst))
b8-table-wipe/b512/step-002000/complete.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "b8",
3
+ "profile": "g1",
4
+ "per_gpu_batch": 16,
5
+ "global_batch": 512,
6
+ "gradient_accumulation_steps": 1,
7
+ "max_steps": 4000,
8
+ "save_every": 2000,
9
+ "recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
10
+ "interface": {
11
+ "profile": "g1",
12
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
13
+ "cameras": [
14
+ "ego_view",
15
+ "chest"
16
+ ],
17
+ "state_dim": 44,
18
+ "action_dim": 76,
19
+ "episodes": 100,
20
+ "frames": 61498,
21
+ "fps": 30,
22
+ "first_episodes_only": null
23
+ },
24
+ "finished_utc": "2026-11-17T21:03:31Z"
25
+ }
b8-table-wipe/b512/step-002000/dataset_meta/hb_fastwam_interface.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "profile": "g1",
3
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
4
+ "cameras": [
5
+ "ego_view",
6
+ "chest"
7
+ ],
8
+ "state_dim": 44,
9
+ "action_dim": 76,
10
+ "episodes": 100,
11
+ "frames": 61498,
12
+ "fps": 30,
13
+ "first_episodes_only": null
14
+ }
b8-table-wipe/b512/step-002000/dataset_meta/info.json ADDED
@@ -0,0 +1,538 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "codebase_version": "v2.1",
3
+ "robot_type": null,
4
+ "total_episodes": 100,
5
+ "total_frames": 61498,
6
+ "total_tasks": 1,
7
+ "total_videos": 200,
8
+ "total_chunks": 1,
9
+ "chunks_size": 1000,
10
+ "fps": 30,
11
+ "splits": {
12
+ "train": "0:100"
13
+ },
14
+ "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
15
+ "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
16
+ "features": {
17
+ "observation.state": {
18
+ "dtype": "float32",
19
+ "shape": [
20
+ 44
21
+ ],
22
+ "names": null
23
+ },
24
+ "observation.eef_state": {
25
+ "dtype": "float64",
26
+ "shape": [
27
+ 14
28
+ ],
29
+ "names": [
30
+ "left_wrist_pos",
31
+ "left_wrist_abs_quat",
32
+ "right_wrist_pos",
33
+ "right_wrist_abs_quat"
34
+ ]
35
+ },
36
+ "action.wbc": {
37
+ "dtype": "float64",
38
+ "shape": [
39
+ 41
40
+ ],
41
+ "names": [
42
+ "left_hip_pitch_joint",
43
+ "left_hip_roll_joint",
44
+ "left_hip_yaw_joint",
45
+ "left_knee_joint",
46
+ "left_ankle_pitch_joint",
47
+ "left_ankle_roll_joint",
48
+ "right_hip_pitch_joint",
49
+ "right_hip_roll_joint",
50
+ "right_hip_yaw_joint",
51
+ "right_knee_joint",
52
+ "right_ankle_pitch_joint",
53
+ "right_ankle_roll_joint",
54
+ "waist_yaw_joint",
55
+ "waist_roll_joint",
56
+ "waist_pitch_joint",
57
+ "left_shoulder_pitch_joint",
58
+ "left_shoulder_roll_joint",
59
+ "left_shoulder_yaw_joint",
60
+ "left_elbow_joint",
61
+ "left_wrist_roll_joint",
62
+ "left_wrist_pitch_joint",
63
+ "left_wrist_yaw_joint",
64
+ "right_shoulder_pitch_joint",
65
+ "right_shoulder_roll_joint",
66
+ "right_shoulder_yaw_joint",
67
+ "right_elbow_joint",
68
+ "right_wrist_roll_joint",
69
+ "right_wrist_pitch_joint",
70
+ "right_wrist_yaw_joint",
71
+ "left_hand_pinky",
72
+ "left_hand_ring",
73
+ "left_hand_middle",
74
+ "left_hand_index",
75
+ "left_hand_thumb_bend",
76
+ "left_hand_thumb_rotation",
77
+ "right_hand_pinky",
78
+ "right_hand_ring",
79
+ "right_hand_middle",
80
+ "right_hand_index",
81
+ "right_hand_thumb_bend",
82
+ "right_hand_thumb_rotation"
83
+ ]
84
+ },
85
+ "observation.root_orientation": {
86
+ "dtype": "float64",
87
+ "shape": [
88
+ 4
89
+ ],
90
+ "names": [
91
+ "base_qw",
92
+ "base_qx",
93
+ "base_qy",
94
+ "base_qz"
95
+ ]
96
+ },
97
+ "observation.projected_gravity": {
98
+ "dtype": "float64",
99
+ "shape": [
100
+ 3
101
+ ],
102
+ "names": [
103
+ "gravity_x",
104
+ "gravity_y",
105
+ "gravity_z"
106
+ ]
107
+ },
108
+ "observation.cpp_rotation_offset": {
109
+ "dtype": "float64",
110
+ "shape": [
111
+ 4
112
+ ],
113
+ "names": [
114
+ "rot_offset_qw",
115
+ "rot_offset_qx",
116
+ "rot_offset_qy",
117
+ "rot_offset_qz"
118
+ ]
119
+ },
120
+ "observation.init_base_quat": {
121
+ "dtype": "float64",
122
+ "shape": [
123
+ 4
124
+ ],
125
+ "names": [
126
+ "init_base_qw",
127
+ "init_base_qx",
128
+ "init_base_qy",
129
+ "init_base_qz"
130
+ ]
131
+ },
132
+ "teleop.delta_heading": {
133
+ "dtype": "float64",
134
+ "shape": [
135
+ 1
136
+ ],
137
+ "names": [
138
+ "delta_heading"
139
+ ]
140
+ },
141
+ "action.motion_token": {
142
+ "dtype": "float64",
143
+ "shape": [
144
+ 64
145
+ ],
146
+ "names": "motion_token"
147
+ },
148
+ "teleop.smpl_joints": {
149
+ "dtype": "float32",
150
+ "shape": [
151
+ 72
152
+ ],
153
+ "names": "smpl_joints"
154
+ },
155
+ "teleop.smpl_pose": {
156
+ "dtype": "float32",
157
+ "shape": [
158
+ 63
159
+ ],
160
+ "names": "smpl_pose"
161
+ },
162
+ "teleop.body_quat_w": {
163
+ "dtype": "float32",
164
+ "shape": [
165
+ 4
166
+ ],
167
+ "names": "body_quat_w"
168
+ },
169
+ "teleop.target_body_orientation": {
170
+ "dtype": "float32",
171
+ "shape": [
172
+ 6
173
+ ],
174
+ "names": [
175
+ "target_body_r00",
176
+ "target_body_r10",
177
+ "target_body_r01",
178
+ "target_body_r11",
179
+ "target_body_r02",
180
+ "target_body_r12"
181
+ ]
182
+ },
183
+ "teleop.left_hand_joints": {
184
+ "dtype": "float32",
185
+ "shape": [
186
+ 7
187
+ ],
188
+ "names": "left_hand_joints"
189
+ },
190
+ "teleop.right_hand_joints": {
191
+ "dtype": "float32",
192
+ "shape": [
193
+ 7
194
+ ],
195
+ "names": "right_hand_joints"
196
+ },
197
+ "teleop.inspire_left_hand_cmd": {
198
+ "dtype": "float32",
199
+ "shape": [
200
+ 6
201
+ ],
202
+ "names": [
203
+ "left_pinky",
204
+ "left_ring",
205
+ "left_middle",
206
+ "left_index",
207
+ "left_thumb_pitch",
208
+ "left_thumb_yaw"
209
+ ]
210
+ },
211
+ "teleop.inspire_right_hand_cmd": {
212
+ "dtype": "float32",
213
+ "shape": [
214
+ 6
215
+ ],
216
+ "names": [
217
+ "right_pinky",
218
+ "right_ring",
219
+ "right_middle",
220
+ "right_index",
221
+ "right_thumb_pitch",
222
+ "right_thumb_yaw"
223
+ ]
224
+ },
225
+ "teleop.inspire_hand_cmd_age_ms": {
226
+ "dtype": "float32",
227
+ "shape": [
228
+ 1
229
+ ],
230
+ "names": [
231
+ "inspire_hand_cmd_age_ms"
232
+ ]
233
+ },
234
+ "teleop.controller": {
235
+ "dtype": "float32",
236
+ "shape": [
237
+ 4
238
+ ],
239
+ "names": [
240
+ "left_trigger",
241
+ "right_trigger",
242
+ "left_grip",
243
+ "right_grip"
244
+ ]
245
+ },
246
+ "teleop.pose_sample_age_ms": {
247
+ "dtype": "float32",
248
+ "shape": [
249
+ 1
250
+ ],
251
+ "names": [
252
+ "pose_sample_age_ms"
253
+ ]
254
+ },
255
+ "teleop.smpl_frame_index": {
256
+ "dtype": "int64",
257
+ "shape": [
258
+ 1
259
+ ],
260
+ "names": [
261
+ "smpl_frame_index"
262
+ ]
263
+ },
264
+ "teleop.left_wrist_joints": {
265
+ "dtype": "float32",
266
+ "shape": [
267
+ 3
268
+ ],
269
+ "names": [
270
+ "left_wrist_roll",
271
+ "left_wrist_pitch",
272
+ "left_wrist_yaw"
273
+ ]
274
+ },
275
+ "teleop.right_wrist_joints": {
276
+ "dtype": "float32",
277
+ "shape": [
278
+ 3
279
+ ],
280
+ "names": [
281
+ "right_wrist_roll",
282
+ "right_wrist_pitch",
283
+ "right_wrist_yaw"
284
+ ]
285
+ },
286
+ "teleop.stream_mode": {
287
+ "dtype": "int32",
288
+ "shape": [
289
+ 1
290
+ ],
291
+ "names": [
292
+ "stream_mode"
293
+ ]
294
+ },
295
+ "teleop.planner_mode": {
296
+ "dtype": "int32",
297
+ "shape": [
298
+ 1
299
+ ],
300
+ "names": [
301
+ "locomotion_mode"
302
+ ]
303
+ },
304
+ "teleop.planner_movement": {
305
+ "dtype": "float32",
306
+ "shape": [
307
+ 3
308
+ ],
309
+ "names": [
310
+ "movement_x",
311
+ "movement_y",
312
+ "movement_z"
313
+ ]
314
+ },
315
+ "teleop.planner_facing": {
316
+ "dtype": "float32",
317
+ "shape": [
318
+ 3
319
+ ],
320
+ "names": [
321
+ "facing_x",
322
+ "facing_y",
323
+ "facing_z"
324
+ ]
325
+ },
326
+ "teleop.planner_speed": {
327
+ "dtype": "float32",
328
+ "shape": [
329
+ 1
330
+ ],
331
+ "names": [
332
+ "speed"
333
+ ]
334
+ },
335
+ "teleop.planner_height": {
336
+ "dtype": "float32",
337
+ "shape": [
338
+ 1
339
+ ],
340
+ "names": [
341
+ "height"
342
+ ]
343
+ },
344
+ "teleop.vr_3pt_position": {
345
+ "dtype": "float32",
346
+ "shape": [
347
+ 9
348
+ ],
349
+ "names": [
350
+ "lwrist_x",
351
+ "lwrist_y",
352
+ "lwrist_z",
353
+ "rwrist_x",
354
+ "rwrist_y",
355
+ "rwrist_z",
356
+ "neck_x",
357
+ "neck_y",
358
+ "neck_z"
359
+ ]
360
+ },
361
+ "teleop.vr_3pt_orientation": {
362
+ "dtype": "float32",
363
+ "shape": [
364
+ 18
365
+ ],
366
+ "names": [
367
+ "lwrist_r00",
368
+ "lwrist_r10",
369
+ "lwrist_r01",
370
+ "lwrist_r11",
371
+ "lwrist_r02",
372
+ "lwrist_r12",
373
+ "rwrist_r00",
374
+ "rwrist_r10",
375
+ "rwrist_r01",
376
+ "rwrist_r11",
377
+ "rwrist_r02",
378
+ "rwrist_r12",
379
+ "neck_r00",
380
+ "neck_r10",
381
+ "neck_r01",
382
+ "neck_r11",
383
+ "neck_r02",
384
+ "neck_r12"
385
+ ]
386
+ },
387
+ "observation.images.ego_view": {
388
+ "dtype": "video",
389
+ "shape": [
390
+ 480,
391
+ 640,
392
+ 3
393
+ ],
394
+ "names": [
395
+ "height",
396
+ "width",
397
+ "channel"
398
+ ],
399
+ "info": {
400
+ "video.height": 480,
401
+ "video.width": 640,
402
+ "video.codec": "h264",
403
+ "video.pix_fmt": "yuv420p",
404
+ "video.is_depth_map": false,
405
+ "video.fps": 30,
406
+ "video.channels": 3,
407
+ "has_audio": false
408
+ }
409
+ },
410
+ "observation.root_position": {
411
+ "dtype": "float64",
412
+ "shape": [
413
+ 3
414
+ ],
415
+ "names": [
416
+ "base_x",
417
+ "base_y",
418
+ "base_z"
419
+ ]
420
+ },
421
+ "observation.inspire_left_hand_state": {
422
+ "dtype": "float32",
423
+ "shape": [
424
+ 6
425
+ ],
426
+ "names": [
427
+ "left_pinky",
428
+ "left_ring",
429
+ "left_middle",
430
+ "left_index",
431
+ "left_thumb_bend",
432
+ "left_thumb_rotation"
433
+ ]
434
+ },
435
+ "observation.inspire_right_hand_state": {
436
+ "dtype": "float32",
437
+ "shape": [
438
+ 6
439
+ ],
440
+ "names": [
441
+ "right_pinky",
442
+ "right_ring",
443
+ "right_middle",
444
+ "right_index",
445
+ "right_thumb_bend",
446
+ "right_thumb_rotation"
447
+ ]
448
+ },
449
+ "observation.images.chest": {
450
+ "dtype": "video",
451
+ "shape": [
452
+ 480,
453
+ 640,
454
+ 3
455
+ ],
456
+ "names": [
457
+ "height",
458
+ "width",
459
+ "channel"
460
+ ],
461
+ "info": {
462
+ "video.height": 480,
463
+ "video.width": 640,
464
+ "video.codec": "h264",
465
+ "video.pix_fmt": "yuv420p",
466
+ "video.is_depth_map": false,
467
+ "video.fps": 30,
468
+ "video.channels": 3,
469
+ "has_audio": false
470
+ }
471
+ },
472
+ "timestamp": {
473
+ "dtype": "float32",
474
+ "shape": [
475
+ 1
476
+ ],
477
+ "names": null
478
+ },
479
+ "frame_index": {
480
+ "dtype": "int64",
481
+ "shape": [
482
+ 1
483
+ ],
484
+ "names": null
485
+ },
486
+ "episode_index": {
487
+ "dtype": "int64",
488
+ "shape": [
489
+ 1
490
+ ],
491
+ "names": null
492
+ },
493
+ "index": {
494
+ "dtype": "int64",
495
+ "shape": [
496
+ 1
497
+ ],
498
+ "names": null
499
+ },
500
+ "task_index": {
501
+ "dtype": "int64",
502
+ "shape": [
503
+ 1
504
+ ],
505
+ "names": null
506
+ },
507
+ "action": {
508
+ "dtype": "float32",
509
+ "shape": [
510
+ 76
511
+ ],
512
+ "names": null
513
+ }
514
+ },
515
+ "script_config": {
516
+ "source": "sim",
517
+ "simulator": "newton",
518
+ "controller": "SONIC g1_deploy_onnx_ref in sim lockstep (planner and upper-body commands, no PICO stream)",
519
+ "exporter": "humanoid_newton.collect.lerobot_export",
520
+ "control_frequency": 50.0,
521
+ "record_wrist_cameras": false,
522
+ "record_chest_camera": true,
523
+ "rotate_chest_camera_180": false,
524
+ "record_depth": false,
525
+ "record_root_position": true,
526
+ "record_inspire_hand_state": true,
527
+ "record_inspire_hand_cmd": true,
528
+ "no_camera": false,
529
+ "camera_backend": "offline_render",
530
+ "camera_note": "ego_view and chest rendered offline (humanoid_newton.rendering.splat_render) from the recorded body poses at the registered D455 mounts; chest rendered upright (the real exporter rotates its upside-down chest stream by 180 deg); cameras not calibrated against the real ones",
531
+ "task": "b8",
532
+ "collect_config": "20261114_black_coffee_training_100_01",
533
+ "scene": "eai_room_fridge_001",
534
+ "task_description": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it.",
535
+ "git_commit": null
536
+ },
537
+ "discarded_episode_indices": []
538
+ }
b8-table-wipe/b512/step-002000/dataset_meta/tasks.jsonl ADDED
@@ -0,0 +1 @@
 
 
1
+ {"task_index": 0, "task": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it."}
b8-table-wipe/b512/step-002000/dataset_stats.json ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-002000/logs/node-0.log ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-002000/logs/node-1.log ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-002000/logs/node-2.log ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-002000/logs/node-3.log ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-002000/loss_summary.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "points": 200,
3
+ "first": {
4
+ "epoch": 0,
5
+ "step": 20,
6
+ "loss": 1.3127,
7
+ "loss_action": 1.0909,
8
+ "loss_video": 0.2218,
9
+ "lr": 1.05e-05,
10
+ "steps_per_s": 0.17
11
+ },
12
+ "last": {
13
+ "epoch": 33,
14
+ "step": 4000,
15
+ "loss": 0.0571,
16
+ "loss_action": 0.0106,
17
+ "loss_video": 0.0464,
18
+ "lr": 1e-06,
19
+ "steps_per_s": 0.24
20
+ },
21
+ "min_loss": 0.0537,
22
+ "mean_steps_per_s": 0.23805000000000018
23
+ }
b8-table-wipe/b512/step-002000/metrics.csv ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ step,epoch,loss,loss_action,loss_video,lr,steps_per_s
2
+ 20,0,1.3127,1.0909,0.2218,1.05e-05,0.17
3
+ 40,0,0.8771,0.6803,0.1968,2.04e-05,0.19
4
+ 60,0,0.4879,0.322,0.166,3.04e-05,0.21
5
+ 80,0,0.347,0.177,0.17,4.03e-05,0.22
6
+ 100,0,0.3127,0.1505,0.1622,5.02e-05,0.21
7
+ 120,0,0.2829,0.1298,0.1531,6.02e-05,0.23
8
+ 140,1,0.2545,0.1109,0.1436,7.01e-05,0.22
9
+ 160,1,0.2666,0.1202,0.1464,8.01e-05,0.22
10
+ 180,1,0.2397,0.1086,0.1311,9e-05,0.22
11
+ 200,1,0.234,0.0977,0.1363,0.0001,0.23
12
+ 220,1,0.2251,0.0856,0.1396,0.0001,0.23
13
+ 240,1,0.2422,0.1015,0.1407,0.0001,0.24
14
+ 260,2,0.1952,0.0765,0.1187,9.99e-05,0.23
15
+ 280,2,0.199,0.0713,0.1276,9.99e-05,0.23
16
+ 300,2,0.1867,0.0643,0.1224,9.98e-05,0.23
17
+ 320,2,0.1877,0.0668,0.1209,9.98e-05,0.23
18
+ 340,2,0.183,0.0612,0.1218,9.97e-05,0.23
19
+ 360,2,0.1771,0.0568,0.1203,9.96e-05,0.24
20
+ 380,3,0.1749,0.0618,0.1131,9.95e-05,0.23
21
+ 400,3,0.1734,0.0591,0.1143,9.93e-05,0.23
22
+ 420,3,0.1651,0.0536,0.1116,9.92e-05,0.23
23
+ 440,3,0.159,0.0489,0.1101,9.9e-05,0.23
24
+ 460,3,0.1598,0.0499,0.1099,9.89e-05,0.24
25
+ 480,3,0.1661,0.051,0.1151,9.87e-05,0.24
26
+ 500,4,0.1518,0.0488,0.103,9.85e-05,0.23
27
+ 520,4,0.1496,0.0463,0.1033,9.83e-05,0.24
28
+ 540,4,0.1465,0.0463,0.1002,9.81e-05,0.24
29
+ 560,4,0.1424,0.0436,0.0988,9.78e-05,0.24
30
+ 580,4,0.1442,0.0449,0.0992,9.76e-05,0.24
31
+ 600,4,0.1472,0.044,0.1033,9.73e-05,0.24
32
+ 620,5,0.1411,0.0398,0.1013,9.7e-05,0.24
33
+ 640,5,0.1367,0.0382,0.0985,9.68e-05,0.24
34
+ 660,5,0.1318,0.0387,0.0931,9.65e-05,0.24
35
+ 680,5,0.1309,0.0381,0.0928,9.62e-05,0.24
36
+ 700,5,0.1304,0.0387,0.0917,9.58e-05,0.24
37
+ 720,5,0.1277,0.037,0.0907,9.55e-05,0.24
38
+ 740,6,0.1294,0.0362,0.0932,9.51e-05,0.24
39
+ 760,6,0.139,0.0362,0.1028,9.48e-05,0.24
40
+ 780,6,0.1241,0.037,0.087,9.44e-05,0.24
41
+ 800,6,0.1339,0.034,0.0999,9.4e-05,0.24
42
+ 820,6,0.1335,0.0355,0.098,9.36e-05,0.24
43
+ 840,6,0.1203,0.0308,0.0895,9.32e-05,0.24
44
+ 860,7,0.1244,0.0317,0.0927,9.28e-05,0.24
45
+ 880,7,0.1211,0.031,0.0901,9.24e-05,0.24
46
+ 900,7,0.1234,0.0312,0.0922,9.19e-05,0.24
47
+ 920,7,0.1253,0.0325,0.0929,9.15e-05,0.24
48
+ 940,7,0.1186,0.0314,0.0872,9.1e-05,0.24
49
+ 960,7,0.1199,0.0294,0.0905,9.05e-05,0.24
50
+ 980,8,0.1223,0.0291,0.0932,9.01e-05,0.24
51
+ 1000,8,0.116,0.0301,0.0859,8.96e-05,0.24
52
+ 1020,8,0.1206,0.0287,0.0918,8.91e-05,0.24
53
+ 1040,8,0.113,0.0297,0.0833,8.85e-05,0.24
54
+ 1060,8,0.1227,0.0306,0.0921,8.8e-05,0.24
55
+ 1080,8,0.1087,0.028,0.0808,8.75e-05,0.24
56
+ 1100,9,0.1058,0.0261,0.0797,8.69e-05,0.24
57
+ 1120,9,0.1183,0.0283,0.09,8.64e-05,0.24
58
+ 1140,9,0.114,0.0267,0.0874,8.58e-05,0.24
59
+ 1160,9,0.1045,0.0259,0.0787,8.52e-05,0.24
60
+ 1180,9,0.1136,0.0271,0.0865,8.46e-05,0.24
61
+ 1200,9,0.1146,0.0255,0.0891,8.4e-05,0.24
62
+ 1220,10,0.107,0.025,0.082,8.34e-05,0.24
63
+ 1240,10,0.1016,0.0253,0.0762,8.28e-05,0.24
64
+ 1260,10,0.1024,0.0254,0.077,8.22e-05,0.24
65
+ 1280,10,0.1036,0.0243,0.0793,8.15e-05,0.24
66
+ 1300,10,0.102,0.0233,0.0786,8.09e-05,0.24
67
+ 1320,10,0.1128,0.0247,0.0881,8.03e-05,0.24
68
+ 1340,11,0.1055,0.0226,0.0829,7.96e-05,0.24
69
+ 1360,11,0.1021,0.0237,0.0784,7.89e-05,0.24
70
+ 1380,11,0.1007,0.0226,0.0782,7.83e-05,0.24
71
+ 1400,11,0.1028,0.0226,0.0801,7.76e-05,0.24
72
+ 1420,11,0.0976,0.0222,0.0753,7.69e-05,0.24
73
+ 1440,11,0.097,0.0213,0.0757,7.62e-05,0.24
74
+ 1460,12,0.0982,0.0215,0.0768,7.55e-05,0.24
75
+ 1480,12,0.0912,0.0216,0.0696,7.48e-05,0.24
76
+ 1500,12,0.0964,0.0217,0.0748,7.41e-05,0.24
77
+ 1520,12,0.1016,0.022,0.0796,7.33e-05,0.24
78
+ 1540,12,0.0943,0.022,0.0723,7.26e-05,0.24
79
+ 1560,12,0.0954,0.0211,0.0743,7.19e-05,0.24
80
+ 1580,13,0.0894,0.0208,0.0687,7.11e-05,0.24
81
+ 1600,13,0.0997,0.0208,0.079,7.04e-05,0.24
82
+ 1620,13,0.087,0.0208,0.0661,6.96e-05,0.24
83
+ 1640,13,0.0924,0.0201,0.0723,6.89e-05,0.24
84
+ 1660,13,0.0944,0.0181,0.0763,6.81e-05,0.24
85
+ 1680,13,0.0972,0.0208,0.0764,6.73e-05,0.24
86
+ 1700,14,0.0931,0.0198,0.0733,6.66e-05,0.24
87
+ 1720,14,0.0829,0.0219,0.061,6.58e-05,0.24
88
+ 1740,14,0.094,0.0193,0.0747,6.5e-05,0.24
89
+ 1760,14,0.0822,0.0194,0.0628,6.42e-05,0.24
90
+ 1780,14,0.0886,0.0182,0.0704,6.34e-05,0.24
91
+ 1800,14,0.0876,0.0195,0.0681,6.27e-05,0.24
92
+ 1820,15,0.0886,0.0195,0.0691,6.19e-05,0.24
93
+ 1840,15,0.0875,0.0193,0.0682,6.11e-05,0.24
94
+ 1860,15,0.0911,0.0195,0.0716,6.03e-05,0.24
95
+ 1880,15,0.0873,0.0178,0.0694,5.95e-05,0.24
96
+ 1900,15,0.0843,0.0181,0.0661,5.86e-05,0.24
97
+ 1920,15,0.0845,0.0174,0.0671,5.78e-05,0.24
98
+ 1940,16,0.0922,0.017,0.0752,5.7e-05,0.24
99
+ 1960,16,0.0852,0.0175,0.0677,5.62e-05,0.24
100
+ 1980,16,0.0851,0.0174,0.0677,5.54e-05,0.24
101
+ 2000,16,0.0816,0.0165,0.065,5.46e-05,0.24
102
+ 2020,16,0.0812,0.017,0.0642,5.38e-05,0.24
103
+ 2040,16,0.0808,0.0164,0.0644,5.3e-05,0.24
104
+ 2060,17,0.0827,0.0174,0.0653,5.21e-05,0.24
105
+ 2080,17,0.0786,0.0179,0.0607,5.13e-05,0.24
106
+ 2100,17,0.0812,0.0165,0.0646,5.05e-05,0.24
107
+ 2120,17,0.083,0.0171,0.0659,4.97e-05,0.24
108
+ 2140,17,0.0738,0.0146,0.0592,4.89e-05,0.24
109
+ 2160,17,0.0777,0.0148,0.0629,4.8e-05,0.24
110
+ 2180,18,0.0788,0.015,0.0638,4.72e-05,0.24
111
+ 2200,18,0.0743,0.0155,0.0588,4.64e-05,0.24
112
+ 2220,18,0.0783,0.0165,0.0618,4.56e-05,0.24
113
+ 2240,18,0.0784,0.014,0.0644,4.48e-05,0.24
114
+ 2260,18,0.0772,0.0146,0.0625,4.4e-05,0.24
115
+ 2280,18,0.0684,0.0142,0.0542,4.32e-05,0.24
116
+ 2300,19,0.0779,0.0162,0.0616,4.24e-05,0.24
117
+ 2320,19,0.0778,0.0161,0.0617,4.15e-05,0.24
118
+ 2340,19,0.074,0.0147,0.0593,4.07e-05,0.24
119
+ 2360,19,0.0693,0.0153,0.054,3.99e-05,0.24
120
+ 2380,19,0.0731,0.0153,0.0578,3.91e-05,0.24
121
+ 2400,19,0.0709,0.0149,0.056,3.83e-05,0.24
122
+ 2420,19,0.0763,0.0144,0.0619,3.76e-05,0.24
123
+ 2440,20,0.075,0.0149,0.06,3.68e-05,0.24
124
+ 2460,20,0.0674,0.0148,0.0526,3.6e-05,0.24
125
+ 2480,20,0.0709,0.0146,0.0563,3.52e-05,0.24
126
+ 2500,20,0.0695,0.015,0.0545,3.44e-05,0.24
127
+ 2520,20,0.0675,0.0131,0.0544,3.37e-05,0.24
128
+ 2540,20,0.0689,0.0141,0.0548,3.29e-05,0.24
129
+ 2560,21,0.0779,0.0142,0.0637,3.21e-05,0.24
130
+ 2580,21,0.0674,0.0142,0.0532,3.14e-05,0.24
131
+ 2600,21,0.0652,0.013,0.0522,3.06e-05,0.24
132
+ 2620,21,0.0612,0.0128,0.0484,2.99e-05,0.24
133
+ 2640,21,0.0654,0.0132,0.0522,2.91e-05,0.24
134
+ 2660,21,0.0649,0.013,0.0519,2.84e-05,0.24
135
+ 2680,22,0.0729,0.0114,0.0615,2.77e-05,0.24
136
+ 2700,22,0.0673,0.0129,0.0544,2.69e-05,0.24
137
+ 2720,22,0.0703,0.0121,0.0582,2.62e-05,0.24
138
+ 2740,22,0.0692,0.0129,0.0563,2.55e-05,0.24
139
+ 2760,22,0.0677,0.0131,0.0546,2.48e-05,0.24
140
+ 2780,22,0.0691,0.0132,0.0559,2.41e-05,0.24
141
+ 2800,23,0.0628,0.0111,0.0517,2.34e-05,0.24
142
+ 2820,23,0.0673,0.013,0.0542,2.27e-05,0.24
143
+ 2840,23,0.0662,0.0133,0.0529,2.21e-05,0.24
144
+ 2860,23,0.0622,0.0119,0.0502,2.14e-05,0.24
145
+ 2880,23,0.0618,0.0111,0.0506,2.07e-05,0.24
146
+ 2900,23,0.0651,0.0124,0.0527,2.01e-05,0.24
147
+ 2920,24,0.0574,0.0111,0.0463,1.95e-05,0.24
148
+ 2940,24,0.0627,0.012,0.0507,1.88e-05,0.24
149
+ 2960,24,0.0669,0.0125,0.0544,1.82e-05,0.24
150
+ 2980,24,0.0628,0.0114,0.0515,1.76e-05,0.24
151
+ 3000,24,0.0645,0.0116,0.0529,1.7e-05,0.24
152
+ 3020,24,0.0667,0.0115,0.0552,1.64e-05,0.24
153
+ 3040,25,0.0588,0.0116,0.0472,1.58e-05,0.24
154
+ 3060,25,0.0645,0.0132,0.0513,1.52e-05,0.24
155
+ 3080,25,0.0582,0.0112,0.047,1.46e-05,0.24
156
+ 3100,25,0.0616,0.0125,0.0491,1.41e-05,0.24
157
+ 3120,25,0.0585,0.0107,0.0478,1.35e-05,0.24
158
+ 3140,25,0.0618,0.0111,0.0507,1.3e-05,0.24
159
+ 3160,26,0.0621,0.0119,0.0502,1.25e-05,0.24
160
+ 3180,26,0.0566,0.0108,0.0458,1.19e-05,0.24
161
+ 3200,26,0.0631,0.0119,0.0512,1.14e-05,0.24
162
+ 3220,26,0.058,0.0111,0.0469,1.09e-05,0.24
163
+ 3240,26,0.0636,0.0106,0.053,1.05e-05,0.24
164
+ 3260,26,0.0641,0.0118,0.0523,9.98e-06,0.24
165
+ 3280,27,0.0631,0.0113,0.0518,9.51e-06,0.24
166
+ 3300,27,0.0639,0.011,0.0529,9.06e-06,0.24
167
+ 3320,27,0.0589,0.0109,0.0479,8.62e-06,0.24
168
+ 3340,27,0.0616,0.011,0.0507,8.19e-06,0.24
169
+ 3360,27,0.0599,0.0115,0.0484,7.77e-06,0.24
170
+ 3380,27,0.0606,0.0117,0.0489,7.36e-06,0.24
171
+ 3400,28,0.058,0.0114,0.0466,6.97e-06,0.24
172
+ 3420,28,0.0613,0.0112,0.0501,6.58e-06,0.24
173
+ 3440,28,0.0595,0.0101,0.0494,6.21e-06,0.24
174
+ 3460,28,0.0573,0.011,0.0463,5.85e-06,0.24
175
+ 3480,28,0.0616,0.0101,0.0515,5.5e-06,0.24
176
+ 3500,28,0.0566,0.0107,0.0459,5.17e-06,0.24
177
+ 3520,29,0.0622,0.0111,0.0512,4.85e-06,0.24
178
+ 3540,29,0.0565,0.0107,0.0458,4.54e-06,0.24
179
+ 3560,29,0.0573,0.0103,0.0469,4.24e-06,0.24
180
+ 3580,29,0.058,0.0108,0.0471,3.95e-06,0.24
181
+ 3600,29,0.0575,0.0104,0.0471,3.68e-06,0.24
182
+ 3620,29,0.0574,0.0098,0.0476,3.42e-06,0.24
183
+ 3640,30,0.055,0.0094,0.0456,3.18e-06,0.24
184
+ 3660,30,0.0624,0.0088,0.0535,2.94e-06,0.24
185
+ 3680,30,0.0546,0.0096,0.045,2.72e-06,0.24
186
+ 3700,30,0.0562,0.0091,0.0471,2.51e-06,0.24
187
+ 3720,30,0.0595,0.0102,0.0494,2.32e-06,0.24
188
+ 3740,30,0.0562,0.0101,0.0461,2.14e-06,0.24
189
+ 3760,31,0.0551,0.0104,0.0447,1.97e-06,0.24
190
+ 3780,31,0.0626,0.0091,0.0535,1.82e-06,0.24
191
+ 3800,31,0.0558,0.0096,0.0462,1.68e-06,0.24
192
+ 3820,31,0.0537,0.0105,0.0431,1.55e-06,0.24
193
+ 3840,31,0.0616,0.0109,0.0507,1.43e-06,0.24
194
+ 3860,31,0.0552,0.01,0.0452,1.33e-06,0.24
195
+ 3880,32,0.0553,0.0097,0.0456,1.24e-06,0.24
196
+ 3900,32,0.0578,0.0107,0.0471,1.17e-06,0.24
197
+ 3920,32,0.0566,0.0097,0.0469,1.11e-06,0.24
198
+ 3940,32,0.0565,0.0101,0.0464,1.06e-06,0.24
199
+ 3960,32,0.0588,0.01,0.0488,1.03e-06,0.24
200
+ 3980,32,0.0547,0.0096,0.0451,1.01e-06,0.24
201
+ 4000,33,0.0571,0.0106,0.0464,1e-06,0.24
b8-table-wipe/b512/step-002000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1b463083f4a14c71f11dc197c546d7f7d81d5a0330c8a67fda8796635d126e6
3
+ size 12042313337
b8-table-wipe/b512/step-002000/model_config.yaml ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _target_: fastwam.runtime.create_fastwam
2
+ model_id: Wan-AI/Wan2.2-TI2V-5B
3
+ tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
4
+ tokenizer_max_len: 128
5
+ load_text_encoder: false
6
+ proprio_dim: 44
7
+ redirect_common_files: false
8
+ mot_checkpoint_mixed_attn: false
9
+ compile_training_denoise: false
10
+ action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt
11
+ skip_dit_load_from_pretrain: false
12
+ video_dit_config:
13
+ has_image_input: false
14
+ patch_size:
15
+ - 1
16
+ - 2
17
+ - 2
18
+ in_dim: 48
19
+ hidden_dim: 3072
20
+ ffn_dim: 14336
21
+ freq_dim: 256
22
+ text_dim: 4096
23
+ out_dim: 48
24
+ num_heads: 24
25
+ attn_head_dim: 128
26
+ num_layers: 30
27
+ eps: 1.0e-06
28
+ seperated_timestep: true
29
+ require_clip_embedding: false
30
+ require_vae_embedding: false
31
+ fuse_vae_embedding_in_latents: true
32
+ use_gradient_checkpointing: false
33
+ video_attention_mask_mode: first_frame_causal
34
+ action_conditioned: false
35
+ action_dim: 76
36
+ action_group_causal_mask_mode: group_diagonal
37
+ action_dit_config:
38
+ action_dim: 76
39
+ hidden_dim: 1024
40
+ ffn_dim: 4096
41
+ num_heads: 24
42
+ attn_head_dim: 128
43
+ num_layers: 30
44
+ text_dim: 4096
45
+ freq_dim: 256
46
+ eps: 1.0e-06
47
+ use_gradient_checkpointing: false
48
+ video_scheduler:
49
+ train_shift: 5.0
50
+ infer_shift: 5.0
51
+ num_train_timesteps: 1000
52
+ action_scheduler:
53
+ train_shift: 1.0
54
+ infer_shift: 1.0
55
+ num_train_timesteps: 1000
56
+ loss:
57
+ lambda_action: 1.0
b8-table-wipe/b512/step-002000/protocol.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "FastWAM (Wan2.2-TI2V-5B video expert + 1.02 B action expert, mixture-of-transformers world action model; upstream github.com/yuantianyuan01/FastWAM)",
3
+ "task": "B8 table wipe (G1)",
4
+ "task_dir": "b8-table-wipe/b512",
5
+ "robot": "Unitree G1",
6
+ "training_data": "b8-table-wipe-expert-v1 @3b2e618",
7
+ "episodes": 100,
8
+ "epoch_completed": null,
9
+ "global_step": 2000,
10
+ "augmentation": "Humanoid Benchmark online augmentation spec v2 (2026-11-09): 15 visual effects, one effect per training sample at 1.5x; proprioception aligned with the GR00T A20 recipe (2026-11-12): per-sample joint bias +-2 deg + Gaussian noise 0.25 deg (hb_aug_v2.perturb_joints), no proprio delay, no history drop; model-native image augmentation off",
11
+ "epochs": null,
12
+ "total_steps": 4000,
13
+ "save_steps": [
14
+ 2000,
15
+ 4000
16
+ ],
17
+ "num_gpus": 32,
18
+ "per_gpu_batch": 16,
19
+ "global_batch": 512,
20
+ "gradient_accumulation_steps": 1,
21
+ "frames": 61498,
22
+ "fps": 30,
23
+ "cameras": [
24
+ "ego_view",
25
+ "chest"
26
+ ],
27
+ "state_dim": 44,
28
+ "action_dim": 76,
29
+ "state_layout": "state44 = 29 body joint angles + measured Inspire hands 6 + 6 + projected gravity 3",
30
+ "action_layout": "action76 = action.motion_token (64) + measured Inspire hands (6 + 6), same-row labels",
31
+ "trainable": "full fine-tune of the mixture-of-transformers DiT: video expert (Wan2.2-TI2V-5B, 5.00 B) + action expert (1.02 B); the Wan2.2 VAE and the T5 text encoder are frozen (prompt embeddings precomputed, text_embeddings/)",
32
+ "recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
33
+ "video": "17-frame clips (4 latent frames + the observation frame), 224 x 448 (the two cameras side by side), action chunk 16",
34
+ "base_model": "Wan-AI/Wan2.2-TI2V-5B (DiT, VAE, T5) + the umt5-xxl tokenizer of Wan-AI/Wan2.1-T2V-1.3B, loaded from their own repositories; ActionDiT initialised by FastWAM's preprocess_action_dit_backbone.py",
35
+ "upstream": "github.com/yuantianyuan01/FastWAM @ 7faa71108368fbb3b6885649f112af607427a2d4",
36
+ "batch_selection": "largest per-GPU batch that fits on A800 80 GB: 16 trains at ~58 GB, 32 runs out of memory (batch_benchmark/)",
37
+ "normalization": "dataset_stats.json (FastWAM min / max statistics of this task)",
38
+ "config": "training_config.yaml (resolved FastWAM config; absolute paths refer to the training NAS), model_config.yaml (the model part)",
39
+ "inference": "FastWAM: build the model from model_config.yaml, load model.pt (DiT state dict), the normalizer from dataset_stats.json and the prompt embedding from text_embeddings/; code/hb_fastwam_convert.py documents the state / action layout of the benchmark interface; code/ad_fastwam.py is the inference adapter of our evaluation (action-only FastWAM.infer_action, 10 denoising steps, seed 42; its default paths refer to the training NAS)",
40
+ "training_records_included": true,
41
+ "resume_state_included": false,
42
+ "files": {
43
+ "batch_benchmark/chosen.json": 210,
44
+ "batch_benchmark/summary.txt": 127,
45
+ "code/ad_fastwam.py": 8413,
46
+ "code/fastwam_prep.sh": 3634,
47
+ "code/fastwam_task.sh": 3719,
48
+ "code/hb_aug_v2.py": 7475,
49
+ "code/hb_eval_common.py": 13471,
50
+ "code/hb_fastwam_aug.py": 3345,
51
+ "code/hb_fastwam_convert.py": 7101,
52
+ "code/hb_fastwam_train.py": 5570,
53
+ "code/hb_inloop_core.py": 11206,
54
+ "code/hb_inloop_fastwam.py": 5431,
55
+ "code/test_fastwam_inputs.py": 2724,
56
+ "complete.json": 653,
57
+ "dataset_meta/hb_fastwam_interface.json": 334,
58
+ "dataset_meta/info.json": 11331,
59
+ "dataset_meta/tasks.jsonl": 213,
60
+ "dataset_stats.json": 194885,
61
+ "logs/node-0.log": 571827,
62
+ "logs/node-1.log": 474197,
63
+ "logs/node-2.log": 474507,
64
+ "logs/node-3.log": 474604,
65
+ "loss_summary.json": 383,
66
+ "metrics.csv": 8445,
67
+ "model.pt": 12042313337,
68
+ "model_config.yaml": 1350,
69
+ "text_embeddings/5a20e43a27564ea8978edc7394a96e0c1295e82528d59dba0c4169f3bba9a630.t5_len128.wan22ti2v5b.pt": 12587613,
70
+ "training_config.yaml": 5872
71
+ }
72
+ }
b8-table-wipe/b512/step-002000/text_embeddings/5a20e43a27564ea8978edc7394a96e0c1295e82528d59dba0c4169f3bba9a630.t5_len128.wan22ti2v5b.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a142c83793090d53a27bdb95e939112cf4007a13c751b7b68cfa7e433f6a5e02
3
+ size 12587613
b8-table-wipe/b512/step-002000/training_config.yaml ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ output_dir: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/runs/b8/out
2
+ batch_size: 16
3
+ num_workers: 8
4
+ lr_scheduler_type: cosine
5
+ learning_rate: 0.0001
6
+ num_epochs: 10
7
+ max_steps: 4000
8
+ log_every: 20
9
+ save_every: 2000
10
+ eval_every: 0
11
+ eval_num_inference_steps: 10
12
+ gradient_accumulation_steps: 1
13
+ mixed_precision: bf16
14
+ seed: 42
15
+ max_grad_norm: 1.0
16
+ weight_decay: 0.01
17
+ resume: null
18
+ wandb:
19
+ enabled: false
20
+ workspace: null
21
+ project: fast-wam
22
+ name: train
23
+ group: null
24
+ mode: online
25
+ data:
26
+ train:
27
+ _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
28
+ dataset_dirs:
29
+ - /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/data/b8
30
+ shape_meta:
31
+ images:
32
+ - key: ego_view
33
+ raw_shape:
34
+ - 3
35
+ - 224
36
+ - 224
37
+ shape:
38
+ - 3
39
+ - 224
40
+ - 224
41
+ - key: chest
42
+ raw_shape:
43
+ - 3
44
+ - 224
45
+ - 224
46
+ shape:
47
+ - 3
48
+ - 224
49
+ - 224
50
+ action:
51
+ - key: default
52
+ raw_shape: 76
53
+ shape: 76
54
+ state:
55
+ - key: default
56
+ raw_shape: 44
57
+ shape: 44
58
+ num_frames: 17
59
+ global_sample_stride: 1
60
+ action_video_freq_ratio: 4
61
+ video_size:
62
+ - 224
63
+ - 448
64
+ camera_key: null
65
+ val_set_proportion: 0.0
66
+ is_training_set: true
67
+ skip_padding_as_possible: false
68
+ concat_multi_camera: horizontal
69
+ processor:
70
+ _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor
71
+ shape_meta:
72
+ images:
73
+ - key: ego_view
74
+ raw_shape:
75
+ - 3
76
+ - 224
77
+ - 224
78
+ shape:
79
+ - 3
80
+ - 224
81
+ - 224
82
+ - key: chest
83
+ raw_shape:
84
+ - 3
85
+ - 224
86
+ - 224
87
+ shape:
88
+ - 3
89
+ - 224
90
+ - 224
91
+ action:
92
+ - key: default
93
+ raw_shape: 76
94
+ shape: 76
95
+ state:
96
+ - key: default
97
+ raw_shape: 44
98
+ shape: 44
99
+ num_obs_steps: 17
100
+ num_output_cameras: 2
101
+ action_output_dim: 76
102
+ proprio_output_dim: 44
103
+ delta_action_dim_mask:
104
+ default:
105
+ - false
106
+ - false
107
+ - false
108
+ - false
109
+ - false
110
+ - false
111
+ - false
112
+ - false
113
+ - false
114
+ - false
115
+ - false
116
+ - false
117
+ - false
118
+ - false
119
+ - false
120
+ - false
121
+ - false
122
+ - false
123
+ - false
124
+ - false
125
+ - false
126
+ - false
127
+ - false
128
+ - false
129
+ - false
130
+ - false
131
+ - false
132
+ - false
133
+ - false
134
+ - false
135
+ - false
136
+ - false
137
+ - false
138
+ - false
139
+ - false
140
+ - false
141
+ - false
142
+ - false
143
+ - false
144
+ - false
145
+ - false
146
+ - false
147
+ - false
148
+ - false
149
+ - false
150
+ - false
151
+ - false
152
+ - false
153
+ - false
154
+ - false
155
+ - false
156
+ - false
157
+ - false
158
+ - false
159
+ - false
160
+ - false
161
+ - false
162
+ - false
163
+ - false
164
+ - false
165
+ - false
166
+ - false
167
+ - false
168
+ - false
169
+ - false
170
+ - false
171
+ - false
172
+ - false
173
+ - false
174
+ - false
175
+ - false
176
+ - false
177
+ - false
178
+ - false
179
+ - false
180
+ - false
181
+ action_state_transforms: null
182
+ use_stepwise_action_norm: false
183
+ norm_default_mode: min/max
184
+ norm_exception_mode: null
185
+ action_state_merger:
186
+ _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
187
+ train_transforms:
188
+ - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
189
+ - _target_: torchvision.transforms.Resize
190
+ size:
191
+ - 224
192
+ - 224
193
+ val_transforms:
194
+ - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
195
+ - _target_: torchvision.transforms.Resize
196
+ size:
197
+ - 224
198
+ - 224
199
+ text_embedding_cache_dir: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/text-cache/all
200
+ use_text_embed_cache: true
201
+ context_len: 128
202
+ model:
203
+ _target_: fastwam.runtime.create_fastwam
204
+ model_id: Wan-AI/Wan2.2-TI2V-5B
205
+ tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
206
+ tokenizer_max_len: 128
207
+ load_text_encoder: false
208
+ proprio_dim: 44
209
+ redirect_common_files: false
210
+ mot_checkpoint_mixed_attn: false
211
+ compile_training_denoise: false
212
+ action_dit_pretrained_path: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/action-init-g1.pt
213
+ skip_dit_load_from_pretrain: false
214
+ video_dit_config:
215
+ has_image_input: false
216
+ patch_size:
217
+ - 1
218
+ - 2
219
+ - 2
220
+ in_dim: 48
221
+ hidden_dim: 3072
222
+ ffn_dim: 14336
223
+ freq_dim: 256
224
+ text_dim: 4096
225
+ out_dim: 48
226
+ num_heads: 24
227
+ attn_head_dim: 128
228
+ num_layers: 30
229
+ eps: 1.0e-06
230
+ seperated_timestep: true
231
+ require_clip_embedding: false
232
+ require_vae_embedding: false
233
+ fuse_vae_embedding_in_latents: true
234
+ use_gradient_checkpointing: false
235
+ video_attention_mask_mode: first_frame_causal
236
+ action_conditioned: false
237
+ action_dim: 76
238
+ action_group_causal_mask_mode: group_diagonal
239
+ action_dit_config:
240
+ action_dim: 76
241
+ hidden_dim: 1024
242
+ ffn_dim: 4096
243
+ num_heads: 24
244
+ attn_head_dim: 128
245
+ num_layers: 30
246
+ text_dim: 4096
247
+ freq_dim: 256
248
+ eps: 1.0e-06
249
+ use_gradient_checkpointing: false
250
+ video_scheduler:
251
+ train_shift: 5.0
252
+ infer_shift: 5.0
253
+ num_train_timesteps: 1000
254
+ action_scheduler:
255
+ train_shift: 1.0
256
+ infer_shift: 1.0
257
+ num_train_timesteps: 1000
258
+ loss:
259
+ lambda_action: 1.0
b8-table-wipe/b512/step-004000/batch_benchmark/chosen.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"per_gpu_batch": 16, "global_batch": 512, "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/bench-20261116-r2/summary.txt"}
b8-table-wipe/b512/step-004000/batch_benchmark/summary.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ bs=16 global=512 rc=[0 0 0 0 ] oom=0 peak_mem_MiB=58421 speed=
2
+ bs=32 global=1024 rc=[1 1 1 1 ] oom=4 peak_mem_MiB=81123 speed=
b8-table-wipe/b512/step-004000/code/ad_fastwam.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM adapter (2026-11-16). Deployment path of FastWAM's own policy (experiments/robotwin/fastwam_policy/
2
+ deploy_policy.py, WorldActionRobotWinPolicy._infer_action_chunk: model.infer_action, the action-only path = first-frame
3
+ video KV cache + action-expert denoising, no future video) fed exactly like our training dataset (RobotVideoDataset +
4
+ FastWAMProcessor of the run's config.yaml, no HB augmentation):
5
+ * cameras in shape_meta order (g1 ego_view + chest, h2 head + chest): uint8 [1,C,H,W] -> processor val transforms
6
+ (ToTensor /255 + Resize 224x224), concatenated horizontally (224 x 448), dataset ResizeSmallestSide/CenterCrop
7
+ (no-op at 224 x 448), Normalize(0.5, 0.5) -> [-1, 1];
8
+ * state: processor.action_state_transform -> normalizer.forward (min/max of the run's
9
+ dataset_stats.json), the proprio of the first frame;
10
+ * text: the run's T5 cache for DEFAULT_PROMPT.format(task=instruction) with context[~mask] = 0 and mask = ones, as in
11
+ training (the text encoder is not loaded in training either; deploy_policy encodes the same prompt with T5 online);
12
+ * infer_action(action_horizon = num_frames - 1 = 16, num_inference_steps = config eval_num_inference_steps (10),
13
+ text_cfg_scale 1, seed 42 = native trainer eval), denormalised with normalizer.normalizers["action"][key].backward.
14
+ ckpt = .../fastwam/runs/<task>/out/checkpoints/weights/step_<N>.pt (config.yaml + dataset_stats.json in .../out).
15
+ Weights: model built from the config (Wan2.2 + ActionDiT init), then the checkpoint's mot + proprio_encoder are
16
+ loaded STRICTLY (FastWAM.load_checkpoint uses strict=False for mot; a missing key here is an error).
17
+ """
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+
24
+ import hb_eval_common as E
25
+
26
+ FW = E.G / "fastwam"
27
+
28
+
29
+ def setup_env():
30
+ os.environ.setdefault("FASTWAM_SRC", str(FW / "FastWAM"))
31
+ os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(FW / "models"))
32
+ os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "true")
33
+ os.environ.setdefault("HF_HUB_OFFLINE", "1")
34
+ src = str(Path(os.environ["FASTWAM_SRC"]) / "src")
35
+ if src not in sys.path:
36
+ sys.path.insert(0, src)
37
+
38
+
39
+ class Inputs:
40
+ """model-agnostic cache query -> FastWAM inputs (shared by the post-hoc adapter and the in-training eval)"""
41
+
42
+ def __init__(self, cfg, stats_path):
43
+ setup_env()
44
+ from hydra.utils import instantiate
45
+ from omegaconf import OmegaConf
46
+ from fastwam.datasets.dataset_utils import CenterCrop, Normalize, ResizeSmallestSideAspectPreserving
47
+ from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT
48
+ from fastwam.datasets.lerobot.utils.normalizer import load_dataset_stats_from_json
49
+ d = cfg.data.train
50
+ self.DEFAULT_PROMPT = DEFAULT_PROMPT
51
+ self.proc = instantiate(d.processor).eval()
52
+ self.proc.set_normalizer_from_stats(load_dataset_stats_from_json(str(stats_path)))
53
+ self.cams = [m["key"] for m in self.proc.shape_meta["images"]]
54
+ if d.concat_multi_camera != "horizontal":
55
+ raise ValueError(f"concat_multi_camera {d.concat_multi_camera} not supported")
56
+ vs = list(d.video_size)
57
+ self.resize = ResizeSmallestSideAspectPreserving(args={"img_w": vs[1], "img_h": vs[0]})
58
+ self.crop = CenterCrop(args={"img_w": vs[1], "img_h": vs[0]})
59
+ self.norm = Normalize(args={"mean": 0.5, "std": 0.5})
60
+ self.cache_dir = d.text_embedding_cache_dir
61
+ self.context_len = int(d.context_len)
62
+ self.horizon = int(d.num_frames) - 1
63
+ self.skey = self.proc.shape_meta["state"][0]["key"]
64
+ self.akey = self.proc.shape_meta["action"][0]["key"]
65
+ if len(self.proc.shape_meta["state"]) != 1 or len(self.proc.shape_meta["action"]) != 1:
66
+ raise ValueError("expected one merged state key and one merged action key")
67
+ self._ctx = {}
68
+ self.OmegaConf = OmegaConf
69
+
70
+ def image(self, images):
71
+ import torch
72
+ per = []
73
+ for c in self.cams:
74
+ x = torch.from_numpy(np.ascontiguousarray(images[c])).permute(2, 0, 1).unsqueeze(0) # [1,C,H,W] uint8
75
+ for tr in self.proc.val_transforms:
76
+ x = tr(x)
77
+ per.append(x)
78
+ v = torch.cat(per, dim=-1) # [1, C, 224, 448]
79
+ v = self.norm(self.crop(self.resize(v)))
80
+ return v # [1, 3, H, W] in [-1, 1], float32
81
+
82
+ def proprio(self, state):
83
+ import torch
84
+ b = {"state": {self.skey: torch.as_tensor(np.asarray(state, dtype=np.float32)).unsqueeze(0)}}
85
+ b = self.proc.action_state_transform(b)
86
+ b = self.proc.normalizer.forward(b) # = deploy_policy._normalize_state (one merged key: merger is identity)
87
+ return b["state"][self.skey] # [1, D]
88
+
89
+ def context(self, instruction):
90
+ import hashlib
91
+ import torch
92
+ prompt = self.DEFAULT_PROMPT.format(task=instruction)
93
+ if prompt not in self._ctx:
94
+ h = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
95
+ p = os.path.join(self.cache_dir, f"{h}.t5_len{self.context_len}.wan22ti2v5b.pt")
96
+ if not os.path.exists(p):
97
+ raise FileNotFoundError(f"no T5 cache for the instruction ({p}): {instruction[:80]!r}")
98
+ obj = torch.load(p, map_location="cpu")
99
+ ctx, mask = obj["context"], obj["mask"]
100
+ ctx = ctx.clone()
101
+ ctx[~mask.bool()] = 0.0
102
+ self._ctx[prompt] = (ctx, torch.ones_like(mask.bool()))
103
+ return self._ctx[prompt]
104
+
105
+ def denorm(self, action):
106
+ import torch
107
+ a = action
108
+ if a.ndim == 2:
109
+ a = a.unsqueeze(0)
110
+ n = self.proc.normalizer.normalizers["action"][self.akey]
111
+ return n.backward(a.to(dtype=torch.float32, device="cpu")).numpy()[0]
112
+
113
+
114
+ def infer(model, inp, obs, steps, seed=42):
115
+ import torch
116
+ img = inp.image(obs["images"]).to(device=model.device, dtype=model.torch_dtype)
117
+ ctx, mask = inp.context(obs["prompt"])
118
+ with torch.no_grad():
119
+ out = model.infer_action(prompt=None, input_image=img, action_horizon=inp.horizon, proprio=inp.proprio(obs["state"]),
120
+ context=ctx, context_mask=mask, text_cfg_scale=1.0, num_inference_steps=steps, seed=seed,
121
+ rand_device="cpu", tiled=False)
122
+ return inp.denorm(out["action"])
123
+
124
+
125
+ class Adapter:
126
+ def __init__(self, ckpt, task, split, meta, device="cuda:0", steps=None, seed="42"):
127
+ setup_env()
128
+ import torch
129
+ from hydra.utils import instantiate
130
+ from omegaconf import OmegaConf
131
+ self.torch = torch
132
+ ckpt = Path(ckpt)
133
+ out = ckpt.parent.parent.parent
134
+ self.cfg = OmegaConf.load(out / "config.yaml")
135
+ self.inp = Inputs(self.cfg, out / "dataset_stats.json")
136
+ miss = [c for c in self.inp.cams if c not in meta["cams"]]
137
+ if miss:
138
+ raise KeyError(f"cameras {miss} not in the cache {meta['cams']}")
139
+ self.device = device if torch.cuda.is_available() else "cpu"
140
+ mp = str(self.cfg.get("mixed_precision", "bf16")).lower()
141
+ dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[mp]
142
+ self.model = instantiate(self.cfg.model, model_dtype=dtype, device=self.device)
143
+ payload = torch.load(str(ckpt), map_location="cpu")
144
+ miss_k, unexp = self.model.mot.load_state_dict(payload["mot"], strict=False)
145
+ if miss_k or unexp:
146
+ raise RuntimeError(f"checkpoint mot keys: missing {list(miss_k)[:5]} unexpected {list(unexp)[:5]}")
147
+ if self.model.proprio_encoder is not None:
148
+ self.model.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
149
+ self.ckpt_step = payload.get("step")
150
+ del payload
151
+ self.model = self.model.to(self.device).eval()
152
+ self.native_steps = int(self.cfg.eval_num_inference_steps)
153
+ self.steps = int(steps or self.native_steps)
154
+ self.seed = int(seed)
155
+ self.ckpt = str(ckpt)
156
+
157
+ def info(self):
158
+ return {"model": "fastwam", "ckpt": self.ckpt, "ckpt_step": self.ckpt_step, "denoising_steps": self.steps,
159
+ "native_steps_in_config": self.native_steps, "cams": self.inp.cams, "chunk": self.inp.horizon,
160
+ "seed": self.seed, "path": "FastWAM.infer_action (action-only, deploy_policy)"}
161
+
162
+ def predict(self, obs):
163
+ return infer(self.model, self.inp, obs, self.steps, self.seed)
b8-table-wipe/b512/step-004000/code/fastwam_prep.sh ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # 2026-11-16: CPU-only preparation of FastWAM for the 12 Delta-flow tasks (no GPU is touched: training is running).
3
+ # Data copies on our interface, ActionDiT initializers per profile, one shared T5 text cache, augmented-sample checks.
4
+ P=/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z
5
+ G=$P/policies-20261109
6
+ F=$G/fastwam
7
+ E=$G/envs/fastwam/bin/python
8
+ exec >> $F/prep.log 2>&1
9
+ echo "$(date) start (v2: 12 tasks)"
10
+ until [ -f $G/envs/fastwam/.hb_ready ] || grep -q ENV_FAILED $F/setup.log; do sleep 30; done
11
+ [ -f $G/envs/fastwam/.hb_ready ] || { echo "ENV_FAILED"; exit 1; }
12
+ grep -q FW_DL_DONE $F/dl2.log || { echo "DOWNLOAD_FAILED"; exit 1; }
13
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true CUDA_VISIBLE_DEVICES="" HF_HUB_OFFLINE=1
14
+ export HB_AUG_DIR=$G/hb_groot
15
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT $(compgen -e | grep -E "^PET_")
16
+ TC=$F/text-cache/all
17
+ # task profile source [first N episodes]
18
+ cat > $F/tasks.txt <<EOF
19
+ a10v3 g1 $P/a10-trolley-push-v3-6e410c5/data
20
+ a3v4 g1 $P/a3-fridge-open-v4-6393d4a/data
21
+ a7v4 g1 $P/a7-cabinet-close-v4-90cc2b5/data
22
+ b7 g1 $P/b7-black-clothes-hamper-v1-1bb002a/data 100
23
+ b5 g1 $P/b5-chair-push-v1-b85b7e9/data
24
+ b8 g1 $P/b8-table-wipe-v1-3b2e618/data
25
+ a17cr2 h2 $P/a17-unscrew-bottle-cropped-v2-3520b70/data
26
+ basketv2 g1 $P/basket-trimv2-c4df4d4/data
27
+ c6 g1 $P/c6-bathroom-door-fetch-v1-3afba69/data
28
+ b6v6 g1 $P/b6-pedal-bin-v6-ad2f5bf/data
29
+ a14v2 h2p $P/a14-unscrew-bulb-v2-8411a6d/data
30
+ b1 g1 $P/b1-riceball-microwave-v1-36da298/data
31
+ EOF
32
+ DIRS=""
33
+ while read t prof src lim; do
34
+ D=$F/data/$t
35
+ if [ ! -f $D/meta/hb_fastwam_interface.json ]; then
36
+ $E $F/tools/hb_fastwam_convert.py $prof $src $D ${lim:+--episodes $lim} || { echo "CONVERT_FAILED $t"; continue; }
37
+ fi
38
+ DIRS="$DIRS,\"$D\""
39
+ done < $F/tasks.txt
40
+ DIRS="[${DIRS#,}]"
41
+ echo "$(date) CONVERTED $DIRS"
42
+ cd $F/FastWAM
43
+ for prof in g1 h2 h2p; do
44
+ AI=$F/action-init-$prof.pt
45
+ [ -f $AI ] && continue
46
+ HB_FW_PROFILE=$prof HB_MODEL_YAML=$F/model-$prof.yaml $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" || { echo "YAML_FAILED $prof"; continue; }
47
+ grep -E 'action_dim|proprio_dim' $F/model-$prof.yaml | tr '\n' ' '; echo
48
+ $E scripts/preprocess_action_dit_backbone.py --model-config $F/model-$prof.yaml --output $AI --device cpu --dtype bfloat16 && echo "$(date) INIT_OK $prof $(stat -c %s $AI)" || echo "INIT_FAILED $prof"
49
+ done
50
+ mkdir -p $TC
51
+ $E scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 model.redirect_common_files=false "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" && echo "$(date) TEXT_OK $(ls $TC | wc -l) files" || echo TEXT_FAILED
52
+ ok=1
53
+ for pair in a10v3:g1 a17cr2:h2 a14v2:h2p; do
54
+ t=${pair%%:*}; prof=${pair##*:}
55
+ HB_FW_PROFILE=$prof HB_DATA_SAMPLE=1 HB_SAMPLE_WORKDIR=/tmp/hbfw-sample-$t $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=[\"$F/data/$t\"]" "data.train.text_embedding_cache_dir=$TC" "model.action_dit_pretrained_path=$F/action-init-$prof.pt" > $F/sample-$t.log 2>&1
56
+ grep -E 'HB_FW_AUG|HBFW_|Error' $F/sample-$t.log | cut -c1-800
57
+ grep -q "HBFW_SAMPLE" $F/sample-$t.log || { ok=0; echo "SAMPLE_FAILED $t"; }
58
+ done
59
+ for t in $(cut -d' ' -f1 $F/tasks.txt); do [ -f $F/data/$t/meta/hb_fastwam_interface.json ] || { ok=0; echo "MISSING_DATA $t"; }; done
60
+ for prof in g1 h2 h2p; do [ -f $F/action-init-$prof.pt ] || { ok=0; echo "MISSING_INIT $prof"; }; done
61
+ [ $ok = 1 ] && touch $F/bench.ready && echo "$(date) BENCH_READY" || echo "$(date) NOT_READY"
b8-table-wipe/b512/step-004000/code/fastwam_task.sh ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Relay cmd item helper (2026-11-16 user): FastWAM on one benchmark task, 4 nodes x 8 GPUs, per-GPU batch from the
3
+ # benchmark ($G/fastwam/chosen.json: largest batch that fits), no gradient accumulation, 4000 steps (user default while
4
+ # the batch is unknown), native Gaussian FM recipe (lr 1e-4 cosine, ZeRO-1, bf16), HB augmentation v2 + proprio
5
+ # perturbation, our state/action interface. Checkpoints at 2000 and 4000. Skips (exit 3) unless the benchmark passed.
6
+ # fastwam_task.sh <task> <profile g1|h2|h2p> <port>
7
+ G=$(dirname "$(dirname "$(realpath "$0")")")
8
+ TASK=$1 PROF=$2 PORT=$3
9
+ F=$G/fastwam; E=$G/envs/fastwam/bin/python; R=$F/runs/$TASK
10
+ RK=${RANK:?}; M0=${MASTER_ADDR:?}
11
+ [ -f $F/fastwam.ok ] && [ -s $F/chosen.json ] || { echo "SKIP fastwam $TASK: benchmark not passed"; exit 3; }
12
+ [ -f $F/data/$TASK/meta/hb_fastwam_interface.json ] && [ -f $F/action-init-$PROF.pt ] || { echo "SKIP fastwam $TASK: data not prepared"; exit 3; }
13
+ BS=$(python3 -c "import json;print(json.load(open('$F/chosen.json'))['per_gpu_batch'])")
14
+ mkdir -p $R/logs
15
+ exec > >(tee -a "$R/logs/node-$RK.log") 2>&1
16
+ date -u; hostname
17
+ [ -f $R/complete.json ] && { echo "already complete"; exit 0; }
18
+ echo "FASTWAM task=$TASK profile=$PROF rank=$RK master=$M0:$PORT per_gpu=$BS global=$((BS * 32)) steps=4000"
19
+ if ! ldconfig -p | grep -q 'libavutil.so.56'; then
20
+ apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
21
+ libavcodec58 libavformat58 libavutil56 libswscale5 libswresample3 libavdevice58 libavfilter7 || true
22
+ fi
23
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT
24
+ unset $(compgen -e | grep -E '^PET_') NPROC_PER_NODE WORLD_SIZE_WITH_STANDBY TORCHELASTIC_RUN_ID
25
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true HF_HUB_OFFLINE=1 HB_AUG_DIR=$G/hb_groot HB_FW_PROFILE=$PROF
26
+ export NCCL_SOCKET_IFNAME=eth GLOO_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false WANDB_MODE=disabled
27
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
28
+ export PATH=$G/envs/fastwam/bin:$PATH # r2 (11-16 05:3x): r1 failed, accelerate called the missing deepspeed CLI (pdsh launcher)
29
+ for i in $(seq 36); do [ -z "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ] && break; sleep 5; done
30
+ cd $F/FastWAM
31
+ $E -m accelerate.commands.launch --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml --deepspeed_multinode_launcher standard \
32
+ --num_machines 4 --machine_rank $RK --main_process_ip $M0 --main_process_port $PORT --num_processes 32 \
33
+ $F/tools/hb_fastwam_train.py "output_dir=$R/out" "batch_size=$BS" max_steps=4000 log_every=20 save_every=2000 eval_every=0 \
34
+ num_workers=8 wandb.enabled=false gradient_accumulation_steps=1 "data.train.dataset_dirs=[\"$F/data/$TASK\"]" \
35
+ "data.train.text_embedding_cache_dir=$F/text-cache/all" "model.action_dit_pretrained_path=$F/action-init-$PROF.pt"
36
+ rc=$?
37
+ echo "$rc" > $R/logs/exit-node-$RK
38
+ if [ "$rc" = 0 ] && [ "$RK" = 0 ]; then
39
+ python3 - "$R/complete.json" "$TASK" "$PROF" "$BS" "$F/data/$TASK/meta/hb_fastwam_interface.json" <<'EOF'
40
+ import json, sys, time
41
+ out, task, prof, bs, iface = sys.argv[1:6]
42
+ d = dict(task=task, profile=prof, per_gpu_batch=int(bs), global_batch=int(bs) * 32, gradient_accumulation_steps=1,
43
+ max_steps=4000, save_every=2000, recipe="native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
44
+ interface=json.load(open(iface)), finished_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
45
+ json.dump(d, open(out, 'w'), indent=1); print('COMPLETE', d)
46
+ EOF
47
+ fi
48
+ echo "EXIT $rc"; exit $rc
b8-table-wipe/b512/step-004000/code/hb_aug_v2.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark data augmentation v2 (project CLAUDE.md), model-agnostic torch implementation.
2
+
3
+ augment(x, rng, obs_mask=None) takes x: float [T, V, 3, H, W] in [0, 1] (T frames, V cameras) and returns the
4
+ augmented tensor. Each call is one training sample: one of the 15 effects is chosen uniformly and its magnitude
5
+ is scaled by 1.5 around identity; the rest stay at x1. Colour is shared across cameras and time; geometry, light
6
+ patches and white balance are drawn per camera and fixed over time; sensor/optics effects (noise, gaussian blur,
7
+ JPEG, motion blur, sharpen, defocus) apply only to observation frames (obs_mask True).
8
+ Pixel-size parameters are defined at 224 px and scaled by H / 224.
9
+ perturb_joints(q, rng) applies the proprio spec: per-joint bias +-2 deg plus N(0, 0.25 deg) clipped at 3 sigma.
10
+ """
11
+ import io
12
+ import math
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from PIL import Image
18
+ from torchvision.transforms import InterpolationMode
19
+ from torchvision.transforms import functional as TF
20
+
21
+ EFFECTS = ("exposure", "contrast", "saturation", "hue", "gamma", "temperature", "rotation_crop", "noise",
22
+ "gaussian_blur", "jpeg", "motion_blur", "sharpen", "defocus", "light_shadow", "wb_channel")
23
+ CENTER = {"exposure": 0.0, "contrast": 1.0, "saturation": 1.0, "hue": 0.0, "gamma": 1.0, "temperature": 0.0}
24
+ RANGE = {"exposure": (-0.22, 0.22), "contrast": (0.88, 1.12), "saturation": (0.76, 1.24), "hue": (-0.015, 0.015),
25
+ "gamma": (0.89, 1.11), "temperature": (-0.066, 0.066)}
26
+ VERSION = "hb-aug-v2-20261109"
27
+
28
+
29
+ def sample(rng, views, height):
30
+ boost = EFFECTS[int(rng.integers(len(EFFECTS)))]
31
+
32
+ def k(name):
33
+ return 1.5 if boost == name else 1.0
34
+
35
+ px = height / 224.0
36
+ p = {"boost": boost}
37
+ for name, (a, b) in RANGE.items():
38
+ c = CENTER[name]
39
+ p[name] = float(rng.uniform(c + k(name) * (a - c), c + k(name) * (b - c)))
40
+ p["geometry"] = []
41
+ for _ in range(views):
42
+ angle = float(rng.uniform(-1.5 * k("rotation_crop"), 1.5 * k("rotation_crop")))
43
+ p["geometry"].append((angle, 1.0 + abs(angle) * 0.015 + float(rng.uniform(0, 0.02 * k("rotation_crop")))))
44
+ p["noise"] = 4.0 / 255.0 * k("noise") if rng.random() < 0.5 else 0.0
45
+ p["gaussian_blur"] = float(rng.uniform(0.25, 0.65)) * k("gaussian_blur") if rng.random() < 0.15 else 0.0
46
+ if rng.random() < 0.35:
47
+ lo = int(round(100 - k("jpeg") * 30))
48
+ hi = int(round(100 - k("jpeg") * 5))
49
+ p["jpeg"] = int(rng.integers(lo, hi + 1))
50
+ else:
51
+ p["jpeg"] = None
52
+ p["motion_blur"] = ((float(rng.uniform(3, 7)) * k("motion_blur") * px, float(rng.uniform(0, 180)))
53
+ if rng.random() < 0.15 else None)
54
+ p["sharpen"] = float(rng.uniform(0.2, 0.6)) * k("sharpen") if rng.random() < 0.15 else 0.0
55
+ p["defocus"] = float(rng.uniform(1, 2)) * k("defocus") * px if rng.random() < 0.10 else 0.0
56
+ if rng.random() < 0.30:
57
+ g = 0.30 * k("light_shadow")
58
+ p["light_shadow"] = [[(float(rng.uniform(0.1, 0.9)), float(rng.uniform(0.1, 0.9)),
59
+ float(rng.uniform(0.15, 0.35)), float(rng.uniform(1 - g, 1 + g)))
60
+ for _ in range(int(rng.integers(1, 3)))] for _ in range(views)]
61
+ else:
62
+ p["light_shadow"] = None
63
+ if rng.random() < 0.30:
64
+ g = 0.10 * k("wb_channel")
65
+ gains = []
66
+ for _ in range(views):
67
+ row = [1.0, 1.0, 1.0]
68
+ row[int(rng.integers(0, 3))] = float(rng.uniform(1 - g, 1 + g))
69
+ gains.append(row)
70
+ p["wb_channel"] = gains
71
+ else:
72
+ p["wb_channel"] = None
73
+ return p
74
+
75
+
76
+ def _conv(x, kernel):
77
+ n, c, h, w = x.shape
78
+ k = kernel.to(x)[None, None].repeat(c, 1, 1, 1)
79
+ pad = kernel.shape[-1] // 2
80
+ return F.conv2d(F.pad(x, (pad, pad, pad, pad), mode="reflect"), k, groups=c)
81
+
82
+
83
+ def motion_kernel(length, angle_deg):
84
+ size = max(3, int(round(length)) | 1)
85
+ k = torch.zeros(size, size)
86
+ c = size // 2
87
+ th = math.radians(angle_deg)
88
+ for s in np.linspace(-(length - 1) / 2, (length - 1) / 2, 8 * size):
89
+ k[int(round(c - s * math.sin(th))), int(round(c + s * math.cos(th)))] = 1.0
90
+ return k / k.sum()
91
+
92
+
93
+ def disk_kernel(r):
94
+ size = 2 * int(math.ceil(r)) + 1
95
+ c = size // 2
96
+ yy, xx = torch.meshgrid(torch.arange(size), torch.arange(size), indexing="ij")
97
+ k = (((yy - c) ** 2 + (xx - c) ** 2) <= (r + 0.5) ** 2).float()
98
+ return k / k.sum()
99
+
100
+
101
+ def apply(x, p, obs_mask=None, noise_seed=0):
102
+ t, v, c, h, w = x.shape
103
+ x = x.clone().clamp(0, 1)
104
+ for vi, (angle, scale) in enumerate(p["geometry"]):
105
+ x[:, vi] = TF.affine(x[:, vi], angle=angle, translate=[0, 0], scale=scale, shear=[0, 0],
106
+ interpolation=InterpolationMode.BILINEAR)
107
+ if p["light_shadow"] is not None:
108
+ yy, xx = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij")
109
+ for vi, blobs in enumerate(p["light_shadow"]):
110
+ m = torch.ones(h, w)
111
+ for cx, cy, s, g in blobs:
112
+ m = m + (g - 1.0) * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * s * s))
113
+ x[:, vi] = (x[:, vi] * m.clamp(min=0).to(x)).clamp(0, 1)
114
+ if p["wb_channel"] is not None:
115
+ for vi, g in enumerate(p["wb_channel"]):
116
+ x[:, vi] = (x[:, vi] * torch.tensor(g, dtype=x.dtype).view(1, 3, 1, 1)).clamp(0, 1)
117
+ y = x.reshape(t * v, c, h, w)
118
+ y = (y * float(2.0 ** p["exposure"])).clamp(0, 1)
119
+ d = p["temperature"]
120
+ y = (y * y.new_tensor([1.0 + d, 1.0, 1.0 - d]).view(1, 3, 1, 1)).clamp(0, 1)
121
+ y = ((y - 0.5) * p["contrast"] + 0.5).clamp(0, 1)
122
+ y = TF.adjust_saturation(y, p["saturation"])
123
+ y = TF.adjust_hue(y, p["hue"])
124
+ y = y.clamp(0, 1).pow(p["gamma"])
125
+ x = y.reshape(t, v, c, h, w)
126
+ mask = torch.ones(t, dtype=torch.bool) if obs_mask is None else torch.as_tensor(obs_mask, dtype=torch.bool)
127
+ if bool(mask.any()):
128
+ o = x[mask].reshape(-1, c, h, w)
129
+ if p["gaussian_blur"] > 0:
130
+ o = TF.gaussian_blur(o, [3, 3], [p["gaussian_blur"]] * 2)
131
+ if p["motion_blur"] is not None:
132
+ o = _conv(o, motion_kernel(*p["motion_blur"]))
133
+ if p["defocus"] > 0:
134
+ o = _conv(o, disk_kernel(p["defocus"]))
135
+ if p["sharpen"] > 0:
136
+ o = o + p["sharpen"] * (o - TF.gaussian_blur(o, [5, 5], [1.0, 1.0]))
137
+ o = o.clamp(0, 1)
138
+ if p["noise"] > 0:
139
+ gen = torch.Generator().manual_seed(int(noise_seed))
140
+ o = (o + torch.randn(o.shape, generator=gen, dtype=o.dtype) * p["noise"]).clamp(0, 1)
141
+ if p["jpeg"] is not None:
142
+ for i in range(o.shape[0]):
143
+ buf = io.BytesIO()
144
+ TF.to_pil_image(o[i]).save(buf, format="JPEG", quality=int(p["jpeg"]))
145
+ buf.seek(0)
146
+ with Image.open(buf) as im:
147
+ o[i] = TF.pil_to_tensor(im.convert("RGB")).to(o.dtype).div(255.0)
148
+ x[mask] = o.reshape(-1, v, c, h, w)
149
+ return x.clamp(0, 1)
150
+
151
+
152
+ def augment(x, rng, obs_mask=None):
153
+ p = sample(rng, x.shape[1], x.shape[-2])
154
+ return apply(x, p, obs_mask, noise_seed=int(rng.integers(0, 2 ** 62))), p
155
+
156
+
157
+ def perturb_joints(q, rng):
158
+ q = np.asarray(q, dtype=np.float64)
159
+ d = q.shape[-1]
160
+ bias = rng.uniform(-math.radians(2.0), math.radians(2.0), size=d)
161
+ sigma = math.radians(0.25)
162
+ noise = np.clip(rng.normal(0.0, sigma, size=q.shape), -3 * sigma, 3 * sigma)
163
+ return (q + bias + noise).astype(np.float32)
b8-table-wipe/b512/step-004000/code/hb_eval_common.py ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark open-loop action-reconstruction evaluation: shared task table, cache layout and metrics.
2
+
3
+ Protocol (identical for every model; mirrors Isaac-GR00T gr00t/eval/open_loop_eval.py):
4
+ * query frames t = 0, 16, 32, ... over the whole episode (non-overlapping 16-step chunks);
5
+ * the model sees the observation at t exactly as at deployment (cameras, raw benchmark state 44/78, instruction;
6
+ Delta Flow also its native 16-step past-action history), all augmentation off;
7
+ * the predicted chunk (first 16 steps, raw action units) is compared with the recorded actions t..t+15;
8
+ rows beyond the episode end are excluded (no clamped / padded targets);
9
+ * metrics: raw MSE / MAE, and nMSE = MSE after dividing every action dim by its std over the task's TRAINING
10
+ frames (floored, see STD_FLOOR_FRAC), overall, per action group and per horizon step.
11
+ Splits: "heldout" = recorded episodes that were never trained on (B6 100-148, trolley 100-102, B7 100-106);
12
+ "train10" = 10 fixed, evenly spaced training episodes (fit on seen data, NOT generalisation).
13
+ Only numpy here: imported by every model environment.
14
+ """
15
+ import json
16
+ import os
17
+ import time
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+
22
+ G = Path(os.environ.get("G", "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/"
23
+ "preflight-20261104T1648Z/policies-20261109"))
24
+ P = G.parent
25
+ ROOT = G / "eval-action-mse"
26
+ CACHE = Path(os.environ.get("HB_EVAL_CACHE", ROOT / "cache"))
27
+ HORIZON = 16
28
+ STRIDE = 16
29
+ N_TRAIN_EVAL = 10
30
+ STD_FLOOR_FRAC = 0.05 # per-dim std floor = 5 % of the median std of the action vector
31
+
32
+ # task -> (source LeRobot v2.1 root, number of TRAINING episodes = fixed first N (None = all recorded), layout)
33
+ TASKS = {
34
+ "b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100, "g1"),
35
+ "basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None, "g1"),
36
+ "trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100, "g1"),
37
+ "b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100, "g1"),
38
+ "b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None, "g1"),
39
+ "a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None, "g1"),
40
+ "a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None, "g1"),
41
+ "a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None, "g1"),
42
+ "c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None, "g1"),
43
+ "a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None, "g1"),
44
+ "a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None, "g1"),
45
+ "a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None, "h2"),
46
+ "a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None, "h2"),
47
+ "a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None, "h2"),
48
+ # 2026-11-15: newer datasets (GR00T in-training eval needs a cache for every task it trains on)
49
+ "a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None, "g1"),
50
+ "a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None, "g1"),
51
+ "b1": (P / "b1-riceball-microwave-v1-36da298/data", None, "g1"),
52
+ "b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None, "g1"),
53
+ "b8": (P / "b8-table-wipe-v1-3b2e618/data", None, "g1"),
54
+ "basketv2": (P / "basket-trimv2-c4df4d4/data", None, "g1"),
55
+ "a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None, "h2p"), # action 82 = hb75 + SONIC planner 7
56
+ # 2026-11-15 evening: v6/v4 datasets (a3v3, a7v3 retired, not registered)
57
+ "b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None, "g1"), # 542 ep, all used (B6 first-100 rule is v5t7 only)
58
+ "b5": (P / "b5-chair-push-v1-b85b7e9/data", None, "g1"),
59
+ "a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None, "g1"),
60
+ "a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None, "g1"),
61
+ # 2026-11-16: B5 standing start (HF b5-chair-push-standing-start-expert-v1 @913f5a16, 100 ep; FastWAM b5ss)
62
+ "b5ss": (P / "b5-chair-push-standing-start-v1-913f5a1/data", None, "g1"),
63
+ # 2026-11-17: B7 black clothes trimmed (HF b7-black-clothes-hamper @949da46, 107 ep; train = first 100, heldout 100-106)
64
+ "b7t": (P / "b7-black-clothes-hamper-trim-949da46/data", 100, "g1"),
65
+ # 2026-11-15 night: H2 cropped v2 (GR00T h32 relay item 71); head+chest only -> build_cache --cams head,chest
66
+ "a18cr2": (P / "a18-pour-water-cropped-v2-44cdd57/data", None, "h2"), # HF a18-pour-water-expert-cropped-v2 @44cdd571
67
+ "a17cr2": (P / "a17-unscrew-bottle-cropped-v2-3520b70/data", None, "h2"), # HF a17-unscrew-bottle-expert-cropped-v2 @3520b70f
68
+ }
69
+
70
+
71
+ def task_for_source(source):
72
+ """benchmark task whose recording root is <source> (groot-data/<view>/hb-dataset-receipt.json "source")"""
73
+ src = Path(source).resolve()
74
+ hits = [t for t, (root, _, _) in TASKS.items() if Path(root).resolve() == src]
75
+ return hits[0] if hits else None
76
+ G1_BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint",
77
+ "left_ankle_pitch_joint", "left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint",
78
+ "right_hip_yaw_joint", "right_knee_joint", "right_ankle_pitch_joint", "right_ankle_roll_joint",
79
+ "waist_yaw_joint", "waist_roll_joint", "waist_pitch_joint", "left_shoulder_pitch_joint",
80
+ "left_shoulder_roll_joint", "left_shoulder_yaw_joint", "left_elbow_joint", "left_wrist_roll_joint",
81
+ "left_wrist_pitch_joint", "left_wrist_yaw_joint", "right_shoulder_pitch_joint",
82
+ "right_shoulder_roll_joint", "right_shoulder_yaw_joint", "right_elbow_joint", "right_wrist_roll_joint",
83
+ "right_wrist_pitch_joint", "right_wrist_yaw_joint"]
84
+ LAYOUTS = {
85
+ # state: body29 | inspire L6 | inspire R6 | projected_gravity3 ; action: motion_token64 | inspire L6 | R6 (same row)
86
+ "g1": dict(state_dim=44, action_dim=76, cams=("ego_view", "chest"),
87
+ groups={"motion_token": (0, 64), "hands": (64, 76)}),
88
+ # state: observation.state75 (legs12 waist3 arms14 head2 hands22+22) | gravity3 ; action: body_target31 | hands44
89
+ "h2": dict(state_dim=78, action_dim=75, cams=("head", "chest", "wrist"),
90
+ groups={"body_target": (0, 31), "hands": (31, 75)}),
91
+ # 2026-11-15 user: A14 v2 is trained on action82 = body_target31 | hands44 | action.planner7 (SONIC planner
92
+ # command); all 82 dims are scored (planner reported as its own group)
93
+ "h2p": dict(state_dim=78, action_dim=82, cams=("head", "chest"),
94
+ groups={"body_target": (0, 31), "hands": (31, 75), "planner": (75, 82)}),
95
+ }
96
+
97
+
98
+ def jl(path):
99
+ return [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
100
+
101
+
102
+ def layout(task):
103
+ return LAYOUTS[TASKS[task][2]]
104
+
105
+
106
+ def split_episodes(task, split, n_recorded):
107
+ _, n_train, _ = TASKS[task]
108
+ n_train = n_recorded if n_train is None else n_train
109
+ if split == "heldout":
110
+ return list(range(n_train, n_recorded))
111
+ if split == "train10":
112
+ return sorted({int(round(x)) for x in np.linspace(0, n_train - 1, N_TRAIN_EVAL)})
113
+ if split.startswith("ep:"): # explicit list, e.g. ep:100,101
114
+ return [int(x) for x in split[3:].split(",")]
115
+ raise ValueError(f"unknown split {split!r}")
116
+
117
+
118
+ def split_dir(task, split):
119
+ return CACHE / task / split
120
+
121
+
122
+ def load_meta(task, split):
123
+ return json.loads((split_dir(task, split) / "meta.json").read_text())
124
+
125
+
126
+ def load_stats(task):
127
+ z = np.load(CACHE / task / "stats.npz")
128
+ return {k: z[k] for k in z.files}
129
+
130
+
131
+ def norm_std(stats):
132
+ std = stats["action_std"].astype(np.float64)
133
+ return np.maximum(std, STD_FLOOR_FRAC * float(np.median(std)))
134
+
135
+
136
+ def iter_episodes(task, split, max_episodes=None):
137
+ meta = load_meta(task, split)
138
+ eps = meta["episodes"][: max_episodes or None]
139
+ for ep in eps:
140
+ with np.load(split_dir(task, split) / f"ep_{ep:06d}.npz") as z:
141
+ yield ep, {k: z[k] for k in z.files}
142
+
143
+
144
+ class Accumulator:
145
+ """Sums of squared / absolute errors; everything is computed in float64 over valid (row, dim) entries."""
146
+
147
+ def __init__(self, std, groups, horizon=HORIZON):
148
+ self.std = np.asarray(std, dtype=np.float64)
149
+ self.groups = groups
150
+ D = len(self.std)
151
+ self.sse_raw = np.zeros(D)
152
+ self.sae_raw = np.zeros(D)
153
+ self.sse_n = np.zeros(D)
154
+ self.rows = 0
155
+ self.step_sse_n = np.zeros(horizon)
156
+ self.step_rows = np.zeros(horizon)
157
+ self.per_episode = {}
158
+
159
+ def add(self, ep, pred, gt, valid):
160
+ pred = np.asarray(pred, dtype=np.float64)
161
+ gt = np.asarray(gt, dtype=np.float64)
162
+ v = np.asarray(valid, dtype=bool)
163
+ if pred.shape != gt.shape:
164
+ raise ValueError(f"pred {pred.shape} vs gt {gt.shape}")
165
+ if not np.isfinite(pred[v]).all():
166
+ raise FloatingPointError("non-finite prediction")
167
+ e = (pred - gt)[v]
168
+ en = e / self.std
169
+ self.sse_raw += (e ** 2).sum(0)
170
+ self.sae_raw += np.abs(e).sum(0)
171
+ self.sse_n += (en ** 2).sum(0)
172
+ self.rows += int(v.sum())
173
+ k = np.nonzero(v)[0]
174
+ self.step_sse_n[k] += (en ** 2).sum(1)
175
+ self.step_rows[k] += 1
176
+ s = self.per_episode.setdefault(int(ep), [0.0, 0])
177
+ s[0] += float((en ** 2).sum())
178
+ s[1] += int(v.sum()) * len(self.std)
179
+
180
+ def result(self):
181
+ D = len(self.std)
182
+ n = max(self.rows, 1)
183
+ out = {"rows": self.rows, "mse_raw": float(self.sse_raw.sum() / (n * D)),
184
+ "mae_raw": float(self.sae_raw.sum() / (n * D)), "nmse": float(self.sse_n.sum() / (n * D)),
185
+ "groups": {}, "nmse_by_step": (self.step_sse_n / np.maximum(self.step_rows, 1) / D).tolist(),
186
+ "nmse_by_dim": (self.sse_n / n).tolist()}
187
+ for g, (a, b) in self.groups.items():
188
+ out["groups"][g] = {"mse_raw": float(self.sse_raw[a:b].sum() / (n * (b - a))),
189
+ "nmse": float(self.sse_n[a:b].sum() / (n * (b - a)))}
190
+ pe = {ep: s / max(c, 1) for ep, (s, c) in sorted(self.per_episode.items())}
191
+ out["nmse_per_episode"] = pe
192
+ vals = np.array(list(pe.values()))
193
+ if len(vals) > 1: # bootstrap-free spread: std error over episodes
194
+ out["nmse_episode_sem"] = float(vals.std(ddof=1) / np.sqrt(len(vals)))
195
+ return out
196
+
197
+
198
+ def queries(T):
199
+ return list(range(0, T, STRIDE))
200
+
201
+
202
+ def gt_chunk(action, t, horizon=HORIZON):
203
+ T = len(action)
204
+ idx = np.arange(t, t + horizon)
205
+ valid = idx < T
206
+ return action[np.minimum(idx, T - 1)], valid
207
+
208
+
209
+ def run(adapter, task, split, out_path, max_episodes=None, max_queries=None, save_pred=True, log_every=50):
210
+ """Evaluate one adapter on one cached split; writes <out_path> (json) and <out_path>.pred.npz."""
211
+ meta = load_meta(task, split)
212
+ lay = LAYOUTS[meta["layout"]]
213
+ stats = load_stats(task)
214
+ acc = Accumulator(norm_std(stats), lay["groups"])
215
+ preds, keys = [], []
216
+ t0 = time.time()
217
+ infer_s = 0.0
218
+ nq = 0
219
+ for ep, z in iter_episodes(task, split, max_episodes):
220
+ state, action = z["state"], z["action"]
221
+ qts = list(z["query_t"])
222
+ for qi, t in enumerate(qts):
223
+ obs = {"episode": ep, "t": int(t), "T": len(action), "state": state[t], "prompt": meta["prompts"][str(int(z["task_index"]))],
224
+ "images": {c: z[f"img_{c}"][qi] for c in meta["cams"]},
225
+ "past_action": action[max(0, t - HORIZON):t],
226
+ "_gt_check": action[t:t + HORIZON]} # only for adapters' input-pipeline consistency checks
227
+ s = time.time()
228
+ pred = np.asarray(adapter.predict(obs), dtype=np.float64)
229
+ infer_s += time.time() - s
230
+ if pred.ndim != 2 or pred.shape[0] < HORIZON or pred.shape[1] != lay["action_dim"]:
231
+ raise ValueError(f"adapter returned {pred.shape}, need (>={HORIZON}, {lay['action_dim']})")
232
+ pred = pred[:HORIZON]
233
+ gt, valid = gt_chunk(action, t)
234
+ acc.add(ep, pred, gt, valid)
235
+ if save_pred:
236
+ preds.append(pred.astype(np.float32))
237
+ keys.append((ep, int(t)))
238
+ nq += 1
239
+ if log_every and nq % log_every == 0:
240
+ print(f"[eval] {task}/{split} {nq} queries, running nMSE {acc.result()['nmse']:.4f}, "
241
+ f"{infer_s / nq:.3f} s/query", flush=True)
242
+ if max_queries and nq >= max_queries:
243
+ break
244
+ if max_queries and nq >= max_queries:
245
+ break
246
+ res = acc.result()
247
+ res.update(task=task, split=split, layout=meta["layout"], episodes=sorted(acc.per_episode), queries=nq,
248
+ horizon=HORIZON, stride=STRIDE, std_floor_frac=STD_FLOOR_FRAC, wall_s=round(time.time() - t0, 1),
249
+ infer_s_per_query=round(infer_s / max(nq, 1), 4), adapter=adapter.info(),
250
+ baselines=meta.get("baselines", {}), cache_created=meta.get("created"),
251
+ partial=bool(max_episodes or max_queries), finished_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
252
+ out_path = Path(out_path)
253
+ out_path.parent.mkdir(parents=True, exist_ok=True)
254
+ tmp = out_path.with_suffix(".tmp")
255
+ tmp.write_text(json.dumps(res, indent=1))
256
+ tmp.replace(out_path)
257
+ if save_pred and preds:
258
+ np.savez_compressed(str(out_path) + ".pred.npz", pred=np.stack(preds).astype(np.float16),
259
+ keys=np.asarray(keys, dtype=np.int64))
260
+ return res
b8-table-wipe/b512/step-004000/code/hb_fastwam_aug.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: Humanoid Benchmark augmentation v2 (project CLAUDE.md) inside the native FastWAM data pipeline.
2
+ - Visual: hb_aug_v2.augment on every training sample's frames before the two cameras are concatenated; frame 0 is the
3
+ current observation (sensor / optics effects only there), the later frames are the future video targets (colour,
4
+ per-camera geometry, light patches and white balance shared with frame 0, as the spec requires).
5
+ - Proprio: hb_aug_v2.perturb_joints on the raw body joints (G1 29, H2 31 = body 29 + head 2) before normalization;
6
+ hands and gravity untouched; supervision labels untouched. Normalization statistics are computed on the raw data.
7
+ FastWAM's own image augmentation is only a resize (no colour jitter / crop), so nothing native is stacked.
8
+ Installed in the main process before the dataloaders fork; HB_FW_JOINTS sets the number of perturbed joints."""
9
+ import os
10
+ import sys
11
+ import time
12
+
13
+ import numpy as np
14
+ import torch
15
+
16
+ sys.path.insert(0, os.environ.get("HB_AUG_DIR", ""))
17
+ import hb_aug_v2 as A # noqa: E402
18
+
19
+ NJ = int(os.environ.get("HB_FW_JOINTS", "29"))
20
+ _seen = {"img": False, "state": False}
21
+
22
+
23
+ def _rng(idx=0):
24
+ return np.random.default_rng([os.getpid(), int(idx) & 0xFFFFFFFF, time.time_ns() & 0xFFFFFFFF])
25
+
26
+
27
+ def _train(proc):
28
+ v = getattr(proc, "is_train", False)
29
+ return bool(v() if callable(v) else v)
30
+
31
+
32
+ def install():
33
+ from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset
34
+ from fastwam.datasets.lerobot.processors.fastwam_processor import FastWAMProcessor
35
+
36
+ pre0 = FastWAMProcessor.preprocess
37
+
38
+ def preprocess(self, data):
39
+ if _train(self) and isinstance(data.get("state"), dict) and "default" in data["state"]:
40
+ st = data["state"]["default"]
41
+ q = st[..., :NJ].detach().cpu().numpy()
42
+ st = st.clone()
43
+ st[..., :NJ] = torch.from_numpy(A.perturb_joints(q, _rng())).to(st.dtype)
44
+ data["state"]["default"] = st
45
+ if not _seen["state"]:
46
+ _seen["state"] = True
47
+ print(f"HB_FW_AUG proprio perturb_joints on state[..., :{NJ}] shape {tuple(st.shape)}", flush=True)
48
+ return pre0(self, data)
49
+
50
+ get0 = BaseLerobotDataset.__getitem__
51
+
52
+ def getitem(self, idx):
53
+ s = get0(self, idx)
54
+ if self.processor is not None and _train(self.processor) and "pixel_values" in s:
55
+ pv = s["pixel_values"]
56
+ five = pv.ndim == 5
57
+ x = (pv if five else pv[None]).permute(1, 0, 2, 3, 4).float() # [T, V, C, H, W]
58
+ hi = float(x.max())
59
+ scale = 255.0 if hi > 1.5 else 1.0
60
+ mask = torch.zeros(x.shape[0], dtype=torch.bool)
61
+ mask[0] = True
62
+ y, p = A.augment((x / scale).clamp(0, 1), _rng(idx), mask)
63
+ y = (y * scale).permute(1, 0, 2, 3, 4)
64
+ s["pixel_values"] = (y if five else y[0]).to(pv.dtype)
65
+ if not _seen["img"]:
66
+ _seen["img"] = True
67
+ print(f"HB_FW_AUG visual {A.VERSION} on pixel_values {tuple(pv.shape)} dtype {pv.dtype} max {hi:.3f} "
68
+ f"(T={x.shape[0]}, obs frame 0) boost={p['boost']}", flush=True)
69
+ return s
70
+
71
+ FastWAMProcessor.preprocess = preprocess
72
+ BaseLerobotDataset.__getitem__ = getitem
73
+ print(f"HB_FW_AUG installed: {A.VERSION}, joints {NJ}", flush=True)
b8-table-wipe/b512/step-004000/code/hb_fastwam_convert.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: LeRobot v2.1 copy of one of our datasets for FastWAM on OUR benchmark interface (CLAUDE.md section 4):
2
+ g1 : state44 = observation.state[0:29] (29 body joints) + measured Inspire 6 + 6 + projected gravity 3;
3
+ action76 = action.motion_token 64 + measured Inspire 6 + 6 (same-row labels, as GR00T / Psi-Zero / Delta Flow)
4
+ h2 : state78 = observation.state[0:75] + projected gravity 3; action75 = action.body_target 31 + action.hands 44
5
+ h2p: as h2 with action82 = action75 + action.planner 7 (A14 v2)
6
+ HumanWAM's fastwam_sonic_data.py expects a 43-D G1 state and takes state[0:22] + state[29:36]; on our 41-D state that
7
+ would put left-hand values into the right arm, so it is not used. Source files are never modified; media are symlinked.
8
+ --episodes N keeps the first N episodes (B7: first 100 of 107, as for every other model).
9
+ Usage: hb_fastwam_convert.py <profile> <source data dir> <fresh output dir> [--episodes N]"""
10
+ import copy, json, shutil, sys, tempfile
11
+ from pathlib import Path
12
+ import numpy as np
13
+ import pyarrow as pa
14
+ import pyarrow.parquet as pq
15
+
16
+ G1 = {"observation.state": 41, "action.motion_token": 64, "observation.inspire_left_hand_state": 6,
17
+ "observation.inspire_right_hand_state": 6, "observation.projected_gravity": 3}
18
+ H2 = {"observation.state": 75, "observation.projected_gravity": 3, "action.body_target": 31, "action.hands": 44}
19
+ H2P = dict(H2, **{"action.planner": 7})
20
+ SPEC = {"g1": (G1, ("ego_view", "chest")), "h2": (H2, ("head", "chest")), "h2p": (H2P, ("head", "chest"))}
21
+
22
+
23
+ def stats(v):
24
+ v = np.asarray(v, dtype=np.float64)
25
+ return {"min": v.min(0).tolist(), "max": v.max(0).tolist(), "mean": v.mean(0).tolist(), "std": v.std(0).tolist(), "count": [len(v)]}
26
+
27
+
28
+ def vec(table, key, values):
29
+ arr = pa.array(values.tolist(), type=pa.list_(pa.float32(), values.shape[1]))
30
+ i = table.schema.get_field_index(key)
31
+ return table.append_column(key, arr) if i < 0 else table.set_column(i, key, arr)
32
+
33
+
34
+ def map_rows(profile, c):
35
+ if profile == "g1":
36
+ L, R = c["observation.inspire_left_hand_state"], c["observation.inspire_right_hand_state"]
37
+ dev = float(np.abs(c["observation.state"][:, 29:41] - np.concatenate([L, R], 1)).max())
38
+ assert dev < 1e-4, f"observation.state[29:41] differs from the Inspire columns by {dev}"
39
+ return (np.concatenate([c["action.motion_token"], L, R], 1),
40
+ np.concatenate([c["observation.state"][:, :29], L, R, c["observation.projected_gravity"]], 1))
41
+ a = [c["action.body_target"], c["action.hands"]] + ([c["action.planner"]] if profile == "h2p" else [])
42
+ return np.concatenate(a, 1), np.concatenate([c["observation.state"], c["observation.projected_gravity"]], 1)
43
+
44
+
45
+ def main(profile, src, out, limit=None):
46
+ raw, cams = SPEC[profile]
47
+ src, out = Path(src).resolve(strict=True), Path(out).resolve()
48
+ if out.exists():
49
+ raise SystemExit(f"{out} exists")
50
+ info = json.loads((src / "meta/info.json").read_text())
51
+ assert info.get("codebase_version") == "v2.1", info.get("codebase_version")
52
+ for k, w in raw.items():
53
+ assert info["features"][k]["shape"] == [w], (k, info["features"][k]["shape"])
54
+ for c in cams:
55
+ assert info["features"]["observation.images." + c]["dtype"] in ("video", "image"), c
56
+ eps = [json.loads(l) for l in (src / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
57
+ assert [e["episode_index"] for e in eps] == list(range(len(eps)))
58
+ if limit is not None:
59
+ eps = eps[:limit]
60
+ old = {r["episode_index"]: r for r in (json.loads(l) for l in (src / "meta/episodes_stats.jsonl").read_text().splitlines() if l.strip())}
61
+ out.parent.mkdir(parents=True, exist_ok=True)
62
+ stg = Path(tempfile.mkdtemp(prefix=".hbfw-", dir=out.parent))
63
+ shutil.copytree(src / "meta", stg / "meta")
64
+ for n in ("videos", "images"):
65
+ if (src / n).exists():
66
+ (stg / n).symlink_to(src / n, target_is_directory=True)
67
+ new, total = [], 0
68
+ for e in eps:
69
+ i = e["episode_index"]
70
+ rel = Path(info["data_path"].format(episode_chunk=i // info["chunks_size"], episode_index=i))
71
+ t = pq.read_table(src / rel)
72
+ c = {k: np.asarray(t[k].to_pylist(), dtype=np.float32) for k in raw}
73
+ act, st = map_rows(profile, c)
74
+ assert len(act) == e["length"] and np.isfinite(act).all() and np.isfinite(st).all()
75
+ total += len(act)
76
+ t = vec(vec(t, "action", act), "observation.state", st)
77
+ md = dict(t.schema.metadata or {})
78
+ if b"huggingface" in md:
79
+ hf = json.loads(md[b"huggingface"])
80
+ for k, w in (("action", act.shape[1]), ("observation.state", st.shape[1])):
81
+ hf["info"]["features"][k] = {"feature": {"dtype": "float32", "_type": "Value"}, "length": w, "_type": "Sequence"}
82
+ md[b"huggingface"] = json.dumps(hf).encode()
83
+ t = t.replace_schema_metadata(md)
84
+ (stg / rel).parent.mkdir(parents=True, exist_ok=True)
85
+ pq.write_table(t, stg / rel)
86
+ r = copy.deepcopy(old[i])
87
+ r["stats"]["action"], r["stats"]["observation.state"] = stats(act), stats(st)
88
+ new.append(r)
89
+ if limit is None:
90
+ assert total == info["total_frames"], (total, info["total_frames"])
91
+ else:
92
+ info["total_episodes"], info["total_frames"] = len(eps), total
93
+ info["total_videos"] = len(eps) * len(cams) if "total_videos" in info else info.get("total_videos")
94
+ info["splits"] = {"train": f"0:{len(eps)}"}
95
+ (stg / "meta/episodes.jsonl").write_text("".join(json.dumps(e) + "\n" for e in eps))
96
+ info["features"]["action"] = {"dtype": "float32", "shape": [int(act.shape[1])], "names": None}
97
+ info["features"]["observation.state"] = {"dtype": "float32", "shape": [int(st.shape[1])], "names": None}
98
+ drop = [k for k, v in info["features"].items() if v["dtype"] in ("video", "image") and k not in ["observation.images." + c for c in cams]]
99
+ for k in drop: # 2-camera rule: unused cameras are not downloaded, FastWAM checks every listed video file
100
+ del info["features"][k]
101
+ for r in new:
102
+ r["stats"].pop(k, None)
103
+ if drop and info.get("total_videos"):
104
+ info["total_videos"] = len(eps) * len(cams)
105
+ (stg / "meta/info.json").write_text(json.dumps(info, indent=2) + "\n")
106
+ (stg / "meta/episodes_stats.jsonl").write_text("".join(json.dumps(r) + "\n" for r in new))
107
+ for s in (stg / "meta").glob("*stats*.json"):
108
+ s.unlink()
109
+ (stg / "meta/hb_fastwam_interface.json").write_text(json.dumps(dict(
110
+ profile=profile, source=str(src), cameras=list(cams), state_dim=int(st.shape[1]), action_dim=int(act.shape[1]),
111
+ episodes=len(eps), frames=total, fps=info["fps"], first_episodes_only=limit), indent=2) + "\n")
112
+ stg.rename(out)
113
+ print("HBFW_CONVERT_OK", profile, out, "episodes", len(eps), "frames", total, "state", st.shape[1], "action", act.shape[1], flush=True)
114
+
115
+
116
+ if __name__ == "__main__":
117
+ a = sys.argv[1:]
118
+ lim = None
119
+ if "--episodes" in a:
120
+ j = a.index("--episodes"); lim = int(a[j + 1]); del a[j:j + 2]
121
+ main(a[0], a[1], a[2], lim)
b8-table-wipe/b512/step-004000/code/hb_fastwam_train.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: native FastWAM Gaussian-FM training on our benchmark interface, with the HB augmentation v2.
2
+ Profiles (HB_FW_PROFILE): g1 = state44 / action76 / cameras ego_view + chest (HumanWAM fastwam_sonic recipe);
3
+ h2 = state78 / action75 / head + chest; h2p = state78 / action82 (+ planner 7) / head + chest. Fixed recipe of HumanWAM
4
+ fastwam_sonic_training.compose_sonic_config: 17 raw samples = 16 actions + 5 video frames (stride 4), 224 x 448 side by
5
+ side, min/max statistics, absolute labels, action scheduler shift 1.0, original Wan files.
6
+ Arguments: key=value Hydra overrides (JSON values parsed). Env: FASTWAM_SRC, DIFFSYNTH_MODEL_BASE_PATH, HB_AUG_DIR.
7
+ HB_MODEL_YAML=<path> writes the resolved model config; HB_COMPOSE_ONLY=1 prints the config; HB_DATA_SAMPLE=1 reads one
8
+ augmented training sample on CPU; HB_FW_AUG=0 disables the augmentation (benchmark only)."""
9
+ import json, os, sys
10
+ from pathlib import Path
11
+
12
+ ROOT = Path(os.environ["FASTWAM_SRC"]).resolve()
13
+ sys.path.insert(0, str(ROOT / "src"))
14
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
15
+ TASK = "libero_uncond_2cam224_1e-4"
16
+ PROFILES = {"g1": (("ego_view", "chest"), 44, 76, 29), "h2": (("head", "chest"), 78, 75, 31),
17
+ "h2p": (("head", "chest"), 78, 82, 31)}
18
+ CAMS, SDIM, ADIM, NJ = PROFILES[os.environ.get("HB_FW_PROFILE", "g1")]
19
+ os.environ.setdefault("HB_FW_JOINTS", str(NJ))
20
+ SHAPE_META = {
21
+ "images": [{"key": n, "raw_shape": [3, 224, 224], "shape": [3, 224, 224]} for n in CAMS],
22
+ "action": [{"key": "default", "raw_shape": ADIM, "shape": ADIM}],
23
+ "state": [{"key": "default", "raw_shape": SDIM, "shape": SDIM}],
24
+ }
25
+ FIXED = {
26
+ "data.train.shape_meta": SHAPE_META, "data.train.num_frames": 17, "data.train.global_sample_stride": 1,
27
+ "data.train.action_video_freq_ratio": 4, "data.train.video_size": [224, 448], "data.train.concat_multi_camera": "horizontal",
28
+ "data.train.processor.action_output_dim": ADIM, "data.train.processor.proprio_output_dim": SDIM,
29
+ "data.train.processor.delta_action_dim_mask": {"default": [False] * ADIM}, "data.train.processor.action_state_transforms": None,
30
+ "data.train.processor.use_stepwise_action_norm": False, "data.train.processor.norm_default_mode": "min/max",
31
+ "data.train.processor.norm_exception_mode": None, "model.redirect_common_files": False,
32
+ "model.action_scheduler.train_shift": 1.0, "model.action_scheduler.infer_shift": 1.0,
33
+ }
34
+
35
+
36
+ def build(overrides):
37
+ from hydra import compose, initialize_config_dir
38
+ from omegaconf import OmegaConf
39
+ from fastwam.utils.config_resolvers import register_default_resolvers
40
+ register_default_resolvers()
41
+ with initialize_config_dir(config_dir=str(ROOT / "configs"), version_base="1.3"):
42
+ cfg = compose(config_name="train", overrides=["task=" + TASK])
43
+ for k, v in {**FIXED, **overrides}.items():
44
+ OmegaConf.update(cfg, k, v, merge=False)
45
+ for k, v in FIXED.items():
46
+ a = OmegaConf.select(cfg, k)
47
+ if OmegaConf.is_config(a):
48
+ a = OmegaConf.to_container(a, resolve=True)
49
+ assert a == v, (k, a, v)
50
+ return cfg
51
+
52
+
53
+ def parse(argv):
54
+ out = {}
55
+ for a in argv:
56
+ k, v = a.split("=", 1)
57
+ try:
58
+ out[k] = json.loads(v)
59
+ except ValueError:
60
+ out[k] = v
61
+ return out
62
+
63
+
64
+ if __name__ == "__main__":
65
+ from omegaconf import OmegaConf
66
+ cfg = build(parse(sys.argv[1:]))
67
+ if os.environ.get("HB_MODEL_YAML"):
68
+ Path(os.environ["HB_MODEL_YAML"]).write_text(OmegaConf.to_yaml(cfg.model, resolve=True))
69
+ print("HBFW_MODEL_YAML", os.environ["HB_MODEL_YAML"], flush=True)
70
+ raise SystemExit(0)
71
+ if os.environ.get("HB_COMPOSE_ONLY"):
72
+ print(OmegaConf.to_yaml(cfg, resolve=True))
73
+ raise SystemExit(0)
74
+ import fastwam.runtime as native
75
+ assert Path(native.__file__).resolve() == ROOT / "src/fastwam/runtime.py", native.__file__
76
+ if os.environ.get("HB_FW_AUG", "1") != "0":
77
+ import hb_fastwam_aug
78
+ hb_fastwam_aug.install()
79
+ print("HBFW_RECIPE", json.dumps(dict(profile=os.environ.get("HB_FW_PROFILE", "g1"), cameras=CAMS, state_dim=SDIM,
80
+ action_dim=ADIM, perturbed_joints=int(os.environ["HB_FW_JOINTS"]),
81
+ aug=os.environ.get("HB_FW_AUG", "1") != "0",
82
+ batch_size=cfg.batch_size, max_steps=cfg.max_steps, lr=cfg.learning_rate)), flush=True)
83
+ if os.environ.get("HB_DATA_SAMPLE"): # CPU check: build the dataset and read augmented samples, no model
84
+ from omegaconf import open_dict
85
+ from fastwam.utils import misc
86
+ misc.register_work_dir(os.environ.get("HB_SAMPLE_WORKDIR", "/tmp/hbfw-sample"))
87
+ tr, _ = native.build_datasets(cfg.data)
88
+ s = tr[len(tr) // 2]
89
+
90
+ def desc(x):
91
+ if isinstance(x, dict):
92
+ return {k: desc(v) for k, v in x.items()}
93
+ if hasattr(x, "shape"):
94
+ return [tuple(x.shape), str(x.dtype), round(float(x.float().min()), 3), round(float(x.float().max()), 3)]
95
+ return type(x).__name__
96
+ print("HBFW_SAMPLE", len(tr), json.dumps(desc(s), default=str), flush=True)
97
+ raise SystemExit(0)
98
+ if os.environ.get("HB_INLOOP_EVERY", "250") != "0": # 2026-11-16: benchmark action nMSE every 250 steps
99
+ try:
100
+ import hb_inloop_fastwam
101
+ hb_inloop_fastwam.install()
102
+ except Exception as e: # noqa: BLE001 - never block training
103
+ print("HB_INLOOP fastwam install failed:", repr(e), flush=True)
104
+ native.run_training(cfg)
b8-table-wipe/b512/step-004000/code/hb_inloop_core.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared machinery of the in-training open-loop action MSE (Psi-Zero, Delta Flow; GR00T has its own tested copy in
2
+ hb_groot/hb_inloop_eval.py with the same protocol).
3
+
4
+ InLoop(model_name, task, split, out_jsonl, prepare, predict):
5
+ prepare(queries) -> optional per-model setup on this rank's queries (first eval only, inside the RNG fork)
6
+ predict(batch, seed) -> np.ndarray (B, >=16, >=Da) in raw action units; the first 16 steps and the first Da dims
7
+ (Da = benchmark action dim of the layout) are scored.
8
+ InLoop.evaluate(step, model): all train/eval modes of model.modules() and every RNG state (torch CPU + this CUDA device,
9
+ numpy, python) are restored afterwards; partial sums are all-reduced once (error flag included, so a failure on one
10
+ rank cannot leave the others waiting); rank 0 appends one json line per eval. Any failure disables later evals.
11
+ Protocol and metrics = hb_eval_common (post-hoc eval): queries every 16 frames, nMSE with the training-frame std.
12
+ """
13
+ import json
14
+ import os
15
+ import random
16
+ import sys
17
+ import time
18
+ import traceback
19
+ from pathlib import Path
20
+
21
+ import numpy as np
22
+ import torch
23
+
24
+ HERE = Path(__file__).resolve().parent
25
+ if str(HERE) not in sys.path:
26
+ sys.path.insert(0, str(HERE))
27
+ import hb_eval_common as E # noqa: E402
28
+
29
+ EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
30
+ BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
31
+ MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
32
+ SEED = 1234
33
+
34
+
35
+ def dist_info():
36
+ import torch.distributed as dist
37
+ if dist.is_available() and dist.is_initialized():
38
+ return dist.get_rank(), dist.get_world_size()
39
+ return int(os.environ.get("RANK", "0")), int(os.environ.get("WORLD_SIZE", "1"))
40
+
41
+
42
+ def pick_split(task):
43
+ split = os.environ.get("HB_INLOOP_SPLIT")
44
+ if not split:
45
+ split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
46
+ if not (E.split_dir(task, split) / "meta.json").exists():
47
+ raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
48
+ return split
49
+
50
+
51
+ class InLoop:
52
+ def __init__(self, model_name, task, split, out_jsonl, predict, prepare=None, cams=None, info=None):
53
+ self.model_name, self.task, self.split = model_name, task, split
54
+ self.out = Path(out_jsonl)
55
+ self.predict, self.prepare = predict, prepare
56
+ self.cams = cams
57
+ self.info = info or {}
58
+ self.disabled = False
59
+ self.ready = False
60
+ self.done_steps = set()
61
+
62
+ def say(self, *a):
63
+ if dist_info()[0] == 0:
64
+ print("HB_INLOOP", *a, flush=True)
65
+
66
+ def _setup(self):
67
+ rank, world = dist_info()
68
+ meta = E.load_meta(self.task, self.split)
69
+ self.meta, self.lay = meta, E.LAYOUTS[meta["layout"]]
70
+ self.std = E.norm_std(E.load_stats(self.task))
71
+ cams = self.cams or meta["cams"]
72
+ miss = [c for c in cams if c not in meta["cams"]]
73
+ if miss:
74
+ raise KeyError(f"cameras {miss} not in the eval cache {meta['cams']}")
75
+ allq = []
76
+ for ep in meta["episodes"]:
77
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
78
+ allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
79
+ if MAX_Q:
80
+ allq = allq[:MAX_Q]
81
+ self.n_total = len(allq)
82
+ mine = [q for i, q in enumerate(allq) if i % world == rank]
83
+ self.eps = list(meta["episodes"])
84
+ by_ep = {}
85
+ for ep, qi, t in mine:
86
+ by_ep.setdefault(ep, []).append((qi, t))
87
+ self.q = []
88
+ for ep, lst in by_ep.items():
89
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
90
+ state, action = z["state"], z["action"]
91
+ prompt = meta["prompts"][str(int(z["task_index"]))]
92
+ imgs = {c: z[f"img_{c}"] for c in cams}
93
+ for qi, t in lst:
94
+ gt, valid = E.gt_chunk(action, t)
95
+ self.q.append(dict(episode=int(ep), t=int(t), T=len(action), state=state[t].copy(), prompt=prompt,
96
+ images={c: imgs[c][qi].copy() for c in cams}, gt=gt, valid=valid,
97
+ past_action=action[max(0, t - E.HORIZON):t].copy()))
98
+ if self.prepare:
99
+ self.prepare(self.q)
100
+ self.ready = True
101
+ self.say(f"{self.model_name} ready: task {self.task} split {self.split} queries {self.n_total} "
102
+ f"(rank {rank}/{world}: {len(self.q)}), every {EVERY} steps, batch {BATCH}, cams {cams}")
103
+
104
+ def evaluate(self, step, model):
105
+ if self.disabled or step in self.done_steps:
106
+ return
107
+ self.done_steps.add(step)
108
+ rank, world = dist_info()
109
+ t0 = time.time()
110
+ err, msg = 0.0, ""
111
+ acc, nq, infer = None, 0, 0.0
112
+ modes = [(m, m.training) for m in model.modules()]
113
+ dev = torch.cuda.current_device() if torch.cuda.is_available() else None
114
+ np_state, py_state = np.random.get_state(), random.getstate()
115
+ try:
116
+ with torch.random.fork_rng(devices=[dev] if dev is not None else []):
117
+ if not self.ready:
118
+ self._setup()
119
+ D = self.lay["action_dim"]
120
+ model.eval()
121
+ acc = E.Accumulator(self.std, self.lay["groups"])
122
+ for bi in range(0, len(self.q), BATCH):
123
+ batch = self.q[bi:bi + BATCH]
124
+ seed = SEED + 1000 * rank + bi
125
+ torch.default_generator.manual_seed(seed)
126
+ if dev is not None:
127
+ torch.cuda.manual_seed(seed)
128
+ s = time.time()
129
+ with torch.no_grad():
130
+ pred = np.asarray(self.predict(batch, seed), dtype=np.float64)
131
+ infer += time.time() - s
132
+ if pred.ndim != 3 or pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] < D:
133
+ raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, >={D})")
134
+ for q, p in zip(batch, pred):
135
+ acc.add(q["episode"], p[:E.HORIZON, :D], q["gt"], q["valid"])
136
+ nq += 1
137
+ except Exception as e: # noqa: BLE001 - never stop training
138
+ err, msg = 1.0, f"{type(e).__name__}: {e}"
139
+ print(f"HB_INLOOP {self.model_name} rank {rank} step {step} FAILED: {msg}\n{traceback.format_exc()}",
140
+ file=sys.stderr, flush=True)
141
+ finally:
142
+ for m, tr in modes:
143
+ m.training = tr
144
+ np.random.set_state(np_state)
145
+ random.setstate(py_state)
146
+ D = self.lay["action_dim"] if self.ready else 0
147
+ H = E.HORIZON
148
+ eps = self.eps if self.ready else []
149
+ nE = len(eps)
150
+ vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
151
+ if acc is not None and not err:
152
+ o = 0
153
+ vec[o:o + D] = acc.sse_raw; o += D
154
+ vec[o:o + D] = acc.sae_raw; o += D
155
+ vec[o:o + D] = acc.sse_n; o += D
156
+ vec[o] = acc.rows; o += 1
157
+ vec[o:o + H] = acc.step_sse_n; o += H
158
+ vec[o:o + H] = acc.step_rows; o += H
159
+ for j, ep in enumerate(eps):
160
+ s_, c_ = acc.per_episode.get(int(ep), (0.0, 0))
161
+ vec[o + j], vec[o + nE + j] = s_, c_
162
+ o += 2 * nE
163
+ vec[o], vec[o + 1] = nq, infer
164
+ vec[-1] = err
165
+ tot = self._allreduce(vec)
166
+ if tot is None or tot[-1] > 0 or not self.ready:
167
+ self.disabled = True
168
+ nfail = None if tot is None else int(tot[-1])
169
+ self._write({"step": step, "error": msg or "failed on another rank / collective failed",
170
+ "ranks_failed": nfail})
171
+ self.say(f"{self.model_name} step {step}: eval failed ({nfail} rank(s)) -> in-training eval disabled: {msg}")
172
+ return
173
+ o = 0
174
+ acc = E.Accumulator(self.std, self.lay["groups"])
175
+ acc.sse_raw = tot[o:o + D]; o += D
176
+ acc.sae_raw = tot[o:o + D]; o += D
177
+ acc.sse_n = tot[o:o + D]; o += D
178
+ acc.rows = int(round(tot[o])); o += 1
179
+ acc.step_sse_n = tot[o:o + H]; o += H
180
+ acc.step_rows = tot[o:o + H]; o += H
181
+ acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
182
+ for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
183
+ o += 2 * nE
184
+ n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
185
+ res = acc.result()
186
+ res.pop("nmse_by_dim", None)
187
+ rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "model": self.model_name,
188
+ "task": self.task, "split": self.split, "queries": n_q, "ranks": world,
189
+ "wall_s": round(time.time() - t0, 1), "infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
190
+ "seed": SEED, **self.info, **res}
191
+ self._write(rec)
192
+ g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
193
+ self.say(f"{self.model_name} step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} "
194
+ f"queries, {rec['wall_s']} s")
195
+
196
+ @staticmethod
197
+ def _allreduce(vec):
198
+ import torch.distributed as dist
199
+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1:
200
+ return vec
201
+ try:
202
+ dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
203
+ n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
204
+ mx, mn = n.clone(), n.clone()
205
+ dist.all_reduce(mx, op=dist.ReduceOp.MAX)
206
+ dist.all_reduce(mn, op=dist.ReduceOp.MIN)
207
+ if mx.item() != mn.item():
208
+ return None
209
+ t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
210
+ dist.all_reduce(t)
211
+ return t.cpu().numpy()
212
+ except Exception as e: # noqa: BLE001
213
+ print(f"HB_INLOOP all_reduce failed: {e!r}", file=sys.stderr, flush=True)
214
+ return None
215
+
216
+ def _write(self, rec):
217
+ if dist_info()[0] != 0:
218
+ return
219
+ try:
220
+ self.out.parent.mkdir(parents=True, exist_ok=True)
221
+ with open(self.out, "a") as f:
222
+ f.write(json.dumps(rec) + "\n")
223
+ except Exception as e: # noqa: BLE001
224
+ print(f"HB_INLOOP write failed: {e!r}", flush=True)
225
+
226
+
227
+ class TempAttrs:
228
+ """set attributes on an object for the duration of a block, then restore (or delete) them"""
229
+ _MISSING = object()
230
+
231
+ def __init__(self, obj, **kw):
232
+ self.obj, self.kw, self.old = obj, kw, {}
233
+
234
+ def __enter__(self):
235
+ for k, v in self.kw.items():
236
+ self.old[k] = self.obj.__dict__.get(k, self._MISSING)
237
+ setattr(self.obj, k, v)
238
+ return self.obj
239
+
240
+ def __exit__(self, *exc):
241
+ for k, v in self.old.items():
242
+ if v is self._MISSING:
243
+ try:
244
+ delattr(self.obj, k)
245
+ except AttributeError:
246
+ pass
247
+ else:
248
+ setattr(self.obj, k, v)
249
+ return False
b8-table-wipe/b512/step-004000/code/hb_inloop_fastwam.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM in-training open-loop action MSE (2026-11-16 user: every 250 steps, like GR00T / Psi-Zero / Delta Flow).
2
+
3
+ install() is called by fastwam/tools/hb_fastwam_train.py right before native.run_training(cfg). It uses the trainer's
4
+ own eval slot: after Wan22Trainer.__init__ it sets eval_every = HB_INLOOP_EVERY (default 250) and a non-None
5
+ val_dataset marker, and replaces Wan22Trainer.evaluate (the native one generates a full video for one random val sample;
6
+ it is off in our runs, eval_every=0 on the command line). The native loop then calls evaluate() on every rank right after
7
+ the optimizer step of every 250th update, followed by accelerator.wait_for_everyone(); returning None logs nothing.
8
+ evaluate(): all ranks score their shard of the benchmark eval cache with the live model (accelerator.unwrap_model; ZeRO-1
9
+ keeps full bf16 weights on every rank) through EXACTLY the post-hoc adapter path (eval-action-mse/adapters/ad_fastwam.py
10
+ Inputs + infer: FastWAM.infer_action, the action-only deployment path; 10 denoising steps = config
11
+ eval_num_inference_steps; seed 42 per query; same cameras / proprio normalisation / T5 cache as training).
12
+ Results: <output_dir>/inloop-mse.jsonl (rank 0) + "HB_INLOOP" log lines. Training is untouched (module modes and every
13
+ RNG restored by hb_inloop_core, no gradients); any failure disables later evals on all ranks.
14
+ """
15
+ import json
16
+ import os
17
+ import sys
18
+ from pathlib import Path
19
+
20
+ G = Path(__file__).resolve().parents[2] # .../policies-20261109 (this file: $G/fastwam/tools/)
21
+ EVAL = G / "eval-action-mse"
22
+ for p in (str(EVAL), str(EVAL / "adapters")):
23
+ if p not in sys.path:
24
+ sys.path.insert(0, p)
25
+ import numpy as np # noqa: E402
26
+ import hb_eval_common as E # noqa: E402
27
+ import hb_inloop_core as C # noqa: E402
28
+ import ad_fastwam as A # noqa: E402
29
+
30
+
31
+ class _Marker:
32
+ """non-None val_dataset so the native loop reaches evaluate(); never indexed"""
33
+
34
+ def __len__(self):
35
+ return 1
36
+
37
+
38
+ class FwEval:
39
+ def __init__(self, trainer):
40
+ cfg = trainer.cfg
41
+ self.cfg = cfg
42
+ ddir = Path(list(cfg.data.train.dataset_dirs)[0])
43
+ iface = json.loads((ddir / "meta/hb_fastwam_interface.json").read_text())
44
+ task = E.task_for_source(iface["source"])
45
+ if task is None:
46
+ raise LookupError(f"no benchmark task for {iface['source']}")
47
+ split = C.pick_split(task)
48
+ self.steps = int(cfg.eval_num_inference_steps)
49
+ self.inp = None
50
+ self.model = None
51
+ out = Path(cfg.output_dir) / "inloop-mse.jsonl"
52
+ self.il = C.InLoop("fastwam", task, split, out, predict=self.predict, prepare=self.prepare,
53
+ info={"denoising_steps": self.steps, "seed_per_query": 42, "dataset": ddir.name,
54
+ "path": "FastWAM.infer_action (ad_fastwam)"})
55
+ self.il.cams = None
56
+
57
+ def prepare(self, queries):
58
+ self.inp = A.Inputs(self.cfg, Path(self.cfg.output_dir) / "dataset_stats.json")
59
+ miss = [c for c in self.inp.cams if c not in self.il.meta["cams"]]
60
+ if miss:
61
+ raise KeyError(f"cameras {miss} not in the eval cache {self.il.meta['cams']}")
62
+ for q in {q["prompt"] for q in queries}:
63
+ self.inp.context(q) # fail early if an instruction has no T5 cache
64
+
65
+ def predict(self, batch, seed):
66
+ return np.stack([np.asarray(A.infer(self.model, self.inp, q, self.steps, 42), dtype=np.float64) for q in batch])
67
+
68
+ def evaluate(self, step, model):
69
+ self.model = model
70
+ try:
71
+ self.il.evaluate(step, model)
72
+ finally:
73
+ self.model = None
74
+
75
+
76
+ class _Broken:
77
+ def __init__(self, err):
78
+ self.err = err
79
+ self.il = C.InLoop("fastwam", "b6", "train10", "/dev/null", predict=None, prepare=self._boom)
80
+
81
+ def _boom(self, _q):
82
+ raise RuntimeError(self.err)
83
+
84
+ def evaluate(self, step, model):
85
+ self.il.evaluate(step, model)
86
+
87
+
88
+ def install():
89
+ if C.EVERY <= 0:
90
+ return
91
+ from fastwam import trainer as T
92
+ WT = T.Wan22Trainer
93
+ orig_init = WT.__init__
94
+
95
+ def __init__(self, *a, **kw):
96
+ orig_init(self, *a, **kw)
97
+ # native eval off (eval_every 0 on our command line). NB runtime.build_datasets sets val_ds = train_ds when
98
+ # there is no data.val, so val_dataset is never None here (11-16 smoke: the old "val_dataset is None" test
99
+ # silently left the in-loop eval off).
100
+ if int(self.eval_every) == 0:
101
+ self.eval_every = C.EVERY
102
+ if self.val_dataset is None:
103
+ self.val_dataset = _Marker()
104
+ self._hb_inloop_on = True
105
+ else: # a native eval is configured: leave it alone
106
+ self._hb_inloop_on = False
107
+
108
+ def evaluate(self):
109
+ if not getattr(self, "_hb_inloop_on", False):
110
+ return orig_eval(self)
111
+ ev = getattr(self, "_hb_inloop", None)
112
+ if ev is None:
113
+ try:
114
+ ev = FwEval(self)
115
+ except Exception as e: # noqa: BLE001
116
+ ev = _Broken(f"{type(e).__name__}: {e}")
117
+ self._hb_inloop = ev
118
+ if not ev.il.disabled:
119
+ ev.evaluate(int(self.global_step), self.accelerator.unwrap_model(self.model))
120
+ return None
121
+
122
+ orig_eval = WT.evaluate
123
+ WT.__init__ = __init__
124
+ WT.evaluate = evaluate
125
+ if int(os.environ.get("RANK", "0")) == 0:
126
+ print(f"HB_INLOOP fastwam installed: every {C.EVERY} steps", flush=True)
b8-table-wipe/b512/step-004000/code/test_fastwam_inputs.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU check (no model): the FastWAM adapter's inputs built from the model-agnostic eval cache == the inputs our FastWAM
2
+ training dataset (RobotVideoDataset of the run's config, no HB augmentation) builds for the same frame.
3
+ python test_fastwam_inputs.py <task> <run out dir> [n_queries]"""
4
+ import json
5
+ import os
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+
11
+ HERE = Path(__file__).resolve().parent
12
+ sys.path.insert(0, str(HERE))
13
+ sys.path.insert(0, str(HERE / "adapters"))
14
+ import hb_eval_common as E # noqa: E402
15
+ import ad_fastwam as A # noqa: E402
16
+
17
+ task, out = sys.argv[1], Path(sys.argv[2])
18
+ nq = int(sys.argv[3]) if len(sys.argv) > 3 else 3
19
+ A.setup_env()
20
+ import torch # noqa: E402
21
+ from omegaconf import OmegaConf # noqa: E402
22
+ import fastwam.runtime as native # noqa: E402
23
+ from fastwam.utils import misc # noqa: E402
24
+
25
+ cfg = OmegaConf.load(out / "config.yaml")
26
+ inp = A.Inputs(cfg, out / "dataset_stats.json")
27
+ misc.register_work_dir("/tmp/hbfw-inputs-check")
28
+ OmegaConf.update(cfg, "data.train.pretrained_norm_stats", str(out / "dataset_stats.json"), merge=False, force_add=True)
29
+ tr, _ = native.build_datasets(cfg.data)
30
+ ddir = Path(cfg.data.train.dataset_dirs[0])
31
+ lens = [json.loads(l)["length"] for l in (ddir / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
32
+ meta = E.load_meta(task, "train10")
33
+ split = "train10"
34
+ worst = {}
35
+ done = 0
36
+ for ep, z in E.iter_episodes(task, split, None):
37
+ base = int(sum(lens[:ep]))
38
+ for qi, t in enumerate(list(z["query_t"])[1:1 + nq], start=1):
39
+ t = int(t)
40
+ s = tr[base + t]
41
+ obs = {"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]}, "state": z["state"][t],
42
+ "prompt": meta["prompts"][str(int(z["task_index"]))]}
43
+ img = inp.image(obs["images"])[0]
44
+ d_img = float((img - s["video"][:, 0].float()).abs().max())
45
+ pr = inp.proprio(obs["state"])[0]
46
+ d_pr = float((pr - s["proprio"][0].float()).abs().max())
47
+ ctx, mask = inp.context(obs["prompt"])
48
+ d_ctx = float((ctx.float() - s["context"].float()).abs().max())
49
+ d_mask = int((mask != s["context_mask"].bool()).sum())
50
+ gt = z["action"][t:t + 16]
51
+ a = inp.denorm(s["action"][:16])
52
+ d_act = float(np.abs(a[:len(gt)] - gt).max())
53
+ row = dict(ep=ep, t=t, idx=base + t, img=d_img, proprio=d_pr, ctx=d_ctx, mask=d_mask, act_denorm_vs_cache=d_act,
54
+ prompt_same=s["prompt"] == inp.DEFAULT_PROMPT.format(task=obs["prompt"]))
55
+ print("CHECK", json.dumps(row), flush=True)
56
+ for k in ("img", "proprio", "ctx", "act_denorm_vs_cache"):
57
+ worst[k] = max(worst.get(k, 0.0), row[k])
58
+ done += 1
59
+ if done >= 2 * nq:
60
+ break
61
+ print("WORST", json.dumps(worst))
b8-table-wipe/b512/step-004000/complete.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "b8",
3
+ "profile": "g1",
4
+ "per_gpu_batch": 16,
5
+ "global_batch": 512,
6
+ "gradient_accumulation_steps": 1,
7
+ "max_steps": 4000,
8
+ "save_every": 2000,
9
+ "recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
10
+ "interface": {
11
+ "profile": "g1",
12
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
13
+ "cameras": [
14
+ "ego_view",
15
+ "chest"
16
+ ],
17
+ "state_dim": 44,
18
+ "action_dim": 76,
19
+ "episodes": 100,
20
+ "frames": 61498,
21
+ "fps": 30,
22
+ "first_episodes_only": null
23
+ },
24
+ "finished_utc": "2026-11-17T21:03:31Z"
25
+ }
b8-table-wipe/b512/step-004000/dataset_meta/hb_fastwam_interface.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "profile": "g1",
3
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
4
+ "cameras": [
5
+ "ego_view",
6
+ "chest"
7
+ ],
8
+ "state_dim": 44,
9
+ "action_dim": 76,
10
+ "episodes": 100,
11
+ "frames": 61498,
12
+ "fps": 30,
13
+ "first_episodes_only": null
14
+ }
b8-table-wipe/b512/step-004000/dataset_meta/info.json ADDED
@@ -0,0 +1,538 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "codebase_version": "v2.1",
3
+ "robot_type": null,
4
+ "total_episodes": 100,
5
+ "total_frames": 61498,
6
+ "total_tasks": 1,
7
+ "total_videos": 200,
8
+ "total_chunks": 1,
9
+ "chunks_size": 1000,
10
+ "fps": 30,
11
+ "splits": {
12
+ "train": "0:100"
13
+ },
14
+ "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
15
+ "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
16
+ "features": {
17
+ "observation.state": {
18
+ "dtype": "float32",
19
+ "shape": [
20
+ 44
21
+ ],
22
+ "names": null
23
+ },
24
+ "observation.eef_state": {
25
+ "dtype": "float64",
26
+ "shape": [
27
+ 14
28
+ ],
29
+ "names": [
30
+ "left_wrist_pos",
31
+ "left_wrist_abs_quat",
32
+ "right_wrist_pos",
33
+ "right_wrist_abs_quat"
34
+ ]
35
+ },
36
+ "action.wbc": {
37
+ "dtype": "float64",
38
+ "shape": [
39
+ 41
40
+ ],
41
+ "names": [
42
+ "left_hip_pitch_joint",
43
+ "left_hip_roll_joint",
44
+ "left_hip_yaw_joint",
45
+ "left_knee_joint",
46
+ "left_ankle_pitch_joint",
47
+ "left_ankle_roll_joint",
48
+ "right_hip_pitch_joint",
49
+ "right_hip_roll_joint",
50
+ "right_hip_yaw_joint",
51
+ "right_knee_joint",
52
+ "right_ankle_pitch_joint",
53
+ "right_ankle_roll_joint",
54
+ "waist_yaw_joint",
55
+ "waist_roll_joint",
56
+ "waist_pitch_joint",
57
+ "left_shoulder_pitch_joint",
58
+ "left_shoulder_roll_joint",
59
+ "left_shoulder_yaw_joint",
60
+ "left_elbow_joint",
61
+ "left_wrist_roll_joint",
62
+ "left_wrist_pitch_joint",
63
+ "left_wrist_yaw_joint",
64
+ "right_shoulder_pitch_joint",
65
+ "right_shoulder_roll_joint",
66
+ "right_shoulder_yaw_joint",
67
+ "right_elbow_joint",
68
+ "right_wrist_roll_joint",
69
+ "right_wrist_pitch_joint",
70
+ "right_wrist_yaw_joint",
71
+ "left_hand_pinky",
72
+ "left_hand_ring",
73
+ "left_hand_middle",
74
+ "left_hand_index",
75
+ "left_hand_thumb_bend",
76
+ "left_hand_thumb_rotation",
77
+ "right_hand_pinky",
78
+ "right_hand_ring",
79
+ "right_hand_middle",
80
+ "right_hand_index",
81
+ "right_hand_thumb_bend",
82
+ "right_hand_thumb_rotation"
83
+ ]
84
+ },
85
+ "observation.root_orientation": {
86
+ "dtype": "float64",
87
+ "shape": [
88
+ 4
89
+ ],
90
+ "names": [
91
+ "base_qw",
92
+ "base_qx",
93
+ "base_qy",
94
+ "base_qz"
95
+ ]
96
+ },
97
+ "observation.projected_gravity": {
98
+ "dtype": "float64",
99
+ "shape": [
100
+ 3
101
+ ],
102
+ "names": [
103
+ "gravity_x",
104
+ "gravity_y",
105
+ "gravity_z"
106
+ ]
107
+ },
108
+ "observation.cpp_rotation_offset": {
109
+ "dtype": "float64",
110
+ "shape": [
111
+ 4
112
+ ],
113
+ "names": [
114
+ "rot_offset_qw",
115
+ "rot_offset_qx",
116
+ "rot_offset_qy",
117
+ "rot_offset_qz"
118
+ ]
119
+ },
120
+ "observation.init_base_quat": {
121
+ "dtype": "float64",
122
+ "shape": [
123
+ 4
124
+ ],
125
+ "names": [
126
+ "init_base_qw",
127
+ "init_base_qx",
128
+ "init_base_qy",
129
+ "init_base_qz"
130
+ ]
131
+ },
132
+ "teleop.delta_heading": {
133
+ "dtype": "float64",
134
+ "shape": [
135
+ 1
136
+ ],
137
+ "names": [
138
+ "delta_heading"
139
+ ]
140
+ },
141
+ "action.motion_token": {
142
+ "dtype": "float64",
143
+ "shape": [
144
+ 64
145
+ ],
146
+ "names": "motion_token"
147
+ },
148
+ "teleop.smpl_joints": {
149
+ "dtype": "float32",
150
+ "shape": [
151
+ 72
152
+ ],
153
+ "names": "smpl_joints"
154
+ },
155
+ "teleop.smpl_pose": {
156
+ "dtype": "float32",
157
+ "shape": [
158
+ 63
159
+ ],
160
+ "names": "smpl_pose"
161
+ },
162
+ "teleop.body_quat_w": {
163
+ "dtype": "float32",
164
+ "shape": [
165
+ 4
166
+ ],
167
+ "names": "body_quat_w"
168
+ },
169
+ "teleop.target_body_orientation": {
170
+ "dtype": "float32",
171
+ "shape": [
172
+ 6
173
+ ],
174
+ "names": [
175
+ "target_body_r00",
176
+ "target_body_r10",
177
+ "target_body_r01",
178
+ "target_body_r11",
179
+ "target_body_r02",
180
+ "target_body_r12"
181
+ ]
182
+ },
183
+ "teleop.left_hand_joints": {
184
+ "dtype": "float32",
185
+ "shape": [
186
+ 7
187
+ ],
188
+ "names": "left_hand_joints"
189
+ },
190
+ "teleop.right_hand_joints": {
191
+ "dtype": "float32",
192
+ "shape": [
193
+ 7
194
+ ],
195
+ "names": "right_hand_joints"
196
+ },
197
+ "teleop.inspire_left_hand_cmd": {
198
+ "dtype": "float32",
199
+ "shape": [
200
+ 6
201
+ ],
202
+ "names": [
203
+ "left_pinky",
204
+ "left_ring",
205
+ "left_middle",
206
+ "left_index",
207
+ "left_thumb_pitch",
208
+ "left_thumb_yaw"
209
+ ]
210
+ },
211
+ "teleop.inspire_right_hand_cmd": {
212
+ "dtype": "float32",
213
+ "shape": [
214
+ 6
215
+ ],
216
+ "names": [
217
+ "right_pinky",
218
+ "right_ring",
219
+ "right_middle",
220
+ "right_index",
221
+ "right_thumb_pitch",
222
+ "right_thumb_yaw"
223
+ ]
224
+ },
225
+ "teleop.inspire_hand_cmd_age_ms": {
226
+ "dtype": "float32",
227
+ "shape": [
228
+ 1
229
+ ],
230
+ "names": [
231
+ "inspire_hand_cmd_age_ms"
232
+ ]
233
+ },
234
+ "teleop.controller": {
235
+ "dtype": "float32",
236
+ "shape": [
237
+ 4
238
+ ],
239
+ "names": [
240
+ "left_trigger",
241
+ "right_trigger",
242
+ "left_grip",
243
+ "right_grip"
244
+ ]
245
+ },
246
+ "teleop.pose_sample_age_ms": {
247
+ "dtype": "float32",
248
+ "shape": [
249
+ 1
250
+ ],
251
+ "names": [
252
+ "pose_sample_age_ms"
253
+ ]
254
+ },
255
+ "teleop.smpl_frame_index": {
256
+ "dtype": "int64",
257
+ "shape": [
258
+ 1
259
+ ],
260
+ "names": [
261
+ "smpl_frame_index"
262
+ ]
263
+ },
264
+ "teleop.left_wrist_joints": {
265
+ "dtype": "float32",
266
+ "shape": [
267
+ 3
268
+ ],
269
+ "names": [
270
+ "left_wrist_roll",
271
+ "left_wrist_pitch",
272
+ "left_wrist_yaw"
273
+ ]
274
+ },
275
+ "teleop.right_wrist_joints": {
276
+ "dtype": "float32",
277
+ "shape": [
278
+ 3
279
+ ],
280
+ "names": [
281
+ "right_wrist_roll",
282
+ "right_wrist_pitch",
283
+ "right_wrist_yaw"
284
+ ]
285
+ },
286
+ "teleop.stream_mode": {
287
+ "dtype": "int32",
288
+ "shape": [
289
+ 1
290
+ ],
291
+ "names": [
292
+ "stream_mode"
293
+ ]
294
+ },
295
+ "teleop.planner_mode": {
296
+ "dtype": "int32",
297
+ "shape": [
298
+ 1
299
+ ],
300
+ "names": [
301
+ "locomotion_mode"
302
+ ]
303
+ },
304
+ "teleop.planner_movement": {
305
+ "dtype": "float32",
306
+ "shape": [
307
+ 3
308
+ ],
309
+ "names": [
310
+ "movement_x",
311
+ "movement_y",
312
+ "movement_z"
313
+ ]
314
+ },
315
+ "teleop.planner_facing": {
316
+ "dtype": "float32",
317
+ "shape": [
318
+ 3
319
+ ],
320
+ "names": [
321
+ "facing_x",
322
+ "facing_y",
323
+ "facing_z"
324
+ ]
325
+ },
326
+ "teleop.planner_speed": {
327
+ "dtype": "float32",
328
+ "shape": [
329
+ 1
330
+ ],
331
+ "names": [
332
+ "speed"
333
+ ]
334
+ },
335
+ "teleop.planner_height": {
336
+ "dtype": "float32",
337
+ "shape": [
338
+ 1
339
+ ],
340
+ "names": [
341
+ "height"
342
+ ]
343
+ },
344
+ "teleop.vr_3pt_position": {
345
+ "dtype": "float32",
346
+ "shape": [
347
+ 9
348
+ ],
349
+ "names": [
350
+ "lwrist_x",
351
+ "lwrist_y",
352
+ "lwrist_z",
353
+ "rwrist_x",
354
+ "rwrist_y",
355
+ "rwrist_z",
356
+ "neck_x",
357
+ "neck_y",
358
+ "neck_z"
359
+ ]
360
+ },
361
+ "teleop.vr_3pt_orientation": {
362
+ "dtype": "float32",
363
+ "shape": [
364
+ 18
365
+ ],
366
+ "names": [
367
+ "lwrist_r00",
368
+ "lwrist_r10",
369
+ "lwrist_r01",
370
+ "lwrist_r11",
371
+ "lwrist_r02",
372
+ "lwrist_r12",
373
+ "rwrist_r00",
374
+ "rwrist_r10",
375
+ "rwrist_r01",
376
+ "rwrist_r11",
377
+ "rwrist_r02",
378
+ "rwrist_r12",
379
+ "neck_r00",
380
+ "neck_r10",
381
+ "neck_r01",
382
+ "neck_r11",
383
+ "neck_r02",
384
+ "neck_r12"
385
+ ]
386
+ },
387
+ "observation.images.ego_view": {
388
+ "dtype": "video",
389
+ "shape": [
390
+ 480,
391
+ 640,
392
+ 3
393
+ ],
394
+ "names": [
395
+ "height",
396
+ "width",
397
+ "channel"
398
+ ],
399
+ "info": {
400
+ "video.height": 480,
401
+ "video.width": 640,
402
+ "video.codec": "h264",
403
+ "video.pix_fmt": "yuv420p",
404
+ "video.is_depth_map": false,
405
+ "video.fps": 30,
406
+ "video.channels": 3,
407
+ "has_audio": false
408
+ }
409
+ },
410
+ "observation.root_position": {
411
+ "dtype": "float64",
412
+ "shape": [
413
+ 3
414
+ ],
415
+ "names": [
416
+ "base_x",
417
+ "base_y",
418
+ "base_z"
419
+ ]
420
+ },
421
+ "observation.inspire_left_hand_state": {
422
+ "dtype": "float32",
423
+ "shape": [
424
+ 6
425
+ ],
426
+ "names": [
427
+ "left_pinky",
428
+ "left_ring",
429
+ "left_middle",
430
+ "left_index",
431
+ "left_thumb_bend",
432
+ "left_thumb_rotation"
433
+ ]
434
+ },
435
+ "observation.inspire_right_hand_state": {
436
+ "dtype": "float32",
437
+ "shape": [
438
+ 6
439
+ ],
440
+ "names": [
441
+ "right_pinky",
442
+ "right_ring",
443
+ "right_middle",
444
+ "right_index",
445
+ "right_thumb_bend",
446
+ "right_thumb_rotation"
447
+ ]
448
+ },
449
+ "observation.images.chest": {
450
+ "dtype": "video",
451
+ "shape": [
452
+ 480,
453
+ 640,
454
+ 3
455
+ ],
456
+ "names": [
457
+ "height",
458
+ "width",
459
+ "channel"
460
+ ],
461
+ "info": {
462
+ "video.height": 480,
463
+ "video.width": 640,
464
+ "video.codec": "h264",
465
+ "video.pix_fmt": "yuv420p",
466
+ "video.is_depth_map": false,
467
+ "video.fps": 30,
468
+ "video.channels": 3,
469
+ "has_audio": false
470
+ }
471
+ },
472
+ "timestamp": {
473
+ "dtype": "float32",
474
+ "shape": [
475
+ 1
476
+ ],
477
+ "names": null
478
+ },
479
+ "frame_index": {
480
+ "dtype": "int64",
481
+ "shape": [
482
+ 1
483
+ ],
484
+ "names": null
485
+ },
486
+ "episode_index": {
487
+ "dtype": "int64",
488
+ "shape": [
489
+ 1
490
+ ],
491
+ "names": null
492
+ },
493
+ "index": {
494
+ "dtype": "int64",
495
+ "shape": [
496
+ 1
497
+ ],
498
+ "names": null
499
+ },
500
+ "task_index": {
501
+ "dtype": "int64",
502
+ "shape": [
503
+ 1
504
+ ],
505
+ "names": null
506
+ },
507
+ "action": {
508
+ "dtype": "float32",
509
+ "shape": [
510
+ 76
511
+ ],
512
+ "names": null
513
+ }
514
+ },
515
+ "script_config": {
516
+ "source": "sim",
517
+ "simulator": "newton",
518
+ "controller": "SONIC g1_deploy_onnx_ref in sim lockstep (planner and upper-body commands, no PICO stream)",
519
+ "exporter": "humanoid_newton.collect.lerobot_export",
520
+ "control_frequency": 50.0,
521
+ "record_wrist_cameras": false,
522
+ "record_chest_camera": true,
523
+ "rotate_chest_camera_180": false,
524
+ "record_depth": false,
525
+ "record_root_position": true,
526
+ "record_inspire_hand_state": true,
527
+ "record_inspire_hand_cmd": true,
528
+ "no_camera": false,
529
+ "camera_backend": "offline_render",
530
+ "camera_note": "ego_view and chest rendered offline (humanoid_newton.rendering.splat_render) from the recorded body poses at the registered D455 mounts; chest rendered upright (the real exporter rotates its upside-down chest stream by 180 deg); cameras not calibrated against the real ones",
531
+ "task": "b8",
532
+ "collect_config": "20261114_black_coffee_training_100_01",
533
+ "scene": "eai_room_fridge_001",
534
+ "task_description": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it.",
535
+ "git_commit": null
536
+ },
537
+ "discarded_episode_indices": []
538
+ }
b8-table-wipe/b512/step-004000/dataset_meta/tasks.jsonl ADDED
@@ -0,0 +1 @@
 
 
1
+ {"task_index": 0, "task": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it."}
b8-table-wipe/b512/step-004000/dataset_stats.json ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-004000/logs/node-0.log ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-004000/logs/node-1.log ADDED
The diff for this file is too large to render. See raw diff
 
b8-table-wipe/b512/step-004000/logs/node-2.log ADDED
The diff for this file is too large to render. See raw diff