BIue-Jay commited on
Commit
aab7217
·
verified ·
1 Parent(s): 2de6b2a

a18cr2: last two checkpoints + eval/deploy files + training records

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. a18-pour-water/b512/step-002000/batch_benchmark/chosen.json +1 -0
  2. a18-pour-water/b512/step-002000/batch_benchmark/summary.txt +2 -0
  3. a18-pour-water/b512/step-002000/code/ad_fastwam.py +163 -0
  4. a18-pour-water/b512/step-002000/code/fastwam_prep.sh +61 -0
  5. a18-pour-water/b512/step-002000/code/fastwam_task.sh +48 -0
  6. a18-pour-water/b512/step-002000/code/hb_aug_v2.py +163 -0
  7. a18-pour-water/b512/step-002000/code/hb_eval_common.py +258 -0
  8. a18-pour-water/b512/step-002000/code/hb_fastwam_aug.py +73 -0
  9. a18-pour-water/b512/step-002000/code/hb_fastwam_convert.py +121 -0
  10. a18-pour-water/b512/step-002000/code/hb_fastwam_train.py +104 -0
  11. a18-pour-water/b512/step-002000/code/hb_inloop_core.py +249 -0
  12. a18-pour-water/b512/step-002000/code/hb_inloop_fastwam.py +126 -0
  13. a18-pour-water/b512/step-002000/code/test_fastwam_inputs.py +61 -0
  14. a18-pour-water/b512/step-002000/complete.json +25 -0
  15. a18-pour-water/b512/step-002000/dataset_meta/hb_fastwam_interface.json +14 -0
  16. a18-pour-water/b512/step-002000/dataset_meta/info.json +450 -0
  17. a18-pour-water/b512/step-002000/dataset_meta/tasks.jsonl +1 -0
  18. a18-pour-water/b512/step-002000/dataset_stats.json +0 -0
  19. a18-pour-water/b512/step-002000/logs/node-0.log +0 -0
  20. a18-pour-water/b512/step-002000/logs/node-1.log +0 -0
  21. a18-pour-water/b512/step-002000/logs/node-2.log +0 -0
  22. a18-pour-water/b512/step-002000/logs/node-3.log +0 -0
  23. a18-pour-water/b512/step-002000/loss_summary.json +23 -0
  24. a18-pour-water/b512/step-002000/metrics.csv +201 -0
  25. a18-pour-water/b512/step-002000/model.pt +3 -0
  26. a18-pour-water/b512/step-002000/model_config.yaml +57 -0
  27. a18-pour-water/b512/step-002000/protocol.json +72 -0
  28. a18-pour-water/b512/step-002000/text_embeddings/cd0ca5d7666fa68fe2141dfac894ceea404ab7cf2a88b8617a489f124dfdfdf2.t5_len128.wan22ti2v5b.pt +3 -0
  29. a18-pour-water/b512/step-002000/training_config.yaml +258 -0
  30. a18-pour-water/b512/step-004000/batch_benchmark/chosen.json +1 -0
  31. a18-pour-water/b512/step-004000/batch_benchmark/summary.txt +2 -0
  32. a18-pour-water/b512/step-004000/code/ad_fastwam.py +163 -0
  33. a18-pour-water/b512/step-004000/code/fastwam_prep.sh +61 -0
  34. a18-pour-water/b512/step-004000/code/fastwam_task.sh +48 -0
  35. a18-pour-water/b512/step-004000/code/hb_aug_v2.py +163 -0
  36. a18-pour-water/b512/step-004000/code/hb_eval_common.py +258 -0
  37. a18-pour-water/b512/step-004000/code/hb_fastwam_aug.py +73 -0
  38. a18-pour-water/b512/step-004000/code/hb_fastwam_convert.py +121 -0
  39. a18-pour-water/b512/step-004000/code/hb_fastwam_train.py +104 -0
  40. a18-pour-water/b512/step-004000/code/hb_inloop_core.py +249 -0
  41. a18-pour-water/b512/step-004000/code/hb_inloop_fastwam.py +126 -0
  42. a18-pour-water/b512/step-004000/code/test_fastwam_inputs.py +61 -0
  43. a18-pour-water/b512/step-004000/complete.json +25 -0
  44. a18-pour-water/b512/step-004000/dataset_meta/hb_fastwam_interface.json +14 -0
  45. a18-pour-water/b512/step-004000/dataset_meta/info.json +450 -0
  46. a18-pour-water/b512/step-004000/dataset_meta/tasks.jsonl +1 -0
  47. a18-pour-water/b512/step-004000/dataset_stats.json +0 -0
  48. a18-pour-water/b512/step-004000/logs/node-0.log +0 -0
  49. a18-pour-water/b512/step-004000/logs/node-1.log +0 -0
  50. a18-pour-water/b512/step-004000/logs/node-2.log +0 -0
a18-pour-water/b512/step-002000/batch_benchmark/chosen.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"per_gpu_batch": 16, "global_batch": 512, "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/bench-20261116-r2/summary.txt"}
a18-pour-water/b512/step-002000/batch_benchmark/summary.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ bs=16 global=512 rc=[0 0 0 0 ] oom=0 peak_mem_MiB=58421 speed=
2
+ bs=32 global=1024 rc=[1 1 1 1 ] oom=4 peak_mem_MiB=81123 speed=
a18-pour-water/b512/step-002000/code/ad_fastwam.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM adapter (2026-11-16). Deployment path of FastWAM's own policy (experiments/robotwin/fastwam_policy/
2
+ deploy_policy.py, WorldActionRobotWinPolicy._infer_action_chunk: model.infer_action, the action-only path = first-frame
3
+ video KV cache + action-expert denoising, no future video) fed exactly like our training dataset (RobotVideoDataset +
4
+ FastWAMProcessor of the run's config.yaml, no HB augmentation):
5
+ * cameras in shape_meta order (g1 ego_view + chest, h2 head + chest): uint8 [1,C,H,W] -> processor val transforms
6
+ (ToTensor /255 + Resize 224x224), concatenated horizontally (224 x 448), dataset ResizeSmallestSide/CenterCrop
7
+ (no-op at 224 x 448), Normalize(0.5, 0.5) -> [-1, 1];
8
+ * state: processor.action_state_transform -> normalizer.forward (min/max of the run's
9
+ dataset_stats.json), the proprio of the first frame;
10
+ * text: the run's T5 cache for DEFAULT_PROMPT.format(task=instruction) with context[~mask] = 0 and mask = ones, as in
11
+ training (the text encoder is not loaded in training either; deploy_policy encodes the same prompt with T5 online);
12
+ * infer_action(action_horizon = num_frames - 1 = 16, num_inference_steps = config eval_num_inference_steps (10),
13
+ text_cfg_scale 1, seed 42 = native trainer eval), denormalised with normalizer.normalizers["action"][key].backward.
14
+ ckpt = .../fastwam/runs/<task>/out/checkpoints/weights/step_<N>.pt (config.yaml + dataset_stats.json in .../out).
15
+ Weights: model built from the config (Wan2.2 + ActionDiT init), then the checkpoint's mot + proprio_encoder are
16
+ loaded STRICTLY (FastWAM.load_checkpoint uses strict=False for mot; a missing key here is an error).
17
+ """
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+
24
+ import hb_eval_common as E
25
+
26
+ FW = E.G / "fastwam"
27
+
28
+
29
+ def setup_env():
30
+ os.environ.setdefault("FASTWAM_SRC", str(FW / "FastWAM"))
31
+ os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(FW / "models"))
32
+ os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "true")
33
+ os.environ.setdefault("HF_HUB_OFFLINE", "1")
34
+ src = str(Path(os.environ["FASTWAM_SRC"]) / "src")
35
+ if src not in sys.path:
36
+ sys.path.insert(0, src)
37
+
38
+
39
+ class Inputs:
40
+ """model-agnostic cache query -> FastWAM inputs (shared by the post-hoc adapter and the in-training eval)"""
41
+
42
+ def __init__(self, cfg, stats_path):
43
+ setup_env()
44
+ from hydra.utils import instantiate
45
+ from omegaconf import OmegaConf
46
+ from fastwam.datasets.dataset_utils import CenterCrop, Normalize, ResizeSmallestSideAspectPreserving
47
+ from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT
48
+ from fastwam.datasets.lerobot.utils.normalizer import load_dataset_stats_from_json
49
+ d = cfg.data.train
50
+ self.DEFAULT_PROMPT = DEFAULT_PROMPT
51
+ self.proc = instantiate(d.processor).eval()
52
+ self.proc.set_normalizer_from_stats(load_dataset_stats_from_json(str(stats_path)))
53
+ self.cams = [m["key"] for m in self.proc.shape_meta["images"]]
54
+ if d.concat_multi_camera != "horizontal":
55
+ raise ValueError(f"concat_multi_camera {d.concat_multi_camera} not supported")
56
+ vs = list(d.video_size)
57
+ self.resize = ResizeSmallestSideAspectPreserving(args={"img_w": vs[1], "img_h": vs[0]})
58
+ self.crop = CenterCrop(args={"img_w": vs[1], "img_h": vs[0]})
59
+ self.norm = Normalize(args={"mean": 0.5, "std": 0.5})
60
+ self.cache_dir = d.text_embedding_cache_dir
61
+ self.context_len = int(d.context_len)
62
+ self.horizon = int(d.num_frames) - 1
63
+ self.skey = self.proc.shape_meta["state"][0]["key"]
64
+ self.akey = self.proc.shape_meta["action"][0]["key"]
65
+ if len(self.proc.shape_meta["state"]) != 1 or len(self.proc.shape_meta["action"]) != 1:
66
+ raise ValueError("expected one merged state key and one merged action key")
67
+ self._ctx = {}
68
+ self.OmegaConf = OmegaConf
69
+
70
+ def image(self, images):
71
+ import torch
72
+ per = []
73
+ for c in self.cams:
74
+ x = torch.from_numpy(np.ascontiguousarray(images[c])).permute(2, 0, 1).unsqueeze(0) # [1,C,H,W] uint8
75
+ for tr in self.proc.val_transforms:
76
+ x = tr(x)
77
+ per.append(x)
78
+ v = torch.cat(per, dim=-1) # [1, C, 224, 448]
79
+ v = self.norm(self.crop(self.resize(v)))
80
+ return v # [1, 3, H, W] in [-1, 1], float32
81
+
82
+ def proprio(self, state):
83
+ import torch
84
+ b = {"state": {self.skey: torch.as_tensor(np.asarray(state, dtype=np.float32)).unsqueeze(0)}}
85
+ b = self.proc.action_state_transform(b)
86
+ b = self.proc.normalizer.forward(b) # = deploy_policy._normalize_state (one merged key: merger is identity)
87
+ return b["state"][self.skey] # [1, D]
88
+
89
+ def context(self, instruction):
90
+ import hashlib
91
+ import torch
92
+ prompt = self.DEFAULT_PROMPT.format(task=instruction)
93
+ if prompt not in self._ctx:
94
+ h = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
95
+ p = os.path.join(self.cache_dir, f"{h}.t5_len{self.context_len}.wan22ti2v5b.pt")
96
+ if not os.path.exists(p):
97
+ raise FileNotFoundError(f"no T5 cache for the instruction ({p}): {instruction[:80]!r}")
98
+ obj = torch.load(p, map_location="cpu")
99
+ ctx, mask = obj["context"], obj["mask"]
100
+ ctx = ctx.clone()
101
+ ctx[~mask.bool()] = 0.0
102
+ self._ctx[prompt] = (ctx, torch.ones_like(mask.bool()))
103
+ return self._ctx[prompt]
104
+
105
+ def denorm(self, action):
106
+ import torch
107
+ a = action
108
+ if a.ndim == 2:
109
+ a = a.unsqueeze(0)
110
+ n = self.proc.normalizer.normalizers["action"][self.akey]
111
+ return n.backward(a.to(dtype=torch.float32, device="cpu")).numpy()[0]
112
+
113
+
114
+ def infer(model, inp, obs, steps, seed=42):
115
+ import torch
116
+ img = inp.image(obs["images"]).to(device=model.device, dtype=model.torch_dtype)
117
+ ctx, mask = inp.context(obs["prompt"])
118
+ with torch.no_grad():
119
+ out = model.infer_action(prompt=None, input_image=img, action_horizon=inp.horizon, proprio=inp.proprio(obs["state"]),
120
+ context=ctx, context_mask=mask, text_cfg_scale=1.0, num_inference_steps=steps, seed=seed,
121
+ rand_device="cpu", tiled=False)
122
+ return inp.denorm(out["action"])
123
+
124
+
125
+ class Adapter:
126
+ def __init__(self, ckpt, task, split, meta, device="cuda:0", steps=None, seed="42"):
127
+ setup_env()
128
+ import torch
129
+ from hydra.utils import instantiate
130
+ from omegaconf import OmegaConf
131
+ self.torch = torch
132
+ ckpt = Path(ckpt)
133
+ out = ckpt.parent.parent.parent
134
+ self.cfg = OmegaConf.load(out / "config.yaml")
135
+ self.inp = Inputs(self.cfg, out / "dataset_stats.json")
136
+ miss = [c for c in self.inp.cams if c not in meta["cams"]]
137
+ if miss:
138
+ raise KeyError(f"cameras {miss} not in the cache {meta['cams']}")
139
+ self.device = device if torch.cuda.is_available() else "cpu"
140
+ mp = str(self.cfg.get("mixed_precision", "bf16")).lower()
141
+ dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[mp]
142
+ self.model = instantiate(self.cfg.model, model_dtype=dtype, device=self.device)
143
+ payload = torch.load(str(ckpt), map_location="cpu")
144
+ miss_k, unexp = self.model.mot.load_state_dict(payload["mot"], strict=False)
145
+ if miss_k or unexp:
146
+ raise RuntimeError(f"checkpoint mot keys: missing {list(miss_k)[:5]} unexpected {list(unexp)[:5]}")
147
+ if self.model.proprio_encoder is not None:
148
+ self.model.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
149
+ self.ckpt_step = payload.get("step")
150
+ del payload
151
+ self.model = self.model.to(self.device).eval()
152
+ self.native_steps = int(self.cfg.eval_num_inference_steps)
153
+ self.steps = int(steps or self.native_steps)
154
+ self.seed = int(seed)
155
+ self.ckpt = str(ckpt)
156
+
157
+ def info(self):
158
+ return {"model": "fastwam", "ckpt": self.ckpt, "ckpt_step": self.ckpt_step, "denoising_steps": self.steps,
159
+ "native_steps_in_config": self.native_steps, "cams": self.inp.cams, "chunk": self.inp.horizon,
160
+ "seed": self.seed, "path": "FastWAM.infer_action (action-only, deploy_policy)"}
161
+
162
+ def predict(self, obs):
163
+ return infer(self.model, self.inp, obs, self.steps, self.seed)
a18-pour-water/b512/step-002000/code/fastwam_prep.sh ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # 2026-11-16: CPU-only preparation of FastWAM for the 12 Delta-flow tasks (no GPU is touched: training is running).
3
+ # Data copies on our interface, ActionDiT initializers per profile, one shared T5 text cache, augmented-sample checks.
4
+ P=/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z
5
+ G=$P/policies-20261109
6
+ F=$G/fastwam
7
+ E=$G/envs/fastwam/bin/python
8
+ exec >> $F/prep.log 2>&1
9
+ echo "$(date) start (v2: 12 tasks)"
10
+ until [ -f $G/envs/fastwam/.hb_ready ] || grep -q ENV_FAILED $F/setup.log; do sleep 30; done
11
+ [ -f $G/envs/fastwam/.hb_ready ] || { echo "ENV_FAILED"; exit 1; }
12
+ grep -q FW_DL_DONE $F/dl2.log || { echo "DOWNLOAD_FAILED"; exit 1; }
13
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true CUDA_VISIBLE_DEVICES="" HF_HUB_OFFLINE=1
14
+ export HB_AUG_DIR=$G/hb_groot
15
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT $(compgen -e | grep -E "^PET_")
16
+ TC=$F/text-cache/all
17
+ # task profile source [first N episodes]
18
+ cat > $F/tasks.txt <<EOF
19
+ a10v3 g1 $P/a10-trolley-push-v3-6e410c5/data
20
+ a3v4 g1 $P/a3-fridge-open-v4-6393d4a/data
21
+ a7v4 g1 $P/a7-cabinet-close-v4-90cc2b5/data
22
+ b7 g1 $P/b7-black-clothes-hamper-v1-1bb002a/data 100
23
+ b5 g1 $P/b5-chair-push-v1-b85b7e9/data
24
+ b8 g1 $P/b8-table-wipe-v1-3b2e618/data
25
+ a17cr2 h2 $P/a17-unscrew-bottle-cropped-v2-3520b70/data
26
+ basketv2 g1 $P/basket-trimv2-c4df4d4/data
27
+ c6 g1 $P/c6-bathroom-door-fetch-v1-3afba69/data
28
+ b6v6 g1 $P/b6-pedal-bin-v6-ad2f5bf/data
29
+ a14v2 h2p $P/a14-unscrew-bulb-v2-8411a6d/data
30
+ b1 g1 $P/b1-riceball-microwave-v1-36da298/data
31
+ EOF
32
+ DIRS=""
33
+ while read t prof src lim; do
34
+ D=$F/data/$t
35
+ if [ ! -f $D/meta/hb_fastwam_interface.json ]; then
36
+ $E $F/tools/hb_fastwam_convert.py $prof $src $D ${lim:+--episodes $lim} || { echo "CONVERT_FAILED $t"; continue; }
37
+ fi
38
+ DIRS="$DIRS,\"$D\""
39
+ done < $F/tasks.txt
40
+ DIRS="[${DIRS#,}]"
41
+ echo "$(date) CONVERTED $DIRS"
42
+ cd $F/FastWAM
43
+ for prof in g1 h2 h2p; do
44
+ AI=$F/action-init-$prof.pt
45
+ [ -f $AI ] && continue
46
+ HB_FW_PROFILE=$prof HB_MODEL_YAML=$F/model-$prof.yaml $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" || { echo "YAML_FAILED $prof"; continue; }
47
+ grep -E 'action_dim|proprio_dim' $F/model-$prof.yaml | tr '\n' ' '; echo
48
+ $E scripts/preprocess_action_dit_backbone.py --model-config $F/model-$prof.yaml --output $AI --device cpu --dtype bfloat16 && echo "$(date) INIT_OK $prof $(stat -c %s $AI)" || echo "INIT_FAILED $prof"
49
+ done
50
+ mkdir -p $TC
51
+ $E scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 model.redirect_common_files=false "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" && echo "$(date) TEXT_OK $(ls $TC | wc -l) files" || echo TEXT_FAILED
52
+ ok=1
53
+ for pair in a10v3:g1 a17cr2:h2 a14v2:h2p; do
54
+ t=${pair%%:*}; prof=${pair##*:}
55
+ HB_FW_PROFILE=$prof HB_DATA_SAMPLE=1 HB_SAMPLE_WORKDIR=/tmp/hbfw-sample-$t $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=[\"$F/data/$t\"]" "data.train.text_embedding_cache_dir=$TC" "model.action_dit_pretrained_path=$F/action-init-$prof.pt" > $F/sample-$t.log 2>&1
56
+ grep -E 'HB_FW_AUG|HBFW_|Error' $F/sample-$t.log | cut -c1-800
57
+ grep -q "HBFW_SAMPLE" $F/sample-$t.log || { ok=0; echo "SAMPLE_FAILED $t"; }
58
+ done
59
+ for t in $(cut -d' ' -f1 $F/tasks.txt); do [ -f $F/data/$t/meta/hb_fastwam_interface.json ] || { ok=0; echo "MISSING_DATA $t"; }; done
60
+ for prof in g1 h2 h2p; do [ -f $F/action-init-$prof.pt ] || { ok=0; echo "MISSING_INIT $prof"; }; done
61
+ [ $ok = 1 ] && touch $F/bench.ready && echo "$(date) BENCH_READY" || echo "$(date) NOT_READY"
a18-pour-water/b512/step-002000/code/fastwam_task.sh ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Relay cmd item helper (2026-11-16 user): FastWAM on one benchmark task, 4 nodes x 8 GPUs, per-GPU batch from the
3
+ # benchmark ($G/fastwam/chosen.json: largest batch that fits), no gradient accumulation, 4000 steps (user default while
4
+ # the batch is unknown), native Gaussian FM recipe (lr 1e-4 cosine, ZeRO-1, bf16), HB augmentation v2 + proprio
5
+ # perturbation, our state/action interface. Checkpoints at 2000 and 4000. Skips (exit 3) unless the benchmark passed.
6
+ # fastwam_task.sh <task> <profile g1|h2|h2p> <port>
7
+ G=$(dirname "$(dirname "$(realpath "$0")")")
8
+ TASK=$1 PROF=$2 PORT=$3
9
+ F=$G/fastwam; E=$G/envs/fastwam/bin/python; R=$F/runs/$TASK
10
+ RK=${RANK:?}; M0=${MASTER_ADDR:?}
11
+ [ -f $F/fastwam.ok ] && [ -s $F/chosen.json ] || { echo "SKIP fastwam $TASK: benchmark not passed"; exit 3; }
12
+ [ -f $F/data/$TASK/meta/hb_fastwam_interface.json ] && [ -f $F/action-init-$PROF.pt ] || { echo "SKIP fastwam $TASK: data not prepared"; exit 3; }
13
+ BS=$(python3 -c "import json;print(json.load(open('$F/chosen.json'))['per_gpu_batch'])")
14
+ mkdir -p $R/logs
15
+ exec > >(tee -a "$R/logs/node-$RK.log") 2>&1
16
+ date -u; hostname
17
+ [ -f $R/complete.json ] && { echo "already complete"; exit 0; }
18
+ echo "FASTWAM task=$TASK profile=$PROF rank=$RK master=$M0:$PORT per_gpu=$BS global=$((BS * 32)) steps=4000"
19
+ if ! ldconfig -p | grep -q 'libavutil.so.56'; then
20
+ apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
21
+ libavcodec58 libavformat58 libavutil56 libswscale5 libswresample3 libavdevice58 libavfilter7 || true
22
+ fi
23
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT
24
+ unset $(compgen -e | grep -E '^PET_') NPROC_PER_NODE WORLD_SIZE_WITH_STANDBY TORCHELASTIC_RUN_ID
25
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true HF_HUB_OFFLINE=1 HB_AUG_DIR=$G/hb_groot HB_FW_PROFILE=$PROF
26
+ export NCCL_SOCKET_IFNAME=eth GLOO_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false WANDB_MODE=disabled
27
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
28
+ export PATH=$G/envs/fastwam/bin:$PATH # r2 (11-16 05:3x): r1 failed, accelerate called the missing deepspeed CLI (pdsh launcher)
29
+ for i in $(seq 36); do [ -z "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ] && break; sleep 5; done
30
+ cd $F/FastWAM
31
+ $E -m accelerate.commands.launch --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml --deepspeed_multinode_launcher standard \
32
+ --num_machines 4 --machine_rank $RK --main_process_ip $M0 --main_process_port $PORT --num_processes 32 \
33
+ $F/tools/hb_fastwam_train.py "output_dir=$R/out" "batch_size=$BS" max_steps=4000 log_every=20 save_every=2000 eval_every=0 \
34
+ num_workers=8 wandb.enabled=false gradient_accumulation_steps=1 "data.train.dataset_dirs=[\"$F/data/$TASK\"]" \
35
+ "data.train.text_embedding_cache_dir=$F/text-cache/all" "model.action_dit_pretrained_path=$F/action-init-$PROF.pt"
36
+ rc=$?
37
+ echo "$rc" > $R/logs/exit-node-$RK
38
+ if [ "$rc" = 0 ] && [ "$RK" = 0 ]; then
39
+ python3 - "$R/complete.json" "$TASK" "$PROF" "$BS" "$F/data/$TASK/meta/hb_fastwam_interface.json" <<'EOF'
40
+ import json, sys, time
41
+ out, task, prof, bs, iface = sys.argv[1:6]
42
+ d = dict(task=task, profile=prof, per_gpu_batch=int(bs), global_batch=int(bs) * 32, gradient_accumulation_steps=1,
43
+ max_steps=4000, save_every=2000, recipe="native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
44
+ interface=json.load(open(iface)), finished_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
45
+ json.dump(d, open(out, 'w'), indent=1); print('COMPLETE', d)
46
+ EOF
47
+ fi
48
+ echo "EXIT $rc"; exit $rc
a18-pour-water/b512/step-002000/code/hb_aug_v2.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark data augmentation v2 (project CLAUDE.md), model-agnostic torch implementation.
2
+
3
+ augment(x, rng, obs_mask=None) takes x: float [T, V, 3, H, W] in [0, 1] (T frames, V cameras) and returns the
4
+ augmented tensor. Each call is one training sample: one of the 15 effects is chosen uniformly and its magnitude
5
+ is scaled by 1.5 around identity; the rest stay at x1. Colour is shared across cameras and time; geometry, light
6
+ patches and white balance are drawn per camera and fixed over time; sensor/optics effects (noise, gaussian blur,
7
+ JPEG, motion blur, sharpen, defocus) apply only to observation frames (obs_mask True).
8
+ Pixel-size parameters are defined at 224 px and scaled by H / 224.
9
+ perturb_joints(q, rng) applies the proprio spec: per-joint bias +-2 deg plus N(0, 0.25 deg) clipped at 3 sigma.
10
+ """
11
+ import io
12
+ import math
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from PIL import Image
18
+ from torchvision.transforms import InterpolationMode
19
+ from torchvision.transforms import functional as TF
20
+
21
+ EFFECTS = ("exposure", "contrast", "saturation", "hue", "gamma", "temperature", "rotation_crop", "noise",
22
+ "gaussian_blur", "jpeg", "motion_blur", "sharpen", "defocus", "light_shadow", "wb_channel")
23
+ CENTER = {"exposure": 0.0, "contrast": 1.0, "saturation": 1.0, "hue": 0.0, "gamma": 1.0, "temperature": 0.0}
24
+ RANGE = {"exposure": (-0.22, 0.22), "contrast": (0.88, 1.12), "saturation": (0.76, 1.24), "hue": (-0.015, 0.015),
25
+ "gamma": (0.89, 1.11), "temperature": (-0.066, 0.066)}
26
+ VERSION = "hb-aug-v2-20261109"
27
+
28
+
29
+ def sample(rng, views, height):
30
+ boost = EFFECTS[int(rng.integers(len(EFFECTS)))]
31
+
32
+ def k(name):
33
+ return 1.5 if boost == name else 1.0
34
+
35
+ px = height / 224.0
36
+ p = {"boost": boost}
37
+ for name, (a, b) in RANGE.items():
38
+ c = CENTER[name]
39
+ p[name] = float(rng.uniform(c + k(name) * (a - c), c + k(name) * (b - c)))
40
+ p["geometry"] = []
41
+ for _ in range(views):
42
+ angle = float(rng.uniform(-1.5 * k("rotation_crop"), 1.5 * k("rotation_crop")))
43
+ p["geometry"].append((angle, 1.0 + abs(angle) * 0.015 + float(rng.uniform(0, 0.02 * k("rotation_crop")))))
44
+ p["noise"] = 4.0 / 255.0 * k("noise") if rng.random() < 0.5 else 0.0
45
+ p["gaussian_blur"] = float(rng.uniform(0.25, 0.65)) * k("gaussian_blur") if rng.random() < 0.15 else 0.0
46
+ if rng.random() < 0.35:
47
+ lo = int(round(100 - k("jpeg") * 30))
48
+ hi = int(round(100 - k("jpeg") * 5))
49
+ p["jpeg"] = int(rng.integers(lo, hi + 1))
50
+ else:
51
+ p["jpeg"] = None
52
+ p["motion_blur"] = ((float(rng.uniform(3, 7)) * k("motion_blur") * px, float(rng.uniform(0, 180)))
53
+ if rng.random() < 0.15 else None)
54
+ p["sharpen"] = float(rng.uniform(0.2, 0.6)) * k("sharpen") if rng.random() < 0.15 else 0.0
55
+ p["defocus"] = float(rng.uniform(1, 2)) * k("defocus") * px if rng.random() < 0.10 else 0.0
56
+ if rng.random() < 0.30:
57
+ g = 0.30 * k("light_shadow")
58
+ p["light_shadow"] = [[(float(rng.uniform(0.1, 0.9)), float(rng.uniform(0.1, 0.9)),
59
+ float(rng.uniform(0.15, 0.35)), float(rng.uniform(1 - g, 1 + g)))
60
+ for _ in range(int(rng.integers(1, 3)))] for _ in range(views)]
61
+ else:
62
+ p["light_shadow"] = None
63
+ if rng.random() < 0.30:
64
+ g = 0.10 * k("wb_channel")
65
+ gains = []
66
+ for _ in range(views):
67
+ row = [1.0, 1.0, 1.0]
68
+ row[int(rng.integers(0, 3))] = float(rng.uniform(1 - g, 1 + g))
69
+ gains.append(row)
70
+ p["wb_channel"] = gains
71
+ else:
72
+ p["wb_channel"] = None
73
+ return p
74
+
75
+
76
+ def _conv(x, kernel):
77
+ n, c, h, w = x.shape
78
+ k = kernel.to(x)[None, None].repeat(c, 1, 1, 1)
79
+ pad = kernel.shape[-1] // 2
80
+ return F.conv2d(F.pad(x, (pad, pad, pad, pad), mode="reflect"), k, groups=c)
81
+
82
+
83
+ def motion_kernel(length, angle_deg):
84
+ size = max(3, int(round(length)) | 1)
85
+ k = torch.zeros(size, size)
86
+ c = size // 2
87
+ th = math.radians(angle_deg)
88
+ for s in np.linspace(-(length - 1) / 2, (length - 1) / 2, 8 * size):
89
+ k[int(round(c - s * math.sin(th))), int(round(c + s * math.cos(th)))] = 1.0
90
+ return k / k.sum()
91
+
92
+
93
+ def disk_kernel(r):
94
+ size = 2 * int(math.ceil(r)) + 1
95
+ c = size // 2
96
+ yy, xx = torch.meshgrid(torch.arange(size), torch.arange(size), indexing="ij")
97
+ k = (((yy - c) ** 2 + (xx - c) ** 2) <= (r + 0.5) ** 2).float()
98
+ return k / k.sum()
99
+
100
+
101
+ def apply(x, p, obs_mask=None, noise_seed=0):
102
+ t, v, c, h, w = x.shape
103
+ x = x.clone().clamp(0, 1)
104
+ for vi, (angle, scale) in enumerate(p["geometry"]):
105
+ x[:, vi] = TF.affine(x[:, vi], angle=angle, translate=[0, 0], scale=scale, shear=[0, 0],
106
+ interpolation=InterpolationMode.BILINEAR)
107
+ if p["light_shadow"] is not None:
108
+ yy, xx = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij")
109
+ for vi, blobs in enumerate(p["light_shadow"]):
110
+ m = torch.ones(h, w)
111
+ for cx, cy, s, g in blobs:
112
+ m = m + (g - 1.0) * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * s * s))
113
+ x[:, vi] = (x[:, vi] * m.clamp(min=0).to(x)).clamp(0, 1)
114
+ if p["wb_channel"] is not None:
115
+ for vi, g in enumerate(p["wb_channel"]):
116
+ x[:, vi] = (x[:, vi] * torch.tensor(g, dtype=x.dtype).view(1, 3, 1, 1)).clamp(0, 1)
117
+ y = x.reshape(t * v, c, h, w)
118
+ y = (y * float(2.0 ** p["exposure"])).clamp(0, 1)
119
+ d = p["temperature"]
120
+ y = (y * y.new_tensor([1.0 + d, 1.0, 1.0 - d]).view(1, 3, 1, 1)).clamp(0, 1)
121
+ y = ((y - 0.5) * p["contrast"] + 0.5).clamp(0, 1)
122
+ y = TF.adjust_saturation(y, p["saturation"])
123
+ y = TF.adjust_hue(y, p["hue"])
124
+ y = y.clamp(0, 1).pow(p["gamma"])
125
+ x = y.reshape(t, v, c, h, w)
126
+ mask = torch.ones(t, dtype=torch.bool) if obs_mask is None else torch.as_tensor(obs_mask, dtype=torch.bool)
127
+ if bool(mask.any()):
128
+ o = x[mask].reshape(-1, c, h, w)
129
+ if p["gaussian_blur"] > 0:
130
+ o = TF.gaussian_blur(o, [3, 3], [p["gaussian_blur"]] * 2)
131
+ if p["motion_blur"] is not None:
132
+ o = _conv(o, motion_kernel(*p["motion_blur"]))
133
+ if p["defocus"] > 0:
134
+ o = _conv(o, disk_kernel(p["defocus"]))
135
+ if p["sharpen"] > 0:
136
+ o = o + p["sharpen"] * (o - TF.gaussian_blur(o, [5, 5], [1.0, 1.0]))
137
+ o = o.clamp(0, 1)
138
+ if p["noise"] > 0:
139
+ gen = torch.Generator().manual_seed(int(noise_seed))
140
+ o = (o + torch.randn(o.shape, generator=gen, dtype=o.dtype) * p["noise"]).clamp(0, 1)
141
+ if p["jpeg"] is not None:
142
+ for i in range(o.shape[0]):
143
+ buf = io.BytesIO()
144
+ TF.to_pil_image(o[i]).save(buf, format="JPEG", quality=int(p["jpeg"]))
145
+ buf.seek(0)
146
+ with Image.open(buf) as im:
147
+ o[i] = TF.pil_to_tensor(im.convert("RGB")).to(o.dtype).div(255.0)
148
+ x[mask] = o.reshape(-1, v, c, h, w)
149
+ return x.clamp(0, 1)
150
+
151
+
152
+ def augment(x, rng, obs_mask=None):
153
+ p = sample(rng, x.shape[1], x.shape[-2])
154
+ return apply(x, p, obs_mask, noise_seed=int(rng.integers(0, 2 ** 62))), p
155
+
156
+
157
+ def perturb_joints(q, rng):
158
+ q = np.asarray(q, dtype=np.float64)
159
+ d = q.shape[-1]
160
+ bias = rng.uniform(-math.radians(2.0), math.radians(2.0), size=d)
161
+ sigma = math.radians(0.25)
162
+ noise = np.clip(rng.normal(0.0, sigma, size=q.shape), -3 * sigma, 3 * sigma)
163
+ return (q + bias + noise).astype(np.float32)
a18-pour-water/b512/step-002000/code/hb_eval_common.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark open-loop action-reconstruction evaluation: shared task table, cache layout and metrics.
2
+
3
+ Protocol (identical for every model; mirrors Isaac-GR00T gr00t/eval/open_loop_eval.py):
4
+ * query frames t = 0, 16, 32, ... over the whole episode (non-overlapping 16-step chunks);
5
+ * the model sees the observation at t exactly as at deployment (cameras, raw benchmark state 44/78, instruction;
6
+ Delta Flow also its native 16-step past-action history), all augmentation off;
7
+ * the predicted chunk (first 16 steps, raw action units) is compared with the recorded actions t..t+15;
8
+ rows beyond the episode end are excluded (no clamped / padded targets);
9
+ * metrics: raw MSE / MAE, and nMSE = MSE after dividing every action dim by its std over the task's TRAINING
10
+ frames (floored, see STD_FLOOR_FRAC), overall, per action group and per horizon step.
11
+ Splits: "heldout" = recorded episodes that were never trained on (B6 100-148, trolley 100-102, B7 100-106);
12
+ "train10" = 10 fixed, evenly spaced training episodes (fit on seen data, NOT generalisation).
13
+ Only numpy here: imported by every model environment.
14
+ """
15
+ import json
16
+ import os
17
+ import time
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+
22
+ G = Path(os.environ.get("G", "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/"
23
+ "preflight-20261104T1648Z/policies-20261109"))
24
+ P = G.parent
25
+ ROOT = G / "eval-action-mse"
26
+ CACHE = Path(os.environ.get("HB_EVAL_CACHE", ROOT / "cache"))
27
+ HORIZON = 16
28
+ STRIDE = 16
29
+ N_TRAIN_EVAL = 10
30
+ STD_FLOOR_FRAC = 0.05 # per-dim std floor = 5 % of the median std of the action vector
31
+
32
+ # task -> (source LeRobot v2.1 root, number of TRAINING episodes = fixed first N (None = all recorded), layout)
33
+ TASKS = {
34
+ "b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100, "g1"),
35
+ "basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None, "g1"),
36
+ "trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100, "g1"),
37
+ "b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100, "g1"),
38
+ "b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None, "g1"),
39
+ "a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None, "g1"),
40
+ "a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None, "g1"),
41
+ "a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None, "g1"),
42
+ "c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None, "g1"),
43
+ "a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None, "g1"),
44
+ "a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None, "g1"),
45
+ "a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None, "h2"),
46
+ "a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None, "h2"),
47
+ "a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None, "h2"),
48
+ # 2026-11-15: newer datasets (GR00T in-training eval needs a cache for every task it trains on)
49
+ "a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None, "g1"),
50
+ "a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None, "g1"),
51
+ "b1": (P / "b1-riceball-microwave-v1-36da298/data", None, "g1"),
52
+ "b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None, "g1"),
53
+ "b8": (P / "b8-table-wipe-v1-3b2e618/data", None, "g1"),
54
+ "basketv2": (P / "basket-trimv2-c4df4d4/data", None, "g1"),
55
+ "a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None, "h2p"), # action 82 = hb75 + SONIC planner 7
56
+ # 2026-11-15 evening: v6/v4 datasets (a3v3, a7v3 retired, not registered)
57
+ "b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None, "g1"), # 542 ep, all used (B6 first-100 rule is v5t7 only)
58
+ "b5": (P / "b5-chair-push-v1-b85b7e9/data", None, "g1"),
59
+ "a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None, "g1"),
60
+ "a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None, "g1"),
61
+ # 2026-11-16: B5 standing start (HF b5-chair-push-standing-start-expert-v1 @913f5a16, 100 ep; FastWAM b5ss)
62
+ "b5ss": (P / "b5-chair-push-standing-start-v1-913f5a1/data", None, "g1"),
63
+ # 2026-11-15 night: H2 cropped v2 (GR00T h32 relay item 71); head+chest only -> build_cache --cams head,chest
64
+ "a18cr2": (P / "a18-pour-water-cropped-v2-44cdd57/data", None, "h2"), # HF a18-pour-water-expert-cropped-v2 @44cdd571
65
+ "a17cr2": (P / "a17-unscrew-bottle-cropped-v2-3520b70/data", None, "h2"), # HF a17-unscrew-bottle-expert-cropped-v2 @3520b70f
66
+ }
67
+
68
+
69
+ def task_for_source(source):
70
+ """benchmark task whose recording root is <source> (groot-data/<view>/hb-dataset-receipt.json "source")"""
71
+ src = Path(source).resolve()
72
+ hits = [t for t, (root, _, _) in TASKS.items() if Path(root).resolve() == src]
73
+ return hits[0] if hits else None
74
+ G1_BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint",
75
+ "left_ankle_pitch_joint", "left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint",
76
+ "right_hip_yaw_joint", "right_knee_joint", "right_ankle_pitch_joint", "right_ankle_roll_joint",
77
+ "waist_yaw_joint", "waist_roll_joint", "waist_pitch_joint", "left_shoulder_pitch_joint",
78
+ "left_shoulder_roll_joint", "left_shoulder_yaw_joint", "left_elbow_joint", "left_wrist_roll_joint",
79
+ "left_wrist_pitch_joint", "left_wrist_yaw_joint", "right_shoulder_pitch_joint",
80
+ "right_shoulder_roll_joint", "right_shoulder_yaw_joint", "right_elbow_joint", "right_wrist_roll_joint",
81
+ "right_wrist_pitch_joint", "right_wrist_yaw_joint"]
82
+ LAYOUTS = {
83
+ # state: body29 | inspire L6 | inspire R6 | projected_gravity3 ; action: motion_token64 | inspire L6 | R6 (same row)
84
+ "g1": dict(state_dim=44, action_dim=76, cams=("ego_view", "chest"),
85
+ groups={"motion_token": (0, 64), "hands": (64, 76)}),
86
+ # state: observation.state75 (legs12 waist3 arms14 head2 hands22+22) | gravity3 ; action: body_target31 | hands44
87
+ "h2": dict(state_dim=78, action_dim=75, cams=("head", "chest", "wrist"),
88
+ groups={"body_target": (0, 31), "hands": (31, 75)}),
89
+ # 2026-11-15 user: A14 v2 is trained on action82 = body_target31 | hands44 | action.planner7 (SONIC planner
90
+ # command); all 82 dims are scored (planner reported as its own group)
91
+ "h2p": dict(state_dim=78, action_dim=82, cams=("head", "chest"),
92
+ groups={"body_target": (0, 31), "hands": (31, 75), "planner": (75, 82)}),
93
+ }
94
+
95
+
96
+ def jl(path):
97
+ return [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
98
+
99
+
100
+ def layout(task):
101
+ return LAYOUTS[TASKS[task][2]]
102
+
103
+
104
+ def split_episodes(task, split, n_recorded):
105
+ _, n_train, _ = TASKS[task]
106
+ n_train = n_recorded if n_train is None else n_train
107
+ if split == "heldout":
108
+ return list(range(n_train, n_recorded))
109
+ if split == "train10":
110
+ return sorted({int(round(x)) for x in np.linspace(0, n_train - 1, N_TRAIN_EVAL)})
111
+ if split.startswith("ep:"): # explicit list, e.g. ep:100,101
112
+ return [int(x) for x in split[3:].split(",")]
113
+ raise ValueError(f"unknown split {split!r}")
114
+
115
+
116
+ def split_dir(task, split):
117
+ return CACHE / task / split
118
+
119
+
120
+ def load_meta(task, split):
121
+ return json.loads((split_dir(task, split) / "meta.json").read_text())
122
+
123
+
124
+ def load_stats(task):
125
+ z = np.load(CACHE / task / "stats.npz")
126
+ return {k: z[k] for k in z.files}
127
+
128
+
129
+ def norm_std(stats):
130
+ std = stats["action_std"].astype(np.float64)
131
+ return np.maximum(std, STD_FLOOR_FRAC * float(np.median(std)))
132
+
133
+
134
+ def iter_episodes(task, split, max_episodes=None):
135
+ meta = load_meta(task, split)
136
+ eps = meta["episodes"][: max_episodes or None]
137
+ for ep in eps:
138
+ with np.load(split_dir(task, split) / f"ep_{ep:06d}.npz") as z:
139
+ yield ep, {k: z[k] for k in z.files}
140
+
141
+
142
+ class Accumulator:
143
+ """Sums of squared / absolute errors; everything is computed in float64 over valid (row, dim) entries."""
144
+
145
+ def __init__(self, std, groups, horizon=HORIZON):
146
+ self.std = np.asarray(std, dtype=np.float64)
147
+ self.groups = groups
148
+ D = len(self.std)
149
+ self.sse_raw = np.zeros(D)
150
+ self.sae_raw = np.zeros(D)
151
+ self.sse_n = np.zeros(D)
152
+ self.rows = 0
153
+ self.step_sse_n = np.zeros(horizon)
154
+ self.step_rows = np.zeros(horizon)
155
+ self.per_episode = {}
156
+
157
+ def add(self, ep, pred, gt, valid):
158
+ pred = np.asarray(pred, dtype=np.float64)
159
+ gt = np.asarray(gt, dtype=np.float64)
160
+ v = np.asarray(valid, dtype=bool)
161
+ if pred.shape != gt.shape:
162
+ raise ValueError(f"pred {pred.shape} vs gt {gt.shape}")
163
+ if not np.isfinite(pred[v]).all():
164
+ raise FloatingPointError("non-finite prediction")
165
+ e = (pred - gt)[v]
166
+ en = e / self.std
167
+ self.sse_raw += (e ** 2).sum(0)
168
+ self.sae_raw += np.abs(e).sum(0)
169
+ self.sse_n += (en ** 2).sum(0)
170
+ self.rows += int(v.sum())
171
+ k = np.nonzero(v)[0]
172
+ self.step_sse_n[k] += (en ** 2).sum(1)
173
+ self.step_rows[k] += 1
174
+ s = self.per_episode.setdefault(int(ep), [0.0, 0])
175
+ s[0] += float((en ** 2).sum())
176
+ s[1] += int(v.sum()) * len(self.std)
177
+
178
+ def result(self):
179
+ D = len(self.std)
180
+ n = max(self.rows, 1)
181
+ out = {"rows": self.rows, "mse_raw": float(self.sse_raw.sum() / (n * D)),
182
+ "mae_raw": float(self.sae_raw.sum() / (n * D)), "nmse": float(self.sse_n.sum() / (n * D)),
183
+ "groups": {}, "nmse_by_step": (self.step_sse_n / np.maximum(self.step_rows, 1) / D).tolist(),
184
+ "nmse_by_dim": (self.sse_n / n).tolist()}
185
+ for g, (a, b) in self.groups.items():
186
+ out["groups"][g] = {"mse_raw": float(self.sse_raw[a:b].sum() / (n * (b - a))),
187
+ "nmse": float(self.sse_n[a:b].sum() / (n * (b - a)))}
188
+ pe = {ep: s / max(c, 1) for ep, (s, c) in sorted(self.per_episode.items())}
189
+ out["nmse_per_episode"] = pe
190
+ vals = np.array(list(pe.values()))
191
+ if len(vals) > 1: # bootstrap-free spread: std error over episodes
192
+ out["nmse_episode_sem"] = float(vals.std(ddof=1) / np.sqrt(len(vals)))
193
+ return out
194
+
195
+
196
+ def queries(T):
197
+ return list(range(0, T, STRIDE))
198
+
199
+
200
+ def gt_chunk(action, t, horizon=HORIZON):
201
+ T = len(action)
202
+ idx = np.arange(t, t + horizon)
203
+ valid = idx < T
204
+ return action[np.minimum(idx, T - 1)], valid
205
+
206
+
207
+ def run(adapter, task, split, out_path, max_episodes=None, max_queries=None, save_pred=True, log_every=50):
208
+ """Evaluate one adapter on one cached split; writes <out_path> (json) and <out_path>.pred.npz."""
209
+ meta = load_meta(task, split)
210
+ lay = LAYOUTS[meta["layout"]]
211
+ stats = load_stats(task)
212
+ acc = Accumulator(norm_std(stats), lay["groups"])
213
+ preds, keys = [], []
214
+ t0 = time.time()
215
+ infer_s = 0.0
216
+ nq = 0
217
+ for ep, z in iter_episodes(task, split, max_episodes):
218
+ state, action = z["state"], z["action"]
219
+ qts = list(z["query_t"])
220
+ for qi, t in enumerate(qts):
221
+ obs = {"episode": ep, "t": int(t), "T": len(action), "state": state[t], "prompt": meta["prompts"][str(int(z["task_index"]))],
222
+ "images": {c: z[f"img_{c}"][qi] for c in meta["cams"]},
223
+ "past_action": action[max(0, t - HORIZON):t],
224
+ "_gt_check": action[t:t + HORIZON]} # only for adapters' input-pipeline consistency checks
225
+ s = time.time()
226
+ pred = np.asarray(adapter.predict(obs), dtype=np.float64)
227
+ infer_s += time.time() - s
228
+ if pred.ndim != 2 or pred.shape[0] < HORIZON or pred.shape[1] != lay["action_dim"]:
229
+ raise ValueError(f"adapter returned {pred.shape}, need (>={HORIZON}, {lay['action_dim']})")
230
+ pred = pred[:HORIZON]
231
+ gt, valid = gt_chunk(action, t)
232
+ acc.add(ep, pred, gt, valid)
233
+ if save_pred:
234
+ preds.append(pred.astype(np.float32))
235
+ keys.append((ep, int(t)))
236
+ nq += 1
237
+ if log_every and nq % log_every == 0:
238
+ print(f"[eval] {task}/{split} {nq} queries, running nMSE {acc.result()['nmse']:.4f}, "
239
+ f"{infer_s / nq:.3f} s/query", flush=True)
240
+ if max_queries and nq >= max_queries:
241
+ break
242
+ if max_queries and nq >= max_queries:
243
+ break
244
+ res = acc.result()
245
+ res.update(task=task, split=split, layout=meta["layout"], episodes=sorted(acc.per_episode), queries=nq,
246
+ horizon=HORIZON, stride=STRIDE, std_floor_frac=STD_FLOOR_FRAC, wall_s=round(time.time() - t0, 1),
247
+ infer_s_per_query=round(infer_s / max(nq, 1), 4), adapter=adapter.info(),
248
+ baselines=meta.get("baselines", {}), cache_created=meta.get("created"),
249
+ partial=bool(max_episodes or max_queries), finished_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
250
+ out_path = Path(out_path)
251
+ out_path.parent.mkdir(parents=True, exist_ok=True)
252
+ tmp = out_path.with_suffix(".tmp")
253
+ tmp.write_text(json.dumps(res, indent=1))
254
+ tmp.replace(out_path)
255
+ if save_pred and preds:
256
+ np.savez_compressed(str(out_path) + ".pred.npz", pred=np.stack(preds).astype(np.float16),
257
+ keys=np.asarray(keys, dtype=np.int64))
258
+ return res
a18-pour-water/b512/step-002000/code/hb_fastwam_aug.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: Humanoid Benchmark augmentation v2 (project CLAUDE.md) inside the native FastWAM data pipeline.
2
+ - Visual: hb_aug_v2.augment on every training sample's frames before the two cameras are concatenated; frame 0 is the
3
+ current observation (sensor / optics effects only there), the later frames are the future video targets (colour,
4
+ per-camera geometry, light patches and white balance shared with frame 0, as the spec requires).
5
+ - Proprio: hb_aug_v2.perturb_joints on the raw body joints (G1 29, H2 31 = body 29 + head 2) before normalization;
6
+ hands and gravity untouched; supervision labels untouched. Normalization statistics are computed on the raw data.
7
+ FastWAM's own image augmentation is only a resize (no colour jitter / crop), so nothing native is stacked.
8
+ Installed in the main process before the dataloaders fork; HB_FW_JOINTS sets the number of perturbed joints."""
9
+ import os
10
+ import sys
11
+ import time
12
+
13
+ import numpy as np
14
+ import torch
15
+
16
+ sys.path.insert(0, os.environ.get("HB_AUG_DIR", ""))
17
+ import hb_aug_v2 as A # noqa: E402
18
+
19
+ NJ = int(os.environ.get("HB_FW_JOINTS", "29"))
20
+ _seen = {"img": False, "state": False}
21
+
22
+
23
+ def _rng(idx=0):
24
+ return np.random.default_rng([os.getpid(), int(idx) & 0xFFFFFFFF, time.time_ns() & 0xFFFFFFFF])
25
+
26
+
27
+ def _train(proc):
28
+ v = getattr(proc, "is_train", False)
29
+ return bool(v() if callable(v) else v)
30
+
31
+
32
+ def install():
33
+ from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset
34
+ from fastwam.datasets.lerobot.processors.fastwam_processor import FastWAMProcessor
35
+
36
+ pre0 = FastWAMProcessor.preprocess
37
+
38
+ def preprocess(self, data):
39
+ if _train(self) and isinstance(data.get("state"), dict) and "default" in data["state"]:
40
+ st = data["state"]["default"]
41
+ q = st[..., :NJ].detach().cpu().numpy()
42
+ st = st.clone()
43
+ st[..., :NJ] = torch.from_numpy(A.perturb_joints(q, _rng())).to(st.dtype)
44
+ data["state"]["default"] = st
45
+ if not _seen["state"]:
46
+ _seen["state"] = True
47
+ print(f"HB_FW_AUG proprio perturb_joints on state[..., :{NJ}] shape {tuple(st.shape)}", flush=True)
48
+ return pre0(self, data)
49
+
50
+ get0 = BaseLerobotDataset.__getitem__
51
+
52
+ def getitem(self, idx):
53
+ s = get0(self, idx)
54
+ if self.processor is not None and _train(self.processor) and "pixel_values" in s:
55
+ pv = s["pixel_values"]
56
+ five = pv.ndim == 5
57
+ x = (pv if five else pv[None]).permute(1, 0, 2, 3, 4).float() # [T, V, C, H, W]
58
+ hi = float(x.max())
59
+ scale = 255.0 if hi > 1.5 else 1.0
60
+ mask = torch.zeros(x.shape[0], dtype=torch.bool)
61
+ mask[0] = True
62
+ y, p = A.augment((x / scale).clamp(0, 1), _rng(idx), mask)
63
+ y = (y * scale).permute(1, 0, 2, 3, 4)
64
+ s["pixel_values"] = (y if five else y[0]).to(pv.dtype)
65
+ if not _seen["img"]:
66
+ _seen["img"] = True
67
+ print(f"HB_FW_AUG visual {A.VERSION} on pixel_values {tuple(pv.shape)} dtype {pv.dtype} max {hi:.3f} "
68
+ f"(T={x.shape[0]}, obs frame 0) boost={p['boost']}", flush=True)
69
+ return s
70
+
71
+ FastWAMProcessor.preprocess = preprocess
72
+ BaseLerobotDataset.__getitem__ = getitem
73
+ print(f"HB_FW_AUG installed: {A.VERSION}, joints {NJ}", flush=True)
a18-pour-water/b512/step-002000/code/hb_fastwam_convert.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: LeRobot v2.1 copy of one of our datasets for FastWAM on OUR benchmark interface (CLAUDE.md section 4):
2
+ g1 : state44 = observation.state[0:29] (29 body joints) + measured Inspire 6 + 6 + projected gravity 3;
3
+ action76 = action.motion_token 64 + measured Inspire 6 + 6 (same-row labels, as GR00T / Psi-Zero / Delta Flow)
4
+ h2 : state78 = observation.state[0:75] + projected gravity 3; action75 = action.body_target 31 + action.hands 44
5
+ h2p: as h2 with action82 = action75 + action.planner 7 (A14 v2)
6
+ HumanWAM's fastwam_sonic_data.py expects a 43-D G1 state and takes state[0:22] + state[29:36]; on our 41-D state that
7
+ would put left-hand values into the right arm, so it is not used. Source files are never modified; media are symlinked.
8
+ --episodes N keeps the first N episodes (B7: first 100 of 107, as for every other model).
9
+ Usage: hb_fastwam_convert.py <profile> <source data dir> <fresh output dir> [--episodes N]"""
10
+ import copy, json, shutil, sys, tempfile
11
+ from pathlib import Path
12
+ import numpy as np
13
+ import pyarrow as pa
14
+ import pyarrow.parquet as pq
15
+
16
+ G1 = {"observation.state": 41, "action.motion_token": 64, "observation.inspire_left_hand_state": 6,
17
+ "observation.inspire_right_hand_state": 6, "observation.projected_gravity": 3}
18
+ H2 = {"observation.state": 75, "observation.projected_gravity": 3, "action.body_target": 31, "action.hands": 44}
19
+ H2P = dict(H2, **{"action.planner": 7})
20
+ SPEC = {"g1": (G1, ("ego_view", "chest")), "h2": (H2, ("head", "chest")), "h2p": (H2P, ("head", "chest"))}
21
+
22
+
23
+ def stats(v):
24
+ v = np.asarray(v, dtype=np.float64)
25
+ return {"min": v.min(0).tolist(), "max": v.max(0).tolist(), "mean": v.mean(0).tolist(), "std": v.std(0).tolist(), "count": [len(v)]}
26
+
27
+
28
+ def vec(table, key, values):
29
+ arr = pa.array(values.tolist(), type=pa.list_(pa.float32(), values.shape[1]))
30
+ i = table.schema.get_field_index(key)
31
+ return table.append_column(key, arr) if i < 0 else table.set_column(i, key, arr)
32
+
33
+
34
+ def map_rows(profile, c):
35
+ if profile == "g1":
36
+ L, R = c["observation.inspire_left_hand_state"], c["observation.inspire_right_hand_state"]
37
+ dev = float(np.abs(c["observation.state"][:, 29:41] - np.concatenate([L, R], 1)).max())
38
+ assert dev < 1e-4, f"observation.state[29:41] differs from the Inspire columns by {dev}"
39
+ return (np.concatenate([c["action.motion_token"], L, R], 1),
40
+ np.concatenate([c["observation.state"][:, :29], L, R, c["observation.projected_gravity"]], 1))
41
+ a = [c["action.body_target"], c["action.hands"]] + ([c["action.planner"]] if profile == "h2p" else [])
42
+ return np.concatenate(a, 1), np.concatenate([c["observation.state"], c["observation.projected_gravity"]], 1)
43
+
44
+
45
+ def main(profile, src, out, limit=None):
46
+ raw, cams = SPEC[profile]
47
+ src, out = Path(src).resolve(strict=True), Path(out).resolve()
48
+ if out.exists():
49
+ raise SystemExit(f"{out} exists")
50
+ info = json.loads((src / "meta/info.json").read_text())
51
+ assert info.get("codebase_version") == "v2.1", info.get("codebase_version")
52
+ for k, w in raw.items():
53
+ assert info["features"][k]["shape"] == [w], (k, info["features"][k]["shape"])
54
+ for c in cams:
55
+ assert info["features"]["observation.images." + c]["dtype"] in ("video", "image"), c
56
+ eps = [json.loads(l) for l in (src / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
57
+ assert [e["episode_index"] for e in eps] == list(range(len(eps)))
58
+ if limit is not None:
59
+ eps = eps[:limit]
60
+ old = {r["episode_index"]: r for r in (json.loads(l) for l in (src / "meta/episodes_stats.jsonl").read_text().splitlines() if l.strip())}
61
+ out.parent.mkdir(parents=True, exist_ok=True)
62
+ stg = Path(tempfile.mkdtemp(prefix=".hbfw-", dir=out.parent))
63
+ shutil.copytree(src / "meta", stg / "meta")
64
+ for n in ("videos", "images"):
65
+ if (src / n).exists():
66
+ (stg / n).symlink_to(src / n, target_is_directory=True)
67
+ new, total = [], 0
68
+ for e in eps:
69
+ i = e["episode_index"]
70
+ rel = Path(info["data_path"].format(episode_chunk=i // info["chunks_size"], episode_index=i))
71
+ t = pq.read_table(src / rel)
72
+ c = {k: np.asarray(t[k].to_pylist(), dtype=np.float32) for k in raw}
73
+ act, st = map_rows(profile, c)
74
+ assert len(act) == e["length"] and np.isfinite(act).all() and np.isfinite(st).all()
75
+ total += len(act)
76
+ t = vec(vec(t, "action", act), "observation.state", st)
77
+ md = dict(t.schema.metadata or {})
78
+ if b"huggingface" in md:
79
+ hf = json.loads(md[b"huggingface"])
80
+ for k, w in (("action", act.shape[1]), ("observation.state", st.shape[1])):
81
+ hf["info"]["features"][k] = {"feature": {"dtype": "float32", "_type": "Value"}, "length": w, "_type": "Sequence"}
82
+ md[b"huggingface"] = json.dumps(hf).encode()
83
+ t = t.replace_schema_metadata(md)
84
+ (stg / rel).parent.mkdir(parents=True, exist_ok=True)
85
+ pq.write_table(t, stg / rel)
86
+ r = copy.deepcopy(old[i])
87
+ r["stats"]["action"], r["stats"]["observation.state"] = stats(act), stats(st)
88
+ new.append(r)
89
+ if limit is None:
90
+ assert total == info["total_frames"], (total, info["total_frames"])
91
+ else:
92
+ info["total_episodes"], info["total_frames"] = len(eps), total
93
+ info["total_videos"] = len(eps) * len(cams) if "total_videos" in info else info.get("total_videos")
94
+ info["splits"] = {"train": f"0:{len(eps)}"}
95
+ (stg / "meta/episodes.jsonl").write_text("".join(json.dumps(e) + "\n" for e in eps))
96
+ info["features"]["action"] = {"dtype": "float32", "shape": [int(act.shape[1])], "names": None}
97
+ info["features"]["observation.state"] = {"dtype": "float32", "shape": [int(st.shape[1])], "names": None}
98
+ drop = [k for k, v in info["features"].items() if v["dtype"] in ("video", "image") and k not in ["observation.images." + c for c in cams]]
99
+ for k in drop: # 2-camera rule: unused cameras are not downloaded, FastWAM checks every listed video file
100
+ del info["features"][k]
101
+ for r in new:
102
+ r["stats"].pop(k, None)
103
+ if drop and info.get("total_videos"):
104
+ info["total_videos"] = len(eps) * len(cams)
105
+ (stg / "meta/info.json").write_text(json.dumps(info, indent=2) + "\n")
106
+ (stg / "meta/episodes_stats.jsonl").write_text("".join(json.dumps(r) + "\n" for r in new))
107
+ for s in (stg / "meta").glob("*stats*.json"):
108
+ s.unlink()
109
+ (stg / "meta/hb_fastwam_interface.json").write_text(json.dumps(dict(
110
+ profile=profile, source=str(src), cameras=list(cams), state_dim=int(st.shape[1]), action_dim=int(act.shape[1]),
111
+ episodes=len(eps), frames=total, fps=info["fps"], first_episodes_only=limit), indent=2) + "\n")
112
+ stg.rename(out)
113
+ print("HBFW_CONVERT_OK", profile, out, "episodes", len(eps), "frames", total, "state", st.shape[1], "action", act.shape[1], flush=True)
114
+
115
+
116
+ if __name__ == "__main__":
117
+ a = sys.argv[1:]
118
+ lim = None
119
+ if "--episodes" in a:
120
+ j = a.index("--episodes"); lim = int(a[j + 1]); del a[j:j + 2]
121
+ main(a[0], a[1], a[2], lim)
a18-pour-water/b512/step-002000/code/hb_fastwam_train.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: native FastWAM Gaussian-FM training on our benchmark interface, with the HB augmentation v2.
2
+ Profiles (HB_FW_PROFILE): g1 = state44 / action76 / cameras ego_view + chest (HumanWAM fastwam_sonic recipe);
3
+ h2 = state78 / action75 / head + chest; h2p = state78 / action82 (+ planner 7) / head + chest. Fixed recipe of HumanWAM
4
+ fastwam_sonic_training.compose_sonic_config: 17 raw samples = 16 actions + 5 video frames (stride 4), 224 x 448 side by
5
+ side, min/max statistics, absolute labels, action scheduler shift 1.0, original Wan files.
6
+ Arguments: key=value Hydra overrides (JSON values parsed). Env: FASTWAM_SRC, DIFFSYNTH_MODEL_BASE_PATH, HB_AUG_DIR.
7
+ HB_MODEL_YAML=<path> writes the resolved model config; HB_COMPOSE_ONLY=1 prints the config; HB_DATA_SAMPLE=1 reads one
8
+ augmented training sample on CPU; HB_FW_AUG=0 disables the augmentation (benchmark only)."""
9
+ import json, os, sys
10
+ from pathlib import Path
11
+
12
+ ROOT = Path(os.environ["FASTWAM_SRC"]).resolve()
13
+ sys.path.insert(0, str(ROOT / "src"))
14
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
15
+ TASK = "libero_uncond_2cam224_1e-4"
16
+ PROFILES = {"g1": (("ego_view", "chest"), 44, 76, 29), "h2": (("head", "chest"), 78, 75, 31),
17
+ "h2p": (("head", "chest"), 78, 82, 31)}
18
+ CAMS, SDIM, ADIM, NJ = PROFILES[os.environ.get("HB_FW_PROFILE", "g1")]
19
+ os.environ.setdefault("HB_FW_JOINTS", str(NJ))
20
+ SHAPE_META = {
21
+ "images": [{"key": n, "raw_shape": [3, 224, 224], "shape": [3, 224, 224]} for n in CAMS],
22
+ "action": [{"key": "default", "raw_shape": ADIM, "shape": ADIM}],
23
+ "state": [{"key": "default", "raw_shape": SDIM, "shape": SDIM}],
24
+ }
25
+ FIXED = {
26
+ "data.train.shape_meta": SHAPE_META, "data.train.num_frames": 17, "data.train.global_sample_stride": 1,
27
+ "data.train.action_video_freq_ratio": 4, "data.train.video_size": [224, 448], "data.train.concat_multi_camera": "horizontal",
28
+ "data.train.processor.action_output_dim": ADIM, "data.train.processor.proprio_output_dim": SDIM,
29
+ "data.train.processor.delta_action_dim_mask": {"default": [False] * ADIM}, "data.train.processor.action_state_transforms": None,
30
+ "data.train.processor.use_stepwise_action_norm": False, "data.train.processor.norm_default_mode": "min/max",
31
+ "data.train.processor.norm_exception_mode": None, "model.redirect_common_files": False,
32
+ "model.action_scheduler.train_shift": 1.0, "model.action_scheduler.infer_shift": 1.0,
33
+ }
34
+
35
+
36
+ def build(overrides):
37
+ from hydra import compose, initialize_config_dir
38
+ from omegaconf import OmegaConf
39
+ from fastwam.utils.config_resolvers import register_default_resolvers
40
+ register_default_resolvers()
41
+ with initialize_config_dir(config_dir=str(ROOT / "configs"), version_base="1.3"):
42
+ cfg = compose(config_name="train", overrides=["task=" + TASK])
43
+ for k, v in {**FIXED, **overrides}.items():
44
+ OmegaConf.update(cfg, k, v, merge=False)
45
+ for k, v in FIXED.items():
46
+ a = OmegaConf.select(cfg, k)
47
+ if OmegaConf.is_config(a):
48
+ a = OmegaConf.to_container(a, resolve=True)
49
+ assert a == v, (k, a, v)
50
+ return cfg
51
+
52
+
53
+ def parse(argv):
54
+ out = {}
55
+ for a in argv:
56
+ k, v = a.split("=", 1)
57
+ try:
58
+ out[k] = json.loads(v)
59
+ except ValueError:
60
+ out[k] = v
61
+ return out
62
+
63
+
64
+ if __name__ == "__main__":
65
+ from omegaconf import OmegaConf
66
+ cfg = build(parse(sys.argv[1:]))
67
+ if os.environ.get("HB_MODEL_YAML"):
68
+ Path(os.environ["HB_MODEL_YAML"]).write_text(OmegaConf.to_yaml(cfg.model, resolve=True))
69
+ print("HBFW_MODEL_YAML", os.environ["HB_MODEL_YAML"], flush=True)
70
+ raise SystemExit(0)
71
+ if os.environ.get("HB_COMPOSE_ONLY"):
72
+ print(OmegaConf.to_yaml(cfg, resolve=True))
73
+ raise SystemExit(0)
74
+ import fastwam.runtime as native
75
+ assert Path(native.__file__).resolve() == ROOT / "src/fastwam/runtime.py", native.__file__
76
+ if os.environ.get("HB_FW_AUG", "1") != "0":
77
+ import hb_fastwam_aug
78
+ hb_fastwam_aug.install()
79
+ print("HBFW_RECIPE", json.dumps(dict(profile=os.environ.get("HB_FW_PROFILE", "g1"), cameras=CAMS, state_dim=SDIM,
80
+ action_dim=ADIM, perturbed_joints=int(os.environ["HB_FW_JOINTS"]),
81
+ aug=os.environ.get("HB_FW_AUG", "1") != "0",
82
+ batch_size=cfg.batch_size, max_steps=cfg.max_steps, lr=cfg.learning_rate)), flush=True)
83
+ if os.environ.get("HB_DATA_SAMPLE"): # CPU check: build the dataset and read augmented samples, no model
84
+ from omegaconf import open_dict
85
+ from fastwam.utils import misc
86
+ misc.register_work_dir(os.environ.get("HB_SAMPLE_WORKDIR", "/tmp/hbfw-sample"))
87
+ tr, _ = native.build_datasets(cfg.data)
88
+ s = tr[len(tr) // 2]
89
+
90
+ def desc(x):
91
+ if isinstance(x, dict):
92
+ return {k: desc(v) for k, v in x.items()}
93
+ if hasattr(x, "shape"):
94
+ return [tuple(x.shape), str(x.dtype), round(float(x.float().min()), 3), round(float(x.float().max()), 3)]
95
+ return type(x).__name__
96
+ print("HBFW_SAMPLE", len(tr), json.dumps(desc(s), default=str), flush=True)
97
+ raise SystemExit(0)
98
+ if os.environ.get("HB_INLOOP_EVERY", "250") != "0": # 2026-11-16: benchmark action nMSE every 250 steps
99
+ try:
100
+ import hb_inloop_fastwam
101
+ hb_inloop_fastwam.install()
102
+ except Exception as e: # noqa: BLE001 - never block training
103
+ print("HB_INLOOP fastwam install failed:", repr(e), flush=True)
104
+ native.run_training(cfg)
a18-pour-water/b512/step-002000/code/hb_inloop_core.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared machinery of the in-training open-loop action MSE (Psi-Zero, Delta Flow; GR00T has its own tested copy in
2
+ hb_groot/hb_inloop_eval.py with the same protocol).
3
+
4
+ InLoop(model_name, task, split, out_jsonl, prepare, predict):
5
+ prepare(queries) -> optional per-model setup on this rank's queries (first eval only, inside the RNG fork)
6
+ predict(batch, seed) -> np.ndarray (B, >=16, >=Da) in raw action units; the first 16 steps and the first Da dims
7
+ (Da = benchmark action dim of the layout) are scored.
8
+ InLoop.evaluate(step, model): all train/eval modes of model.modules() and every RNG state (torch CPU + this CUDA device,
9
+ numpy, python) are restored afterwards; partial sums are all-reduced once (error flag included, so a failure on one
10
+ rank cannot leave the others waiting); rank 0 appends one json line per eval. Any failure disables later evals.
11
+ Protocol and metrics = hb_eval_common (post-hoc eval): queries every 16 frames, nMSE with the training-frame std.
12
+ """
13
+ import json
14
+ import os
15
+ import random
16
+ import sys
17
+ import time
18
+ import traceback
19
+ from pathlib import Path
20
+
21
+ import numpy as np
22
+ import torch
23
+
24
+ HERE = Path(__file__).resolve().parent
25
+ if str(HERE) not in sys.path:
26
+ sys.path.insert(0, str(HERE))
27
+ import hb_eval_common as E # noqa: E402
28
+
29
+ EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
30
+ BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
31
+ MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
32
+ SEED = 1234
33
+
34
+
35
+ def dist_info():
36
+ import torch.distributed as dist
37
+ if dist.is_available() and dist.is_initialized():
38
+ return dist.get_rank(), dist.get_world_size()
39
+ return int(os.environ.get("RANK", "0")), int(os.environ.get("WORLD_SIZE", "1"))
40
+
41
+
42
+ def pick_split(task):
43
+ split = os.environ.get("HB_INLOOP_SPLIT")
44
+ if not split:
45
+ split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
46
+ if not (E.split_dir(task, split) / "meta.json").exists():
47
+ raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
48
+ return split
49
+
50
+
51
+ class InLoop:
52
+ def __init__(self, model_name, task, split, out_jsonl, predict, prepare=None, cams=None, info=None):
53
+ self.model_name, self.task, self.split = model_name, task, split
54
+ self.out = Path(out_jsonl)
55
+ self.predict, self.prepare = predict, prepare
56
+ self.cams = cams
57
+ self.info = info or {}
58
+ self.disabled = False
59
+ self.ready = False
60
+ self.done_steps = set()
61
+
62
+ def say(self, *a):
63
+ if dist_info()[0] == 0:
64
+ print("HB_INLOOP", *a, flush=True)
65
+
66
+ def _setup(self):
67
+ rank, world = dist_info()
68
+ meta = E.load_meta(self.task, self.split)
69
+ self.meta, self.lay = meta, E.LAYOUTS[meta["layout"]]
70
+ self.std = E.norm_std(E.load_stats(self.task))
71
+ cams = self.cams or meta["cams"]
72
+ miss = [c for c in cams if c not in meta["cams"]]
73
+ if miss:
74
+ raise KeyError(f"cameras {miss} not in the eval cache {meta['cams']}")
75
+ allq = []
76
+ for ep in meta["episodes"]:
77
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
78
+ allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
79
+ if MAX_Q:
80
+ allq = allq[:MAX_Q]
81
+ self.n_total = len(allq)
82
+ mine = [q for i, q in enumerate(allq) if i % world == rank]
83
+ self.eps = list(meta["episodes"])
84
+ by_ep = {}
85
+ for ep, qi, t in mine:
86
+ by_ep.setdefault(ep, []).append((qi, t))
87
+ self.q = []
88
+ for ep, lst in by_ep.items():
89
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
90
+ state, action = z["state"], z["action"]
91
+ prompt = meta["prompts"][str(int(z["task_index"]))]
92
+ imgs = {c: z[f"img_{c}"] for c in cams}
93
+ for qi, t in lst:
94
+ gt, valid = E.gt_chunk(action, t)
95
+ self.q.append(dict(episode=int(ep), t=int(t), T=len(action), state=state[t].copy(), prompt=prompt,
96
+ images={c: imgs[c][qi].copy() for c in cams}, gt=gt, valid=valid,
97
+ past_action=action[max(0, t - E.HORIZON):t].copy()))
98
+ if self.prepare:
99
+ self.prepare(self.q)
100
+ self.ready = True
101
+ self.say(f"{self.model_name} ready: task {self.task} split {self.split} queries {self.n_total} "
102
+ f"(rank {rank}/{world}: {len(self.q)}), every {EVERY} steps, batch {BATCH}, cams {cams}")
103
+
104
+ def evaluate(self, step, model):
105
+ if self.disabled or step in self.done_steps:
106
+ return
107
+ self.done_steps.add(step)
108
+ rank, world = dist_info()
109
+ t0 = time.time()
110
+ err, msg = 0.0, ""
111
+ acc, nq, infer = None, 0, 0.0
112
+ modes = [(m, m.training) for m in model.modules()]
113
+ dev = torch.cuda.current_device() if torch.cuda.is_available() else None
114
+ np_state, py_state = np.random.get_state(), random.getstate()
115
+ try:
116
+ with torch.random.fork_rng(devices=[dev] if dev is not None else []):
117
+ if not self.ready:
118
+ self._setup()
119
+ D = self.lay["action_dim"]
120
+ model.eval()
121
+ acc = E.Accumulator(self.std, self.lay["groups"])
122
+ for bi in range(0, len(self.q), BATCH):
123
+ batch = self.q[bi:bi + BATCH]
124
+ seed = SEED + 1000 * rank + bi
125
+ torch.default_generator.manual_seed(seed)
126
+ if dev is not None:
127
+ torch.cuda.manual_seed(seed)
128
+ s = time.time()
129
+ with torch.no_grad():
130
+ pred = np.asarray(self.predict(batch, seed), dtype=np.float64)
131
+ infer += time.time() - s
132
+ if pred.ndim != 3 or pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] < D:
133
+ raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, >={D})")
134
+ for q, p in zip(batch, pred):
135
+ acc.add(q["episode"], p[:E.HORIZON, :D], q["gt"], q["valid"])
136
+ nq += 1
137
+ except Exception as e: # noqa: BLE001 - never stop training
138
+ err, msg = 1.0, f"{type(e).__name__}: {e}"
139
+ print(f"HB_INLOOP {self.model_name} rank {rank} step {step} FAILED: {msg}\n{traceback.format_exc()}",
140
+ file=sys.stderr, flush=True)
141
+ finally:
142
+ for m, tr in modes:
143
+ m.training = tr
144
+ np.random.set_state(np_state)
145
+ random.setstate(py_state)
146
+ D = self.lay["action_dim"] if self.ready else 0
147
+ H = E.HORIZON
148
+ eps = self.eps if self.ready else []
149
+ nE = len(eps)
150
+ vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
151
+ if acc is not None and not err:
152
+ o = 0
153
+ vec[o:o + D] = acc.sse_raw; o += D
154
+ vec[o:o + D] = acc.sae_raw; o += D
155
+ vec[o:o + D] = acc.sse_n; o += D
156
+ vec[o] = acc.rows; o += 1
157
+ vec[o:o + H] = acc.step_sse_n; o += H
158
+ vec[o:o + H] = acc.step_rows; o += H
159
+ for j, ep in enumerate(eps):
160
+ s_, c_ = acc.per_episode.get(int(ep), (0.0, 0))
161
+ vec[o + j], vec[o + nE + j] = s_, c_
162
+ o += 2 * nE
163
+ vec[o], vec[o + 1] = nq, infer
164
+ vec[-1] = err
165
+ tot = self._allreduce(vec)
166
+ if tot is None or tot[-1] > 0 or not self.ready:
167
+ self.disabled = True
168
+ nfail = None if tot is None else int(tot[-1])
169
+ self._write({"step": step, "error": msg or "failed on another rank / collective failed",
170
+ "ranks_failed": nfail})
171
+ self.say(f"{self.model_name} step {step}: eval failed ({nfail} rank(s)) -> in-training eval disabled: {msg}")
172
+ return
173
+ o = 0
174
+ acc = E.Accumulator(self.std, self.lay["groups"])
175
+ acc.sse_raw = tot[o:o + D]; o += D
176
+ acc.sae_raw = tot[o:o + D]; o += D
177
+ acc.sse_n = tot[o:o + D]; o += D
178
+ acc.rows = int(round(tot[o])); o += 1
179
+ acc.step_sse_n = tot[o:o + H]; o += H
180
+ acc.step_rows = tot[o:o + H]; o += H
181
+ acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
182
+ for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
183
+ o += 2 * nE
184
+ n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
185
+ res = acc.result()
186
+ res.pop("nmse_by_dim", None)
187
+ rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "model": self.model_name,
188
+ "task": self.task, "split": self.split, "queries": n_q, "ranks": world,
189
+ "wall_s": round(time.time() - t0, 1), "infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
190
+ "seed": SEED, **self.info, **res}
191
+ self._write(rec)
192
+ g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
193
+ self.say(f"{self.model_name} step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} "
194
+ f"queries, {rec['wall_s']} s")
195
+
196
+ @staticmethod
197
+ def _allreduce(vec):
198
+ import torch.distributed as dist
199
+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1:
200
+ return vec
201
+ try:
202
+ dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
203
+ n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
204
+ mx, mn = n.clone(), n.clone()
205
+ dist.all_reduce(mx, op=dist.ReduceOp.MAX)
206
+ dist.all_reduce(mn, op=dist.ReduceOp.MIN)
207
+ if mx.item() != mn.item():
208
+ return None
209
+ t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
210
+ dist.all_reduce(t)
211
+ return t.cpu().numpy()
212
+ except Exception as e: # noqa: BLE001
213
+ print(f"HB_INLOOP all_reduce failed: {e!r}", file=sys.stderr, flush=True)
214
+ return None
215
+
216
+ def _write(self, rec):
217
+ if dist_info()[0] != 0:
218
+ return
219
+ try:
220
+ self.out.parent.mkdir(parents=True, exist_ok=True)
221
+ with open(self.out, "a") as f:
222
+ f.write(json.dumps(rec) + "\n")
223
+ except Exception as e: # noqa: BLE001
224
+ print(f"HB_INLOOP write failed: {e!r}", flush=True)
225
+
226
+
227
+ class TempAttrs:
228
+ """set attributes on an object for the duration of a block, then restore (or delete) them"""
229
+ _MISSING = object()
230
+
231
+ def __init__(self, obj, **kw):
232
+ self.obj, self.kw, self.old = obj, kw, {}
233
+
234
+ def __enter__(self):
235
+ for k, v in self.kw.items():
236
+ self.old[k] = self.obj.__dict__.get(k, self._MISSING)
237
+ setattr(self.obj, k, v)
238
+ return self.obj
239
+
240
+ def __exit__(self, *exc):
241
+ for k, v in self.old.items():
242
+ if v is self._MISSING:
243
+ try:
244
+ delattr(self.obj, k)
245
+ except AttributeError:
246
+ pass
247
+ else:
248
+ setattr(self.obj, k, v)
249
+ return False
a18-pour-water/b512/step-002000/code/hb_inloop_fastwam.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM in-training open-loop action MSE (2026-11-16 user: every 250 steps, like GR00T / Psi-Zero / Delta Flow).
2
+
3
+ install() is called by fastwam/tools/hb_fastwam_train.py right before native.run_training(cfg). It uses the trainer's
4
+ own eval slot: after Wan22Trainer.__init__ it sets eval_every = HB_INLOOP_EVERY (default 250) and a non-None
5
+ val_dataset marker, and replaces Wan22Trainer.evaluate (the native one generates a full video for one random val sample;
6
+ it is off in our runs, eval_every=0 on the command line). The native loop then calls evaluate() on every rank right after
7
+ the optimizer step of every 250th update, followed by accelerator.wait_for_everyone(); returning None logs nothing.
8
+ evaluate(): all ranks score their shard of the benchmark eval cache with the live model (accelerator.unwrap_model; ZeRO-1
9
+ keeps full bf16 weights on every rank) through EXACTLY the post-hoc adapter path (eval-action-mse/adapters/ad_fastwam.py
10
+ Inputs + infer: FastWAM.infer_action, the action-only deployment path; 10 denoising steps = config
11
+ eval_num_inference_steps; seed 42 per query; same cameras / proprio normalisation / T5 cache as training).
12
+ Results: <output_dir>/inloop-mse.jsonl (rank 0) + "HB_INLOOP" log lines. Training is untouched (module modes and every
13
+ RNG restored by hb_inloop_core, no gradients); any failure disables later evals on all ranks.
14
+ """
15
+ import json
16
+ import os
17
+ import sys
18
+ from pathlib import Path
19
+
20
+ G = Path(__file__).resolve().parents[2] # .../policies-20261109 (this file: $G/fastwam/tools/)
21
+ EVAL = G / "eval-action-mse"
22
+ for p in (str(EVAL), str(EVAL / "adapters")):
23
+ if p not in sys.path:
24
+ sys.path.insert(0, p)
25
+ import numpy as np # noqa: E402
26
+ import hb_eval_common as E # noqa: E402
27
+ import hb_inloop_core as C # noqa: E402
28
+ import ad_fastwam as A # noqa: E402
29
+
30
+
31
+ class _Marker:
32
+ """non-None val_dataset so the native loop reaches evaluate(); never indexed"""
33
+
34
+ def __len__(self):
35
+ return 1
36
+
37
+
38
+ class FwEval:
39
+ def __init__(self, trainer):
40
+ cfg = trainer.cfg
41
+ self.cfg = cfg
42
+ ddir = Path(list(cfg.data.train.dataset_dirs)[0])
43
+ iface = json.loads((ddir / "meta/hb_fastwam_interface.json").read_text())
44
+ task = E.task_for_source(iface["source"])
45
+ if task is None:
46
+ raise LookupError(f"no benchmark task for {iface['source']}")
47
+ split = C.pick_split(task)
48
+ self.steps = int(cfg.eval_num_inference_steps)
49
+ self.inp = None
50
+ self.model = None
51
+ out = Path(cfg.output_dir) / "inloop-mse.jsonl"
52
+ self.il = C.InLoop("fastwam", task, split, out, predict=self.predict, prepare=self.prepare,
53
+ info={"denoising_steps": self.steps, "seed_per_query": 42, "dataset": ddir.name,
54
+ "path": "FastWAM.infer_action (ad_fastwam)"})
55
+ self.il.cams = None
56
+
57
+ def prepare(self, queries):
58
+ self.inp = A.Inputs(self.cfg, Path(self.cfg.output_dir) / "dataset_stats.json")
59
+ miss = [c for c in self.inp.cams if c not in self.il.meta["cams"]]
60
+ if miss:
61
+ raise KeyError(f"cameras {miss} not in the eval cache {self.il.meta['cams']}")
62
+ for q in {q["prompt"] for q in queries}:
63
+ self.inp.context(q) # fail early if an instruction has no T5 cache
64
+
65
+ def predict(self, batch, seed):
66
+ return np.stack([np.asarray(A.infer(self.model, self.inp, q, self.steps, 42), dtype=np.float64) for q in batch])
67
+
68
+ def evaluate(self, step, model):
69
+ self.model = model
70
+ try:
71
+ self.il.evaluate(step, model)
72
+ finally:
73
+ self.model = None
74
+
75
+
76
+ class _Broken:
77
+ def __init__(self, err):
78
+ self.err = err
79
+ self.il = C.InLoop("fastwam", "b6", "train10", "/dev/null", predict=None, prepare=self._boom)
80
+
81
+ def _boom(self, _q):
82
+ raise RuntimeError(self.err)
83
+
84
+ def evaluate(self, step, model):
85
+ self.il.evaluate(step, model)
86
+
87
+
88
+ def install():
89
+ if C.EVERY <= 0:
90
+ return
91
+ from fastwam import trainer as T
92
+ WT = T.Wan22Trainer
93
+ orig_init = WT.__init__
94
+
95
+ def __init__(self, *a, **kw):
96
+ orig_init(self, *a, **kw)
97
+ # native eval off (eval_every 0 on our command line). NB runtime.build_datasets sets val_ds = train_ds when
98
+ # there is no data.val, so val_dataset is never None here (11-16 smoke: the old "val_dataset is None" test
99
+ # silently left the in-loop eval off).
100
+ if int(self.eval_every) == 0:
101
+ self.eval_every = C.EVERY
102
+ if self.val_dataset is None:
103
+ self.val_dataset = _Marker()
104
+ self._hb_inloop_on = True
105
+ else: # a native eval is configured: leave it alone
106
+ self._hb_inloop_on = False
107
+
108
+ def evaluate(self):
109
+ if not getattr(self, "_hb_inloop_on", False):
110
+ return orig_eval(self)
111
+ ev = getattr(self, "_hb_inloop", None)
112
+ if ev is None:
113
+ try:
114
+ ev = FwEval(self)
115
+ except Exception as e: # noqa: BLE001
116
+ ev = _Broken(f"{type(e).__name__}: {e}")
117
+ self._hb_inloop = ev
118
+ if not ev.il.disabled:
119
+ ev.evaluate(int(self.global_step), self.accelerator.unwrap_model(self.model))
120
+ return None
121
+
122
+ orig_eval = WT.evaluate
123
+ WT.__init__ = __init__
124
+ WT.evaluate = evaluate
125
+ if int(os.environ.get("RANK", "0")) == 0:
126
+ print(f"HB_INLOOP fastwam installed: every {C.EVERY} steps", flush=True)
a18-pour-water/b512/step-002000/code/test_fastwam_inputs.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU check (no model): the FastWAM adapter's inputs built from the model-agnostic eval cache == the inputs our FastWAM
2
+ training dataset (RobotVideoDataset of the run's config, no HB augmentation) builds for the same frame.
3
+ python test_fastwam_inputs.py <task> <run out dir> [n_queries]"""
4
+ import json
5
+ import os
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+
11
+ HERE = Path(__file__).resolve().parent
12
+ sys.path.insert(0, str(HERE))
13
+ sys.path.insert(0, str(HERE / "adapters"))
14
+ import hb_eval_common as E # noqa: E402
15
+ import ad_fastwam as A # noqa: E402
16
+
17
+ task, out = sys.argv[1], Path(sys.argv[2])
18
+ nq = int(sys.argv[3]) if len(sys.argv) > 3 else 3
19
+ A.setup_env()
20
+ import torch # noqa: E402
21
+ from omegaconf import OmegaConf # noqa: E402
22
+ import fastwam.runtime as native # noqa: E402
23
+ from fastwam.utils import misc # noqa: E402
24
+
25
+ cfg = OmegaConf.load(out / "config.yaml")
26
+ inp = A.Inputs(cfg, out / "dataset_stats.json")
27
+ misc.register_work_dir("/tmp/hbfw-inputs-check")
28
+ OmegaConf.update(cfg, "data.train.pretrained_norm_stats", str(out / "dataset_stats.json"), merge=False, force_add=True)
29
+ tr, _ = native.build_datasets(cfg.data)
30
+ ddir = Path(cfg.data.train.dataset_dirs[0])
31
+ lens = [json.loads(l)["length"] for l in (ddir / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
32
+ meta = E.load_meta(task, "train10")
33
+ split = "train10"
34
+ worst = {}
35
+ done = 0
36
+ for ep, z in E.iter_episodes(task, split, None):
37
+ base = int(sum(lens[:ep]))
38
+ for qi, t in enumerate(list(z["query_t"])[1:1 + nq], start=1):
39
+ t = int(t)
40
+ s = tr[base + t]
41
+ obs = {"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]}, "state": z["state"][t],
42
+ "prompt": meta["prompts"][str(int(z["task_index"]))]}
43
+ img = inp.image(obs["images"])[0]
44
+ d_img = float((img - s["video"][:, 0].float()).abs().max())
45
+ pr = inp.proprio(obs["state"])[0]
46
+ d_pr = float((pr - s["proprio"][0].float()).abs().max())
47
+ ctx, mask = inp.context(obs["prompt"])
48
+ d_ctx = float((ctx.float() - s["context"].float()).abs().max())
49
+ d_mask = int((mask != s["context_mask"].bool()).sum())
50
+ gt = z["action"][t:t + 16]
51
+ a = inp.denorm(s["action"][:16])
52
+ d_act = float(np.abs(a[:len(gt)] - gt).max())
53
+ row = dict(ep=ep, t=t, idx=base + t, img=d_img, proprio=d_pr, ctx=d_ctx, mask=d_mask, act_denorm_vs_cache=d_act,
54
+ prompt_same=s["prompt"] == inp.DEFAULT_PROMPT.format(task=obs["prompt"]))
55
+ print("CHECK", json.dumps(row), flush=True)
56
+ for k in ("img", "proprio", "ctx", "act_denorm_vs_cache"):
57
+ worst[k] = max(worst.get(k, 0.0), row[k])
58
+ done += 1
59
+ if done >= 2 * nq:
60
+ break
61
+ print("WORST", json.dumps(worst))
a18-pour-water/b512/step-002000/complete.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "a18cr2",
3
+ "profile": "h2",
4
+ "per_gpu_batch": 16,
5
+ "global_batch": 512,
6
+ "gradient_accumulation_steps": 1,
7
+ "max_steps": 4000,
8
+ "save_every": 2000,
9
+ "recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
10
+ "interface": {
11
+ "profile": "h2",
12
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/a18-pour-water-cropped-v2-44cdd57/data",
13
+ "cameras": [
14
+ "head",
15
+ "chest"
16
+ ],
17
+ "state_dim": 78,
18
+ "action_dim": 75,
19
+ "episodes": 100,
20
+ "frames": 66850,
21
+ "fps": 30,
22
+ "first_episodes_only": null
23
+ },
24
+ "finished_utc": "2026-11-17T02:31:21Z"
25
+ }
a18-pour-water/b512/step-002000/dataset_meta/hb_fastwam_interface.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "profile": "h2",
3
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/a18-pour-water-cropped-v2-44cdd57/data",
4
+ "cameras": [
5
+ "head",
6
+ "chest"
7
+ ],
8
+ "state_dim": 78,
9
+ "action_dim": 75,
10
+ "episodes": 100,
11
+ "frames": 66850,
12
+ "fps": 30,
13
+ "first_episodes_only": null
14
+ }
a18-pour-water/b512/step-002000/dataset_meta/info.json ADDED
@@ -0,0 +1,450 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "codebase_version": "v2.1",
3
+ "robot_type": "h2_sharpa",
4
+ "total_episodes": 100,
5
+ "total_frames": 66850,
6
+ "total_tasks": 1,
7
+ "total_videos": 200,
8
+ "total_chunks": 1,
9
+ "chunks_size": 1000,
10
+ "fps": 30,
11
+ "splits": {
12
+ "train": "0:100"
13
+ },
14
+ "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
15
+ "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
16
+ "features": {
17
+ "observation.state": {
18
+ "dtype": "float32",
19
+ "shape": [
20
+ 78
21
+ ],
22
+ "names": null
23
+ },
24
+ "observation.eef_state": {
25
+ "dtype": "float32",
26
+ "shape": [
27
+ 44
28
+ ],
29
+ "names": [
30
+ "left_wrist_x",
31
+ "left_wrist_y",
32
+ "left_wrist_z",
33
+ "left_wrist_qw",
34
+ "left_wrist_qx",
35
+ "left_wrist_qy",
36
+ "left_wrist_qz",
37
+ "left_thumb_DP_x",
38
+ "left_thumb_DP_y",
39
+ "left_thumb_DP_z",
40
+ "left_index_DP_x",
41
+ "left_index_DP_y",
42
+ "left_index_DP_z",
43
+ "left_middle_DP_x",
44
+ "left_middle_DP_y",
45
+ "left_middle_DP_z",
46
+ "left_ring_DP_x",
47
+ "left_ring_DP_y",
48
+ "left_ring_DP_z",
49
+ "left_pinky_DP_x",
50
+ "left_pinky_DP_y",
51
+ "left_pinky_DP_z",
52
+ "right_wrist_x",
53
+ "right_wrist_y",
54
+ "right_wrist_z",
55
+ "right_wrist_qw",
56
+ "right_wrist_qx",
57
+ "right_wrist_qy",
58
+ "right_wrist_qz",
59
+ "right_thumb_DP_x",
60
+ "right_thumb_DP_y",
61
+ "right_thumb_DP_z",
62
+ "right_index_DP_x",
63
+ "right_index_DP_y",
64
+ "right_index_DP_z",
65
+ "right_middle_DP_x",
66
+ "right_middle_DP_y",
67
+ "right_middle_DP_z",
68
+ "right_ring_DP_x",
69
+ "right_ring_DP_y",
70
+ "right_ring_DP_z",
71
+ "right_pinky_DP_x",
72
+ "right_pinky_DP_y",
73
+ "right_pinky_DP_z"
74
+ ]
75
+ },
76
+ "observation.root_orientation": {
77
+ "dtype": "float32",
78
+ "shape": [
79
+ 4
80
+ ],
81
+ "names": [
82
+ "qw",
83
+ "qx",
84
+ "qy",
85
+ "qz"
86
+ ]
87
+ },
88
+ "observation.projected_gravity": {
89
+ "dtype": "float32",
90
+ "shape": [
91
+ 3
92
+ ],
93
+ "names": [
94
+ "gx",
95
+ "gy",
96
+ "gz"
97
+ ]
98
+ },
99
+ "observation.base_velocity": {
100
+ "dtype": "float32",
101
+ "shape": [
102
+ 3
103
+ ],
104
+ "names": [
105
+ "vx",
106
+ "vy",
107
+ "yaw_rate"
108
+ ]
109
+ },
110
+ "observation.object_state": {
111
+ "dtype": "float32",
112
+ "shape": [
113
+ 14
114
+ ],
115
+ "names": [
116
+ "pitcher_x",
117
+ "pitcher_y",
118
+ "pitcher_z",
119
+ "pitcher_qw",
120
+ "pitcher_qx",
121
+ "pitcher_qy",
122
+ "pitcher_qz",
123
+ "cup_x",
124
+ "cup_y",
125
+ "cup_z",
126
+ "cup_qw",
127
+ "cup_qx",
128
+ "cup_qy",
129
+ "cup_qz"
130
+ ]
131
+ },
132
+ "observation.task_state": {
133
+ "dtype": "float32",
134
+ "shape": [
135
+ 5
136
+ ],
137
+ "names": [
138
+ "cup_ml",
139
+ "spill_ml",
140
+ "source_ml",
141
+ "pouring",
142
+ "tilt_deg"
143
+ ]
144
+ },
145
+ "observation.tactile": {
146
+ "dtype": "float32",
147
+ "shape": [
148
+ 10,
149
+ 24,
150
+ 24
151
+ ],
152
+ "names": {
153
+ "finger": [
154
+ "left_thumb",
155
+ "left_index",
156
+ "left_middle",
157
+ "left_ring",
158
+ "left_pinky",
159
+ "right_thumb",
160
+ "right_index",
161
+ "right_middle",
162
+ "right_ring",
163
+ "right_pinky"
164
+ ],
165
+ "row": "u: pad joint end -> fingertip",
166
+ "column": "v: across the pad, thumb side first"
167
+ }
168
+ },
169
+ "observation.tactile_force": {
170
+ "dtype": "float32",
171
+ "shape": [
172
+ 10
173
+ ],
174
+ "names": [
175
+ "left_thumb",
176
+ "left_index",
177
+ "left_middle",
178
+ "left_ring",
179
+ "left_pinky",
180
+ "right_thumb",
181
+ "right_index",
182
+ "right_middle",
183
+ "right_ring",
184
+ "right_pinky"
185
+ ]
186
+ },
187
+ "action.arms": {
188
+ "dtype": "float32",
189
+ "shape": [
190
+ 14
191
+ ],
192
+ "names": [
193
+ "left_shoulder_pitch_joint",
194
+ "left_shoulder_roll_joint",
195
+ "left_shoulder_yaw_joint",
196
+ "left_elbow_joint",
197
+ "left_wrist_roll_joint",
198
+ "left_wrist_pitch_joint",
199
+ "left_wrist_yaw_joint",
200
+ "right_shoulder_pitch_joint",
201
+ "right_shoulder_roll_joint",
202
+ "right_shoulder_yaw_joint",
203
+ "right_elbow_joint",
204
+ "right_wrist_roll_joint",
205
+ "right_wrist_pitch_joint",
206
+ "right_wrist_yaw_joint"
207
+ ]
208
+ },
209
+ "action.hands": {
210
+ "dtype": "float32",
211
+ "shape": [
212
+ 44
213
+ ],
214
+ "names": [
215
+ "left_index_MCP_FE",
216
+ "left_middle_MCP_FE",
217
+ "left_pinky_CMC",
218
+ "left_ring_MCP_FE",
219
+ "left_thumb_CMC_FE",
220
+ "left_index_MCP_AA",
221
+ "left_middle_MCP_AA",
222
+ "left_pinky_MCP_FE",
223
+ "left_ring_MCP_AA",
224
+ "left_thumb_CMC_AA",
225
+ "left_index_PIP",
226
+ "left_middle_PIP",
227
+ "left_pinky_MCP_AA",
228
+ "left_ring_PIP",
229
+ "left_thumb_MCP_FE",
230
+ "left_index_DIP",
231
+ "left_middle_DIP",
232
+ "left_pinky_PIP",
233
+ "left_ring_DIP",
234
+ "left_thumb_MCP_AA",
235
+ "left_pinky_DIP",
236
+ "left_thumb_IP",
237
+ "right_index_MCP_FE",
238
+ "right_middle_MCP_FE",
239
+ "right_pinky_CMC",
240
+ "right_ring_MCP_FE",
241
+ "right_thumb_CMC_FE",
242
+ "right_index_MCP_AA",
243
+ "right_middle_MCP_AA",
244
+ "right_pinky_MCP_FE",
245
+ "right_ring_MCP_AA",
246
+ "right_thumb_CMC_AA",
247
+ "right_index_PIP",
248
+ "right_middle_PIP",
249
+ "right_pinky_MCP_AA",
250
+ "right_ring_PIP",
251
+ "right_thumb_MCP_FE",
252
+ "right_index_DIP",
253
+ "right_middle_DIP",
254
+ "right_pinky_PIP",
255
+ "right_ring_DIP",
256
+ "right_thumb_MCP_AA",
257
+ "right_pinky_DIP",
258
+ "right_thumb_IP"
259
+ ]
260
+ },
261
+ "action.body_target": {
262
+ "dtype": "float32",
263
+ "shape": [
264
+ 31
265
+ ],
266
+ "names": [
267
+ "left_hip_pitch_joint",
268
+ "left_hip_roll_joint",
269
+ "left_hip_yaw_joint",
270
+ "left_knee_joint",
271
+ "left_ankle_roll_joint",
272
+ "left_ankle_pitch_joint",
273
+ "right_hip_pitch_joint",
274
+ "right_hip_roll_joint",
275
+ "right_hip_yaw_joint",
276
+ "right_knee_joint",
277
+ "right_ankle_roll_joint",
278
+ "right_ankle_pitch_joint",
279
+ "waist_yaw_joint",
280
+ "waist_roll_joint",
281
+ "waist_pitch_joint",
282
+ "left_shoulder_pitch_joint",
283
+ "left_shoulder_roll_joint",
284
+ "left_shoulder_yaw_joint",
285
+ "left_elbow_joint",
286
+ "left_wrist_roll_joint",
287
+ "left_wrist_pitch_joint",
288
+ "left_wrist_yaw_joint",
289
+ "right_shoulder_pitch_joint",
290
+ "right_shoulder_roll_joint",
291
+ "right_shoulder_yaw_joint",
292
+ "right_elbow_joint",
293
+ "right_wrist_roll_joint",
294
+ "right_wrist_pitch_joint",
295
+ "right_wrist_yaw_joint",
296
+ "head_pitch_joint",
297
+ "head_yaw_joint"
298
+ ]
299
+ },
300
+ "action.locomotion": {
301
+ "dtype": "float32",
302
+ "shape": [
303
+ 3
304
+ ],
305
+ "names": [
306
+ "walking",
307
+ "desired_heading_rel_rad",
308
+ "playback_speed"
309
+ ]
310
+ },
311
+ "annotation.phase": {
312
+ "dtype": "int64",
313
+ "shape": [
314
+ 1
315
+ ],
316
+ "names": [
317
+ "phase"
318
+ ]
319
+ },
320
+ "observation.images.head": {
321
+ "dtype": "video",
322
+ "shape": [
323
+ 480,
324
+ 640,
325
+ 3
326
+ ],
327
+ "names": [
328
+ "height",
329
+ "width",
330
+ "channel"
331
+ ],
332
+ "info": {
333
+ "video.height": 480,
334
+ "video.width": 640,
335
+ "video.codec": "h264",
336
+ "video.pix_fmt": "yuv420p",
337
+ "video.is_depth_map": false,
338
+ "video.fps": 30,
339
+ "video.channels": 3,
340
+ "has_audio": false
341
+ }
342
+ },
343
+ "observation.images.chest": {
344
+ "dtype": "video",
345
+ "shape": [
346
+ 480,
347
+ 640,
348
+ 3
349
+ ],
350
+ "names": [
351
+ "height",
352
+ "width",
353
+ "channel"
354
+ ],
355
+ "info": {
356
+ "video.height": 480,
357
+ "video.width": 640,
358
+ "video.codec": "h264",
359
+ "video.pix_fmt": "yuv420p",
360
+ "video.is_depth_map": false,
361
+ "video.fps": 30,
362
+ "video.channels": 3,
363
+ "has_audio": false
364
+ }
365
+ },
366
+ "timestamp": {
367
+ "dtype": "float32",
368
+ "shape": [
369
+ 1
370
+ ],
371
+ "names": null
372
+ },
373
+ "frame_index": {
374
+ "dtype": "int64",
375
+ "shape": [
376
+ 1
377
+ ],
378
+ "names": null
379
+ },
380
+ "episode_index": {
381
+ "dtype": "int64",
382
+ "shape": [
383
+ 1
384
+ ],
385
+ "names": null
386
+ },
387
+ "index": {
388
+ "dtype": "int64",
389
+ "shape": [
390
+ 1
391
+ ],
392
+ "names": null
393
+ },
394
+ "task_index": {
395
+ "dtype": "int64",
396
+ "shape": [
397
+ 1
398
+ ],
399
+ "names": null
400
+ },
401
+ "action": {
402
+ "dtype": "float32",
403
+ "shape": [
404
+ 75
405
+ ],
406
+ "names": null
407
+ }
408
+ },
409
+ "script_config": {
410
+ "source": "sim",
411
+ "simulator": "newton",
412
+ "robot": "H2 + Sharpa (robot:h2_sharpa, experimental)",
413
+ "controller": "TRACE SONIC native runtime (legs, waist, balance; torso support kept for the tabletop tasks) plus the skill-server arm joint overrides and Sharpa position targets; scripted privileged expert",
414
+ "control_frequency": 50.0,
415
+ "camera_backend": "offline_render",
416
+ "camera_note": "head, chest and wrist rendered offline (humanoid_newton.rendering.splat_render) at the registered robot camera_mounts; third_person fixed; not calibrated cameras",
417
+ "tactile_note": "simulated contact footprints on the Sharpa elastomer pads (no sensor model)",
418
+ "task": "h2_pour_water",
419
+ "profile": "h2_pour_water_r1",
420
+ "task_description": "Grasp the pitcher's handle with the right hand, lift it, pour water into the glass tumbler, then put the pitcher back where it stood and let go.",
421
+ "phases": [
422
+ "idle",
423
+ "reach_pitcher",
424
+ "grasp_pitcher",
425
+ "lift_carry",
426
+ "pour",
427
+ "return",
428
+ "release"
429
+ ],
430
+ "tactile": {
431
+ "fingers": [
432
+ "left_thumb",
433
+ "left_index",
434
+ "left_middle",
435
+ "left_ring",
436
+ "left_pinky",
437
+ "right_thumb",
438
+ "right_index",
439
+ "right_middle",
440
+ "right_ring",
441
+ "right_pinky"
442
+ ],
443
+ "grid": 24,
444
+ "units": "N per taxel",
445
+ "module": "debug/newton/h2_sharpa_tactile.py"
446
+ },
447
+ "git_commit": "69c6ae8351f490ff9ffb87a56e95b11c91bf2f55"
448
+ },
449
+ "discarded_episode_indices": []
450
+ }
a18-pour-water/b512/step-002000/dataset_meta/tasks.jsonl ADDED
@@ -0,0 +1 @@
 
 
1
+ {"task_index": 0, "task": "Grasp the pitcher's handle with the right hand, lift it, pour water into the glass tumbler, then put the pitcher back where it stood and let go."}
a18-pour-water/b512/step-002000/dataset_stats.json ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-002000/logs/node-0.log ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-002000/logs/node-1.log ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-002000/logs/node-2.log ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-002000/logs/node-3.log ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-002000/loss_summary.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "points": 200,
3
+ "first": {
4
+ "epoch": 0,
5
+ "step": 20,
6
+ "loss": 1.325,
7
+ "loss_action": 1.0898,
8
+ "loss_video": 0.2352,
9
+ "lr": 1.05e-05,
10
+ "steps_per_s": 0.15
11
+ },
12
+ "last": {
13
+ "epoch": 30,
14
+ "step": 4000,
15
+ "loss": 0.0427,
16
+ "loss_action": 0.0035,
17
+ "loss_video": 0.0392,
18
+ "lr": 1e-06,
19
+ "steps_per_s": 0.22
20
+ },
21
+ "min_loss": 0.0395,
22
+ "mean_steps_per_s": 0.21804999999999938
23
+ }
a18-pour-water/b512/step-002000/metrics.csv ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ step,epoch,loss,loss_action,loss_video,lr,steps_per_s
2
+ 20,0,1.325,1.0898,0.2352,1.05e-05,0.15
3
+ 40,0,0.8912,0.6885,0.2026,2.04e-05,0.17
4
+ 60,0,0.4459,0.2836,0.1624,3.04e-05,0.19
5
+ 80,0,0.3124,0.155,0.1573,4.03e-05,0.19
6
+ 100,0,0.2571,0.1173,0.1398,5.02e-05,0.2
7
+ 120,0,0.2246,0.094,0.1306,6.02e-05,0.21
8
+ 140,1,0.2086,0.0922,0.1163,7.01e-05,0.2
9
+ 160,1,0.2059,0.0843,0.1217,8.01e-05,0.21
10
+ 180,1,0.1737,0.0686,0.1051,9e-05,0.21
11
+ 200,1,0.1653,0.0607,0.1047,0.0001,0.21
12
+ 220,1,0.1621,0.0514,0.1107,0.0001,0.21
13
+ 240,1,0.1666,0.0513,0.1153,0.0001,0.21
14
+ 260,1,0.1449,0.0478,0.0971,9.99e-05,0.22
15
+ 280,2,0.1413,0.0373,0.1041,9.99e-05,0.21
16
+ 300,2,0.1349,0.0368,0.0981,9.98e-05,0.21
17
+ 320,2,0.1307,0.0365,0.0942,9.98e-05,0.21
18
+ 340,2,0.126,0.032,0.094,9.97e-05,0.21
19
+ 360,2,0.124,0.0302,0.0937,9.96e-05,0.21
20
+ 380,2,0.1184,0.0291,0.0894,9.95e-05,0.22
21
+ 400,3,0.1162,0.0268,0.0895,9.93e-05,0.21
22
+ 420,3,0.1185,0.0298,0.0887,9.92e-05,0.21
23
+ 440,3,0.115,0.0326,0.0825,9.9e-05,0.21
24
+ 460,3,0.1143,0.0267,0.0877,9.89e-05,0.21
25
+ 480,3,0.1186,0.0261,0.0925,9.87e-05,0.22
26
+ 500,3,0.104,0.0257,0.0784,9.85e-05,0.22
27
+ 520,3,0.1087,0.0266,0.0821,9.83e-05,0.22
28
+ 540,4,0.1008,0.0249,0.0759,9.81e-05,0.21
29
+ 560,4,0.098,0.0223,0.0757,9.78e-05,0.22
30
+ 580,4,0.0988,0.0216,0.0772,9.76e-05,0.22
31
+ 600,4,0.1017,0.0204,0.0813,9.73e-05,0.22
32
+ 620,4,0.0999,0.0178,0.0821,9.7e-05,0.22
33
+ 640,4,0.0959,0.018,0.0779,9.68e-05,0.22
34
+ 660,5,0.0914,0.0181,0.0733,9.65e-05,0.22
35
+ 680,5,0.0873,0.0171,0.0702,9.62e-05,0.21
36
+ 700,5,0.09,0.0177,0.0724,9.58e-05,0.22
37
+ 720,5,0.0862,0.0167,0.0696,9.55e-05,0.22
38
+ 740,5,0.0887,0.0154,0.0733,9.51e-05,0.22
39
+ 760,5,0.1012,0.0166,0.0846,9.48e-05,0.22
40
+ 780,5,0.0838,0.0155,0.0683,9.44e-05,0.22
41
+ 800,6,0.0897,0.0143,0.0753,9.4e-05,0.22
42
+ 820,6,0.0923,0.0148,0.0776,9.36e-05,0.22
43
+ 840,6,0.0839,0.0138,0.0702,9.32e-05,0.22
44
+ 860,6,0.0899,0.0137,0.0763,9.28e-05,0.22
45
+ 880,6,0.0859,0.0143,0.0716,9.24e-05,0.22
46
+ 900,6,0.0877,0.0135,0.0742,9.19e-05,0.22
47
+ 920,7,0.091,0.0143,0.0767,9.15e-05,0.22
48
+ 940,7,0.0827,0.0132,0.0695,9.1e-05,0.22
49
+ 960,7,0.0878,0.0127,0.0751,9.05e-05,0.22
50
+ 980,7,0.0868,0.0118,0.075,9.01e-05,0.22
51
+ 1000,7,0.0863,0.0131,0.0732,8.96e-05,0.22
52
+ 1020,7,0.0892,0.0127,0.0765,8.91e-05,0.22
53
+ 1040,7,0.0797,0.0121,0.0676,8.85e-05,0.22
54
+ 1060,8,0.0853,0.0121,0.0733,8.8e-05,0.22
55
+ 1080,8,0.0783,0.0113,0.067,8.75e-05,0.22
56
+ 1100,8,0.0745,0.0108,0.0637,8.69e-05,0.22
57
+ 1120,8,0.0859,0.0124,0.0736,8.64e-05,0.22
58
+ 1140,8,0.0823,0.0114,0.0709,8.58e-05,0.22
59
+ 1160,8,0.0745,0.0122,0.0623,8.52e-05,0.22
60
+ 1180,9,0.0816,0.0113,0.0703,8.46e-05,0.22
61
+ 1200,9,0.0828,0.0107,0.0721,8.4e-05,0.22
62
+ 1220,9,0.0776,0.0096,0.0679,8.34e-05,0.22
63
+ 1240,9,0.0733,0.0102,0.0631,8.28e-05,0.22
64
+ 1260,9,0.0778,0.0117,0.0661,8.22e-05,0.22
65
+ 1280,9,0.0739,0.0093,0.0647,8.15e-05,0.22
66
+ 1300,9,0.0737,0.0101,0.0636,8.09e-05,0.22
67
+ 1320,10,0.0818,0.0094,0.0724,8.03e-05,0.22
68
+ 1340,10,0.0769,0.0093,0.0676,7.96e-05,0.22
69
+ 1360,10,0.0745,0.0099,0.0646,7.89e-05,0.22
70
+ 1380,10,0.0778,0.0088,0.069,7.83e-05,0.22
71
+ 1400,10,0.0756,0.0091,0.0664,7.76e-05,0.22
72
+ 1420,10,0.0714,0.0088,0.0625,7.69e-05,0.22
73
+ 1440,10,0.0711,0.0092,0.0619,7.62e-05,0.22
74
+ 1460,11,0.0698,0.0081,0.0616,7.55e-05,0.22
75
+ 1480,11,0.0657,0.0087,0.0569,7.48e-05,0.22
76
+ 1500,11,0.0673,0.0088,0.0585,7.41e-05,0.22
77
+ 1520,11,0.0747,0.0091,0.0656,7.33e-05,0.22
78
+ 1540,11,0.0665,0.008,0.0585,7.26e-05,0.22
79
+ 1560,11,0.0694,0.0092,0.0602,7.19e-05,0.22
80
+ 1580,12,0.0625,0.0079,0.0545,7.11e-05,0.22
81
+ 1600,12,0.0736,0.0088,0.0647,7.04e-05,0.22
82
+ 1620,12,0.0625,0.0081,0.0545,6.96e-05,0.22
83
+ 1640,12,0.0691,0.0083,0.0608,6.89e-05,0.22
84
+ 1660,12,0.0692,0.0067,0.0625,6.81e-05,0.22
85
+ 1680,12,0.0705,0.0073,0.0632,6.73e-05,0.22
86
+ 1700,12,0.0688,0.008,0.0607,6.66e-05,0.22
87
+ 1720,13,0.0586,0.0085,0.0501,6.58e-05,0.22
88
+ 1740,13,0.0686,0.0073,0.0614,6.5e-05,0.22
89
+ 1760,13,0.0604,0.0077,0.0527,6.42e-05,0.22
90
+ 1780,13,0.0677,0.0071,0.0606,6.34e-05,0.22
91
+ 1800,13,0.063,0.0077,0.0554,6.27e-05,0.22
92
+ 1820,13,0.0652,0.0074,0.0579,6.19e-05,0.22
93
+ 1840,14,0.0635,0.0076,0.0559,6.11e-05,0.22
94
+ 1860,14,0.0663,0.0073,0.059,6.03e-05,0.22
95
+ 1880,14,0.0642,0.0073,0.0569,5.95e-05,0.22
96
+ 1900,14,0.0614,0.0071,0.0543,5.86e-05,0.22
97
+ 1920,14,0.0609,0.007,0.0539,5.78e-05,0.22
98
+ 1940,14,0.07,0.0074,0.0626,5.7e-05,0.22
99
+ 1960,14,0.0644,0.0069,0.0575,5.62e-05,0.22
100
+ 1980,15,0.0633,0.0063,0.057,5.54e-05,0.22
101
+ 2000,15,0.0593,0.0063,0.053,5.46e-05,0.22
102
+ 2020,15,0.0566,0.0059,0.0507,5.38e-05,0.22
103
+ 2040,15,0.0587,0.0066,0.0522,5.3e-05,0.22
104
+ 2060,15,0.0606,0.0063,0.0543,5.21e-05,0.22
105
+ 2080,15,0.0596,0.0063,0.0533,5.13e-05,0.22
106
+ 2100,16,0.0601,0.0056,0.0545,5.05e-05,0.22
107
+ 2120,16,0.0605,0.0062,0.0543,4.97e-05,0.22
108
+ 2140,16,0.0537,0.0054,0.0483,4.89e-05,0.22
109
+ 2160,16,0.0569,0.0059,0.051,4.8e-05,0.22
110
+ 2180,16,0.0587,0.006,0.0527,4.72e-05,0.22
111
+ 2200,16,0.057,0.0062,0.0508,4.64e-05,0.22
112
+ 2220,16,0.0557,0.006,0.0497,4.56e-05,0.22
113
+ 2240,17,0.056,0.0055,0.0505,4.48e-05,0.22
114
+ 2260,17,0.0561,0.005,0.0511,4.4e-05,0.22
115
+ 2280,17,0.0495,0.0055,0.044,4.32e-05,0.22
116
+ 2300,17,0.0584,0.0057,0.0526,4.24e-05,0.22
117
+ 2320,17,0.0571,0.0057,0.0514,4.15e-05,0.22
118
+ 2340,17,0.0534,0.0056,0.0478,4.07e-05,0.22
119
+ 2360,18,0.0492,0.0055,0.0437,3.99e-05,0.22
120
+ 2380,18,0.0532,0.0054,0.0477,3.91e-05,0.22
121
+ 2400,18,0.0519,0.0055,0.0464,3.83e-05,0.22
122
+ 2420,18,0.0589,0.0058,0.0531,3.76e-05,0.22
123
+ 2440,18,0.0569,0.0049,0.052,3.68e-05,0.22
124
+ 2460,18,0.0474,0.0056,0.0418,3.6e-05,0.22
125
+ 2480,18,0.0503,0.0048,0.0455,3.52e-05,0.22
126
+ 2500,19,0.0501,0.0054,0.0448,3.44e-05,0.22
127
+ 2520,19,0.0515,0.0046,0.0469,3.37e-05,0.22
128
+ 2540,19,0.0501,0.0048,0.0453,3.29e-05,0.22
129
+ 2560,19,0.0563,0.0046,0.0517,3.21e-05,0.22
130
+ 2580,19,0.0476,0.0049,0.0427,3.14e-05,0.22
131
+ 2600,19,0.0468,0.0047,0.0421,3.06e-05,0.22
132
+ 2620,19,0.0455,0.0049,0.0406,2.99e-05,0.22
133
+ 2640,20,0.0472,0.0048,0.0424,2.91e-05,0.22
134
+ 2660,20,0.0467,0.0048,0.042,2.84e-05,0.22
135
+ 2680,20,0.0567,0.0043,0.0524,2.77e-05,0.22
136
+ 2700,20,0.05,0.0048,0.0452,2.69e-05,0.22
137
+ 2720,20,0.0534,0.0039,0.0495,2.62e-05,0.22
138
+ 2740,20,0.0535,0.0047,0.0488,2.55e-05,0.22
139
+ 2760,21,0.0499,0.0046,0.0453,2.48e-05,0.22
140
+ 2780,21,0.0499,0.0044,0.0455,2.41e-05,0.22
141
+ 2800,21,0.046,0.0044,0.0416,2.34e-05,0.22
142
+ 2820,21,0.0489,0.0043,0.0446,2.27e-05,0.22
143
+ 2840,21,0.0472,0.0048,0.0424,2.21e-05,0.22
144
+ 2860,21,0.0474,0.0044,0.043,2.14e-05,0.22
145
+ 2880,21,0.0455,0.0041,0.0414,2.07e-05,0.22
146
+ 2900,22,0.0481,0.0043,0.0439,2.01e-05,0.22
147
+ 2920,22,0.0426,0.0044,0.0382,1.95e-05,0.22
148
+ 2940,22,0.0462,0.0042,0.042,1.88e-05,0.22
149
+ 2960,22,0.0516,0.0045,0.0471,1.82e-05,0.22
150
+ 2980,22,0.0449,0.004,0.041,1.76e-05,0.22
151
+ 3000,22,0.0474,0.0041,0.0432,1.7e-05,0.22
152
+ 3020,23,0.0505,0.0041,0.0465,1.64e-05,0.22
153
+ 3040,23,0.044,0.004,0.0401,1.58e-05,0.22
154
+ 3060,23,0.0507,0.0042,0.0466,1.52e-05,0.22
155
+ 3080,23,0.0429,0.0041,0.0389,1.46e-05,0.22
156
+ 3100,23,0.045,0.0041,0.041,1.41e-05,0.22
157
+ 3120,23,0.0439,0.0039,0.0401,1.35e-05,0.22
158
+ 3140,23,0.0449,0.0039,0.041,1.3e-05,0.22
159
+ 3160,24,0.0477,0.0041,0.0435,1.25e-05,0.22
160
+ 3180,24,0.0439,0.0037,0.0402,1.19e-05,0.22
161
+ 3200,24,0.0475,0.0044,0.0431,1.14e-05,0.22
162
+ 3220,24,0.0417,0.0039,0.0377,1.09e-05,0.22
163
+ 3240,24,0.0481,0.0038,0.0443,1.05e-05,0.22
164
+ 3260,24,0.0473,0.0039,0.0434,9.98e-06,0.22
165
+ 3280,25,0.046,0.0036,0.0424,9.51e-06,0.22
166
+ 3300,25,0.0487,0.0035,0.0453,9.06e-06,0.22
167
+ 3320,25,0.0453,0.0044,0.0409,8.62e-06,0.22
168
+ 3340,25,0.0458,0.0037,0.0422,8.19e-06,0.22
169
+ 3360,25,0.0429,0.0039,0.0391,7.77e-06,0.22
170
+ 3380,25,0.044,0.0041,0.0399,7.36e-06,0.22
171
+ 3400,25,0.0432,0.0038,0.0394,6.97e-06,0.22
172
+ 3420,26,0.0464,0.0038,0.0426,6.58e-06,0.22
173
+ 3440,26,0.0455,0.0035,0.042,6.21e-06,0.22
174
+ 3460,26,0.041,0.0036,0.0374,5.85e-06,0.22
175
+ 3480,26,0.0451,0.0031,0.042,5.5e-06,0.22
176
+ 3500,26,0.0416,0.0035,0.0381,5.17e-06,0.22
177
+ 3520,26,0.0459,0.0039,0.042,4.85e-06,0.22
178
+ 3540,27,0.0419,0.0034,0.0385,4.54e-06,0.22
179
+ 3560,27,0.0433,0.0033,0.04,4.24e-06,0.22
180
+ 3580,27,0.0428,0.004,0.0389,3.95e-06,0.22
181
+ 3600,27,0.0422,0.0032,0.039,3.68e-06,0.22
182
+ 3620,27,0.0426,0.0032,0.0393,3.42e-06,0.22
183
+ 3640,27,0.0426,0.0036,0.039,3.18e-06,0.22
184
+ 3660,27,0.0474,0.0035,0.044,2.94e-06,0.22
185
+ 3680,28,0.041,0.0034,0.0376,2.72e-06,0.22
186
+ 3700,28,0.0424,0.0037,0.0387,2.51e-06,0.22
187
+ 3720,28,0.0448,0.0037,0.0411,2.32e-06,0.22
188
+ 3740,28,0.0402,0.0032,0.0369,2.14e-06,0.22
189
+ 3760,28,0.0415,0.0036,0.0379,1.97e-06,0.22
190
+ 3780,28,0.0491,0.0035,0.0456,1.82e-06,0.22
191
+ 3800,29,0.0417,0.0029,0.0388,1.68e-06,0.22
192
+ 3820,29,0.0395,0.0035,0.0359,1.55e-06,0.22
193
+ 3840,29,0.0444,0.0039,0.0405,1.43e-06,0.22
194
+ 3860,29,0.0405,0.0033,0.0372,1.33e-06,0.22
195
+ 3880,29,0.0422,0.0034,0.0388,1.24e-06,0.22
196
+ 3900,29,0.0428,0.0037,0.0391,1.17e-06,0.22
197
+ 3920,29,0.0437,0.0033,0.0403,1.11e-06,0.22
198
+ 3940,30,0.0406,0.0032,0.0374,1.06e-06,0.22
199
+ 3960,30,0.0458,0.0035,0.0423,1.03e-06,0.22
200
+ 3980,30,0.0404,0.0033,0.0371,1.01e-06,0.22
201
+ 4000,30,0.0427,0.0035,0.0392,1e-06,0.22
a18-pour-water/b512/step-002000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdbe894e6758b8ad8bc738bc79ea36382304256a50bae63a24f7d84565eef213
3
+ size 12042587769
a18-pour-water/b512/step-002000/model_config.yaml ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _target_: fastwam.runtime.create_fastwam
2
+ model_id: Wan-AI/Wan2.2-TI2V-5B
3
+ tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
4
+ tokenizer_max_len: 128
5
+ load_text_encoder: false
6
+ proprio_dim: 78
7
+ redirect_common_files: false
8
+ mot_checkpoint_mixed_attn: false
9
+ compile_training_denoise: false
10
+ action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt
11
+ skip_dit_load_from_pretrain: false
12
+ video_dit_config:
13
+ has_image_input: false
14
+ patch_size:
15
+ - 1
16
+ - 2
17
+ - 2
18
+ in_dim: 48
19
+ hidden_dim: 3072
20
+ ffn_dim: 14336
21
+ freq_dim: 256
22
+ text_dim: 4096
23
+ out_dim: 48
24
+ num_heads: 24
25
+ attn_head_dim: 128
26
+ num_layers: 30
27
+ eps: 1.0e-06
28
+ seperated_timestep: true
29
+ require_clip_embedding: false
30
+ require_vae_embedding: false
31
+ fuse_vae_embedding_in_latents: true
32
+ use_gradient_checkpointing: false
33
+ video_attention_mask_mode: first_frame_causal
34
+ action_conditioned: false
35
+ action_dim: 75
36
+ action_group_causal_mask_mode: group_diagonal
37
+ action_dit_config:
38
+ action_dim: 75
39
+ hidden_dim: 1024
40
+ ffn_dim: 4096
41
+ num_heads: 24
42
+ attn_head_dim: 128
43
+ num_layers: 30
44
+ text_dim: 4096
45
+ freq_dim: 256
46
+ eps: 1.0e-06
47
+ use_gradient_checkpointing: false
48
+ video_scheduler:
49
+ train_shift: 5.0
50
+ infer_shift: 5.0
51
+ num_train_timesteps: 1000
52
+ action_scheduler:
53
+ train_shift: 1.0
54
+ infer_shift: 1.0
55
+ num_train_timesteps: 1000
56
+ loss:
57
+ lambda_action: 1.0
a18-pour-water/b512/step-002000/protocol.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "FastWAM (Wan2.2-TI2V-5B video expert + 1.02 B action expert, mixture-of-transformers world action model; upstream github.com/yuantianyuan01/FastWAM)",
3
+ "task": "A18 pour water (H2), 2 cameras",
4
+ "task_dir": "a18-pour-water/b512",
5
+ "robot": "H2 (Sharpa hands)",
6
+ "training_data": "a18-pour-water-expert-cropped-v2 @44cdd57 (100 episodes, waits trimmed)",
7
+ "episodes": 100,
8
+ "epoch_completed": null,
9
+ "global_step": 2000,
10
+ "augmentation": "Humanoid Benchmark online augmentation spec v2 (2026-11-09): 15 visual effects, one effect per training sample at 1.5x; proprioception aligned with the GR00T A20 recipe (2026-11-12): per-sample joint bias +-2 deg + Gaussian noise 0.25 deg (hb_aug_v2.perturb_joints), no proprio delay, no history drop; model-native image augmentation off",
11
+ "epochs": null,
12
+ "total_steps": 4000,
13
+ "save_steps": [
14
+ 2000,
15
+ 4000
16
+ ],
17
+ "num_gpus": 32,
18
+ "per_gpu_batch": 16,
19
+ "global_batch": 512,
20
+ "gradient_accumulation_steps": 1,
21
+ "frames": 66850,
22
+ "fps": 30,
23
+ "cameras": [
24
+ "head",
25
+ "chest"
26
+ ],
27
+ "state_dim": 78,
28
+ "action_dim": 75,
29
+ "state_layout": "state78 = 75 joint angles + projected gravity 3",
30
+ "action_layout": "action75 = action.body_target (31) + action.hands (44)",
31
+ "trainable": "full fine-tune of the mixture-of-transformers DiT: video expert (Wan2.2-TI2V-5B, 5.00 B) + action expert (1.02 B); the Wan2.2 VAE and the T5 text encoder are frozen (prompt embeddings precomputed, text_embeddings/)",
32
+ "recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
33
+ "video": "17-frame clips (4 latent frames + the observation frame), 224 x 448 (the two cameras side by side), action chunk 16",
34
+ "base_model": "Wan-AI/Wan2.2-TI2V-5B (DiT, VAE, T5) + the umt5-xxl tokenizer of Wan-AI/Wan2.1-T2V-1.3B, loaded from their own repositories; ActionDiT initialised by FastWAM's preprocess_action_dit_backbone.py",
35
+ "upstream": "github.com/yuantianyuan01/FastWAM @ 7faa71108368fbb3b6885649f112af607427a2d4",
36
+ "batch_selection": "largest per-GPU batch that fits on A800 80 GB: 16 trains at ~58 GB, 32 runs out of memory (batch_benchmark/)",
37
+ "normalization": "dataset_stats.json (FastWAM min / max statistics of this task)",
38
+ "config": "training_config.yaml (resolved FastWAM config; absolute paths refer to the training NAS), model_config.yaml (the model part)",
39
+ "inference": "FastWAM: build the model from model_config.yaml, load model.pt (DiT state dict), the normalizer from dataset_stats.json and the prompt embedding from text_embeddings/; code/hb_fastwam_convert.py documents the state / action layout of the benchmark interface; code/ad_fastwam.py is the inference adapter of our evaluation (action-only FastWAM.infer_action, 10 denoising steps, seed 42; its default paths refer to the training NAS)",
40
+ "training_records_included": true,
41
+ "resume_state_included": false,
42
+ "files": {
43
+ "batch_benchmark/chosen.json": 210,
44
+ "batch_benchmark/summary.txt": 127,
45
+ "code/ad_fastwam.py": 8413,
46
+ "code/fastwam_prep.sh": 3634,
47
+ "code/fastwam_task.sh": 3719,
48
+ "code/hb_aug_v2.py": 7475,
49
+ "code/hb_eval_common.py": 13273,
50
+ "code/hb_fastwam_aug.py": 3345,
51
+ "code/hb_fastwam_convert.py": 7101,
52
+ "code/hb_fastwam_train.py": 5570,
53
+ "code/hb_inloop_core.py": 11206,
54
+ "code/hb_inloop_fastwam.py": 5431,
55
+ "code/test_fastwam_inputs.py": 2724,
56
+ "complete.json": 662,
57
+ "dataset_meta/hb_fastwam_interface.json": 339,
58
+ "dataset_meta/info.json": 10450,
59
+ "dataset_meta/tasks.jsonl": 174,
60
+ "dataset_stats.json": 216001,
61
+ "logs/node-0.log": 529655,
62
+ "logs/node-1.log": 439077,
63
+ "logs/node-2.log": 438866,
64
+ "logs/node-3.log": 438908,
65
+ "loss_summary.json": 382,
66
+ "metrics.csv": 8447,
67
+ "model.pt": 12042587769,
68
+ "model_config.yaml": 1350,
69
+ "text_embeddings/cd0ca5d7666fa68fe2141dfac894ceea404ab7cf2a88b8617a489f124dfdfdf2.t5_len128.wan22ti2v5b.pt": 1051869,
70
+ "training_config.yaml": 5856
71
+ }
72
+ }
a18-pour-water/b512/step-002000/text_embeddings/cd0ca5d7666fa68fe2141dfac894ceea404ab7cf2a88b8617a489f124dfdfdf2.t5_len128.wan22ti2v5b.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:812855e3448be40732596a0cb9828e6c760b3c2468f751d9ef72bf9f9bcca8c3
3
+ size 1051869
a18-pour-water/b512/step-002000/training_config.yaml ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ output_dir: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/runs/a18cr2/out
2
+ batch_size: 16
3
+ num_workers: 8
4
+ lr_scheduler_type: cosine
5
+ learning_rate: 0.0001
6
+ num_epochs: 10
7
+ max_steps: 4000
8
+ log_every: 20
9
+ save_every: 2000
10
+ eval_every: 0
11
+ eval_num_inference_steps: 10
12
+ gradient_accumulation_steps: 1
13
+ mixed_precision: bf16
14
+ seed: 42
15
+ max_grad_norm: 1.0
16
+ weight_decay: 0.01
17
+ resume: null
18
+ wandb:
19
+ enabled: false
20
+ workspace: null
21
+ project: fast-wam
22
+ name: train
23
+ group: null
24
+ mode: online
25
+ data:
26
+ train:
27
+ _target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
28
+ dataset_dirs:
29
+ - /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/data/a18cr2
30
+ shape_meta:
31
+ images:
32
+ - key: head
33
+ raw_shape:
34
+ - 3
35
+ - 224
36
+ - 224
37
+ shape:
38
+ - 3
39
+ - 224
40
+ - 224
41
+ - key: chest
42
+ raw_shape:
43
+ - 3
44
+ - 224
45
+ - 224
46
+ shape:
47
+ - 3
48
+ - 224
49
+ - 224
50
+ action:
51
+ - key: default
52
+ raw_shape: 75
53
+ shape: 75
54
+ state:
55
+ - key: default
56
+ raw_shape: 78
57
+ shape: 78
58
+ num_frames: 17
59
+ global_sample_stride: 1
60
+ action_video_freq_ratio: 4
61
+ video_size:
62
+ - 224
63
+ - 448
64
+ camera_key: null
65
+ val_set_proportion: 0.0
66
+ is_training_set: true
67
+ skip_padding_as_possible: false
68
+ concat_multi_camera: horizontal
69
+ processor:
70
+ _target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor
71
+ shape_meta:
72
+ images:
73
+ - key: head
74
+ raw_shape:
75
+ - 3
76
+ - 224
77
+ - 224
78
+ shape:
79
+ - 3
80
+ - 224
81
+ - 224
82
+ - key: chest
83
+ raw_shape:
84
+ - 3
85
+ - 224
86
+ - 224
87
+ shape:
88
+ - 3
89
+ - 224
90
+ - 224
91
+ action:
92
+ - key: default
93
+ raw_shape: 75
94
+ shape: 75
95
+ state:
96
+ - key: default
97
+ raw_shape: 78
98
+ shape: 78
99
+ num_obs_steps: 17
100
+ num_output_cameras: 2
101
+ action_output_dim: 75
102
+ proprio_output_dim: 78
103
+ delta_action_dim_mask:
104
+ default:
105
+ - false
106
+ - false
107
+ - false
108
+ - false
109
+ - false
110
+ - false
111
+ - false
112
+ - false
113
+ - false
114
+ - false
115
+ - false
116
+ - false
117
+ - false
118
+ - false
119
+ - false
120
+ - false
121
+ - false
122
+ - false
123
+ - false
124
+ - false
125
+ - false
126
+ - false
127
+ - false
128
+ - false
129
+ - false
130
+ - false
131
+ - false
132
+ - false
133
+ - false
134
+ - false
135
+ - false
136
+ - false
137
+ - false
138
+ - false
139
+ - false
140
+ - false
141
+ - false
142
+ - false
143
+ - false
144
+ - false
145
+ - false
146
+ - false
147
+ - false
148
+ - false
149
+ - false
150
+ - false
151
+ - false
152
+ - false
153
+ - false
154
+ - false
155
+ - false
156
+ - false
157
+ - false
158
+ - false
159
+ - false
160
+ - false
161
+ - false
162
+ - false
163
+ - false
164
+ - false
165
+ - false
166
+ - false
167
+ - false
168
+ - false
169
+ - false
170
+ - false
171
+ - false
172
+ - false
173
+ - false
174
+ - false
175
+ - false
176
+ - false
177
+ - false
178
+ - false
179
+ - false
180
+ action_state_transforms: null
181
+ use_stepwise_action_norm: false
182
+ norm_default_mode: min/max
183
+ norm_exception_mode: null
184
+ action_state_merger:
185
+ _target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
186
+ train_transforms:
187
+ - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
188
+ - _target_: torchvision.transforms.Resize
189
+ size:
190
+ - 224
191
+ - 224
192
+ val_transforms:
193
+ - _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
194
+ - _target_: torchvision.transforms.Resize
195
+ size:
196
+ - 224
197
+ - 224
198
+ text_embedding_cache_dir: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/text-cache/all
199
+ use_text_embed_cache: true
200
+ context_len: 128
201
+ model:
202
+ _target_: fastwam.runtime.create_fastwam
203
+ model_id: Wan-AI/Wan2.2-TI2V-5B
204
+ tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
205
+ tokenizer_max_len: 128
206
+ load_text_encoder: false
207
+ proprio_dim: 78
208
+ redirect_common_files: false
209
+ mot_checkpoint_mixed_attn: false
210
+ compile_training_denoise: false
211
+ action_dit_pretrained_path: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/action-init-h2.pt
212
+ skip_dit_load_from_pretrain: false
213
+ video_dit_config:
214
+ has_image_input: false
215
+ patch_size:
216
+ - 1
217
+ - 2
218
+ - 2
219
+ in_dim: 48
220
+ hidden_dim: 3072
221
+ ffn_dim: 14336
222
+ freq_dim: 256
223
+ text_dim: 4096
224
+ out_dim: 48
225
+ num_heads: 24
226
+ attn_head_dim: 128
227
+ num_layers: 30
228
+ eps: 1.0e-06
229
+ seperated_timestep: true
230
+ require_clip_embedding: false
231
+ require_vae_embedding: false
232
+ fuse_vae_embedding_in_latents: true
233
+ use_gradient_checkpointing: false
234
+ video_attention_mask_mode: first_frame_causal
235
+ action_conditioned: false
236
+ action_dim: 75
237
+ action_group_causal_mask_mode: group_diagonal
238
+ action_dit_config:
239
+ action_dim: 75
240
+ hidden_dim: 1024
241
+ ffn_dim: 4096
242
+ num_heads: 24
243
+ attn_head_dim: 128
244
+ num_layers: 30
245
+ text_dim: 4096
246
+ freq_dim: 256
247
+ eps: 1.0e-06
248
+ use_gradient_checkpointing: false
249
+ video_scheduler:
250
+ train_shift: 5.0
251
+ infer_shift: 5.0
252
+ num_train_timesteps: 1000
253
+ action_scheduler:
254
+ train_shift: 1.0
255
+ infer_shift: 1.0
256
+ num_train_timesteps: 1000
257
+ loss:
258
+ lambda_action: 1.0
a18-pour-water/b512/step-004000/batch_benchmark/chosen.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"per_gpu_batch": 16, "global_batch": 512, "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/bench-20261116-r2/summary.txt"}
a18-pour-water/b512/step-004000/batch_benchmark/summary.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ bs=16 global=512 rc=[0 0 0 0 ] oom=0 peak_mem_MiB=58421 speed=
2
+ bs=32 global=1024 rc=[1 1 1 1 ] oom=4 peak_mem_MiB=81123 speed=
a18-pour-water/b512/step-004000/code/ad_fastwam.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM adapter (2026-11-16). Deployment path of FastWAM's own policy (experiments/robotwin/fastwam_policy/
2
+ deploy_policy.py, WorldActionRobotWinPolicy._infer_action_chunk: model.infer_action, the action-only path = first-frame
3
+ video KV cache + action-expert denoising, no future video) fed exactly like our training dataset (RobotVideoDataset +
4
+ FastWAMProcessor of the run's config.yaml, no HB augmentation):
5
+ * cameras in shape_meta order (g1 ego_view + chest, h2 head + chest): uint8 [1,C,H,W] -> processor val transforms
6
+ (ToTensor /255 + Resize 224x224), concatenated horizontally (224 x 448), dataset ResizeSmallestSide/CenterCrop
7
+ (no-op at 224 x 448), Normalize(0.5, 0.5) -> [-1, 1];
8
+ * state: processor.action_state_transform -> normalizer.forward (min/max of the run's
9
+ dataset_stats.json), the proprio of the first frame;
10
+ * text: the run's T5 cache for DEFAULT_PROMPT.format(task=instruction) with context[~mask] = 0 and mask = ones, as in
11
+ training (the text encoder is not loaded in training either; deploy_policy encodes the same prompt with T5 online);
12
+ * infer_action(action_horizon = num_frames - 1 = 16, num_inference_steps = config eval_num_inference_steps (10),
13
+ text_cfg_scale 1, seed 42 = native trainer eval), denormalised with normalizer.normalizers["action"][key].backward.
14
+ ckpt = .../fastwam/runs/<task>/out/checkpoints/weights/step_<N>.pt (config.yaml + dataset_stats.json in .../out).
15
+ Weights: model built from the config (Wan2.2 + ActionDiT init), then the checkpoint's mot + proprio_encoder are
16
+ loaded STRICTLY (FastWAM.load_checkpoint uses strict=False for mot; a missing key here is an error).
17
+ """
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+
24
+ import hb_eval_common as E
25
+
26
+ FW = E.G / "fastwam"
27
+
28
+
29
+ def setup_env():
30
+ os.environ.setdefault("FASTWAM_SRC", str(FW / "FastWAM"))
31
+ os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(FW / "models"))
32
+ os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "true")
33
+ os.environ.setdefault("HF_HUB_OFFLINE", "1")
34
+ src = str(Path(os.environ["FASTWAM_SRC"]) / "src")
35
+ if src not in sys.path:
36
+ sys.path.insert(0, src)
37
+
38
+
39
+ class Inputs:
40
+ """model-agnostic cache query -> FastWAM inputs (shared by the post-hoc adapter and the in-training eval)"""
41
+
42
+ def __init__(self, cfg, stats_path):
43
+ setup_env()
44
+ from hydra.utils import instantiate
45
+ from omegaconf import OmegaConf
46
+ from fastwam.datasets.dataset_utils import CenterCrop, Normalize, ResizeSmallestSideAspectPreserving
47
+ from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT
48
+ from fastwam.datasets.lerobot.utils.normalizer import load_dataset_stats_from_json
49
+ d = cfg.data.train
50
+ self.DEFAULT_PROMPT = DEFAULT_PROMPT
51
+ self.proc = instantiate(d.processor).eval()
52
+ self.proc.set_normalizer_from_stats(load_dataset_stats_from_json(str(stats_path)))
53
+ self.cams = [m["key"] for m in self.proc.shape_meta["images"]]
54
+ if d.concat_multi_camera != "horizontal":
55
+ raise ValueError(f"concat_multi_camera {d.concat_multi_camera} not supported")
56
+ vs = list(d.video_size)
57
+ self.resize = ResizeSmallestSideAspectPreserving(args={"img_w": vs[1], "img_h": vs[0]})
58
+ self.crop = CenterCrop(args={"img_w": vs[1], "img_h": vs[0]})
59
+ self.norm = Normalize(args={"mean": 0.5, "std": 0.5})
60
+ self.cache_dir = d.text_embedding_cache_dir
61
+ self.context_len = int(d.context_len)
62
+ self.horizon = int(d.num_frames) - 1
63
+ self.skey = self.proc.shape_meta["state"][0]["key"]
64
+ self.akey = self.proc.shape_meta["action"][0]["key"]
65
+ if len(self.proc.shape_meta["state"]) != 1 or len(self.proc.shape_meta["action"]) != 1:
66
+ raise ValueError("expected one merged state key and one merged action key")
67
+ self._ctx = {}
68
+ self.OmegaConf = OmegaConf
69
+
70
+ def image(self, images):
71
+ import torch
72
+ per = []
73
+ for c in self.cams:
74
+ x = torch.from_numpy(np.ascontiguousarray(images[c])).permute(2, 0, 1).unsqueeze(0) # [1,C,H,W] uint8
75
+ for tr in self.proc.val_transforms:
76
+ x = tr(x)
77
+ per.append(x)
78
+ v = torch.cat(per, dim=-1) # [1, C, 224, 448]
79
+ v = self.norm(self.crop(self.resize(v)))
80
+ return v # [1, 3, H, W] in [-1, 1], float32
81
+
82
+ def proprio(self, state):
83
+ import torch
84
+ b = {"state": {self.skey: torch.as_tensor(np.asarray(state, dtype=np.float32)).unsqueeze(0)}}
85
+ b = self.proc.action_state_transform(b)
86
+ b = self.proc.normalizer.forward(b) # = deploy_policy._normalize_state (one merged key: merger is identity)
87
+ return b["state"][self.skey] # [1, D]
88
+
89
+ def context(self, instruction):
90
+ import hashlib
91
+ import torch
92
+ prompt = self.DEFAULT_PROMPT.format(task=instruction)
93
+ if prompt not in self._ctx:
94
+ h = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
95
+ p = os.path.join(self.cache_dir, f"{h}.t5_len{self.context_len}.wan22ti2v5b.pt")
96
+ if not os.path.exists(p):
97
+ raise FileNotFoundError(f"no T5 cache for the instruction ({p}): {instruction[:80]!r}")
98
+ obj = torch.load(p, map_location="cpu")
99
+ ctx, mask = obj["context"], obj["mask"]
100
+ ctx = ctx.clone()
101
+ ctx[~mask.bool()] = 0.0
102
+ self._ctx[prompt] = (ctx, torch.ones_like(mask.bool()))
103
+ return self._ctx[prompt]
104
+
105
+ def denorm(self, action):
106
+ import torch
107
+ a = action
108
+ if a.ndim == 2:
109
+ a = a.unsqueeze(0)
110
+ n = self.proc.normalizer.normalizers["action"][self.akey]
111
+ return n.backward(a.to(dtype=torch.float32, device="cpu")).numpy()[0]
112
+
113
+
114
+ def infer(model, inp, obs, steps, seed=42):
115
+ import torch
116
+ img = inp.image(obs["images"]).to(device=model.device, dtype=model.torch_dtype)
117
+ ctx, mask = inp.context(obs["prompt"])
118
+ with torch.no_grad():
119
+ out = model.infer_action(prompt=None, input_image=img, action_horizon=inp.horizon, proprio=inp.proprio(obs["state"]),
120
+ context=ctx, context_mask=mask, text_cfg_scale=1.0, num_inference_steps=steps, seed=seed,
121
+ rand_device="cpu", tiled=False)
122
+ return inp.denorm(out["action"])
123
+
124
+
125
+ class Adapter:
126
+ def __init__(self, ckpt, task, split, meta, device="cuda:0", steps=None, seed="42"):
127
+ setup_env()
128
+ import torch
129
+ from hydra.utils import instantiate
130
+ from omegaconf import OmegaConf
131
+ self.torch = torch
132
+ ckpt = Path(ckpt)
133
+ out = ckpt.parent.parent.parent
134
+ self.cfg = OmegaConf.load(out / "config.yaml")
135
+ self.inp = Inputs(self.cfg, out / "dataset_stats.json")
136
+ miss = [c for c in self.inp.cams if c not in meta["cams"]]
137
+ if miss:
138
+ raise KeyError(f"cameras {miss} not in the cache {meta['cams']}")
139
+ self.device = device if torch.cuda.is_available() else "cpu"
140
+ mp = str(self.cfg.get("mixed_precision", "bf16")).lower()
141
+ dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[mp]
142
+ self.model = instantiate(self.cfg.model, model_dtype=dtype, device=self.device)
143
+ payload = torch.load(str(ckpt), map_location="cpu")
144
+ miss_k, unexp = self.model.mot.load_state_dict(payload["mot"], strict=False)
145
+ if miss_k or unexp:
146
+ raise RuntimeError(f"checkpoint mot keys: missing {list(miss_k)[:5]} unexpected {list(unexp)[:5]}")
147
+ if self.model.proprio_encoder is not None:
148
+ self.model.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
149
+ self.ckpt_step = payload.get("step")
150
+ del payload
151
+ self.model = self.model.to(self.device).eval()
152
+ self.native_steps = int(self.cfg.eval_num_inference_steps)
153
+ self.steps = int(steps or self.native_steps)
154
+ self.seed = int(seed)
155
+ self.ckpt = str(ckpt)
156
+
157
+ def info(self):
158
+ return {"model": "fastwam", "ckpt": self.ckpt, "ckpt_step": self.ckpt_step, "denoising_steps": self.steps,
159
+ "native_steps_in_config": self.native_steps, "cams": self.inp.cams, "chunk": self.inp.horizon,
160
+ "seed": self.seed, "path": "FastWAM.infer_action (action-only, deploy_policy)"}
161
+
162
+ def predict(self, obs):
163
+ return infer(self.model, self.inp, obs, self.steps, self.seed)
a18-pour-water/b512/step-004000/code/fastwam_prep.sh ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # 2026-11-16: CPU-only preparation of FastWAM for the 12 Delta-flow tasks (no GPU is touched: training is running).
3
+ # Data copies on our interface, ActionDiT initializers per profile, one shared T5 text cache, augmented-sample checks.
4
+ P=/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z
5
+ G=$P/policies-20261109
6
+ F=$G/fastwam
7
+ E=$G/envs/fastwam/bin/python
8
+ exec >> $F/prep.log 2>&1
9
+ echo "$(date) start (v2: 12 tasks)"
10
+ until [ -f $G/envs/fastwam/.hb_ready ] || grep -q ENV_FAILED $F/setup.log; do sleep 30; done
11
+ [ -f $G/envs/fastwam/.hb_ready ] || { echo "ENV_FAILED"; exit 1; }
12
+ grep -q FW_DL_DONE $F/dl2.log || { echo "DOWNLOAD_FAILED"; exit 1; }
13
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true CUDA_VISIBLE_DEVICES="" HF_HUB_OFFLINE=1
14
+ export HB_AUG_DIR=$G/hb_groot
15
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT $(compgen -e | grep -E "^PET_")
16
+ TC=$F/text-cache/all
17
+ # task profile source [first N episodes]
18
+ cat > $F/tasks.txt <<EOF
19
+ a10v3 g1 $P/a10-trolley-push-v3-6e410c5/data
20
+ a3v4 g1 $P/a3-fridge-open-v4-6393d4a/data
21
+ a7v4 g1 $P/a7-cabinet-close-v4-90cc2b5/data
22
+ b7 g1 $P/b7-black-clothes-hamper-v1-1bb002a/data 100
23
+ b5 g1 $P/b5-chair-push-v1-b85b7e9/data
24
+ b8 g1 $P/b8-table-wipe-v1-3b2e618/data
25
+ a17cr2 h2 $P/a17-unscrew-bottle-cropped-v2-3520b70/data
26
+ basketv2 g1 $P/basket-trimv2-c4df4d4/data
27
+ c6 g1 $P/c6-bathroom-door-fetch-v1-3afba69/data
28
+ b6v6 g1 $P/b6-pedal-bin-v6-ad2f5bf/data
29
+ a14v2 h2p $P/a14-unscrew-bulb-v2-8411a6d/data
30
+ b1 g1 $P/b1-riceball-microwave-v1-36da298/data
31
+ EOF
32
+ DIRS=""
33
+ while read t prof src lim; do
34
+ D=$F/data/$t
35
+ if [ ! -f $D/meta/hb_fastwam_interface.json ]; then
36
+ $E $F/tools/hb_fastwam_convert.py $prof $src $D ${lim:+--episodes $lim} || { echo "CONVERT_FAILED $t"; continue; }
37
+ fi
38
+ DIRS="$DIRS,\"$D\""
39
+ done < $F/tasks.txt
40
+ DIRS="[${DIRS#,}]"
41
+ echo "$(date) CONVERTED $DIRS"
42
+ cd $F/FastWAM
43
+ for prof in g1 h2 h2p; do
44
+ AI=$F/action-init-$prof.pt
45
+ [ -f $AI ] && continue
46
+ HB_FW_PROFILE=$prof HB_MODEL_YAML=$F/model-$prof.yaml $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" || { echo "YAML_FAILED $prof"; continue; }
47
+ grep -E 'action_dim|proprio_dim' $F/model-$prof.yaml | tr '\n' ' '; echo
48
+ $E scripts/preprocess_action_dit_backbone.py --model-config $F/model-$prof.yaml --output $AI --device cpu --dtype bfloat16 && echo "$(date) INIT_OK $prof $(stat -c %s $AI)" || echo "INIT_FAILED $prof"
49
+ done
50
+ mkdir -p $TC
51
+ $E scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 model.redirect_common_files=false "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" && echo "$(date) TEXT_OK $(ls $TC | wc -l) files" || echo TEXT_FAILED
52
+ ok=1
53
+ for pair in a10v3:g1 a17cr2:h2 a14v2:h2p; do
54
+ t=${pair%%:*}; prof=${pair##*:}
55
+ HB_FW_PROFILE=$prof HB_DATA_SAMPLE=1 HB_SAMPLE_WORKDIR=/tmp/hbfw-sample-$t $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=[\"$F/data/$t\"]" "data.train.text_embedding_cache_dir=$TC" "model.action_dit_pretrained_path=$F/action-init-$prof.pt" > $F/sample-$t.log 2>&1
56
+ grep -E 'HB_FW_AUG|HBFW_|Error' $F/sample-$t.log | cut -c1-800
57
+ grep -q "HBFW_SAMPLE" $F/sample-$t.log || { ok=0; echo "SAMPLE_FAILED $t"; }
58
+ done
59
+ for t in $(cut -d' ' -f1 $F/tasks.txt); do [ -f $F/data/$t/meta/hb_fastwam_interface.json ] || { ok=0; echo "MISSING_DATA $t"; }; done
60
+ for prof in g1 h2 h2p; do [ -f $F/action-init-$prof.pt ] || { ok=0; echo "MISSING_INIT $prof"; }; done
61
+ [ $ok = 1 ] && touch $F/bench.ready && echo "$(date) BENCH_READY" || echo "$(date) NOT_READY"
a18-pour-water/b512/step-004000/code/fastwam_task.sh ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Relay cmd item helper (2026-11-16 user): FastWAM on one benchmark task, 4 nodes x 8 GPUs, per-GPU batch from the
3
+ # benchmark ($G/fastwam/chosen.json: largest batch that fits), no gradient accumulation, 4000 steps (user default while
4
+ # the batch is unknown), native Gaussian FM recipe (lr 1e-4 cosine, ZeRO-1, bf16), HB augmentation v2 + proprio
5
+ # perturbation, our state/action interface. Checkpoints at 2000 and 4000. Skips (exit 3) unless the benchmark passed.
6
+ # fastwam_task.sh <task> <profile g1|h2|h2p> <port>
7
+ G=$(dirname "$(dirname "$(realpath "$0")")")
8
+ TASK=$1 PROF=$2 PORT=$3
9
+ F=$G/fastwam; E=$G/envs/fastwam/bin/python; R=$F/runs/$TASK
10
+ RK=${RANK:?}; M0=${MASTER_ADDR:?}
11
+ [ -f $F/fastwam.ok ] && [ -s $F/chosen.json ] || { echo "SKIP fastwam $TASK: benchmark not passed"; exit 3; }
12
+ [ -f $F/data/$TASK/meta/hb_fastwam_interface.json ] && [ -f $F/action-init-$PROF.pt ] || { echo "SKIP fastwam $TASK: data not prepared"; exit 3; }
13
+ BS=$(python3 -c "import json;print(json.load(open('$F/chosen.json'))['per_gpu_batch'])")
14
+ mkdir -p $R/logs
15
+ exec > >(tee -a "$R/logs/node-$RK.log") 2>&1
16
+ date -u; hostname
17
+ [ -f $R/complete.json ] && { echo "already complete"; exit 0; }
18
+ echo "FASTWAM task=$TASK profile=$PROF rank=$RK master=$M0:$PORT per_gpu=$BS global=$((BS * 32)) steps=4000"
19
+ if ! ldconfig -p | grep -q 'libavutil.so.56'; then
20
+ apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
21
+ libavcodec58 libavformat58 libavutil56 libswscale5 libswresample3 libavdevice58 libavfilter7 || true
22
+ fi
23
+ unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT
24
+ unset $(compgen -e | grep -E '^PET_') NPROC_PER_NODE WORLD_SIZE_WITH_STANDBY TORCHELASTIC_RUN_ID
25
+ export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true HF_HUB_OFFLINE=1 HB_AUG_DIR=$G/hb_groot HB_FW_PROFILE=$PROF
26
+ export NCCL_SOCKET_IFNAME=eth GLOO_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false WANDB_MODE=disabled
27
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
28
+ export PATH=$G/envs/fastwam/bin:$PATH # r2 (11-16 05:3x): r1 failed, accelerate called the missing deepspeed CLI (pdsh launcher)
29
+ for i in $(seq 36); do [ -z "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ] && break; sleep 5; done
30
+ cd $F/FastWAM
31
+ $E -m accelerate.commands.launch --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml --deepspeed_multinode_launcher standard \
32
+ --num_machines 4 --machine_rank $RK --main_process_ip $M0 --main_process_port $PORT --num_processes 32 \
33
+ $F/tools/hb_fastwam_train.py "output_dir=$R/out" "batch_size=$BS" max_steps=4000 log_every=20 save_every=2000 eval_every=0 \
34
+ num_workers=8 wandb.enabled=false gradient_accumulation_steps=1 "data.train.dataset_dirs=[\"$F/data/$TASK\"]" \
35
+ "data.train.text_embedding_cache_dir=$F/text-cache/all" "model.action_dit_pretrained_path=$F/action-init-$PROF.pt"
36
+ rc=$?
37
+ echo "$rc" > $R/logs/exit-node-$RK
38
+ if [ "$rc" = 0 ] && [ "$RK" = 0 ]; then
39
+ python3 - "$R/complete.json" "$TASK" "$PROF" "$BS" "$F/data/$TASK/meta/hb_fastwam_interface.json" <<'EOF'
40
+ import json, sys, time
41
+ out, task, prof, bs, iface = sys.argv[1:6]
42
+ d = dict(task=task, profile=prof, per_gpu_batch=int(bs), global_batch=int(bs) * 32, gradient_accumulation_steps=1,
43
+ max_steps=4000, save_every=2000, recipe="native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
44
+ interface=json.load(open(iface)), finished_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
45
+ json.dump(d, open(out, 'w'), indent=1); print('COMPLETE', d)
46
+ EOF
47
+ fi
48
+ echo "EXIT $rc"; exit $rc
a18-pour-water/b512/step-004000/code/hb_aug_v2.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark data augmentation v2 (project CLAUDE.md), model-agnostic torch implementation.
2
+
3
+ augment(x, rng, obs_mask=None) takes x: float [T, V, 3, H, W] in [0, 1] (T frames, V cameras) and returns the
4
+ augmented tensor. Each call is one training sample: one of the 15 effects is chosen uniformly and its magnitude
5
+ is scaled by 1.5 around identity; the rest stay at x1. Colour is shared across cameras and time; geometry, light
6
+ patches and white balance are drawn per camera and fixed over time; sensor/optics effects (noise, gaussian blur,
7
+ JPEG, motion blur, sharpen, defocus) apply only to observation frames (obs_mask True).
8
+ Pixel-size parameters are defined at 224 px and scaled by H / 224.
9
+ perturb_joints(q, rng) applies the proprio spec: per-joint bias +-2 deg plus N(0, 0.25 deg) clipped at 3 sigma.
10
+ """
11
+ import io
12
+ import math
13
+
14
+ import numpy as np
15
+ import torch
16
+ import torch.nn.functional as F
17
+ from PIL import Image
18
+ from torchvision.transforms import InterpolationMode
19
+ from torchvision.transforms import functional as TF
20
+
21
+ EFFECTS = ("exposure", "contrast", "saturation", "hue", "gamma", "temperature", "rotation_crop", "noise",
22
+ "gaussian_blur", "jpeg", "motion_blur", "sharpen", "defocus", "light_shadow", "wb_channel")
23
+ CENTER = {"exposure": 0.0, "contrast": 1.0, "saturation": 1.0, "hue": 0.0, "gamma": 1.0, "temperature": 0.0}
24
+ RANGE = {"exposure": (-0.22, 0.22), "contrast": (0.88, 1.12), "saturation": (0.76, 1.24), "hue": (-0.015, 0.015),
25
+ "gamma": (0.89, 1.11), "temperature": (-0.066, 0.066)}
26
+ VERSION = "hb-aug-v2-20261109"
27
+
28
+
29
+ def sample(rng, views, height):
30
+ boost = EFFECTS[int(rng.integers(len(EFFECTS)))]
31
+
32
+ def k(name):
33
+ return 1.5 if boost == name else 1.0
34
+
35
+ px = height / 224.0
36
+ p = {"boost": boost}
37
+ for name, (a, b) in RANGE.items():
38
+ c = CENTER[name]
39
+ p[name] = float(rng.uniform(c + k(name) * (a - c), c + k(name) * (b - c)))
40
+ p["geometry"] = []
41
+ for _ in range(views):
42
+ angle = float(rng.uniform(-1.5 * k("rotation_crop"), 1.5 * k("rotation_crop")))
43
+ p["geometry"].append((angle, 1.0 + abs(angle) * 0.015 + float(rng.uniform(0, 0.02 * k("rotation_crop")))))
44
+ p["noise"] = 4.0 / 255.0 * k("noise") if rng.random() < 0.5 else 0.0
45
+ p["gaussian_blur"] = float(rng.uniform(0.25, 0.65)) * k("gaussian_blur") if rng.random() < 0.15 else 0.0
46
+ if rng.random() < 0.35:
47
+ lo = int(round(100 - k("jpeg") * 30))
48
+ hi = int(round(100 - k("jpeg") * 5))
49
+ p["jpeg"] = int(rng.integers(lo, hi + 1))
50
+ else:
51
+ p["jpeg"] = None
52
+ p["motion_blur"] = ((float(rng.uniform(3, 7)) * k("motion_blur") * px, float(rng.uniform(0, 180)))
53
+ if rng.random() < 0.15 else None)
54
+ p["sharpen"] = float(rng.uniform(0.2, 0.6)) * k("sharpen") if rng.random() < 0.15 else 0.0
55
+ p["defocus"] = float(rng.uniform(1, 2)) * k("defocus") * px if rng.random() < 0.10 else 0.0
56
+ if rng.random() < 0.30:
57
+ g = 0.30 * k("light_shadow")
58
+ p["light_shadow"] = [[(float(rng.uniform(0.1, 0.9)), float(rng.uniform(0.1, 0.9)),
59
+ float(rng.uniform(0.15, 0.35)), float(rng.uniform(1 - g, 1 + g)))
60
+ for _ in range(int(rng.integers(1, 3)))] for _ in range(views)]
61
+ else:
62
+ p["light_shadow"] = None
63
+ if rng.random() < 0.30:
64
+ g = 0.10 * k("wb_channel")
65
+ gains = []
66
+ for _ in range(views):
67
+ row = [1.0, 1.0, 1.0]
68
+ row[int(rng.integers(0, 3))] = float(rng.uniform(1 - g, 1 + g))
69
+ gains.append(row)
70
+ p["wb_channel"] = gains
71
+ else:
72
+ p["wb_channel"] = None
73
+ return p
74
+
75
+
76
+ def _conv(x, kernel):
77
+ n, c, h, w = x.shape
78
+ k = kernel.to(x)[None, None].repeat(c, 1, 1, 1)
79
+ pad = kernel.shape[-1] // 2
80
+ return F.conv2d(F.pad(x, (pad, pad, pad, pad), mode="reflect"), k, groups=c)
81
+
82
+
83
+ def motion_kernel(length, angle_deg):
84
+ size = max(3, int(round(length)) | 1)
85
+ k = torch.zeros(size, size)
86
+ c = size // 2
87
+ th = math.radians(angle_deg)
88
+ for s in np.linspace(-(length - 1) / 2, (length - 1) / 2, 8 * size):
89
+ k[int(round(c - s * math.sin(th))), int(round(c + s * math.cos(th)))] = 1.0
90
+ return k / k.sum()
91
+
92
+
93
+ def disk_kernel(r):
94
+ size = 2 * int(math.ceil(r)) + 1
95
+ c = size // 2
96
+ yy, xx = torch.meshgrid(torch.arange(size), torch.arange(size), indexing="ij")
97
+ k = (((yy - c) ** 2 + (xx - c) ** 2) <= (r + 0.5) ** 2).float()
98
+ return k / k.sum()
99
+
100
+
101
+ def apply(x, p, obs_mask=None, noise_seed=0):
102
+ t, v, c, h, w = x.shape
103
+ x = x.clone().clamp(0, 1)
104
+ for vi, (angle, scale) in enumerate(p["geometry"]):
105
+ x[:, vi] = TF.affine(x[:, vi], angle=angle, translate=[0, 0], scale=scale, shear=[0, 0],
106
+ interpolation=InterpolationMode.BILINEAR)
107
+ if p["light_shadow"] is not None:
108
+ yy, xx = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij")
109
+ for vi, blobs in enumerate(p["light_shadow"]):
110
+ m = torch.ones(h, w)
111
+ for cx, cy, s, g in blobs:
112
+ m = m + (g - 1.0) * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * s * s))
113
+ x[:, vi] = (x[:, vi] * m.clamp(min=0).to(x)).clamp(0, 1)
114
+ if p["wb_channel"] is not None:
115
+ for vi, g in enumerate(p["wb_channel"]):
116
+ x[:, vi] = (x[:, vi] * torch.tensor(g, dtype=x.dtype).view(1, 3, 1, 1)).clamp(0, 1)
117
+ y = x.reshape(t * v, c, h, w)
118
+ y = (y * float(2.0 ** p["exposure"])).clamp(0, 1)
119
+ d = p["temperature"]
120
+ y = (y * y.new_tensor([1.0 + d, 1.0, 1.0 - d]).view(1, 3, 1, 1)).clamp(0, 1)
121
+ y = ((y - 0.5) * p["contrast"] + 0.5).clamp(0, 1)
122
+ y = TF.adjust_saturation(y, p["saturation"])
123
+ y = TF.adjust_hue(y, p["hue"])
124
+ y = y.clamp(0, 1).pow(p["gamma"])
125
+ x = y.reshape(t, v, c, h, w)
126
+ mask = torch.ones(t, dtype=torch.bool) if obs_mask is None else torch.as_tensor(obs_mask, dtype=torch.bool)
127
+ if bool(mask.any()):
128
+ o = x[mask].reshape(-1, c, h, w)
129
+ if p["gaussian_blur"] > 0:
130
+ o = TF.gaussian_blur(o, [3, 3], [p["gaussian_blur"]] * 2)
131
+ if p["motion_blur"] is not None:
132
+ o = _conv(o, motion_kernel(*p["motion_blur"]))
133
+ if p["defocus"] > 0:
134
+ o = _conv(o, disk_kernel(p["defocus"]))
135
+ if p["sharpen"] > 0:
136
+ o = o + p["sharpen"] * (o - TF.gaussian_blur(o, [5, 5], [1.0, 1.0]))
137
+ o = o.clamp(0, 1)
138
+ if p["noise"] > 0:
139
+ gen = torch.Generator().manual_seed(int(noise_seed))
140
+ o = (o + torch.randn(o.shape, generator=gen, dtype=o.dtype) * p["noise"]).clamp(0, 1)
141
+ if p["jpeg"] is not None:
142
+ for i in range(o.shape[0]):
143
+ buf = io.BytesIO()
144
+ TF.to_pil_image(o[i]).save(buf, format="JPEG", quality=int(p["jpeg"]))
145
+ buf.seek(0)
146
+ with Image.open(buf) as im:
147
+ o[i] = TF.pil_to_tensor(im.convert("RGB")).to(o.dtype).div(255.0)
148
+ x[mask] = o.reshape(-1, v, c, h, w)
149
+ return x.clamp(0, 1)
150
+
151
+
152
+ def augment(x, rng, obs_mask=None):
153
+ p = sample(rng, x.shape[1], x.shape[-2])
154
+ return apply(x, p, obs_mask, noise_seed=int(rng.integers(0, 2 ** 62))), p
155
+
156
+
157
+ def perturb_joints(q, rng):
158
+ q = np.asarray(q, dtype=np.float64)
159
+ d = q.shape[-1]
160
+ bias = rng.uniform(-math.radians(2.0), math.radians(2.0), size=d)
161
+ sigma = math.radians(0.25)
162
+ noise = np.clip(rng.normal(0.0, sigma, size=q.shape), -3 * sigma, 3 * sigma)
163
+ return (q + bias + noise).astype(np.float32)
a18-pour-water/b512/step-004000/code/hb_eval_common.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Humanoid Benchmark open-loop action-reconstruction evaluation: shared task table, cache layout and metrics.
2
+
3
+ Protocol (identical for every model; mirrors Isaac-GR00T gr00t/eval/open_loop_eval.py):
4
+ * query frames t = 0, 16, 32, ... over the whole episode (non-overlapping 16-step chunks);
5
+ * the model sees the observation at t exactly as at deployment (cameras, raw benchmark state 44/78, instruction;
6
+ Delta Flow also its native 16-step past-action history), all augmentation off;
7
+ * the predicted chunk (first 16 steps, raw action units) is compared with the recorded actions t..t+15;
8
+ rows beyond the episode end are excluded (no clamped / padded targets);
9
+ * metrics: raw MSE / MAE, and nMSE = MSE after dividing every action dim by its std over the task's TRAINING
10
+ frames (floored, see STD_FLOOR_FRAC), overall, per action group and per horizon step.
11
+ Splits: "heldout" = recorded episodes that were never trained on (B6 100-148, trolley 100-102, B7 100-106);
12
+ "train10" = 10 fixed, evenly spaced training episodes (fit on seen data, NOT generalisation).
13
+ Only numpy here: imported by every model environment.
14
+ """
15
+ import json
16
+ import os
17
+ import time
18
+ from pathlib import Path
19
+
20
+ import numpy as np
21
+
22
+ G = Path(os.environ.get("G", "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/"
23
+ "preflight-20261104T1648Z/policies-20261109"))
24
+ P = G.parent
25
+ ROOT = G / "eval-action-mse"
26
+ CACHE = Path(os.environ.get("HB_EVAL_CACHE", ROOT / "cache"))
27
+ HORIZON = 16
28
+ STRIDE = 16
29
+ N_TRAIN_EVAL = 10
30
+ STD_FLOOR_FRAC = 0.05 # per-dim std floor = 5 % of the median std of the action vector
31
+
32
+ # task -> (source LeRobot v2.1 root, number of TRAINING episodes = fixed first N (None = all recorded), layout)
33
+ TASKS = {
34
+ "b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100, "g1"),
35
+ "basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None, "g1"),
36
+ "trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100, "g1"),
37
+ "b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100, "g1"),
38
+ "b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None, "g1"),
39
+ "a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None, "g1"),
40
+ "a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None, "g1"),
41
+ "a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None, "g1"),
42
+ "c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None, "g1"),
43
+ "a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None, "g1"),
44
+ "a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None, "g1"),
45
+ "a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None, "h2"),
46
+ "a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None, "h2"),
47
+ "a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None, "h2"),
48
+ # 2026-11-15: newer datasets (GR00T in-training eval needs a cache for every task it trains on)
49
+ "a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None, "g1"),
50
+ "a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None, "g1"),
51
+ "b1": (P / "b1-riceball-microwave-v1-36da298/data", None, "g1"),
52
+ "b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None, "g1"),
53
+ "b8": (P / "b8-table-wipe-v1-3b2e618/data", None, "g1"),
54
+ "basketv2": (P / "basket-trimv2-c4df4d4/data", None, "g1"),
55
+ "a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None, "h2p"), # action 82 = hb75 + SONIC planner 7
56
+ # 2026-11-15 evening: v6/v4 datasets (a3v3, a7v3 retired, not registered)
57
+ "b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None, "g1"), # 542 ep, all used (B6 first-100 rule is v5t7 only)
58
+ "b5": (P / "b5-chair-push-v1-b85b7e9/data", None, "g1"),
59
+ "a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None, "g1"),
60
+ "a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None, "g1"),
61
+ # 2026-11-16: B5 standing start (HF b5-chair-push-standing-start-expert-v1 @913f5a16, 100 ep; FastWAM b5ss)
62
+ "b5ss": (P / "b5-chair-push-standing-start-v1-913f5a1/data", None, "g1"),
63
+ # 2026-11-15 night: H2 cropped v2 (GR00T h32 relay item 71); head+chest only -> build_cache --cams head,chest
64
+ "a18cr2": (P / "a18-pour-water-cropped-v2-44cdd57/data", None, "h2"), # HF a18-pour-water-expert-cropped-v2 @44cdd571
65
+ "a17cr2": (P / "a17-unscrew-bottle-cropped-v2-3520b70/data", None, "h2"), # HF a17-unscrew-bottle-expert-cropped-v2 @3520b70f
66
+ }
67
+
68
+
69
+ def task_for_source(source):
70
+ """benchmark task whose recording root is <source> (groot-data/<view>/hb-dataset-receipt.json "source")"""
71
+ src = Path(source).resolve()
72
+ hits = [t for t, (root, _, _) in TASKS.items() if Path(root).resolve() == src]
73
+ return hits[0] if hits else None
74
+ G1_BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint",
75
+ "left_ankle_pitch_joint", "left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint",
76
+ "right_hip_yaw_joint", "right_knee_joint", "right_ankle_pitch_joint", "right_ankle_roll_joint",
77
+ "waist_yaw_joint", "waist_roll_joint", "waist_pitch_joint", "left_shoulder_pitch_joint",
78
+ "left_shoulder_roll_joint", "left_shoulder_yaw_joint", "left_elbow_joint", "left_wrist_roll_joint",
79
+ "left_wrist_pitch_joint", "left_wrist_yaw_joint", "right_shoulder_pitch_joint",
80
+ "right_shoulder_roll_joint", "right_shoulder_yaw_joint", "right_elbow_joint", "right_wrist_roll_joint",
81
+ "right_wrist_pitch_joint", "right_wrist_yaw_joint"]
82
+ LAYOUTS = {
83
+ # state: body29 | inspire L6 | inspire R6 | projected_gravity3 ; action: motion_token64 | inspire L6 | R6 (same row)
84
+ "g1": dict(state_dim=44, action_dim=76, cams=("ego_view", "chest"),
85
+ groups={"motion_token": (0, 64), "hands": (64, 76)}),
86
+ # state: observation.state75 (legs12 waist3 arms14 head2 hands22+22) | gravity3 ; action: body_target31 | hands44
87
+ "h2": dict(state_dim=78, action_dim=75, cams=("head", "chest", "wrist"),
88
+ groups={"body_target": (0, 31), "hands": (31, 75)}),
89
+ # 2026-11-15 user: A14 v2 is trained on action82 = body_target31 | hands44 | action.planner7 (SONIC planner
90
+ # command); all 82 dims are scored (planner reported as its own group)
91
+ "h2p": dict(state_dim=78, action_dim=82, cams=("head", "chest"),
92
+ groups={"body_target": (0, 31), "hands": (31, 75), "planner": (75, 82)}),
93
+ }
94
+
95
+
96
+ def jl(path):
97
+ return [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
98
+
99
+
100
+ def layout(task):
101
+ return LAYOUTS[TASKS[task][2]]
102
+
103
+
104
+ def split_episodes(task, split, n_recorded):
105
+ _, n_train, _ = TASKS[task]
106
+ n_train = n_recorded if n_train is None else n_train
107
+ if split == "heldout":
108
+ return list(range(n_train, n_recorded))
109
+ if split == "train10":
110
+ return sorted({int(round(x)) for x in np.linspace(0, n_train - 1, N_TRAIN_EVAL)})
111
+ if split.startswith("ep:"): # explicit list, e.g. ep:100,101
112
+ return [int(x) for x in split[3:].split(",")]
113
+ raise ValueError(f"unknown split {split!r}")
114
+
115
+
116
+ def split_dir(task, split):
117
+ return CACHE / task / split
118
+
119
+
120
+ def load_meta(task, split):
121
+ return json.loads((split_dir(task, split) / "meta.json").read_text())
122
+
123
+
124
+ def load_stats(task):
125
+ z = np.load(CACHE / task / "stats.npz")
126
+ return {k: z[k] for k in z.files}
127
+
128
+
129
+ def norm_std(stats):
130
+ std = stats["action_std"].astype(np.float64)
131
+ return np.maximum(std, STD_FLOOR_FRAC * float(np.median(std)))
132
+
133
+
134
+ def iter_episodes(task, split, max_episodes=None):
135
+ meta = load_meta(task, split)
136
+ eps = meta["episodes"][: max_episodes or None]
137
+ for ep in eps:
138
+ with np.load(split_dir(task, split) / f"ep_{ep:06d}.npz") as z:
139
+ yield ep, {k: z[k] for k in z.files}
140
+
141
+
142
+ class Accumulator:
143
+ """Sums of squared / absolute errors; everything is computed in float64 over valid (row, dim) entries."""
144
+
145
+ def __init__(self, std, groups, horizon=HORIZON):
146
+ self.std = np.asarray(std, dtype=np.float64)
147
+ self.groups = groups
148
+ D = len(self.std)
149
+ self.sse_raw = np.zeros(D)
150
+ self.sae_raw = np.zeros(D)
151
+ self.sse_n = np.zeros(D)
152
+ self.rows = 0
153
+ self.step_sse_n = np.zeros(horizon)
154
+ self.step_rows = np.zeros(horizon)
155
+ self.per_episode = {}
156
+
157
+ def add(self, ep, pred, gt, valid):
158
+ pred = np.asarray(pred, dtype=np.float64)
159
+ gt = np.asarray(gt, dtype=np.float64)
160
+ v = np.asarray(valid, dtype=bool)
161
+ if pred.shape != gt.shape:
162
+ raise ValueError(f"pred {pred.shape} vs gt {gt.shape}")
163
+ if not np.isfinite(pred[v]).all():
164
+ raise FloatingPointError("non-finite prediction")
165
+ e = (pred - gt)[v]
166
+ en = e / self.std
167
+ self.sse_raw += (e ** 2).sum(0)
168
+ self.sae_raw += np.abs(e).sum(0)
169
+ self.sse_n += (en ** 2).sum(0)
170
+ self.rows += int(v.sum())
171
+ k = np.nonzero(v)[0]
172
+ self.step_sse_n[k] += (en ** 2).sum(1)
173
+ self.step_rows[k] += 1
174
+ s = self.per_episode.setdefault(int(ep), [0.0, 0])
175
+ s[0] += float((en ** 2).sum())
176
+ s[1] += int(v.sum()) * len(self.std)
177
+
178
+ def result(self):
179
+ D = len(self.std)
180
+ n = max(self.rows, 1)
181
+ out = {"rows": self.rows, "mse_raw": float(self.sse_raw.sum() / (n * D)),
182
+ "mae_raw": float(self.sae_raw.sum() / (n * D)), "nmse": float(self.sse_n.sum() / (n * D)),
183
+ "groups": {}, "nmse_by_step": (self.step_sse_n / np.maximum(self.step_rows, 1) / D).tolist(),
184
+ "nmse_by_dim": (self.sse_n / n).tolist()}
185
+ for g, (a, b) in self.groups.items():
186
+ out["groups"][g] = {"mse_raw": float(self.sse_raw[a:b].sum() / (n * (b - a))),
187
+ "nmse": float(self.sse_n[a:b].sum() / (n * (b - a)))}
188
+ pe = {ep: s / max(c, 1) for ep, (s, c) in sorted(self.per_episode.items())}
189
+ out["nmse_per_episode"] = pe
190
+ vals = np.array(list(pe.values()))
191
+ if len(vals) > 1: # bootstrap-free spread: std error over episodes
192
+ out["nmse_episode_sem"] = float(vals.std(ddof=1) / np.sqrt(len(vals)))
193
+ return out
194
+
195
+
196
+ def queries(T):
197
+ return list(range(0, T, STRIDE))
198
+
199
+
200
+ def gt_chunk(action, t, horizon=HORIZON):
201
+ T = len(action)
202
+ idx = np.arange(t, t + horizon)
203
+ valid = idx < T
204
+ return action[np.minimum(idx, T - 1)], valid
205
+
206
+
207
+ def run(adapter, task, split, out_path, max_episodes=None, max_queries=None, save_pred=True, log_every=50):
208
+ """Evaluate one adapter on one cached split; writes <out_path> (json) and <out_path>.pred.npz."""
209
+ meta = load_meta(task, split)
210
+ lay = LAYOUTS[meta["layout"]]
211
+ stats = load_stats(task)
212
+ acc = Accumulator(norm_std(stats), lay["groups"])
213
+ preds, keys = [], []
214
+ t0 = time.time()
215
+ infer_s = 0.0
216
+ nq = 0
217
+ for ep, z in iter_episodes(task, split, max_episodes):
218
+ state, action = z["state"], z["action"]
219
+ qts = list(z["query_t"])
220
+ for qi, t in enumerate(qts):
221
+ obs = {"episode": ep, "t": int(t), "T": len(action), "state": state[t], "prompt": meta["prompts"][str(int(z["task_index"]))],
222
+ "images": {c: z[f"img_{c}"][qi] for c in meta["cams"]},
223
+ "past_action": action[max(0, t - HORIZON):t],
224
+ "_gt_check": action[t:t + HORIZON]} # only for adapters' input-pipeline consistency checks
225
+ s = time.time()
226
+ pred = np.asarray(adapter.predict(obs), dtype=np.float64)
227
+ infer_s += time.time() - s
228
+ if pred.ndim != 2 or pred.shape[0] < HORIZON or pred.shape[1] != lay["action_dim"]:
229
+ raise ValueError(f"adapter returned {pred.shape}, need (>={HORIZON}, {lay['action_dim']})")
230
+ pred = pred[:HORIZON]
231
+ gt, valid = gt_chunk(action, t)
232
+ acc.add(ep, pred, gt, valid)
233
+ if save_pred:
234
+ preds.append(pred.astype(np.float32))
235
+ keys.append((ep, int(t)))
236
+ nq += 1
237
+ if log_every and nq % log_every == 0:
238
+ print(f"[eval] {task}/{split} {nq} queries, running nMSE {acc.result()['nmse']:.4f}, "
239
+ f"{infer_s / nq:.3f} s/query", flush=True)
240
+ if max_queries and nq >= max_queries:
241
+ break
242
+ if max_queries and nq >= max_queries:
243
+ break
244
+ res = acc.result()
245
+ res.update(task=task, split=split, layout=meta["layout"], episodes=sorted(acc.per_episode), queries=nq,
246
+ horizon=HORIZON, stride=STRIDE, std_floor_frac=STD_FLOOR_FRAC, wall_s=round(time.time() - t0, 1),
247
+ infer_s_per_query=round(infer_s / max(nq, 1), 4), adapter=adapter.info(),
248
+ baselines=meta.get("baselines", {}), cache_created=meta.get("created"),
249
+ partial=bool(max_episodes or max_queries), finished_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
250
+ out_path = Path(out_path)
251
+ out_path.parent.mkdir(parents=True, exist_ok=True)
252
+ tmp = out_path.with_suffix(".tmp")
253
+ tmp.write_text(json.dumps(res, indent=1))
254
+ tmp.replace(out_path)
255
+ if save_pred and preds:
256
+ np.savez_compressed(str(out_path) + ".pred.npz", pred=np.stack(preds).astype(np.float16),
257
+ keys=np.asarray(keys, dtype=np.int64))
258
+ return res
a18-pour-water/b512/step-004000/code/hb_fastwam_aug.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: Humanoid Benchmark augmentation v2 (project CLAUDE.md) inside the native FastWAM data pipeline.
2
+ - Visual: hb_aug_v2.augment on every training sample's frames before the two cameras are concatenated; frame 0 is the
3
+ current observation (sensor / optics effects only there), the later frames are the future video targets (colour,
4
+ per-camera geometry, light patches and white balance shared with frame 0, as the spec requires).
5
+ - Proprio: hb_aug_v2.perturb_joints on the raw body joints (G1 29, H2 31 = body 29 + head 2) before normalization;
6
+ hands and gravity untouched; supervision labels untouched. Normalization statistics are computed on the raw data.
7
+ FastWAM's own image augmentation is only a resize (no colour jitter / crop), so nothing native is stacked.
8
+ Installed in the main process before the dataloaders fork; HB_FW_JOINTS sets the number of perturbed joints."""
9
+ import os
10
+ import sys
11
+ import time
12
+
13
+ import numpy as np
14
+ import torch
15
+
16
+ sys.path.insert(0, os.environ.get("HB_AUG_DIR", ""))
17
+ import hb_aug_v2 as A # noqa: E402
18
+
19
+ NJ = int(os.environ.get("HB_FW_JOINTS", "29"))
20
+ _seen = {"img": False, "state": False}
21
+
22
+
23
+ def _rng(idx=0):
24
+ return np.random.default_rng([os.getpid(), int(idx) & 0xFFFFFFFF, time.time_ns() & 0xFFFFFFFF])
25
+
26
+
27
+ def _train(proc):
28
+ v = getattr(proc, "is_train", False)
29
+ return bool(v() if callable(v) else v)
30
+
31
+
32
+ def install():
33
+ from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset
34
+ from fastwam.datasets.lerobot.processors.fastwam_processor import FastWAMProcessor
35
+
36
+ pre0 = FastWAMProcessor.preprocess
37
+
38
+ def preprocess(self, data):
39
+ if _train(self) and isinstance(data.get("state"), dict) and "default" in data["state"]:
40
+ st = data["state"]["default"]
41
+ q = st[..., :NJ].detach().cpu().numpy()
42
+ st = st.clone()
43
+ st[..., :NJ] = torch.from_numpy(A.perturb_joints(q, _rng())).to(st.dtype)
44
+ data["state"]["default"] = st
45
+ if not _seen["state"]:
46
+ _seen["state"] = True
47
+ print(f"HB_FW_AUG proprio perturb_joints on state[..., :{NJ}] shape {tuple(st.shape)}", flush=True)
48
+ return pre0(self, data)
49
+
50
+ get0 = BaseLerobotDataset.__getitem__
51
+
52
+ def getitem(self, idx):
53
+ s = get0(self, idx)
54
+ if self.processor is not None and _train(self.processor) and "pixel_values" in s:
55
+ pv = s["pixel_values"]
56
+ five = pv.ndim == 5
57
+ x = (pv if five else pv[None]).permute(1, 0, 2, 3, 4).float() # [T, V, C, H, W]
58
+ hi = float(x.max())
59
+ scale = 255.0 if hi > 1.5 else 1.0
60
+ mask = torch.zeros(x.shape[0], dtype=torch.bool)
61
+ mask[0] = True
62
+ y, p = A.augment((x / scale).clamp(0, 1), _rng(idx), mask)
63
+ y = (y * scale).permute(1, 0, 2, 3, 4)
64
+ s["pixel_values"] = (y if five else y[0]).to(pv.dtype)
65
+ if not _seen["img"]:
66
+ _seen["img"] = True
67
+ print(f"HB_FW_AUG visual {A.VERSION} on pixel_values {tuple(pv.shape)} dtype {pv.dtype} max {hi:.3f} "
68
+ f"(T={x.shape[0]}, obs frame 0) boost={p['boost']}", flush=True)
69
+ return s
70
+
71
+ FastWAMProcessor.preprocess = preprocess
72
+ BaseLerobotDataset.__getitem__ = getitem
73
+ print(f"HB_FW_AUG installed: {A.VERSION}, joints {NJ}", flush=True)
a18-pour-water/b512/step-004000/code/hb_fastwam_convert.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: LeRobot v2.1 copy of one of our datasets for FastWAM on OUR benchmark interface (CLAUDE.md section 4):
2
+ g1 : state44 = observation.state[0:29] (29 body joints) + measured Inspire 6 + 6 + projected gravity 3;
3
+ action76 = action.motion_token 64 + measured Inspire 6 + 6 (same-row labels, as GR00T / Psi-Zero / Delta Flow)
4
+ h2 : state78 = observation.state[0:75] + projected gravity 3; action75 = action.body_target 31 + action.hands 44
5
+ h2p: as h2 with action82 = action75 + action.planner 7 (A14 v2)
6
+ HumanWAM's fastwam_sonic_data.py expects a 43-D G1 state and takes state[0:22] + state[29:36]; on our 41-D state that
7
+ would put left-hand values into the right arm, so it is not used. Source files are never modified; media are symlinked.
8
+ --episodes N keeps the first N episodes (B7: first 100 of 107, as for every other model).
9
+ Usage: hb_fastwam_convert.py <profile> <source data dir> <fresh output dir> [--episodes N]"""
10
+ import copy, json, shutil, sys, tempfile
11
+ from pathlib import Path
12
+ import numpy as np
13
+ import pyarrow as pa
14
+ import pyarrow.parquet as pq
15
+
16
+ G1 = {"observation.state": 41, "action.motion_token": 64, "observation.inspire_left_hand_state": 6,
17
+ "observation.inspire_right_hand_state": 6, "observation.projected_gravity": 3}
18
+ H2 = {"observation.state": 75, "observation.projected_gravity": 3, "action.body_target": 31, "action.hands": 44}
19
+ H2P = dict(H2, **{"action.planner": 7})
20
+ SPEC = {"g1": (G1, ("ego_view", "chest")), "h2": (H2, ("head", "chest")), "h2p": (H2P, ("head", "chest"))}
21
+
22
+
23
+ def stats(v):
24
+ v = np.asarray(v, dtype=np.float64)
25
+ return {"min": v.min(0).tolist(), "max": v.max(0).tolist(), "mean": v.mean(0).tolist(), "std": v.std(0).tolist(), "count": [len(v)]}
26
+
27
+
28
+ def vec(table, key, values):
29
+ arr = pa.array(values.tolist(), type=pa.list_(pa.float32(), values.shape[1]))
30
+ i = table.schema.get_field_index(key)
31
+ return table.append_column(key, arr) if i < 0 else table.set_column(i, key, arr)
32
+
33
+
34
+ def map_rows(profile, c):
35
+ if profile == "g1":
36
+ L, R = c["observation.inspire_left_hand_state"], c["observation.inspire_right_hand_state"]
37
+ dev = float(np.abs(c["observation.state"][:, 29:41] - np.concatenate([L, R], 1)).max())
38
+ assert dev < 1e-4, f"observation.state[29:41] differs from the Inspire columns by {dev}"
39
+ return (np.concatenate([c["action.motion_token"], L, R], 1),
40
+ np.concatenate([c["observation.state"][:, :29], L, R, c["observation.projected_gravity"]], 1))
41
+ a = [c["action.body_target"], c["action.hands"]] + ([c["action.planner"]] if profile == "h2p" else [])
42
+ return np.concatenate(a, 1), np.concatenate([c["observation.state"], c["observation.projected_gravity"]], 1)
43
+
44
+
45
+ def main(profile, src, out, limit=None):
46
+ raw, cams = SPEC[profile]
47
+ src, out = Path(src).resolve(strict=True), Path(out).resolve()
48
+ if out.exists():
49
+ raise SystemExit(f"{out} exists")
50
+ info = json.loads((src / "meta/info.json").read_text())
51
+ assert info.get("codebase_version") == "v2.1", info.get("codebase_version")
52
+ for k, w in raw.items():
53
+ assert info["features"][k]["shape"] == [w], (k, info["features"][k]["shape"])
54
+ for c in cams:
55
+ assert info["features"]["observation.images." + c]["dtype"] in ("video", "image"), c
56
+ eps = [json.loads(l) for l in (src / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
57
+ assert [e["episode_index"] for e in eps] == list(range(len(eps)))
58
+ if limit is not None:
59
+ eps = eps[:limit]
60
+ old = {r["episode_index"]: r for r in (json.loads(l) for l in (src / "meta/episodes_stats.jsonl").read_text().splitlines() if l.strip())}
61
+ out.parent.mkdir(parents=True, exist_ok=True)
62
+ stg = Path(tempfile.mkdtemp(prefix=".hbfw-", dir=out.parent))
63
+ shutil.copytree(src / "meta", stg / "meta")
64
+ for n in ("videos", "images"):
65
+ if (src / n).exists():
66
+ (stg / n).symlink_to(src / n, target_is_directory=True)
67
+ new, total = [], 0
68
+ for e in eps:
69
+ i = e["episode_index"]
70
+ rel = Path(info["data_path"].format(episode_chunk=i // info["chunks_size"], episode_index=i))
71
+ t = pq.read_table(src / rel)
72
+ c = {k: np.asarray(t[k].to_pylist(), dtype=np.float32) for k in raw}
73
+ act, st = map_rows(profile, c)
74
+ assert len(act) == e["length"] and np.isfinite(act).all() and np.isfinite(st).all()
75
+ total += len(act)
76
+ t = vec(vec(t, "action", act), "observation.state", st)
77
+ md = dict(t.schema.metadata or {})
78
+ if b"huggingface" in md:
79
+ hf = json.loads(md[b"huggingface"])
80
+ for k, w in (("action", act.shape[1]), ("observation.state", st.shape[1])):
81
+ hf["info"]["features"][k] = {"feature": {"dtype": "float32", "_type": "Value"}, "length": w, "_type": "Sequence"}
82
+ md[b"huggingface"] = json.dumps(hf).encode()
83
+ t = t.replace_schema_metadata(md)
84
+ (stg / rel).parent.mkdir(parents=True, exist_ok=True)
85
+ pq.write_table(t, stg / rel)
86
+ r = copy.deepcopy(old[i])
87
+ r["stats"]["action"], r["stats"]["observation.state"] = stats(act), stats(st)
88
+ new.append(r)
89
+ if limit is None:
90
+ assert total == info["total_frames"], (total, info["total_frames"])
91
+ else:
92
+ info["total_episodes"], info["total_frames"] = len(eps), total
93
+ info["total_videos"] = len(eps) * len(cams) if "total_videos" in info else info.get("total_videos")
94
+ info["splits"] = {"train": f"0:{len(eps)}"}
95
+ (stg / "meta/episodes.jsonl").write_text("".join(json.dumps(e) + "\n" for e in eps))
96
+ info["features"]["action"] = {"dtype": "float32", "shape": [int(act.shape[1])], "names": None}
97
+ info["features"]["observation.state"] = {"dtype": "float32", "shape": [int(st.shape[1])], "names": None}
98
+ drop = [k for k, v in info["features"].items() if v["dtype"] in ("video", "image") and k not in ["observation.images." + c for c in cams]]
99
+ for k in drop: # 2-camera rule: unused cameras are not downloaded, FastWAM checks every listed video file
100
+ del info["features"][k]
101
+ for r in new:
102
+ r["stats"].pop(k, None)
103
+ if drop and info.get("total_videos"):
104
+ info["total_videos"] = len(eps) * len(cams)
105
+ (stg / "meta/info.json").write_text(json.dumps(info, indent=2) + "\n")
106
+ (stg / "meta/episodes_stats.jsonl").write_text("".join(json.dumps(r) + "\n" for r in new))
107
+ for s in (stg / "meta").glob("*stats*.json"):
108
+ s.unlink()
109
+ (stg / "meta/hb_fastwam_interface.json").write_text(json.dumps(dict(
110
+ profile=profile, source=str(src), cameras=list(cams), state_dim=int(st.shape[1]), action_dim=int(act.shape[1]),
111
+ episodes=len(eps), frames=total, fps=info["fps"], first_episodes_only=limit), indent=2) + "\n")
112
+ stg.rename(out)
113
+ print("HBFW_CONVERT_OK", profile, out, "episodes", len(eps), "frames", total, "state", st.shape[1], "action", act.shape[1], flush=True)
114
+
115
+
116
+ if __name__ == "__main__":
117
+ a = sys.argv[1:]
118
+ lim = None
119
+ if "--episodes" in a:
120
+ j = a.index("--episodes"); lim = int(a[j + 1]); del a[j:j + 2]
121
+ main(a[0], a[1], a[2], lim)
a18-pour-water/b512/step-004000/code/hb_fastwam_train.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """2026-11-16: native FastWAM Gaussian-FM training on our benchmark interface, with the HB augmentation v2.
2
+ Profiles (HB_FW_PROFILE): g1 = state44 / action76 / cameras ego_view + chest (HumanWAM fastwam_sonic recipe);
3
+ h2 = state78 / action75 / head + chest; h2p = state78 / action82 (+ planner 7) / head + chest. Fixed recipe of HumanWAM
4
+ fastwam_sonic_training.compose_sonic_config: 17 raw samples = 16 actions + 5 video frames (stride 4), 224 x 448 side by
5
+ side, min/max statistics, absolute labels, action scheduler shift 1.0, original Wan files.
6
+ Arguments: key=value Hydra overrides (JSON values parsed). Env: FASTWAM_SRC, DIFFSYNTH_MODEL_BASE_PATH, HB_AUG_DIR.
7
+ HB_MODEL_YAML=<path> writes the resolved model config; HB_COMPOSE_ONLY=1 prints the config; HB_DATA_SAMPLE=1 reads one
8
+ augmented training sample on CPU; HB_FW_AUG=0 disables the augmentation (benchmark only)."""
9
+ import json, os, sys
10
+ from pathlib import Path
11
+
12
+ ROOT = Path(os.environ["FASTWAM_SRC"]).resolve()
13
+ sys.path.insert(0, str(ROOT / "src"))
14
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
15
+ TASK = "libero_uncond_2cam224_1e-4"
16
+ PROFILES = {"g1": (("ego_view", "chest"), 44, 76, 29), "h2": (("head", "chest"), 78, 75, 31),
17
+ "h2p": (("head", "chest"), 78, 82, 31)}
18
+ CAMS, SDIM, ADIM, NJ = PROFILES[os.environ.get("HB_FW_PROFILE", "g1")]
19
+ os.environ.setdefault("HB_FW_JOINTS", str(NJ))
20
+ SHAPE_META = {
21
+ "images": [{"key": n, "raw_shape": [3, 224, 224], "shape": [3, 224, 224]} for n in CAMS],
22
+ "action": [{"key": "default", "raw_shape": ADIM, "shape": ADIM}],
23
+ "state": [{"key": "default", "raw_shape": SDIM, "shape": SDIM}],
24
+ }
25
+ FIXED = {
26
+ "data.train.shape_meta": SHAPE_META, "data.train.num_frames": 17, "data.train.global_sample_stride": 1,
27
+ "data.train.action_video_freq_ratio": 4, "data.train.video_size": [224, 448], "data.train.concat_multi_camera": "horizontal",
28
+ "data.train.processor.action_output_dim": ADIM, "data.train.processor.proprio_output_dim": SDIM,
29
+ "data.train.processor.delta_action_dim_mask": {"default": [False] * ADIM}, "data.train.processor.action_state_transforms": None,
30
+ "data.train.processor.use_stepwise_action_norm": False, "data.train.processor.norm_default_mode": "min/max",
31
+ "data.train.processor.norm_exception_mode": None, "model.redirect_common_files": False,
32
+ "model.action_scheduler.train_shift": 1.0, "model.action_scheduler.infer_shift": 1.0,
33
+ }
34
+
35
+
36
+ def build(overrides):
37
+ from hydra import compose, initialize_config_dir
38
+ from omegaconf import OmegaConf
39
+ from fastwam.utils.config_resolvers import register_default_resolvers
40
+ register_default_resolvers()
41
+ with initialize_config_dir(config_dir=str(ROOT / "configs"), version_base="1.3"):
42
+ cfg = compose(config_name="train", overrides=["task=" + TASK])
43
+ for k, v in {**FIXED, **overrides}.items():
44
+ OmegaConf.update(cfg, k, v, merge=False)
45
+ for k, v in FIXED.items():
46
+ a = OmegaConf.select(cfg, k)
47
+ if OmegaConf.is_config(a):
48
+ a = OmegaConf.to_container(a, resolve=True)
49
+ assert a == v, (k, a, v)
50
+ return cfg
51
+
52
+
53
+ def parse(argv):
54
+ out = {}
55
+ for a in argv:
56
+ k, v = a.split("=", 1)
57
+ try:
58
+ out[k] = json.loads(v)
59
+ except ValueError:
60
+ out[k] = v
61
+ return out
62
+
63
+
64
+ if __name__ == "__main__":
65
+ from omegaconf import OmegaConf
66
+ cfg = build(parse(sys.argv[1:]))
67
+ if os.environ.get("HB_MODEL_YAML"):
68
+ Path(os.environ["HB_MODEL_YAML"]).write_text(OmegaConf.to_yaml(cfg.model, resolve=True))
69
+ print("HBFW_MODEL_YAML", os.environ["HB_MODEL_YAML"], flush=True)
70
+ raise SystemExit(0)
71
+ if os.environ.get("HB_COMPOSE_ONLY"):
72
+ print(OmegaConf.to_yaml(cfg, resolve=True))
73
+ raise SystemExit(0)
74
+ import fastwam.runtime as native
75
+ assert Path(native.__file__).resolve() == ROOT / "src/fastwam/runtime.py", native.__file__
76
+ if os.environ.get("HB_FW_AUG", "1") != "0":
77
+ import hb_fastwam_aug
78
+ hb_fastwam_aug.install()
79
+ print("HBFW_RECIPE", json.dumps(dict(profile=os.environ.get("HB_FW_PROFILE", "g1"), cameras=CAMS, state_dim=SDIM,
80
+ action_dim=ADIM, perturbed_joints=int(os.environ["HB_FW_JOINTS"]),
81
+ aug=os.environ.get("HB_FW_AUG", "1") != "0",
82
+ batch_size=cfg.batch_size, max_steps=cfg.max_steps, lr=cfg.learning_rate)), flush=True)
83
+ if os.environ.get("HB_DATA_SAMPLE"): # CPU check: build the dataset and read augmented samples, no model
84
+ from omegaconf import open_dict
85
+ from fastwam.utils import misc
86
+ misc.register_work_dir(os.environ.get("HB_SAMPLE_WORKDIR", "/tmp/hbfw-sample"))
87
+ tr, _ = native.build_datasets(cfg.data)
88
+ s = tr[len(tr) // 2]
89
+
90
+ def desc(x):
91
+ if isinstance(x, dict):
92
+ return {k: desc(v) for k, v in x.items()}
93
+ if hasattr(x, "shape"):
94
+ return [tuple(x.shape), str(x.dtype), round(float(x.float().min()), 3), round(float(x.float().max()), 3)]
95
+ return type(x).__name__
96
+ print("HBFW_SAMPLE", len(tr), json.dumps(desc(s), default=str), flush=True)
97
+ raise SystemExit(0)
98
+ if os.environ.get("HB_INLOOP_EVERY", "250") != "0": # 2026-11-16: benchmark action nMSE every 250 steps
99
+ try:
100
+ import hb_inloop_fastwam
101
+ hb_inloop_fastwam.install()
102
+ except Exception as e: # noqa: BLE001 - never block training
103
+ print("HB_INLOOP fastwam install failed:", repr(e), flush=True)
104
+ native.run_training(cfg)
a18-pour-water/b512/step-004000/code/hb_inloop_core.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared machinery of the in-training open-loop action MSE (Psi-Zero, Delta Flow; GR00T has its own tested copy in
2
+ hb_groot/hb_inloop_eval.py with the same protocol).
3
+
4
+ InLoop(model_name, task, split, out_jsonl, prepare, predict):
5
+ prepare(queries) -> optional per-model setup on this rank's queries (first eval only, inside the RNG fork)
6
+ predict(batch, seed) -> np.ndarray (B, >=16, >=Da) in raw action units; the first 16 steps and the first Da dims
7
+ (Da = benchmark action dim of the layout) are scored.
8
+ InLoop.evaluate(step, model): all train/eval modes of model.modules() and every RNG state (torch CPU + this CUDA device,
9
+ numpy, python) are restored afterwards; partial sums are all-reduced once (error flag included, so a failure on one
10
+ rank cannot leave the others waiting); rank 0 appends one json line per eval. Any failure disables later evals.
11
+ Protocol and metrics = hb_eval_common (post-hoc eval): queries every 16 frames, nMSE with the training-frame std.
12
+ """
13
+ import json
14
+ import os
15
+ import random
16
+ import sys
17
+ import time
18
+ import traceback
19
+ from pathlib import Path
20
+
21
+ import numpy as np
22
+ import torch
23
+
24
+ HERE = Path(__file__).resolve().parent
25
+ if str(HERE) not in sys.path:
26
+ sys.path.insert(0, str(HERE))
27
+ import hb_eval_common as E # noqa: E402
28
+
29
+ EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
30
+ BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
31
+ MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
32
+ SEED = 1234
33
+
34
+
35
+ def dist_info():
36
+ import torch.distributed as dist
37
+ if dist.is_available() and dist.is_initialized():
38
+ return dist.get_rank(), dist.get_world_size()
39
+ return int(os.environ.get("RANK", "0")), int(os.environ.get("WORLD_SIZE", "1"))
40
+
41
+
42
+ def pick_split(task):
43
+ split = os.environ.get("HB_INLOOP_SPLIT")
44
+ if not split:
45
+ split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
46
+ if not (E.split_dir(task, split) / "meta.json").exists():
47
+ raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
48
+ return split
49
+
50
+
51
+ class InLoop:
52
+ def __init__(self, model_name, task, split, out_jsonl, predict, prepare=None, cams=None, info=None):
53
+ self.model_name, self.task, self.split = model_name, task, split
54
+ self.out = Path(out_jsonl)
55
+ self.predict, self.prepare = predict, prepare
56
+ self.cams = cams
57
+ self.info = info or {}
58
+ self.disabled = False
59
+ self.ready = False
60
+ self.done_steps = set()
61
+
62
+ def say(self, *a):
63
+ if dist_info()[0] == 0:
64
+ print("HB_INLOOP", *a, flush=True)
65
+
66
+ def _setup(self):
67
+ rank, world = dist_info()
68
+ meta = E.load_meta(self.task, self.split)
69
+ self.meta, self.lay = meta, E.LAYOUTS[meta["layout"]]
70
+ self.std = E.norm_std(E.load_stats(self.task))
71
+ cams = self.cams or meta["cams"]
72
+ miss = [c for c in cams if c not in meta["cams"]]
73
+ if miss:
74
+ raise KeyError(f"cameras {miss} not in the eval cache {meta['cams']}")
75
+ allq = []
76
+ for ep in meta["episodes"]:
77
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
78
+ allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
79
+ if MAX_Q:
80
+ allq = allq[:MAX_Q]
81
+ self.n_total = len(allq)
82
+ mine = [q for i, q in enumerate(allq) if i % world == rank]
83
+ self.eps = list(meta["episodes"])
84
+ by_ep = {}
85
+ for ep, qi, t in mine:
86
+ by_ep.setdefault(ep, []).append((qi, t))
87
+ self.q = []
88
+ for ep, lst in by_ep.items():
89
+ with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
90
+ state, action = z["state"], z["action"]
91
+ prompt = meta["prompts"][str(int(z["task_index"]))]
92
+ imgs = {c: z[f"img_{c}"] for c in cams}
93
+ for qi, t in lst:
94
+ gt, valid = E.gt_chunk(action, t)
95
+ self.q.append(dict(episode=int(ep), t=int(t), T=len(action), state=state[t].copy(), prompt=prompt,
96
+ images={c: imgs[c][qi].copy() for c in cams}, gt=gt, valid=valid,
97
+ past_action=action[max(0, t - E.HORIZON):t].copy()))
98
+ if self.prepare:
99
+ self.prepare(self.q)
100
+ self.ready = True
101
+ self.say(f"{self.model_name} ready: task {self.task} split {self.split} queries {self.n_total} "
102
+ f"(rank {rank}/{world}: {len(self.q)}), every {EVERY} steps, batch {BATCH}, cams {cams}")
103
+
104
+ def evaluate(self, step, model):
105
+ if self.disabled or step in self.done_steps:
106
+ return
107
+ self.done_steps.add(step)
108
+ rank, world = dist_info()
109
+ t0 = time.time()
110
+ err, msg = 0.0, ""
111
+ acc, nq, infer = None, 0, 0.0
112
+ modes = [(m, m.training) for m in model.modules()]
113
+ dev = torch.cuda.current_device() if torch.cuda.is_available() else None
114
+ np_state, py_state = np.random.get_state(), random.getstate()
115
+ try:
116
+ with torch.random.fork_rng(devices=[dev] if dev is not None else []):
117
+ if not self.ready:
118
+ self._setup()
119
+ D = self.lay["action_dim"]
120
+ model.eval()
121
+ acc = E.Accumulator(self.std, self.lay["groups"])
122
+ for bi in range(0, len(self.q), BATCH):
123
+ batch = self.q[bi:bi + BATCH]
124
+ seed = SEED + 1000 * rank + bi
125
+ torch.default_generator.manual_seed(seed)
126
+ if dev is not None:
127
+ torch.cuda.manual_seed(seed)
128
+ s = time.time()
129
+ with torch.no_grad():
130
+ pred = np.asarray(self.predict(batch, seed), dtype=np.float64)
131
+ infer += time.time() - s
132
+ if pred.ndim != 3 or pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] < D:
133
+ raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, >={D})")
134
+ for q, p in zip(batch, pred):
135
+ acc.add(q["episode"], p[:E.HORIZON, :D], q["gt"], q["valid"])
136
+ nq += 1
137
+ except Exception as e: # noqa: BLE001 - never stop training
138
+ err, msg = 1.0, f"{type(e).__name__}: {e}"
139
+ print(f"HB_INLOOP {self.model_name} rank {rank} step {step} FAILED: {msg}\n{traceback.format_exc()}",
140
+ file=sys.stderr, flush=True)
141
+ finally:
142
+ for m, tr in modes:
143
+ m.training = tr
144
+ np.random.set_state(np_state)
145
+ random.setstate(py_state)
146
+ D = self.lay["action_dim"] if self.ready else 0
147
+ H = E.HORIZON
148
+ eps = self.eps if self.ready else []
149
+ nE = len(eps)
150
+ vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
151
+ if acc is not None and not err:
152
+ o = 0
153
+ vec[o:o + D] = acc.sse_raw; o += D
154
+ vec[o:o + D] = acc.sae_raw; o += D
155
+ vec[o:o + D] = acc.sse_n; o += D
156
+ vec[o] = acc.rows; o += 1
157
+ vec[o:o + H] = acc.step_sse_n; o += H
158
+ vec[o:o + H] = acc.step_rows; o += H
159
+ for j, ep in enumerate(eps):
160
+ s_, c_ = acc.per_episode.get(int(ep), (0.0, 0))
161
+ vec[o + j], vec[o + nE + j] = s_, c_
162
+ o += 2 * nE
163
+ vec[o], vec[o + 1] = nq, infer
164
+ vec[-1] = err
165
+ tot = self._allreduce(vec)
166
+ if tot is None or tot[-1] > 0 or not self.ready:
167
+ self.disabled = True
168
+ nfail = None if tot is None else int(tot[-1])
169
+ self._write({"step": step, "error": msg or "failed on another rank / collective failed",
170
+ "ranks_failed": nfail})
171
+ self.say(f"{self.model_name} step {step}: eval failed ({nfail} rank(s)) -> in-training eval disabled: {msg}")
172
+ return
173
+ o = 0
174
+ acc = E.Accumulator(self.std, self.lay["groups"])
175
+ acc.sse_raw = tot[o:o + D]; o += D
176
+ acc.sae_raw = tot[o:o + D]; o += D
177
+ acc.sse_n = tot[o:o + D]; o += D
178
+ acc.rows = int(round(tot[o])); o += 1
179
+ acc.step_sse_n = tot[o:o + H]; o += H
180
+ acc.step_rows = tot[o:o + H]; o += H
181
+ acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
182
+ for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
183
+ o += 2 * nE
184
+ n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
185
+ res = acc.result()
186
+ res.pop("nmse_by_dim", None)
187
+ rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "model": self.model_name,
188
+ "task": self.task, "split": self.split, "queries": n_q, "ranks": world,
189
+ "wall_s": round(time.time() - t0, 1), "infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
190
+ "seed": SEED, **self.info, **res}
191
+ self._write(rec)
192
+ g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
193
+ self.say(f"{self.model_name} step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} "
194
+ f"queries, {rec['wall_s']} s")
195
+
196
+ @staticmethod
197
+ def _allreduce(vec):
198
+ import torch.distributed as dist
199
+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1:
200
+ return vec
201
+ try:
202
+ dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
203
+ n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
204
+ mx, mn = n.clone(), n.clone()
205
+ dist.all_reduce(mx, op=dist.ReduceOp.MAX)
206
+ dist.all_reduce(mn, op=dist.ReduceOp.MIN)
207
+ if mx.item() != mn.item():
208
+ return None
209
+ t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
210
+ dist.all_reduce(t)
211
+ return t.cpu().numpy()
212
+ except Exception as e: # noqa: BLE001
213
+ print(f"HB_INLOOP all_reduce failed: {e!r}", file=sys.stderr, flush=True)
214
+ return None
215
+
216
+ def _write(self, rec):
217
+ if dist_info()[0] != 0:
218
+ return
219
+ try:
220
+ self.out.parent.mkdir(parents=True, exist_ok=True)
221
+ with open(self.out, "a") as f:
222
+ f.write(json.dumps(rec) + "\n")
223
+ except Exception as e: # noqa: BLE001
224
+ print(f"HB_INLOOP write failed: {e!r}", flush=True)
225
+
226
+
227
+ class TempAttrs:
228
+ """set attributes on an object for the duration of a block, then restore (or delete) them"""
229
+ _MISSING = object()
230
+
231
+ def __init__(self, obj, **kw):
232
+ self.obj, self.kw, self.old = obj, kw, {}
233
+
234
+ def __enter__(self):
235
+ for k, v in self.kw.items():
236
+ self.old[k] = self.obj.__dict__.get(k, self._MISSING)
237
+ setattr(self.obj, k, v)
238
+ return self.obj
239
+
240
+ def __exit__(self, *exc):
241
+ for k, v in self.old.items():
242
+ if v is self._MISSING:
243
+ try:
244
+ delattr(self.obj, k)
245
+ except AttributeError:
246
+ pass
247
+ else:
248
+ setattr(self.obj, k, v)
249
+ return False
a18-pour-water/b512/step-004000/code/hb_inloop_fastwam.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastWAM in-training open-loop action MSE (2026-11-16 user: every 250 steps, like GR00T / Psi-Zero / Delta Flow).
2
+
3
+ install() is called by fastwam/tools/hb_fastwam_train.py right before native.run_training(cfg). It uses the trainer's
4
+ own eval slot: after Wan22Trainer.__init__ it sets eval_every = HB_INLOOP_EVERY (default 250) and a non-None
5
+ val_dataset marker, and replaces Wan22Trainer.evaluate (the native one generates a full video for one random val sample;
6
+ it is off in our runs, eval_every=0 on the command line). The native loop then calls evaluate() on every rank right after
7
+ the optimizer step of every 250th update, followed by accelerator.wait_for_everyone(); returning None logs nothing.
8
+ evaluate(): all ranks score their shard of the benchmark eval cache with the live model (accelerator.unwrap_model; ZeRO-1
9
+ keeps full bf16 weights on every rank) through EXACTLY the post-hoc adapter path (eval-action-mse/adapters/ad_fastwam.py
10
+ Inputs + infer: FastWAM.infer_action, the action-only deployment path; 10 denoising steps = config
11
+ eval_num_inference_steps; seed 42 per query; same cameras / proprio normalisation / T5 cache as training).
12
+ Results: <output_dir>/inloop-mse.jsonl (rank 0) + "HB_INLOOP" log lines. Training is untouched (module modes and every
13
+ RNG restored by hb_inloop_core, no gradients); any failure disables later evals on all ranks.
14
+ """
15
+ import json
16
+ import os
17
+ import sys
18
+ from pathlib import Path
19
+
20
+ G = Path(__file__).resolve().parents[2] # .../policies-20261109 (this file: $G/fastwam/tools/)
21
+ EVAL = G / "eval-action-mse"
22
+ for p in (str(EVAL), str(EVAL / "adapters")):
23
+ if p not in sys.path:
24
+ sys.path.insert(0, p)
25
+ import numpy as np # noqa: E402
26
+ import hb_eval_common as E # noqa: E402
27
+ import hb_inloop_core as C # noqa: E402
28
+ import ad_fastwam as A # noqa: E402
29
+
30
+
31
+ class _Marker:
32
+ """non-None val_dataset so the native loop reaches evaluate(); never indexed"""
33
+
34
+ def __len__(self):
35
+ return 1
36
+
37
+
38
+ class FwEval:
39
+ def __init__(self, trainer):
40
+ cfg = trainer.cfg
41
+ self.cfg = cfg
42
+ ddir = Path(list(cfg.data.train.dataset_dirs)[0])
43
+ iface = json.loads((ddir / "meta/hb_fastwam_interface.json").read_text())
44
+ task = E.task_for_source(iface["source"])
45
+ if task is None:
46
+ raise LookupError(f"no benchmark task for {iface['source']}")
47
+ split = C.pick_split(task)
48
+ self.steps = int(cfg.eval_num_inference_steps)
49
+ self.inp = None
50
+ self.model = None
51
+ out = Path(cfg.output_dir) / "inloop-mse.jsonl"
52
+ self.il = C.InLoop("fastwam", task, split, out, predict=self.predict, prepare=self.prepare,
53
+ info={"denoising_steps": self.steps, "seed_per_query": 42, "dataset": ddir.name,
54
+ "path": "FastWAM.infer_action (ad_fastwam)"})
55
+ self.il.cams = None
56
+
57
+ def prepare(self, queries):
58
+ self.inp = A.Inputs(self.cfg, Path(self.cfg.output_dir) / "dataset_stats.json")
59
+ miss = [c for c in self.inp.cams if c not in self.il.meta["cams"]]
60
+ if miss:
61
+ raise KeyError(f"cameras {miss} not in the eval cache {self.il.meta['cams']}")
62
+ for q in {q["prompt"] for q in queries}:
63
+ self.inp.context(q) # fail early if an instruction has no T5 cache
64
+
65
+ def predict(self, batch, seed):
66
+ return np.stack([np.asarray(A.infer(self.model, self.inp, q, self.steps, 42), dtype=np.float64) for q in batch])
67
+
68
+ def evaluate(self, step, model):
69
+ self.model = model
70
+ try:
71
+ self.il.evaluate(step, model)
72
+ finally:
73
+ self.model = None
74
+
75
+
76
+ class _Broken:
77
+ def __init__(self, err):
78
+ self.err = err
79
+ self.il = C.InLoop("fastwam", "b6", "train10", "/dev/null", predict=None, prepare=self._boom)
80
+
81
+ def _boom(self, _q):
82
+ raise RuntimeError(self.err)
83
+
84
+ def evaluate(self, step, model):
85
+ self.il.evaluate(step, model)
86
+
87
+
88
+ def install():
89
+ if C.EVERY <= 0:
90
+ return
91
+ from fastwam import trainer as T
92
+ WT = T.Wan22Trainer
93
+ orig_init = WT.__init__
94
+
95
+ def __init__(self, *a, **kw):
96
+ orig_init(self, *a, **kw)
97
+ # native eval off (eval_every 0 on our command line). NB runtime.build_datasets sets val_ds = train_ds when
98
+ # there is no data.val, so val_dataset is never None here (11-16 smoke: the old "val_dataset is None" test
99
+ # silently left the in-loop eval off).
100
+ if int(self.eval_every) == 0:
101
+ self.eval_every = C.EVERY
102
+ if self.val_dataset is None:
103
+ self.val_dataset = _Marker()
104
+ self._hb_inloop_on = True
105
+ else: # a native eval is configured: leave it alone
106
+ self._hb_inloop_on = False
107
+
108
+ def evaluate(self):
109
+ if not getattr(self, "_hb_inloop_on", False):
110
+ return orig_eval(self)
111
+ ev = getattr(self, "_hb_inloop", None)
112
+ if ev is None:
113
+ try:
114
+ ev = FwEval(self)
115
+ except Exception as e: # noqa: BLE001
116
+ ev = _Broken(f"{type(e).__name__}: {e}")
117
+ self._hb_inloop = ev
118
+ if not ev.il.disabled:
119
+ ev.evaluate(int(self.global_step), self.accelerator.unwrap_model(self.model))
120
+ return None
121
+
122
+ orig_eval = WT.evaluate
123
+ WT.__init__ = __init__
124
+ WT.evaluate = evaluate
125
+ if int(os.environ.get("RANK", "0")) == 0:
126
+ print(f"HB_INLOOP fastwam installed: every {C.EVERY} steps", flush=True)
a18-pour-water/b512/step-004000/code/test_fastwam_inputs.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU check (no model): the FastWAM adapter's inputs built from the model-agnostic eval cache == the inputs our FastWAM
2
+ training dataset (RobotVideoDataset of the run's config, no HB augmentation) builds for the same frame.
3
+ python test_fastwam_inputs.py <task> <run out dir> [n_queries]"""
4
+ import json
5
+ import os
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+
11
+ HERE = Path(__file__).resolve().parent
12
+ sys.path.insert(0, str(HERE))
13
+ sys.path.insert(0, str(HERE / "adapters"))
14
+ import hb_eval_common as E # noqa: E402
15
+ import ad_fastwam as A # noqa: E402
16
+
17
+ task, out = sys.argv[1], Path(sys.argv[2])
18
+ nq = int(sys.argv[3]) if len(sys.argv) > 3 else 3
19
+ A.setup_env()
20
+ import torch # noqa: E402
21
+ from omegaconf import OmegaConf # noqa: E402
22
+ import fastwam.runtime as native # noqa: E402
23
+ from fastwam.utils import misc # noqa: E402
24
+
25
+ cfg = OmegaConf.load(out / "config.yaml")
26
+ inp = A.Inputs(cfg, out / "dataset_stats.json")
27
+ misc.register_work_dir("/tmp/hbfw-inputs-check")
28
+ OmegaConf.update(cfg, "data.train.pretrained_norm_stats", str(out / "dataset_stats.json"), merge=False, force_add=True)
29
+ tr, _ = native.build_datasets(cfg.data)
30
+ ddir = Path(cfg.data.train.dataset_dirs[0])
31
+ lens = [json.loads(l)["length"] for l in (ddir / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
32
+ meta = E.load_meta(task, "train10")
33
+ split = "train10"
34
+ worst = {}
35
+ done = 0
36
+ for ep, z in E.iter_episodes(task, split, None):
37
+ base = int(sum(lens[:ep]))
38
+ for qi, t in enumerate(list(z["query_t"])[1:1 + nq], start=1):
39
+ t = int(t)
40
+ s = tr[base + t]
41
+ obs = {"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]}, "state": z["state"][t],
42
+ "prompt": meta["prompts"][str(int(z["task_index"]))]}
43
+ img = inp.image(obs["images"])[0]
44
+ d_img = float((img - s["video"][:, 0].float()).abs().max())
45
+ pr = inp.proprio(obs["state"])[0]
46
+ d_pr = float((pr - s["proprio"][0].float()).abs().max())
47
+ ctx, mask = inp.context(obs["prompt"])
48
+ d_ctx = float((ctx.float() - s["context"].float()).abs().max())
49
+ d_mask = int((mask != s["context_mask"].bool()).sum())
50
+ gt = z["action"][t:t + 16]
51
+ a = inp.denorm(s["action"][:16])
52
+ d_act = float(np.abs(a[:len(gt)] - gt).max())
53
+ row = dict(ep=ep, t=t, idx=base + t, img=d_img, proprio=d_pr, ctx=d_ctx, mask=d_mask, act_denorm_vs_cache=d_act,
54
+ prompt_same=s["prompt"] == inp.DEFAULT_PROMPT.format(task=obs["prompt"]))
55
+ print("CHECK", json.dumps(row), flush=True)
56
+ for k in ("img", "proprio", "ctx", "act_denorm_vs_cache"):
57
+ worst[k] = max(worst.get(k, 0.0), row[k])
58
+ done += 1
59
+ if done >= 2 * nq:
60
+ break
61
+ print("WORST", json.dumps(worst))
a18-pour-water/b512/step-004000/complete.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "a18cr2",
3
+ "profile": "h2",
4
+ "per_gpu_batch": 16,
5
+ "global_batch": 512,
6
+ "gradient_accumulation_steps": 1,
7
+ "max_steps": 4000,
8
+ "save_every": 2000,
9
+ "recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
10
+ "interface": {
11
+ "profile": "h2",
12
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/a18-pour-water-cropped-v2-44cdd57/data",
13
+ "cameras": [
14
+ "head",
15
+ "chest"
16
+ ],
17
+ "state_dim": 78,
18
+ "action_dim": 75,
19
+ "episodes": 100,
20
+ "frames": 66850,
21
+ "fps": 30,
22
+ "first_episodes_only": null
23
+ },
24
+ "finished_utc": "2026-11-17T02:31:21Z"
25
+ }
a18-pour-water/b512/step-004000/dataset_meta/hb_fastwam_interface.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "profile": "h2",
3
+ "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/a18-pour-water-cropped-v2-44cdd57/data",
4
+ "cameras": [
5
+ "head",
6
+ "chest"
7
+ ],
8
+ "state_dim": 78,
9
+ "action_dim": 75,
10
+ "episodes": 100,
11
+ "frames": 66850,
12
+ "fps": 30,
13
+ "first_episodes_only": null
14
+ }
a18-pour-water/b512/step-004000/dataset_meta/info.json ADDED
@@ -0,0 +1,450 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "codebase_version": "v2.1",
3
+ "robot_type": "h2_sharpa",
4
+ "total_episodes": 100,
5
+ "total_frames": 66850,
6
+ "total_tasks": 1,
7
+ "total_videos": 200,
8
+ "total_chunks": 1,
9
+ "chunks_size": 1000,
10
+ "fps": 30,
11
+ "splits": {
12
+ "train": "0:100"
13
+ },
14
+ "data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
15
+ "video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
16
+ "features": {
17
+ "observation.state": {
18
+ "dtype": "float32",
19
+ "shape": [
20
+ 78
21
+ ],
22
+ "names": null
23
+ },
24
+ "observation.eef_state": {
25
+ "dtype": "float32",
26
+ "shape": [
27
+ 44
28
+ ],
29
+ "names": [
30
+ "left_wrist_x",
31
+ "left_wrist_y",
32
+ "left_wrist_z",
33
+ "left_wrist_qw",
34
+ "left_wrist_qx",
35
+ "left_wrist_qy",
36
+ "left_wrist_qz",
37
+ "left_thumb_DP_x",
38
+ "left_thumb_DP_y",
39
+ "left_thumb_DP_z",
40
+ "left_index_DP_x",
41
+ "left_index_DP_y",
42
+ "left_index_DP_z",
43
+ "left_middle_DP_x",
44
+ "left_middle_DP_y",
45
+ "left_middle_DP_z",
46
+ "left_ring_DP_x",
47
+ "left_ring_DP_y",
48
+ "left_ring_DP_z",
49
+ "left_pinky_DP_x",
50
+ "left_pinky_DP_y",
51
+ "left_pinky_DP_z",
52
+ "right_wrist_x",
53
+ "right_wrist_y",
54
+ "right_wrist_z",
55
+ "right_wrist_qw",
56
+ "right_wrist_qx",
57
+ "right_wrist_qy",
58
+ "right_wrist_qz",
59
+ "right_thumb_DP_x",
60
+ "right_thumb_DP_y",
61
+ "right_thumb_DP_z",
62
+ "right_index_DP_x",
63
+ "right_index_DP_y",
64
+ "right_index_DP_z",
65
+ "right_middle_DP_x",
66
+ "right_middle_DP_y",
67
+ "right_middle_DP_z",
68
+ "right_ring_DP_x",
69
+ "right_ring_DP_y",
70
+ "right_ring_DP_z",
71
+ "right_pinky_DP_x",
72
+ "right_pinky_DP_y",
73
+ "right_pinky_DP_z"
74
+ ]
75
+ },
76
+ "observation.root_orientation": {
77
+ "dtype": "float32",
78
+ "shape": [
79
+ 4
80
+ ],
81
+ "names": [
82
+ "qw",
83
+ "qx",
84
+ "qy",
85
+ "qz"
86
+ ]
87
+ },
88
+ "observation.projected_gravity": {
89
+ "dtype": "float32",
90
+ "shape": [
91
+ 3
92
+ ],
93
+ "names": [
94
+ "gx",
95
+ "gy",
96
+ "gz"
97
+ ]
98
+ },
99
+ "observation.base_velocity": {
100
+ "dtype": "float32",
101
+ "shape": [
102
+ 3
103
+ ],
104
+ "names": [
105
+ "vx",
106
+ "vy",
107
+ "yaw_rate"
108
+ ]
109
+ },
110
+ "observation.object_state": {
111
+ "dtype": "float32",
112
+ "shape": [
113
+ 14
114
+ ],
115
+ "names": [
116
+ "pitcher_x",
117
+ "pitcher_y",
118
+ "pitcher_z",
119
+ "pitcher_qw",
120
+ "pitcher_qx",
121
+ "pitcher_qy",
122
+ "pitcher_qz",
123
+ "cup_x",
124
+ "cup_y",
125
+ "cup_z",
126
+ "cup_qw",
127
+ "cup_qx",
128
+ "cup_qy",
129
+ "cup_qz"
130
+ ]
131
+ },
132
+ "observation.task_state": {
133
+ "dtype": "float32",
134
+ "shape": [
135
+ 5
136
+ ],
137
+ "names": [
138
+ "cup_ml",
139
+ "spill_ml",
140
+ "source_ml",
141
+ "pouring",
142
+ "tilt_deg"
143
+ ]
144
+ },
145
+ "observation.tactile": {
146
+ "dtype": "float32",
147
+ "shape": [
148
+ 10,
149
+ 24,
150
+ 24
151
+ ],
152
+ "names": {
153
+ "finger": [
154
+ "left_thumb",
155
+ "left_index",
156
+ "left_middle",
157
+ "left_ring",
158
+ "left_pinky",
159
+ "right_thumb",
160
+ "right_index",
161
+ "right_middle",
162
+ "right_ring",
163
+ "right_pinky"
164
+ ],
165
+ "row": "u: pad joint end -> fingertip",
166
+ "column": "v: across the pad, thumb side first"
167
+ }
168
+ },
169
+ "observation.tactile_force": {
170
+ "dtype": "float32",
171
+ "shape": [
172
+ 10
173
+ ],
174
+ "names": [
175
+ "left_thumb",
176
+ "left_index",
177
+ "left_middle",
178
+ "left_ring",
179
+ "left_pinky",
180
+ "right_thumb",
181
+ "right_index",
182
+ "right_middle",
183
+ "right_ring",
184
+ "right_pinky"
185
+ ]
186
+ },
187
+ "action.arms": {
188
+ "dtype": "float32",
189
+ "shape": [
190
+ 14
191
+ ],
192
+ "names": [
193
+ "left_shoulder_pitch_joint",
194
+ "left_shoulder_roll_joint",
195
+ "left_shoulder_yaw_joint",
196
+ "left_elbow_joint",
197
+ "left_wrist_roll_joint",
198
+ "left_wrist_pitch_joint",
199
+ "left_wrist_yaw_joint",
200
+ "right_shoulder_pitch_joint",
201
+ "right_shoulder_roll_joint",
202
+ "right_shoulder_yaw_joint",
203
+ "right_elbow_joint",
204
+ "right_wrist_roll_joint",
205
+ "right_wrist_pitch_joint",
206
+ "right_wrist_yaw_joint"
207
+ ]
208
+ },
209
+ "action.hands": {
210
+ "dtype": "float32",
211
+ "shape": [
212
+ 44
213
+ ],
214
+ "names": [
215
+ "left_index_MCP_FE",
216
+ "left_middle_MCP_FE",
217
+ "left_pinky_CMC",
218
+ "left_ring_MCP_FE",
219
+ "left_thumb_CMC_FE",
220
+ "left_index_MCP_AA",
221
+ "left_middle_MCP_AA",
222
+ "left_pinky_MCP_FE",
223
+ "left_ring_MCP_AA",
224
+ "left_thumb_CMC_AA",
225
+ "left_index_PIP",
226
+ "left_middle_PIP",
227
+ "left_pinky_MCP_AA",
228
+ "left_ring_PIP",
229
+ "left_thumb_MCP_FE",
230
+ "left_index_DIP",
231
+ "left_middle_DIP",
232
+ "left_pinky_PIP",
233
+ "left_ring_DIP",
234
+ "left_thumb_MCP_AA",
235
+ "left_pinky_DIP",
236
+ "left_thumb_IP",
237
+ "right_index_MCP_FE",
238
+ "right_middle_MCP_FE",
239
+ "right_pinky_CMC",
240
+ "right_ring_MCP_FE",
241
+ "right_thumb_CMC_FE",
242
+ "right_index_MCP_AA",
243
+ "right_middle_MCP_AA",
244
+ "right_pinky_MCP_FE",
245
+ "right_ring_MCP_AA",
246
+ "right_thumb_CMC_AA",
247
+ "right_index_PIP",
248
+ "right_middle_PIP",
249
+ "right_pinky_MCP_AA",
250
+ "right_ring_PIP",
251
+ "right_thumb_MCP_FE",
252
+ "right_index_DIP",
253
+ "right_middle_DIP",
254
+ "right_pinky_PIP",
255
+ "right_ring_DIP",
256
+ "right_thumb_MCP_AA",
257
+ "right_pinky_DIP",
258
+ "right_thumb_IP"
259
+ ]
260
+ },
261
+ "action.body_target": {
262
+ "dtype": "float32",
263
+ "shape": [
264
+ 31
265
+ ],
266
+ "names": [
267
+ "left_hip_pitch_joint",
268
+ "left_hip_roll_joint",
269
+ "left_hip_yaw_joint",
270
+ "left_knee_joint",
271
+ "left_ankle_roll_joint",
272
+ "left_ankle_pitch_joint",
273
+ "right_hip_pitch_joint",
274
+ "right_hip_roll_joint",
275
+ "right_hip_yaw_joint",
276
+ "right_knee_joint",
277
+ "right_ankle_roll_joint",
278
+ "right_ankle_pitch_joint",
279
+ "waist_yaw_joint",
280
+ "waist_roll_joint",
281
+ "waist_pitch_joint",
282
+ "left_shoulder_pitch_joint",
283
+ "left_shoulder_roll_joint",
284
+ "left_shoulder_yaw_joint",
285
+ "left_elbow_joint",
286
+ "left_wrist_roll_joint",
287
+ "left_wrist_pitch_joint",
288
+ "left_wrist_yaw_joint",
289
+ "right_shoulder_pitch_joint",
290
+ "right_shoulder_roll_joint",
291
+ "right_shoulder_yaw_joint",
292
+ "right_elbow_joint",
293
+ "right_wrist_roll_joint",
294
+ "right_wrist_pitch_joint",
295
+ "right_wrist_yaw_joint",
296
+ "head_pitch_joint",
297
+ "head_yaw_joint"
298
+ ]
299
+ },
300
+ "action.locomotion": {
301
+ "dtype": "float32",
302
+ "shape": [
303
+ 3
304
+ ],
305
+ "names": [
306
+ "walking",
307
+ "desired_heading_rel_rad",
308
+ "playback_speed"
309
+ ]
310
+ },
311
+ "annotation.phase": {
312
+ "dtype": "int64",
313
+ "shape": [
314
+ 1
315
+ ],
316
+ "names": [
317
+ "phase"
318
+ ]
319
+ },
320
+ "observation.images.head": {
321
+ "dtype": "video",
322
+ "shape": [
323
+ 480,
324
+ 640,
325
+ 3
326
+ ],
327
+ "names": [
328
+ "height",
329
+ "width",
330
+ "channel"
331
+ ],
332
+ "info": {
333
+ "video.height": 480,
334
+ "video.width": 640,
335
+ "video.codec": "h264",
336
+ "video.pix_fmt": "yuv420p",
337
+ "video.is_depth_map": false,
338
+ "video.fps": 30,
339
+ "video.channels": 3,
340
+ "has_audio": false
341
+ }
342
+ },
343
+ "observation.images.chest": {
344
+ "dtype": "video",
345
+ "shape": [
346
+ 480,
347
+ 640,
348
+ 3
349
+ ],
350
+ "names": [
351
+ "height",
352
+ "width",
353
+ "channel"
354
+ ],
355
+ "info": {
356
+ "video.height": 480,
357
+ "video.width": 640,
358
+ "video.codec": "h264",
359
+ "video.pix_fmt": "yuv420p",
360
+ "video.is_depth_map": false,
361
+ "video.fps": 30,
362
+ "video.channels": 3,
363
+ "has_audio": false
364
+ }
365
+ },
366
+ "timestamp": {
367
+ "dtype": "float32",
368
+ "shape": [
369
+ 1
370
+ ],
371
+ "names": null
372
+ },
373
+ "frame_index": {
374
+ "dtype": "int64",
375
+ "shape": [
376
+ 1
377
+ ],
378
+ "names": null
379
+ },
380
+ "episode_index": {
381
+ "dtype": "int64",
382
+ "shape": [
383
+ 1
384
+ ],
385
+ "names": null
386
+ },
387
+ "index": {
388
+ "dtype": "int64",
389
+ "shape": [
390
+ 1
391
+ ],
392
+ "names": null
393
+ },
394
+ "task_index": {
395
+ "dtype": "int64",
396
+ "shape": [
397
+ 1
398
+ ],
399
+ "names": null
400
+ },
401
+ "action": {
402
+ "dtype": "float32",
403
+ "shape": [
404
+ 75
405
+ ],
406
+ "names": null
407
+ }
408
+ },
409
+ "script_config": {
410
+ "source": "sim",
411
+ "simulator": "newton",
412
+ "robot": "H2 + Sharpa (robot:h2_sharpa, experimental)",
413
+ "controller": "TRACE SONIC native runtime (legs, waist, balance; torso support kept for the tabletop tasks) plus the skill-server arm joint overrides and Sharpa position targets; scripted privileged expert",
414
+ "control_frequency": 50.0,
415
+ "camera_backend": "offline_render",
416
+ "camera_note": "head, chest and wrist rendered offline (humanoid_newton.rendering.splat_render) at the registered robot camera_mounts; third_person fixed; not calibrated cameras",
417
+ "tactile_note": "simulated contact footprints on the Sharpa elastomer pads (no sensor model)",
418
+ "task": "h2_pour_water",
419
+ "profile": "h2_pour_water_r1",
420
+ "task_description": "Grasp the pitcher's handle with the right hand, lift it, pour water into the glass tumbler, then put the pitcher back where it stood and let go.",
421
+ "phases": [
422
+ "idle",
423
+ "reach_pitcher",
424
+ "grasp_pitcher",
425
+ "lift_carry",
426
+ "pour",
427
+ "return",
428
+ "release"
429
+ ],
430
+ "tactile": {
431
+ "fingers": [
432
+ "left_thumb",
433
+ "left_index",
434
+ "left_middle",
435
+ "left_ring",
436
+ "left_pinky",
437
+ "right_thumb",
438
+ "right_index",
439
+ "right_middle",
440
+ "right_ring",
441
+ "right_pinky"
442
+ ],
443
+ "grid": 24,
444
+ "units": "N per taxel",
445
+ "module": "debug/newton/h2_sharpa_tactile.py"
446
+ },
447
+ "git_commit": "69c6ae8351f490ff9ffb87a56e95b11c91bf2f55"
448
+ },
449
+ "discarded_episode_indices": []
450
+ }
a18-pour-water/b512/step-004000/dataset_meta/tasks.jsonl ADDED
@@ -0,0 +1 @@
 
 
1
+ {"task_index": 0, "task": "Grasp the pitcher's handle with the right hand, lift it, pour water into the glass tumbler, then put the pitcher back where it stood and let go."}
a18-pour-water/b512/step-004000/dataset_stats.json ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-004000/logs/node-0.log ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-004000/logs/node-1.log ADDED
The diff for this file is too large to render. See raw diff
 
a18-pour-water/b512/step-004000/logs/node-2.log ADDED
The diff for this file is too large to render. See raw diff