b8: last two checkpoints + eval/deploy files + training records
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- b8-table-wipe/b512/step-002000/batch_benchmark/chosen.json +1 -0
- b8-table-wipe/b512/step-002000/batch_benchmark/summary.txt +2 -0
- b8-table-wipe/b512/step-002000/code/ad_fastwam.py +163 -0
- b8-table-wipe/b512/step-002000/code/fastwam_prep.sh +61 -0
- b8-table-wipe/b512/step-002000/code/fastwam_task.sh +48 -0
- b8-table-wipe/b512/step-002000/code/hb_aug_v2.py +163 -0
- b8-table-wipe/b512/step-002000/code/hb_eval_common.py +260 -0
- b8-table-wipe/b512/step-002000/code/hb_fastwam_aug.py +73 -0
- b8-table-wipe/b512/step-002000/code/hb_fastwam_convert.py +121 -0
- b8-table-wipe/b512/step-002000/code/hb_fastwam_train.py +104 -0
- b8-table-wipe/b512/step-002000/code/hb_inloop_core.py +249 -0
- b8-table-wipe/b512/step-002000/code/hb_inloop_fastwam.py +126 -0
- b8-table-wipe/b512/step-002000/code/test_fastwam_inputs.py +61 -0
- b8-table-wipe/b512/step-002000/complete.json +25 -0
- b8-table-wipe/b512/step-002000/dataset_meta/hb_fastwam_interface.json +14 -0
- b8-table-wipe/b512/step-002000/dataset_meta/info.json +538 -0
- b8-table-wipe/b512/step-002000/dataset_meta/tasks.jsonl +1 -0
- b8-table-wipe/b512/step-002000/dataset_stats.json +0 -0
- b8-table-wipe/b512/step-002000/logs/node-0.log +0 -0
- b8-table-wipe/b512/step-002000/logs/node-1.log +0 -0
- b8-table-wipe/b512/step-002000/logs/node-2.log +0 -0
- b8-table-wipe/b512/step-002000/logs/node-3.log +0 -0
- b8-table-wipe/b512/step-002000/loss_summary.json +23 -0
- b8-table-wipe/b512/step-002000/metrics.csv +201 -0
- b8-table-wipe/b512/step-002000/model.pt +3 -0
- b8-table-wipe/b512/step-002000/model_config.yaml +57 -0
- b8-table-wipe/b512/step-002000/protocol.json +72 -0
- b8-table-wipe/b512/step-002000/text_embeddings/5a20e43a27564ea8978edc7394a96e0c1295e82528d59dba0c4169f3bba9a630.t5_len128.wan22ti2v5b.pt +3 -0
- b8-table-wipe/b512/step-002000/training_config.yaml +259 -0
- b8-table-wipe/b512/step-004000/batch_benchmark/chosen.json +1 -0
- b8-table-wipe/b512/step-004000/batch_benchmark/summary.txt +2 -0
- b8-table-wipe/b512/step-004000/code/ad_fastwam.py +163 -0
- b8-table-wipe/b512/step-004000/code/fastwam_prep.sh +61 -0
- b8-table-wipe/b512/step-004000/code/fastwam_task.sh +48 -0
- b8-table-wipe/b512/step-004000/code/hb_aug_v2.py +163 -0
- b8-table-wipe/b512/step-004000/code/hb_eval_common.py +260 -0
- b8-table-wipe/b512/step-004000/code/hb_fastwam_aug.py +73 -0
- b8-table-wipe/b512/step-004000/code/hb_fastwam_convert.py +121 -0
- b8-table-wipe/b512/step-004000/code/hb_fastwam_train.py +104 -0
- b8-table-wipe/b512/step-004000/code/hb_inloop_core.py +249 -0
- b8-table-wipe/b512/step-004000/code/hb_inloop_fastwam.py +126 -0
- b8-table-wipe/b512/step-004000/code/test_fastwam_inputs.py +61 -0
- b8-table-wipe/b512/step-004000/complete.json +25 -0
- b8-table-wipe/b512/step-004000/dataset_meta/hb_fastwam_interface.json +14 -0
- b8-table-wipe/b512/step-004000/dataset_meta/info.json +538 -0
- b8-table-wipe/b512/step-004000/dataset_meta/tasks.jsonl +1 -0
- b8-table-wipe/b512/step-004000/dataset_stats.json +0 -0
- b8-table-wipe/b512/step-004000/logs/node-0.log +0 -0
- b8-table-wipe/b512/step-004000/logs/node-1.log +0 -0
- b8-table-wipe/b512/step-004000/logs/node-2.log +0 -0
b8-table-wipe/b512/step-002000/batch_benchmark/chosen.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"per_gpu_batch": 16, "global_batch": 512, "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/bench-20261116-r2/summary.txt"}
|
b8-table-wipe/b512/step-002000/batch_benchmark/summary.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
bs=16 global=512 rc=[0 0 0 0 ] oom=0 peak_mem_MiB=58421 speed=
|
| 2 |
+
bs=32 global=1024 rc=[1 1 1 1 ] oom=4 peak_mem_MiB=81123 speed=
|
b8-table-wipe/b512/step-002000/code/ad_fastwam.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastWAM adapter (2026-11-16). Deployment path of FastWAM's own policy (experiments/robotwin/fastwam_policy/
|
| 2 |
+
deploy_policy.py, WorldActionRobotWinPolicy._infer_action_chunk: model.infer_action, the action-only path = first-frame
|
| 3 |
+
video KV cache + action-expert denoising, no future video) fed exactly like our training dataset (RobotVideoDataset +
|
| 4 |
+
FastWAMProcessor of the run's config.yaml, no HB augmentation):
|
| 5 |
+
* cameras in shape_meta order (g1 ego_view + chest, h2 head + chest): uint8 [1,C,H,W] -> processor val transforms
|
| 6 |
+
(ToTensor /255 + Resize 224x224), concatenated horizontally (224 x 448), dataset ResizeSmallestSide/CenterCrop
|
| 7 |
+
(no-op at 224 x 448), Normalize(0.5, 0.5) -> [-1, 1];
|
| 8 |
+
* state: processor.action_state_transform -> normalizer.forward (min/max of the run's
|
| 9 |
+
dataset_stats.json), the proprio of the first frame;
|
| 10 |
+
* text: the run's T5 cache for DEFAULT_PROMPT.format(task=instruction) with context[~mask] = 0 and mask = ones, as in
|
| 11 |
+
training (the text encoder is not loaded in training either; deploy_policy encodes the same prompt with T5 online);
|
| 12 |
+
* infer_action(action_horizon = num_frames - 1 = 16, num_inference_steps = config eval_num_inference_steps (10),
|
| 13 |
+
text_cfg_scale 1, seed 42 = native trainer eval), denormalised with normalizer.normalizers["action"][key].backward.
|
| 14 |
+
ckpt = .../fastwam/runs/<task>/out/checkpoints/weights/step_<N>.pt (config.yaml + dataset_stats.json in .../out).
|
| 15 |
+
Weights: model built from the config (Wan2.2 + ActionDiT init), then the checkpoint's mot + proprio_encoder are
|
| 16 |
+
loaded STRICTLY (FastWAM.load_checkpoint uses strict=False for mot; a missing key here is an error).
|
| 17 |
+
"""
|
| 18 |
+
import os
|
| 19 |
+
import sys
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
|
| 24 |
+
import hb_eval_common as E
|
| 25 |
+
|
| 26 |
+
FW = E.G / "fastwam"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def setup_env():
|
| 30 |
+
os.environ.setdefault("FASTWAM_SRC", str(FW / "FastWAM"))
|
| 31 |
+
os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(FW / "models"))
|
| 32 |
+
os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "true")
|
| 33 |
+
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
| 34 |
+
src = str(Path(os.environ["FASTWAM_SRC"]) / "src")
|
| 35 |
+
if src not in sys.path:
|
| 36 |
+
sys.path.insert(0, src)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class Inputs:
|
| 40 |
+
"""model-agnostic cache query -> FastWAM inputs (shared by the post-hoc adapter and the in-training eval)"""
|
| 41 |
+
|
| 42 |
+
def __init__(self, cfg, stats_path):
|
| 43 |
+
setup_env()
|
| 44 |
+
from hydra.utils import instantiate
|
| 45 |
+
from omegaconf import OmegaConf
|
| 46 |
+
from fastwam.datasets.dataset_utils import CenterCrop, Normalize, ResizeSmallestSideAspectPreserving
|
| 47 |
+
from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT
|
| 48 |
+
from fastwam.datasets.lerobot.utils.normalizer import load_dataset_stats_from_json
|
| 49 |
+
d = cfg.data.train
|
| 50 |
+
self.DEFAULT_PROMPT = DEFAULT_PROMPT
|
| 51 |
+
self.proc = instantiate(d.processor).eval()
|
| 52 |
+
self.proc.set_normalizer_from_stats(load_dataset_stats_from_json(str(stats_path)))
|
| 53 |
+
self.cams = [m["key"] for m in self.proc.shape_meta["images"]]
|
| 54 |
+
if d.concat_multi_camera != "horizontal":
|
| 55 |
+
raise ValueError(f"concat_multi_camera {d.concat_multi_camera} not supported")
|
| 56 |
+
vs = list(d.video_size)
|
| 57 |
+
self.resize = ResizeSmallestSideAspectPreserving(args={"img_w": vs[1], "img_h": vs[0]})
|
| 58 |
+
self.crop = CenterCrop(args={"img_w": vs[1], "img_h": vs[0]})
|
| 59 |
+
self.norm = Normalize(args={"mean": 0.5, "std": 0.5})
|
| 60 |
+
self.cache_dir = d.text_embedding_cache_dir
|
| 61 |
+
self.context_len = int(d.context_len)
|
| 62 |
+
self.horizon = int(d.num_frames) - 1
|
| 63 |
+
self.skey = self.proc.shape_meta["state"][0]["key"]
|
| 64 |
+
self.akey = self.proc.shape_meta["action"][0]["key"]
|
| 65 |
+
if len(self.proc.shape_meta["state"]) != 1 or len(self.proc.shape_meta["action"]) != 1:
|
| 66 |
+
raise ValueError("expected one merged state key and one merged action key")
|
| 67 |
+
self._ctx = {}
|
| 68 |
+
self.OmegaConf = OmegaConf
|
| 69 |
+
|
| 70 |
+
def image(self, images):
|
| 71 |
+
import torch
|
| 72 |
+
per = []
|
| 73 |
+
for c in self.cams:
|
| 74 |
+
x = torch.from_numpy(np.ascontiguousarray(images[c])).permute(2, 0, 1).unsqueeze(0) # [1,C,H,W] uint8
|
| 75 |
+
for tr in self.proc.val_transforms:
|
| 76 |
+
x = tr(x)
|
| 77 |
+
per.append(x)
|
| 78 |
+
v = torch.cat(per, dim=-1) # [1, C, 224, 448]
|
| 79 |
+
v = self.norm(self.crop(self.resize(v)))
|
| 80 |
+
return v # [1, 3, H, W] in [-1, 1], float32
|
| 81 |
+
|
| 82 |
+
def proprio(self, state):
|
| 83 |
+
import torch
|
| 84 |
+
b = {"state": {self.skey: torch.as_tensor(np.asarray(state, dtype=np.float32)).unsqueeze(0)}}
|
| 85 |
+
b = self.proc.action_state_transform(b)
|
| 86 |
+
b = self.proc.normalizer.forward(b) # = deploy_policy._normalize_state (one merged key: merger is identity)
|
| 87 |
+
return b["state"][self.skey] # [1, D]
|
| 88 |
+
|
| 89 |
+
def context(self, instruction):
|
| 90 |
+
import hashlib
|
| 91 |
+
import torch
|
| 92 |
+
prompt = self.DEFAULT_PROMPT.format(task=instruction)
|
| 93 |
+
if prompt not in self._ctx:
|
| 94 |
+
h = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
|
| 95 |
+
p = os.path.join(self.cache_dir, f"{h}.t5_len{self.context_len}.wan22ti2v5b.pt")
|
| 96 |
+
if not os.path.exists(p):
|
| 97 |
+
raise FileNotFoundError(f"no T5 cache for the instruction ({p}): {instruction[:80]!r}")
|
| 98 |
+
obj = torch.load(p, map_location="cpu")
|
| 99 |
+
ctx, mask = obj["context"], obj["mask"]
|
| 100 |
+
ctx = ctx.clone()
|
| 101 |
+
ctx[~mask.bool()] = 0.0
|
| 102 |
+
self._ctx[prompt] = (ctx, torch.ones_like(mask.bool()))
|
| 103 |
+
return self._ctx[prompt]
|
| 104 |
+
|
| 105 |
+
def denorm(self, action):
|
| 106 |
+
import torch
|
| 107 |
+
a = action
|
| 108 |
+
if a.ndim == 2:
|
| 109 |
+
a = a.unsqueeze(0)
|
| 110 |
+
n = self.proc.normalizer.normalizers["action"][self.akey]
|
| 111 |
+
return n.backward(a.to(dtype=torch.float32, device="cpu")).numpy()[0]
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def infer(model, inp, obs, steps, seed=42):
|
| 115 |
+
import torch
|
| 116 |
+
img = inp.image(obs["images"]).to(device=model.device, dtype=model.torch_dtype)
|
| 117 |
+
ctx, mask = inp.context(obs["prompt"])
|
| 118 |
+
with torch.no_grad():
|
| 119 |
+
out = model.infer_action(prompt=None, input_image=img, action_horizon=inp.horizon, proprio=inp.proprio(obs["state"]),
|
| 120 |
+
context=ctx, context_mask=mask, text_cfg_scale=1.0, num_inference_steps=steps, seed=seed,
|
| 121 |
+
rand_device="cpu", tiled=False)
|
| 122 |
+
return inp.denorm(out["action"])
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class Adapter:
|
| 126 |
+
def __init__(self, ckpt, task, split, meta, device="cuda:0", steps=None, seed="42"):
|
| 127 |
+
setup_env()
|
| 128 |
+
import torch
|
| 129 |
+
from hydra.utils import instantiate
|
| 130 |
+
from omegaconf import OmegaConf
|
| 131 |
+
self.torch = torch
|
| 132 |
+
ckpt = Path(ckpt)
|
| 133 |
+
out = ckpt.parent.parent.parent
|
| 134 |
+
self.cfg = OmegaConf.load(out / "config.yaml")
|
| 135 |
+
self.inp = Inputs(self.cfg, out / "dataset_stats.json")
|
| 136 |
+
miss = [c for c in self.inp.cams if c not in meta["cams"]]
|
| 137 |
+
if miss:
|
| 138 |
+
raise KeyError(f"cameras {miss} not in the cache {meta['cams']}")
|
| 139 |
+
self.device = device if torch.cuda.is_available() else "cpu"
|
| 140 |
+
mp = str(self.cfg.get("mixed_precision", "bf16")).lower()
|
| 141 |
+
dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[mp]
|
| 142 |
+
self.model = instantiate(self.cfg.model, model_dtype=dtype, device=self.device)
|
| 143 |
+
payload = torch.load(str(ckpt), map_location="cpu")
|
| 144 |
+
miss_k, unexp = self.model.mot.load_state_dict(payload["mot"], strict=False)
|
| 145 |
+
if miss_k or unexp:
|
| 146 |
+
raise RuntimeError(f"checkpoint mot keys: missing {list(miss_k)[:5]} unexpected {list(unexp)[:5]}")
|
| 147 |
+
if self.model.proprio_encoder is not None:
|
| 148 |
+
self.model.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
|
| 149 |
+
self.ckpt_step = payload.get("step")
|
| 150 |
+
del payload
|
| 151 |
+
self.model = self.model.to(self.device).eval()
|
| 152 |
+
self.native_steps = int(self.cfg.eval_num_inference_steps)
|
| 153 |
+
self.steps = int(steps or self.native_steps)
|
| 154 |
+
self.seed = int(seed)
|
| 155 |
+
self.ckpt = str(ckpt)
|
| 156 |
+
|
| 157 |
+
def info(self):
|
| 158 |
+
return {"model": "fastwam", "ckpt": self.ckpt, "ckpt_step": self.ckpt_step, "denoising_steps": self.steps,
|
| 159 |
+
"native_steps_in_config": self.native_steps, "cams": self.inp.cams, "chunk": self.inp.horizon,
|
| 160 |
+
"seed": self.seed, "path": "FastWAM.infer_action (action-only, deploy_policy)"}
|
| 161 |
+
|
| 162 |
+
def predict(self, obs):
|
| 163 |
+
return infer(self.model, self.inp, obs, self.steps, self.seed)
|
b8-table-wipe/b512/step-002000/code/fastwam_prep.sh
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# 2026-11-16: CPU-only preparation of FastWAM for the 12 Delta-flow tasks (no GPU is touched: training is running).
|
| 3 |
+
# Data copies on our interface, ActionDiT initializers per profile, one shared T5 text cache, augmented-sample checks.
|
| 4 |
+
P=/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z
|
| 5 |
+
G=$P/policies-20261109
|
| 6 |
+
F=$G/fastwam
|
| 7 |
+
E=$G/envs/fastwam/bin/python
|
| 8 |
+
exec >> $F/prep.log 2>&1
|
| 9 |
+
echo "$(date) start (v2: 12 tasks)"
|
| 10 |
+
until [ -f $G/envs/fastwam/.hb_ready ] || grep -q ENV_FAILED $F/setup.log; do sleep 30; done
|
| 11 |
+
[ -f $G/envs/fastwam/.hb_ready ] || { echo "ENV_FAILED"; exit 1; }
|
| 12 |
+
grep -q FW_DL_DONE $F/dl2.log || { echo "DOWNLOAD_FAILED"; exit 1; }
|
| 13 |
+
export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true CUDA_VISIBLE_DEVICES="" HF_HUB_OFFLINE=1
|
| 14 |
+
export HB_AUG_DIR=$G/hb_groot
|
| 15 |
+
unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT $(compgen -e | grep -E "^PET_")
|
| 16 |
+
TC=$F/text-cache/all
|
| 17 |
+
# task profile source [first N episodes]
|
| 18 |
+
cat > $F/tasks.txt <<EOF
|
| 19 |
+
a10v3 g1 $P/a10-trolley-push-v3-6e410c5/data
|
| 20 |
+
a3v4 g1 $P/a3-fridge-open-v4-6393d4a/data
|
| 21 |
+
a7v4 g1 $P/a7-cabinet-close-v4-90cc2b5/data
|
| 22 |
+
b7 g1 $P/b7-black-clothes-hamper-v1-1bb002a/data 100
|
| 23 |
+
b5 g1 $P/b5-chair-push-v1-b85b7e9/data
|
| 24 |
+
b8 g1 $P/b8-table-wipe-v1-3b2e618/data
|
| 25 |
+
a17cr2 h2 $P/a17-unscrew-bottle-cropped-v2-3520b70/data
|
| 26 |
+
basketv2 g1 $P/basket-trimv2-c4df4d4/data
|
| 27 |
+
c6 g1 $P/c6-bathroom-door-fetch-v1-3afba69/data
|
| 28 |
+
b6v6 g1 $P/b6-pedal-bin-v6-ad2f5bf/data
|
| 29 |
+
a14v2 h2p $P/a14-unscrew-bulb-v2-8411a6d/data
|
| 30 |
+
b1 g1 $P/b1-riceball-microwave-v1-36da298/data
|
| 31 |
+
EOF
|
| 32 |
+
DIRS=""
|
| 33 |
+
while read t prof src lim; do
|
| 34 |
+
D=$F/data/$t
|
| 35 |
+
if [ ! -f $D/meta/hb_fastwam_interface.json ]; then
|
| 36 |
+
$E $F/tools/hb_fastwam_convert.py $prof $src $D ${lim:+--episodes $lim} || { echo "CONVERT_FAILED $t"; continue; }
|
| 37 |
+
fi
|
| 38 |
+
DIRS="$DIRS,\"$D\""
|
| 39 |
+
done < $F/tasks.txt
|
| 40 |
+
DIRS="[${DIRS#,}]"
|
| 41 |
+
echo "$(date) CONVERTED $DIRS"
|
| 42 |
+
cd $F/FastWAM
|
| 43 |
+
for prof in g1 h2 h2p; do
|
| 44 |
+
AI=$F/action-init-$prof.pt
|
| 45 |
+
[ -f $AI ] && continue
|
| 46 |
+
HB_FW_PROFILE=$prof HB_MODEL_YAML=$F/model-$prof.yaml $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" || { echo "YAML_FAILED $prof"; continue; }
|
| 47 |
+
grep -E 'action_dim|proprio_dim' $F/model-$prof.yaml | tr '\n' ' '; echo
|
| 48 |
+
$E scripts/preprocess_action_dit_backbone.py --model-config $F/model-$prof.yaml --output $AI --device cpu --dtype bfloat16 && echo "$(date) INIT_OK $prof $(stat -c %s $AI)" || echo "INIT_FAILED $prof"
|
| 49 |
+
done
|
| 50 |
+
mkdir -p $TC
|
| 51 |
+
$E scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 model.redirect_common_files=false "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" && echo "$(date) TEXT_OK $(ls $TC | wc -l) files" || echo TEXT_FAILED
|
| 52 |
+
ok=1
|
| 53 |
+
for pair in a10v3:g1 a17cr2:h2 a14v2:h2p; do
|
| 54 |
+
t=${pair%%:*}; prof=${pair##*:}
|
| 55 |
+
HB_FW_PROFILE=$prof HB_DATA_SAMPLE=1 HB_SAMPLE_WORKDIR=/tmp/hbfw-sample-$t $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=[\"$F/data/$t\"]" "data.train.text_embedding_cache_dir=$TC" "model.action_dit_pretrained_path=$F/action-init-$prof.pt" > $F/sample-$t.log 2>&1
|
| 56 |
+
grep -E 'HB_FW_AUG|HBFW_|Error' $F/sample-$t.log | cut -c1-800
|
| 57 |
+
grep -q "HBFW_SAMPLE" $F/sample-$t.log || { ok=0; echo "SAMPLE_FAILED $t"; }
|
| 58 |
+
done
|
| 59 |
+
for t in $(cut -d' ' -f1 $F/tasks.txt); do [ -f $F/data/$t/meta/hb_fastwam_interface.json ] || { ok=0; echo "MISSING_DATA $t"; }; done
|
| 60 |
+
for prof in g1 h2 h2p; do [ -f $F/action-init-$prof.pt ] || { ok=0; echo "MISSING_INIT $prof"; }; done
|
| 61 |
+
[ $ok = 1 ] && touch $F/bench.ready && echo "$(date) BENCH_READY" || echo "$(date) NOT_READY"
|
b8-table-wipe/b512/step-002000/code/fastwam_task.sh
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Relay cmd item helper (2026-11-16 user): FastWAM on one benchmark task, 4 nodes x 8 GPUs, per-GPU batch from the
|
| 3 |
+
# benchmark ($G/fastwam/chosen.json: largest batch that fits), no gradient accumulation, 4000 steps (user default while
|
| 4 |
+
# the batch is unknown), native Gaussian FM recipe (lr 1e-4 cosine, ZeRO-1, bf16), HB augmentation v2 + proprio
|
| 5 |
+
# perturbation, our state/action interface. Checkpoints at 2000 and 4000. Skips (exit 3) unless the benchmark passed.
|
| 6 |
+
# fastwam_task.sh <task> <profile g1|h2|h2p> <port>
|
| 7 |
+
G=$(dirname "$(dirname "$(realpath "$0")")")
|
| 8 |
+
TASK=$1 PROF=$2 PORT=$3
|
| 9 |
+
F=$G/fastwam; E=$G/envs/fastwam/bin/python; R=$F/runs/$TASK
|
| 10 |
+
RK=${RANK:?}; M0=${MASTER_ADDR:?}
|
| 11 |
+
[ -f $F/fastwam.ok ] && [ -s $F/chosen.json ] || { echo "SKIP fastwam $TASK: benchmark not passed"; exit 3; }
|
| 12 |
+
[ -f $F/data/$TASK/meta/hb_fastwam_interface.json ] && [ -f $F/action-init-$PROF.pt ] || { echo "SKIP fastwam $TASK: data not prepared"; exit 3; }
|
| 13 |
+
BS=$(python3 -c "import json;print(json.load(open('$F/chosen.json'))['per_gpu_batch'])")
|
| 14 |
+
mkdir -p $R/logs
|
| 15 |
+
exec > >(tee -a "$R/logs/node-$RK.log") 2>&1
|
| 16 |
+
date -u; hostname
|
| 17 |
+
[ -f $R/complete.json ] && { echo "already complete"; exit 0; }
|
| 18 |
+
echo "FASTWAM task=$TASK profile=$PROF rank=$RK master=$M0:$PORT per_gpu=$BS global=$((BS * 32)) steps=4000"
|
| 19 |
+
if ! ldconfig -p | grep -q 'libavutil.so.56'; then
|
| 20 |
+
apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
|
| 21 |
+
libavcodec58 libavformat58 libavutil56 libswscale5 libswresample3 libavdevice58 libavfilter7 || true
|
| 22 |
+
fi
|
| 23 |
+
unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT
|
| 24 |
+
unset $(compgen -e | grep -E '^PET_') NPROC_PER_NODE WORLD_SIZE_WITH_STANDBY TORCHELASTIC_RUN_ID
|
| 25 |
+
export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true HF_HUB_OFFLINE=1 HB_AUG_DIR=$G/hb_groot HB_FW_PROFILE=$PROF
|
| 26 |
+
export NCCL_SOCKET_IFNAME=eth GLOO_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false WANDB_MODE=disabled
|
| 27 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
| 28 |
+
export PATH=$G/envs/fastwam/bin:$PATH # r2 (11-16 05:3x): r1 failed, accelerate called the missing deepspeed CLI (pdsh launcher)
|
| 29 |
+
for i in $(seq 36); do [ -z "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ] && break; sleep 5; done
|
| 30 |
+
cd $F/FastWAM
|
| 31 |
+
$E -m accelerate.commands.launch --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml --deepspeed_multinode_launcher standard \
|
| 32 |
+
--num_machines 4 --machine_rank $RK --main_process_ip $M0 --main_process_port $PORT --num_processes 32 \
|
| 33 |
+
$F/tools/hb_fastwam_train.py "output_dir=$R/out" "batch_size=$BS" max_steps=4000 log_every=20 save_every=2000 eval_every=0 \
|
| 34 |
+
num_workers=8 wandb.enabled=false gradient_accumulation_steps=1 "data.train.dataset_dirs=[\"$F/data/$TASK\"]" \
|
| 35 |
+
"data.train.text_embedding_cache_dir=$F/text-cache/all" "model.action_dit_pretrained_path=$F/action-init-$PROF.pt"
|
| 36 |
+
rc=$?
|
| 37 |
+
echo "$rc" > $R/logs/exit-node-$RK
|
| 38 |
+
if [ "$rc" = 0 ] && [ "$RK" = 0 ]; then
|
| 39 |
+
python3 - "$R/complete.json" "$TASK" "$PROF" "$BS" "$F/data/$TASK/meta/hb_fastwam_interface.json" <<'EOF'
|
| 40 |
+
import json, sys, time
|
| 41 |
+
out, task, prof, bs, iface = sys.argv[1:6]
|
| 42 |
+
d = dict(task=task, profile=prof, per_gpu_batch=int(bs), global_batch=int(bs) * 32, gradient_accumulation_steps=1,
|
| 43 |
+
max_steps=4000, save_every=2000, recipe="native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
|
| 44 |
+
interface=json.load(open(iface)), finished_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
|
| 45 |
+
json.dump(d, open(out, 'w'), indent=1); print('COMPLETE', d)
|
| 46 |
+
EOF
|
| 47 |
+
fi
|
| 48 |
+
echo "EXIT $rc"; exit $rc
|
b8-table-wipe/b512/step-002000/code/hb_aug_v2.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Humanoid Benchmark data augmentation v2 (project CLAUDE.md), model-agnostic torch implementation.
|
| 2 |
+
|
| 3 |
+
augment(x, rng, obs_mask=None) takes x: float [T, V, 3, H, W] in [0, 1] (T frames, V cameras) and returns the
|
| 4 |
+
augmented tensor. Each call is one training sample: one of the 15 effects is chosen uniformly and its magnitude
|
| 5 |
+
is scaled by 1.5 around identity; the rest stay at x1. Colour is shared across cameras and time; geometry, light
|
| 6 |
+
patches and white balance are drawn per camera and fixed over time; sensor/optics effects (noise, gaussian blur,
|
| 7 |
+
JPEG, motion blur, sharpen, defocus) apply only to observation frames (obs_mask True).
|
| 8 |
+
Pixel-size parameters are defined at 224 px and scaled by H / 224.
|
| 9 |
+
perturb_joints(q, rng) applies the proprio spec: per-joint bias +-2 deg plus N(0, 0.25 deg) clipped at 3 sigma.
|
| 10 |
+
"""
|
| 11 |
+
import io
|
| 12 |
+
import math
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from PIL import Image
|
| 18 |
+
from torchvision.transforms import InterpolationMode
|
| 19 |
+
from torchvision.transforms import functional as TF
|
| 20 |
+
|
| 21 |
+
EFFECTS = ("exposure", "contrast", "saturation", "hue", "gamma", "temperature", "rotation_crop", "noise",
|
| 22 |
+
"gaussian_blur", "jpeg", "motion_blur", "sharpen", "defocus", "light_shadow", "wb_channel")
|
| 23 |
+
CENTER = {"exposure": 0.0, "contrast": 1.0, "saturation": 1.0, "hue": 0.0, "gamma": 1.0, "temperature": 0.0}
|
| 24 |
+
RANGE = {"exposure": (-0.22, 0.22), "contrast": (0.88, 1.12), "saturation": (0.76, 1.24), "hue": (-0.015, 0.015),
|
| 25 |
+
"gamma": (0.89, 1.11), "temperature": (-0.066, 0.066)}
|
| 26 |
+
VERSION = "hb-aug-v2-20261109"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def sample(rng, views, height):
|
| 30 |
+
boost = EFFECTS[int(rng.integers(len(EFFECTS)))]
|
| 31 |
+
|
| 32 |
+
def k(name):
|
| 33 |
+
return 1.5 if boost == name else 1.0
|
| 34 |
+
|
| 35 |
+
px = height / 224.0
|
| 36 |
+
p = {"boost": boost}
|
| 37 |
+
for name, (a, b) in RANGE.items():
|
| 38 |
+
c = CENTER[name]
|
| 39 |
+
p[name] = float(rng.uniform(c + k(name) * (a - c), c + k(name) * (b - c)))
|
| 40 |
+
p["geometry"] = []
|
| 41 |
+
for _ in range(views):
|
| 42 |
+
angle = float(rng.uniform(-1.5 * k("rotation_crop"), 1.5 * k("rotation_crop")))
|
| 43 |
+
p["geometry"].append((angle, 1.0 + abs(angle) * 0.015 + float(rng.uniform(0, 0.02 * k("rotation_crop")))))
|
| 44 |
+
p["noise"] = 4.0 / 255.0 * k("noise") if rng.random() < 0.5 else 0.0
|
| 45 |
+
p["gaussian_blur"] = float(rng.uniform(0.25, 0.65)) * k("gaussian_blur") if rng.random() < 0.15 else 0.0
|
| 46 |
+
if rng.random() < 0.35:
|
| 47 |
+
lo = int(round(100 - k("jpeg") * 30))
|
| 48 |
+
hi = int(round(100 - k("jpeg") * 5))
|
| 49 |
+
p["jpeg"] = int(rng.integers(lo, hi + 1))
|
| 50 |
+
else:
|
| 51 |
+
p["jpeg"] = None
|
| 52 |
+
p["motion_blur"] = ((float(rng.uniform(3, 7)) * k("motion_blur") * px, float(rng.uniform(0, 180)))
|
| 53 |
+
if rng.random() < 0.15 else None)
|
| 54 |
+
p["sharpen"] = float(rng.uniform(0.2, 0.6)) * k("sharpen") if rng.random() < 0.15 else 0.0
|
| 55 |
+
p["defocus"] = float(rng.uniform(1, 2)) * k("defocus") * px if rng.random() < 0.10 else 0.0
|
| 56 |
+
if rng.random() < 0.30:
|
| 57 |
+
g = 0.30 * k("light_shadow")
|
| 58 |
+
p["light_shadow"] = [[(float(rng.uniform(0.1, 0.9)), float(rng.uniform(0.1, 0.9)),
|
| 59 |
+
float(rng.uniform(0.15, 0.35)), float(rng.uniform(1 - g, 1 + g)))
|
| 60 |
+
for _ in range(int(rng.integers(1, 3)))] for _ in range(views)]
|
| 61 |
+
else:
|
| 62 |
+
p["light_shadow"] = None
|
| 63 |
+
if rng.random() < 0.30:
|
| 64 |
+
g = 0.10 * k("wb_channel")
|
| 65 |
+
gains = []
|
| 66 |
+
for _ in range(views):
|
| 67 |
+
row = [1.0, 1.0, 1.0]
|
| 68 |
+
row[int(rng.integers(0, 3))] = float(rng.uniform(1 - g, 1 + g))
|
| 69 |
+
gains.append(row)
|
| 70 |
+
p["wb_channel"] = gains
|
| 71 |
+
else:
|
| 72 |
+
p["wb_channel"] = None
|
| 73 |
+
return p
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _conv(x, kernel):
|
| 77 |
+
n, c, h, w = x.shape
|
| 78 |
+
k = kernel.to(x)[None, None].repeat(c, 1, 1, 1)
|
| 79 |
+
pad = kernel.shape[-1] // 2
|
| 80 |
+
return F.conv2d(F.pad(x, (pad, pad, pad, pad), mode="reflect"), k, groups=c)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def motion_kernel(length, angle_deg):
|
| 84 |
+
size = max(3, int(round(length)) | 1)
|
| 85 |
+
k = torch.zeros(size, size)
|
| 86 |
+
c = size // 2
|
| 87 |
+
th = math.radians(angle_deg)
|
| 88 |
+
for s in np.linspace(-(length - 1) / 2, (length - 1) / 2, 8 * size):
|
| 89 |
+
k[int(round(c - s * math.sin(th))), int(round(c + s * math.cos(th)))] = 1.0
|
| 90 |
+
return k / k.sum()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def disk_kernel(r):
|
| 94 |
+
size = 2 * int(math.ceil(r)) + 1
|
| 95 |
+
c = size // 2
|
| 96 |
+
yy, xx = torch.meshgrid(torch.arange(size), torch.arange(size), indexing="ij")
|
| 97 |
+
k = (((yy - c) ** 2 + (xx - c) ** 2) <= (r + 0.5) ** 2).float()
|
| 98 |
+
return k / k.sum()
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def apply(x, p, obs_mask=None, noise_seed=0):
|
| 102 |
+
t, v, c, h, w = x.shape
|
| 103 |
+
x = x.clone().clamp(0, 1)
|
| 104 |
+
for vi, (angle, scale) in enumerate(p["geometry"]):
|
| 105 |
+
x[:, vi] = TF.affine(x[:, vi], angle=angle, translate=[0, 0], scale=scale, shear=[0, 0],
|
| 106 |
+
interpolation=InterpolationMode.BILINEAR)
|
| 107 |
+
if p["light_shadow"] is not None:
|
| 108 |
+
yy, xx = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij")
|
| 109 |
+
for vi, blobs in enumerate(p["light_shadow"]):
|
| 110 |
+
m = torch.ones(h, w)
|
| 111 |
+
for cx, cy, s, g in blobs:
|
| 112 |
+
m = m + (g - 1.0) * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * s * s))
|
| 113 |
+
x[:, vi] = (x[:, vi] * m.clamp(min=0).to(x)).clamp(0, 1)
|
| 114 |
+
if p["wb_channel"] is not None:
|
| 115 |
+
for vi, g in enumerate(p["wb_channel"]):
|
| 116 |
+
x[:, vi] = (x[:, vi] * torch.tensor(g, dtype=x.dtype).view(1, 3, 1, 1)).clamp(0, 1)
|
| 117 |
+
y = x.reshape(t * v, c, h, w)
|
| 118 |
+
y = (y * float(2.0 ** p["exposure"])).clamp(0, 1)
|
| 119 |
+
d = p["temperature"]
|
| 120 |
+
y = (y * y.new_tensor([1.0 + d, 1.0, 1.0 - d]).view(1, 3, 1, 1)).clamp(0, 1)
|
| 121 |
+
y = ((y - 0.5) * p["contrast"] + 0.5).clamp(0, 1)
|
| 122 |
+
y = TF.adjust_saturation(y, p["saturation"])
|
| 123 |
+
y = TF.adjust_hue(y, p["hue"])
|
| 124 |
+
y = y.clamp(0, 1).pow(p["gamma"])
|
| 125 |
+
x = y.reshape(t, v, c, h, w)
|
| 126 |
+
mask = torch.ones(t, dtype=torch.bool) if obs_mask is None else torch.as_tensor(obs_mask, dtype=torch.bool)
|
| 127 |
+
if bool(mask.any()):
|
| 128 |
+
o = x[mask].reshape(-1, c, h, w)
|
| 129 |
+
if p["gaussian_blur"] > 0:
|
| 130 |
+
o = TF.gaussian_blur(o, [3, 3], [p["gaussian_blur"]] * 2)
|
| 131 |
+
if p["motion_blur"] is not None:
|
| 132 |
+
o = _conv(o, motion_kernel(*p["motion_blur"]))
|
| 133 |
+
if p["defocus"] > 0:
|
| 134 |
+
o = _conv(o, disk_kernel(p["defocus"]))
|
| 135 |
+
if p["sharpen"] > 0:
|
| 136 |
+
o = o + p["sharpen"] * (o - TF.gaussian_blur(o, [5, 5], [1.0, 1.0]))
|
| 137 |
+
o = o.clamp(0, 1)
|
| 138 |
+
if p["noise"] > 0:
|
| 139 |
+
gen = torch.Generator().manual_seed(int(noise_seed))
|
| 140 |
+
o = (o + torch.randn(o.shape, generator=gen, dtype=o.dtype) * p["noise"]).clamp(0, 1)
|
| 141 |
+
if p["jpeg"] is not None:
|
| 142 |
+
for i in range(o.shape[0]):
|
| 143 |
+
buf = io.BytesIO()
|
| 144 |
+
TF.to_pil_image(o[i]).save(buf, format="JPEG", quality=int(p["jpeg"]))
|
| 145 |
+
buf.seek(0)
|
| 146 |
+
with Image.open(buf) as im:
|
| 147 |
+
o[i] = TF.pil_to_tensor(im.convert("RGB")).to(o.dtype).div(255.0)
|
| 148 |
+
x[mask] = o.reshape(-1, v, c, h, w)
|
| 149 |
+
return x.clamp(0, 1)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def augment(x, rng, obs_mask=None):
|
| 153 |
+
p = sample(rng, x.shape[1], x.shape[-2])
|
| 154 |
+
return apply(x, p, obs_mask, noise_seed=int(rng.integers(0, 2 ** 62))), p
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def perturb_joints(q, rng):
|
| 158 |
+
q = np.asarray(q, dtype=np.float64)
|
| 159 |
+
d = q.shape[-1]
|
| 160 |
+
bias = rng.uniform(-math.radians(2.0), math.radians(2.0), size=d)
|
| 161 |
+
sigma = math.radians(0.25)
|
| 162 |
+
noise = np.clip(rng.normal(0.0, sigma, size=q.shape), -3 * sigma, 3 * sigma)
|
| 163 |
+
return (q + bias + noise).astype(np.float32)
|
b8-table-wipe/b512/step-002000/code/hb_eval_common.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Humanoid Benchmark open-loop action-reconstruction evaluation: shared task table, cache layout and metrics.
|
| 2 |
+
|
| 3 |
+
Protocol (identical for every model; mirrors Isaac-GR00T gr00t/eval/open_loop_eval.py):
|
| 4 |
+
* query frames t = 0, 16, 32, ... over the whole episode (non-overlapping 16-step chunks);
|
| 5 |
+
* the model sees the observation at t exactly as at deployment (cameras, raw benchmark state 44/78, instruction;
|
| 6 |
+
Delta Flow also its native 16-step past-action history), all augmentation off;
|
| 7 |
+
* the predicted chunk (first 16 steps, raw action units) is compared with the recorded actions t..t+15;
|
| 8 |
+
rows beyond the episode end are excluded (no clamped / padded targets);
|
| 9 |
+
* metrics: raw MSE / MAE, and nMSE = MSE after dividing every action dim by its std over the task's TRAINING
|
| 10 |
+
frames (floored, see STD_FLOOR_FRAC), overall, per action group and per horizon step.
|
| 11 |
+
Splits: "heldout" = recorded episodes that were never trained on (B6 100-148, trolley 100-102, B7 100-106);
|
| 12 |
+
"train10" = 10 fixed, evenly spaced training episodes (fit on seen data, NOT generalisation).
|
| 13 |
+
Only numpy here: imported by every model environment.
|
| 14 |
+
"""
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import time
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
|
| 22 |
+
G = Path(os.environ.get("G", "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/"
|
| 23 |
+
"preflight-20261104T1648Z/policies-20261109"))
|
| 24 |
+
P = G.parent
|
| 25 |
+
ROOT = G / "eval-action-mse"
|
| 26 |
+
CACHE = Path(os.environ.get("HB_EVAL_CACHE", ROOT / "cache"))
|
| 27 |
+
HORIZON = 16
|
| 28 |
+
STRIDE = 16
|
| 29 |
+
N_TRAIN_EVAL = 10
|
| 30 |
+
STD_FLOOR_FRAC = 0.05 # per-dim std floor = 5 % of the median std of the action vector
|
| 31 |
+
|
| 32 |
+
# task -> (source LeRobot v2.1 root, number of TRAINING episodes = fixed first N (None = all recorded), layout)
|
| 33 |
+
TASKS = {
|
| 34 |
+
"b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100, "g1"),
|
| 35 |
+
"basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None, "g1"),
|
| 36 |
+
"trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100, "g1"),
|
| 37 |
+
"b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100, "g1"),
|
| 38 |
+
"b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None, "g1"),
|
| 39 |
+
"a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None, "g1"),
|
| 40 |
+
"a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None, "g1"),
|
| 41 |
+
"a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None, "g1"),
|
| 42 |
+
"c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None, "g1"),
|
| 43 |
+
"a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None, "g1"),
|
| 44 |
+
"a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None, "g1"),
|
| 45 |
+
"a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None, "h2"),
|
| 46 |
+
"a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None, "h2"),
|
| 47 |
+
"a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None, "h2"),
|
| 48 |
+
# 2026-11-15: newer datasets (GR00T in-training eval needs a cache for every task it trains on)
|
| 49 |
+
"a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None, "g1"),
|
| 50 |
+
"a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None, "g1"),
|
| 51 |
+
"b1": (P / "b1-riceball-microwave-v1-36da298/data", None, "g1"),
|
| 52 |
+
"b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None, "g1"),
|
| 53 |
+
"b8": (P / "b8-table-wipe-v1-3b2e618/data", None, "g1"),
|
| 54 |
+
"basketv2": (P / "basket-trimv2-c4df4d4/data", None, "g1"),
|
| 55 |
+
"a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None, "h2p"), # action 82 = hb75 + SONIC planner 7
|
| 56 |
+
# 2026-11-15 evening: v6/v4 datasets (a3v3, a7v3 retired, not registered)
|
| 57 |
+
"b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None, "g1"), # 542 ep, all used (B6 first-100 rule is v5t7 only)
|
| 58 |
+
"b5": (P / "b5-chair-push-v1-b85b7e9/data", None, "g1"),
|
| 59 |
+
"a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None, "g1"),
|
| 60 |
+
"a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None, "g1"),
|
| 61 |
+
# 2026-11-16: B5 standing start (HF b5-chair-push-standing-start-expert-v1 @913f5a16, 100 ep; FastWAM b5ss)
|
| 62 |
+
"b5ss": (P / "b5-chair-push-standing-start-v1-913f5a1/data", None, "g1"),
|
| 63 |
+
# 2026-11-17: B7 black clothes trimmed (HF b7-black-clothes-hamper @949da46, 107 ep; train = first 100, heldout 100-106)
|
| 64 |
+
"b7t": (P / "b7-black-clothes-hamper-trim-949da46/data", 100, "g1"),
|
| 65 |
+
# 2026-11-15 night: H2 cropped v2 (GR00T h32 relay item 71); head+chest only -> build_cache --cams head,chest
|
| 66 |
+
"a18cr2": (P / "a18-pour-water-cropped-v2-44cdd57/data", None, "h2"), # HF a18-pour-water-expert-cropped-v2 @44cdd571
|
| 67 |
+
"a17cr2": (P / "a17-unscrew-bottle-cropped-v2-3520b70/data", None, "h2"), # HF a17-unscrew-bottle-expert-cropped-v2 @3520b70f
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def task_for_source(source):
|
| 72 |
+
"""benchmark task whose recording root is <source> (groot-data/<view>/hb-dataset-receipt.json "source")"""
|
| 73 |
+
src = Path(source).resolve()
|
| 74 |
+
hits = [t for t, (root, _, _) in TASKS.items() if Path(root).resolve() == src]
|
| 75 |
+
return hits[0] if hits else None
|
| 76 |
+
G1_BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint",
|
| 77 |
+
"left_ankle_pitch_joint", "left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint",
|
| 78 |
+
"right_hip_yaw_joint", "right_knee_joint", "right_ankle_pitch_joint", "right_ankle_roll_joint",
|
| 79 |
+
"waist_yaw_joint", "waist_roll_joint", "waist_pitch_joint", "left_shoulder_pitch_joint",
|
| 80 |
+
"left_shoulder_roll_joint", "left_shoulder_yaw_joint", "left_elbow_joint", "left_wrist_roll_joint",
|
| 81 |
+
"left_wrist_pitch_joint", "left_wrist_yaw_joint", "right_shoulder_pitch_joint",
|
| 82 |
+
"right_shoulder_roll_joint", "right_shoulder_yaw_joint", "right_elbow_joint", "right_wrist_roll_joint",
|
| 83 |
+
"right_wrist_pitch_joint", "right_wrist_yaw_joint"]
|
| 84 |
+
LAYOUTS = {
|
| 85 |
+
# state: body29 | inspire L6 | inspire R6 | projected_gravity3 ; action: motion_token64 | inspire L6 | R6 (same row)
|
| 86 |
+
"g1": dict(state_dim=44, action_dim=76, cams=("ego_view", "chest"),
|
| 87 |
+
groups={"motion_token": (0, 64), "hands": (64, 76)}),
|
| 88 |
+
# state: observation.state75 (legs12 waist3 arms14 head2 hands22+22) | gravity3 ; action: body_target31 | hands44
|
| 89 |
+
"h2": dict(state_dim=78, action_dim=75, cams=("head", "chest", "wrist"),
|
| 90 |
+
groups={"body_target": (0, 31), "hands": (31, 75)}),
|
| 91 |
+
# 2026-11-15 user: A14 v2 is trained on action82 = body_target31 | hands44 | action.planner7 (SONIC planner
|
| 92 |
+
# command); all 82 dims are scored (planner reported as its own group)
|
| 93 |
+
"h2p": dict(state_dim=78, action_dim=82, cams=("head", "chest"),
|
| 94 |
+
groups={"body_target": (0, 31), "hands": (31, 75), "planner": (75, 82)}),
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def jl(path):
|
| 99 |
+
return [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def layout(task):
|
| 103 |
+
return LAYOUTS[TASKS[task][2]]
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def split_episodes(task, split, n_recorded):
|
| 107 |
+
_, n_train, _ = TASKS[task]
|
| 108 |
+
n_train = n_recorded if n_train is None else n_train
|
| 109 |
+
if split == "heldout":
|
| 110 |
+
return list(range(n_train, n_recorded))
|
| 111 |
+
if split == "train10":
|
| 112 |
+
return sorted({int(round(x)) for x in np.linspace(0, n_train - 1, N_TRAIN_EVAL)})
|
| 113 |
+
if split.startswith("ep:"): # explicit list, e.g. ep:100,101
|
| 114 |
+
return [int(x) for x in split[3:].split(",")]
|
| 115 |
+
raise ValueError(f"unknown split {split!r}")
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def split_dir(task, split):
|
| 119 |
+
return CACHE / task / split
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def load_meta(task, split):
|
| 123 |
+
return json.loads((split_dir(task, split) / "meta.json").read_text())
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def load_stats(task):
|
| 127 |
+
z = np.load(CACHE / task / "stats.npz")
|
| 128 |
+
return {k: z[k] for k in z.files}
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def norm_std(stats):
|
| 132 |
+
std = stats["action_std"].astype(np.float64)
|
| 133 |
+
return np.maximum(std, STD_FLOOR_FRAC * float(np.median(std)))
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def iter_episodes(task, split, max_episodes=None):
|
| 137 |
+
meta = load_meta(task, split)
|
| 138 |
+
eps = meta["episodes"][: max_episodes or None]
|
| 139 |
+
for ep in eps:
|
| 140 |
+
with np.load(split_dir(task, split) / f"ep_{ep:06d}.npz") as z:
|
| 141 |
+
yield ep, {k: z[k] for k in z.files}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
class Accumulator:
|
| 145 |
+
"""Sums of squared / absolute errors; everything is computed in float64 over valid (row, dim) entries."""
|
| 146 |
+
|
| 147 |
+
def __init__(self, std, groups, horizon=HORIZON):
|
| 148 |
+
self.std = np.asarray(std, dtype=np.float64)
|
| 149 |
+
self.groups = groups
|
| 150 |
+
D = len(self.std)
|
| 151 |
+
self.sse_raw = np.zeros(D)
|
| 152 |
+
self.sae_raw = np.zeros(D)
|
| 153 |
+
self.sse_n = np.zeros(D)
|
| 154 |
+
self.rows = 0
|
| 155 |
+
self.step_sse_n = np.zeros(horizon)
|
| 156 |
+
self.step_rows = np.zeros(horizon)
|
| 157 |
+
self.per_episode = {}
|
| 158 |
+
|
| 159 |
+
def add(self, ep, pred, gt, valid):
|
| 160 |
+
pred = np.asarray(pred, dtype=np.float64)
|
| 161 |
+
gt = np.asarray(gt, dtype=np.float64)
|
| 162 |
+
v = np.asarray(valid, dtype=bool)
|
| 163 |
+
if pred.shape != gt.shape:
|
| 164 |
+
raise ValueError(f"pred {pred.shape} vs gt {gt.shape}")
|
| 165 |
+
if not np.isfinite(pred[v]).all():
|
| 166 |
+
raise FloatingPointError("non-finite prediction")
|
| 167 |
+
e = (pred - gt)[v]
|
| 168 |
+
en = e / self.std
|
| 169 |
+
self.sse_raw += (e ** 2).sum(0)
|
| 170 |
+
self.sae_raw += np.abs(e).sum(0)
|
| 171 |
+
self.sse_n += (en ** 2).sum(0)
|
| 172 |
+
self.rows += int(v.sum())
|
| 173 |
+
k = np.nonzero(v)[0]
|
| 174 |
+
self.step_sse_n[k] += (en ** 2).sum(1)
|
| 175 |
+
self.step_rows[k] += 1
|
| 176 |
+
s = self.per_episode.setdefault(int(ep), [0.0, 0])
|
| 177 |
+
s[0] += float((en ** 2).sum())
|
| 178 |
+
s[1] += int(v.sum()) * len(self.std)
|
| 179 |
+
|
| 180 |
+
def result(self):
|
| 181 |
+
D = len(self.std)
|
| 182 |
+
n = max(self.rows, 1)
|
| 183 |
+
out = {"rows": self.rows, "mse_raw": float(self.sse_raw.sum() / (n * D)),
|
| 184 |
+
"mae_raw": float(self.sae_raw.sum() / (n * D)), "nmse": float(self.sse_n.sum() / (n * D)),
|
| 185 |
+
"groups": {}, "nmse_by_step": (self.step_sse_n / np.maximum(self.step_rows, 1) / D).tolist(),
|
| 186 |
+
"nmse_by_dim": (self.sse_n / n).tolist()}
|
| 187 |
+
for g, (a, b) in self.groups.items():
|
| 188 |
+
out["groups"][g] = {"mse_raw": float(self.sse_raw[a:b].sum() / (n * (b - a))),
|
| 189 |
+
"nmse": float(self.sse_n[a:b].sum() / (n * (b - a)))}
|
| 190 |
+
pe = {ep: s / max(c, 1) for ep, (s, c) in sorted(self.per_episode.items())}
|
| 191 |
+
out["nmse_per_episode"] = pe
|
| 192 |
+
vals = np.array(list(pe.values()))
|
| 193 |
+
if len(vals) > 1: # bootstrap-free spread: std error over episodes
|
| 194 |
+
out["nmse_episode_sem"] = float(vals.std(ddof=1) / np.sqrt(len(vals)))
|
| 195 |
+
return out
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def queries(T):
|
| 199 |
+
return list(range(0, T, STRIDE))
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def gt_chunk(action, t, horizon=HORIZON):
|
| 203 |
+
T = len(action)
|
| 204 |
+
idx = np.arange(t, t + horizon)
|
| 205 |
+
valid = idx < T
|
| 206 |
+
return action[np.minimum(idx, T - 1)], valid
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def run(adapter, task, split, out_path, max_episodes=None, max_queries=None, save_pred=True, log_every=50):
|
| 210 |
+
"""Evaluate one adapter on one cached split; writes <out_path> (json) and <out_path>.pred.npz."""
|
| 211 |
+
meta = load_meta(task, split)
|
| 212 |
+
lay = LAYOUTS[meta["layout"]]
|
| 213 |
+
stats = load_stats(task)
|
| 214 |
+
acc = Accumulator(norm_std(stats), lay["groups"])
|
| 215 |
+
preds, keys = [], []
|
| 216 |
+
t0 = time.time()
|
| 217 |
+
infer_s = 0.0
|
| 218 |
+
nq = 0
|
| 219 |
+
for ep, z in iter_episodes(task, split, max_episodes):
|
| 220 |
+
state, action = z["state"], z["action"]
|
| 221 |
+
qts = list(z["query_t"])
|
| 222 |
+
for qi, t in enumerate(qts):
|
| 223 |
+
obs = {"episode": ep, "t": int(t), "T": len(action), "state": state[t], "prompt": meta["prompts"][str(int(z["task_index"]))],
|
| 224 |
+
"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]},
|
| 225 |
+
"past_action": action[max(0, t - HORIZON):t],
|
| 226 |
+
"_gt_check": action[t:t + HORIZON]} # only for adapters' input-pipeline consistency checks
|
| 227 |
+
s = time.time()
|
| 228 |
+
pred = np.asarray(adapter.predict(obs), dtype=np.float64)
|
| 229 |
+
infer_s += time.time() - s
|
| 230 |
+
if pred.ndim != 2 or pred.shape[0] < HORIZON or pred.shape[1] != lay["action_dim"]:
|
| 231 |
+
raise ValueError(f"adapter returned {pred.shape}, need (>={HORIZON}, {lay['action_dim']})")
|
| 232 |
+
pred = pred[:HORIZON]
|
| 233 |
+
gt, valid = gt_chunk(action, t)
|
| 234 |
+
acc.add(ep, pred, gt, valid)
|
| 235 |
+
if save_pred:
|
| 236 |
+
preds.append(pred.astype(np.float32))
|
| 237 |
+
keys.append((ep, int(t)))
|
| 238 |
+
nq += 1
|
| 239 |
+
if log_every and nq % log_every == 0:
|
| 240 |
+
print(f"[eval] {task}/{split} {nq} queries, running nMSE {acc.result()['nmse']:.4f}, "
|
| 241 |
+
f"{infer_s / nq:.3f} s/query", flush=True)
|
| 242 |
+
if max_queries and nq >= max_queries:
|
| 243 |
+
break
|
| 244 |
+
if max_queries and nq >= max_queries:
|
| 245 |
+
break
|
| 246 |
+
res = acc.result()
|
| 247 |
+
res.update(task=task, split=split, layout=meta["layout"], episodes=sorted(acc.per_episode), queries=nq,
|
| 248 |
+
horizon=HORIZON, stride=STRIDE, std_floor_frac=STD_FLOOR_FRAC, wall_s=round(time.time() - t0, 1),
|
| 249 |
+
infer_s_per_query=round(infer_s / max(nq, 1), 4), adapter=adapter.info(),
|
| 250 |
+
baselines=meta.get("baselines", {}), cache_created=meta.get("created"),
|
| 251 |
+
partial=bool(max_episodes or max_queries), finished_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
| 252 |
+
out_path = Path(out_path)
|
| 253 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 254 |
+
tmp = out_path.with_suffix(".tmp")
|
| 255 |
+
tmp.write_text(json.dumps(res, indent=1))
|
| 256 |
+
tmp.replace(out_path)
|
| 257 |
+
if save_pred and preds:
|
| 258 |
+
np.savez_compressed(str(out_path) + ".pred.npz", pred=np.stack(preds).astype(np.float16),
|
| 259 |
+
keys=np.asarray(keys, dtype=np.int64))
|
| 260 |
+
return res
|
b8-table-wipe/b512/step-002000/code/hb_fastwam_aug.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-16: Humanoid Benchmark augmentation v2 (project CLAUDE.md) inside the native FastWAM data pipeline.
|
| 2 |
+
- Visual: hb_aug_v2.augment on every training sample's frames before the two cameras are concatenated; frame 0 is the
|
| 3 |
+
current observation (sensor / optics effects only there), the later frames are the future video targets (colour,
|
| 4 |
+
per-camera geometry, light patches and white balance shared with frame 0, as the spec requires).
|
| 5 |
+
- Proprio: hb_aug_v2.perturb_joints on the raw body joints (G1 29, H2 31 = body 29 + head 2) before normalization;
|
| 6 |
+
hands and gravity untouched; supervision labels untouched. Normalization statistics are computed on the raw data.
|
| 7 |
+
FastWAM's own image augmentation is only a resize (no colour jitter / crop), so nothing native is stacked.
|
| 8 |
+
Installed in the main process before the dataloaders fork; HB_FW_JOINTS sets the number of perturbed joints."""
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
sys.path.insert(0, os.environ.get("HB_AUG_DIR", ""))
|
| 17 |
+
import hb_aug_v2 as A # noqa: E402
|
| 18 |
+
|
| 19 |
+
NJ = int(os.environ.get("HB_FW_JOINTS", "29"))
|
| 20 |
+
_seen = {"img": False, "state": False}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _rng(idx=0):
|
| 24 |
+
return np.random.default_rng([os.getpid(), int(idx) & 0xFFFFFFFF, time.time_ns() & 0xFFFFFFFF])
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _train(proc):
|
| 28 |
+
v = getattr(proc, "is_train", False)
|
| 29 |
+
return bool(v() if callable(v) else v)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def install():
|
| 33 |
+
from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset
|
| 34 |
+
from fastwam.datasets.lerobot.processors.fastwam_processor import FastWAMProcessor
|
| 35 |
+
|
| 36 |
+
pre0 = FastWAMProcessor.preprocess
|
| 37 |
+
|
| 38 |
+
def preprocess(self, data):
|
| 39 |
+
if _train(self) and isinstance(data.get("state"), dict) and "default" in data["state"]:
|
| 40 |
+
st = data["state"]["default"]
|
| 41 |
+
q = st[..., :NJ].detach().cpu().numpy()
|
| 42 |
+
st = st.clone()
|
| 43 |
+
st[..., :NJ] = torch.from_numpy(A.perturb_joints(q, _rng())).to(st.dtype)
|
| 44 |
+
data["state"]["default"] = st
|
| 45 |
+
if not _seen["state"]:
|
| 46 |
+
_seen["state"] = True
|
| 47 |
+
print(f"HB_FW_AUG proprio perturb_joints on state[..., :{NJ}] shape {tuple(st.shape)}", flush=True)
|
| 48 |
+
return pre0(self, data)
|
| 49 |
+
|
| 50 |
+
get0 = BaseLerobotDataset.__getitem__
|
| 51 |
+
|
| 52 |
+
def getitem(self, idx):
|
| 53 |
+
s = get0(self, idx)
|
| 54 |
+
if self.processor is not None and _train(self.processor) and "pixel_values" in s:
|
| 55 |
+
pv = s["pixel_values"]
|
| 56 |
+
five = pv.ndim == 5
|
| 57 |
+
x = (pv if five else pv[None]).permute(1, 0, 2, 3, 4).float() # [T, V, C, H, W]
|
| 58 |
+
hi = float(x.max())
|
| 59 |
+
scale = 255.0 if hi > 1.5 else 1.0
|
| 60 |
+
mask = torch.zeros(x.shape[0], dtype=torch.bool)
|
| 61 |
+
mask[0] = True
|
| 62 |
+
y, p = A.augment((x / scale).clamp(0, 1), _rng(idx), mask)
|
| 63 |
+
y = (y * scale).permute(1, 0, 2, 3, 4)
|
| 64 |
+
s["pixel_values"] = (y if five else y[0]).to(pv.dtype)
|
| 65 |
+
if not _seen["img"]:
|
| 66 |
+
_seen["img"] = True
|
| 67 |
+
print(f"HB_FW_AUG visual {A.VERSION} on pixel_values {tuple(pv.shape)} dtype {pv.dtype} max {hi:.3f} "
|
| 68 |
+
f"(T={x.shape[0]}, obs frame 0) boost={p['boost']}", flush=True)
|
| 69 |
+
return s
|
| 70 |
+
|
| 71 |
+
FastWAMProcessor.preprocess = preprocess
|
| 72 |
+
BaseLerobotDataset.__getitem__ = getitem
|
| 73 |
+
print(f"HB_FW_AUG installed: {A.VERSION}, joints {NJ}", flush=True)
|
b8-table-wipe/b512/step-002000/code/hb_fastwam_convert.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-16: LeRobot v2.1 copy of one of our datasets for FastWAM on OUR benchmark interface (CLAUDE.md section 4):
|
| 2 |
+
g1 : state44 = observation.state[0:29] (29 body joints) + measured Inspire 6 + 6 + projected gravity 3;
|
| 3 |
+
action76 = action.motion_token 64 + measured Inspire 6 + 6 (same-row labels, as GR00T / Psi-Zero / Delta Flow)
|
| 4 |
+
h2 : state78 = observation.state[0:75] + projected gravity 3; action75 = action.body_target 31 + action.hands 44
|
| 5 |
+
h2p: as h2 with action82 = action75 + action.planner 7 (A14 v2)
|
| 6 |
+
HumanWAM's fastwam_sonic_data.py expects a 43-D G1 state and takes state[0:22] + state[29:36]; on our 41-D state that
|
| 7 |
+
would put left-hand values into the right arm, so it is not used. Source files are never modified; media are symlinked.
|
| 8 |
+
--episodes N keeps the first N episodes (B7: first 100 of 107, as for every other model).
|
| 9 |
+
Usage: hb_fastwam_convert.py <profile> <source data dir> <fresh output dir> [--episodes N]"""
|
| 10 |
+
import copy, json, shutil, sys, tempfile
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import numpy as np
|
| 13 |
+
import pyarrow as pa
|
| 14 |
+
import pyarrow.parquet as pq
|
| 15 |
+
|
| 16 |
+
G1 = {"observation.state": 41, "action.motion_token": 64, "observation.inspire_left_hand_state": 6,
|
| 17 |
+
"observation.inspire_right_hand_state": 6, "observation.projected_gravity": 3}
|
| 18 |
+
H2 = {"observation.state": 75, "observation.projected_gravity": 3, "action.body_target": 31, "action.hands": 44}
|
| 19 |
+
H2P = dict(H2, **{"action.planner": 7})
|
| 20 |
+
SPEC = {"g1": (G1, ("ego_view", "chest")), "h2": (H2, ("head", "chest")), "h2p": (H2P, ("head", "chest"))}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def stats(v):
|
| 24 |
+
v = np.asarray(v, dtype=np.float64)
|
| 25 |
+
return {"min": v.min(0).tolist(), "max": v.max(0).tolist(), "mean": v.mean(0).tolist(), "std": v.std(0).tolist(), "count": [len(v)]}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def vec(table, key, values):
|
| 29 |
+
arr = pa.array(values.tolist(), type=pa.list_(pa.float32(), values.shape[1]))
|
| 30 |
+
i = table.schema.get_field_index(key)
|
| 31 |
+
return table.append_column(key, arr) if i < 0 else table.set_column(i, key, arr)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def map_rows(profile, c):
|
| 35 |
+
if profile == "g1":
|
| 36 |
+
L, R = c["observation.inspire_left_hand_state"], c["observation.inspire_right_hand_state"]
|
| 37 |
+
dev = float(np.abs(c["observation.state"][:, 29:41] - np.concatenate([L, R], 1)).max())
|
| 38 |
+
assert dev < 1e-4, f"observation.state[29:41] differs from the Inspire columns by {dev}"
|
| 39 |
+
return (np.concatenate([c["action.motion_token"], L, R], 1),
|
| 40 |
+
np.concatenate([c["observation.state"][:, :29], L, R, c["observation.projected_gravity"]], 1))
|
| 41 |
+
a = [c["action.body_target"], c["action.hands"]] + ([c["action.planner"]] if profile == "h2p" else [])
|
| 42 |
+
return np.concatenate(a, 1), np.concatenate([c["observation.state"], c["observation.projected_gravity"]], 1)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def main(profile, src, out, limit=None):
|
| 46 |
+
raw, cams = SPEC[profile]
|
| 47 |
+
src, out = Path(src).resolve(strict=True), Path(out).resolve()
|
| 48 |
+
if out.exists():
|
| 49 |
+
raise SystemExit(f"{out} exists")
|
| 50 |
+
info = json.loads((src / "meta/info.json").read_text())
|
| 51 |
+
assert info.get("codebase_version") == "v2.1", info.get("codebase_version")
|
| 52 |
+
for k, w in raw.items():
|
| 53 |
+
assert info["features"][k]["shape"] == [w], (k, info["features"][k]["shape"])
|
| 54 |
+
for c in cams:
|
| 55 |
+
assert info["features"]["observation.images." + c]["dtype"] in ("video", "image"), c
|
| 56 |
+
eps = [json.loads(l) for l in (src / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
|
| 57 |
+
assert [e["episode_index"] for e in eps] == list(range(len(eps)))
|
| 58 |
+
if limit is not None:
|
| 59 |
+
eps = eps[:limit]
|
| 60 |
+
old = {r["episode_index"]: r for r in (json.loads(l) for l in (src / "meta/episodes_stats.jsonl").read_text().splitlines() if l.strip())}
|
| 61 |
+
out.parent.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
stg = Path(tempfile.mkdtemp(prefix=".hbfw-", dir=out.parent))
|
| 63 |
+
shutil.copytree(src / "meta", stg / "meta")
|
| 64 |
+
for n in ("videos", "images"):
|
| 65 |
+
if (src / n).exists():
|
| 66 |
+
(stg / n).symlink_to(src / n, target_is_directory=True)
|
| 67 |
+
new, total = [], 0
|
| 68 |
+
for e in eps:
|
| 69 |
+
i = e["episode_index"]
|
| 70 |
+
rel = Path(info["data_path"].format(episode_chunk=i // info["chunks_size"], episode_index=i))
|
| 71 |
+
t = pq.read_table(src / rel)
|
| 72 |
+
c = {k: np.asarray(t[k].to_pylist(), dtype=np.float32) for k in raw}
|
| 73 |
+
act, st = map_rows(profile, c)
|
| 74 |
+
assert len(act) == e["length"] and np.isfinite(act).all() and np.isfinite(st).all()
|
| 75 |
+
total += len(act)
|
| 76 |
+
t = vec(vec(t, "action", act), "observation.state", st)
|
| 77 |
+
md = dict(t.schema.metadata or {})
|
| 78 |
+
if b"huggingface" in md:
|
| 79 |
+
hf = json.loads(md[b"huggingface"])
|
| 80 |
+
for k, w in (("action", act.shape[1]), ("observation.state", st.shape[1])):
|
| 81 |
+
hf["info"]["features"][k] = {"feature": {"dtype": "float32", "_type": "Value"}, "length": w, "_type": "Sequence"}
|
| 82 |
+
md[b"huggingface"] = json.dumps(hf).encode()
|
| 83 |
+
t = t.replace_schema_metadata(md)
|
| 84 |
+
(stg / rel).parent.mkdir(parents=True, exist_ok=True)
|
| 85 |
+
pq.write_table(t, stg / rel)
|
| 86 |
+
r = copy.deepcopy(old[i])
|
| 87 |
+
r["stats"]["action"], r["stats"]["observation.state"] = stats(act), stats(st)
|
| 88 |
+
new.append(r)
|
| 89 |
+
if limit is None:
|
| 90 |
+
assert total == info["total_frames"], (total, info["total_frames"])
|
| 91 |
+
else:
|
| 92 |
+
info["total_episodes"], info["total_frames"] = len(eps), total
|
| 93 |
+
info["total_videos"] = len(eps) * len(cams) if "total_videos" in info else info.get("total_videos")
|
| 94 |
+
info["splits"] = {"train": f"0:{len(eps)}"}
|
| 95 |
+
(stg / "meta/episodes.jsonl").write_text("".join(json.dumps(e) + "\n" for e in eps))
|
| 96 |
+
info["features"]["action"] = {"dtype": "float32", "shape": [int(act.shape[1])], "names": None}
|
| 97 |
+
info["features"]["observation.state"] = {"dtype": "float32", "shape": [int(st.shape[1])], "names": None}
|
| 98 |
+
drop = [k for k, v in info["features"].items() if v["dtype"] in ("video", "image") and k not in ["observation.images." + c for c in cams]]
|
| 99 |
+
for k in drop: # 2-camera rule: unused cameras are not downloaded, FastWAM checks every listed video file
|
| 100 |
+
del info["features"][k]
|
| 101 |
+
for r in new:
|
| 102 |
+
r["stats"].pop(k, None)
|
| 103 |
+
if drop and info.get("total_videos"):
|
| 104 |
+
info["total_videos"] = len(eps) * len(cams)
|
| 105 |
+
(stg / "meta/info.json").write_text(json.dumps(info, indent=2) + "\n")
|
| 106 |
+
(stg / "meta/episodes_stats.jsonl").write_text("".join(json.dumps(r) + "\n" for r in new))
|
| 107 |
+
for s in (stg / "meta").glob("*stats*.json"):
|
| 108 |
+
s.unlink()
|
| 109 |
+
(stg / "meta/hb_fastwam_interface.json").write_text(json.dumps(dict(
|
| 110 |
+
profile=profile, source=str(src), cameras=list(cams), state_dim=int(st.shape[1]), action_dim=int(act.shape[1]),
|
| 111 |
+
episodes=len(eps), frames=total, fps=info["fps"], first_episodes_only=limit), indent=2) + "\n")
|
| 112 |
+
stg.rename(out)
|
| 113 |
+
print("HBFW_CONVERT_OK", profile, out, "episodes", len(eps), "frames", total, "state", st.shape[1], "action", act.shape[1], flush=True)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
if __name__ == "__main__":
|
| 117 |
+
a = sys.argv[1:]
|
| 118 |
+
lim = None
|
| 119 |
+
if "--episodes" in a:
|
| 120 |
+
j = a.index("--episodes"); lim = int(a[j + 1]); del a[j:j + 2]
|
| 121 |
+
main(a[0], a[1], a[2], lim)
|
b8-table-wipe/b512/step-002000/code/hb_fastwam_train.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-16: native FastWAM Gaussian-FM training on our benchmark interface, with the HB augmentation v2.
|
| 2 |
+
Profiles (HB_FW_PROFILE): g1 = state44 / action76 / cameras ego_view + chest (HumanWAM fastwam_sonic recipe);
|
| 3 |
+
h2 = state78 / action75 / head + chest; h2p = state78 / action82 (+ planner 7) / head + chest. Fixed recipe of HumanWAM
|
| 4 |
+
fastwam_sonic_training.compose_sonic_config: 17 raw samples = 16 actions + 5 video frames (stride 4), 224 x 448 side by
|
| 5 |
+
side, min/max statistics, absolute labels, action scheduler shift 1.0, original Wan files.
|
| 6 |
+
Arguments: key=value Hydra overrides (JSON values parsed). Env: FASTWAM_SRC, DIFFSYNTH_MODEL_BASE_PATH, HB_AUG_DIR.
|
| 7 |
+
HB_MODEL_YAML=<path> writes the resolved model config; HB_COMPOSE_ONLY=1 prints the config; HB_DATA_SAMPLE=1 reads one
|
| 8 |
+
augmented training sample on CPU; HB_FW_AUG=0 disables the augmentation (benchmark only)."""
|
| 9 |
+
import json, os, sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
ROOT = Path(os.environ["FASTWAM_SRC"]).resolve()
|
| 13 |
+
sys.path.insert(0, str(ROOT / "src"))
|
| 14 |
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
| 15 |
+
TASK = "libero_uncond_2cam224_1e-4"
|
| 16 |
+
PROFILES = {"g1": (("ego_view", "chest"), 44, 76, 29), "h2": (("head", "chest"), 78, 75, 31),
|
| 17 |
+
"h2p": (("head", "chest"), 78, 82, 31)}
|
| 18 |
+
CAMS, SDIM, ADIM, NJ = PROFILES[os.environ.get("HB_FW_PROFILE", "g1")]
|
| 19 |
+
os.environ.setdefault("HB_FW_JOINTS", str(NJ))
|
| 20 |
+
SHAPE_META = {
|
| 21 |
+
"images": [{"key": n, "raw_shape": [3, 224, 224], "shape": [3, 224, 224]} for n in CAMS],
|
| 22 |
+
"action": [{"key": "default", "raw_shape": ADIM, "shape": ADIM}],
|
| 23 |
+
"state": [{"key": "default", "raw_shape": SDIM, "shape": SDIM}],
|
| 24 |
+
}
|
| 25 |
+
FIXED = {
|
| 26 |
+
"data.train.shape_meta": SHAPE_META, "data.train.num_frames": 17, "data.train.global_sample_stride": 1,
|
| 27 |
+
"data.train.action_video_freq_ratio": 4, "data.train.video_size": [224, 448], "data.train.concat_multi_camera": "horizontal",
|
| 28 |
+
"data.train.processor.action_output_dim": ADIM, "data.train.processor.proprio_output_dim": SDIM,
|
| 29 |
+
"data.train.processor.delta_action_dim_mask": {"default": [False] * ADIM}, "data.train.processor.action_state_transforms": None,
|
| 30 |
+
"data.train.processor.use_stepwise_action_norm": False, "data.train.processor.norm_default_mode": "min/max",
|
| 31 |
+
"data.train.processor.norm_exception_mode": None, "model.redirect_common_files": False,
|
| 32 |
+
"model.action_scheduler.train_shift": 1.0, "model.action_scheduler.infer_shift": 1.0,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def build(overrides):
|
| 37 |
+
from hydra import compose, initialize_config_dir
|
| 38 |
+
from omegaconf import OmegaConf
|
| 39 |
+
from fastwam.utils.config_resolvers import register_default_resolvers
|
| 40 |
+
register_default_resolvers()
|
| 41 |
+
with initialize_config_dir(config_dir=str(ROOT / "configs"), version_base="1.3"):
|
| 42 |
+
cfg = compose(config_name="train", overrides=["task=" + TASK])
|
| 43 |
+
for k, v in {**FIXED, **overrides}.items():
|
| 44 |
+
OmegaConf.update(cfg, k, v, merge=False)
|
| 45 |
+
for k, v in FIXED.items():
|
| 46 |
+
a = OmegaConf.select(cfg, k)
|
| 47 |
+
if OmegaConf.is_config(a):
|
| 48 |
+
a = OmegaConf.to_container(a, resolve=True)
|
| 49 |
+
assert a == v, (k, a, v)
|
| 50 |
+
return cfg
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def parse(argv):
|
| 54 |
+
out = {}
|
| 55 |
+
for a in argv:
|
| 56 |
+
k, v = a.split("=", 1)
|
| 57 |
+
try:
|
| 58 |
+
out[k] = json.loads(v)
|
| 59 |
+
except ValueError:
|
| 60 |
+
out[k] = v
|
| 61 |
+
return out
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
from omegaconf import OmegaConf
|
| 66 |
+
cfg = build(parse(sys.argv[1:]))
|
| 67 |
+
if os.environ.get("HB_MODEL_YAML"):
|
| 68 |
+
Path(os.environ["HB_MODEL_YAML"]).write_text(OmegaConf.to_yaml(cfg.model, resolve=True))
|
| 69 |
+
print("HBFW_MODEL_YAML", os.environ["HB_MODEL_YAML"], flush=True)
|
| 70 |
+
raise SystemExit(0)
|
| 71 |
+
if os.environ.get("HB_COMPOSE_ONLY"):
|
| 72 |
+
print(OmegaConf.to_yaml(cfg, resolve=True))
|
| 73 |
+
raise SystemExit(0)
|
| 74 |
+
import fastwam.runtime as native
|
| 75 |
+
assert Path(native.__file__).resolve() == ROOT / "src/fastwam/runtime.py", native.__file__
|
| 76 |
+
if os.environ.get("HB_FW_AUG", "1") != "0":
|
| 77 |
+
import hb_fastwam_aug
|
| 78 |
+
hb_fastwam_aug.install()
|
| 79 |
+
print("HBFW_RECIPE", json.dumps(dict(profile=os.environ.get("HB_FW_PROFILE", "g1"), cameras=CAMS, state_dim=SDIM,
|
| 80 |
+
action_dim=ADIM, perturbed_joints=int(os.environ["HB_FW_JOINTS"]),
|
| 81 |
+
aug=os.environ.get("HB_FW_AUG", "1") != "0",
|
| 82 |
+
batch_size=cfg.batch_size, max_steps=cfg.max_steps, lr=cfg.learning_rate)), flush=True)
|
| 83 |
+
if os.environ.get("HB_DATA_SAMPLE"): # CPU check: build the dataset and read augmented samples, no model
|
| 84 |
+
from omegaconf import open_dict
|
| 85 |
+
from fastwam.utils import misc
|
| 86 |
+
misc.register_work_dir(os.environ.get("HB_SAMPLE_WORKDIR", "/tmp/hbfw-sample"))
|
| 87 |
+
tr, _ = native.build_datasets(cfg.data)
|
| 88 |
+
s = tr[len(tr) // 2]
|
| 89 |
+
|
| 90 |
+
def desc(x):
|
| 91 |
+
if isinstance(x, dict):
|
| 92 |
+
return {k: desc(v) for k, v in x.items()}
|
| 93 |
+
if hasattr(x, "shape"):
|
| 94 |
+
return [tuple(x.shape), str(x.dtype), round(float(x.float().min()), 3), round(float(x.float().max()), 3)]
|
| 95 |
+
return type(x).__name__
|
| 96 |
+
print("HBFW_SAMPLE", len(tr), json.dumps(desc(s), default=str), flush=True)
|
| 97 |
+
raise SystemExit(0)
|
| 98 |
+
if os.environ.get("HB_INLOOP_EVERY", "250") != "0": # 2026-11-16: benchmark action nMSE every 250 steps
|
| 99 |
+
try:
|
| 100 |
+
import hb_inloop_fastwam
|
| 101 |
+
hb_inloop_fastwam.install()
|
| 102 |
+
except Exception as e: # noqa: BLE001 - never block training
|
| 103 |
+
print("HB_INLOOP fastwam install failed:", repr(e), flush=True)
|
| 104 |
+
native.run_training(cfg)
|
b8-table-wipe/b512/step-002000/code/hb_inloop_core.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared machinery of the in-training open-loop action MSE (Psi-Zero, Delta Flow; GR00T has its own tested copy in
|
| 2 |
+
hb_groot/hb_inloop_eval.py with the same protocol).
|
| 3 |
+
|
| 4 |
+
InLoop(model_name, task, split, out_jsonl, prepare, predict):
|
| 5 |
+
prepare(queries) -> optional per-model setup on this rank's queries (first eval only, inside the RNG fork)
|
| 6 |
+
predict(batch, seed) -> np.ndarray (B, >=16, >=Da) in raw action units; the first 16 steps and the first Da dims
|
| 7 |
+
(Da = benchmark action dim of the layout) are scored.
|
| 8 |
+
InLoop.evaluate(step, model): all train/eval modes of model.modules() and every RNG state (torch CPU + this CUDA device,
|
| 9 |
+
numpy, python) are restored afterwards; partial sums are all-reduced once (error flag included, so a failure on one
|
| 10 |
+
rank cannot leave the others waiting); rank 0 appends one json line per eval. Any failure disables later evals.
|
| 11 |
+
Protocol and metrics = hb_eval_common (post-hoc eval): queries every 16 frames, nMSE with the training-frame std.
|
| 12 |
+
"""
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import random
|
| 16 |
+
import sys
|
| 17 |
+
import time
|
| 18 |
+
import traceback
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
HERE = Path(__file__).resolve().parent
|
| 25 |
+
if str(HERE) not in sys.path:
|
| 26 |
+
sys.path.insert(0, str(HERE))
|
| 27 |
+
import hb_eval_common as E # noqa: E402
|
| 28 |
+
|
| 29 |
+
EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
|
| 30 |
+
BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
|
| 31 |
+
MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
|
| 32 |
+
SEED = 1234
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def dist_info():
|
| 36 |
+
import torch.distributed as dist
|
| 37 |
+
if dist.is_available() and dist.is_initialized():
|
| 38 |
+
return dist.get_rank(), dist.get_world_size()
|
| 39 |
+
return int(os.environ.get("RANK", "0")), int(os.environ.get("WORLD_SIZE", "1"))
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def pick_split(task):
|
| 43 |
+
split = os.environ.get("HB_INLOOP_SPLIT")
|
| 44 |
+
if not split:
|
| 45 |
+
split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
|
| 46 |
+
if not (E.split_dir(task, split) / "meta.json").exists():
|
| 47 |
+
raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
|
| 48 |
+
return split
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class InLoop:
|
| 52 |
+
def __init__(self, model_name, task, split, out_jsonl, predict, prepare=None, cams=None, info=None):
|
| 53 |
+
self.model_name, self.task, self.split = model_name, task, split
|
| 54 |
+
self.out = Path(out_jsonl)
|
| 55 |
+
self.predict, self.prepare = predict, prepare
|
| 56 |
+
self.cams = cams
|
| 57 |
+
self.info = info or {}
|
| 58 |
+
self.disabled = False
|
| 59 |
+
self.ready = False
|
| 60 |
+
self.done_steps = set()
|
| 61 |
+
|
| 62 |
+
def say(self, *a):
|
| 63 |
+
if dist_info()[0] == 0:
|
| 64 |
+
print("HB_INLOOP", *a, flush=True)
|
| 65 |
+
|
| 66 |
+
def _setup(self):
|
| 67 |
+
rank, world = dist_info()
|
| 68 |
+
meta = E.load_meta(self.task, self.split)
|
| 69 |
+
self.meta, self.lay = meta, E.LAYOUTS[meta["layout"]]
|
| 70 |
+
self.std = E.norm_std(E.load_stats(self.task))
|
| 71 |
+
cams = self.cams or meta["cams"]
|
| 72 |
+
miss = [c for c in cams if c not in meta["cams"]]
|
| 73 |
+
if miss:
|
| 74 |
+
raise KeyError(f"cameras {miss} not in the eval cache {meta['cams']}")
|
| 75 |
+
allq = []
|
| 76 |
+
for ep in meta["episodes"]:
|
| 77 |
+
with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
|
| 78 |
+
allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
|
| 79 |
+
if MAX_Q:
|
| 80 |
+
allq = allq[:MAX_Q]
|
| 81 |
+
self.n_total = len(allq)
|
| 82 |
+
mine = [q for i, q in enumerate(allq) if i % world == rank]
|
| 83 |
+
self.eps = list(meta["episodes"])
|
| 84 |
+
by_ep = {}
|
| 85 |
+
for ep, qi, t in mine:
|
| 86 |
+
by_ep.setdefault(ep, []).append((qi, t))
|
| 87 |
+
self.q = []
|
| 88 |
+
for ep, lst in by_ep.items():
|
| 89 |
+
with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
|
| 90 |
+
state, action = z["state"], z["action"]
|
| 91 |
+
prompt = meta["prompts"][str(int(z["task_index"]))]
|
| 92 |
+
imgs = {c: z[f"img_{c}"] for c in cams}
|
| 93 |
+
for qi, t in lst:
|
| 94 |
+
gt, valid = E.gt_chunk(action, t)
|
| 95 |
+
self.q.append(dict(episode=int(ep), t=int(t), T=len(action), state=state[t].copy(), prompt=prompt,
|
| 96 |
+
images={c: imgs[c][qi].copy() for c in cams}, gt=gt, valid=valid,
|
| 97 |
+
past_action=action[max(0, t - E.HORIZON):t].copy()))
|
| 98 |
+
if self.prepare:
|
| 99 |
+
self.prepare(self.q)
|
| 100 |
+
self.ready = True
|
| 101 |
+
self.say(f"{self.model_name} ready: task {self.task} split {self.split} queries {self.n_total} "
|
| 102 |
+
f"(rank {rank}/{world}: {len(self.q)}), every {EVERY} steps, batch {BATCH}, cams {cams}")
|
| 103 |
+
|
| 104 |
+
def evaluate(self, step, model):
|
| 105 |
+
if self.disabled or step in self.done_steps:
|
| 106 |
+
return
|
| 107 |
+
self.done_steps.add(step)
|
| 108 |
+
rank, world = dist_info()
|
| 109 |
+
t0 = time.time()
|
| 110 |
+
err, msg = 0.0, ""
|
| 111 |
+
acc, nq, infer = None, 0, 0.0
|
| 112 |
+
modes = [(m, m.training) for m in model.modules()]
|
| 113 |
+
dev = torch.cuda.current_device() if torch.cuda.is_available() else None
|
| 114 |
+
np_state, py_state = np.random.get_state(), random.getstate()
|
| 115 |
+
try:
|
| 116 |
+
with torch.random.fork_rng(devices=[dev] if dev is not None else []):
|
| 117 |
+
if not self.ready:
|
| 118 |
+
self._setup()
|
| 119 |
+
D = self.lay["action_dim"]
|
| 120 |
+
model.eval()
|
| 121 |
+
acc = E.Accumulator(self.std, self.lay["groups"])
|
| 122 |
+
for bi in range(0, len(self.q), BATCH):
|
| 123 |
+
batch = self.q[bi:bi + BATCH]
|
| 124 |
+
seed = SEED + 1000 * rank + bi
|
| 125 |
+
torch.default_generator.manual_seed(seed)
|
| 126 |
+
if dev is not None:
|
| 127 |
+
torch.cuda.manual_seed(seed)
|
| 128 |
+
s = time.time()
|
| 129 |
+
with torch.no_grad():
|
| 130 |
+
pred = np.asarray(self.predict(batch, seed), dtype=np.float64)
|
| 131 |
+
infer += time.time() - s
|
| 132 |
+
if pred.ndim != 3 or pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] < D:
|
| 133 |
+
raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, >={D})")
|
| 134 |
+
for q, p in zip(batch, pred):
|
| 135 |
+
acc.add(q["episode"], p[:E.HORIZON, :D], q["gt"], q["valid"])
|
| 136 |
+
nq += 1
|
| 137 |
+
except Exception as e: # noqa: BLE001 - never stop training
|
| 138 |
+
err, msg = 1.0, f"{type(e).__name__}: {e}"
|
| 139 |
+
print(f"HB_INLOOP {self.model_name} rank {rank} step {step} FAILED: {msg}\n{traceback.format_exc()}",
|
| 140 |
+
file=sys.stderr, flush=True)
|
| 141 |
+
finally:
|
| 142 |
+
for m, tr in modes:
|
| 143 |
+
m.training = tr
|
| 144 |
+
np.random.set_state(np_state)
|
| 145 |
+
random.setstate(py_state)
|
| 146 |
+
D = self.lay["action_dim"] if self.ready else 0
|
| 147 |
+
H = E.HORIZON
|
| 148 |
+
eps = self.eps if self.ready else []
|
| 149 |
+
nE = len(eps)
|
| 150 |
+
vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
|
| 151 |
+
if acc is not None and not err:
|
| 152 |
+
o = 0
|
| 153 |
+
vec[o:o + D] = acc.sse_raw; o += D
|
| 154 |
+
vec[o:o + D] = acc.sae_raw; o += D
|
| 155 |
+
vec[o:o + D] = acc.sse_n; o += D
|
| 156 |
+
vec[o] = acc.rows; o += 1
|
| 157 |
+
vec[o:o + H] = acc.step_sse_n; o += H
|
| 158 |
+
vec[o:o + H] = acc.step_rows; o += H
|
| 159 |
+
for j, ep in enumerate(eps):
|
| 160 |
+
s_, c_ = acc.per_episode.get(int(ep), (0.0, 0))
|
| 161 |
+
vec[o + j], vec[o + nE + j] = s_, c_
|
| 162 |
+
o += 2 * nE
|
| 163 |
+
vec[o], vec[o + 1] = nq, infer
|
| 164 |
+
vec[-1] = err
|
| 165 |
+
tot = self._allreduce(vec)
|
| 166 |
+
if tot is None or tot[-1] > 0 or not self.ready:
|
| 167 |
+
self.disabled = True
|
| 168 |
+
nfail = None if tot is None else int(tot[-1])
|
| 169 |
+
self._write({"step": step, "error": msg or "failed on another rank / collective failed",
|
| 170 |
+
"ranks_failed": nfail})
|
| 171 |
+
self.say(f"{self.model_name} step {step}: eval failed ({nfail} rank(s)) -> in-training eval disabled: {msg}")
|
| 172 |
+
return
|
| 173 |
+
o = 0
|
| 174 |
+
acc = E.Accumulator(self.std, self.lay["groups"])
|
| 175 |
+
acc.sse_raw = tot[o:o + D]; o += D
|
| 176 |
+
acc.sae_raw = tot[o:o + D]; o += D
|
| 177 |
+
acc.sse_n = tot[o:o + D]; o += D
|
| 178 |
+
acc.rows = int(round(tot[o])); o += 1
|
| 179 |
+
acc.step_sse_n = tot[o:o + H]; o += H
|
| 180 |
+
acc.step_rows = tot[o:o + H]; o += H
|
| 181 |
+
acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
|
| 182 |
+
for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
|
| 183 |
+
o += 2 * nE
|
| 184 |
+
n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
|
| 185 |
+
res = acc.result()
|
| 186 |
+
res.pop("nmse_by_dim", None)
|
| 187 |
+
rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "model": self.model_name,
|
| 188 |
+
"task": self.task, "split": self.split, "queries": n_q, "ranks": world,
|
| 189 |
+
"wall_s": round(time.time() - t0, 1), "infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
|
| 190 |
+
"seed": SEED, **self.info, **res}
|
| 191 |
+
self._write(rec)
|
| 192 |
+
g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
|
| 193 |
+
self.say(f"{self.model_name} step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} "
|
| 194 |
+
f"queries, {rec['wall_s']} s")
|
| 195 |
+
|
| 196 |
+
@staticmethod
|
| 197 |
+
def _allreduce(vec):
|
| 198 |
+
import torch.distributed as dist
|
| 199 |
+
if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1:
|
| 200 |
+
return vec
|
| 201 |
+
try:
|
| 202 |
+
dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
|
| 203 |
+
n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
|
| 204 |
+
mx, mn = n.clone(), n.clone()
|
| 205 |
+
dist.all_reduce(mx, op=dist.ReduceOp.MAX)
|
| 206 |
+
dist.all_reduce(mn, op=dist.ReduceOp.MIN)
|
| 207 |
+
if mx.item() != mn.item():
|
| 208 |
+
return None
|
| 209 |
+
t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
|
| 210 |
+
dist.all_reduce(t)
|
| 211 |
+
return t.cpu().numpy()
|
| 212 |
+
except Exception as e: # noqa: BLE001
|
| 213 |
+
print(f"HB_INLOOP all_reduce failed: {e!r}", file=sys.stderr, flush=True)
|
| 214 |
+
return None
|
| 215 |
+
|
| 216 |
+
def _write(self, rec):
|
| 217 |
+
if dist_info()[0] != 0:
|
| 218 |
+
return
|
| 219 |
+
try:
|
| 220 |
+
self.out.parent.mkdir(parents=True, exist_ok=True)
|
| 221 |
+
with open(self.out, "a") as f:
|
| 222 |
+
f.write(json.dumps(rec) + "\n")
|
| 223 |
+
except Exception as e: # noqa: BLE001
|
| 224 |
+
print(f"HB_INLOOP write failed: {e!r}", flush=True)
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
class TempAttrs:
|
| 228 |
+
"""set attributes on an object for the duration of a block, then restore (or delete) them"""
|
| 229 |
+
_MISSING = object()
|
| 230 |
+
|
| 231 |
+
def __init__(self, obj, **kw):
|
| 232 |
+
self.obj, self.kw, self.old = obj, kw, {}
|
| 233 |
+
|
| 234 |
+
def __enter__(self):
|
| 235 |
+
for k, v in self.kw.items():
|
| 236 |
+
self.old[k] = self.obj.__dict__.get(k, self._MISSING)
|
| 237 |
+
setattr(self.obj, k, v)
|
| 238 |
+
return self.obj
|
| 239 |
+
|
| 240 |
+
def __exit__(self, *exc):
|
| 241 |
+
for k, v in self.old.items():
|
| 242 |
+
if v is self._MISSING:
|
| 243 |
+
try:
|
| 244 |
+
delattr(self.obj, k)
|
| 245 |
+
except AttributeError:
|
| 246 |
+
pass
|
| 247 |
+
else:
|
| 248 |
+
setattr(self.obj, k, v)
|
| 249 |
+
return False
|
b8-table-wipe/b512/step-002000/code/hb_inloop_fastwam.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastWAM in-training open-loop action MSE (2026-11-16 user: every 250 steps, like GR00T / Psi-Zero / Delta Flow).
|
| 2 |
+
|
| 3 |
+
install() is called by fastwam/tools/hb_fastwam_train.py right before native.run_training(cfg). It uses the trainer's
|
| 4 |
+
own eval slot: after Wan22Trainer.__init__ it sets eval_every = HB_INLOOP_EVERY (default 250) and a non-None
|
| 5 |
+
val_dataset marker, and replaces Wan22Trainer.evaluate (the native one generates a full video for one random val sample;
|
| 6 |
+
it is off in our runs, eval_every=0 on the command line). The native loop then calls evaluate() on every rank right after
|
| 7 |
+
the optimizer step of every 250th update, followed by accelerator.wait_for_everyone(); returning None logs nothing.
|
| 8 |
+
evaluate(): all ranks score their shard of the benchmark eval cache with the live model (accelerator.unwrap_model; ZeRO-1
|
| 9 |
+
keeps full bf16 weights on every rank) through EXACTLY the post-hoc adapter path (eval-action-mse/adapters/ad_fastwam.py
|
| 10 |
+
Inputs + infer: FastWAM.infer_action, the action-only deployment path; 10 denoising steps = config
|
| 11 |
+
eval_num_inference_steps; seed 42 per query; same cameras / proprio normalisation / T5 cache as training).
|
| 12 |
+
Results: <output_dir>/inloop-mse.jsonl (rank 0) + "HB_INLOOP" log lines. Training is untouched (module modes and every
|
| 13 |
+
RNG restored by hb_inloop_core, no gradients); any failure disables later evals on all ranks.
|
| 14 |
+
"""
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import sys
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
G = Path(__file__).resolve().parents[2] # .../policies-20261109 (this file: $G/fastwam/tools/)
|
| 21 |
+
EVAL = G / "eval-action-mse"
|
| 22 |
+
for p in (str(EVAL), str(EVAL / "adapters")):
|
| 23 |
+
if p not in sys.path:
|
| 24 |
+
sys.path.insert(0, p)
|
| 25 |
+
import numpy as np # noqa: E402
|
| 26 |
+
import hb_eval_common as E # noqa: E402
|
| 27 |
+
import hb_inloop_core as C # noqa: E402
|
| 28 |
+
import ad_fastwam as A # noqa: E402
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class _Marker:
|
| 32 |
+
"""non-None val_dataset so the native loop reaches evaluate(); never indexed"""
|
| 33 |
+
|
| 34 |
+
def __len__(self):
|
| 35 |
+
return 1
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class FwEval:
|
| 39 |
+
def __init__(self, trainer):
|
| 40 |
+
cfg = trainer.cfg
|
| 41 |
+
self.cfg = cfg
|
| 42 |
+
ddir = Path(list(cfg.data.train.dataset_dirs)[0])
|
| 43 |
+
iface = json.loads((ddir / "meta/hb_fastwam_interface.json").read_text())
|
| 44 |
+
task = E.task_for_source(iface["source"])
|
| 45 |
+
if task is None:
|
| 46 |
+
raise LookupError(f"no benchmark task for {iface['source']}")
|
| 47 |
+
split = C.pick_split(task)
|
| 48 |
+
self.steps = int(cfg.eval_num_inference_steps)
|
| 49 |
+
self.inp = None
|
| 50 |
+
self.model = None
|
| 51 |
+
out = Path(cfg.output_dir) / "inloop-mse.jsonl"
|
| 52 |
+
self.il = C.InLoop("fastwam", task, split, out, predict=self.predict, prepare=self.prepare,
|
| 53 |
+
info={"denoising_steps": self.steps, "seed_per_query": 42, "dataset": ddir.name,
|
| 54 |
+
"path": "FastWAM.infer_action (ad_fastwam)"})
|
| 55 |
+
self.il.cams = None
|
| 56 |
+
|
| 57 |
+
def prepare(self, queries):
|
| 58 |
+
self.inp = A.Inputs(self.cfg, Path(self.cfg.output_dir) / "dataset_stats.json")
|
| 59 |
+
miss = [c for c in self.inp.cams if c not in self.il.meta["cams"]]
|
| 60 |
+
if miss:
|
| 61 |
+
raise KeyError(f"cameras {miss} not in the eval cache {self.il.meta['cams']}")
|
| 62 |
+
for q in {q["prompt"] for q in queries}:
|
| 63 |
+
self.inp.context(q) # fail early if an instruction has no T5 cache
|
| 64 |
+
|
| 65 |
+
def predict(self, batch, seed):
|
| 66 |
+
return np.stack([np.asarray(A.infer(self.model, self.inp, q, self.steps, 42), dtype=np.float64) for q in batch])
|
| 67 |
+
|
| 68 |
+
def evaluate(self, step, model):
|
| 69 |
+
self.model = model
|
| 70 |
+
try:
|
| 71 |
+
self.il.evaluate(step, model)
|
| 72 |
+
finally:
|
| 73 |
+
self.model = None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class _Broken:
|
| 77 |
+
def __init__(self, err):
|
| 78 |
+
self.err = err
|
| 79 |
+
self.il = C.InLoop("fastwam", "b6", "train10", "/dev/null", predict=None, prepare=self._boom)
|
| 80 |
+
|
| 81 |
+
def _boom(self, _q):
|
| 82 |
+
raise RuntimeError(self.err)
|
| 83 |
+
|
| 84 |
+
def evaluate(self, step, model):
|
| 85 |
+
self.il.evaluate(step, model)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def install():
|
| 89 |
+
if C.EVERY <= 0:
|
| 90 |
+
return
|
| 91 |
+
from fastwam import trainer as T
|
| 92 |
+
WT = T.Wan22Trainer
|
| 93 |
+
orig_init = WT.__init__
|
| 94 |
+
|
| 95 |
+
def __init__(self, *a, **kw):
|
| 96 |
+
orig_init(self, *a, **kw)
|
| 97 |
+
# native eval off (eval_every 0 on our command line). NB runtime.build_datasets sets val_ds = train_ds when
|
| 98 |
+
# there is no data.val, so val_dataset is never None here (11-16 smoke: the old "val_dataset is None" test
|
| 99 |
+
# silently left the in-loop eval off).
|
| 100 |
+
if int(self.eval_every) == 0:
|
| 101 |
+
self.eval_every = C.EVERY
|
| 102 |
+
if self.val_dataset is None:
|
| 103 |
+
self.val_dataset = _Marker()
|
| 104 |
+
self._hb_inloop_on = True
|
| 105 |
+
else: # a native eval is configured: leave it alone
|
| 106 |
+
self._hb_inloop_on = False
|
| 107 |
+
|
| 108 |
+
def evaluate(self):
|
| 109 |
+
if not getattr(self, "_hb_inloop_on", False):
|
| 110 |
+
return orig_eval(self)
|
| 111 |
+
ev = getattr(self, "_hb_inloop", None)
|
| 112 |
+
if ev is None:
|
| 113 |
+
try:
|
| 114 |
+
ev = FwEval(self)
|
| 115 |
+
except Exception as e: # noqa: BLE001
|
| 116 |
+
ev = _Broken(f"{type(e).__name__}: {e}")
|
| 117 |
+
self._hb_inloop = ev
|
| 118 |
+
if not ev.il.disabled:
|
| 119 |
+
ev.evaluate(int(self.global_step), self.accelerator.unwrap_model(self.model))
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
orig_eval = WT.evaluate
|
| 123 |
+
WT.__init__ = __init__
|
| 124 |
+
WT.evaluate = evaluate
|
| 125 |
+
if int(os.environ.get("RANK", "0")) == 0:
|
| 126 |
+
print(f"HB_INLOOP fastwam installed: every {C.EVERY} steps", flush=True)
|
b8-table-wipe/b512/step-002000/code/test_fastwam_inputs.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU check (no model): the FastWAM adapter's inputs built from the model-agnostic eval cache == the inputs our FastWAM
|
| 2 |
+
training dataset (RobotVideoDataset of the run's config, no HB augmentation) builds for the same frame.
|
| 3 |
+
python test_fastwam_inputs.py <task> <run out dir> [n_queries]"""
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
HERE = Path(__file__).resolve().parent
|
| 12 |
+
sys.path.insert(0, str(HERE))
|
| 13 |
+
sys.path.insert(0, str(HERE / "adapters"))
|
| 14 |
+
import hb_eval_common as E # noqa: E402
|
| 15 |
+
import ad_fastwam as A # noqa: E402
|
| 16 |
+
|
| 17 |
+
task, out = sys.argv[1], Path(sys.argv[2])
|
| 18 |
+
nq = int(sys.argv[3]) if len(sys.argv) > 3 else 3
|
| 19 |
+
A.setup_env()
|
| 20 |
+
import torch # noqa: E402
|
| 21 |
+
from omegaconf import OmegaConf # noqa: E402
|
| 22 |
+
import fastwam.runtime as native # noqa: E402
|
| 23 |
+
from fastwam.utils import misc # noqa: E402
|
| 24 |
+
|
| 25 |
+
cfg = OmegaConf.load(out / "config.yaml")
|
| 26 |
+
inp = A.Inputs(cfg, out / "dataset_stats.json")
|
| 27 |
+
misc.register_work_dir("/tmp/hbfw-inputs-check")
|
| 28 |
+
OmegaConf.update(cfg, "data.train.pretrained_norm_stats", str(out / "dataset_stats.json"), merge=False, force_add=True)
|
| 29 |
+
tr, _ = native.build_datasets(cfg.data)
|
| 30 |
+
ddir = Path(cfg.data.train.dataset_dirs[0])
|
| 31 |
+
lens = [json.loads(l)["length"] for l in (ddir / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
|
| 32 |
+
meta = E.load_meta(task, "train10")
|
| 33 |
+
split = "train10"
|
| 34 |
+
worst = {}
|
| 35 |
+
done = 0
|
| 36 |
+
for ep, z in E.iter_episodes(task, split, None):
|
| 37 |
+
base = int(sum(lens[:ep]))
|
| 38 |
+
for qi, t in enumerate(list(z["query_t"])[1:1 + nq], start=1):
|
| 39 |
+
t = int(t)
|
| 40 |
+
s = tr[base + t]
|
| 41 |
+
obs = {"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]}, "state": z["state"][t],
|
| 42 |
+
"prompt": meta["prompts"][str(int(z["task_index"]))]}
|
| 43 |
+
img = inp.image(obs["images"])[0]
|
| 44 |
+
d_img = float((img - s["video"][:, 0].float()).abs().max())
|
| 45 |
+
pr = inp.proprio(obs["state"])[0]
|
| 46 |
+
d_pr = float((pr - s["proprio"][0].float()).abs().max())
|
| 47 |
+
ctx, mask = inp.context(obs["prompt"])
|
| 48 |
+
d_ctx = float((ctx.float() - s["context"].float()).abs().max())
|
| 49 |
+
d_mask = int((mask != s["context_mask"].bool()).sum())
|
| 50 |
+
gt = z["action"][t:t + 16]
|
| 51 |
+
a = inp.denorm(s["action"][:16])
|
| 52 |
+
d_act = float(np.abs(a[:len(gt)] - gt).max())
|
| 53 |
+
row = dict(ep=ep, t=t, idx=base + t, img=d_img, proprio=d_pr, ctx=d_ctx, mask=d_mask, act_denorm_vs_cache=d_act,
|
| 54 |
+
prompt_same=s["prompt"] == inp.DEFAULT_PROMPT.format(task=obs["prompt"]))
|
| 55 |
+
print("CHECK", json.dumps(row), flush=True)
|
| 56 |
+
for k in ("img", "proprio", "ctx", "act_denorm_vs_cache"):
|
| 57 |
+
worst[k] = max(worst.get(k, 0.0), row[k])
|
| 58 |
+
done += 1
|
| 59 |
+
if done >= 2 * nq:
|
| 60 |
+
break
|
| 61 |
+
print("WORST", json.dumps(worst))
|
b8-table-wipe/b512/step-002000/complete.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"task": "b8",
|
| 3 |
+
"profile": "g1",
|
| 4 |
+
"per_gpu_batch": 16,
|
| 5 |
+
"global_batch": 512,
|
| 6 |
+
"gradient_accumulation_steps": 1,
|
| 7 |
+
"max_steps": 4000,
|
| 8 |
+
"save_every": 2000,
|
| 9 |
+
"recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
|
| 10 |
+
"interface": {
|
| 11 |
+
"profile": "g1",
|
| 12 |
+
"source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
|
| 13 |
+
"cameras": [
|
| 14 |
+
"ego_view",
|
| 15 |
+
"chest"
|
| 16 |
+
],
|
| 17 |
+
"state_dim": 44,
|
| 18 |
+
"action_dim": 76,
|
| 19 |
+
"episodes": 100,
|
| 20 |
+
"frames": 61498,
|
| 21 |
+
"fps": 30,
|
| 22 |
+
"first_episodes_only": null
|
| 23 |
+
},
|
| 24 |
+
"finished_utc": "2026-11-17T21:03:31Z"
|
| 25 |
+
}
|
b8-table-wipe/b512/step-002000/dataset_meta/hb_fastwam_interface.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"profile": "g1",
|
| 3 |
+
"source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
|
| 4 |
+
"cameras": [
|
| 5 |
+
"ego_view",
|
| 6 |
+
"chest"
|
| 7 |
+
],
|
| 8 |
+
"state_dim": 44,
|
| 9 |
+
"action_dim": 76,
|
| 10 |
+
"episodes": 100,
|
| 11 |
+
"frames": 61498,
|
| 12 |
+
"fps": 30,
|
| 13 |
+
"first_episodes_only": null
|
| 14 |
+
}
|
b8-table-wipe/b512/step-002000/dataset_meta/info.json
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"codebase_version": "v2.1",
|
| 3 |
+
"robot_type": null,
|
| 4 |
+
"total_episodes": 100,
|
| 5 |
+
"total_frames": 61498,
|
| 6 |
+
"total_tasks": 1,
|
| 7 |
+
"total_videos": 200,
|
| 8 |
+
"total_chunks": 1,
|
| 9 |
+
"chunks_size": 1000,
|
| 10 |
+
"fps": 30,
|
| 11 |
+
"splits": {
|
| 12 |
+
"train": "0:100"
|
| 13 |
+
},
|
| 14 |
+
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
|
| 15 |
+
"video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
|
| 16 |
+
"features": {
|
| 17 |
+
"observation.state": {
|
| 18 |
+
"dtype": "float32",
|
| 19 |
+
"shape": [
|
| 20 |
+
44
|
| 21 |
+
],
|
| 22 |
+
"names": null
|
| 23 |
+
},
|
| 24 |
+
"observation.eef_state": {
|
| 25 |
+
"dtype": "float64",
|
| 26 |
+
"shape": [
|
| 27 |
+
14
|
| 28 |
+
],
|
| 29 |
+
"names": [
|
| 30 |
+
"left_wrist_pos",
|
| 31 |
+
"left_wrist_abs_quat",
|
| 32 |
+
"right_wrist_pos",
|
| 33 |
+
"right_wrist_abs_quat"
|
| 34 |
+
]
|
| 35 |
+
},
|
| 36 |
+
"action.wbc": {
|
| 37 |
+
"dtype": "float64",
|
| 38 |
+
"shape": [
|
| 39 |
+
41
|
| 40 |
+
],
|
| 41 |
+
"names": [
|
| 42 |
+
"left_hip_pitch_joint",
|
| 43 |
+
"left_hip_roll_joint",
|
| 44 |
+
"left_hip_yaw_joint",
|
| 45 |
+
"left_knee_joint",
|
| 46 |
+
"left_ankle_pitch_joint",
|
| 47 |
+
"left_ankle_roll_joint",
|
| 48 |
+
"right_hip_pitch_joint",
|
| 49 |
+
"right_hip_roll_joint",
|
| 50 |
+
"right_hip_yaw_joint",
|
| 51 |
+
"right_knee_joint",
|
| 52 |
+
"right_ankle_pitch_joint",
|
| 53 |
+
"right_ankle_roll_joint",
|
| 54 |
+
"waist_yaw_joint",
|
| 55 |
+
"waist_roll_joint",
|
| 56 |
+
"waist_pitch_joint",
|
| 57 |
+
"left_shoulder_pitch_joint",
|
| 58 |
+
"left_shoulder_roll_joint",
|
| 59 |
+
"left_shoulder_yaw_joint",
|
| 60 |
+
"left_elbow_joint",
|
| 61 |
+
"left_wrist_roll_joint",
|
| 62 |
+
"left_wrist_pitch_joint",
|
| 63 |
+
"left_wrist_yaw_joint",
|
| 64 |
+
"right_shoulder_pitch_joint",
|
| 65 |
+
"right_shoulder_roll_joint",
|
| 66 |
+
"right_shoulder_yaw_joint",
|
| 67 |
+
"right_elbow_joint",
|
| 68 |
+
"right_wrist_roll_joint",
|
| 69 |
+
"right_wrist_pitch_joint",
|
| 70 |
+
"right_wrist_yaw_joint",
|
| 71 |
+
"left_hand_pinky",
|
| 72 |
+
"left_hand_ring",
|
| 73 |
+
"left_hand_middle",
|
| 74 |
+
"left_hand_index",
|
| 75 |
+
"left_hand_thumb_bend",
|
| 76 |
+
"left_hand_thumb_rotation",
|
| 77 |
+
"right_hand_pinky",
|
| 78 |
+
"right_hand_ring",
|
| 79 |
+
"right_hand_middle",
|
| 80 |
+
"right_hand_index",
|
| 81 |
+
"right_hand_thumb_bend",
|
| 82 |
+
"right_hand_thumb_rotation"
|
| 83 |
+
]
|
| 84 |
+
},
|
| 85 |
+
"observation.root_orientation": {
|
| 86 |
+
"dtype": "float64",
|
| 87 |
+
"shape": [
|
| 88 |
+
4
|
| 89 |
+
],
|
| 90 |
+
"names": [
|
| 91 |
+
"base_qw",
|
| 92 |
+
"base_qx",
|
| 93 |
+
"base_qy",
|
| 94 |
+
"base_qz"
|
| 95 |
+
]
|
| 96 |
+
},
|
| 97 |
+
"observation.projected_gravity": {
|
| 98 |
+
"dtype": "float64",
|
| 99 |
+
"shape": [
|
| 100 |
+
3
|
| 101 |
+
],
|
| 102 |
+
"names": [
|
| 103 |
+
"gravity_x",
|
| 104 |
+
"gravity_y",
|
| 105 |
+
"gravity_z"
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"observation.cpp_rotation_offset": {
|
| 109 |
+
"dtype": "float64",
|
| 110 |
+
"shape": [
|
| 111 |
+
4
|
| 112 |
+
],
|
| 113 |
+
"names": [
|
| 114 |
+
"rot_offset_qw",
|
| 115 |
+
"rot_offset_qx",
|
| 116 |
+
"rot_offset_qy",
|
| 117 |
+
"rot_offset_qz"
|
| 118 |
+
]
|
| 119 |
+
},
|
| 120 |
+
"observation.init_base_quat": {
|
| 121 |
+
"dtype": "float64",
|
| 122 |
+
"shape": [
|
| 123 |
+
4
|
| 124 |
+
],
|
| 125 |
+
"names": [
|
| 126 |
+
"init_base_qw",
|
| 127 |
+
"init_base_qx",
|
| 128 |
+
"init_base_qy",
|
| 129 |
+
"init_base_qz"
|
| 130 |
+
]
|
| 131 |
+
},
|
| 132 |
+
"teleop.delta_heading": {
|
| 133 |
+
"dtype": "float64",
|
| 134 |
+
"shape": [
|
| 135 |
+
1
|
| 136 |
+
],
|
| 137 |
+
"names": [
|
| 138 |
+
"delta_heading"
|
| 139 |
+
]
|
| 140 |
+
},
|
| 141 |
+
"action.motion_token": {
|
| 142 |
+
"dtype": "float64",
|
| 143 |
+
"shape": [
|
| 144 |
+
64
|
| 145 |
+
],
|
| 146 |
+
"names": "motion_token"
|
| 147 |
+
},
|
| 148 |
+
"teleop.smpl_joints": {
|
| 149 |
+
"dtype": "float32",
|
| 150 |
+
"shape": [
|
| 151 |
+
72
|
| 152 |
+
],
|
| 153 |
+
"names": "smpl_joints"
|
| 154 |
+
},
|
| 155 |
+
"teleop.smpl_pose": {
|
| 156 |
+
"dtype": "float32",
|
| 157 |
+
"shape": [
|
| 158 |
+
63
|
| 159 |
+
],
|
| 160 |
+
"names": "smpl_pose"
|
| 161 |
+
},
|
| 162 |
+
"teleop.body_quat_w": {
|
| 163 |
+
"dtype": "float32",
|
| 164 |
+
"shape": [
|
| 165 |
+
4
|
| 166 |
+
],
|
| 167 |
+
"names": "body_quat_w"
|
| 168 |
+
},
|
| 169 |
+
"teleop.target_body_orientation": {
|
| 170 |
+
"dtype": "float32",
|
| 171 |
+
"shape": [
|
| 172 |
+
6
|
| 173 |
+
],
|
| 174 |
+
"names": [
|
| 175 |
+
"target_body_r00",
|
| 176 |
+
"target_body_r10",
|
| 177 |
+
"target_body_r01",
|
| 178 |
+
"target_body_r11",
|
| 179 |
+
"target_body_r02",
|
| 180 |
+
"target_body_r12"
|
| 181 |
+
]
|
| 182 |
+
},
|
| 183 |
+
"teleop.left_hand_joints": {
|
| 184 |
+
"dtype": "float32",
|
| 185 |
+
"shape": [
|
| 186 |
+
7
|
| 187 |
+
],
|
| 188 |
+
"names": "left_hand_joints"
|
| 189 |
+
},
|
| 190 |
+
"teleop.right_hand_joints": {
|
| 191 |
+
"dtype": "float32",
|
| 192 |
+
"shape": [
|
| 193 |
+
7
|
| 194 |
+
],
|
| 195 |
+
"names": "right_hand_joints"
|
| 196 |
+
},
|
| 197 |
+
"teleop.inspire_left_hand_cmd": {
|
| 198 |
+
"dtype": "float32",
|
| 199 |
+
"shape": [
|
| 200 |
+
6
|
| 201 |
+
],
|
| 202 |
+
"names": [
|
| 203 |
+
"left_pinky",
|
| 204 |
+
"left_ring",
|
| 205 |
+
"left_middle",
|
| 206 |
+
"left_index",
|
| 207 |
+
"left_thumb_pitch",
|
| 208 |
+
"left_thumb_yaw"
|
| 209 |
+
]
|
| 210 |
+
},
|
| 211 |
+
"teleop.inspire_right_hand_cmd": {
|
| 212 |
+
"dtype": "float32",
|
| 213 |
+
"shape": [
|
| 214 |
+
6
|
| 215 |
+
],
|
| 216 |
+
"names": [
|
| 217 |
+
"right_pinky",
|
| 218 |
+
"right_ring",
|
| 219 |
+
"right_middle",
|
| 220 |
+
"right_index",
|
| 221 |
+
"right_thumb_pitch",
|
| 222 |
+
"right_thumb_yaw"
|
| 223 |
+
]
|
| 224 |
+
},
|
| 225 |
+
"teleop.inspire_hand_cmd_age_ms": {
|
| 226 |
+
"dtype": "float32",
|
| 227 |
+
"shape": [
|
| 228 |
+
1
|
| 229 |
+
],
|
| 230 |
+
"names": [
|
| 231 |
+
"inspire_hand_cmd_age_ms"
|
| 232 |
+
]
|
| 233 |
+
},
|
| 234 |
+
"teleop.controller": {
|
| 235 |
+
"dtype": "float32",
|
| 236 |
+
"shape": [
|
| 237 |
+
4
|
| 238 |
+
],
|
| 239 |
+
"names": [
|
| 240 |
+
"left_trigger",
|
| 241 |
+
"right_trigger",
|
| 242 |
+
"left_grip",
|
| 243 |
+
"right_grip"
|
| 244 |
+
]
|
| 245 |
+
},
|
| 246 |
+
"teleop.pose_sample_age_ms": {
|
| 247 |
+
"dtype": "float32",
|
| 248 |
+
"shape": [
|
| 249 |
+
1
|
| 250 |
+
],
|
| 251 |
+
"names": [
|
| 252 |
+
"pose_sample_age_ms"
|
| 253 |
+
]
|
| 254 |
+
},
|
| 255 |
+
"teleop.smpl_frame_index": {
|
| 256 |
+
"dtype": "int64",
|
| 257 |
+
"shape": [
|
| 258 |
+
1
|
| 259 |
+
],
|
| 260 |
+
"names": [
|
| 261 |
+
"smpl_frame_index"
|
| 262 |
+
]
|
| 263 |
+
},
|
| 264 |
+
"teleop.left_wrist_joints": {
|
| 265 |
+
"dtype": "float32",
|
| 266 |
+
"shape": [
|
| 267 |
+
3
|
| 268 |
+
],
|
| 269 |
+
"names": [
|
| 270 |
+
"left_wrist_roll",
|
| 271 |
+
"left_wrist_pitch",
|
| 272 |
+
"left_wrist_yaw"
|
| 273 |
+
]
|
| 274 |
+
},
|
| 275 |
+
"teleop.right_wrist_joints": {
|
| 276 |
+
"dtype": "float32",
|
| 277 |
+
"shape": [
|
| 278 |
+
3
|
| 279 |
+
],
|
| 280 |
+
"names": [
|
| 281 |
+
"right_wrist_roll",
|
| 282 |
+
"right_wrist_pitch",
|
| 283 |
+
"right_wrist_yaw"
|
| 284 |
+
]
|
| 285 |
+
},
|
| 286 |
+
"teleop.stream_mode": {
|
| 287 |
+
"dtype": "int32",
|
| 288 |
+
"shape": [
|
| 289 |
+
1
|
| 290 |
+
],
|
| 291 |
+
"names": [
|
| 292 |
+
"stream_mode"
|
| 293 |
+
]
|
| 294 |
+
},
|
| 295 |
+
"teleop.planner_mode": {
|
| 296 |
+
"dtype": "int32",
|
| 297 |
+
"shape": [
|
| 298 |
+
1
|
| 299 |
+
],
|
| 300 |
+
"names": [
|
| 301 |
+
"locomotion_mode"
|
| 302 |
+
]
|
| 303 |
+
},
|
| 304 |
+
"teleop.planner_movement": {
|
| 305 |
+
"dtype": "float32",
|
| 306 |
+
"shape": [
|
| 307 |
+
3
|
| 308 |
+
],
|
| 309 |
+
"names": [
|
| 310 |
+
"movement_x",
|
| 311 |
+
"movement_y",
|
| 312 |
+
"movement_z"
|
| 313 |
+
]
|
| 314 |
+
},
|
| 315 |
+
"teleop.planner_facing": {
|
| 316 |
+
"dtype": "float32",
|
| 317 |
+
"shape": [
|
| 318 |
+
3
|
| 319 |
+
],
|
| 320 |
+
"names": [
|
| 321 |
+
"facing_x",
|
| 322 |
+
"facing_y",
|
| 323 |
+
"facing_z"
|
| 324 |
+
]
|
| 325 |
+
},
|
| 326 |
+
"teleop.planner_speed": {
|
| 327 |
+
"dtype": "float32",
|
| 328 |
+
"shape": [
|
| 329 |
+
1
|
| 330 |
+
],
|
| 331 |
+
"names": [
|
| 332 |
+
"speed"
|
| 333 |
+
]
|
| 334 |
+
},
|
| 335 |
+
"teleop.planner_height": {
|
| 336 |
+
"dtype": "float32",
|
| 337 |
+
"shape": [
|
| 338 |
+
1
|
| 339 |
+
],
|
| 340 |
+
"names": [
|
| 341 |
+
"height"
|
| 342 |
+
]
|
| 343 |
+
},
|
| 344 |
+
"teleop.vr_3pt_position": {
|
| 345 |
+
"dtype": "float32",
|
| 346 |
+
"shape": [
|
| 347 |
+
9
|
| 348 |
+
],
|
| 349 |
+
"names": [
|
| 350 |
+
"lwrist_x",
|
| 351 |
+
"lwrist_y",
|
| 352 |
+
"lwrist_z",
|
| 353 |
+
"rwrist_x",
|
| 354 |
+
"rwrist_y",
|
| 355 |
+
"rwrist_z",
|
| 356 |
+
"neck_x",
|
| 357 |
+
"neck_y",
|
| 358 |
+
"neck_z"
|
| 359 |
+
]
|
| 360 |
+
},
|
| 361 |
+
"teleop.vr_3pt_orientation": {
|
| 362 |
+
"dtype": "float32",
|
| 363 |
+
"shape": [
|
| 364 |
+
18
|
| 365 |
+
],
|
| 366 |
+
"names": [
|
| 367 |
+
"lwrist_r00",
|
| 368 |
+
"lwrist_r10",
|
| 369 |
+
"lwrist_r01",
|
| 370 |
+
"lwrist_r11",
|
| 371 |
+
"lwrist_r02",
|
| 372 |
+
"lwrist_r12",
|
| 373 |
+
"rwrist_r00",
|
| 374 |
+
"rwrist_r10",
|
| 375 |
+
"rwrist_r01",
|
| 376 |
+
"rwrist_r11",
|
| 377 |
+
"rwrist_r02",
|
| 378 |
+
"rwrist_r12",
|
| 379 |
+
"neck_r00",
|
| 380 |
+
"neck_r10",
|
| 381 |
+
"neck_r01",
|
| 382 |
+
"neck_r11",
|
| 383 |
+
"neck_r02",
|
| 384 |
+
"neck_r12"
|
| 385 |
+
]
|
| 386 |
+
},
|
| 387 |
+
"observation.images.ego_view": {
|
| 388 |
+
"dtype": "video",
|
| 389 |
+
"shape": [
|
| 390 |
+
480,
|
| 391 |
+
640,
|
| 392 |
+
3
|
| 393 |
+
],
|
| 394 |
+
"names": [
|
| 395 |
+
"height",
|
| 396 |
+
"width",
|
| 397 |
+
"channel"
|
| 398 |
+
],
|
| 399 |
+
"info": {
|
| 400 |
+
"video.height": 480,
|
| 401 |
+
"video.width": 640,
|
| 402 |
+
"video.codec": "h264",
|
| 403 |
+
"video.pix_fmt": "yuv420p",
|
| 404 |
+
"video.is_depth_map": false,
|
| 405 |
+
"video.fps": 30,
|
| 406 |
+
"video.channels": 3,
|
| 407 |
+
"has_audio": false
|
| 408 |
+
}
|
| 409 |
+
},
|
| 410 |
+
"observation.root_position": {
|
| 411 |
+
"dtype": "float64",
|
| 412 |
+
"shape": [
|
| 413 |
+
3
|
| 414 |
+
],
|
| 415 |
+
"names": [
|
| 416 |
+
"base_x",
|
| 417 |
+
"base_y",
|
| 418 |
+
"base_z"
|
| 419 |
+
]
|
| 420 |
+
},
|
| 421 |
+
"observation.inspire_left_hand_state": {
|
| 422 |
+
"dtype": "float32",
|
| 423 |
+
"shape": [
|
| 424 |
+
6
|
| 425 |
+
],
|
| 426 |
+
"names": [
|
| 427 |
+
"left_pinky",
|
| 428 |
+
"left_ring",
|
| 429 |
+
"left_middle",
|
| 430 |
+
"left_index",
|
| 431 |
+
"left_thumb_bend",
|
| 432 |
+
"left_thumb_rotation"
|
| 433 |
+
]
|
| 434 |
+
},
|
| 435 |
+
"observation.inspire_right_hand_state": {
|
| 436 |
+
"dtype": "float32",
|
| 437 |
+
"shape": [
|
| 438 |
+
6
|
| 439 |
+
],
|
| 440 |
+
"names": [
|
| 441 |
+
"right_pinky",
|
| 442 |
+
"right_ring",
|
| 443 |
+
"right_middle",
|
| 444 |
+
"right_index",
|
| 445 |
+
"right_thumb_bend",
|
| 446 |
+
"right_thumb_rotation"
|
| 447 |
+
]
|
| 448 |
+
},
|
| 449 |
+
"observation.images.chest": {
|
| 450 |
+
"dtype": "video",
|
| 451 |
+
"shape": [
|
| 452 |
+
480,
|
| 453 |
+
640,
|
| 454 |
+
3
|
| 455 |
+
],
|
| 456 |
+
"names": [
|
| 457 |
+
"height",
|
| 458 |
+
"width",
|
| 459 |
+
"channel"
|
| 460 |
+
],
|
| 461 |
+
"info": {
|
| 462 |
+
"video.height": 480,
|
| 463 |
+
"video.width": 640,
|
| 464 |
+
"video.codec": "h264",
|
| 465 |
+
"video.pix_fmt": "yuv420p",
|
| 466 |
+
"video.is_depth_map": false,
|
| 467 |
+
"video.fps": 30,
|
| 468 |
+
"video.channels": 3,
|
| 469 |
+
"has_audio": false
|
| 470 |
+
}
|
| 471 |
+
},
|
| 472 |
+
"timestamp": {
|
| 473 |
+
"dtype": "float32",
|
| 474 |
+
"shape": [
|
| 475 |
+
1
|
| 476 |
+
],
|
| 477 |
+
"names": null
|
| 478 |
+
},
|
| 479 |
+
"frame_index": {
|
| 480 |
+
"dtype": "int64",
|
| 481 |
+
"shape": [
|
| 482 |
+
1
|
| 483 |
+
],
|
| 484 |
+
"names": null
|
| 485 |
+
},
|
| 486 |
+
"episode_index": {
|
| 487 |
+
"dtype": "int64",
|
| 488 |
+
"shape": [
|
| 489 |
+
1
|
| 490 |
+
],
|
| 491 |
+
"names": null
|
| 492 |
+
},
|
| 493 |
+
"index": {
|
| 494 |
+
"dtype": "int64",
|
| 495 |
+
"shape": [
|
| 496 |
+
1
|
| 497 |
+
],
|
| 498 |
+
"names": null
|
| 499 |
+
},
|
| 500 |
+
"task_index": {
|
| 501 |
+
"dtype": "int64",
|
| 502 |
+
"shape": [
|
| 503 |
+
1
|
| 504 |
+
],
|
| 505 |
+
"names": null
|
| 506 |
+
},
|
| 507 |
+
"action": {
|
| 508 |
+
"dtype": "float32",
|
| 509 |
+
"shape": [
|
| 510 |
+
76
|
| 511 |
+
],
|
| 512 |
+
"names": null
|
| 513 |
+
}
|
| 514 |
+
},
|
| 515 |
+
"script_config": {
|
| 516 |
+
"source": "sim",
|
| 517 |
+
"simulator": "newton",
|
| 518 |
+
"controller": "SONIC g1_deploy_onnx_ref in sim lockstep (planner and upper-body commands, no PICO stream)",
|
| 519 |
+
"exporter": "humanoid_newton.collect.lerobot_export",
|
| 520 |
+
"control_frequency": 50.0,
|
| 521 |
+
"record_wrist_cameras": false,
|
| 522 |
+
"record_chest_camera": true,
|
| 523 |
+
"rotate_chest_camera_180": false,
|
| 524 |
+
"record_depth": false,
|
| 525 |
+
"record_root_position": true,
|
| 526 |
+
"record_inspire_hand_state": true,
|
| 527 |
+
"record_inspire_hand_cmd": true,
|
| 528 |
+
"no_camera": false,
|
| 529 |
+
"camera_backend": "offline_render",
|
| 530 |
+
"camera_note": "ego_view and chest rendered offline (humanoid_newton.rendering.splat_render) from the recorded body poses at the registered D455 mounts; chest rendered upright (the real exporter rotates its upside-down chest stream by 180 deg); cameras not calibrated against the real ones",
|
| 531 |
+
"task": "b8",
|
| 532 |
+
"collect_config": "20261114_black_coffee_training_100_01",
|
| 533 |
+
"scene": "eai_room_fridge_001",
|
| 534 |
+
"task_description": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it.",
|
| 535 |
+
"git_commit": null
|
| 536 |
+
},
|
| 537 |
+
"discarded_episode_indices": []
|
| 538 |
+
}
|
b8-table-wipe/b512/step-002000/dataset_meta/tasks.jsonl
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"task_index": 0, "task": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it."}
|
b8-table-wipe/b512/step-002000/dataset_stats.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-002000/logs/node-0.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-002000/logs/node-1.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-002000/logs/node-2.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-002000/logs/node-3.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-002000/loss_summary.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"points": 200,
|
| 3 |
+
"first": {
|
| 4 |
+
"epoch": 0,
|
| 5 |
+
"step": 20,
|
| 6 |
+
"loss": 1.3127,
|
| 7 |
+
"loss_action": 1.0909,
|
| 8 |
+
"loss_video": 0.2218,
|
| 9 |
+
"lr": 1.05e-05,
|
| 10 |
+
"steps_per_s": 0.17
|
| 11 |
+
},
|
| 12 |
+
"last": {
|
| 13 |
+
"epoch": 33,
|
| 14 |
+
"step": 4000,
|
| 15 |
+
"loss": 0.0571,
|
| 16 |
+
"loss_action": 0.0106,
|
| 17 |
+
"loss_video": 0.0464,
|
| 18 |
+
"lr": 1e-06,
|
| 19 |
+
"steps_per_s": 0.24
|
| 20 |
+
},
|
| 21 |
+
"min_loss": 0.0537,
|
| 22 |
+
"mean_steps_per_s": 0.23805000000000018
|
| 23 |
+
}
|
b8-table-wipe/b512/step-002000/metrics.csv
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
step,epoch,loss,loss_action,loss_video,lr,steps_per_s
|
| 2 |
+
20,0,1.3127,1.0909,0.2218,1.05e-05,0.17
|
| 3 |
+
40,0,0.8771,0.6803,0.1968,2.04e-05,0.19
|
| 4 |
+
60,0,0.4879,0.322,0.166,3.04e-05,0.21
|
| 5 |
+
80,0,0.347,0.177,0.17,4.03e-05,0.22
|
| 6 |
+
100,0,0.3127,0.1505,0.1622,5.02e-05,0.21
|
| 7 |
+
120,0,0.2829,0.1298,0.1531,6.02e-05,0.23
|
| 8 |
+
140,1,0.2545,0.1109,0.1436,7.01e-05,0.22
|
| 9 |
+
160,1,0.2666,0.1202,0.1464,8.01e-05,0.22
|
| 10 |
+
180,1,0.2397,0.1086,0.1311,9e-05,0.22
|
| 11 |
+
200,1,0.234,0.0977,0.1363,0.0001,0.23
|
| 12 |
+
220,1,0.2251,0.0856,0.1396,0.0001,0.23
|
| 13 |
+
240,1,0.2422,0.1015,0.1407,0.0001,0.24
|
| 14 |
+
260,2,0.1952,0.0765,0.1187,9.99e-05,0.23
|
| 15 |
+
280,2,0.199,0.0713,0.1276,9.99e-05,0.23
|
| 16 |
+
300,2,0.1867,0.0643,0.1224,9.98e-05,0.23
|
| 17 |
+
320,2,0.1877,0.0668,0.1209,9.98e-05,0.23
|
| 18 |
+
340,2,0.183,0.0612,0.1218,9.97e-05,0.23
|
| 19 |
+
360,2,0.1771,0.0568,0.1203,9.96e-05,0.24
|
| 20 |
+
380,3,0.1749,0.0618,0.1131,9.95e-05,0.23
|
| 21 |
+
400,3,0.1734,0.0591,0.1143,9.93e-05,0.23
|
| 22 |
+
420,3,0.1651,0.0536,0.1116,9.92e-05,0.23
|
| 23 |
+
440,3,0.159,0.0489,0.1101,9.9e-05,0.23
|
| 24 |
+
460,3,0.1598,0.0499,0.1099,9.89e-05,0.24
|
| 25 |
+
480,3,0.1661,0.051,0.1151,9.87e-05,0.24
|
| 26 |
+
500,4,0.1518,0.0488,0.103,9.85e-05,0.23
|
| 27 |
+
520,4,0.1496,0.0463,0.1033,9.83e-05,0.24
|
| 28 |
+
540,4,0.1465,0.0463,0.1002,9.81e-05,0.24
|
| 29 |
+
560,4,0.1424,0.0436,0.0988,9.78e-05,0.24
|
| 30 |
+
580,4,0.1442,0.0449,0.0992,9.76e-05,0.24
|
| 31 |
+
600,4,0.1472,0.044,0.1033,9.73e-05,0.24
|
| 32 |
+
620,5,0.1411,0.0398,0.1013,9.7e-05,0.24
|
| 33 |
+
640,5,0.1367,0.0382,0.0985,9.68e-05,0.24
|
| 34 |
+
660,5,0.1318,0.0387,0.0931,9.65e-05,0.24
|
| 35 |
+
680,5,0.1309,0.0381,0.0928,9.62e-05,0.24
|
| 36 |
+
700,5,0.1304,0.0387,0.0917,9.58e-05,0.24
|
| 37 |
+
720,5,0.1277,0.037,0.0907,9.55e-05,0.24
|
| 38 |
+
740,6,0.1294,0.0362,0.0932,9.51e-05,0.24
|
| 39 |
+
760,6,0.139,0.0362,0.1028,9.48e-05,0.24
|
| 40 |
+
780,6,0.1241,0.037,0.087,9.44e-05,0.24
|
| 41 |
+
800,6,0.1339,0.034,0.0999,9.4e-05,0.24
|
| 42 |
+
820,6,0.1335,0.0355,0.098,9.36e-05,0.24
|
| 43 |
+
840,6,0.1203,0.0308,0.0895,9.32e-05,0.24
|
| 44 |
+
860,7,0.1244,0.0317,0.0927,9.28e-05,0.24
|
| 45 |
+
880,7,0.1211,0.031,0.0901,9.24e-05,0.24
|
| 46 |
+
900,7,0.1234,0.0312,0.0922,9.19e-05,0.24
|
| 47 |
+
920,7,0.1253,0.0325,0.0929,9.15e-05,0.24
|
| 48 |
+
940,7,0.1186,0.0314,0.0872,9.1e-05,0.24
|
| 49 |
+
960,7,0.1199,0.0294,0.0905,9.05e-05,0.24
|
| 50 |
+
980,8,0.1223,0.0291,0.0932,9.01e-05,0.24
|
| 51 |
+
1000,8,0.116,0.0301,0.0859,8.96e-05,0.24
|
| 52 |
+
1020,8,0.1206,0.0287,0.0918,8.91e-05,0.24
|
| 53 |
+
1040,8,0.113,0.0297,0.0833,8.85e-05,0.24
|
| 54 |
+
1060,8,0.1227,0.0306,0.0921,8.8e-05,0.24
|
| 55 |
+
1080,8,0.1087,0.028,0.0808,8.75e-05,0.24
|
| 56 |
+
1100,9,0.1058,0.0261,0.0797,8.69e-05,0.24
|
| 57 |
+
1120,9,0.1183,0.0283,0.09,8.64e-05,0.24
|
| 58 |
+
1140,9,0.114,0.0267,0.0874,8.58e-05,0.24
|
| 59 |
+
1160,9,0.1045,0.0259,0.0787,8.52e-05,0.24
|
| 60 |
+
1180,9,0.1136,0.0271,0.0865,8.46e-05,0.24
|
| 61 |
+
1200,9,0.1146,0.0255,0.0891,8.4e-05,0.24
|
| 62 |
+
1220,10,0.107,0.025,0.082,8.34e-05,0.24
|
| 63 |
+
1240,10,0.1016,0.0253,0.0762,8.28e-05,0.24
|
| 64 |
+
1260,10,0.1024,0.0254,0.077,8.22e-05,0.24
|
| 65 |
+
1280,10,0.1036,0.0243,0.0793,8.15e-05,0.24
|
| 66 |
+
1300,10,0.102,0.0233,0.0786,8.09e-05,0.24
|
| 67 |
+
1320,10,0.1128,0.0247,0.0881,8.03e-05,0.24
|
| 68 |
+
1340,11,0.1055,0.0226,0.0829,7.96e-05,0.24
|
| 69 |
+
1360,11,0.1021,0.0237,0.0784,7.89e-05,0.24
|
| 70 |
+
1380,11,0.1007,0.0226,0.0782,7.83e-05,0.24
|
| 71 |
+
1400,11,0.1028,0.0226,0.0801,7.76e-05,0.24
|
| 72 |
+
1420,11,0.0976,0.0222,0.0753,7.69e-05,0.24
|
| 73 |
+
1440,11,0.097,0.0213,0.0757,7.62e-05,0.24
|
| 74 |
+
1460,12,0.0982,0.0215,0.0768,7.55e-05,0.24
|
| 75 |
+
1480,12,0.0912,0.0216,0.0696,7.48e-05,0.24
|
| 76 |
+
1500,12,0.0964,0.0217,0.0748,7.41e-05,0.24
|
| 77 |
+
1520,12,0.1016,0.022,0.0796,7.33e-05,0.24
|
| 78 |
+
1540,12,0.0943,0.022,0.0723,7.26e-05,0.24
|
| 79 |
+
1560,12,0.0954,0.0211,0.0743,7.19e-05,0.24
|
| 80 |
+
1580,13,0.0894,0.0208,0.0687,7.11e-05,0.24
|
| 81 |
+
1600,13,0.0997,0.0208,0.079,7.04e-05,0.24
|
| 82 |
+
1620,13,0.087,0.0208,0.0661,6.96e-05,0.24
|
| 83 |
+
1640,13,0.0924,0.0201,0.0723,6.89e-05,0.24
|
| 84 |
+
1660,13,0.0944,0.0181,0.0763,6.81e-05,0.24
|
| 85 |
+
1680,13,0.0972,0.0208,0.0764,6.73e-05,0.24
|
| 86 |
+
1700,14,0.0931,0.0198,0.0733,6.66e-05,0.24
|
| 87 |
+
1720,14,0.0829,0.0219,0.061,6.58e-05,0.24
|
| 88 |
+
1740,14,0.094,0.0193,0.0747,6.5e-05,0.24
|
| 89 |
+
1760,14,0.0822,0.0194,0.0628,6.42e-05,0.24
|
| 90 |
+
1780,14,0.0886,0.0182,0.0704,6.34e-05,0.24
|
| 91 |
+
1800,14,0.0876,0.0195,0.0681,6.27e-05,0.24
|
| 92 |
+
1820,15,0.0886,0.0195,0.0691,6.19e-05,0.24
|
| 93 |
+
1840,15,0.0875,0.0193,0.0682,6.11e-05,0.24
|
| 94 |
+
1860,15,0.0911,0.0195,0.0716,6.03e-05,0.24
|
| 95 |
+
1880,15,0.0873,0.0178,0.0694,5.95e-05,0.24
|
| 96 |
+
1900,15,0.0843,0.0181,0.0661,5.86e-05,0.24
|
| 97 |
+
1920,15,0.0845,0.0174,0.0671,5.78e-05,0.24
|
| 98 |
+
1940,16,0.0922,0.017,0.0752,5.7e-05,0.24
|
| 99 |
+
1960,16,0.0852,0.0175,0.0677,5.62e-05,0.24
|
| 100 |
+
1980,16,0.0851,0.0174,0.0677,5.54e-05,0.24
|
| 101 |
+
2000,16,0.0816,0.0165,0.065,5.46e-05,0.24
|
| 102 |
+
2020,16,0.0812,0.017,0.0642,5.38e-05,0.24
|
| 103 |
+
2040,16,0.0808,0.0164,0.0644,5.3e-05,0.24
|
| 104 |
+
2060,17,0.0827,0.0174,0.0653,5.21e-05,0.24
|
| 105 |
+
2080,17,0.0786,0.0179,0.0607,5.13e-05,0.24
|
| 106 |
+
2100,17,0.0812,0.0165,0.0646,5.05e-05,0.24
|
| 107 |
+
2120,17,0.083,0.0171,0.0659,4.97e-05,0.24
|
| 108 |
+
2140,17,0.0738,0.0146,0.0592,4.89e-05,0.24
|
| 109 |
+
2160,17,0.0777,0.0148,0.0629,4.8e-05,0.24
|
| 110 |
+
2180,18,0.0788,0.015,0.0638,4.72e-05,0.24
|
| 111 |
+
2200,18,0.0743,0.0155,0.0588,4.64e-05,0.24
|
| 112 |
+
2220,18,0.0783,0.0165,0.0618,4.56e-05,0.24
|
| 113 |
+
2240,18,0.0784,0.014,0.0644,4.48e-05,0.24
|
| 114 |
+
2260,18,0.0772,0.0146,0.0625,4.4e-05,0.24
|
| 115 |
+
2280,18,0.0684,0.0142,0.0542,4.32e-05,0.24
|
| 116 |
+
2300,19,0.0779,0.0162,0.0616,4.24e-05,0.24
|
| 117 |
+
2320,19,0.0778,0.0161,0.0617,4.15e-05,0.24
|
| 118 |
+
2340,19,0.074,0.0147,0.0593,4.07e-05,0.24
|
| 119 |
+
2360,19,0.0693,0.0153,0.054,3.99e-05,0.24
|
| 120 |
+
2380,19,0.0731,0.0153,0.0578,3.91e-05,0.24
|
| 121 |
+
2400,19,0.0709,0.0149,0.056,3.83e-05,0.24
|
| 122 |
+
2420,19,0.0763,0.0144,0.0619,3.76e-05,0.24
|
| 123 |
+
2440,20,0.075,0.0149,0.06,3.68e-05,0.24
|
| 124 |
+
2460,20,0.0674,0.0148,0.0526,3.6e-05,0.24
|
| 125 |
+
2480,20,0.0709,0.0146,0.0563,3.52e-05,0.24
|
| 126 |
+
2500,20,0.0695,0.015,0.0545,3.44e-05,0.24
|
| 127 |
+
2520,20,0.0675,0.0131,0.0544,3.37e-05,0.24
|
| 128 |
+
2540,20,0.0689,0.0141,0.0548,3.29e-05,0.24
|
| 129 |
+
2560,21,0.0779,0.0142,0.0637,3.21e-05,0.24
|
| 130 |
+
2580,21,0.0674,0.0142,0.0532,3.14e-05,0.24
|
| 131 |
+
2600,21,0.0652,0.013,0.0522,3.06e-05,0.24
|
| 132 |
+
2620,21,0.0612,0.0128,0.0484,2.99e-05,0.24
|
| 133 |
+
2640,21,0.0654,0.0132,0.0522,2.91e-05,0.24
|
| 134 |
+
2660,21,0.0649,0.013,0.0519,2.84e-05,0.24
|
| 135 |
+
2680,22,0.0729,0.0114,0.0615,2.77e-05,0.24
|
| 136 |
+
2700,22,0.0673,0.0129,0.0544,2.69e-05,0.24
|
| 137 |
+
2720,22,0.0703,0.0121,0.0582,2.62e-05,0.24
|
| 138 |
+
2740,22,0.0692,0.0129,0.0563,2.55e-05,0.24
|
| 139 |
+
2760,22,0.0677,0.0131,0.0546,2.48e-05,0.24
|
| 140 |
+
2780,22,0.0691,0.0132,0.0559,2.41e-05,0.24
|
| 141 |
+
2800,23,0.0628,0.0111,0.0517,2.34e-05,0.24
|
| 142 |
+
2820,23,0.0673,0.013,0.0542,2.27e-05,0.24
|
| 143 |
+
2840,23,0.0662,0.0133,0.0529,2.21e-05,0.24
|
| 144 |
+
2860,23,0.0622,0.0119,0.0502,2.14e-05,0.24
|
| 145 |
+
2880,23,0.0618,0.0111,0.0506,2.07e-05,0.24
|
| 146 |
+
2900,23,0.0651,0.0124,0.0527,2.01e-05,0.24
|
| 147 |
+
2920,24,0.0574,0.0111,0.0463,1.95e-05,0.24
|
| 148 |
+
2940,24,0.0627,0.012,0.0507,1.88e-05,0.24
|
| 149 |
+
2960,24,0.0669,0.0125,0.0544,1.82e-05,0.24
|
| 150 |
+
2980,24,0.0628,0.0114,0.0515,1.76e-05,0.24
|
| 151 |
+
3000,24,0.0645,0.0116,0.0529,1.7e-05,0.24
|
| 152 |
+
3020,24,0.0667,0.0115,0.0552,1.64e-05,0.24
|
| 153 |
+
3040,25,0.0588,0.0116,0.0472,1.58e-05,0.24
|
| 154 |
+
3060,25,0.0645,0.0132,0.0513,1.52e-05,0.24
|
| 155 |
+
3080,25,0.0582,0.0112,0.047,1.46e-05,0.24
|
| 156 |
+
3100,25,0.0616,0.0125,0.0491,1.41e-05,0.24
|
| 157 |
+
3120,25,0.0585,0.0107,0.0478,1.35e-05,0.24
|
| 158 |
+
3140,25,0.0618,0.0111,0.0507,1.3e-05,0.24
|
| 159 |
+
3160,26,0.0621,0.0119,0.0502,1.25e-05,0.24
|
| 160 |
+
3180,26,0.0566,0.0108,0.0458,1.19e-05,0.24
|
| 161 |
+
3200,26,0.0631,0.0119,0.0512,1.14e-05,0.24
|
| 162 |
+
3220,26,0.058,0.0111,0.0469,1.09e-05,0.24
|
| 163 |
+
3240,26,0.0636,0.0106,0.053,1.05e-05,0.24
|
| 164 |
+
3260,26,0.0641,0.0118,0.0523,9.98e-06,0.24
|
| 165 |
+
3280,27,0.0631,0.0113,0.0518,9.51e-06,0.24
|
| 166 |
+
3300,27,0.0639,0.011,0.0529,9.06e-06,0.24
|
| 167 |
+
3320,27,0.0589,0.0109,0.0479,8.62e-06,0.24
|
| 168 |
+
3340,27,0.0616,0.011,0.0507,8.19e-06,0.24
|
| 169 |
+
3360,27,0.0599,0.0115,0.0484,7.77e-06,0.24
|
| 170 |
+
3380,27,0.0606,0.0117,0.0489,7.36e-06,0.24
|
| 171 |
+
3400,28,0.058,0.0114,0.0466,6.97e-06,0.24
|
| 172 |
+
3420,28,0.0613,0.0112,0.0501,6.58e-06,0.24
|
| 173 |
+
3440,28,0.0595,0.0101,0.0494,6.21e-06,0.24
|
| 174 |
+
3460,28,0.0573,0.011,0.0463,5.85e-06,0.24
|
| 175 |
+
3480,28,0.0616,0.0101,0.0515,5.5e-06,0.24
|
| 176 |
+
3500,28,0.0566,0.0107,0.0459,5.17e-06,0.24
|
| 177 |
+
3520,29,0.0622,0.0111,0.0512,4.85e-06,0.24
|
| 178 |
+
3540,29,0.0565,0.0107,0.0458,4.54e-06,0.24
|
| 179 |
+
3560,29,0.0573,0.0103,0.0469,4.24e-06,0.24
|
| 180 |
+
3580,29,0.058,0.0108,0.0471,3.95e-06,0.24
|
| 181 |
+
3600,29,0.0575,0.0104,0.0471,3.68e-06,0.24
|
| 182 |
+
3620,29,0.0574,0.0098,0.0476,3.42e-06,0.24
|
| 183 |
+
3640,30,0.055,0.0094,0.0456,3.18e-06,0.24
|
| 184 |
+
3660,30,0.0624,0.0088,0.0535,2.94e-06,0.24
|
| 185 |
+
3680,30,0.0546,0.0096,0.045,2.72e-06,0.24
|
| 186 |
+
3700,30,0.0562,0.0091,0.0471,2.51e-06,0.24
|
| 187 |
+
3720,30,0.0595,0.0102,0.0494,2.32e-06,0.24
|
| 188 |
+
3740,30,0.0562,0.0101,0.0461,2.14e-06,0.24
|
| 189 |
+
3760,31,0.0551,0.0104,0.0447,1.97e-06,0.24
|
| 190 |
+
3780,31,0.0626,0.0091,0.0535,1.82e-06,0.24
|
| 191 |
+
3800,31,0.0558,0.0096,0.0462,1.68e-06,0.24
|
| 192 |
+
3820,31,0.0537,0.0105,0.0431,1.55e-06,0.24
|
| 193 |
+
3840,31,0.0616,0.0109,0.0507,1.43e-06,0.24
|
| 194 |
+
3860,31,0.0552,0.01,0.0452,1.33e-06,0.24
|
| 195 |
+
3880,32,0.0553,0.0097,0.0456,1.24e-06,0.24
|
| 196 |
+
3900,32,0.0578,0.0107,0.0471,1.17e-06,0.24
|
| 197 |
+
3920,32,0.0566,0.0097,0.0469,1.11e-06,0.24
|
| 198 |
+
3940,32,0.0565,0.0101,0.0464,1.06e-06,0.24
|
| 199 |
+
3960,32,0.0588,0.01,0.0488,1.03e-06,0.24
|
| 200 |
+
3980,32,0.0547,0.0096,0.0451,1.01e-06,0.24
|
| 201 |
+
4000,33,0.0571,0.0106,0.0464,1e-06,0.24
|
b8-table-wipe/b512/step-002000/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c1b463083f4a14c71f11dc197c546d7f7d81d5a0330c8a67fda8796635d126e6
|
| 3 |
+
size 12042313337
|
b8-table-wipe/b512/step-002000/model_config.yaml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_target_: fastwam.runtime.create_fastwam
|
| 2 |
+
model_id: Wan-AI/Wan2.2-TI2V-5B
|
| 3 |
+
tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
|
| 4 |
+
tokenizer_max_len: 128
|
| 5 |
+
load_text_encoder: false
|
| 6 |
+
proprio_dim: 44
|
| 7 |
+
redirect_common_files: false
|
| 8 |
+
mot_checkpoint_mixed_attn: false
|
| 9 |
+
compile_training_denoise: false
|
| 10 |
+
action_dit_pretrained_path: checkpoints/ActionDiT_linear_interp_Wan22_alphascale_1024hdim.pt
|
| 11 |
+
skip_dit_load_from_pretrain: false
|
| 12 |
+
video_dit_config:
|
| 13 |
+
has_image_input: false
|
| 14 |
+
patch_size:
|
| 15 |
+
- 1
|
| 16 |
+
- 2
|
| 17 |
+
- 2
|
| 18 |
+
in_dim: 48
|
| 19 |
+
hidden_dim: 3072
|
| 20 |
+
ffn_dim: 14336
|
| 21 |
+
freq_dim: 256
|
| 22 |
+
text_dim: 4096
|
| 23 |
+
out_dim: 48
|
| 24 |
+
num_heads: 24
|
| 25 |
+
attn_head_dim: 128
|
| 26 |
+
num_layers: 30
|
| 27 |
+
eps: 1.0e-06
|
| 28 |
+
seperated_timestep: true
|
| 29 |
+
require_clip_embedding: false
|
| 30 |
+
require_vae_embedding: false
|
| 31 |
+
fuse_vae_embedding_in_latents: true
|
| 32 |
+
use_gradient_checkpointing: false
|
| 33 |
+
video_attention_mask_mode: first_frame_causal
|
| 34 |
+
action_conditioned: false
|
| 35 |
+
action_dim: 76
|
| 36 |
+
action_group_causal_mask_mode: group_diagonal
|
| 37 |
+
action_dit_config:
|
| 38 |
+
action_dim: 76
|
| 39 |
+
hidden_dim: 1024
|
| 40 |
+
ffn_dim: 4096
|
| 41 |
+
num_heads: 24
|
| 42 |
+
attn_head_dim: 128
|
| 43 |
+
num_layers: 30
|
| 44 |
+
text_dim: 4096
|
| 45 |
+
freq_dim: 256
|
| 46 |
+
eps: 1.0e-06
|
| 47 |
+
use_gradient_checkpointing: false
|
| 48 |
+
video_scheduler:
|
| 49 |
+
train_shift: 5.0
|
| 50 |
+
infer_shift: 5.0
|
| 51 |
+
num_train_timesteps: 1000
|
| 52 |
+
action_scheduler:
|
| 53 |
+
train_shift: 1.0
|
| 54 |
+
infer_shift: 1.0
|
| 55 |
+
num_train_timesteps: 1000
|
| 56 |
+
loss:
|
| 57 |
+
lambda_action: 1.0
|
b8-table-wipe/b512/step-002000/protocol.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "FastWAM (Wan2.2-TI2V-5B video expert + 1.02 B action expert, mixture-of-transformers world action model; upstream github.com/yuantianyuan01/FastWAM)",
|
| 3 |
+
"task": "B8 table wipe (G1)",
|
| 4 |
+
"task_dir": "b8-table-wipe/b512",
|
| 5 |
+
"robot": "Unitree G1",
|
| 6 |
+
"training_data": "b8-table-wipe-expert-v1 @3b2e618",
|
| 7 |
+
"episodes": 100,
|
| 8 |
+
"epoch_completed": null,
|
| 9 |
+
"global_step": 2000,
|
| 10 |
+
"augmentation": "Humanoid Benchmark online augmentation spec v2 (2026-11-09): 15 visual effects, one effect per training sample at 1.5x; proprioception aligned with the GR00T A20 recipe (2026-11-12): per-sample joint bias +-2 deg + Gaussian noise 0.25 deg (hb_aug_v2.perturb_joints), no proprio delay, no history drop; model-native image augmentation off",
|
| 11 |
+
"epochs": null,
|
| 12 |
+
"total_steps": 4000,
|
| 13 |
+
"save_steps": [
|
| 14 |
+
2000,
|
| 15 |
+
4000
|
| 16 |
+
],
|
| 17 |
+
"num_gpus": 32,
|
| 18 |
+
"per_gpu_batch": 16,
|
| 19 |
+
"global_batch": 512,
|
| 20 |
+
"gradient_accumulation_steps": 1,
|
| 21 |
+
"frames": 61498,
|
| 22 |
+
"fps": 30,
|
| 23 |
+
"cameras": [
|
| 24 |
+
"ego_view",
|
| 25 |
+
"chest"
|
| 26 |
+
],
|
| 27 |
+
"state_dim": 44,
|
| 28 |
+
"action_dim": 76,
|
| 29 |
+
"state_layout": "state44 = 29 body joint angles + measured Inspire hands 6 + 6 + projected gravity 3",
|
| 30 |
+
"action_layout": "action76 = action.motion_token (64) + measured Inspire hands (6 + 6), same-row labels",
|
| 31 |
+
"trainable": "full fine-tune of the mixture-of-transformers DiT: video expert (Wan2.2-TI2V-5B, 5.00 B) + action expert (1.02 B); the Wan2.2 VAE and the T5 text encoder are frozen (prompt embeddings precomputed, text_embeddings/)",
|
| 32 |
+
"recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
|
| 33 |
+
"video": "17-frame clips (4 latent frames + the observation frame), 224 x 448 (the two cameras side by side), action chunk 16",
|
| 34 |
+
"base_model": "Wan-AI/Wan2.2-TI2V-5B (DiT, VAE, T5) + the umt5-xxl tokenizer of Wan-AI/Wan2.1-T2V-1.3B, loaded from their own repositories; ActionDiT initialised by FastWAM's preprocess_action_dit_backbone.py",
|
| 35 |
+
"upstream": "github.com/yuantianyuan01/FastWAM @ 7faa71108368fbb3b6885649f112af607427a2d4",
|
| 36 |
+
"batch_selection": "largest per-GPU batch that fits on A800 80 GB: 16 trains at ~58 GB, 32 runs out of memory (batch_benchmark/)",
|
| 37 |
+
"normalization": "dataset_stats.json (FastWAM min / max statistics of this task)",
|
| 38 |
+
"config": "training_config.yaml (resolved FastWAM config; absolute paths refer to the training NAS), model_config.yaml (the model part)",
|
| 39 |
+
"inference": "FastWAM: build the model from model_config.yaml, load model.pt (DiT state dict), the normalizer from dataset_stats.json and the prompt embedding from text_embeddings/; code/hb_fastwam_convert.py documents the state / action layout of the benchmark interface; code/ad_fastwam.py is the inference adapter of our evaluation (action-only FastWAM.infer_action, 10 denoising steps, seed 42; its default paths refer to the training NAS)",
|
| 40 |
+
"training_records_included": true,
|
| 41 |
+
"resume_state_included": false,
|
| 42 |
+
"files": {
|
| 43 |
+
"batch_benchmark/chosen.json": 210,
|
| 44 |
+
"batch_benchmark/summary.txt": 127,
|
| 45 |
+
"code/ad_fastwam.py": 8413,
|
| 46 |
+
"code/fastwam_prep.sh": 3634,
|
| 47 |
+
"code/fastwam_task.sh": 3719,
|
| 48 |
+
"code/hb_aug_v2.py": 7475,
|
| 49 |
+
"code/hb_eval_common.py": 13471,
|
| 50 |
+
"code/hb_fastwam_aug.py": 3345,
|
| 51 |
+
"code/hb_fastwam_convert.py": 7101,
|
| 52 |
+
"code/hb_fastwam_train.py": 5570,
|
| 53 |
+
"code/hb_inloop_core.py": 11206,
|
| 54 |
+
"code/hb_inloop_fastwam.py": 5431,
|
| 55 |
+
"code/test_fastwam_inputs.py": 2724,
|
| 56 |
+
"complete.json": 653,
|
| 57 |
+
"dataset_meta/hb_fastwam_interface.json": 334,
|
| 58 |
+
"dataset_meta/info.json": 11331,
|
| 59 |
+
"dataset_meta/tasks.jsonl": 213,
|
| 60 |
+
"dataset_stats.json": 194885,
|
| 61 |
+
"logs/node-0.log": 571827,
|
| 62 |
+
"logs/node-1.log": 474197,
|
| 63 |
+
"logs/node-2.log": 474507,
|
| 64 |
+
"logs/node-3.log": 474604,
|
| 65 |
+
"loss_summary.json": 383,
|
| 66 |
+
"metrics.csv": 8445,
|
| 67 |
+
"model.pt": 12042313337,
|
| 68 |
+
"model_config.yaml": 1350,
|
| 69 |
+
"text_embeddings/5a20e43a27564ea8978edc7394a96e0c1295e82528d59dba0c4169f3bba9a630.t5_len128.wan22ti2v5b.pt": 12587613,
|
| 70 |
+
"training_config.yaml": 5872
|
| 71 |
+
}
|
| 72 |
+
}
|
b8-table-wipe/b512/step-002000/text_embeddings/5a20e43a27564ea8978edc7394a96e0c1295e82528d59dba0c4169f3bba9a630.t5_len128.wan22ti2v5b.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a142c83793090d53a27bdb95e939112cf4007a13c751b7b68cfa7e433f6a5e02
|
| 3 |
+
size 12587613
|
b8-table-wipe/b512/step-002000/training_config.yaml
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
output_dir: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/runs/b8/out
|
| 2 |
+
batch_size: 16
|
| 3 |
+
num_workers: 8
|
| 4 |
+
lr_scheduler_type: cosine
|
| 5 |
+
learning_rate: 0.0001
|
| 6 |
+
num_epochs: 10
|
| 7 |
+
max_steps: 4000
|
| 8 |
+
log_every: 20
|
| 9 |
+
save_every: 2000
|
| 10 |
+
eval_every: 0
|
| 11 |
+
eval_num_inference_steps: 10
|
| 12 |
+
gradient_accumulation_steps: 1
|
| 13 |
+
mixed_precision: bf16
|
| 14 |
+
seed: 42
|
| 15 |
+
max_grad_norm: 1.0
|
| 16 |
+
weight_decay: 0.01
|
| 17 |
+
resume: null
|
| 18 |
+
wandb:
|
| 19 |
+
enabled: false
|
| 20 |
+
workspace: null
|
| 21 |
+
project: fast-wam
|
| 22 |
+
name: train
|
| 23 |
+
group: null
|
| 24 |
+
mode: online
|
| 25 |
+
data:
|
| 26 |
+
train:
|
| 27 |
+
_target_: fastwam.datasets.lerobot.robot_video_dataset.RobotVideoDataset
|
| 28 |
+
dataset_dirs:
|
| 29 |
+
- /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/data/b8
|
| 30 |
+
shape_meta:
|
| 31 |
+
images:
|
| 32 |
+
- key: ego_view
|
| 33 |
+
raw_shape:
|
| 34 |
+
- 3
|
| 35 |
+
- 224
|
| 36 |
+
- 224
|
| 37 |
+
shape:
|
| 38 |
+
- 3
|
| 39 |
+
- 224
|
| 40 |
+
- 224
|
| 41 |
+
- key: chest
|
| 42 |
+
raw_shape:
|
| 43 |
+
- 3
|
| 44 |
+
- 224
|
| 45 |
+
- 224
|
| 46 |
+
shape:
|
| 47 |
+
- 3
|
| 48 |
+
- 224
|
| 49 |
+
- 224
|
| 50 |
+
action:
|
| 51 |
+
- key: default
|
| 52 |
+
raw_shape: 76
|
| 53 |
+
shape: 76
|
| 54 |
+
state:
|
| 55 |
+
- key: default
|
| 56 |
+
raw_shape: 44
|
| 57 |
+
shape: 44
|
| 58 |
+
num_frames: 17
|
| 59 |
+
global_sample_stride: 1
|
| 60 |
+
action_video_freq_ratio: 4
|
| 61 |
+
video_size:
|
| 62 |
+
- 224
|
| 63 |
+
- 448
|
| 64 |
+
camera_key: null
|
| 65 |
+
val_set_proportion: 0.0
|
| 66 |
+
is_training_set: true
|
| 67 |
+
skip_padding_as_possible: false
|
| 68 |
+
concat_multi_camera: horizontal
|
| 69 |
+
processor:
|
| 70 |
+
_target_: fastwam.datasets.lerobot.processors.fastwam_processor.FastWAMProcessor
|
| 71 |
+
shape_meta:
|
| 72 |
+
images:
|
| 73 |
+
- key: ego_view
|
| 74 |
+
raw_shape:
|
| 75 |
+
- 3
|
| 76 |
+
- 224
|
| 77 |
+
- 224
|
| 78 |
+
shape:
|
| 79 |
+
- 3
|
| 80 |
+
- 224
|
| 81 |
+
- 224
|
| 82 |
+
- key: chest
|
| 83 |
+
raw_shape:
|
| 84 |
+
- 3
|
| 85 |
+
- 224
|
| 86 |
+
- 224
|
| 87 |
+
shape:
|
| 88 |
+
- 3
|
| 89 |
+
- 224
|
| 90 |
+
- 224
|
| 91 |
+
action:
|
| 92 |
+
- key: default
|
| 93 |
+
raw_shape: 76
|
| 94 |
+
shape: 76
|
| 95 |
+
state:
|
| 96 |
+
- key: default
|
| 97 |
+
raw_shape: 44
|
| 98 |
+
shape: 44
|
| 99 |
+
num_obs_steps: 17
|
| 100 |
+
num_output_cameras: 2
|
| 101 |
+
action_output_dim: 76
|
| 102 |
+
proprio_output_dim: 44
|
| 103 |
+
delta_action_dim_mask:
|
| 104 |
+
default:
|
| 105 |
+
- false
|
| 106 |
+
- false
|
| 107 |
+
- false
|
| 108 |
+
- false
|
| 109 |
+
- false
|
| 110 |
+
- false
|
| 111 |
+
- false
|
| 112 |
+
- false
|
| 113 |
+
- false
|
| 114 |
+
- false
|
| 115 |
+
- false
|
| 116 |
+
- false
|
| 117 |
+
- false
|
| 118 |
+
- false
|
| 119 |
+
- false
|
| 120 |
+
- false
|
| 121 |
+
- false
|
| 122 |
+
- false
|
| 123 |
+
- false
|
| 124 |
+
- false
|
| 125 |
+
- false
|
| 126 |
+
- false
|
| 127 |
+
- false
|
| 128 |
+
- false
|
| 129 |
+
- false
|
| 130 |
+
- false
|
| 131 |
+
- false
|
| 132 |
+
- false
|
| 133 |
+
- false
|
| 134 |
+
- false
|
| 135 |
+
- false
|
| 136 |
+
- false
|
| 137 |
+
- false
|
| 138 |
+
- false
|
| 139 |
+
- false
|
| 140 |
+
- false
|
| 141 |
+
- false
|
| 142 |
+
- false
|
| 143 |
+
- false
|
| 144 |
+
- false
|
| 145 |
+
- false
|
| 146 |
+
- false
|
| 147 |
+
- false
|
| 148 |
+
- false
|
| 149 |
+
- false
|
| 150 |
+
- false
|
| 151 |
+
- false
|
| 152 |
+
- false
|
| 153 |
+
- false
|
| 154 |
+
- false
|
| 155 |
+
- false
|
| 156 |
+
- false
|
| 157 |
+
- false
|
| 158 |
+
- false
|
| 159 |
+
- false
|
| 160 |
+
- false
|
| 161 |
+
- false
|
| 162 |
+
- false
|
| 163 |
+
- false
|
| 164 |
+
- false
|
| 165 |
+
- false
|
| 166 |
+
- false
|
| 167 |
+
- false
|
| 168 |
+
- false
|
| 169 |
+
- false
|
| 170 |
+
- false
|
| 171 |
+
- false
|
| 172 |
+
- false
|
| 173 |
+
- false
|
| 174 |
+
- false
|
| 175 |
+
- false
|
| 176 |
+
- false
|
| 177 |
+
- false
|
| 178 |
+
- false
|
| 179 |
+
- false
|
| 180 |
+
- false
|
| 181 |
+
action_state_transforms: null
|
| 182 |
+
use_stepwise_action_norm: false
|
| 183 |
+
norm_default_mode: min/max
|
| 184 |
+
norm_exception_mode: null
|
| 185 |
+
action_state_merger:
|
| 186 |
+
_target_: fastwam.datasets.lerobot.transforms.action_state_merger.ConcatLeftAlign
|
| 187 |
+
train_transforms:
|
| 188 |
+
- _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
|
| 189 |
+
- _target_: torchvision.transforms.Resize
|
| 190 |
+
size:
|
| 191 |
+
- 224
|
| 192 |
+
- 224
|
| 193 |
+
val_transforms:
|
| 194 |
+
- _target_: fastwam.datasets.lerobot.transforms.image.ToTensor
|
| 195 |
+
- _target_: torchvision.transforms.Resize
|
| 196 |
+
size:
|
| 197 |
+
- 224
|
| 198 |
+
- 224
|
| 199 |
+
text_embedding_cache_dir: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/text-cache/all
|
| 200 |
+
use_text_embed_cache: true
|
| 201 |
+
context_len: 128
|
| 202 |
+
model:
|
| 203 |
+
_target_: fastwam.runtime.create_fastwam
|
| 204 |
+
model_id: Wan-AI/Wan2.2-TI2V-5B
|
| 205 |
+
tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
|
| 206 |
+
tokenizer_max_len: 128
|
| 207 |
+
load_text_encoder: false
|
| 208 |
+
proprio_dim: 44
|
| 209 |
+
redirect_common_files: false
|
| 210 |
+
mot_checkpoint_mixed_attn: false
|
| 211 |
+
compile_training_denoise: false
|
| 212 |
+
action_dit_pretrained_path: /mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/action-init-g1.pt
|
| 213 |
+
skip_dit_load_from_pretrain: false
|
| 214 |
+
video_dit_config:
|
| 215 |
+
has_image_input: false
|
| 216 |
+
patch_size:
|
| 217 |
+
- 1
|
| 218 |
+
- 2
|
| 219 |
+
- 2
|
| 220 |
+
in_dim: 48
|
| 221 |
+
hidden_dim: 3072
|
| 222 |
+
ffn_dim: 14336
|
| 223 |
+
freq_dim: 256
|
| 224 |
+
text_dim: 4096
|
| 225 |
+
out_dim: 48
|
| 226 |
+
num_heads: 24
|
| 227 |
+
attn_head_dim: 128
|
| 228 |
+
num_layers: 30
|
| 229 |
+
eps: 1.0e-06
|
| 230 |
+
seperated_timestep: true
|
| 231 |
+
require_clip_embedding: false
|
| 232 |
+
require_vae_embedding: false
|
| 233 |
+
fuse_vae_embedding_in_latents: true
|
| 234 |
+
use_gradient_checkpointing: false
|
| 235 |
+
video_attention_mask_mode: first_frame_causal
|
| 236 |
+
action_conditioned: false
|
| 237 |
+
action_dim: 76
|
| 238 |
+
action_group_causal_mask_mode: group_diagonal
|
| 239 |
+
action_dit_config:
|
| 240 |
+
action_dim: 76
|
| 241 |
+
hidden_dim: 1024
|
| 242 |
+
ffn_dim: 4096
|
| 243 |
+
num_heads: 24
|
| 244 |
+
attn_head_dim: 128
|
| 245 |
+
num_layers: 30
|
| 246 |
+
text_dim: 4096
|
| 247 |
+
freq_dim: 256
|
| 248 |
+
eps: 1.0e-06
|
| 249 |
+
use_gradient_checkpointing: false
|
| 250 |
+
video_scheduler:
|
| 251 |
+
train_shift: 5.0
|
| 252 |
+
infer_shift: 5.0
|
| 253 |
+
num_train_timesteps: 1000
|
| 254 |
+
action_scheduler:
|
| 255 |
+
train_shift: 1.0
|
| 256 |
+
infer_shift: 1.0
|
| 257 |
+
num_train_timesteps: 1000
|
| 258 |
+
loss:
|
| 259 |
+
lambda_action: 1.0
|
b8-table-wipe/b512/step-004000/batch_benchmark/chosen.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"per_gpu_batch": 16, "global_batch": 512, "source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/policies-20261109/fastwam/bench-20261116-r2/summary.txt"}
|
b8-table-wipe/b512/step-004000/batch_benchmark/summary.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
bs=16 global=512 rc=[0 0 0 0 ] oom=0 peak_mem_MiB=58421 speed=
|
| 2 |
+
bs=32 global=1024 rc=[1 1 1 1 ] oom=4 peak_mem_MiB=81123 speed=
|
b8-table-wipe/b512/step-004000/code/ad_fastwam.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastWAM adapter (2026-11-16). Deployment path of FastWAM's own policy (experiments/robotwin/fastwam_policy/
|
| 2 |
+
deploy_policy.py, WorldActionRobotWinPolicy._infer_action_chunk: model.infer_action, the action-only path = first-frame
|
| 3 |
+
video KV cache + action-expert denoising, no future video) fed exactly like our training dataset (RobotVideoDataset +
|
| 4 |
+
FastWAMProcessor of the run's config.yaml, no HB augmentation):
|
| 5 |
+
* cameras in shape_meta order (g1 ego_view + chest, h2 head + chest): uint8 [1,C,H,W] -> processor val transforms
|
| 6 |
+
(ToTensor /255 + Resize 224x224), concatenated horizontally (224 x 448), dataset ResizeSmallestSide/CenterCrop
|
| 7 |
+
(no-op at 224 x 448), Normalize(0.5, 0.5) -> [-1, 1];
|
| 8 |
+
* state: processor.action_state_transform -> normalizer.forward (min/max of the run's
|
| 9 |
+
dataset_stats.json), the proprio of the first frame;
|
| 10 |
+
* text: the run's T5 cache for DEFAULT_PROMPT.format(task=instruction) with context[~mask] = 0 and mask = ones, as in
|
| 11 |
+
training (the text encoder is not loaded in training either; deploy_policy encodes the same prompt with T5 online);
|
| 12 |
+
* infer_action(action_horizon = num_frames - 1 = 16, num_inference_steps = config eval_num_inference_steps (10),
|
| 13 |
+
text_cfg_scale 1, seed 42 = native trainer eval), denormalised with normalizer.normalizers["action"][key].backward.
|
| 14 |
+
ckpt = .../fastwam/runs/<task>/out/checkpoints/weights/step_<N>.pt (config.yaml + dataset_stats.json in .../out).
|
| 15 |
+
Weights: model built from the config (Wan2.2 + ActionDiT init), then the checkpoint's mot + proprio_encoder are
|
| 16 |
+
loaded STRICTLY (FastWAM.load_checkpoint uses strict=False for mot; a missing key here is an error).
|
| 17 |
+
"""
|
| 18 |
+
import os
|
| 19 |
+
import sys
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
|
| 24 |
+
import hb_eval_common as E
|
| 25 |
+
|
| 26 |
+
FW = E.G / "fastwam"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def setup_env():
|
| 30 |
+
os.environ.setdefault("FASTWAM_SRC", str(FW / "FastWAM"))
|
| 31 |
+
os.environ.setdefault("DIFFSYNTH_MODEL_BASE_PATH", str(FW / "models"))
|
| 32 |
+
os.environ.setdefault("DIFFSYNTH_SKIP_DOWNLOAD", "true")
|
| 33 |
+
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
| 34 |
+
src = str(Path(os.environ["FASTWAM_SRC"]) / "src")
|
| 35 |
+
if src not in sys.path:
|
| 36 |
+
sys.path.insert(0, src)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class Inputs:
|
| 40 |
+
"""model-agnostic cache query -> FastWAM inputs (shared by the post-hoc adapter and the in-training eval)"""
|
| 41 |
+
|
| 42 |
+
def __init__(self, cfg, stats_path):
|
| 43 |
+
setup_env()
|
| 44 |
+
from hydra.utils import instantiate
|
| 45 |
+
from omegaconf import OmegaConf
|
| 46 |
+
from fastwam.datasets.dataset_utils import CenterCrop, Normalize, ResizeSmallestSideAspectPreserving
|
| 47 |
+
from fastwam.datasets.lerobot.robot_video_dataset import DEFAULT_PROMPT
|
| 48 |
+
from fastwam.datasets.lerobot.utils.normalizer import load_dataset_stats_from_json
|
| 49 |
+
d = cfg.data.train
|
| 50 |
+
self.DEFAULT_PROMPT = DEFAULT_PROMPT
|
| 51 |
+
self.proc = instantiate(d.processor).eval()
|
| 52 |
+
self.proc.set_normalizer_from_stats(load_dataset_stats_from_json(str(stats_path)))
|
| 53 |
+
self.cams = [m["key"] for m in self.proc.shape_meta["images"]]
|
| 54 |
+
if d.concat_multi_camera != "horizontal":
|
| 55 |
+
raise ValueError(f"concat_multi_camera {d.concat_multi_camera} not supported")
|
| 56 |
+
vs = list(d.video_size)
|
| 57 |
+
self.resize = ResizeSmallestSideAspectPreserving(args={"img_w": vs[1], "img_h": vs[0]})
|
| 58 |
+
self.crop = CenterCrop(args={"img_w": vs[1], "img_h": vs[0]})
|
| 59 |
+
self.norm = Normalize(args={"mean": 0.5, "std": 0.5})
|
| 60 |
+
self.cache_dir = d.text_embedding_cache_dir
|
| 61 |
+
self.context_len = int(d.context_len)
|
| 62 |
+
self.horizon = int(d.num_frames) - 1
|
| 63 |
+
self.skey = self.proc.shape_meta["state"][0]["key"]
|
| 64 |
+
self.akey = self.proc.shape_meta["action"][0]["key"]
|
| 65 |
+
if len(self.proc.shape_meta["state"]) != 1 or len(self.proc.shape_meta["action"]) != 1:
|
| 66 |
+
raise ValueError("expected one merged state key and one merged action key")
|
| 67 |
+
self._ctx = {}
|
| 68 |
+
self.OmegaConf = OmegaConf
|
| 69 |
+
|
| 70 |
+
def image(self, images):
|
| 71 |
+
import torch
|
| 72 |
+
per = []
|
| 73 |
+
for c in self.cams:
|
| 74 |
+
x = torch.from_numpy(np.ascontiguousarray(images[c])).permute(2, 0, 1).unsqueeze(0) # [1,C,H,W] uint8
|
| 75 |
+
for tr in self.proc.val_transforms:
|
| 76 |
+
x = tr(x)
|
| 77 |
+
per.append(x)
|
| 78 |
+
v = torch.cat(per, dim=-1) # [1, C, 224, 448]
|
| 79 |
+
v = self.norm(self.crop(self.resize(v)))
|
| 80 |
+
return v # [1, 3, H, W] in [-1, 1], float32
|
| 81 |
+
|
| 82 |
+
def proprio(self, state):
|
| 83 |
+
import torch
|
| 84 |
+
b = {"state": {self.skey: torch.as_tensor(np.asarray(state, dtype=np.float32)).unsqueeze(0)}}
|
| 85 |
+
b = self.proc.action_state_transform(b)
|
| 86 |
+
b = self.proc.normalizer.forward(b) # = deploy_policy._normalize_state (one merged key: merger is identity)
|
| 87 |
+
return b["state"][self.skey] # [1, D]
|
| 88 |
+
|
| 89 |
+
def context(self, instruction):
|
| 90 |
+
import hashlib
|
| 91 |
+
import torch
|
| 92 |
+
prompt = self.DEFAULT_PROMPT.format(task=instruction)
|
| 93 |
+
if prompt not in self._ctx:
|
| 94 |
+
h = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
|
| 95 |
+
p = os.path.join(self.cache_dir, f"{h}.t5_len{self.context_len}.wan22ti2v5b.pt")
|
| 96 |
+
if not os.path.exists(p):
|
| 97 |
+
raise FileNotFoundError(f"no T5 cache for the instruction ({p}): {instruction[:80]!r}")
|
| 98 |
+
obj = torch.load(p, map_location="cpu")
|
| 99 |
+
ctx, mask = obj["context"], obj["mask"]
|
| 100 |
+
ctx = ctx.clone()
|
| 101 |
+
ctx[~mask.bool()] = 0.0
|
| 102 |
+
self._ctx[prompt] = (ctx, torch.ones_like(mask.bool()))
|
| 103 |
+
return self._ctx[prompt]
|
| 104 |
+
|
| 105 |
+
def denorm(self, action):
|
| 106 |
+
import torch
|
| 107 |
+
a = action
|
| 108 |
+
if a.ndim == 2:
|
| 109 |
+
a = a.unsqueeze(0)
|
| 110 |
+
n = self.proc.normalizer.normalizers["action"][self.akey]
|
| 111 |
+
return n.backward(a.to(dtype=torch.float32, device="cpu")).numpy()[0]
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def infer(model, inp, obs, steps, seed=42):
|
| 115 |
+
import torch
|
| 116 |
+
img = inp.image(obs["images"]).to(device=model.device, dtype=model.torch_dtype)
|
| 117 |
+
ctx, mask = inp.context(obs["prompt"])
|
| 118 |
+
with torch.no_grad():
|
| 119 |
+
out = model.infer_action(prompt=None, input_image=img, action_horizon=inp.horizon, proprio=inp.proprio(obs["state"]),
|
| 120 |
+
context=ctx, context_mask=mask, text_cfg_scale=1.0, num_inference_steps=steps, seed=seed,
|
| 121 |
+
rand_device="cpu", tiled=False)
|
| 122 |
+
return inp.denorm(out["action"])
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class Adapter:
|
| 126 |
+
def __init__(self, ckpt, task, split, meta, device="cuda:0", steps=None, seed="42"):
|
| 127 |
+
setup_env()
|
| 128 |
+
import torch
|
| 129 |
+
from hydra.utils import instantiate
|
| 130 |
+
from omegaconf import OmegaConf
|
| 131 |
+
self.torch = torch
|
| 132 |
+
ckpt = Path(ckpt)
|
| 133 |
+
out = ckpt.parent.parent.parent
|
| 134 |
+
self.cfg = OmegaConf.load(out / "config.yaml")
|
| 135 |
+
self.inp = Inputs(self.cfg, out / "dataset_stats.json")
|
| 136 |
+
miss = [c for c in self.inp.cams if c not in meta["cams"]]
|
| 137 |
+
if miss:
|
| 138 |
+
raise KeyError(f"cameras {miss} not in the cache {meta['cams']}")
|
| 139 |
+
self.device = device if torch.cuda.is_available() else "cpu"
|
| 140 |
+
mp = str(self.cfg.get("mixed_precision", "bf16")).lower()
|
| 141 |
+
dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[mp]
|
| 142 |
+
self.model = instantiate(self.cfg.model, model_dtype=dtype, device=self.device)
|
| 143 |
+
payload = torch.load(str(ckpt), map_location="cpu")
|
| 144 |
+
miss_k, unexp = self.model.mot.load_state_dict(payload["mot"], strict=False)
|
| 145 |
+
if miss_k or unexp:
|
| 146 |
+
raise RuntimeError(f"checkpoint mot keys: missing {list(miss_k)[:5]} unexpected {list(unexp)[:5]}")
|
| 147 |
+
if self.model.proprio_encoder is not None:
|
| 148 |
+
self.model.proprio_encoder.load_state_dict(payload["proprio_encoder"], strict=True)
|
| 149 |
+
self.ckpt_step = payload.get("step")
|
| 150 |
+
del payload
|
| 151 |
+
self.model = self.model.to(self.device).eval()
|
| 152 |
+
self.native_steps = int(self.cfg.eval_num_inference_steps)
|
| 153 |
+
self.steps = int(steps or self.native_steps)
|
| 154 |
+
self.seed = int(seed)
|
| 155 |
+
self.ckpt = str(ckpt)
|
| 156 |
+
|
| 157 |
+
def info(self):
|
| 158 |
+
return {"model": "fastwam", "ckpt": self.ckpt, "ckpt_step": self.ckpt_step, "denoising_steps": self.steps,
|
| 159 |
+
"native_steps_in_config": self.native_steps, "cams": self.inp.cams, "chunk": self.inp.horizon,
|
| 160 |
+
"seed": self.seed, "path": "FastWAM.infer_action (action-only, deploy_policy)"}
|
| 161 |
+
|
| 162 |
+
def predict(self, obs):
|
| 163 |
+
return infer(self.model, self.inp, obs, self.steps, self.seed)
|
b8-table-wipe/b512/step-004000/code/fastwam_prep.sh
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# 2026-11-16: CPU-only preparation of FastWAM for the 12 Delta-flow tasks (no GPU is touched: training is running).
|
| 3 |
+
# Data copies on our interface, ActionDiT initializers per profile, one shared T5 text cache, augmented-sample checks.
|
| 4 |
+
P=/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z
|
| 5 |
+
G=$P/policies-20261109
|
| 6 |
+
F=$G/fastwam
|
| 7 |
+
E=$G/envs/fastwam/bin/python
|
| 8 |
+
exec >> $F/prep.log 2>&1
|
| 9 |
+
echo "$(date) start (v2: 12 tasks)"
|
| 10 |
+
until [ -f $G/envs/fastwam/.hb_ready ] || grep -q ENV_FAILED $F/setup.log; do sleep 30; done
|
| 11 |
+
[ -f $G/envs/fastwam/.hb_ready ] || { echo "ENV_FAILED"; exit 1; }
|
| 12 |
+
grep -q FW_DL_DONE $F/dl2.log || { echo "DOWNLOAD_FAILED"; exit 1; }
|
| 13 |
+
export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true CUDA_VISIBLE_DEVICES="" HF_HUB_OFFLINE=1
|
| 14 |
+
export HB_AUG_DIR=$G/hb_groot
|
| 15 |
+
unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT $(compgen -e | grep -E "^PET_")
|
| 16 |
+
TC=$F/text-cache/all
|
| 17 |
+
# task profile source [first N episodes]
|
| 18 |
+
cat > $F/tasks.txt <<EOF
|
| 19 |
+
a10v3 g1 $P/a10-trolley-push-v3-6e410c5/data
|
| 20 |
+
a3v4 g1 $P/a3-fridge-open-v4-6393d4a/data
|
| 21 |
+
a7v4 g1 $P/a7-cabinet-close-v4-90cc2b5/data
|
| 22 |
+
b7 g1 $P/b7-black-clothes-hamper-v1-1bb002a/data 100
|
| 23 |
+
b5 g1 $P/b5-chair-push-v1-b85b7e9/data
|
| 24 |
+
b8 g1 $P/b8-table-wipe-v1-3b2e618/data
|
| 25 |
+
a17cr2 h2 $P/a17-unscrew-bottle-cropped-v2-3520b70/data
|
| 26 |
+
basketv2 g1 $P/basket-trimv2-c4df4d4/data
|
| 27 |
+
c6 g1 $P/c6-bathroom-door-fetch-v1-3afba69/data
|
| 28 |
+
b6v6 g1 $P/b6-pedal-bin-v6-ad2f5bf/data
|
| 29 |
+
a14v2 h2p $P/a14-unscrew-bulb-v2-8411a6d/data
|
| 30 |
+
b1 g1 $P/b1-riceball-microwave-v1-36da298/data
|
| 31 |
+
EOF
|
| 32 |
+
DIRS=""
|
| 33 |
+
while read t prof src lim; do
|
| 34 |
+
D=$F/data/$t
|
| 35 |
+
if [ ! -f $D/meta/hb_fastwam_interface.json ]; then
|
| 36 |
+
$E $F/tools/hb_fastwam_convert.py $prof $src $D ${lim:+--episodes $lim} || { echo "CONVERT_FAILED $t"; continue; }
|
| 37 |
+
fi
|
| 38 |
+
DIRS="$DIRS,\"$D\""
|
| 39 |
+
done < $F/tasks.txt
|
| 40 |
+
DIRS="[${DIRS#,}]"
|
| 41 |
+
echo "$(date) CONVERTED $DIRS"
|
| 42 |
+
cd $F/FastWAM
|
| 43 |
+
for prof in g1 h2 h2p; do
|
| 44 |
+
AI=$F/action-init-$prof.pt
|
| 45 |
+
[ -f $AI ] && continue
|
| 46 |
+
HB_FW_PROFILE=$prof HB_MODEL_YAML=$F/model-$prof.yaml $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" || { echo "YAML_FAILED $prof"; continue; }
|
| 47 |
+
grep -E 'action_dim|proprio_dim' $F/model-$prof.yaml | tr '\n' ' '; echo
|
| 48 |
+
$E scripts/preprocess_action_dit_backbone.py --model-config $F/model-$prof.yaml --output $AI --device cpu --dtype bfloat16 && echo "$(date) INIT_OK $prof $(stat -c %s $AI)" || echo "INIT_FAILED $prof"
|
| 49 |
+
done
|
| 50 |
+
mkdir -p $TC
|
| 51 |
+
$E scripts/precompute_text_embeds.py task=libero_uncond_2cam224_1e-4 model.redirect_common_files=false "data.train.dataset_dirs=$DIRS" "data.train.text_embedding_cache_dir=$TC" && echo "$(date) TEXT_OK $(ls $TC | wc -l) files" || echo TEXT_FAILED
|
| 52 |
+
ok=1
|
| 53 |
+
for pair in a10v3:g1 a17cr2:h2 a14v2:h2p; do
|
| 54 |
+
t=${pair%%:*}; prof=${pair##*:}
|
| 55 |
+
HB_FW_PROFILE=$prof HB_DATA_SAMPLE=1 HB_SAMPLE_WORKDIR=/tmp/hbfw-sample-$t $E $F/tools/hb_fastwam_train.py "data.train.dataset_dirs=[\"$F/data/$t\"]" "data.train.text_embedding_cache_dir=$TC" "model.action_dit_pretrained_path=$F/action-init-$prof.pt" > $F/sample-$t.log 2>&1
|
| 56 |
+
grep -E 'HB_FW_AUG|HBFW_|Error' $F/sample-$t.log | cut -c1-800
|
| 57 |
+
grep -q "HBFW_SAMPLE" $F/sample-$t.log || { ok=0; echo "SAMPLE_FAILED $t"; }
|
| 58 |
+
done
|
| 59 |
+
for t in $(cut -d' ' -f1 $F/tasks.txt); do [ -f $F/data/$t/meta/hb_fastwam_interface.json ] || { ok=0; echo "MISSING_DATA $t"; }; done
|
| 60 |
+
for prof in g1 h2 h2p; do [ -f $F/action-init-$prof.pt ] || { ok=0; echo "MISSING_INIT $prof"; }; done
|
| 61 |
+
[ $ok = 1 ] && touch $F/bench.ready && echo "$(date) BENCH_READY" || echo "$(date) NOT_READY"
|
b8-table-wipe/b512/step-004000/code/fastwam_task.sh
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Relay cmd item helper (2026-11-16 user): FastWAM on one benchmark task, 4 nodes x 8 GPUs, per-GPU batch from the
|
| 3 |
+
# benchmark ($G/fastwam/chosen.json: largest batch that fits), no gradient accumulation, 4000 steps (user default while
|
| 4 |
+
# the batch is unknown), native Gaussian FM recipe (lr 1e-4 cosine, ZeRO-1, bf16), HB augmentation v2 + proprio
|
| 5 |
+
# perturbation, our state/action interface. Checkpoints at 2000 and 4000. Skips (exit 3) unless the benchmark passed.
|
| 6 |
+
# fastwam_task.sh <task> <profile g1|h2|h2p> <port>
|
| 7 |
+
G=$(dirname "$(dirname "$(realpath "$0")")")
|
| 8 |
+
TASK=$1 PROF=$2 PORT=$3
|
| 9 |
+
F=$G/fastwam; E=$G/envs/fastwam/bin/python; R=$F/runs/$TASK
|
| 10 |
+
RK=${RANK:?}; M0=${MASTER_ADDR:?}
|
| 11 |
+
[ -f $F/fastwam.ok ] && [ -s $F/chosen.json ] || { echo "SKIP fastwam $TASK: benchmark not passed"; exit 3; }
|
| 12 |
+
[ -f $F/data/$TASK/meta/hb_fastwam_interface.json ] && [ -f $F/action-init-$PROF.pt ] || { echo "SKIP fastwam $TASK: data not prepared"; exit 3; }
|
| 13 |
+
BS=$(python3 -c "import json;print(json.load(open('$F/chosen.json'))['per_gpu_batch'])")
|
| 14 |
+
mkdir -p $R/logs
|
| 15 |
+
exec > >(tee -a "$R/logs/node-$RK.log") 2>&1
|
| 16 |
+
date -u; hostname
|
| 17 |
+
[ -f $R/complete.json ] && { echo "already complete"; exit 0; }
|
| 18 |
+
echo "FASTWAM task=$TASK profile=$PROF rank=$RK master=$M0:$PORT per_gpu=$BS global=$((BS * 32)) steps=4000"
|
| 19 |
+
if ! ldconfig -p | grep -q 'libavutil.so.56'; then
|
| 20 |
+
apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
|
| 21 |
+
libavcodec58 libavformat58 libavutil56 libswscale5 libswresample3 libavdevice58 libavfilter7 || true
|
| 22 |
+
fi
|
| 23 |
+
unset RANK WORLD_SIZE LOCAL_RANK GROUP_RANK ROLE_RANK LOCAL_WORLD_SIZE MASTER_ADDR MASTER_PORT
|
| 24 |
+
unset $(compgen -e | grep -E '^PET_') NPROC_PER_NODE WORLD_SIZE_WITH_STANDBY TORCHELASTIC_RUN_ID
|
| 25 |
+
export FASTWAM_SRC=$F/FastWAM DIFFSYNTH_MODEL_BASE_PATH=$F/models DIFFSYNTH_SKIP_DOWNLOAD=true HF_HUB_OFFLINE=1 HB_AUG_DIR=$G/hb_groot HB_FW_PROFILE=$PROF
|
| 26 |
+
export NCCL_SOCKET_IFNAME=eth GLOO_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false WANDB_MODE=disabled
|
| 27 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
| 28 |
+
export PATH=$G/envs/fastwam/bin:$PATH # r2 (11-16 05:3x): r1 failed, accelerate called the missing deepspeed CLI (pdsh launcher)
|
| 29 |
+
for i in $(seq 36); do [ -z "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ] && break; sleep 5; done
|
| 30 |
+
cd $F/FastWAM
|
| 31 |
+
$E -m accelerate.commands.launch --config_file scripts/accelerate_configs/accelerate_zero1_ds.yaml --deepspeed_multinode_launcher standard \
|
| 32 |
+
--num_machines 4 --machine_rank $RK --main_process_ip $M0 --main_process_port $PORT --num_processes 32 \
|
| 33 |
+
$F/tools/hb_fastwam_train.py "output_dir=$R/out" "batch_size=$BS" max_steps=4000 log_every=20 save_every=2000 eval_every=0 \
|
| 34 |
+
num_workers=8 wandb.enabled=false gradient_accumulation_steps=1 "data.train.dataset_dirs=[\"$F/data/$TASK\"]" \
|
| 35 |
+
"data.train.text_embedding_cache_dir=$F/text-cache/all" "model.action_dit_pretrained_path=$F/action-init-$PROF.pt"
|
| 36 |
+
rc=$?
|
| 37 |
+
echo "$rc" > $R/logs/exit-node-$RK
|
| 38 |
+
if [ "$rc" = 0 ] && [ "$RK" = 0 ]; then
|
| 39 |
+
python3 - "$R/complete.json" "$TASK" "$PROF" "$BS" "$F/data/$TASK/meta/hb_fastwam_interface.json" <<'EOF'
|
| 40 |
+
import json, sys, time
|
| 41 |
+
out, task, prof, bs, iface = sys.argv[1:6]
|
| 42 |
+
d = dict(task=task, profile=prof, per_gpu_batch=int(bs), global_batch=int(bs) * 32, gradient_accumulation_steps=1,
|
| 43 |
+
max_steps=4000, save_every=2000, recipe="native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
|
| 44 |
+
interface=json.load(open(iface)), finished_utc=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
|
| 45 |
+
json.dump(d, open(out, 'w'), indent=1); print('COMPLETE', d)
|
| 46 |
+
EOF
|
| 47 |
+
fi
|
| 48 |
+
echo "EXIT $rc"; exit $rc
|
b8-table-wipe/b512/step-004000/code/hb_aug_v2.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Humanoid Benchmark data augmentation v2 (project CLAUDE.md), model-agnostic torch implementation.
|
| 2 |
+
|
| 3 |
+
augment(x, rng, obs_mask=None) takes x: float [T, V, 3, H, W] in [0, 1] (T frames, V cameras) and returns the
|
| 4 |
+
augmented tensor. Each call is one training sample: one of the 15 effects is chosen uniformly and its magnitude
|
| 5 |
+
is scaled by 1.5 around identity; the rest stay at x1. Colour is shared across cameras and time; geometry, light
|
| 6 |
+
patches and white balance are drawn per camera and fixed over time; sensor/optics effects (noise, gaussian blur,
|
| 7 |
+
JPEG, motion blur, sharpen, defocus) apply only to observation frames (obs_mask True).
|
| 8 |
+
Pixel-size parameters are defined at 224 px and scaled by H / 224.
|
| 9 |
+
perturb_joints(q, rng) applies the proprio spec: per-joint bias +-2 deg plus N(0, 0.25 deg) clipped at 3 sigma.
|
| 10 |
+
"""
|
| 11 |
+
import io
|
| 12 |
+
import math
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
from PIL import Image
|
| 18 |
+
from torchvision.transforms import InterpolationMode
|
| 19 |
+
from torchvision.transforms import functional as TF
|
| 20 |
+
|
| 21 |
+
EFFECTS = ("exposure", "contrast", "saturation", "hue", "gamma", "temperature", "rotation_crop", "noise",
|
| 22 |
+
"gaussian_blur", "jpeg", "motion_blur", "sharpen", "defocus", "light_shadow", "wb_channel")
|
| 23 |
+
CENTER = {"exposure": 0.0, "contrast": 1.0, "saturation": 1.0, "hue": 0.0, "gamma": 1.0, "temperature": 0.0}
|
| 24 |
+
RANGE = {"exposure": (-0.22, 0.22), "contrast": (0.88, 1.12), "saturation": (0.76, 1.24), "hue": (-0.015, 0.015),
|
| 25 |
+
"gamma": (0.89, 1.11), "temperature": (-0.066, 0.066)}
|
| 26 |
+
VERSION = "hb-aug-v2-20261109"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def sample(rng, views, height):
|
| 30 |
+
boost = EFFECTS[int(rng.integers(len(EFFECTS)))]
|
| 31 |
+
|
| 32 |
+
def k(name):
|
| 33 |
+
return 1.5 if boost == name else 1.0
|
| 34 |
+
|
| 35 |
+
px = height / 224.0
|
| 36 |
+
p = {"boost": boost}
|
| 37 |
+
for name, (a, b) in RANGE.items():
|
| 38 |
+
c = CENTER[name]
|
| 39 |
+
p[name] = float(rng.uniform(c + k(name) * (a - c), c + k(name) * (b - c)))
|
| 40 |
+
p["geometry"] = []
|
| 41 |
+
for _ in range(views):
|
| 42 |
+
angle = float(rng.uniform(-1.5 * k("rotation_crop"), 1.5 * k("rotation_crop")))
|
| 43 |
+
p["geometry"].append((angle, 1.0 + abs(angle) * 0.015 + float(rng.uniform(0, 0.02 * k("rotation_crop")))))
|
| 44 |
+
p["noise"] = 4.0 / 255.0 * k("noise") if rng.random() < 0.5 else 0.0
|
| 45 |
+
p["gaussian_blur"] = float(rng.uniform(0.25, 0.65)) * k("gaussian_blur") if rng.random() < 0.15 else 0.0
|
| 46 |
+
if rng.random() < 0.35:
|
| 47 |
+
lo = int(round(100 - k("jpeg") * 30))
|
| 48 |
+
hi = int(round(100 - k("jpeg") * 5))
|
| 49 |
+
p["jpeg"] = int(rng.integers(lo, hi + 1))
|
| 50 |
+
else:
|
| 51 |
+
p["jpeg"] = None
|
| 52 |
+
p["motion_blur"] = ((float(rng.uniform(3, 7)) * k("motion_blur") * px, float(rng.uniform(0, 180)))
|
| 53 |
+
if rng.random() < 0.15 else None)
|
| 54 |
+
p["sharpen"] = float(rng.uniform(0.2, 0.6)) * k("sharpen") if rng.random() < 0.15 else 0.0
|
| 55 |
+
p["defocus"] = float(rng.uniform(1, 2)) * k("defocus") * px if rng.random() < 0.10 else 0.0
|
| 56 |
+
if rng.random() < 0.30:
|
| 57 |
+
g = 0.30 * k("light_shadow")
|
| 58 |
+
p["light_shadow"] = [[(float(rng.uniform(0.1, 0.9)), float(rng.uniform(0.1, 0.9)),
|
| 59 |
+
float(rng.uniform(0.15, 0.35)), float(rng.uniform(1 - g, 1 + g)))
|
| 60 |
+
for _ in range(int(rng.integers(1, 3)))] for _ in range(views)]
|
| 61 |
+
else:
|
| 62 |
+
p["light_shadow"] = None
|
| 63 |
+
if rng.random() < 0.30:
|
| 64 |
+
g = 0.10 * k("wb_channel")
|
| 65 |
+
gains = []
|
| 66 |
+
for _ in range(views):
|
| 67 |
+
row = [1.0, 1.0, 1.0]
|
| 68 |
+
row[int(rng.integers(0, 3))] = float(rng.uniform(1 - g, 1 + g))
|
| 69 |
+
gains.append(row)
|
| 70 |
+
p["wb_channel"] = gains
|
| 71 |
+
else:
|
| 72 |
+
p["wb_channel"] = None
|
| 73 |
+
return p
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _conv(x, kernel):
|
| 77 |
+
n, c, h, w = x.shape
|
| 78 |
+
k = kernel.to(x)[None, None].repeat(c, 1, 1, 1)
|
| 79 |
+
pad = kernel.shape[-1] // 2
|
| 80 |
+
return F.conv2d(F.pad(x, (pad, pad, pad, pad), mode="reflect"), k, groups=c)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def motion_kernel(length, angle_deg):
|
| 84 |
+
size = max(3, int(round(length)) | 1)
|
| 85 |
+
k = torch.zeros(size, size)
|
| 86 |
+
c = size // 2
|
| 87 |
+
th = math.radians(angle_deg)
|
| 88 |
+
for s in np.linspace(-(length - 1) / 2, (length - 1) / 2, 8 * size):
|
| 89 |
+
k[int(round(c - s * math.sin(th))), int(round(c + s * math.cos(th)))] = 1.0
|
| 90 |
+
return k / k.sum()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def disk_kernel(r):
|
| 94 |
+
size = 2 * int(math.ceil(r)) + 1
|
| 95 |
+
c = size // 2
|
| 96 |
+
yy, xx = torch.meshgrid(torch.arange(size), torch.arange(size), indexing="ij")
|
| 97 |
+
k = (((yy - c) ** 2 + (xx - c) ** 2) <= (r + 0.5) ** 2).float()
|
| 98 |
+
return k / k.sum()
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def apply(x, p, obs_mask=None, noise_seed=0):
|
| 102 |
+
t, v, c, h, w = x.shape
|
| 103 |
+
x = x.clone().clamp(0, 1)
|
| 104 |
+
for vi, (angle, scale) in enumerate(p["geometry"]):
|
| 105 |
+
x[:, vi] = TF.affine(x[:, vi], angle=angle, translate=[0, 0], scale=scale, shear=[0, 0],
|
| 106 |
+
interpolation=InterpolationMode.BILINEAR)
|
| 107 |
+
if p["light_shadow"] is not None:
|
| 108 |
+
yy, xx = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij")
|
| 109 |
+
for vi, blobs in enumerate(p["light_shadow"]):
|
| 110 |
+
m = torch.ones(h, w)
|
| 111 |
+
for cx, cy, s, g in blobs:
|
| 112 |
+
m = m + (g - 1.0) * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * s * s))
|
| 113 |
+
x[:, vi] = (x[:, vi] * m.clamp(min=0).to(x)).clamp(0, 1)
|
| 114 |
+
if p["wb_channel"] is not None:
|
| 115 |
+
for vi, g in enumerate(p["wb_channel"]):
|
| 116 |
+
x[:, vi] = (x[:, vi] * torch.tensor(g, dtype=x.dtype).view(1, 3, 1, 1)).clamp(0, 1)
|
| 117 |
+
y = x.reshape(t * v, c, h, w)
|
| 118 |
+
y = (y * float(2.0 ** p["exposure"])).clamp(0, 1)
|
| 119 |
+
d = p["temperature"]
|
| 120 |
+
y = (y * y.new_tensor([1.0 + d, 1.0, 1.0 - d]).view(1, 3, 1, 1)).clamp(0, 1)
|
| 121 |
+
y = ((y - 0.5) * p["contrast"] + 0.5).clamp(0, 1)
|
| 122 |
+
y = TF.adjust_saturation(y, p["saturation"])
|
| 123 |
+
y = TF.adjust_hue(y, p["hue"])
|
| 124 |
+
y = y.clamp(0, 1).pow(p["gamma"])
|
| 125 |
+
x = y.reshape(t, v, c, h, w)
|
| 126 |
+
mask = torch.ones(t, dtype=torch.bool) if obs_mask is None else torch.as_tensor(obs_mask, dtype=torch.bool)
|
| 127 |
+
if bool(mask.any()):
|
| 128 |
+
o = x[mask].reshape(-1, c, h, w)
|
| 129 |
+
if p["gaussian_blur"] > 0:
|
| 130 |
+
o = TF.gaussian_blur(o, [3, 3], [p["gaussian_blur"]] * 2)
|
| 131 |
+
if p["motion_blur"] is not None:
|
| 132 |
+
o = _conv(o, motion_kernel(*p["motion_blur"]))
|
| 133 |
+
if p["defocus"] > 0:
|
| 134 |
+
o = _conv(o, disk_kernel(p["defocus"]))
|
| 135 |
+
if p["sharpen"] > 0:
|
| 136 |
+
o = o + p["sharpen"] * (o - TF.gaussian_blur(o, [5, 5], [1.0, 1.0]))
|
| 137 |
+
o = o.clamp(0, 1)
|
| 138 |
+
if p["noise"] > 0:
|
| 139 |
+
gen = torch.Generator().manual_seed(int(noise_seed))
|
| 140 |
+
o = (o + torch.randn(o.shape, generator=gen, dtype=o.dtype) * p["noise"]).clamp(0, 1)
|
| 141 |
+
if p["jpeg"] is not None:
|
| 142 |
+
for i in range(o.shape[0]):
|
| 143 |
+
buf = io.BytesIO()
|
| 144 |
+
TF.to_pil_image(o[i]).save(buf, format="JPEG", quality=int(p["jpeg"]))
|
| 145 |
+
buf.seek(0)
|
| 146 |
+
with Image.open(buf) as im:
|
| 147 |
+
o[i] = TF.pil_to_tensor(im.convert("RGB")).to(o.dtype).div(255.0)
|
| 148 |
+
x[mask] = o.reshape(-1, v, c, h, w)
|
| 149 |
+
return x.clamp(0, 1)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def augment(x, rng, obs_mask=None):
|
| 153 |
+
p = sample(rng, x.shape[1], x.shape[-2])
|
| 154 |
+
return apply(x, p, obs_mask, noise_seed=int(rng.integers(0, 2 ** 62))), p
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def perturb_joints(q, rng):
|
| 158 |
+
q = np.asarray(q, dtype=np.float64)
|
| 159 |
+
d = q.shape[-1]
|
| 160 |
+
bias = rng.uniform(-math.radians(2.0), math.radians(2.0), size=d)
|
| 161 |
+
sigma = math.radians(0.25)
|
| 162 |
+
noise = np.clip(rng.normal(0.0, sigma, size=q.shape), -3 * sigma, 3 * sigma)
|
| 163 |
+
return (q + bias + noise).astype(np.float32)
|
b8-table-wipe/b512/step-004000/code/hb_eval_common.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Humanoid Benchmark open-loop action-reconstruction evaluation: shared task table, cache layout and metrics.
|
| 2 |
+
|
| 3 |
+
Protocol (identical for every model; mirrors Isaac-GR00T gr00t/eval/open_loop_eval.py):
|
| 4 |
+
* query frames t = 0, 16, 32, ... over the whole episode (non-overlapping 16-step chunks);
|
| 5 |
+
* the model sees the observation at t exactly as at deployment (cameras, raw benchmark state 44/78, instruction;
|
| 6 |
+
Delta Flow also its native 16-step past-action history), all augmentation off;
|
| 7 |
+
* the predicted chunk (first 16 steps, raw action units) is compared with the recorded actions t..t+15;
|
| 8 |
+
rows beyond the episode end are excluded (no clamped / padded targets);
|
| 9 |
+
* metrics: raw MSE / MAE, and nMSE = MSE after dividing every action dim by its std over the task's TRAINING
|
| 10 |
+
frames (floored, see STD_FLOOR_FRAC), overall, per action group and per horizon step.
|
| 11 |
+
Splits: "heldout" = recorded episodes that were never trained on (B6 100-148, trolley 100-102, B7 100-106);
|
| 12 |
+
"train10" = 10 fixed, evenly spaced training episodes (fit on seen data, NOT generalisation).
|
| 13 |
+
Only numpy here: imported by every model environment.
|
| 14 |
+
"""
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import time
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
|
| 22 |
+
G = Path(os.environ.get("G", "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/"
|
| 23 |
+
"preflight-20261104T1648Z/policies-20261109"))
|
| 24 |
+
P = G.parent
|
| 25 |
+
ROOT = G / "eval-action-mse"
|
| 26 |
+
CACHE = Path(os.environ.get("HB_EVAL_CACHE", ROOT / "cache"))
|
| 27 |
+
HORIZON = 16
|
| 28 |
+
STRIDE = 16
|
| 29 |
+
N_TRAIN_EVAL = 10
|
| 30 |
+
STD_FLOOR_FRAC = 0.05 # per-dim std floor = 5 % of the median std of the action vector
|
| 31 |
+
|
| 32 |
+
# task -> (source LeRobot v2.1 root, number of TRAINING episodes = fixed first N (None = all recorded), layout)
|
| 33 |
+
TASKS = {
|
| 34 |
+
"b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100, "g1"),
|
| 35 |
+
"basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None, "g1"),
|
| 36 |
+
"trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100, "g1"),
|
| 37 |
+
"b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100, "g1"),
|
| 38 |
+
"b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None, "g1"),
|
| 39 |
+
"a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None, "g1"),
|
| 40 |
+
"a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None, "g1"),
|
| 41 |
+
"a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None, "g1"),
|
| 42 |
+
"c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None, "g1"),
|
| 43 |
+
"a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None, "g1"),
|
| 44 |
+
"a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None, "g1"),
|
| 45 |
+
"a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None, "h2"),
|
| 46 |
+
"a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None, "h2"),
|
| 47 |
+
"a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None, "h2"),
|
| 48 |
+
# 2026-11-15: newer datasets (GR00T in-training eval needs a cache for every task it trains on)
|
| 49 |
+
"a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None, "g1"),
|
| 50 |
+
"a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None, "g1"),
|
| 51 |
+
"b1": (P / "b1-riceball-microwave-v1-36da298/data", None, "g1"),
|
| 52 |
+
"b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None, "g1"),
|
| 53 |
+
"b8": (P / "b8-table-wipe-v1-3b2e618/data", None, "g1"),
|
| 54 |
+
"basketv2": (P / "basket-trimv2-c4df4d4/data", None, "g1"),
|
| 55 |
+
"a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None, "h2p"), # action 82 = hb75 + SONIC planner 7
|
| 56 |
+
# 2026-11-15 evening: v6/v4 datasets (a3v3, a7v3 retired, not registered)
|
| 57 |
+
"b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None, "g1"), # 542 ep, all used (B6 first-100 rule is v5t7 only)
|
| 58 |
+
"b5": (P / "b5-chair-push-v1-b85b7e9/data", None, "g1"),
|
| 59 |
+
"a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None, "g1"),
|
| 60 |
+
"a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None, "g1"),
|
| 61 |
+
# 2026-11-16: B5 standing start (HF b5-chair-push-standing-start-expert-v1 @913f5a16, 100 ep; FastWAM b5ss)
|
| 62 |
+
"b5ss": (P / "b5-chair-push-standing-start-v1-913f5a1/data", None, "g1"),
|
| 63 |
+
# 2026-11-17: B7 black clothes trimmed (HF b7-black-clothes-hamper @949da46, 107 ep; train = first 100, heldout 100-106)
|
| 64 |
+
"b7t": (P / "b7-black-clothes-hamper-trim-949da46/data", 100, "g1"),
|
| 65 |
+
# 2026-11-15 night: H2 cropped v2 (GR00T h32 relay item 71); head+chest only -> build_cache --cams head,chest
|
| 66 |
+
"a18cr2": (P / "a18-pour-water-cropped-v2-44cdd57/data", None, "h2"), # HF a18-pour-water-expert-cropped-v2 @44cdd571
|
| 67 |
+
"a17cr2": (P / "a17-unscrew-bottle-cropped-v2-3520b70/data", None, "h2"), # HF a17-unscrew-bottle-expert-cropped-v2 @3520b70f
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def task_for_source(source):
|
| 72 |
+
"""benchmark task whose recording root is <source> (groot-data/<view>/hb-dataset-receipt.json "source")"""
|
| 73 |
+
src = Path(source).resolve()
|
| 74 |
+
hits = [t for t, (root, _, _) in TASKS.items() if Path(root).resolve() == src]
|
| 75 |
+
return hits[0] if hits else None
|
| 76 |
+
G1_BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint",
|
| 77 |
+
"left_ankle_pitch_joint", "left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint",
|
| 78 |
+
"right_hip_yaw_joint", "right_knee_joint", "right_ankle_pitch_joint", "right_ankle_roll_joint",
|
| 79 |
+
"waist_yaw_joint", "waist_roll_joint", "waist_pitch_joint", "left_shoulder_pitch_joint",
|
| 80 |
+
"left_shoulder_roll_joint", "left_shoulder_yaw_joint", "left_elbow_joint", "left_wrist_roll_joint",
|
| 81 |
+
"left_wrist_pitch_joint", "left_wrist_yaw_joint", "right_shoulder_pitch_joint",
|
| 82 |
+
"right_shoulder_roll_joint", "right_shoulder_yaw_joint", "right_elbow_joint", "right_wrist_roll_joint",
|
| 83 |
+
"right_wrist_pitch_joint", "right_wrist_yaw_joint"]
|
| 84 |
+
LAYOUTS = {
|
| 85 |
+
# state: body29 | inspire L6 | inspire R6 | projected_gravity3 ; action: motion_token64 | inspire L6 | R6 (same row)
|
| 86 |
+
"g1": dict(state_dim=44, action_dim=76, cams=("ego_view", "chest"),
|
| 87 |
+
groups={"motion_token": (0, 64), "hands": (64, 76)}),
|
| 88 |
+
# state: observation.state75 (legs12 waist3 arms14 head2 hands22+22) | gravity3 ; action: body_target31 | hands44
|
| 89 |
+
"h2": dict(state_dim=78, action_dim=75, cams=("head", "chest", "wrist"),
|
| 90 |
+
groups={"body_target": (0, 31), "hands": (31, 75)}),
|
| 91 |
+
# 2026-11-15 user: A14 v2 is trained on action82 = body_target31 | hands44 | action.planner7 (SONIC planner
|
| 92 |
+
# command); all 82 dims are scored (planner reported as its own group)
|
| 93 |
+
"h2p": dict(state_dim=78, action_dim=82, cams=("head", "chest"),
|
| 94 |
+
groups={"body_target": (0, 31), "hands": (31, 75), "planner": (75, 82)}),
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def jl(path):
|
| 99 |
+
return [json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def layout(task):
|
| 103 |
+
return LAYOUTS[TASKS[task][2]]
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def split_episodes(task, split, n_recorded):
|
| 107 |
+
_, n_train, _ = TASKS[task]
|
| 108 |
+
n_train = n_recorded if n_train is None else n_train
|
| 109 |
+
if split == "heldout":
|
| 110 |
+
return list(range(n_train, n_recorded))
|
| 111 |
+
if split == "train10":
|
| 112 |
+
return sorted({int(round(x)) for x in np.linspace(0, n_train - 1, N_TRAIN_EVAL)})
|
| 113 |
+
if split.startswith("ep:"): # explicit list, e.g. ep:100,101
|
| 114 |
+
return [int(x) for x in split[3:].split(",")]
|
| 115 |
+
raise ValueError(f"unknown split {split!r}")
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def split_dir(task, split):
|
| 119 |
+
return CACHE / task / split
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def load_meta(task, split):
|
| 123 |
+
return json.loads((split_dir(task, split) / "meta.json").read_text())
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def load_stats(task):
|
| 127 |
+
z = np.load(CACHE / task / "stats.npz")
|
| 128 |
+
return {k: z[k] for k in z.files}
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def norm_std(stats):
|
| 132 |
+
std = stats["action_std"].astype(np.float64)
|
| 133 |
+
return np.maximum(std, STD_FLOOR_FRAC * float(np.median(std)))
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def iter_episodes(task, split, max_episodes=None):
|
| 137 |
+
meta = load_meta(task, split)
|
| 138 |
+
eps = meta["episodes"][: max_episodes or None]
|
| 139 |
+
for ep in eps:
|
| 140 |
+
with np.load(split_dir(task, split) / f"ep_{ep:06d}.npz") as z:
|
| 141 |
+
yield ep, {k: z[k] for k in z.files}
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
class Accumulator:
|
| 145 |
+
"""Sums of squared / absolute errors; everything is computed in float64 over valid (row, dim) entries."""
|
| 146 |
+
|
| 147 |
+
def __init__(self, std, groups, horizon=HORIZON):
|
| 148 |
+
self.std = np.asarray(std, dtype=np.float64)
|
| 149 |
+
self.groups = groups
|
| 150 |
+
D = len(self.std)
|
| 151 |
+
self.sse_raw = np.zeros(D)
|
| 152 |
+
self.sae_raw = np.zeros(D)
|
| 153 |
+
self.sse_n = np.zeros(D)
|
| 154 |
+
self.rows = 0
|
| 155 |
+
self.step_sse_n = np.zeros(horizon)
|
| 156 |
+
self.step_rows = np.zeros(horizon)
|
| 157 |
+
self.per_episode = {}
|
| 158 |
+
|
| 159 |
+
def add(self, ep, pred, gt, valid):
|
| 160 |
+
pred = np.asarray(pred, dtype=np.float64)
|
| 161 |
+
gt = np.asarray(gt, dtype=np.float64)
|
| 162 |
+
v = np.asarray(valid, dtype=bool)
|
| 163 |
+
if pred.shape != gt.shape:
|
| 164 |
+
raise ValueError(f"pred {pred.shape} vs gt {gt.shape}")
|
| 165 |
+
if not np.isfinite(pred[v]).all():
|
| 166 |
+
raise FloatingPointError("non-finite prediction")
|
| 167 |
+
e = (pred - gt)[v]
|
| 168 |
+
en = e / self.std
|
| 169 |
+
self.sse_raw += (e ** 2).sum(0)
|
| 170 |
+
self.sae_raw += np.abs(e).sum(0)
|
| 171 |
+
self.sse_n += (en ** 2).sum(0)
|
| 172 |
+
self.rows += int(v.sum())
|
| 173 |
+
k = np.nonzero(v)[0]
|
| 174 |
+
self.step_sse_n[k] += (en ** 2).sum(1)
|
| 175 |
+
self.step_rows[k] += 1
|
| 176 |
+
s = self.per_episode.setdefault(int(ep), [0.0, 0])
|
| 177 |
+
s[0] += float((en ** 2).sum())
|
| 178 |
+
s[1] += int(v.sum()) * len(self.std)
|
| 179 |
+
|
| 180 |
+
def result(self):
|
| 181 |
+
D = len(self.std)
|
| 182 |
+
n = max(self.rows, 1)
|
| 183 |
+
out = {"rows": self.rows, "mse_raw": float(self.sse_raw.sum() / (n * D)),
|
| 184 |
+
"mae_raw": float(self.sae_raw.sum() / (n * D)), "nmse": float(self.sse_n.sum() / (n * D)),
|
| 185 |
+
"groups": {}, "nmse_by_step": (self.step_sse_n / np.maximum(self.step_rows, 1) / D).tolist(),
|
| 186 |
+
"nmse_by_dim": (self.sse_n / n).tolist()}
|
| 187 |
+
for g, (a, b) in self.groups.items():
|
| 188 |
+
out["groups"][g] = {"mse_raw": float(self.sse_raw[a:b].sum() / (n * (b - a))),
|
| 189 |
+
"nmse": float(self.sse_n[a:b].sum() / (n * (b - a)))}
|
| 190 |
+
pe = {ep: s / max(c, 1) for ep, (s, c) in sorted(self.per_episode.items())}
|
| 191 |
+
out["nmse_per_episode"] = pe
|
| 192 |
+
vals = np.array(list(pe.values()))
|
| 193 |
+
if len(vals) > 1: # bootstrap-free spread: std error over episodes
|
| 194 |
+
out["nmse_episode_sem"] = float(vals.std(ddof=1) / np.sqrt(len(vals)))
|
| 195 |
+
return out
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def queries(T):
|
| 199 |
+
return list(range(0, T, STRIDE))
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def gt_chunk(action, t, horizon=HORIZON):
|
| 203 |
+
T = len(action)
|
| 204 |
+
idx = np.arange(t, t + horizon)
|
| 205 |
+
valid = idx < T
|
| 206 |
+
return action[np.minimum(idx, T - 1)], valid
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def run(adapter, task, split, out_path, max_episodes=None, max_queries=None, save_pred=True, log_every=50):
|
| 210 |
+
"""Evaluate one adapter on one cached split; writes <out_path> (json) and <out_path>.pred.npz."""
|
| 211 |
+
meta = load_meta(task, split)
|
| 212 |
+
lay = LAYOUTS[meta["layout"]]
|
| 213 |
+
stats = load_stats(task)
|
| 214 |
+
acc = Accumulator(norm_std(stats), lay["groups"])
|
| 215 |
+
preds, keys = [], []
|
| 216 |
+
t0 = time.time()
|
| 217 |
+
infer_s = 0.0
|
| 218 |
+
nq = 0
|
| 219 |
+
for ep, z in iter_episodes(task, split, max_episodes):
|
| 220 |
+
state, action = z["state"], z["action"]
|
| 221 |
+
qts = list(z["query_t"])
|
| 222 |
+
for qi, t in enumerate(qts):
|
| 223 |
+
obs = {"episode": ep, "t": int(t), "T": len(action), "state": state[t], "prompt": meta["prompts"][str(int(z["task_index"]))],
|
| 224 |
+
"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]},
|
| 225 |
+
"past_action": action[max(0, t - HORIZON):t],
|
| 226 |
+
"_gt_check": action[t:t + HORIZON]} # only for adapters' input-pipeline consistency checks
|
| 227 |
+
s = time.time()
|
| 228 |
+
pred = np.asarray(adapter.predict(obs), dtype=np.float64)
|
| 229 |
+
infer_s += time.time() - s
|
| 230 |
+
if pred.ndim != 2 or pred.shape[0] < HORIZON or pred.shape[1] != lay["action_dim"]:
|
| 231 |
+
raise ValueError(f"adapter returned {pred.shape}, need (>={HORIZON}, {lay['action_dim']})")
|
| 232 |
+
pred = pred[:HORIZON]
|
| 233 |
+
gt, valid = gt_chunk(action, t)
|
| 234 |
+
acc.add(ep, pred, gt, valid)
|
| 235 |
+
if save_pred:
|
| 236 |
+
preds.append(pred.astype(np.float32))
|
| 237 |
+
keys.append((ep, int(t)))
|
| 238 |
+
nq += 1
|
| 239 |
+
if log_every and nq % log_every == 0:
|
| 240 |
+
print(f"[eval] {task}/{split} {nq} queries, running nMSE {acc.result()['nmse']:.4f}, "
|
| 241 |
+
f"{infer_s / nq:.3f} s/query", flush=True)
|
| 242 |
+
if max_queries and nq >= max_queries:
|
| 243 |
+
break
|
| 244 |
+
if max_queries and nq >= max_queries:
|
| 245 |
+
break
|
| 246 |
+
res = acc.result()
|
| 247 |
+
res.update(task=task, split=split, layout=meta["layout"], episodes=sorted(acc.per_episode), queries=nq,
|
| 248 |
+
horizon=HORIZON, stride=STRIDE, std_floor_frac=STD_FLOOR_FRAC, wall_s=round(time.time() - t0, 1),
|
| 249 |
+
infer_s_per_query=round(infer_s / max(nq, 1), 4), adapter=adapter.info(),
|
| 250 |
+
baselines=meta.get("baselines", {}), cache_created=meta.get("created"),
|
| 251 |
+
partial=bool(max_episodes or max_queries), finished_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
| 252 |
+
out_path = Path(out_path)
|
| 253 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 254 |
+
tmp = out_path.with_suffix(".tmp")
|
| 255 |
+
tmp.write_text(json.dumps(res, indent=1))
|
| 256 |
+
tmp.replace(out_path)
|
| 257 |
+
if save_pred and preds:
|
| 258 |
+
np.savez_compressed(str(out_path) + ".pred.npz", pred=np.stack(preds).astype(np.float16),
|
| 259 |
+
keys=np.asarray(keys, dtype=np.int64))
|
| 260 |
+
return res
|
b8-table-wipe/b512/step-004000/code/hb_fastwam_aug.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-16: Humanoid Benchmark augmentation v2 (project CLAUDE.md) inside the native FastWAM data pipeline.
|
| 2 |
+
- Visual: hb_aug_v2.augment on every training sample's frames before the two cameras are concatenated; frame 0 is the
|
| 3 |
+
current observation (sensor / optics effects only there), the later frames are the future video targets (colour,
|
| 4 |
+
per-camera geometry, light patches and white balance shared with frame 0, as the spec requires).
|
| 5 |
+
- Proprio: hb_aug_v2.perturb_joints on the raw body joints (G1 29, H2 31 = body 29 + head 2) before normalization;
|
| 6 |
+
hands and gravity untouched; supervision labels untouched. Normalization statistics are computed on the raw data.
|
| 7 |
+
FastWAM's own image augmentation is only a resize (no colour jitter / crop), so nothing native is stacked.
|
| 8 |
+
Installed in the main process before the dataloaders fork; HB_FW_JOINTS sets the number of perturbed joints."""
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
sys.path.insert(0, os.environ.get("HB_AUG_DIR", ""))
|
| 17 |
+
import hb_aug_v2 as A # noqa: E402
|
| 18 |
+
|
| 19 |
+
NJ = int(os.environ.get("HB_FW_JOINTS", "29"))
|
| 20 |
+
_seen = {"img": False, "state": False}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _rng(idx=0):
|
| 24 |
+
return np.random.default_rng([os.getpid(), int(idx) & 0xFFFFFFFF, time.time_ns() & 0xFFFFFFFF])
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _train(proc):
|
| 28 |
+
v = getattr(proc, "is_train", False)
|
| 29 |
+
return bool(v() if callable(v) else v)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def install():
|
| 33 |
+
from fastwam.datasets.lerobot.base_lerobot_dataset import BaseLerobotDataset
|
| 34 |
+
from fastwam.datasets.lerobot.processors.fastwam_processor import FastWAMProcessor
|
| 35 |
+
|
| 36 |
+
pre0 = FastWAMProcessor.preprocess
|
| 37 |
+
|
| 38 |
+
def preprocess(self, data):
|
| 39 |
+
if _train(self) and isinstance(data.get("state"), dict) and "default" in data["state"]:
|
| 40 |
+
st = data["state"]["default"]
|
| 41 |
+
q = st[..., :NJ].detach().cpu().numpy()
|
| 42 |
+
st = st.clone()
|
| 43 |
+
st[..., :NJ] = torch.from_numpy(A.perturb_joints(q, _rng())).to(st.dtype)
|
| 44 |
+
data["state"]["default"] = st
|
| 45 |
+
if not _seen["state"]:
|
| 46 |
+
_seen["state"] = True
|
| 47 |
+
print(f"HB_FW_AUG proprio perturb_joints on state[..., :{NJ}] shape {tuple(st.shape)}", flush=True)
|
| 48 |
+
return pre0(self, data)
|
| 49 |
+
|
| 50 |
+
get0 = BaseLerobotDataset.__getitem__
|
| 51 |
+
|
| 52 |
+
def getitem(self, idx):
|
| 53 |
+
s = get0(self, idx)
|
| 54 |
+
if self.processor is not None and _train(self.processor) and "pixel_values" in s:
|
| 55 |
+
pv = s["pixel_values"]
|
| 56 |
+
five = pv.ndim == 5
|
| 57 |
+
x = (pv if five else pv[None]).permute(1, 0, 2, 3, 4).float() # [T, V, C, H, W]
|
| 58 |
+
hi = float(x.max())
|
| 59 |
+
scale = 255.0 if hi > 1.5 else 1.0
|
| 60 |
+
mask = torch.zeros(x.shape[0], dtype=torch.bool)
|
| 61 |
+
mask[0] = True
|
| 62 |
+
y, p = A.augment((x / scale).clamp(0, 1), _rng(idx), mask)
|
| 63 |
+
y = (y * scale).permute(1, 0, 2, 3, 4)
|
| 64 |
+
s["pixel_values"] = (y if five else y[0]).to(pv.dtype)
|
| 65 |
+
if not _seen["img"]:
|
| 66 |
+
_seen["img"] = True
|
| 67 |
+
print(f"HB_FW_AUG visual {A.VERSION} on pixel_values {tuple(pv.shape)} dtype {pv.dtype} max {hi:.3f} "
|
| 68 |
+
f"(T={x.shape[0]}, obs frame 0) boost={p['boost']}", flush=True)
|
| 69 |
+
return s
|
| 70 |
+
|
| 71 |
+
FastWAMProcessor.preprocess = preprocess
|
| 72 |
+
BaseLerobotDataset.__getitem__ = getitem
|
| 73 |
+
print(f"HB_FW_AUG installed: {A.VERSION}, joints {NJ}", flush=True)
|
b8-table-wipe/b512/step-004000/code/hb_fastwam_convert.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-16: LeRobot v2.1 copy of one of our datasets for FastWAM on OUR benchmark interface (CLAUDE.md section 4):
|
| 2 |
+
g1 : state44 = observation.state[0:29] (29 body joints) + measured Inspire 6 + 6 + projected gravity 3;
|
| 3 |
+
action76 = action.motion_token 64 + measured Inspire 6 + 6 (same-row labels, as GR00T / Psi-Zero / Delta Flow)
|
| 4 |
+
h2 : state78 = observation.state[0:75] + projected gravity 3; action75 = action.body_target 31 + action.hands 44
|
| 5 |
+
h2p: as h2 with action82 = action75 + action.planner 7 (A14 v2)
|
| 6 |
+
HumanWAM's fastwam_sonic_data.py expects a 43-D G1 state and takes state[0:22] + state[29:36]; on our 41-D state that
|
| 7 |
+
would put left-hand values into the right arm, so it is not used. Source files are never modified; media are symlinked.
|
| 8 |
+
--episodes N keeps the first N episodes (B7: first 100 of 107, as for every other model).
|
| 9 |
+
Usage: hb_fastwam_convert.py <profile> <source data dir> <fresh output dir> [--episodes N]"""
|
| 10 |
+
import copy, json, shutil, sys, tempfile
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
import numpy as np
|
| 13 |
+
import pyarrow as pa
|
| 14 |
+
import pyarrow.parquet as pq
|
| 15 |
+
|
| 16 |
+
G1 = {"observation.state": 41, "action.motion_token": 64, "observation.inspire_left_hand_state": 6,
|
| 17 |
+
"observation.inspire_right_hand_state": 6, "observation.projected_gravity": 3}
|
| 18 |
+
H2 = {"observation.state": 75, "observation.projected_gravity": 3, "action.body_target": 31, "action.hands": 44}
|
| 19 |
+
H2P = dict(H2, **{"action.planner": 7})
|
| 20 |
+
SPEC = {"g1": (G1, ("ego_view", "chest")), "h2": (H2, ("head", "chest")), "h2p": (H2P, ("head", "chest"))}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def stats(v):
|
| 24 |
+
v = np.asarray(v, dtype=np.float64)
|
| 25 |
+
return {"min": v.min(0).tolist(), "max": v.max(0).tolist(), "mean": v.mean(0).tolist(), "std": v.std(0).tolist(), "count": [len(v)]}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def vec(table, key, values):
|
| 29 |
+
arr = pa.array(values.tolist(), type=pa.list_(pa.float32(), values.shape[1]))
|
| 30 |
+
i = table.schema.get_field_index(key)
|
| 31 |
+
return table.append_column(key, arr) if i < 0 else table.set_column(i, key, arr)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def map_rows(profile, c):
|
| 35 |
+
if profile == "g1":
|
| 36 |
+
L, R = c["observation.inspire_left_hand_state"], c["observation.inspire_right_hand_state"]
|
| 37 |
+
dev = float(np.abs(c["observation.state"][:, 29:41] - np.concatenate([L, R], 1)).max())
|
| 38 |
+
assert dev < 1e-4, f"observation.state[29:41] differs from the Inspire columns by {dev}"
|
| 39 |
+
return (np.concatenate([c["action.motion_token"], L, R], 1),
|
| 40 |
+
np.concatenate([c["observation.state"][:, :29], L, R, c["observation.projected_gravity"]], 1))
|
| 41 |
+
a = [c["action.body_target"], c["action.hands"]] + ([c["action.planner"]] if profile == "h2p" else [])
|
| 42 |
+
return np.concatenate(a, 1), np.concatenate([c["observation.state"], c["observation.projected_gravity"]], 1)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def main(profile, src, out, limit=None):
|
| 46 |
+
raw, cams = SPEC[profile]
|
| 47 |
+
src, out = Path(src).resolve(strict=True), Path(out).resolve()
|
| 48 |
+
if out.exists():
|
| 49 |
+
raise SystemExit(f"{out} exists")
|
| 50 |
+
info = json.loads((src / "meta/info.json").read_text())
|
| 51 |
+
assert info.get("codebase_version") == "v2.1", info.get("codebase_version")
|
| 52 |
+
for k, w in raw.items():
|
| 53 |
+
assert info["features"][k]["shape"] == [w], (k, info["features"][k]["shape"])
|
| 54 |
+
for c in cams:
|
| 55 |
+
assert info["features"]["observation.images." + c]["dtype"] in ("video", "image"), c
|
| 56 |
+
eps = [json.loads(l) for l in (src / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
|
| 57 |
+
assert [e["episode_index"] for e in eps] == list(range(len(eps)))
|
| 58 |
+
if limit is not None:
|
| 59 |
+
eps = eps[:limit]
|
| 60 |
+
old = {r["episode_index"]: r for r in (json.loads(l) for l in (src / "meta/episodes_stats.jsonl").read_text().splitlines() if l.strip())}
|
| 61 |
+
out.parent.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
stg = Path(tempfile.mkdtemp(prefix=".hbfw-", dir=out.parent))
|
| 63 |
+
shutil.copytree(src / "meta", stg / "meta")
|
| 64 |
+
for n in ("videos", "images"):
|
| 65 |
+
if (src / n).exists():
|
| 66 |
+
(stg / n).symlink_to(src / n, target_is_directory=True)
|
| 67 |
+
new, total = [], 0
|
| 68 |
+
for e in eps:
|
| 69 |
+
i = e["episode_index"]
|
| 70 |
+
rel = Path(info["data_path"].format(episode_chunk=i // info["chunks_size"], episode_index=i))
|
| 71 |
+
t = pq.read_table(src / rel)
|
| 72 |
+
c = {k: np.asarray(t[k].to_pylist(), dtype=np.float32) for k in raw}
|
| 73 |
+
act, st = map_rows(profile, c)
|
| 74 |
+
assert len(act) == e["length"] and np.isfinite(act).all() and np.isfinite(st).all()
|
| 75 |
+
total += len(act)
|
| 76 |
+
t = vec(vec(t, "action", act), "observation.state", st)
|
| 77 |
+
md = dict(t.schema.metadata or {})
|
| 78 |
+
if b"huggingface" in md:
|
| 79 |
+
hf = json.loads(md[b"huggingface"])
|
| 80 |
+
for k, w in (("action", act.shape[1]), ("observation.state", st.shape[1])):
|
| 81 |
+
hf["info"]["features"][k] = {"feature": {"dtype": "float32", "_type": "Value"}, "length": w, "_type": "Sequence"}
|
| 82 |
+
md[b"huggingface"] = json.dumps(hf).encode()
|
| 83 |
+
t = t.replace_schema_metadata(md)
|
| 84 |
+
(stg / rel).parent.mkdir(parents=True, exist_ok=True)
|
| 85 |
+
pq.write_table(t, stg / rel)
|
| 86 |
+
r = copy.deepcopy(old[i])
|
| 87 |
+
r["stats"]["action"], r["stats"]["observation.state"] = stats(act), stats(st)
|
| 88 |
+
new.append(r)
|
| 89 |
+
if limit is None:
|
| 90 |
+
assert total == info["total_frames"], (total, info["total_frames"])
|
| 91 |
+
else:
|
| 92 |
+
info["total_episodes"], info["total_frames"] = len(eps), total
|
| 93 |
+
info["total_videos"] = len(eps) * len(cams) if "total_videos" in info else info.get("total_videos")
|
| 94 |
+
info["splits"] = {"train": f"0:{len(eps)}"}
|
| 95 |
+
(stg / "meta/episodes.jsonl").write_text("".join(json.dumps(e) + "\n" for e in eps))
|
| 96 |
+
info["features"]["action"] = {"dtype": "float32", "shape": [int(act.shape[1])], "names": None}
|
| 97 |
+
info["features"]["observation.state"] = {"dtype": "float32", "shape": [int(st.shape[1])], "names": None}
|
| 98 |
+
drop = [k for k, v in info["features"].items() if v["dtype"] in ("video", "image") and k not in ["observation.images." + c for c in cams]]
|
| 99 |
+
for k in drop: # 2-camera rule: unused cameras are not downloaded, FastWAM checks every listed video file
|
| 100 |
+
del info["features"][k]
|
| 101 |
+
for r in new:
|
| 102 |
+
r["stats"].pop(k, None)
|
| 103 |
+
if drop and info.get("total_videos"):
|
| 104 |
+
info["total_videos"] = len(eps) * len(cams)
|
| 105 |
+
(stg / "meta/info.json").write_text(json.dumps(info, indent=2) + "\n")
|
| 106 |
+
(stg / "meta/episodes_stats.jsonl").write_text("".join(json.dumps(r) + "\n" for r in new))
|
| 107 |
+
for s in (stg / "meta").glob("*stats*.json"):
|
| 108 |
+
s.unlink()
|
| 109 |
+
(stg / "meta/hb_fastwam_interface.json").write_text(json.dumps(dict(
|
| 110 |
+
profile=profile, source=str(src), cameras=list(cams), state_dim=int(st.shape[1]), action_dim=int(act.shape[1]),
|
| 111 |
+
episodes=len(eps), frames=total, fps=info["fps"], first_episodes_only=limit), indent=2) + "\n")
|
| 112 |
+
stg.rename(out)
|
| 113 |
+
print("HBFW_CONVERT_OK", profile, out, "episodes", len(eps), "frames", total, "state", st.shape[1], "action", act.shape[1], flush=True)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
if __name__ == "__main__":
|
| 117 |
+
a = sys.argv[1:]
|
| 118 |
+
lim = None
|
| 119 |
+
if "--episodes" in a:
|
| 120 |
+
j = a.index("--episodes"); lim = int(a[j + 1]); del a[j:j + 2]
|
| 121 |
+
main(a[0], a[1], a[2], lim)
|
b8-table-wipe/b512/step-004000/code/hb_fastwam_train.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-16: native FastWAM Gaussian-FM training on our benchmark interface, with the HB augmentation v2.
|
| 2 |
+
Profiles (HB_FW_PROFILE): g1 = state44 / action76 / cameras ego_view + chest (HumanWAM fastwam_sonic recipe);
|
| 3 |
+
h2 = state78 / action75 / head + chest; h2p = state78 / action82 (+ planner 7) / head + chest. Fixed recipe of HumanWAM
|
| 4 |
+
fastwam_sonic_training.compose_sonic_config: 17 raw samples = 16 actions + 5 video frames (stride 4), 224 x 448 side by
|
| 5 |
+
side, min/max statistics, absolute labels, action scheduler shift 1.0, original Wan files.
|
| 6 |
+
Arguments: key=value Hydra overrides (JSON values parsed). Env: FASTWAM_SRC, DIFFSYNTH_MODEL_BASE_PATH, HB_AUG_DIR.
|
| 7 |
+
HB_MODEL_YAML=<path> writes the resolved model config; HB_COMPOSE_ONLY=1 prints the config; HB_DATA_SAMPLE=1 reads one
|
| 8 |
+
augmented training sample on CPU; HB_FW_AUG=0 disables the augmentation (benchmark only)."""
|
| 9 |
+
import json, os, sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
ROOT = Path(os.environ["FASTWAM_SRC"]).resolve()
|
| 13 |
+
sys.path.insert(0, str(ROOT / "src"))
|
| 14 |
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
| 15 |
+
TASK = "libero_uncond_2cam224_1e-4"
|
| 16 |
+
PROFILES = {"g1": (("ego_view", "chest"), 44, 76, 29), "h2": (("head", "chest"), 78, 75, 31),
|
| 17 |
+
"h2p": (("head", "chest"), 78, 82, 31)}
|
| 18 |
+
CAMS, SDIM, ADIM, NJ = PROFILES[os.environ.get("HB_FW_PROFILE", "g1")]
|
| 19 |
+
os.environ.setdefault("HB_FW_JOINTS", str(NJ))
|
| 20 |
+
SHAPE_META = {
|
| 21 |
+
"images": [{"key": n, "raw_shape": [3, 224, 224], "shape": [3, 224, 224]} for n in CAMS],
|
| 22 |
+
"action": [{"key": "default", "raw_shape": ADIM, "shape": ADIM}],
|
| 23 |
+
"state": [{"key": "default", "raw_shape": SDIM, "shape": SDIM}],
|
| 24 |
+
}
|
| 25 |
+
FIXED = {
|
| 26 |
+
"data.train.shape_meta": SHAPE_META, "data.train.num_frames": 17, "data.train.global_sample_stride": 1,
|
| 27 |
+
"data.train.action_video_freq_ratio": 4, "data.train.video_size": [224, 448], "data.train.concat_multi_camera": "horizontal",
|
| 28 |
+
"data.train.processor.action_output_dim": ADIM, "data.train.processor.proprio_output_dim": SDIM,
|
| 29 |
+
"data.train.processor.delta_action_dim_mask": {"default": [False] * ADIM}, "data.train.processor.action_state_transforms": None,
|
| 30 |
+
"data.train.processor.use_stepwise_action_norm": False, "data.train.processor.norm_default_mode": "min/max",
|
| 31 |
+
"data.train.processor.norm_exception_mode": None, "model.redirect_common_files": False,
|
| 32 |
+
"model.action_scheduler.train_shift": 1.0, "model.action_scheduler.infer_shift": 1.0,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def build(overrides):
|
| 37 |
+
from hydra import compose, initialize_config_dir
|
| 38 |
+
from omegaconf import OmegaConf
|
| 39 |
+
from fastwam.utils.config_resolvers import register_default_resolvers
|
| 40 |
+
register_default_resolvers()
|
| 41 |
+
with initialize_config_dir(config_dir=str(ROOT / "configs"), version_base="1.3"):
|
| 42 |
+
cfg = compose(config_name="train", overrides=["task=" + TASK])
|
| 43 |
+
for k, v in {**FIXED, **overrides}.items():
|
| 44 |
+
OmegaConf.update(cfg, k, v, merge=False)
|
| 45 |
+
for k, v in FIXED.items():
|
| 46 |
+
a = OmegaConf.select(cfg, k)
|
| 47 |
+
if OmegaConf.is_config(a):
|
| 48 |
+
a = OmegaConf.to_container(a, resolve=True)
|
| 49 |
+
assert a == v, (k, a, v)
|
| 50 |
+
return cfg
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def parse(argv):
|
| 54 |
+
out = {}
|
| 55 |
+
for a in argv:
|
| 56 |
+
k, v = a.split("=", 1)
|
| 57 |
+
try:
|
| 58 |
+
out[k] = json.loads(v)
|
| 59 |
+
except ValueError:
|
| 60 |
+
out[k] = v
|
| 61 |
+
return out
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
from omegaconf import OmegaConf
|
| 66 |
+
cfg = build(parse(sys.argv[1:]))
|
| 67 |
+
if os.environ.get("HB_MODEL_YAML"):
|
| 68 |
+
Path(os.environ["HB_MODEL_YAML"]).write_text(OmegaConf.to_yaml(cfg.model, resolve=True))
|
| 69 |
+
print("HBFW_MODEL_YAML", os.environ["HB_MODEL_YAML"], flush=True)
|
| 70 |
+
raise SystemExit(0)
|
| 71 |
+
if os.environ.get("HB_COMPOSE_ONLY"):
|
| 72 |
+
print(OmegaConf.to_yaml(cfg, resolve=True))
|
| 73 |
+
raise SystemExit(0)
|
| 74 |
+
import fastwam.runtime as native
|
| 75 |
+
assert Path(native.__file__).resolve() == ROOT / "src/fastwam/runtime.py", native.__file__
|
| 76 |
+
if os.environ.get("HB_FW_AUG", "1") != "0":
|
| 77 |
+
import hb_fastwam_aug
|
| 78 |
+
hb_fastwam_aug.install()
|
| 79 |
+
print("HBFW_RECIPE", json.dumps(dict(profile=os.environ.get("HB_FW_PROFILE", "g1"), cameras=CAMS, state_dim=SDIM,
|
| 80 |
+
action_dim=ADIM, perturbed_joints=int(os.environ["HB_FW_JOINTS"]),
|
| 81 |
+
aug=os.environ.get("HB_FW_AUG", "1") != "0",
|
| 82 |
+
batch_size=cfg.batch_size, max_steps=cfg.max_steps, lr=cfg.learning_rate)), flush=True)
|
| 83 |
+
if os.environ.get("HB_DATA_SAMPLE"): # CPU check: build the dataset and read augmented samples, no model
|
| 84 |
+
from omegaconf import open_dict
|
| 85 |
+
from fastwam.utils import misc
|
| 86 |
+
misc.register_work_dir(os.environ.get("HB_SAMPLE_WORKDIR", "/tmp/hbfw-sample"))
|
| 87 |
+
tr, _ = native.build_datasets(cfg.data)
|
| 88 |
+
s = tr[len(tr) // 2]
|
| 89 |
+
|
| 90 |
+
def desc(x):
|
| 91 |
+
if isinstance(x, dict):
|
| 92 |
+
return {k: desc(v) for k, v in x.items()}
|
| 93 |
+
if hasattr(x, "shape"):
|
| 94 |
+
return [tuple(x.shape), str(x.dtype), round(float(x.float().min()), 3), round(float(x.float().max()), 3)]
|
| 95 |
+
return type(x).__name__
|
| 96 |
+
print("HBFW_SAMPLE", len(tr), json.dumps(desc(s), default=str), flush=True)
|
| 97 |
+
raise SystemExit(0)
|
| 98 |
+
if os.environ.get("HB_INLOOP_EVERY", "250") != "0": # 2026-11-16: benchmark action nMSE every 250 steps
|
| 99 |
+
try:
|
| 100 |
+
import hb_inloop_fastwam
|
| 101 |
+
hb_inloop_fastwam.install()
|
| 102 |
+
except Exception as e: # noqa: BLE001 - never block training
|
| 103 |
+
print("HB_INLOOP fastwam install failed:", repr(e), flush=True)
|
| 104 |
+
native.run_training(cfg)
|
b8-table-wipe/b512/step-004000/code/hb_inloop_core.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared machinery of the in-training open-loop action MSE (Psi-Zero, Delta Flow; GR00T has its own tested copy in
|
| 2 |
+
hb_groot/hb_inloop_eval.py with the same protocol).
|
| 3 |
+
|
| 4 |
+
InLoop(model_name, task, split, out_jsonl, prepare, predict):
|
| 5 |
+
prepare(queries) -> optional per-model setup on this rank's queries (first eval only, inside the RNG fork)
|
| 6 |
+
predict(batch, seed) -> np.ndarray (B, >=16, >=Da) in raw action units; the first 16 steps and the first Da dims
|
| 7 |
+
(Da = benchmark action dim of the layout) are scored.
|
| 8 |
+
InLoop.evaluate(step, model): all train/eval modes of model.modules() and every RNG state (torch CPU + this CUDA device,
|
| 9 |
+
numpy, python) are restored afterwards; partial sums are all-reduced once (error flag included, so a failure on one
|
| 10 |
+
rank cannot leave the others waiting); rank 0 appends one json line per eval. Any failure disables later evals.
|
| 11 |
+
Protocol and metrics = hb_eval_common (post-hoc eval): queries every 16 frames, nMSE with the training-frame std.
|
| 12 |
+
"""
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import random
|
| 16 |
+
import sys
|
| 17 |
+
import time
|
| 18 |
+
import traceback
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
HERE = Path(__file__).resolve().parent
|
| 25 |
+
if str(HERE) not in sys.path:
|
| 26 |
+
sys.path.insert(0, str(HERE))
|
| 27 |
+
import hb_eval_common as E # noqa: E402
|
| 28 |
+
|
| 29 |
+
EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
|
| 30 |
+
BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
|
| 31 |
+
MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
|
| 32 |
+
SEED = 1234
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def dist_info():
|
| 36 |
+
import torch.distributed as dist
|
| 37 |
+
if dist.is_available() and dist.is_initialized():
|
| 38 |
+
return dist.get_rank(), dist.get_world_size()
|
| 39 |
+
return int(os.environ.get("RANK", "0")), int(os.environ.get("WORLD_SIZE", "1"))
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def pick_split(task):
|
| 43 |
+
split = os.environ.get("HB_INLOOP_SPLIT")
|
| 44 |
+
if not split:
|
| 45 |
+
split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
|
| 46 |
+
if not (E.split_dir(task, split) / "meta.json").exists():
|
| 47 |
+
raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
|
| 48 |
+
return split
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class InLoop:
|
| 52 |
+
def __init__(self, model_name, task, split, out_jsonl, predict, prepare=None, cams=None, info=None):
|
| 53 |
+
self.model_name, self.task, self.split = model_name, task, split
|
| 54 |
+
self.out = Path(out_jsonl)
|
| 55 |
+
self.predict, self.prepare = predict, prepare
|
| 56 |
+
self.cams = cams
|
| 57 |
+
self.info = info or {}
|
| 58 |
+
self.disabled = False
|
| 59 |
+
self.ready = False
|
| 60 |
+
self.done_steps = set()
|
| 61 |
+
|
| 62 |
+
def say(self, *a):
|
| 63 |
+
if dist_info()[0] == 0:
|
| 64 |
+
print("HB_INLOOP", *a, flush=True)
|
| 65 |
+
|
| 66 |
+
def _setup(self):
|
| 67 |
+
rank, world = dist_info()
|
| 68 |
+
meta = E.load_meta(self.task, self.split)
|
| 69 |
+
self.meta, self.lay = meta, E.LAYOUTS[meta["layout"]]
|
| 70 |
+
self.std = E.norm_std(E.load_stats(self.task))
|
| 71 |
+
cams = self.cams or meta["cams"]
|
| 72 |
+
miss = [c for c in cams if c not in meta["cams"]]
|
| 73 |
+
if miss:
|
| 74 |
+
raise KeyError(f"cameras {miss} not in the eval cache {meta['cams']}")
|
| 75 |
+
allq = []
|
| 76 |
+
for ep in meta["episodes"]:
|
| 77 |
+
with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
|
| 78 |
+
allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
|
| 79 |
+
if MAX_Q:
|
| 80 |
+
allq = allq[:MAX_Q]
|
| 81 |
+
self.n_total = len(allq)
|
| 82 |
+
mine = [q for i, q in enumerate(allq) if i % world == rank]
|
| 83 |
+
self.eps = list(meta["episodes"])
|
| 84 |
+
by_ep = {}
|
| 85 |
+
for ep, qi, t in mine:
|
| 86 |
+
by_ep.setdefault(ep, []).append((qi, t))
|
| 87 |
+
self.q = []
|
| 88 |
+
for ep, lst in by_ep.items():
|
| 89 |
+
with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
|
| 90 |
+
state, action = z["state"], z["action"]
|
| 91 |
+
prompt = meta["prompts"][str(int(z["task_index"]))]
|
| 92 |
+
imgs = {c: z[f"img_{c}"] for c in cams}
|
| 93 |
+
for qi, t in lst:
|
| 94 |
+
gt, valid = E.gt_chunk(action, t)
|
| 95 |
+
self.q.append(dict(episode=int(ep), t=int(t), T=len(action), state=state[t].copy(), prompt=prompt,
|
| 96 |
+
images={c: imgs[c][qi].copy() for c in cams}, gt=gt, valid=valid,
|
| 97 |
+
past_action=action[max(0, t - E.HORIZON):t].copy()))
|
| 98 |
+
if self.prepare:
|
| 99 |
+
self.prepare(self.q)
|
| 100 |
+
self.ready = True
|
| 101 |
+
self.say(f"{self.model_name} ready: task {self.task} split {self.split} queries {self.n_total} "
|
| 102 |
+
f"(rank {rank}/{world}: {len(self.q)}), every {EVERY} steps, batch {BATCH}, cams {cams}")
|
| 103 |
+
|
| 104 |
+
def evaluate(self, step, model):
|
| 105 |
+
if self.disabled or step in self.done_steps:
|
| 106 |
+
return
|
| 107 |
+
self.done_steps.add(step)
|
| 108 |
+
rank, world = dist_info()
|
| 109 |
+
t0 = time.time()
|
| 110 |
+
err, msg = 0.0, ""
|
| 111 |
+
acc, nq, infer = None, 0, 0.0
|
| 112 |
+
modes = [(m, m.training) for m in model.modules()]
|
| 113 |
+
dev = torch.cuda.current_device() if torch.cuda.is_available() else None
|
| 114 |
+
np_state, py_state = np.random.get_state(), random.getstate()
|
| 115 |
+
try:
|
| 116 |
+
with torch.random.fork_rng(devices=[dev] if dev is not None else []):
|
| 117 |
+
if not self.ready:
|
| 118 |
+
self._setup()
|
| 119 |
+
D = self.lay["action_dim"]
|
| 120 |
+
model.eval()
|
| 121 |
+
acc = E.Accumulator(self.std, self.lay["groups"])
|
| 122 |
+
for bi in range(0, len(self.q), BATCH):
|
| 123 |
+
batch = self.q[bi:bi + BATCH]
|
| 124 |
+
seed = SEED + 1000 * rank + bi
|
| 125 |
+
torch.default_generator.manual_seed(seed)
|
| 126 |
+
if dev is not None:
|
| 127 |
+
torch.cuda.manual_seed(seed)
|
| 128 |
+
s = time.time()
|
| 129 |
+
with torch.no_grad():
|
| 130 |
+
pred = np.asarray(self.predict(batch, seed), dtype=np.float64)
|
| 131 |
+
infer += time.time() - s
|
| 132 |
+
if pred.ndim != 3 or pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] < D:
|
| 133 |
+
raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, >={D})")
|
| 134 |
+
for q, p in zip(batch, pred):
|
| 135 |
+
acc.add(q["episode"], p[:E.HORIZON, :D], q["gt"], q["valid"])
|
| 136 |
+
nq += 1
|
| 137 |
+
except Exception as e: # noqa: BLE001 - never stop training
|
| 138 |
+
err, msg = 1.0, f"{type(e).__name__}: {e}"
|
| 139 |
+
print(f"HB_INLOOP {self.model_name} rank {rank} step {step} FAILED: {msg}\n{traceback.format_exc()}",
|
| 140 |
+
file=sys.stderr, flush=True)
|
| 141 |
+
finally:
|
| 142 |
+
for m, tr in modes:
|
| 143 |
+
m.training = tr
|
| 144 |
+
np.random.set_state(np_state)
|
| 145 |
+
random.setstate(py_state)
|
| 146 |
+
D = self.lay["action_dim"] if self.ready else 0
|
| 147 |
+
H = E.HORIZON
|
| 148 |
+
eps = self.eps if self.ready else []
|
| 149 |
+
nE = len(eps)
|
| 150 |
+
vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
|
| 151 |
+
if acc is not None and not err:
|
| 152 |
+
o = 0
|
| 153 |
+
vec[o:o + D] = acc.sse_raw; o += D
|
| 154 |
+
vec[o:o + D] = acc.sae_raw; o += D
|
| 155 |
+
vec[o:o + D] = acc.sse_n; o += D
|
| 156 |
+
vec[o] = acc.rows; o += 1
|
| 157 |
+
vec[o:o + H] = acc.step_sse_n; o += H
|
| 158 |
+
vec[o:o + H] = acc.step_rows; o += H
|
| 159 |
+
for j, ep in enumerate(eps):
|
| 160 |
+
s_, c_ = acc.per_episode.get(int(ep), (0.0, 0))
|
| 161 |
+
vec[o + j], vec[o + nE + j] = s_, c_
|
| 162 |
+
o += 2 * nE
|
| 163 |
+
vec[o], vec[o + 1] = nq, infer
|
| 164 |
+
vec[-1] = err
|
| 165 |
+
tot = self._allreduce(vec)
|
| 166 |
+
if tot is None or tot[-1] > 0 or not self.ready:
|
| 167 |
+
self.disabled = True
|
| 168 |
+
nfail = None if tot is None else int(tot[-1])
|
| 169 |
+
self._write({"step": step, "error": msg or "failed on another rank / collective failed",
|
| 170 |
+
"ranks_failed": nfail})
|
| 171 |
+
self.say(f"{self.model_name} step {step}: eval failed ({nfail} rank(s)) -> in-training eval disabled: {msg}")
|
| 172 |
+
return
|
| 173 |
+
o = 0
|
| 174 |
+
acc = E.Accumulator(self.std, self.lay["groups"])
|
| 175 |
+
acc.sse_raw = tot[o:o + D]; o += D
|
| 176 |
+
acc.sae_raw = tot[o:o + D]; o += D
|
| 177 |
+
acc.sse_n = tot[o:o + D]; o += D
|
| 178 |
+
acc.rows = int(round(tot[o])); o += 1
|
| 179 |
+
acc.step_sse_n = tot[o:o + H]; o += H
|
| 180 |
+
acc.step_rows = tot[o:o + H]; o += H
|
| 181 |
+
acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
|
| 182 |
+
for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
|
| 183 |
+
o += 2 * nE
|
| 184 |
+
n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
|
| 185 |
+
res = acc.result()
|
| 186 |
+
res.pop("nmse_by_dim", None)
|
| 187 |
+
rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "model": self.model_name,
|
| 188 |
+
"task": self.task, "split": self.split, "queries": n_q, "ranks": world,
|
| 189 |
+
"wall_s": round(time.time() - t0, 1), "infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
|
| 190 |
+
"seed": SEED, **self.info, **res}
|
| 191 |
+
self._write(rec)
|
| 192 |
+
g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
|
| 193 |
+
self.say(f"{self.model_name} step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} "
|
| 194 |
+
f"queries, {rec['wall_s']} s")
|
| 195 |
+
|
| 196 |
+
@staticmethod
|
| 197 |
+
def _allreduce(vec):
|
| 198 |
+
import torch.distributed as dist
|
| 199 |
+
if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() == 1:
|
| 200 |
+
return vec
|
| 201 |
+
try:
|
| 202 |
+
dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
|
| 203 |
+
n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
|
| 204 |
+
mx, mn = n.clone(), n.clone()
|
| 205 |
+
dist.all_reduce(mx, op=dist.ReduceOp.MAX)
|
| 206 |
+
dist.all_reduce(mn, op=dist.ReduceOp.MIN)
|
| 207 |
+
if mx.item() != mn.item():
|
| 208 |
+
return None
|
| 209 |
+
t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
|
| 210 |
+
dist.all_reduce(t)
|
| 211 |
+
return t.cpu().numpy()
|
| 212 |
+
except Exception as e: # noqa: BLE001
|
| 213 |
+
print(f"HB_INLOOP all_reduce failed: {e!r}", file=sys.stderr, flush=True)
|
| 214 |
+
return None
|
| 215 |
+
|
| 216 |
+
def _write(self, rec):
|
| 217 |
+
if dist_info()[0] != 0:
|
| 218 |
+
return
|
| 219 |
+
try:
|
| 220 |
+
self.out.parent.mkdir(parents=True, exist_ok=True)
|
| 221 |
+
with open(self.out, "a") as f:
|
| 222 |
+
f.write(json.dumps(rec) + "\n")
|
| 223 |
+
except Exception as e: # noqa: BLE001
|
| 224 |
+
print(f"HB_INLOOP write failed: {e!r}", flush=True)
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
class TempAttrs:
|
| 228 |
+
"""set attributes on an object for the duration of a block, then restore (or delete) them"""
|
| 229 |
+
_MISSING = object()
|
| 230 |
+
|
| 231 |
+
def __init__(self, obj, **kw):
|
| 232 |
+
self.obj, self.kw, self.old = obj, kw, {}
|
| 233 |
+
|
| 234 |
+
def __enter__(self):
|
| 235 |
+
for k, v in self.kw.items():
|
| 236 |
+
self.old[k] = self.obj.__dict__.get(k, self._MISSING)
|
| 237 |
+
setattr(self.obj, k, v)
|
| 238 |
+
return self.obj
|
| 239 |
+
|
| 240 |
+
def __exit__(self, *exc):
|
| 241 |
+
for k, v in self.old.items():
|
| 242 |
+
if v is self._MISSING:
|
| 243 |
+
try:
|
| 244 |
+
delattr(self.obj, k)
|
| 245 |
+
except AttributeError:
|
| 246 |
+
pass
|
| 247 |
+
else:
|
| 248 |
+
setattr(self.obj, k, v)
|
| 249 |
+
return False
|
b8-table-wipe/b512/step-004000/code/hb_inloop_fastwam.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastWAM in-training open-loop action MSE (2026-11-16 user: every 250 steps, like GR00T / Psi-Zero / Delta Flow).
|
| 2 |
+
|
| 3 |
+
install() is called by fastwam/tools/hb_fastwam_train.py right before native.run_training(cfg). It uses the trainer's
|
| 4 |
+
own eval slot: after Wan22Trainer.__init__ it sets eval_every = HB_INLOOP_EVERY (default 250) and a non-None
|
| 5 |
+
val_dataset marker, and replaces Wan22Trainer.evaluate (the native one generates a full video for one random val sample;
|
| 6 |
+
it is off in our runs, eval_every=0 on the command line). The native loop then calls evaluate() on every rank right after
|
| 7 |
+
the optimizer step of every 250th update, followed by accelerator.wait_for_everyone(); returning None logs nothing.
|
| 8 |
+
evaluate(): all ranks score their shard of the benchmark eval cache with the live model (accelerator.unwrap_model; ZeRO-1
|
| 9 |
+
keeps full bf16 weights on every rank) through EXACTLY the post-hoc adapter path (eval-action-mse/adapters/ad_fastwam.py
|
| 10 |
+
Inputs + infer: FastWAM.infer_action, the action-only deployment path; 10 denoising steps = config
|
| 11 |
+
eval_num_inference_steps; seed 42 per query; same cameras / proprio normalisation / T5 cache as training).
|
| 12 |
+
Results: <output_dir>/inloop-mse.jsonl (rank 0) + "HB_INLOOP" log lines. Training is untouched (module modes and every
|
| 13 |
+
RNG restored by hb_inloop_core, no gradients); any failure disables later evals on all ranks.
|
| 14 |
+
"""
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import sys
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
G = Path(__file__).resolve().parents[2] # .../policies-20261109 (this file: $G/fastwam/tools/)
|
| 21 |
+
EVAL = G / "eval-action-mse"
|
| 22 |
+
for p in (str(EVAL), str(EVAL / "adapters")):
|
| 23 |
+
if p not in sys.path:
|
| 24 |
+
sys.path.insert(0, p)
|
| 25 |
+
import numpy as np # noqa: E402
|
| 26 |
+
import hb_eval_common as E # noqa: E402
|
| 27 |
+
import hb_inloop_core as C # noqa: E402
|
| 28 |
+
import ad_fastwam as A # noqa: E402
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class _Marker:
|
| 32 |
+
"""non-None val_dataset so the native loop reaches evaluate(); never indexed"""
|
| 33 |
+
|
| 34 |
+
def __len__(self):
|
| 35 |
+
return 1
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class FwEval:
|
| 39 |
+
def __init__(self, trainer):
|
| 40 |
+
cfg = trainer.cfg
|
| 41 |
+
self.cfg = cfg
|
| 42 |
+
ddir = Path(list(cfg.data.train.dataset_dirs)[0])
|
| 43 |
+
iface = json.loads((ddir / "meta/hb_fastwam_interface.json").read_text())
|
| 44 |
+
task = E.task_for_source(iface["source"])
|
| 45 |
+
if task is None:
|
| 46 |
+
raise LookupError(f"no benchmark task for {iface['source']}")
|
| 47 |
+
split = C.pick_split(task)
|
| 48 |
+
self.steps = int(cfg.eval_num_inference_steps)
|
| 49 |
+
self.inp = None
|
| 50 |
+
self.model = None
|
| 51 |
+
out = Path(cfg.output_dir) / "inloop-mse.jsonl"
|
| 52 |
+
self.il = C.InLoop("fastwam", task, split, out, predict=self.predict, prepare=self.prepare,
|
| 53 |
+
info={"denoising_steps": self.steps, "seed_per_query": 42, "dataset": ddir.name,
|
| 54 |
+
"path": "FastWAM.infer_action (ad_fastwam)"})
|
| 55 |
+
self.il.cams = None
|
| 56 |
+
|
| 57 |
+
def prepare(self, queries):
|
| 58 |
+
self.inp = A.Inputs(self.cfg, Path(self.cfg.output_dir) / "dataset_stats.json")
|
| 59 |
+
miss = [c for c in self.inp.cams if c not in self.il.meta["cams"]]
|
| 60 |
+
if miss:
|
| 61 |
+
raise KeyError(f"cameras {miss} not in the eval cache {self.il.meta['cams']}")
|
| 62 |
+
for q in {q["prompt"] for q in queries}:
|
| 63 |
+
self.inp.context(q) # fail early if an instruction has no T5 cache
|
| 64 |
+
|
| 65 |
+
def predict(self, batch, seed):
|
| 66 |
+
return np.stack([np.asarray(A.infer(self.model, self.inp, q, self.steps, 42), dtype=np.float64) for q in batch])
|
| 67 |
+
|
| 68 |
+
def evaluate(self, step, model):
|
| 69 |
+
self.model = model
|
| 70 |
+
try:
|
| 71 |
+
self.il.evaluate(step, model)
|
| 72 |
+
finally:
|
| 73 |
+
self.model = None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class _Broken:
|
| 77 |
+
def __init__(self, err):
|
| 78 |
+
self.err = err
|
| 79 |
+
self.il = C.InLoop("fastwam", "b6", "train10", "/dev/null", predict=None, prepare=self._boom)
|
| 80 |
+
|
| 81 |
+
def _boom(self, _q):
|
| 82 |
+
raise RuntimeError(self.err)
|
| 83 |
+
|
| 84 |
+
def evaluate(self, step, model):
|
| 85 |
+
self.il.evaluate(step, model)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def install():
|
| 89 |
+
if C.EVERY <= 0:
|
| 90 |
+
return
|
| 91 |
+
from fastwam import trainer as T
|
| 92 |
+
WT = T.Wan22Trainer
|
| 93 |
+
orig_init = WT.__init__
|
| 94 |
+
|
| 95 |
+
def __init__(self, *a, **kw):
|
| 96 |
+
orig_init(self, *a, **kw)
|
| 97 |
+
# native eval off (eval_every 0 on our command line). NB runtime.build_datasets sets val_ds = train_ds when
|
| 98 |
+
# there is no data.val, so val_dataset is never None here (11-16 smoke: the old "val_dataset is None" test
|
| 99 |
+
# silently left the in-loop eval off).
|
| 100 |
+
if int(self.eval_every) == 0:
|
| 101 |
+
self.eval_every = C.EVERY
|
| 102 |
+
if self.val_dataset is None:
|
| 103 |
+
self.val_dataset = _Marker()
|
| 104 |
+
self._hb_inloop_on = True
|
| 105 |
+
else: # a native eval is configured: leave it alone
|
| 106 |
+
self._hb_inloop_on = False
|
| 107 |
+
|
| 108 |
+
def evaluate(self):
|
| 109 |
+
if not getattr(self, "_hb_inloop_on", False):
|
| 110 |
+
return orig_eval(self)
|
| 111 |
+
ev = getattr(self, "_hb_inloop", None)
|
| 112 |
+
if ev is None:
|
| 113 |
+
try:
|
| 114 |
+
ev = FwEval(self)
|
| 115 |
+
except Exception as e: # noqa: BLE001
|
| 116 |
+
ev = _Broken(f"{type(e).__name__}: {e}")
|
| 117 |
+
self._hb_inloop = ev
|
| 118 |
+
if not ev.il.disabled:
|
| 119 |
+
ev.evaluate(int(self.global_step), self.accelerator.unwrap_model(self.model))
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
orig_eval = WT.evaluate
|
| 123 |
+
WT.__init__ = __init__
|
| 124 |
+
WT.evaluate = evaluate
|
| 125 |
+
if int(os.environ.get("RANK", "0")) == 0:
|
| 126 |
+
print(f"HB_INLOOP fastwam installed: every {C.EVERY} steps", flush=True)
|
b8-table-wipe/b512/step-004000/code/test_fastwam_inputs.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU check (no model): the FastWAM adapter's inputs built from the model-agnostic eval cache == the inputs our FastWAM
|
| 2 |
+
training dataset (RobotVideoDataset of the run's config, no HB augmentation) builds for the same frame.
|
| 3 |
+
python test_fastwam_inputs.py <task> <run out dir> [n_queries]"""
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
HERE = Path(__file__).resolve().parent
|
| 12 |
+
sys.path.insert(0, str(HERE))
|
| 13 |
+
sys.path.insert(0, str(HERE / "adapters"))
|
| 14 |
+
import hb_eval_common as E # noqa: E402
|
| 15 |
+
import ad_fastwam as A # noqa: E402
|
| 16 |
+
|
| 17 |
+
task, out = sys.argv[1], Path(sys.argv[2])
|
| 18 |
+
nq = int(sys.argv[3]) if len(sys.argv) > 3 else 3
|
| 19 |
+
A.setup_env()
|
| 20 |
+
import torch # noqa: E402
|
| 21 |
+
from omegaconf import OmegaConf # noqa: E402
|
| 22 |
+
import fastwam.runtime as native # noqa: E402
|
| 23 |
+
from fastwam.utils import misc # noqa: E402
|
| 24 |
+
|
| 25 |
+
cfg = OmegaConf.load(out / "config.yaml")
|
| 26 |
+
inp = A.Inputs(cfg, out / "dataset_stats.json")
|
| 27 |
+
misc.register_work_dir("/tmp/hbfw-inputs-check")
|
| 28 |
+
OmegaConf.update(cfg, "data.train.pretrained_norm_stats", str(out / "dataset_stats.json"), merge=False, force_add=True)
|
| 29 |
+
tr, _ = native.build_datasets(cfg.data)
|
| 30 |
+
ddir = Path(cfg.data.train.dataset_dirs[0])
|
| 31 |
+
lens = [json.loads(l)["length"] for l in (ddir / "meta/episodes.jsonl").read_text().splitlines() if l.strip()]
|
| 32 |
+
meta = E.load_meta(task, "train10")
|
| 33 |
+
split = "train10"
|
| 34 |
+
worst = {}
|
| 35 |
+
done = 0
|
| 36 |
+
for ep, z in E.iter_episodes(task, split, None):
|
| 37 |
+
base = int(sum(lens[:ep]))
|
| 38 |
+
for qi, t in enumerate(list(z["query_t"])[1:1 + nq], start=1):
|
| 39 |
+
t = int(t)
|
| 40 |
+
s = tr[base + t]
|
| 41 |
+
obs = {"images": {c: z[f"img_{c}"][qi] for c in meta["cams"]}, "state": z["state"][t],
|
| 42 |
+
"prompt": meta["prompts"][str(int(z["task_index"]))]}
|
| 43 |
+
img = inp.image(obs["images"])[0]
|
| 44 |
+
d_img = float((img - s["video"][:, 0].float()).abs().max())
|
| 45 |
+
pr = inp.proprio(obs["state"])[0]
|
| 46 |
+
d_pr = float((pr - s["proprio"][0].float()).abs().max())
|
| 47 |
+
ctx, mask = inp.context(obs["prompt"])
|
| 48 |
+
d_ctx = float((ctx.float() - s["context"].float()).abs().max())
|
| 49 |
+
d_mask = int((mask != s["context_mask"].bool()).sum())
|
| 50 |
+
gt = z["action"][t:t + 16]
|
| 51 |
+
a = inp.denorm(s["action"][:16])
|
| 52 |
+
d_act = float(np.abs(a[:len(gt)] - gt).max())
|
| 53 |
+
row = dict(ep=ep, t=t, idx=base + t, img=d_img, proprio=d_pr, ctx=d_ctx, mask=d_mask, act_denorm_vs_cache=d_act,
|
| 54 |
+
prompt_same=s["prompt"] == inp.DEFAULT_PROMPT.format(task=obs["prompt"]))
|
| 55 |
+
print("CHECK", json.dumps(row), flush=True)
|
| 56 |
+
for k in ("img", "proprio", "ctx", "act_denorm_vs_cache"):
|
| 57 |
+
worst[k] = max(worst.get(k, 0.0), row[k])
|
| 58 |
+
done += 1
|
| 59 |
+
if done >= 2 * nq:
|
| 60 |
+
break
|
| 61 |
+
print("WORST", json.dumps(worst))
|
b8-table-wipe/b512/step-004000/complete.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"task": "b8",
|
| 3 |
+
"profile": "g1",
|
| 4 |
+
"per_gpu_batch": 16,
|
| 5 |
+
"global_batch": 512,
|
| 6 |
+
"gradient_accumulation_steps": 1,
|
| 7 |
+
"max_steps": 4000,
|
| 8 |
+
"save_every": 2000,
|
| 9 |
+
"recipe": "native FastWAM Gaussian FM (libero_uncond_2cam224_1e-4 base), HB aug v2 + proprio perturbation",
|
| 10 |
+
"interface": {
|
| 11 |
+
"profile": "g1",
|
| 12 |
+
"source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
|
| 13 |
+
"cameras": [
|
| 14 |
+
"ego_view",
|
| 15 |
+
"chest"
|
| 16 |
+
],
|
| 17 |
+
"state_dim": 44,
|
| 18 |
+
"action_dim": 76,
|
| 19 |
+
"episodes": 100,
|
| 20 |
+
"frames": 61498,
|
| 21 |
+
"fps": 30,
|
| 22 |
+
"first_episodes_only": null
|
| 23 |
+
},
|
| 24 |
+
"finished_utc": "2026-11-17T21:03:31Z"
|
| 25 |
+
}
|
b8-table-wipe/b512/step-004000/dataset_meta/hb_fastwam_interface.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"profile": "g1",
|
| 3 |
+
"source": "/mnt/nas/cross/Team/RobotAgent/Personal/baijiaqi/humanoid-benchmark-work/preflight-20261104T1648Z/b8-table-wipe-v1-3b2e618/data",
|
| 4 |
+
"cameras": [
|
| 5 |
+
"ego_view",
|
| 6 |
+
"chest"
|
| 7 |
+
],
|
| 8 |
+
"state_dim": 44,
|
| 9 |
+
"action_dim": 76,
|
| 10 |
+
"episodes": 100,
|
| 11 |
+
"frames": 61498,
|
| 12 |
+
"fps": 30,
|
| 13 |
+
"first_episodes_only": null
|
| 14 |
+
}
|
b8-table-wipe/b512/step-004000/dataset_meta/info.json
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"codebase_version": "v2.1",
|
| 3 |
+
"robot_type": null,
|
| 4 |
+
"total_episodes": 100,
|
| 5 |
+
"total_frames": 61498,
|
| 6 |
+
"total_tasks": 1,
|
| 7 |
+
"total_videos": 200,
|
| 8 |
+
"total_chunks": 1,
|
| 9 |
+
"chunks_size": 1000,
|
| 10 |
+
"fps": 30,
|
| 11 |
+
"splits": {
|
| 12 |
+
"train": "0:100"
|
| 13 |
+
},
|
| 14 |
+
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
|
| 15 |
+
"video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
|
| 16 |
+
"features": {
|
| 17 |
+
"observation.state": {
|
| 18 |
+
"dtype": "float32",
|
| 19 |
+
"shape": [
|
| 20 |
+
44
|
| 21 |
+
],
|
| 22 |
+
"names": null
|
| 23 |
+
},
|
| 24 |
+
"observation.eef_state": {
|
| 25 |
+
"dtype": "float64",
|
| 26 |
+
"shape": [
|
| 27 |
+
14
|
| 28 |
+
],
|
| 29 |
+
"names": [
|
| 30 |
+
"left_wrist_pos",
|
| 31 |
+
"left_wrist_abs_quat",
|
| 32 |
+
"right_wrist_pos",
|
| 33 |
+
"right_wrist_abs_quat"
|
| 34 |
+
]
|
| 35 |
+
},
|
| 36 |
+
"action.wbc": {
|
| 37 |
+
"dtype": "float64",
|
| 38 |
+
"shape": [
|
| 39 |
+
41
|
| 40 |
+
],
|
| 41 |
+
"names": [
|
| 42 |
+
"left_hip_pitch_joint",
|
| 43 |
+
"left_hip_roll_joint",
|
| 44 |
+
"left_hip_yaw_joint",
|
| 45 |
+
"left_knee_joint",
|
| 46 |
+
"left_ankle_pitch_joint",
|
| 47 |
+
"left_ankle_roll_joint",
|
| 48 |
+
"right_hip_pitch_joint",
|
| 49 |
+
"right_hip_roll_joint",
|
| 50 |
+
"right_hip_yaw_joint",
|
| 51 |
+
"right_knee_joint",
|
| 52 |
+
"right_ankle_pitch_joint",
|
| 53 |
+
"right_ankle_roll_joint",
|
| 54 |
+
"waist_yaw_joint",
|
| 55 |
+
"waist_roll_joint",
|
| 56 |
+
"waist_pitch_joint",
|
| 57 |
+
"left_shoulder_pitch_joint",
|
| 58 |
+
"left_shoulder_roll_joint",
|
| 59 |
+
"left_shoulder_yaw_joint",
|
| 60 |
+
"left_elbow_joint",
|
| 61 |
+
"left_wrist_roll_joint",
|
| 62 |
+
"left_wrist_pitch_joint",
|
| 63 |
+
"left_wrist_yaw_joint",
|
| 64 |
+
"right_shoulder_pitch_joint",
|
| 65 |
+
"right_shoulder_roll_joint",
|
| 66 |
+
"right_shoulder_yaw_joint",
|
| 67 |
+
"right_elbow_joint",
|
| 68 |
+
"right_wrist_roll_joint",
|
| 69 |
+
"right_wrist_pitch_joint",
|
| 70 |
+
"right_wrist_yaw_joint",
|
| 71 |
+
"left_hand_pinky",
|
| 72 |
+
"left_hand_ring",
|
| 73 |
+
"left_hand_middle",
|
| 74 |
+
"left_hand_index",
|
| 75 |
+
"left_hand_thumb_bend",
|
| 76 |
+
"left_hand_thumb_rotation",
|
| 77 |
+
"right_hand_pinky",
|
| 78 |
+
"right_hand_ring",
|
| 79 |
+
"right_hand_middle",
|
| 80 |
+
"right_hand_index",
|
| 81 |
+
"right_hand_thumb_bend",
|
| 82 |
+
"right_hand_thumb_rotation"
|
| 83 |
+
]
|
| 84 |
+
},
|
| 85 |
+
"observation.root_orientation": {
|
| 86 |
+
"dtype": "float64",
|
| 87 |
+
"shape": [
|
| 88 |
+
4
|
| 89 |
+
],
|
| 90 |
+
"names": [
|
| 91 |
+
"base_qw",
|
| 92 |
+
"base_qx",
|
| 93 |
+
"base_qy",
|
| 94 |
+
"base_qz"
|
| 95 |
+
]
|
| 96 |
+
},
|
| 97 |
+
"observation.projected_gravity": {
|
| 98 |
+
"dtype": "float64",
|
| 99 |
+
"shape": [
|
| 100 |
+
3
|
| 101 |
+
],
|
| 102 |
+
"names": [
|
| 103 |
+
"gravity_x",
|
| 104 |
+
"gravity_y",
|
| 105 |
+
"gravity_z"
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"observation.cpp_rotation_offset": {
|
| 109 |
+
"dtype": "float64",
|
| 110 |
+
"shape": [
|
| 111 |
+
4
|
| 112 |
+
],
|
| 113 |
+
"names": [
|
| 114 |
+
"rot_offset_qw",
|
| 115 |
+
"rot_offset_qx",
|
| 116 |
+
"rot_offset_qy",
|
| 117 |
+
"rot_offset_qz"
|
| 118 |
+
]
|
| 119 |
+
},
|
| 120 |
+
"observation.init_base_quat": {
|
| 121 |
+
"dtype": "float64",
|
| 122 |
+
"shape": [
|
| 123 |
+
4
|
| 124 |
+
],
|
| 125 |
+
"names": [
|
| 126 |
+
"init_base_qw",
|
| 127 |
+
"init_base_qx",
|
| 128 |
+
"init_base_qy",
|
| 129 |
+
"init_base_qz"
|
| 130 |
+
]
|
| 131 |
+
},
|
| 132 |
+
"teleop.delta_heading": {
|
| 133 |
+
"dtype": "float64",
|
| 134 |
+
"shape": [
|
| 135 |
+
1
|
| 136 |
+
],
|
| 137 |
+
"names": [
|
| 138 |
+
"delta_heading"
|
| 139 |
+
]
|
| 140 |
+
},
|
| 141 |
+
"action.motion_token": {
|
| 142 |
+
"dtype": "float64",
|
| 143 |
+
"shape": [
|
| 144 |
+
64
|
| 145 |
+
],
|
| 146 |
+
"names": "motion_token"
|
| 147 |
+
},
|
| 148 |
+
"teleop.smpl_joints": {
|
| 149 |
+
"dtype": "float32",
|
| 150 |
+
"shape": [
|
| 151 |
+
72
|
| 152 |
+
],
|
| 153 |
+
"names": "smpl_joints"
|
| 154 |
+
},
|
| 155 |
+
"teleop.smpl_pose": {
|
| 156 |
+
"dtype": "float32",
|
| 157 |
+
"shape": [
|
| 158 |
+
63
|
| 159 |
+
],
|
| 160 |
+
"names": "smpl_pose"
|
| 161 |
+
},
|
| 162 |
+
"teleop.body_quat_w": {
|
| 163 |
+
"dtype": "float32",
|
| 164 |
+
"shape": [
|
| 165 |
+
4
|
| 166 |
+
],
|
| 167 |
+
"names": "body_quat_w"
|
| 168 |
+
},
|
| 169 |
+
"teleop.target_body_orientation": {
|
| 170 |
+
"dtype": "float32",
|
| 171 |
+
"shape": [
|
| 172 |
+
6
|
| 173 |
+
],
|
| 174 |
+
"names": [
|
| 175 |
+
"target_body_r00",
|
| 176 |
+
"target_body_r10",
|
| 177 |
+
"target_body_r01",
|
| 178 |
+
"target_body_r11",
|
| 179 |
+
"target_body_r02",
|
| 180 |
+
"target_body_r12"
|
| 181 |
+
]
|
| 182 |
+
},
|
| 183 |
+
"teleop.left_hand_joints": {
|
| 184 |
+
"dtype": "float32",
|
| 185 |
+
"shape": [
|
| 186 |
+
7
|
| 187 |
+
],
|
| 188 |
+
"names": "left_hand_joints"
|
| 189 |
+
},
|
| 190 |
+
"teleop.right_hand_joints": {
|
| 191 |
+
"dtype": "float32",
|
| 192 |
+
"shape": [
|
| 193 |
+
7
|
| 194 |
+
],
|
| 195 |
+
"names": "right_hand_joints"
|
| 196 |
+
},
|
| 197 |
+
"teleop.inspire_left_hand_cmd": {
|
| 198 |
+
"dtype": "float32",
|
| 199 |
+
"shape": [
|
| 200 |
+
6
|
| 201 |
+
],
|
| 202 |
+
"names": [
|
| 203 |
+
"left_pinky",
|
| 204 |
+
"left_ring",
|
| 205 |
+
"left_middle",
|
| 206 |
+
"left_index",
|
| 207 |
+
"left_thumb_pitch",
|
| 208 |
+
"left_thumb_yaw"
|
| 209 |
+
]
|
| 210 |
+
},
|
| 211 |
+
"teleop.inspire_right_hand_cmd": {
|
| 212 |
+
"dtype": "float32",
|
| 213 |
+
"shape": [
|
| 214 |
+
6
|
| 215 |
+
],
|
| 216 |
+
"names": [
|
| 217 |
+
"right_pinky",
|
| 218 |
+
"right_ring",
|
| 219 |
+
"right_middle",
|
| 220 |
+
"right_index",
|
| 221 |
+
"right_thumb_pitch",
|
| 222 |
+
"right_thumb_yaw"
|
| 223 |
+
]
|
| 224 |
+
},
|
| 225 |
+
"teleop.inspire_hand_cmd_age_ms": {
|
| 226 |
+
"dtype": "float32",
|
| 227 |
+
"shape": [
|
| 228 |
+
1
|
| 229 |
+
],
|
| 230 |
+
"names": [
|
| 231 |
+
"inspire_hand_cmd_age_ms"
|
| 232 |
+
]
|
| 233 |
+
},
|
| 234 |
+
"teleop.controller": {
|
| 235 |
+
"dtype": "float32",
|
| 236 |
+
"shape": [
|
| 237 |
+
4
|
| 238 |
+
],
|
| 239 |
+
"names": [
|
| 240 |
+
"left_trigger",
|
| 241 |
+
"right_trigger",
|
| 242 |
+
"left_grip",
|
| 243 |
+
"right_grip"
|
| 244 |
+
]
|
| 245 |
+
},
|
| 246 |
+
"teleop.pose_sample_age_ms": {
|
| 247 |
+
"dtype": "float32",
|
| 248 |
+
"shape": [
|
| 249 |
+
1
|
| 250 |
+
],
|
| 251 |
+
"names": [
|
| 252 |
+
"pose_sample_age_ms"
|
| 253 |
+
]
|
| 254 |
+
},
|
| 255 |
+
"teleop.smpl_frame_index": {
|
| 256 |
+
"dtype": "int64",
|
| 257 |
+
"shape": [
|
| 258 |
+
1
|
| 259 |
+
],
|
| 260 |
+
"names": [
|
| 261 |
+
"smpl_frame_index"
|
| 262 |
+
]
|
| 263 |
+
},
|
| 264 |
+
"teleop.left_wrist_joints": {
|
| 265 |
+
"dtype": "float32",
|
| 266 |
+
"shape": [
|
| 267 |
+
3
|
| 268 |
+
],
|
| 269 |
+
"names": [
|
| 270 |
+
"left_wrist_roll",
|
| 271 |
+
"left_wrist_pitch",
|
| 272 |
+
"left_wrist_yaw"
|
| 273 |
+
]
|
| 274 |
+
},
|
| 275 |
+
"teleop.right_wrist_joints": {
|
| 276 |
+
"dtype": "float32",
|
| 277 |
+
"shape": [
|
| 278 |
+
3
|
| 279 |
+
],
|
| 280 |
+
"names": [
|
| 281 |
+
"right_wrist_roll",
|
| 282 |
+
"right_wrist_pitch",
|
| 283 |
+
"right_wrist_yaw"
|
| 284 |
+
]
|
| 285 |
+
},
|
| 286 |
+
"teleop.stream_mode": {
|
| 287 |
+
"dtype": "int32",
|
| 288 |
+
"shape": [
|
| 289 |
+
1
|
| 290 |
+
],
|
| 291 |
+
"names": [
|
| 292 |
+
"stream_mode"
|
| 293 |
+
]
|
| 294 |
+
},
|
| 295 |
+
"teleop.planner_mode": {
|
| 296 |
+
"dtype": "int32",
|
| 297 |
+
"shape": [
|
| 298 |
+
1
|
| 299 |
+
],
|
| 300 |
+
"names": [
|
| 301 |
+
"locomotion_mode"
|
| 302 |
+
]
|
| 303 |
+
},
|
| 304 |
+
"teleop.planner_movement": {
|
| 305 |
+
"dtype": "float32",
|
| 306 |
+
"shape": [
|
| 307 |
+
3
|
| 308 |
+
],
|
| 309 |
+
"names": [
|
| 310 |
+
"movement_x",
|
| 311 |
+
"movement_y",
|
| 312 |
+
"movement_z"
|
| 313 |
+
]
|
| 314 |
+
},
|
| 315 |
+
"teleop.planner_facing": {
|
| 316 |
+
"dtype": "float32",
|
| 317 |
+
"shape": [
|
| 318 |
+
3
|
| 319 |
+
],
|
| 320 |
+
"names": [
|
| 321 |
+
"facing_x",
|
| 322 |
+
"facing_y",
|
| 323 |
+
"facing_z"
|
| 324 |
+
]
|
| 325 |
+
},
|
| 326 |
+
"teleop.planner_speed": {
|
| 327 |
+
"dtype": "float32",
|
| 328 |
+
"shape": [
|
| 329 |
+
1
|
| 330 |
+
],
|
| 331 |
+
"names": [
|
| 332 |
+
"speed"
|
| 333 |
+
]
|
| 334 |
+
},
|
| 335 |
+
"teleop.planner_height": {
|
| 336 |
+
"dtype": "float32",
|
| 337 |
+
"shape": [
|
| 338 |
+
1
|
| 339 |
+
],
|
| 340 |
+
"names": [
|
| 341 |
+
"height"
|
| 342 |
+
]
|
| 343 |
+
},
|
| 344 |
+
"teleop.vr_3pt_position": {
|
| 345 |
+
"dtype": "float32",
|
| 346 |
+
"shape": [
|
| 347 |
+
9
|
| 348 |
+
],
|
| 349 |
+
"names": [
|
| 350 |
+
"lwrist_x",
|
| 351 |
+
"lwrist_y",
|
| 352 |
+
"lwrist_z",
|
| 353 |
+
"rwrist_x",
|
| 354 |
+
"rwrist_y",
|
| 355 |
+
"rwrist_z",
|
| 356 |
+
"neck_x",
|
| 357 |
+
"neck_y",
|
| 358 |
+
"neck_z"
|
| 359 |
+
]
|
| 360 |
+
},
|
| 361 |
+
"teleop.vr_3pt_orientation": {
|
| 362 |
+
"dtype": "float32",
|
| 363 |
+
"shape": [
|
| 364 |
+
18
|
| 365 |
+
],
|
| 366 |
+
"names": [
|
| 367 |
+
"lwrist_r00",
|
| 368 |
+
"lwrist_r10",
|
| 369 |
+
"lwrist_r01",
|
| 370 |
+
"lwrist_r11",
|
| 371 |
+
"lwrist_r02",
|
| 372 |
+
"lwrist_r12",
|
| 373 |
+
"rwrist_r00",
|
| 374 |
+
"rwrist_r10",
|
| 375 |
+
"rwrist_r01",
|
| 376 |
+
"rwrist_r11",
|
| 377 |
+
"rwrist_r02",
|
| 378 |
+
"rwrist_r12",
|
| 379 |
+
"neck_r00",
|
| 380 |
+
"neck_r10",
|
| 381 |
+
"neck_r01",
|
| 382 |
+
"neck_r11",
|
| 383 |
+
"neck_r02",
|
| 384 |
+
"neck_r12"
|
| 385 |
+
]
|
| 386 |
+
},
|
| 387 |
+
"observation.images.ego_view": {
|
| 388 |
+
"dtype": "video",
|
| 389 |
+
"shape": [
|
| 390 |
+
480,
|
| 391 |
+
640,
|
| 392 |
+
3
|
| 393 |
+
],
|
| 394 |
+
"names": [
|
| 395 |
+
"height",
|
| 396 |
+
"width",
|
| 397 |
+
"channel"
|
| 398 |
+
],
|
| 399 |
+
"info": {
|
| 400 |
+
"video.height": 480,
|
| 401 |
+
"video.width": 640,
|
| 402 |
+
"video.codec": "h264",
|
| 403 |
+
"video.pix_fmt": "yuv420p",
|
| 404 |
+
"video.is_depth_map": false,
|
| 405 |
+
"video.fps": 30,
|
| 406 |
+
"video.channels": 3,
|
| 407 |
+
"has_audio": false
|
| 408 |
+
}
|
| 409 |
+
},
|
| 410 |
+
"observation.root_position": {
|
| 411 |
+
"dtype": "float64",
|
| 412 |
+
"shape": [
|
| 413 |
+
3
|
| 414 |
+
],
|
| 415 |
+
"names": [
|
| 416 |
+
"base_x",
|
| 417 |
+
"base_y",
|
| 418 |
+
"base_z"
|
| 419 |
+
]
|
| 420 |
+
},
|
| 421 |
+
"observation.inspire_left_hand_state": {
|
| 422 |
+
"dtype": "float32",
|
| 423 |
+
"shape": [
|
| 424 |
+
6
|
| 425 |
+
],
|
| 426 |
+
"names": [
|
| 427 |
+
"left_pinky",
|
| 428 |
+
"left_ring",
|
| 429 |
+
"left_middle",
|
| 430 |
+
"left_index",
|
| 431 |
+
"left_thumb_bend",
|
| 432 |
+
"left_thumb_rotation"
|
| 433 |
+
]
|
| 434 |
+
},
|
| 435 |
+
"observation.inspire_right_hand_state": {
|
| 436 |
+
"dtype": "float32",
|
| 437 |
+
"shape": [
|
| 438 |
+
6
|
| 439 |
+
],
|
| 440 |
+
"names": [
|
| 441 |
+
"right_pinky",
|
| 442 |
+
"right_ring",
|
| 443 |
+
"right_middle",
|
| 444 |
+
"right_index",
|
| 445 |
+
"right_thumb_bend",
|
| 446 |
+
"right_thumb_rotation"
|
| 447 |
+
]
|
| 448 |
+
},
|
| 449 |
+
"observation.images.chest": {
|
| 450 |
+
"dtype": "video",
|
| 451 |
+
"shape": [
|
| 452 |
+
480,
|
| 453 |
+
640,
|
| 454 |
+
3
|
| 455 |
+
],
|
| 456 |
+
"names": [
|
| 457 |
+
"height",
|
| 458 |
+
"width",
|
| 459 |
+
"channel"
|
| 460 |
+
],
|
| 461 |
+
"info": {
|
| 462 |
+
"video.height": 480,
|
| 463 |
+
"video.width": 640,
|
| 464 |
+
"video.codec": "h264",
|
| 465 |
+
"video.pix_fmt": "yuv420p",
|
| 466 |
+
"video.is_depth_map": false,
|
| 467 |
+
"video.fps": 30,
|
| 468 |
+
"video.channels": 3,
|
| 469 |
+
"has_audio": false
|
| 470 |
+
}
|
| 471 |
+
},
|
| 472 |
+
"timestamp": {
|
| 473 |
+
"dtype": "float32",
|
| 474 |
+
"shape": [
|
| 475 |
+
1
|
| 476 |
+
],
|
| 477 |
+
"names": null
|
| 478 |
+
},
|
| 479 |
+
"frame_index": {
|
| 480 |
+
"dtype": "int64",
|
| 481 |
+
"shape": [
|
| 482 |
+
1
|
| 483 |
+
],
|
| 484 |
+
"names": null
|
| 485 |
+
},
|
| 486 |
+
"episode_index": {
|
| 487 |
+
"dtype": "int64",
|
| 488 |
+
"shape": [
|
| 489 |
+
1
|
| 490 |
+
],
|
| 491 |
+
"names": null
|
| 492 |
+
},
|
| 493 |
+
"index": {
|
| 494 |
+
"dtype": "int64",
|
| 495 |
+
"shape": [
|
| 496 |
+
1
|
| 497 |
+
],
|
| 498 |
+
"names": null
|
| 499 |
+
},
|
| 500 |
+
"task_index": {
|
| 501 |
+
"dtype": "int64",
|
| 502 |
+
"shape": [
|
| 503 |
+
1
|
| 504 |
+
],
|
| 505 |
+
"names": null
|
| 506 |
+
},
|
| 507 |
+
"action": {
|
| 508 |
+
"dtype": "float32",
|
| 509 |
+
"shape": [
|
| 510 |
+
76
|
| 511 |
+
],
|
| 512 |
+
"names": null
|
| 513 |
+
}
|
| 514 |
+
},
|
| 515 |
+
"script_config": {
|
| 516 |
+
"source": "sim",
|
| 517 |
+
"simulator": "newton",
|
| 518 |
+
"controller": "SONIC g1_deploy_onnx_ref in sim lockstep (planner and upper-body commands, no PICO stream)",
|
| 519 |
+
"exporter": "humanoid_newton.collect.lerobot_export",
|
| 520 |
+
"control_frequency": 50.0,
|
| 521 |
+
"record_wrist_cameras": false,
|
| 522 |
+
"record_chest_camera": true,
|
| 523 |
+
"rotate_chest_camera_180": false,
|
| 524 |
+
"record_depth": false,
|
| 525 |
+
"record_root_position": true,
|
| 526 |
+
"record_inspire_hand_state": true,
|
| 527 |
+
"record_inspire_hand_cmd": true,
|
| 528 |
+
"no_camera": false,
|
| 529 |
+
"camera_backend": "offline_render",
|
| 530 |
+
"camera_note": "ego_view and chest rendered offline (humanoid_newton.rendering.splat_render) from the recorded body poses at the registered D455 mounts; chest rendered upright (the real exporter rotates its upside-down chest stream by 180 deg); cameras not calibrated against the real ones",
|
| 531 |
+
"task": "b8",
|
| 532 |
+
"collect_config": "20261114_black_coffee_training_100_01",
|
| 533 |
+
"scene": "eai_room_fridge_001",
|
| 534 |
+
"task_description": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it.",
|
| 535 |
+
"git_commit": null
|
| 536 |
+
},
|
| 537 |
+
"discarded_episode_indices": []
|
| 538 |
+
}
|
b8-table-wipe/b512/step-004000/dataset_meta/tasks.jsonl
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"task_index": 0, "task": "Approach the flat block with a naturally aligned wrist, stop before closing the hand, wipe sideways at a top-view 45-degree heading, then set the block on the tabletop and release it."}
|
b8-table-wipe/b512/step-004000/dataset_stats.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-004000/logs/node-0.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-004000/logs/node-1.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
b8-table-wipe/b512/step-004000/logs/node-2.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|