Update README and adapter code
Browse files- code/hb_groot/h2_body31_hand44_h32.py +24 -0
- code/hb_groot/h2_body31_hand44_planner7_h16.py +25 -0
- code/hb_groot/h2w3_body31_hand44_h16.py +24 -0
- code/hb_groot/hb_inloop_eval.py +347 -0
- code/hb_groot/hb_launch.py +8 -0
- code/hb_groot/hb_launch_v.py +102 -0
- code/hb_groot/prepare_groot_data.py +35 -7
- code/hb_groot/verify_groot_cpu.py +8 -2
code/hb_groot/h2_body31_hand44_h32.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-15 user: action horizon 32 copy of h2_body31_hand44_h16.py. A18 H2 (Sharpa hands) GR00T modality config, same protocol as the A18 Delta-flow run
|
| 2 |
+
(schema a18-h2-sharpa-body31-hand44-state78-chunk16-v1): cameras head+chest, state = 75 joints + projected
|
| 3 |
+
gravity, action = action.body_target (31) + action.hands (44), horizon 16, absolute targets."""
|
| 4 |
+
from gr00t.configs.data.embodiment_configs import register_modality_config
|
| 5 |
+
from gr00t.data.embodiment_tags import EmbodimentTag
|
| 6 |
+
from gr00t.data.types import ActionConfig, ActionFormat, ActionRepresentation, ActionType, ModalityConfig
|
| 7 |
+
|
| 8 |
+
h2_body31_hand44_h32 = {
|
| 9 |
+
"video": ModalityConfig(delta_indices=[0], modality_keys=["head", "chest"]),
|
| 10 |
+
"state": ModalityConfig(
|
| 11 |
+
delta_indices=[0],
|
| 12 |
+
modality_keys=["left_leg", "right_leg", "waist", "left_arm", "right_arm", "head", "left_hand", "right_hand",
|
| 13 |
+
"projected_gravity"],
|
| 14 |
+
),
|
| 15 |
+
"action": ModalityConfig(
|
| 16 |
+
delta_indices=list(range(32)),
|
| 17 |
+
modality_keys=["body_target", "left_hand_target", "right_hand_target"],
|
| 18 |
+
action_configs=[ActionConfig(rep=ActionRepresentation.ABSOLUTE, type=ActionType.NON_EEF,
|
| 19 |
+
format=ActionFormat.DEFAULT) for _ in range(3)],
|
| 20 |
+
),
|
| 21 |
+
"language": ModalityConfig(delta_indices=[0], modality_keys=["annotation.human.task_description"]),
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
register_modality_config(h2_body31_hand44_h32, embodiment_tag=EmbodimentTag.NEW_EMBODIMENT)
|
code/hb_groot/h2_body31_hand44_planner7_h16.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A14 v2 H2 (Sharpa hands) GR00T modality config (H2 training guide hb_training/docs/h2/a14_training_guide.md 4.1):
|
| 2 |
+
cameras head + chest, state78 = 75 joints + projected gravity, action82 = action.body_target (31) + action.hands (44)
|
| 3 |
+
+ action.planner (7, the SONIC planner command), horizon 16, absolute targets. Identical to h2_body31_hand44_h16.py
|
| 4 |
+
except the added `planner` action modality."""
|
| 5 |
+
from gr00t.configs.data.embodiment_configs import register_modality_config
|
| 6 |
+
from gr00t.data.embodiment_tags import EmbodimentTag
|
| 7 |
+
from gr00t.data.types import ActionConfig, ActionFormat, ActionRepresentation, ActionType, ModalityConfig
|
| 8 |
+
|
| 9 |
+
h2_body31_hand44_planner7_h16 = {
|
| 10 |
+
"video": ModalityConfig(delta_indices=[0], modality_keys=["head", "chest"]),
|
| 11 |
+
"state": ModalityConfig(
|
| 12 |
+
delta_indices=[0],
|
| 13 |
+
modality_keys=["left_leg", "right_leg", "waist", "left_arm", "right_arm", "head", "left_hand", "right_hand",
|
| 14 |
+
"projected_gravity"],
|
| 15 |
+
),
|
| 16 |
+
"action": ModalityConfig(
|
| 17 |
+
delta_indices=list(range(16)),
|
| 18 |
+
modality_keys=["body_target", "left_hand_target", "right_hand_target", "planner"],
|
| 19 |
+
action_configs=[ActionConfig(rep=ActionRepresentation.ABSOLUTE, type=ActionType.NON_EEF,
|
| 20 |
+
format=ActionFormat.DEFAULT) for _ in range(4)],
|
| 21 |
+
),
|
| 22 |
+
"language": ModalityConfig(delta_indices=[0], modality_keys=["annotation.human.task_description"]),
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
register_modality_config(h2_body31_hand44_planner7_h16, embodiment_tag=EmbodimentTag.NEW_EMBODIMENT)
|
code/hb_groot/h2w3_body31_hand44_h16.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A14 H2 (Sharpa hands) GR00T modality config with THREE cameras head + chest + wrist (user 2026-11-12; the third-person camera is never a model input). Otherwise identical to h2_body31_hand44_h16.py:, same protocol as the A18 Delta-flow run
|
| 2 |
+
(schema a18-h2-sharpa-body31-hand44-state78-chunk16-v1): cameras head+chest, state = 75 joints + projected
|
| 3 |
+
gravity, action = action.body_target (31) + action.hands (44), horizon 16, absolute targets."""
|
| 4 |
+
from gr00t.configs.data.embodiment_configs import register_modality_config
|
| 5 |
+
from gr00t.data.embodiment_tags import EmbodimentTag
|
| 6 |
+
from gr00t.data.types import ActionConfig, ActionFormat, ActionRepresentation, ActionType, ModalityConfig
|
| 7 |
+
|
| 8 |
+
h2w3_body31_hand44_h16 = {
|
| 9 |
+
"video": ModalityConfig(delta_indices=[0], modality_keys=["head", "chest", "wrist"]),
|
| 10 |
+
"state": ModalityConfig(
|
| 11 |
+
delta_indices=[0],
|
| 12 |
+
modality_keys=["left_leg", "right_leg", "waist", "left_arm", "right_arm", "head", "left_hand", "right_hand",
|
| 13 |
+
"projected_gravity"],
|
| 14 |
+
),
|
| 15 |
+
"action": ModalityConfig(
|
| 16 |
+
delta_indices=list(range(16)),
|
| 17 |
+
modality_keys=["body_target", "left_hand_target", "right_hand_target"],
|
| 18 |
+
action_configs=[ActionConfig(rep=ActionRepresentation.ABSOLUTE, type=ActionType.NON_EEF,
|
| 19 |
+
format=ActionFormat.DEFAULT) for _ in range(3)],
|
| 20 |
+
),
|
| 21 |
+
"language": ModalityConfig(delta_indices=[0], modality_keys=["annotation.human.task_description"]),
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
register_modality_config(h2w3_body31_hand44_h16, embodiment_tag=EmbodimentTag.NEW_EMBODIMENT)
|
code/hb_groot/hb_inloop_eval.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GR00T in-training open-loop action MSE (2026-11-15 user: every 250 training steps).
|
| 2 |
+
|
| 3 |
+
Imported by hb_groot/hb_launch.py (hb_launch_v.py runs hb_launch.py). It wraps Gr00tTrainer.__init__ so that every
|
| 4 |
+
trainer gets a TrainerCallback. At global_step % HB_INLOOP_EVERY == 0 (default 250; 0 = off) every rank evaluates its
|
| 5 |
+
shard of the benchmark eval cache (eval-action-mse/cache) with the weights being trained right now, exactly as deployed:
|
| 6 |
+
* Gr00tPolicy pipeline: eval processor from <output_dir>/processor (the files every checkpoint ships),
|
| 7 |
+
parse_observation_gr00t inputs, bf16 inputs, model.get_action under inference_mode, processor.decode_action;
|
| 8 |
+
* same protocol and metrics as the post-hoc eval (hb_eval_common): queries every 16 frames, first 16 predicted steps
|
| 9 |
+
vs the recorded actions, nMSE with the training-frame std, per group / per step;
|
| 10 |
+
* split: "heldout" when the task has recorded episodes that are never trained on (B6, trolley, B7), else "train10".
|
| 11 |
+
Training is not changed: module train/eval modes are restored module by module, torch CPU + CUDA (this device), numpy
|
| 12 |
+
and python RNG states are restored after each eval; the flow-matching noise uses a fixed seed (rank, batch) so every
|
| 13 |
+
eval sees the same noise (smoother curve). ZeRO-2 keeps the full bf16 weights on every rank, so ranks run independently;
|
| 14 |
+
partial sums are all-reduced once. Results: <run>/logs/inloop-mse.jsonl (rank 0), one json line per eval, and an
|
| 15 |
+
"HB_INLOOP" log line. Failures never stop training: an exception on any rank disables further evals on all ranks
|
| 16 |
+
(error flag in the same all-reduce, so no rank waits on another).
|
| 17 |
+
Env: HB_INLOOP_EVERY (250), HB_INLOOP_BATCH (8 queries per forward), HB_INLOOP_MAX_QUERIES (0 = all; tests only),
|
| 18 |
+
HB_INLOOP_SPLIT (override), HB_INLOOP_PROCESSOR_DIR (tests only).
|
| 19 |
+
"""
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import random
|
| 23 |
+
import sys
|
| 24 |
+
import time
|
| 25 |
+
import traceback
|
| 26 |
+
from contextlib import nullcontext
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
import numpy as np
|
| 30 |
+
import torch
|
| 31 |
+
|
| 32 |
+
EVERY = int(os.environ.get("HB_INLOOP_EVERY", "250"))
|
| 33 |
+
BATCH = int(os.environ.get("HB_INLOOP_BATCH", "8"))
|
| 34 |
+
MAX_Q = int(os.environ.get("HB_INLOOP_MAX_QUERIES", "0"))
|
| 35 |
+
SEED = 1234
|
| 36 |
+
G = Path(__file__).resolve().parent.parent
|
| 37 |
+
EVAL_DIR = G / "eval-action-mse"
|
| 38 |
+
for _p in (str(EVAL_DIR), str(EVAL_DIR / "adapters")):
|
| 39 |
+
if _p not in sys.path:
|
| 40 |
+
sys.path.insert(0, _p)
|
| 41 |
+
import hb_eval_common as E # noqa: E402
|
| 42 |
+
import ad_groot as AG # noqa: E402 (canonical state -> GR00T state groups, same mapping as the post-hoc eval)
|
| 43 |
+
|
| 44 |
+
RANK = int(os.environ.get("RANK", "0"))
|
| 45 |
+
WORLD = int(os.environ.get("WORLD_SIZE", "1"))
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def say(*a):
|
| 49 |
+
if RANK == 0:
|
| 50 |
+
print("HB_INLOOP", *a, flush=True)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _argv(name):
|
| 54 |
+
a = sys.argv
|
| 55 |
+
for i, x in enumerate(a):
|
| 56 |
+
if x == name and i + 1 < len(a):
|
| 57 |
+
return a[i + 1]
|
| 58 |
+
if x.startswith(name + "="):
|
| 59 |
+
return x.split("=", 1)[1]
|
| 60 |
+
return None
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def resolve(dataset_path):
|
| 64 |
+
"""groot-data/<view> -> (benchmark task, split, view dir) or raises with the reason"""
|
| 65 |
+
view = Path(dataset_path)
|
| 66 |
+
rec = json.loads((view / "hb-dataset-receipt.json").read_text())
|
| 67 |
+
task = E.task_for_source(rec["source"])
|
| 68 |
+
if task is None:
|
| 69 |
+
raise LookupError(f"no benchmark task for source {rec['source']}")
|
| 70 |
+
split = os.environ.get("HB_INLOOP_SPLIT")
|
| 71 |
+
if not split:
|
| 72 |
+
split = "heldout" if (E.split_dir(task, "heldout") / "meta.json").exists() else "train10"
|
| 73 |
+
if not (E.split_dir(task, split) / "meta.json").exists():
|
| 74 |
+
raise FileNotFoundError(f"eval cache {task}/{split} missing (build_cache.py {task} {split})")
|
| 75 |
+
return task, split, view
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
class InLoopMSE:
|
| 79 |
+
"""transformers TrainerCallback (duck-typed: the callback handler only calls the hooks that exist)"""
|
| 80 |
+
|
| 81 |
+
def __init__(self, task, split, view, processor_dir=None):
|
| 82 |
+
self.task, self.split, self.view = task, split, view
|
| 83 |
+
self.processor_dir = processor_dir
|
| 84 |
+
self.disabled = False
|
| 85 |
+
self.ready = False
|
| 86 |
+
self.results = []
|
| 87 |
+
|
| 88 |
+
# ---------------------------------------------------------------- setup (first eval; needs the live model)
|
| 89 |
+
def _setup(self, model, output_dir):
|
| 90 |
+
from transformers import AutoProcessor
|
| 91 |
+
from gr00t.data.embodiment_tags import EmbodimentTag
|
| 92 |
+
from gr00t.policy.gr00t_policy import Gr00tPolicy
|
| 93 |
+
from gr00t.policy.policy import BasePolicy
|
| 94 |
+
|
| 95 |
+
meta = E.load_meta(self.task, self.split)
|
| 96 |
+
self.meta = meta
|
| 97 |
+
self.lay = E.LAYOUTS[meta["layout"]]
|
| 98 |
+
self.std = E.norm_std(E.load_stats(self.task))
|
| 99 |
+
# the Gr00tPolicy of a checkpoint, built around the live model instead of loading weights from disk
|
| 100 |
+
pol = Gr00tPolicy.__new__(Gr00tPolicy)
|
| 101 |
+
BasePolicy.__init__(pol, strict=True)
|
| 102 |
+
pdir = Path(self.processor_dir or os.environ.get("HB_INLOOP_PROCESSOR_DIR") or Path(output_dir) / "processor")
|
| 103 |
+
proc = AutoProcessor.from_pretrained(str(pdir))
|
| 104 |
+
proc.eval()
|
| 105 |
+
proc.validate_strict_action_padding_mask(model.config)
|
| 106 |
+
tag = EmbodimentTag.NEW_EMBODIMENT
|
| 107 |
+
pol.model, pol.processor, pol.embodiment_tag = model, proc, tag
|
| 108 |
+
pol.modality_configs = {k: v for k, v in proc.get_modality_configs()[tag.value].items() if k != "rl_info"}
|
| 109 |
+
pol.collate_fn = proc.collator
|
| 110 |
+
pol.language_key = pol.modality_configs["language"].modality_keys[0]
|
| 111 |
+
self.policy = pol
|
| 112 |
+
self.mod = pol.modality_configs
|
| 113 |
+
self.state_map = AG.H2_STATE if meta["layout"].startswith("h2") else AG.G1_STATE
|
| 114 |
+
self.video_keys = list(self.mod["video"].modality_keys)
|
| 115 |
+
self.state_keys = list(self.mod["state"].modality_keys)
|
| 116 |
+
self.action_keys = list(self.mod["action"].modality_keys)
|
| 117 |
+
missing = [k for k in self.state_keys if k not in self.state_map] + \
|
| 118 |
+
[k for k in self.video_keys if k not in meta["cams"]]
|
| 119 |
+
if missing:
|
| 120 |
+
raise KeyError(f"model expects {missing}, not in the benchmark state / cache cameras {meta['cams']}")
|
| 121 |
+
tj = self.view / "meta/tasks.jsonl"
|
| 122 |
+
self.prompts = {str(r["task_index"]): r["task"] for r in E.jl(tj)} if tj.exists() else {}
|
| 123 |
+
# all queries of the split in a fixed order; this rank keeps every WORLD-th one (images only for those)
|
| 124 |
+
allq = []
|
| 125 |
+
for ep in meta["episodes"]:
|
| 126 |
+
with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
|
| 127 |
+
allq += [(ep, qi, int(t)) for qi, t in enumerate(z["query_t"])]
|
| 128 |
+
if MAX_Q:
|
| 129 |
+
allq = allq[:MAX_Q]
|
| 130 |
+
self.n_total = len(allq)
|
| 131 |
+
mine = [q for i, q in enumerate(allq) if i % WORLD == RANK]
|
| 132 |
+
self.eps = list(meta["episodes"])
|
| 133 |
+
self.q = [] # (ep, t, T, state[t], prompt, {cam: img}, gt(16, Da), valid(16))
|
| 134 |
+
by_ep = {}
|
| 135 |
+
for ep, qi, t in mine:
|
| 136 |
+
by_ep.setdefault(ep, []).append((qi, t))
|
| 137 |
+
for ep, lst in by_ep.items():
|
| 138 |
+
with np.load(E.split_dir(self.task, self.split) / f"ep_{ep:06d}.npz") as z:
|
| 139 |
+
state, action = z["state"], z["action"]
|
| 140 |
+
prompt = meta["prompts"][str(int(z["task_index"]))]
|
| 141 |
+
imgs = {c: z[f"img_{c}"] for c in self.video_keys}
|
| 142 |
+
for qi, t in lst:
|
| 143 |
+
gt, valid = E.gt_chunk(action, t)
|
| 144 |
+
self.q.append((ep, t, len(action), state[t].copy(), prompt,
|
| 145 |
+
{c: imgs[c][qi].copy() for c in self.video_keys}, gt, valid))
|
| 146 |
+
self.ready = True
|
| 147 |
+
say(f"ready: task {self.task} split {self.split} queries {self.n_total} (rank {RANK}: {len(self.q)}), "
|
| 148 |
+
f"every {EVERY} steps, batch {BATCH}, cams {self.video_keys}, processor {pdir}")
|
| 149 |
+
|
| 150 |
+
# ---------------------------------------------------------------- one forward on a batch of queries
|
| 151 |
+
def _obs(self, q):
|
| 152 |
+
from gr00t.data.utils import parse_observation_gr00t
|
| 153 |
+
ep, t, T, st, prompt, imgs, _, _ = q
|
| 154 |
+
o = {}
|
| 155 |
+
for k in self.state_keys:
|
| 156 |
+
a, b = self.state_map[k]
|
| 157 |
+
o[f"state.{k}"] = np.asarray(st[a:b], dtype=np.float32)[None]
|
| 158 |
+
for k in self.video_keys:
|
| 159 |
+
o[f"video.{k}"] = imgs[k][None]
|
| 160 |
+
o[self.language_key()] = self.prompts.get("0", prompt) if len(self.prompts) == 1 else prompt
|
| 161 |
+
return parse_observation_gr00t(o, self.mod)
|
| 162 |
+
|
| 163 |
+
def language_key(self):
|
| 164 |
+
return self.policy.language_key
|
| 165 |
+
|
| 166 |
+
@staticmethod
|
| 167 |
+
def _merge(obs_list):
|
| 168 |
+
out = {}
|
| 169 |
+
for m in obs_list[0]:
|
| 170 |
+
out[m] = {}
|
| 171 |
+
for k, v in obs_list[0][m].items():
|
| 172 |
+
vals = [o[m][k] for o in obs_list]
|
| 173 |
+
out[m][k] = np.concatenate(vals, axis=0) if isinstance(v, np.ndarray) else sum(vals, [])
|
| 174 |
+
return out
|
| 175 |
+
|
| 176 |
+
def _predict(self, batch, model):
|
| 177 |
+
obs = self._merge([self._obs(q) for q in batch])
|
| 178 |
+
dtype = next(model.parameters()).dtype
|
| 179 |
+
ctx = torch.autocast("cuda", dtype=torch.bfloat16) if (dtype != torch.bfloat16 and torch.cuda.is_available()) \
|
| 180 |
+
else nullcontext()
|
| 181 |
+
with ctx:
|
| 182 |
+
act, _ = self.policy.get_action(obs)
|
| 183 |
+
return np.concatenate([np.asarray(act[k], dtype=np.float64) for k in self.action_keys], axis=-1)
|
| 184 |
+
|
| 185 |
+
# ---------------------------------------------------------------- the eval
|
| 186 |
+
def evaluate(self, step, model, output_dir):
|
| 187 |
+
t0 = time.time()
|
| 188 |
+
D = self.lay["action_dim"] if self.ready else 0
|
| 189 |
+
err, msg = 0.0, ""
|
| 190 |
+
acc = None
|
| 191 |
+
nq = 0
|
| 192 |
+
infer = 0.0
|
| 193 |
+
modes = [(m, m.training) for m in model.modules()]
|
| 194 |
+
dev = torch.cuda.current_device() if torch.cuda.is_available() else None
|
| 195 |
+
np_state, py_state = np.random.get_state(), random.getstate()
|
| 196 |
+
try:
|
| 197 |
+
with torch.random.fork_rng(devices=[dev] if dev is not None else []):
|
| 198 |
+
if not self.ready:
|
| 199 |
+
self._setup(model, output_dir)
|
| 200 |
+
D = self.lay["action_dim"]
|
| 201 |
+
model.eval()
|
| 202 |
+
acc = E.Accumulator(self.std, self.lay["groups"])
|
| 203 |
+
for bi in range(0, len(self.q), BATCH):
|
| 204 |
+
batch = self.q[bi:bi + BATCH]
|
| 205 |
+
torch.default_generator.manual_seed(SEED + 1000 * RANK + bi)
|
| 206 |
+
if dev is not None:
|
| 207 |
+
torch.cuda.manual_seed(SEED + 1000 * RANK + bi)
|
| 208 |
+
s = time.time()
|
| 209 |
+
pred = self._predict(batch, model)
|
| 210 |
+
infer += time.time() - s
|
| 211 |
+
if pred.shape[0] != len(batch) or pred.shape[1] < E.HORIZON or pred.shape[2] != D:
|
| 212 |
+
raise ValueError(f"prediction {pred.shape}, need ({len(batch)}, >={E.HORIZON}, {D})")
|
| 213 |
+
for q, p in zip(batch, pred):
|
| 214 |
+
acc.add(q[0], p[:E.HORIZON], q[6], q[7])
|
| 215 |
+
nq += 1
|
| 216 |
+
except Exception as e: # noqa: BLE001 - never stop training
|
| 217 |
+
err, msg = 1.0, f"{type(e).__name__}: {e}"
|
| 218 |
+
print(f"HB_INLOOP rank {RANK} step {step} FAILED: {msg}\n{traceback.format_exc()}", file=sys.stderr,
|
| 219 |
+
flush=True)
|
| 220 |
+
finally:
|
| 221 |
+
for m, tr in modes:
|
| 222 |
+
m.training = tr
|
| 223 |
+
np.random.set_state(np_state)
|
| 224 |
+
random.setstate(py_state)
|
| 225 |
+
# ---- one all-reduce of all partial sums (+ error flag); D / eps are identical on every rank once ready
|
| 226 |
+
H = E.HORIZON
|
| 227 |
+
eps = self.eps if self.ready else []
|
| 228 |
+
nE = len(eps)
|
| 229 |
+
vec = np.zeros(3 * D + 1 + 2 * H + 2 * nE + 3)
|
| 230 |
+
if acc is not None and not err:
|
| 231 |
+
o = 0
|
| 232 |
+
vec[o:o + D] = acc.sse_raw; o += D
|
| 233 |
+
vec[o:o + D] = acc.sae_raw; o += D
|
| 234 |
+
vec[o:o + D] = acc.sse_n; o += D
|
| 235 |
+
vec[o] = acc.rows; o += 1
|
| 236 |
+
vec[o:o + H] = acc.step_sse_n; o += H
|
| 237 |
+
vec[o:o + H] = acc.step_rows; o += H
|
| 238 |
+
for j, ep in enumerate(eps):
|
| 239 |
+
s, c = acc.per_episode.get(int(ep), (0.0, 0))
|
| 240 |
+
vec[o + j], vec[o + nE + j] = s, c
|
| 241 |
+
o += 2 * nE
|
| 242 |
+
vec[o], vec[o + 1] = nq, infer
|
| 243 |
+
vec[-1] = err
|
| 244 |
+
tot = self._allreduce(vec)
|
| 245 |
+
if tot is None: # ranks disagree on the vector length (setup failed somewhere) -> stop for good
|
| 246 |
+
self.disabled = True
|
| 247 |
+
self._write(output_dir, {"step": step, "error": "length mismatch across ranks / collective failed"})
|
| 248 |
+
return
|
| 249 |
+
if tot[-1] > 0 or not self.ready:
|
| 250 |
+
self.disabled = True
|
| 251 |
+
self._write(output_dir, {"step": step, "error": msg or "failed on another rank", "ranks_failed": tot[-1]})
|
| 252 |
+
say(f"step {step}: eval failed on {int(tot[-1])} rank(s) -> in-training eval disabled ({msg})")
|
| 253 |
+
return
|
| 254 |
+
o = 0
|
| 255 |
+
acc = E.Accumulator(self.std, self.lay["groups"])
|
| 256 |
+
acc.sse_raw = tot[o:o + D]; o += D
|
| 257 |
+
acc.sae_raw = tot[o:o + D]; o += D
|
| 258 |
+
acc.sse_n = tot[o:o + D]; o += D
|
| 259 |
+
acc.rows = int(round(tot[o])); o += 1
|
| 260 |
+
acc.step_sse_n = tot[o:o + H]; o += H
|
| 261 |
+
acc.step_rows = tot[o:o + H]; o += H
|
| 262 |
+
acc.per_episode = {int(ep): [float(tot[o + j]), int(round(tot[o + nE + j]))]
|
| 263 |
+
for j, ep in enumerate(eps) if tot[o + nE + j] > 0}
|
| 264 |
+
o += 2 * nE
|
| 265 |
+
n_q, infer_sum = int(round(tot[o])), float(tot[o + 1])
|
| 266 |
+
res = acc.result()
|
| 267 |
+
res.pop("nmse_by_dim", None)
|
| 268 |
+
rec = {"step": step, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "task": self.task,
|
| 269 |
+
"split": self.split, "queries": n_q, "ranks": WORLD, "wall_s": round(time.time() - t0, 1),
|
| 270 |
+
"infer_s_per_query": round(infer_sum / max(n_q, 1), 4),
|
| 271 |
+
"denoising_steps": int(getattr(model.action_head, "num_inference_timesteps", -1))
|
| 272 |
+
if hasattr(model, "action_head") else None, "seed": SEED, **res}
|
| 273 |
+
self.results.append(rec)
|
| 274 |
+
self._write(output_dir, rec)
|
| 275 |
+
g = " ".join(f"{k} {v['nmse']:.4f}" for k, v in res["groups"].items())
|
| 276 |
+
say(f"step {step}: {self.task}/{self.split} nMSE {res['nmse']:.4f} ({g}) on {n_q} queries, "
|
| 277 |
+
f"{rec['wall_s']} s")
|
| 278 |
+
|
| 279 |
+
def _allreduce(self, vec):
|
| 280 |
+
import torch.distributed as dist
|
| 281 |
+
if not (dist.is_available() and dist.is_initialized()) or WORLD == 1:
|
| 282 |
+
return vec
|
| 283 |
+
try:
|
| 284 |
+
dev = "cuda" if dist.get_backend() == "nccl" else "cpu"
|
| 285 |
+
n = torch.tensor([float(len(vec))], dtype=torch.float64, device=dev)
|
| 286 |
+
mx = n.clone()
|
| 287 |
+
dist.all_reduce(mx, op=dist.ReduceOp.MAX)
|
| 288 |
+
mn = n.clone()
|
| 289 |
+
dist.all_reduce(mn, op=dist.ReduceOp.MIN)
|
| 290 |
+
if mx.item() != mn.item():
|
| 291 |
+
return None
|
| 292 |
+
t = torch.from_numpy(vec).to(device=dev, dtype=torch.float64)
|
| 293 |
+
dist.all_reduce(t)
|
| 294 |
+
return t.cpu().numpy()
|
| 295 |
+
except Exception as e: # noqa: BLE001
|
| 296 |
+
print(f"HB_INLOOP rank {RANK} all_reduce failed: {e!r}", file=sys.stderr, flush=True)
|
| 297 |
+
return None
|
| 298 |
+
|
| 299 |
+
def _write(self, output_dir, rec):
|
| 300 |
+
if RANK != 0:
|
| 301 |
+
return
|
| 302 |
+
try:
|
| 303 |
+
out = Path(output_dir).parent / "logs" / "inloop-mse.jsonl"
|
| 304 |
+
out.parent.mkdir(parents=True, exist_ok=True)
|
| 305 |
+
with open(out, "a") as f:
|
| 306 |
+
f.write(json.dumps(rec) + "\n")
|
| 307 |
+
except Exception as e: # noqa: BLE001
|
| 308 |
+
print(f"HB_INLOOP write failed: {e!r}", flush=True)
|
| 309 |
+
|
| 310 |
+
# ---------------------------------------------------------------- TrainerCallback hooks
|
| 311 |
+
def on_step_end(self, args, state, control, model=None, **kw):
|
| 312 |
+
step = int(state.global_step)
|
| 313 |
+
if self.disabled or EVERY <= 0 or step <= 0 or step % EVERY or model is None:
|
| 314 |
+
return control
|
| 315 |
+
self.evaluate(step, model, args.output_dir)
|
| 316 |
+
return control
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _install():
|
| 320 |
+
if EVERY <= 0:
|
| 321 |
+
say("disabled (HB_INLOOP_EVERY=0)")
|
| 322 |
+
return
|
| 323 |
+
ds = _argv("--dataset_path") or _argv("--dataset-path")
|
| 324 |
+
if not ds:
|
| 325 |
+
say("disabled: no --dataset_path in argv")
|
| 326 |
+
return
|
| 327 |
+
try:
|
| 328 |
+
task, split, view = resolve(ds)
|
| 329 |
+
except Exception as e: # noqa: BLE001
|
| 330 |
+
say(f"disabled: {e}")
|
| 331 |
+
return
|
| 332 |
+
from transformers import TrainerCallback
|
| 333 |
+
from gr00t.experiment.trainer import Gr00tTrainer
|
| 334 |
+
|
| 335 |
+
cb_cls = type("HBInLoopMSE", (InLoopMSE, TrainerCallback), {})
|
| 336 |
+
_orig_init = Gr00tTrainer.__init__
|
| 337 |
+
|
| 338 |
+
def __init__(self, *a, **k):
|
| 339 |
+
_orig_init(self, *a, **k)
|
| 340 |
+
self.add_callback(cb_cls(task, split, view))
|
| 341 |
+
|
| 342 |
+
Gr00tTrainer.__init__ = __init__
|
| 343 |
+
say(f"installed: task {task} split {split} every {EVERY} steps (view {view.name})")
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
if os.environ.get("HB_INLOOP_NO_INSTALL") != "1":
|
| 347 |
+
_install()
|
code/hb_groot/hb_launch.py
CHANGED
|
@@ -74,6 +74,14 @@ Proc._get_vlm_inputs = _get_vlm_inputs
|
|
| 74 |
Proc._apply_vlm_processing = _apply_vlm_processing
|
| 75 |
Proc.__call__ = _call
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
if os.environ.get("HB_PATCH_ONLY") != "1":
|
| 78 |
entry = Path(os.environ["GROOT_SRC"]) / "gr00t/experiment/launch_finetune.py"
|
| 79 |
sys.argv = [str(entry)] + sys.argv[1:]
|
|
|
|
| 74 |
Proc._apply_vlm_processing = _apply_vlm_processing
|
| 75 |
Proc.__call__ = _call
|
| 76 |
|
| 77 |
+
# 2026-11-15 user: in-training open-loop action MSE every 250 steps (hb_inloop_eval.py adds a TrainerCallback to
|
| 78 |
+
# Gr00tTrainer; HB_INLOOP_EVERY=0 turns it off). It never blocks training: any setup problem only disables it.
|
| 79 |
+
if os.environ.get("HB_PATCH_ONLY") != "1":
|
| 80 |
+
try:
|
| 81 |
+
import hb_inloop_eval # noqa: E402,F401
|
| 82 |
+
except Exception as _e: # noqa: BLE001
|
| 83 |
+
print(f"HB_INLOOP disabled: {_e!r}", flush=True)
|
| 84 |
+
|
| 85 |
if os.environ.get("HB_PATCH_ONLY") != "1":
|
| 86 |
entry = Path(os.environ["GROOT_SRC"]) / "gr00t/experiment/launch_finetune.py"
|
| 87 |
sys.argv = [str(entry)] + sys.argv[1:]
|
code/hb_groot/hb_launch_v.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""2026-11-15 user: GR00T variant launcher = hb_launch.py (HB aug v2 + A20 proprio + processor patch, then the pinned
|
| 2 |
+
upstream launch_finetune.py) plus, from environment variables, before the run starts:
|
| 3 |
+
HB_GROOT_MODEL_OVERRIDES JSON {"attn_dropout": 0.0, "diffusion_model_cfg.dropout": 0.0, "num_inference_timesteps": 16,
|
| 4 |
+
...}: set on config.model (dotted keys go into nested dicts / attributes)
|
| 5 |
+
HB_GROOT_LR_GROUPS JSON {"language": 1e-6, "vision": 1e-5, "projector": 1e-4, "rest": 1e-4}: layer-wise learning
|
| 6 |
+
rates (Psi-Zero style) through Gr00tTrainer.create_optimizer:
|
| 7 |
+
language = backbone ... language_model.* and lm_head
|
| 8 |
+
vision = backbone ... visual.* except the vision->language mergers
|
| 9 |
+
projector = visual.merger.* and visual.deepstack_merger_list.*
|
| 10 |
+
rest = action head (DiT, VL self-attention, VLLN, encoders / decoders)
|
| 11 |
+
weight decay as HF Trainer (no decay on biases / norms); cosine + warmup scale every group.
|
| 12 |
+
Prints HB_LAUNCH_V lines (overrides applied, parameter count and lr per group) on rank 0 for verification."""
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import runpy
|
| 16 |
+
import sys
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
H = Path(__file__).resolve().parent
|
| 20 |
+
sys.path.insert(0, str(H))
|
| 21 |
+
RANK0 = os.environ.get("RANK", "0") == "0"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def say(*a):
|
| 25 |
+
if RANK0:
|
| 26 |
+
print("HB_LAUNCH_V", *a, flush=True)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
OV = json.loads(os.environ.get("HB_GROOT_MODEL_OVERRIDES", "{}") or "{}")
|
| 30 |
+
LRG = json.loads(os.environ.get("HB_GROOT_LR_GROUPS", "{}") or "{}")
|
| 31 |
+
|
| 32 |
+
import gr00t.experiment.experiment as E # noqa: E402
|
| 33 |
+
|
| 34 |
+
_run = E.run
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _set(obj, dotted, value):
|
| 38 |
+
parts = dotted.split(".")
|
| 39 |
+
for p in parts[:-1]:
|
| 40 |
+
obj = obj[p] if isinstance(obj, dict) else getattr(obj, p)
|
| 41 |
+
last = parts[-1]
|
| 42 |
+
if isinstance(obj, dict):
|
| 43 |
+
if last not in obj:
|
| 44 |
+
raise KeyError(f"model override {dotted}: no key {last}")
|
| 45 |
+
old = obj[last]
|
| 46 |
+
obj[last] = value
|
| 47 |
+
else:
|
| 48 |
+
if not hasattr(obj, last):
|
| 49 |
+
raise AttributeError(f"model override {dotted}: no attribute {last}")
|
| 50 |
+
old = getattr(obj, last)
|
| 51 |
+
setattr(obj, last, value)
|
| 52 |
+
say(f"override model.{dotted}: {old!r} -> {value!r}")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def run(config):
|
| 56 |
+
for k, v in OV.items():
|
| 57 |
+
_set(config.model, k, v)
|
| 58 |
+
return _run(config)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
E.run = run
|
| 62 |
+
|
| 63 |
+
if LRG:
|
| 64 |
+
from transformers import Trainer
|
| 65 |
+
from gr00t.experiment.trainer import Gr00tTrainer
|
| 66 |
+
|
| 67 |
+
def group_of(name):
|
| 68 |
+
if ".language_model." in name or "lm_head" in name:
|
| 69 |
+
return "language"
|
| 70 |
+
if ".visual." in name:
|
| 71 |
+
return "projector" if (".visual.merger." in name or ".visual.deepstack_merger_list." in name) else "vision"
|
| 72 |
+
return "rest"
|
| 73 |
+
|
| 74 |
+
def create_optimizer(self):
|
| 75 |
+
if self.optimizer is not None:
|
| 76 |
+
return self.optimizer
|
| 77 |
+
model = self.model
|
| 78 |
+
decay = set(self.get_decay_parameter_names(model))
|
| 79 |
+
groups, stats = {}, {}
|
| 80 |
+
for n, p in model.named_parameters():
|
| 81 |
+
if not p.requires_grad:
|
| 82 |
+
continue
|
| 83 |
+
g = group_of(n)
|
| 84 |
+
lr = float(LRG.get(g, LRG.get("rest", self.args.learning_rate)))
|
| 85 |
+
wd = self.args.weight_decay if n in decay else 0.0
|
| 86 |
+
groups.setdefault((g, lr, wd), []).append(p)
|
| 87 |
+
s = stats.setdefault(g, [lr, 0, 0])
|
| 88 |
+
s[1] += 1
|
| 89 |
+
s[2] += p.numel()
|
| 90 |
+
opt_cls, opt_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args, model)
|
| 91 |
+
opt_kwargs.pop("lr", None)
|
| 92 |
+
self.optimizer = opt_cls([{"params": ps, "lr": lr, "weight_decay": wd} for (g, lr, wd), ps in groups.items()],
|
| 93 |
+
lr=self.args.learning_rate, **opt_kwargs)
|
| 94 |
+
for g, (lr, n_t, n_p) in sorted(stats.items()):
|
| 95 |
+
say(f"lr group {g}: lr {lr:g}, {n_t} tensors, {n_p / 1e6:.1f} M params")
|
| 96 |
+
return self.optimizer
|
| 97 |
+
|
| 98 |
+
Gr00tTrainer.create_optimizer = create_optimizer
|
| 99 |
+
say("layer-wise lr groups installed", json.dumps(LRG))
|
| 100 |
+
|
| 101 |
+
say("model overrides", json.dumps(OV))
|
| 102 |
+
runpy.run_path(str(H / "hb_launch.py"), run_name="__main__")
|
code/hb_groot/prepare_groot_data.py
CHANGED
|
@@ -17,22 +17,47 @@ OUT = G / "groot-data"
|
|
| 17 |
TASKS = {
|
| 18 |
"b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100),
|
| 19 |
"basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None), # HF basket-transfer-expert-v1 @current-dynamics-100-v21 b0ff522 (old ed77719 retired)
|
|
|
|
| 20 |
"trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100), # HF trolley-push-expert-v1 @ef9024a: fixed first 100 of 103
|
| 21 |
"b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100), # HF b7-black-clothes-hamper-expert-v1 @1bb002a, first 100 of 107
|
| 22 |
"a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None), # HF door-pull-expert-v1 @cf3ee9b (A6)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
"c6t": (P / "c6-test8-06c231d/data", None), # c6 8-episode adaptation test only
|
| 24 |
"c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None), # HF c6-bathroom-door-fetch-expert-v1 @3afba69 (C6)
|
| 25 |
"a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None),
|
| 26 |
"a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None),
|
| 27 |
"a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None),
|
| 28 |
"a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None), # HF a17-unscrew-bottle-expert-v1 @407e6e2
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
}
|
| 30 |
-
CONFIG = {"a18": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
H2_GROUPS = {"left_leg": (0, 6), "right_leg": (6, 12), "waist": (12, 15), "left_arm": (15, 22), "right_arm": (22, 29),
|
| 32 |
"head": (29, 31), "left_hand": (31, 53), "right_hand": (53, 75)}
|
| 33 |
|
| 34 |
|
| 35 |
-
def h2_modality(info):
|
| 36 |
f = info["features"]
|
| 37 |
names = f["observation.state"]["names"]
|
| 38 |
assert len(names) == 75 and names[:31] == f["action.body_target"]["names"] and names[31:] == f["action.hands"]["names"]
|
|
@@ -43,8 +68,7 @@ def h2_modality(info):
|
|
| 43 |
"action": {"body_target": {"original_key": "action.body_target", "start": 0, "end": 31},
|
| 44 |
"left_hand_target": {"original_key": "action.hands", "start": 0, "end": 22},
|
| 45 |
"right_hand_target": {"original_key": "action.hands", "start": 22, "end": 44}},
|
| 46 |
-
"video": {
|
| 47 |
-
"chest": {"original_key": "observation.images.chest"}},
|
| 48 |
"annotation": {"human.task_description": {"original_key": "task_index"}}}
|
| 49 |
BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint", "left_ankle_pitch_joint",
|
| 50 |
"left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint", "right_hip_yaw_joint",
|
|
@@ -105,8 +129,9 @@ def build(task):
|
|
| 105 |
# stats.py computes stats for every low-dim feature in info.json and cannot handle e.g. observation.tactile
|
| 106 |
# [10,24,24]; keep only what the H2 modality uses (parquet columns stay untouched behind the symlinks).
|
| 107 |
keep = {"observation.state", "observation.projected_gravity", "action.body_target", "action.hands",
|
| 108 |
-
"
|
| 109 |
-
|
|
|
|
| 110 |
info["features"] = {k: v for k, v in info["features"].items() if k in keep}
|
| 111 |
vkeys = [k for k in vkeys if k in keep]
|
| 112 |
stats = [{**s, "stats": {k: v for k, v in s["stats"].items() if k in keep}} if "stats" in s else s
|
|
@@ -118,7 +143,10 @@ def build(task):
|
|
| 118 |
wl(dst / "meta/episodes_stats.jsonl", stats)
|
| 119 |
(dst / "meta/tasks.jsonl").write_text((src / "meta/tasks.jsonl").read_text())
|
| 120 |
if task in CONFIG:
|
| 121 |
-
modality = h2_modality(info)
|
|
|
|
|
|
|
|
|
|
| 122 |
else:
|
| 123 |
state = {g: {"original_key": "observation.state", "start": idx[a], "end": idx[a] + (b - a)}
|
| 124 |
for g, (a, b) in GROUPS.items()}
|
|
|
|
| 17 |
TASKS = {
|
| 18 |
"b6": (P / "b6-v5t7-ready-start-60ep-20261109/data/deliveries/g1/b6-v5t7-ready-start-20261109", 100),
|
| 19 |
"basket": (P / "basket-current-dynamics-100-v21-b0ff522/data", None), # HF basket-transfer-expert-v1 @current-dynamics-100-v21 b0ff522 (old ed77719 retired)
|
| 20 |
+
"basketv2": (P / "basket-trimv2-c4df4d4/data", None), # HF a20-basket-transfer-pre-reach-trim-expert-v2 @c4df4d40 (user 2026-11-14)
|
| 21 |
"trolley": (P / "trolley-push-expert-v1-20261110/data/trolley_push", 100), # HF trolley-push-expert-v1 @ef9024a: fixed first 100 of 103
|
| 22 |
"b7": (P / "b7-black-clothes-hamper-v1-1bb002a/data", 100), # HF b7-black-clothes-hamper-expert-v1 @1bb002a, first 100 of 107
|
| 23 |
"a6": (P / "a6-door-pull-v1-cf3ee9b/data/door_pull", None), # HF door-pull-expert-v1 @cf3ee9b (A6)
|
| 24 |
+
"b7green": (P / "b7-green-clothes-hamper-v1-ada288c/data", None), # HF b7-green-clothes-hamper-expert-v1 @ada288c, 100 episodes (user 2026-11-13)
|
| 25 |
+
"a7v2": (P / "a7-cabinet-close-v2-aecf070/data", None), # HF a7-cabinet-close-expert-v2 @aecf070, 100 episodes (user 2026-11-14)
|
| 26 |
+
"b8": (P / "b8-table-wipe-v1-3b2e618/data", None), # HF b8-table-wipe-expert-v1 @3b2e618, 100 episodes (user 2026-11-14)
|
| 27 |
+
"b1": (P / "b1-riceball-microwave-v1-36da298/data", None), # HF b1-riceball-microwave-expert-v1 @36da298, all 115 episodes (user 2026-11-14)
|
| 28 |
+
"a10v3": (P / "a10-trolley-push-v3-6e410c5/data", None), # HF a10-trolley-push-expert-v3 @6e410c5, all 148 episodes (user 2026-11-15)
|
| 29 |
+
"a3v3": (P / "a3-fridge-open-v3-24882d5/data", None), # HF a3-fridge-open-expert-v3 @24882d5, all 125 episodes, measured hands (user 2026-11-15)
|
| 30 |
+
"a7v3": (P / "a7-cabinet-close-v3-c2e15b0/data", None), # HF a7-cabinet-close-expert-v3 @c2e15b0, all 100 episodes, measured hands (user 2026-11-15)
|
| 31 |
+
"b6v6": (P / "b6-pedal-bin-v6-ad2f5bf/data", None), # HF b6-pedal-bin @ad2f5bf0 (v6 at the repo root), all 542 episodes (user 2026-11-15)
|
| 32 |
+
"b5": (P / "b5-chair-push-v1-b85b7e9/data", None), # HF b5-chair-push-expert-v1 @b85b7e9, all 100 episodes (user 2026-11-15)
|
| 33 |
+
"a3v4": (P / "a3-fridge-open-v4-6393d4a/data", None), # HF a3-fridge-open-expert-v4 @6393d4a, all 125 episodes (user 2026-11-15: v4 replaces v3)
|
| 34 |
+
"a7v4": (P / "a7-cabinet-close-v4-90cc2b5/data", None), # HF a7-cabinet-close-expert-v4 @90cc2b5, all 100 episodes (user 2026-11-15: v4 replaces v3)
|
| 35 |
+
"b7greenv2": (P / "b7-green-clothes-hamper-v2-c12757e/data", None), # HF b7-green-clothes-hamper @c12757e (v2 at the repo root), all 120 episodes (user 2026-11-14)
|
| 36 |
+
"a10v2": (P / "a10-trolley-push-v2-d3abb4c/data", None), # HF a10-trolley-push-expert-v2 @d3abb4c, all 150 episodes (user 2026-11-13)
|
| 37 |
+
"a1": (P / "a1-ground-object-pickup-v1-f47f07e/data", None), # HF a1-ground-object-pickup-expert-v1 @f47f07e (A1)
|
| 38 |
"c6t": (P / "c6-test8-06c231d/data", None), # c6 8-episode adaptation test only
|
| 39 |
"c6": (P / "c6-bathroom-door-fetch-v1-3afba69/data", None), # HF c6-bathroom-door-fetch-expert-v1 @3afba69 (C6)
|
| 40 |
"a3": (P / "a3-a7-20261109/a3-fridge-open-expert-v1/780fcf1e4bd7db947a8c2d8f7b92e61f24f71b45", None),
|
| 41 |
"a7": (P / "a3-a7-20261109/a7-cabinet-close-expert-v1/1935a98e8f55d4d1d2d1615ddb97c2679713595c", None),
|
| 42 |
"a18": (P / "a18-h2-dataset-audit-8e956a10/dataset", None),
|
| 43 |
"a17": (P / "a17-unscrew-bottle-v1-407e6e2/data", None), # HF a17-unscrew-bottle-expert-v1 @407e6e2
|
| 44 |
+
"a18c2": (P / "a18-h2-dataset-audit-8e956a10/dataset", None), # A18, 2 cameras head + chest (user 2026-11-14 18:2x)
|
| 45 |
+
"a17c2": (P / "a17-unscrew-bottle-v1-407e6e2/data", None), # A17, 2 cameras head + chest (user 2026-11-14 18:2x)
|
| 46 |
+
"a14": (P / "a14-unscrew-bulb-v1-b4390fc/data", None), # HF a14-unscrew-bulb-expert-v1 @b4390fc (H2, 3 cameras)
|
| 47 |
+
"a14v2": (P / "a14-unscrew-bulb-v2-8411a6d/data", None), # HF a14-unscrew-bulb-expert-v2 @8411a6da: + action.planner (H2 guide 2026-11-14)
|
| 48 |
}
|
| 49 |
+
CONFIG = {"a18": "h2w3_body31_hand44_h16.py", "a17": "h2w3_body31_hand44_h16.py", "a14": "h2w3_body31_hand44_h16.py",
|
| 50 |
+
"a18c2": "h2_body31_hand44_h16.py", "a17c2": "h2_body31_hand44_h16.py",
|
| 51 |
+
"a14v2": "h2_body31_hand44_planner7_h16.py"} # 2026-11-14 H2 guide: action82 incl. the planner command
|
| 52 |
+
# model-input cameras per H2 task (user 2026-11-12 22:0x: EVERY H2 task uses head + chest + wrist; third_person never)
|
| 53 |
+
CAMS = {"a18": ("head", "chest", "wrist"), "a17": ("head", "chest", "wrist"), "a14": ("head", "chest", "wrist"),
|
| 54 |
+
"a18c2": ("head", "chest"), "a17c2": ("head", "chest"), "a14v2": ("head", "chest")} # 2026-11-14: 2-camera re-train (user)
|
| 55 |
+
PLANNER = {"a14v2"} # tasks whose action adds action.planner (7) after body_target + hands (H2 guide a14 2.2)
|
| 56 |
H2_GROUPS = {"left_leg": (0, 6), "right_leg": (6, 12), "waist": (12, 15), "left_arm": (15, 22), "right_arm": (22, 29),
|
| 57 |
"head": (29, 31), "left_hand": (31, 53), "right_hand": (53, 75)}
|
| 58 |
|
| 59 |
|
| 60 |
+
def h2_modality(info, cams=("head", "chest")):
|
| 61 |
f = info["features"]
|
| 62 |
names = f["observation.state"]["names"]
|
| 63 |
assert len(names) == 75 and names[:31] == f["action.body_target"]["names"] and names[31:] == f["action.hands"]["names"]
|
|
|
|
| 68 |
"action": {"body_target": {"original_key": "action.body_target", "start": 0, "end": 31},
|
| 69 |
"left_hand_target": {"original_key": "action.hands", "start": 0, "end": 22},
|
| 70 |
"right_hand_target": {"original_key": "action.hands", "start": 22, "end": 44}},
|
| 71 |
+
"video": {c: {"original_key": f"observation.images.{c}"} for c in cams},
|
|
|
|
| 72 |
"annotation": {"human.task_description": {"original_key": "task_index"}}}
|
| 73 |
BODY = ["left_hip_pitch_joint", "left_hip_roll_joint", "left_hip_yaw_joint", "left_knee_joint", "left_ankle_pitch_joint",
|
| 74 |
"left_ankle_roll_joint", "right_hip_pitch_joint", "right_hip_roll_joint", "right_hip_yaw_joint",
|
|
|
|
| 129 |
# stats.py computes stats for every low-dim feature in info.json and cannot handle e.g. observation.tactile
|
| 130 |
# [10,24,24]; keep only what the H2 modality uses (parquet columns stay untouched behind the symlinks).
|
| 131 |
keep = {"observation.state", "observation.projected_gravity", "action.body_target", "action.hands",
|
| 132 |
+
"timestamp", "frame_index", "episode_index", "index", "task_index"}
|
| 133 |
+
keep |= {f"observation.images.{c}" for c in CAMS[task]}
|
| 134 |
+
keep |= {"action.planner"} if task in PLANNER else set()
|
| 135 |
info["features"] = {k: v for k, v in info["features"].items() if k in keep}
|
| 136 |
vkeys = [k for k in vkeys if k in keep]
|
| 137 |
stats = [{**s, "stats": {k: v for k, v in s["stats"].items() if k in keep}} if "stats" in s else s
|
|
|
|
| 143 |
wl(dst / "meta/episodes_stats.jsonl", stats)
|
| 144 |
(dst / "meta/tasks.jsonl").write_text((src / "meta/tasks.jsonl").read_text())
|
| 145 |
if task in CONFIG:
|
| 146 |
+
modality = h2_modality(info, CAMS[task])
|
| 147 |
+
if task in PLANNER:
|
| 148 |
+
assert info["features"]["action.planner"]["shape"] == [7]
|
| 149 |
+
modality["action"]["planner"] = {"original_key": "action.planner", "start": 0, "end": 7}
|
| 150 |
else:
|
| 151 |
state = {g: {"original_key": "observation.state", "start": idx[a], "end": idx[a] + (b - a)}
|
| 152 |
for g, (a, b) in GROUPS.items()}
|
code/hb_groot/verify_groot_cpu.py
CHANGED
|
@@ -16,9 +16,15 @@ from gr00t.data.dataset.lerobot_episode_loader import LeRobotEpisodeLoader # no
|
|
| 16 |
from gr00t.model.gr00t_n1d7.processing_gr00t_n1d7 import Gr00tN1d7Processor # noqa: E402
|
| 17 |
|
| 18 |
view = Path(sys.argv[1])
|
| 19 |
-
if view.name == "
|
|
|
|
|
|
|
|
|
|
| 20 |
import h2_body31_hand44_h16 as C # noqa: E402
|
| 21 |
-
cfg = C.
|
|
|
|
|
|
|
|
|
|
| 22 |
else:
|
| 23 |
import g1_feedback76_h16 as C # noqa: E402
|
| 24 |
cfg = C.feedback76_h16
|
|
|
|
| 16 |
from gr00t.model.gr00t_n1d7.processing_gr00t_n1d7 import Gr00tN1d7Processor # noqa: E402
|
| 17 |
|
| 18 |
view = Path(sys.argv[1])
|
| 19 |
+
if view.name == "a14v2": # 2026-11-14 H2 guide: head + chest, action82 = body_target + hands + planner
|
| 20 |
+
import h2_body31_hand44_planner7_h16 as C # noqa: E402
|
| 21 |
+
cfg = C.h2_body31_hand44_planner7_h16
|
| 22 |
+
elif view.name in ("a18c2", "a17c2"): # 2026-11-14: H2 2-camera re-train (head + chest)
|
| 23 |
import h2_body31_hand44_h16 as C # noqa: E402
|
| 24 |
+
cfg = next(v for k, v in vars(C).items() if isinstance(v, dict) and "video" in v)
|
| 25 |
+
elif view.name in ("a14", "a18", "a17"): # every H2 task: 3 cameras head + chest + wrist (user 2026-11-12)
|
| 26 |
+
import h2w3_body31_hand44_h16 as C # noqa: E402
|
| 27 |
+
cfg = C.h2w3_body31_hand44_h16
|
| 28 |
else:
|
| 29 |
import g1_feedback76_h16 as C # noqa: E402
|
| 30 |
cfg = C.feedback76_h16
|