Download code/hb_baselines_v1/baselines/data.py from humanoid-benchmark/dp: direct link, hf CLI and curl.
- Browser
- Download file 14.5 kB
-
https://huggingface.co/humanoid-benchmark/dp/resolve/main/code/hb_baselines_v1/baselines/data.py
- Command line
-
hf download hf://humanoid-benchmark/dp/code/hb_baselines_v1/baselines/data.py
-
curl -L -o data.py https://huggingface.co/humanoid-benchmark/dp/resolve/main/code/hb_baselines_v1/baselines/data.py
14.5 kB
| """Training data of the baselines: a task's LeRobot v2.1 expert dataset read through a robot contract. | |
| A contract names the cameras and which dataset columns form the policy state and the action rows, the same layouts | |
| the registered GR00T / Psi0 checkpoints were trained on: | |
| - `g1_sonic_dfx6` (G1 + INSPIRE, the evaluation/README.md interface; ego_view, chest): state44 = | |
| `observation.state[0:29]` body, measured `observation.inspire_{left,right}_hand_state`, | |
| `observation.projected_gravity`; action76 (`sonic_dfx6`) = `action.motion_token` 64 and the measured DFX6 of the | |
| same row (row 0 = the current opening); | |
| - `h2_sharpa` (H2 + Sharpa, debug/newton/h2_sharpa_policy_eval.py; head, chest): state78 = `observation.state` 75, | |
| `observation.projected_gravity` 3; action75 = `action.body_target` 31, `action.hands` 44 (absolute targets); | |
| - `h2_sharpa_planner` (H2 + Sharpa a14 v2, h2_policy_planner_eval.py; head, chest): state78; action82 = action75 + | |
| `action.planner` 7. | |
| Frames: every camera's video is decoded once (PyAV), each frame resized with `resize_frame` (OpenCV INTER_AREA, the | |
| same function the policy runtime applies to live frames) and stored as a uint8 memory map in a cache directory keyed by | |
| the dataset, episodes, camera and size. A training sample is the frame t of an episode: the cameras and the state at t, | |
| and the action rows t..t+horizon-1 (rows past the episode's end repeat its last row, as DextraVerse's windows). | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import sys | |
| import numpy as np | |
| CACHE_VERSION = 1 | |
| class Contract: | |
| name: str | |
| robot: str | |
| cameras: tuple | |
| state: tuple # ((column, start, end), ...) concatenated in order | |
| action: tuple | |
| action_space: str | None = None # the G1 evaluation interface's action space, if any | |
| body_dims: int = 0 # leading raw-radian body joint angles of the state (augmentation spec v2 perturbs these) | |
| def state_dim(self) -> int: | |
| return sum(end - start for _, start, end in self.state) | |
| def action_dim(self) -> int: | |
| return sum(end - start for _, start, end in self.action) | |
| def to_dict(self, cameras=None) -> dict: | |
| return dict(name=self.name, robot=self.robot, cameras=list(cameras or self.cameras), | |
| state=[list(s) for s in self.state], action=[list(a) for a in self.action], | |
| state_dim=self.state_dim, action_dim=self.action_dim, action_space=self.action_space, | |
| body_dims=self.body_dims) | |
| _G1_HANDS = (('observation.inspire_left_hand_state', 0, 6), ('observation.inspire_right_hand_state', 0, 6)) | |
| _H2_STATE = (('observation.state', 0, 75), ('observation.projected_gravity', 0, 3)) | |
| _H2_ACTION = (('action.body_target', 0, 31), ('action.hands', 0, 44)) | |
| # body_dims: G1 observation.state[0:29] are the 29 body joints; H2 observation.state[0:31] are legs 6+6, waist 3, | |
| # head 2, arms 7+7 (docs/unitree_h2_sharpa.md), then the Sharpa hands. | |
| CONTRACTS = {c.name: c for c in ( | |
| Contract('g1_sonic_dfx6', 'g1', ('ego_view', 'chest'), | |
| (('observation.state', 0, 29), *_G1_HANDS, ('observation.projected_gravity', 0, 3)), | |
| (('action.motion_token', 0, 64), *_G1_HANDS), action_space='sonic_dfx6', body_dims=29), | |
| Contract('h2_sharpa', 'h2_sharpa', ('head', 'chest'), _H2_STATE, _H2_ACTION, body_dims=31), | |
| Contract('h2_sharpa_planner', 'h2_sharpa', ('head', 'chest'), _H2_STATE, | |
| (*_H2_ACTION, ('action.planner', 0, 7)), body_dims=31), | |
| )} | |
| def augmentation_module(): | |
| """evaluation/augmentation: the augmentation spec v2 shared by every model's training (imported on demand).""" | |
| evaluation = str(Path(__file__).resolve().parents[1]) | |
| if evaluation not in sys.path: | |
| sys.path.insert(0, evaluation) | |
| import augmentation | |
| return augmentation | |
| def resize_frame(frame: np.ndarray, hw) -> np.ndarray: | |
| """(H, W, 3) uint8 RGB -> (h, w, 3) uint8, OpenCV INTER_AREA (training cache and live frames alike).""" | |
| import cv2 | |
| frame = np.ascontiguousarray(frame, dtype=np.uint8) | |
| if frame.ndim != 3 or frame.shape[2] != 3: | |
| raise ValueError(f'expected an (H, W, 3) RGB frame, got {frame.shape}') | |
| h, w = int(hw[0]), int(hw[1]) | |
| if frame.shape[:2] == (h, w): | |
| return frame | |
| return cv2.resize(frame, (w, h), interpolation=cv2.INTER_AREA) | |
| class LeRobotDataset: | |
| """A LeRobot v2.1 dataset directory (meta/info.json, meta/episodes.jsonl, data/, videos/).""" | |
| def __init__(self, root): | |
| self.root = Path(root).expanduser().resolve() | |
| self.info = json.loads((self.root / 'meta/info.json').read_text()) | |
| if self.info.get('codebase_version') != 'v2.1': | |
| raise ValueError(f"{self.root}: LeRobot {self.info.get('codebase_version')}, expected v2.1") | |
| rows = [json.loads(line) for line in (self.root / 'meta/episodes.jsonl').read_text().splitlines() if line] | |
| self.lengths = {int(r['episode_index']): int(r['length']) for r in rows} | |
| tasks = [json.loads(line) for line in (self.root / 'meta/tasks.jsonl').read_text().splitlines() if line] | |
| self.tasks = {int(t['task_index']): t['task'] for t in tasks} | |
| self.fps = float(self.info['fps']) | |
| def name(self) -> str: | |
| """The dataset's folder name (its parent's for an export folder named `lerobot`).""" | |
| return self.root.parent.name if self.root.name == 'lerobot' else self.root.name | |
| def chunk(self, episode: int) -> int: | |
| return episode // int(self.info['chunks_size']) | |
| def parquet(self, episode: int) -> Path: | |
| return self.root / self.info['data_path'].format(episode_chunk=self.chunk(episode), episode_index=episode) | |
| def video(self, episode: int, camera: str) -> Path: | |
| return self.root / self.info['video_path'].format(episode_chunk=self.chunk(episode), episode_index=episode, | |
| video_key=f'observation.images.{camera}') | |
| def check_contract(self, contract: Contract, cameras): | |
| features = self.info['features'] | |
| for column, start, end in (*contract.state, *contract.action): | |
| if column not in features: | |
| raise KeyError(f'{self.root.name}: no column {column!r} (contract {contract.name})') | |
| width = int(np.prod(features[column]['shape'])) | |
| if end > width: | |
| raise ValueError(f'{column} has {width} values; the contract reads {start}:{end}') | |
| for camera in cameras: | |
| key = f'observation.images.{camera}' | |
| if features.get(key, {}).get('dtype') != 'video': | |
| raise KeyError(f'{self.root.name}: no video {key!r}') | |
| def parse_episodes(spec: str | None, available) -> list: | |
| """'0:100', '0,3,7' or None (all), restricted to the dataset's episodes, sorted.""" | |
| available = sorted(available) | |
| if not spec: | |
| return available | |
| chosen = set() | |
| for part in spec.split(','): | |
| if ':' in part: | |
| lo, hi = part.split(':') | |
| chosen.update(range(int(lo or 0), int(hi) if hi else available[-1] + 1)) | |
| else: | |
| chosen.add(int(part)) | |
| missing = sorted(chosen - set(available)) | |
| if missing: | |
| raise ValueError(f'episodes not in the dataset: {missing[:10]}') | |
| return sorted(chosen) | |
| def load_lowdim(dataset: LeRobotDataset, contract: Contract, episodes) -> dict: | |
| """State (N, state_dim) and action (N, action_dim) float32 of the episodes in order, plus each episode's start | |
| row, length and task text.""" | |
| import pyarrow.parquet as pq | |
| columns = sorted({c for c, _, _ in (*contract.state, *contract.action)} | {'frame_index', 'task_index'}) | |
| states, actions, starts, lengths, tasks, offset = [], [], [], [], set(), 0 | |
| for episode in episodes: | |
| table = pq.read_table(dataset.parquet(episode), columns=columns) | |
| data = {c: np.asarray(table.column(c).to_pylist(), dtype=np.float64) for c in columns} | |
| n = len(data['frame_index']) | |
| if n != dataset.lengths[episode] or not np.array_equal(data['frame_index'], np.arange(n)): | |
| raise ValueError(f'episode {episode}: {n} rows / frame_index differ from episodes.jsonl') | |
| def pick(spec): | |
| return np.concatenate([data[c].reshape(n, -1)[:, s:e] for c, s, e in spec], axis=1).astype(np.float32) | |
| states.append(pick(contract.state)) | |
| actions.append(pick(contract.action)) | |
| tasks.update(dataset.tasks[int(i)] for i in np.unique(data['task_index'])) | |
| starts.append(offset) | |
| lengths.append(n) | |
| offset += n | |
| state, action = np.concatenate(states), np.concatenate(actions) | |
| if not (np.isfinite(state).all() and np.isfinite(action).all()): | |
| raise ValueError('non-finite state or action values') | |
| return dict(state=state, action=action, starts=np.asarray(starts), lengths=np.asarray(lengths), | |
| episodes=list(episodes), tasks=sorted(tasks)) | |
| def statistics(lowdim: dict, rows: np.ndarray) -> dict: | |
| out = {} | |
| for key in ('state', 'action'): | |
| x = lowdim[key][rows].astype(np.float64) | |
| out[key] = dict(min=x.min(0).tolist(), max=x.max(0).tolist(), mean=x.mean(0).tolist(), std=x.std(0).tolist()) | |
| return out | |
| # -- frame cache -------------------------------------------------------------------------------------------------- | |
| def _decode_into(job): | |
| path, out_path, total, start, length, hw = job | |
| import av | |
| frames = np.memmap(out_path, dtype=np.uint8, mode='r+', shape=(total, hw[0], hw[1], 3)) | |
| count = 0 | |
| with av.open(str(path)) as container: | |
| stream = container.streams.video[0] | |
| stream.thread_type = 'AUTO' | |
| for frame in container.decode(stream): | |
| if count < length: | |
| frames[start + count] = resize_frame(frame.to_ndarray(format='rgb24'), hw) | |
| count += 1 | |
| frames.flush() | |
| del frames | |
| if count != length: | |
| raise ValueError(f'{path}: {count} video frames, {length} data rows') | |
| return length | |
| def frame_cache(dataset: LeRobotDataset, lowdim: dict, cameras, hw, cache_root, workers=8) -> dict: | |
| """{camera: path of the (N, h, w, 3) uint8 memory map} for the episodes of `lowdim`, decoded once and reused.""" | |
| from multiprocessing import get_context | |
| total = int(lowdim['lengths'].sum()) | |
| paths = {} | |
| for camera in cameras: | |
| key = json.dumps(dict(version=CACHE_VERSION, root=str(dataset.root), episodes=lowdim['episodes'], | |
| lengths=lowdim['lengths'].tolist(), camera=camera, hw=list(hw)), sort_keys=True) | |
| digest = hashlib.sha256(key.encode()).hexdigest()[:16] | |
| folder = Path(cache_root).expanduser() / f'{dataset.name}_{camera}_{hw[0]}x{hw[1]}_{digest}' | |
| folder.mkdir(parents=True, exist_ok=True) | |
| out, meta = folder / 'frames.u8', folder / 'meta.json' | |
| if meta.exists() and json.loads(meta.read_text()).get('complete'): | |
| paths[camera] = out | |
| continue | |
| np.memmap(out, dtype=np.uint8, mode='w+', shape=(total, hw[0], hw[1], 3)).flush() | |
| jobs = [(dataset.video(e, camera), out, total, int(s), int(n), tuple(hw)) | |
| for e, s, n in zip(lowdim['episodes'], lowdim['starts'], lowdim['lengths'])] | |
| with get_context('spawn').Pool(max(1, int(workers))) as pool: # the caller may hold torch threads | |
| for done, _ in enumerate(pool.imap_unordered(_decode_into, jobs), 1): | |
| if done % 20 == 0 or done == len(jobs): | |
| print(json.dumps(dict(event='frame_cache', camera=camera, episodes=done, of=len(jobs))), | |
| flush=True) | |
| meta.write_text(json.dumps(dict(json.loads(key), frames=total, complete=True), indent=1)) | |
| paths[camera] = out | |
| return paths | |
| class ChunkDataset: | |
| """torch-style dataset of (images (n_cameras, h, w, 3) uint8, state (state_dim,), action (horizon, action_dim)) | |
| over the frames of the selected episodes (positions into lowdim['episodes']). Memory maps open lazily per | |
| process (DataLoader workers). | |
| `body_dims` > 0 turns on the training augmentation (docs/augmentation_v2.md, evaluation/augmentation): per item, | |
| the cached frames of all cameras get one hb_aug_v2.augment call (one time step, no future targets) and the first | |
| `body_dims` state values the per-sample joint bias + noise; the action rows are never touched. 0 (evaluation, | |
| offline checks) returns the cached frames and the recorded state unchanged, deterministically.""" | |
| def __init__(self, lowdim: dict, frames: dict, cameras, hw, horizon: int, positions, body_dims: int = 0): | |
| self.lowdim, self.paths, self.cameras, self.hw, self.horizon = lowdim, frames, list(cameras), tuple(hw), horizon | |
| self.positions, self.body_dims = list(positions), int(body_dims) | |
| if self.body_dims: | |
| augmentation_module() # asserts the spec version before any worker starts | |
| self.total = int(lowdim['lengths'].sum()) | |
| index, end = [], np.zeros(self.total, np.int64) | |
| for p in positions: | |
| s, n = int(lowdim['starts'][p]), int(lowdim['lengths'][p]) | |
| index.append(np.arange(s, s + n)) | |
| end[s:s + n] = s + n | |
| self.index, self.end = np.concatenate(index), end | |
| self._maps, self._pid = None, None | |
| def __len__(self): | |
| return len(self.index) | |
| def _frames(self): | |
| if self._maps is None or self._pid != os.getpid(): | |
| self._maps = {c: np.memmap(self.paths[c], dtype=np.uint8, mode='r', shape=(self.total, *self.hw, 3)) | |
| for c in self.cameras} | |
| self._pid = os.getpid() | |
| return self._maps | |
| def rows(self, i: int) -> np.ndarray: | |
| return np.minimum(np.arange(i, i + self.horizon), self.end[i] - 1) | |
| def __getitem__(self, k): | |
| import torch | |
| i = int(self.index[k]) | |
| maps = self._frames() | |
| images = torch.from_numpy(np.stack([maps[c][i] for c in self.cameras])) | |
| state = self.lowdim['state'][i] | |
| if self.body_dims: | |
| augmentation = augmentation_module() | |
| images, _ = augmentation.augment_frames_uint8(images) | |
| state = augmentation.perturb_body(state, self.body_dims) | |
| return images, torch.from_numpy(state), torch.from_numpy(self.lowdim['action'][self.rows(i)]) | |