"""Training data of the baselines: a task's LeRobot v2.1 expert dataset read through a robot contract. A contract names the cameras and which dataset columns form the policy state and the action rows, the same layouts the registered GR00T / Psi0 checkpoints were trained on: - `g1_sonic_dfx6` (G1 + INSPIRE, the evaluation/README.md interface; ego_view, chest): state44 = `observation.state[0:29]` body, measured `observation.inspire_{left,right}_hand_state`, `observation.projected_gravity`; action76 (`sonic_dfx6`) = `action.motion_token` 64 and the measured DFX6 of the same row (row 0 = the current opening); - `h2_sharpa` (H2 + Sharpa, debug/newton/h2_sharpa_policy_eval.py; head, chest): state78 = `observation.state` 75, `observation.projected_gravity` 3; action75 = `action.body_target` 31, `action.hands` 44 (absolute targets); - `h2_sharpa_planner` (H2 + Sharpa a14 v2, h2_policy_planner_eval.py; head, chest): state78; action82 = action75 + `action.planner` 7. Frames: every camera's video is decoded once (PyAV), each frame resized with `resize_frame` (OpenCV INTER_AREA, the same function the policy runtime applies to live frames) and stored as a uint8 memory map in a cache directory keyed by the dataset, episodes, camera and size. A training sample is the frame t of an episode: the cameras and the state at t, and the action rows t..t+horizon-1 (rows past the episode's end repeat its last row, as DextraVerse's windows). """ from __future__ import annotations from dataclasses import dataclass import hashlib import json import os from pathlib import Path import sys import numpy as np CACHE_VERSION = 1 @dataclass(frozen=True) class Contract: name: str robot: str cameras: tuple state: tuple # ((column, start, end), ...) concatenated in order action: tuple action_space: str | None = None # the G1 evaluation interface's action space, if any body_dims: int = 0 # leading raw-radian body joint angles of the state (augmentation spec v2 perturbs these) @property def state_dim(self) -> int: return sum(end - start for _, start, end in self.state) @property def action_dim(self) -> int: return sum(end - start for _, start, end in self.action) def to_dict(self, cameras=None) -> dict: return dict(name=self.name, robot=self.robot, cameras=list(cameras or self.cameras), state=[list(s) for s in self.state], action=[list(a) for a in self.action], state_dim=self.state_dim, action_dim=self.action_dim, action_space=self.action_space, body_dims=self.body_dims) _G1_HANDS = (('observation.inspire_left_hand_state', 0, 6), ('observation.inspire_right_hand_state', 0, 6)) _H2_STATE = (('observation.state', 0, 75), ('observation.projected_gravity', 0, 3)) _H2_ACTION = (('action.body_target', 0, 31), ('action.hands', 0, 44)) # body_dims: G1 observation.state[0:29] are the 29 body joints; H2 observation.state[0:31] are legs 6+6, waist 3, # head 2, arms 7+7 (docs/unitree_h2_sharpa.md), then the Sharpa hands. CONTRACTS = {c.name: c for c in ( Contract('g1_sonic_dfx6', 'g1', ('ego_view', 'chest'), (('observation.state', 0, 29), *_G1_HANDS, ('observation.projected_gravity', 0, 3)), (('action.motion_token', 0, 64), *_G1_HANDS), action_space='sonic_dfx6', body_dims=29), Contract('h2_sharpa', 'h2_sharpa', ('head', 'chest'), _H2_STATE, _H2_ACTION, body_dims=31), Contract('h2_sharpa_planner', 'h2_sharpa', ('head', 'chest'), _H2_STATE, (*_H2_ACTION, ('action.planner', 0, 7)), body_dims=31), )} def augmentation_module(): """evaluation/augmentation: the augmentation spec v2 shared by every model's training (imported on demand).""" evaluation = str(Path(__file__).resolve().parents[1]) if evaluation not in sys.path: sys.path.insert(0, evaluation) import augmentation return augmentation def resize_frame(frame: np.ndarray, hw) -> np.ndarray: """(H, W, 3) uint8 RGB -> (h, w, 3) uint8, OpenCV INTER_AREA (training cache and live frames alike).""" import cv2 frame = np.ascontiguousarray(frame, dtype=np.uint8) if frame.ndim != 3 or frame.shape[2] != 3: raise ValueError(f'expected an (H, W, 3) RGB frame, got {frame.shape}') h, w = int(hw[0]), int(hw[1]) if frame.shape[:2] == (h, w): return frame return cv2.resize(frame, (w, h), interpolation=cv2.INTER_AREA) class LeRobotDataset: """A LeRobot v2.1 dataset directory (meta/info.json, meta/episodes.jsonl, data/, videos/).""" def __init__(self, root): self.root = Path(root).expanduser().resolve() self.info = json.loads((self.root / 'meta/info.json').read_text()) if self.info.get('codebase_version') != 'v2.1': raise ValueError(f"{self.root}: LeRobot {self.info.get('codebase_version')}, expected v2.1") rows = [json.loads(line) for line in (self.root / 'meta/episodes.jsonl').read_text().splitlines() if line] self.lengths = {int(r['episode_index']): int(r['length']) for r in rows} tasks = [json.loads(line) for line in (self.root / 'meta/tasks.jsonl').read_text().splitlines() if line] self.tasks = {int(t['task_index']): t['task'] for t in tasks} self.fps = float(self.info['fps']) @property def name(self) -> str: """The dataset's folder name (its parent's for an export folder named `lerobot`).""" return self.root.parent.name if self.root.name == 'lerobot' else self.root.name def chunk(self, episode: int) -> int: return episode // int(self.info['chunks_size']) def parquet(self, episode: int) -> Path: return self.root / self.info['data_path'].format(episode_chunk=self.chunk(episode), episode_index=episode) def video(self, episode: int, camera: str) -> Path: return self.root / self.info['video_path'].format(episode_chunk=self.chunk(episode), episode_index=episode, video_key=f'observation.images.{camera}') def check_contract(self, contract: Contract, cameras): features = self.info['features'] for column, start, end in (*contract.state, *contract.action): if column not in features: raise KeyError(f'{self.root.name}: no column {column!r} (contract {contract.name})') width = int(np.prod(features[column]['shape'])) if end > width: raise ValueError(f'{column} has {width} values; the contract reads {start}:{end}') for camera in cameras: key = f'observation.images.{camera}' if features.get(key, {}).get('dtype') != 'video': raise KeyError(f'{self.root.name}: no video {key!r}') def parse_episodes(spec: str | None, available) -> list: """'0:100', '0,3,7' or None (all), restricted to the dataset's episodes, sorted.""" available = sorted(available) if not spec: return available chosen = set() for part in spec.split(','): if ':' in part: lo, hi = part.split(':') chosen.update(range(int(lo or 0), int(hi) if hi else available[-1] + 1)) else: chosen.add(int(part)) missing = sorted(chosen - set(available)) if missing: raise ValueError(f'episodes not in the dataset: {missing[:10]}') return sorted(chosen) def load_lowdim(dataset: LeRobotDataset, contract: Contract, episodes) -> dict: """State (N, state_dim) and action (N, action_dim) float32 of the episodes in order, plus each episode's start row, length and task text.""" import pyarrow.parquet as pq columns = sorted({c for c, _, _ in (*contract.state, *contract.action)} | {'frame_index', 'task_index'}) states, actions, starts, lengths, tasks, offset = [], [], [], [], set(), 0 for episode in episodes: table = pq.read_table(dataset.parquet(episode), columns=columns) data = {c: np.asarray(table.column(c).to_pylist(), dtype=np.float64) for c in columns} n = len(data['frame_index']) if n != dataset.lengths[episode] or not np.array_equal(data['frame_index'], np.arange(n)): raise ValueError(f'episode {episode}: {n} rows / frame_index differ from episodes.jsonl') def pick(spec): return np.concatenate([data[c].reshape(n, -1)[:, s:e] for c, s, e in spec], axis=1).astype(np.float32) states.append(pick(contract.state)) actions.append(pick(contract.action)) tasks.update(dataset.tasks[int(i)] for i in np.unique(data['task_index'])) starts.append(offset) lengths.append(n) offset += n state, action = np.concatenate(states), np.concatenate(actions) if not (np.isfinite(state).all() and np.isfinite(action).all()): raise ValueError('non-finite state or action values') return dict(state=state, action=action, starts=np.asarray(starts), lengths=np.asarray(lengths), episodes=list(episodes), tasks=sorted(tasks)) def statistics(lowdim: dict, rows: np.ndarray) -> dict: out = {} for key in ('state', 'action'): x = lowdim[key][rows].astype(np.float64) out[key] = dict(min=x.min(0).tolist(), max=x.max(0).tolist(), mean=x.mean(0).tolist(), std=x.std(0).tolist()) return out # -- frame cache -------------------------------------------------------------------------------------------------- def _decode_into(job): path, out_path, total, start, length, hw = job import av frames = np.memmap(out_path, dtype=np.uint8, mode='r+', shape=(total, hw[0], hw[1], 3)) count = 0 with av.open(str(path)) as container: stream = container.streams.video[0] stream.thread_type = 'AUTO' for frame in container.decode(stream): if count < length: frames[start + count] = resize_frame(frame.to_ndarray(format='rgb24'), hw) count += 1 frames.flush() del frames if count != length: raise ValueError(f'{path}: {count} video frames, {length} data rows') return length def frame_cache(dataset: LeRobotDataset, lowdim: dict, cameras, hw, cache_root, workers=8) -> dict: """{camera: path of the (N, h, w, 3) uint8 memory map} for the episodes of `lowdim`, decoded once and reused.""" from multiprocessing import get_context total = int(lowdim['lengths'].sum()) paths = {} for camera in cameras: key = json.dumps(dict(version=CACHE_VERSION, root=str(dataset.root), episodes=lowdim['episodes'], lengths=lowdim['lengths'].tolist(), camera=camera, hw=list(hw)), sort_keys=True) digest = hashlib.sha256(key.encode()).hexdigest()[:16] folder = Path(cache_root).expanduser() / f'{dataset.name}_{camera}_{hw[0]}x{hw[1]}_{digest}' folder.mkdir(parents=True, exist_ok=True) out, meta = folder / 'frames.u8', folder / 'meta.json' if meta.exists() and json.loads(meta.read_text()).get('complete'): paths[camera] = out continue np.memmap(out, dtype=np.uint8, mode='w+', shape=(total, hw[0], hw[1], 3)).flush() jobs = [(dataset.video(e, camera), out, total, int(s), int(n), tuple(hw)) for e, s, n in zip(lowdim['episodes'], lowdim['starts'], lowdim['lengths'])] with get_context('spawn').Pool(max(1, int(workers))) as pool: # the caller may hold torch threads for done, _ in enumerate(pool.imap_unordered(_decode_into, jobs), 1): if done % 20 == 0 or done == len(jobs): print(json.dumps(dict(event='frame_cache', camera=camera, episodes=done, of=len(jobs))), flush=True) meta.write_text(json.dumps(dict(json.loads(key), frames=total, complete=True), indent=1)) paths[camera] = out return paths class ChunkDataset: """torch-style dataset of (images (n_cameras, h, w, 3) uint8, state (state_dim,), action (horizon, action_dim)) over the frames of the selected episodes (positions into lowdim['episodes']). Memory maps open lazily per process (DataLoader workers). `body_dims` > 0 turns on the training augmentation (docs/augmentation_v2.md, evaluation/augmentation): per item, the cached frames of all cameras get one hb_aug_v2.augment call (one time step, no future targets) and the first `body_dims` state values the per-sample joint bias + noise; the action rows are never touched. 0 (evaluation, offline checks) returns the cached frames and the recorded state unchanged, deterministically.""" def __init__(self, lowdim: dict, frames: dict, cameras, hw, horizon: int, positions, body_dims: int = 0): self.lowdim, self.paths, self.cameras, self.hw, self.horizon = lowdim, frames, list(cameras), tuple(hw), horizon self.positions, self.body_dims = list(positions), int(body_dims) if self.body_dims: augmentation_module() # asserts the spec version before any worker starts self.total = int(lowdim['lengths'].sum()) index, end = [], np.zeros(self.total, np.int64) for p in positions: s, n = int(lowdim['starts'][p]), int(lowdim['lengths'][p]) index.append(np.arange(s, s + n)) end[s:s + n] = s + n self.index, self.end = np.concatenate(index), end self._maps, self._pid = None, None def __len__(self): return len(self.index) def _frames(self): if self._maps is None or self._pid != os.getpid(): self._maps = {c: np.memmap(self.paths[c], dtype=np.uint8, mode='r', shape=(self.total, *self.hw, 3)) for c in self.cameras} self._pid = os.getpid() return self._maps def rows(self, i: int) -> np.ndarray: return np.minimum(np.arange(i, i + self.horizon), self.end[i] - 1) def __getitem__(self, k): import torch i = int(self.index[k]) maps = self._frames() images = torch.from_numpy(np.stack([maps[c][i] for c in self.cameras])) state = self.lowdim['state'][i] if self.body_dims: augmentation = augmentation_module() images, _ = augmentation.augment_frames_uint8(images) state = augmentation.perturb_body(state, self.body_dims) return images, torch.from_numpy(state), torch.from_numpy(self.lowdim['action'][self.rows(i)])