Rye-Catcher's picture
Add A18 and A17 FM-DiT batch-512 step-4000 checkpoints with resume state
b935b85 verified
Raw History Blame Contribute Delete
14.5 kB
"""Training data of the baselines: a task's LeRobot v2.1 expert dataset read through a robot contract.
A contract names the cameras and which dataset columns form the policy state and the action rows, the same layouts
the registered GR00T / Psi0 checkpoints were trained on:
- `g1_sonic_dfx6` (G1 + INSPIRE, the evaluation/README.md interface; ego_view, chest): state44 =
`observation.state[0:29]` body, measured `observation.inspire_{left,right}_hand_state`,
`observation.projected_gravity`; action76 (`sonic_dfx6`) = `action.motion_token` 64 and the measured DFX6 of the
same row (row 0 = the current opening);
- `h2_sharpa` (H2 + Sharpa, debug/newton/h2_sharpa_policy_eval.py; head, chest): state78 = `observation.state` 75,
`observation.projected_gravity` 3; action75 = `action.body_target` 31, `action.hands` 44 (absolute targets);
- `h2_sharpa_planner` (H2 + Sharpa a14 v2, h2_policy_planner_eval.py; head, chest): state78; action82 = action75 +
`action.planner` 7.
Frames: every camera's video is decoded once (PyAV), each frame resized with `resize_frame` (OpenCV INTER_AREA, the
same function the policy runtime applies to live frames) and stored as a uint8 memory map in a cache directory keyed by
the dataset, episodes, camera and size. A training sample is the frame t of an episode: the cameras and the state at t,
and the action rows t..t+horizon-1 (rows past the episode's end repeat its last row, as DextraVerse's windows).
"""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import json
import os
from pathlib import Path
import sys
import numpy as np
CACHE_VERSION = 1
@dataclass(frozen=True)
class Contract:
name: str
robot: str
cameras: tuple
state: tuple # ((column, start, end), ...) concatenated in order
action: tuple
action_space: str | None = None # the G1 evaluation interface's action space, if any
body_dims: int = 0 # leading raw-radian body joint angles of the state (augmentation spec v2 perturbs these)
@property
def state_dim(self) -> int:
return sum(end - start for _, start, end in self.state)
@property
def action_dim(self) -> int:
return sum(end - start for _, start, end in self.action)
def to_dict(self, cameras=None) -> dict:
return dict(name=self.name, robot=self.robot, cameras=list(cameras or self.cameras),
state=[list(s) for s in self.state], action=[list(a) for a in self.action],
state_dim=self.state_dim, action_dim=self.action_dim, action_space=self.action_space,
body_dims=self.body_dims)
_G1_HANDS = (('observation.inspire_left_hand_state', 0, 6), ('observation.inspire_right_hand_state', 0, 6))
_H2_STATE = (('observation.state', 0, 75), ('observation.projected_gravity', 0, 3))
_H2_ACTION = (('action.body_target', 0, 31), ('action.hands', 0, 44))
# body_dims: G1 observation.state[0:29] are the 29 body joints; H2 observation.state[0:31] are legs 6+6, waist 3,
# head 2, arms 7+7 (docs/unitree_h2_sharpa.md), then the Sharpa hands.
CONTRACTS = {c.name: c for c in (
Contract('g1_sonic_dfx6', 'g1', ('ego_view', 'chest'),
(('observation.state', 0, 29), *_G1_HANDS, ('observation.projected_gravity', 0, 3)),
(('action.motion_token', 0, 64), *_G1_HANDS), action_space='sonic_dfx6', body_dims=29),
Contract('h2_sharpa', 'h2_sharpa', ('head', 'chest'), _H2_STATE, _H2_ACTION, body_dims=31),
Contract('h2_sharpa_planner', 'h2_sharpa', ('head', 'chest'), _H2_STATE,
(*_H2_ACTION, ('action.planner', 0, 7)), body_dims=31),
)}
def augmentation_module():
"""evaluation/augmentation: the augmentation spec v2 shared by every model's training (imported on demand)."""
evaluation = str(Path(__file__).resolve().parents[1])
if evaluation not in sys.path:
sys.path.insert(0, evaluation)
import augmentation
return augmentation
def resize_frame(frame: np.ndarray, hw) -> np.ndarray:
"""(H, W, 3) uint8 RGB -> (h, w, 3) uint8, OpenCV INTER_AREA (training cache and live frames alike)."""
import cv2
frame = np.ascontiguousarray(frame, dtype=np.uint8)
if frame.ndim != 3 or frame.shape[2] != 3:
raise ValueError(f'expected an (H, W, 3) RGB frame, got {frame.shape}')
h, w = int(hw[0]), int(hw[1])
if frame.shape[:2] == (h, w):
return frame
return cv2.resize(frame, (w, h), interpolation=cv2.INTER_AREA)
class LeRobotDataset:
"""A LeRobot v2.1 dataset directory (meta/info.json, meta/episodes.jsonl, data/, videos/)."""
def __init__(self, root):
self.root = Path(root).expanduser().resolve()
self.info = json.loads((self.root / 'meta/info.json').read_text())
if self.info.get('codebase_version') != 'v2.1':
raise ValueError(f"{self.root}: LeRobot {self.info.get('codebase_version')}, expected v2.1")
rows = [json.loads(line) for line in (self.root / 'meta/episodes.jsonl').read_text().splitlines() if line]
self.lengths = {int(r['episode_index']): int(r['length']) for r in rows}
tasks = [json.loads(line) for line in (self.root / 'meta/tasks.jsonl').read_text().splitlines() if line]
self.tasks = {int(t['task_index']): t['task'] for t in tasks}
self.fps = float(self.info['fps'])
@property
def name(self) -> str:
"""The dataset's folder name (its parent's for an export folder named `lerobot`)."""
return self.root.parent.name if self.root.name == 'lerobot' else self.root.name
def chunk(self, episode: int) -> int:
return episode // int(self.info['chunks_size'])
def parquet(self, episode: int) -> Path:
return self.root / self.info['data_path'].format(episode_chunk=self.chunk(episode), episode_index=episode)
def video(self, episode: int, camera: str) -> Path:
return self.root / self.info['video_path'].format(episode_chunk=self.chunk(episode), episode_index=episode,
video_key=f'observation.images.{camera}')
def check_contract(self, contract: Contract, cameras):
features = self.info['features']
for column, start, end in (*contract.state, *contract.action):
if column not in features:
raise KeyError(f'{self.root.name}: no column {column!r} (contract {contract.name})')
width = int(np.prod(features[column]['shape']))
if end > width:
raise ValueError(f'{column} has {width} values; the contract reads {start}:{end}')
for camera in cameras:
key = f'observation.images.{camera}'
if features.get(key, {}).get('dtype') != 'video':
raise KeyError(f'{self.root.name}: no video {key!r}')
def parse_episodes(spec: str | None, available) -> list:
"""'0:100', '0,3,7' or None (all), restricted to the dataset's episodes, sorted."""
available = sorted(available)
if not spec:
return available
chosen = set()
for part in spec.split(','):
if ':' in part:
lo, hi = part.split(':')
chosen.update(range(int(lo or 0), int(hi) if hi else available[-1] + 1))
else:
chosen.add(int(part))
missing = sorted(chosen - set(available))
if missing:
raise ValueError(f'episodes not in the dataset: {missing[:10]}')
return sorted(chosen)
def load_lowdim(dataset: LeRobotDataset, contract: Contract, episodes) -> dict:
"""State (N, state_dim) and action (N, action_dim) float32 of the episodes in order, plus each episode's start
row, length and task text."""
import pyarrow.parquet as pq
columns = sorted({c for c, _, _ in (*contract.state, *contract.action)} | {'frame_index', 'task_index'})
states, actions, starts, lengths, tasks, offset = [], [], [], [], set(), 0
for episode in episodes:
table = pq.read_table(dataset.parquet(episode), columns=columns)
data = {c: np.asarray(table.column(c).to_pylist(), dtype=np.float64) for c in columns}
n = len(data['frame_index'])
if n != dataset.lengths[episode] or not np.array_equal(data['frame_index'], np.arange(n)):
raise ValueError(f'episode {episode}: {n} rows / frame_index differ from episodes.jsonl')
def pick(spec):
return np.concatenate([data[c].reshape(n, -1)[:, s:e] for c, s, e in spec], axis=1).astype(np.float32)
states.append(pick(contract.state))
actions.append(pick(contract.action))
tasks.update(dataset.tasks[int(i)] for i in np.unique(data['task_index']))
starts.append(offset)
lengths.append(n)
offset += n
state, action = np.concatenate(states), np.concatenate(actions)
if not (np.isfinite(state).all() and np.isfinite(action).all()):
raise ValueError('non-finite state or action values')
return dict(state=state, action=action, starts=np.asarray(starts), lengths=np.asarray(lengths),
episodes=list(episodes), tasks=sorted(tasks))
def statistics(lowdim: dict, rows: np.ndarray) -> dict:
out = {}
for key in ('state', 'action'):
x = lowdim[key][rows].astype(np.float64)
out[key] = dict(min=x.min(0).tolist(), max=x.max(0).tolist(), mean=x.mean(0).tolist(), std=x.std(0).tolist())
return out
# -- frame cache --------------------------------------------------------------------------------------------------
def _decode_into(job):
path, out_path, total, start, length, hw = job
import av
frames = np.memmap(out_path, dtype=np.uint8, mode='r+', shape=(total, hw[0], hw[1], 3))
count = 0
with av.open(str(path)) as container:
stream = container.streams.video[0]
stream.thread_type = 'AUTO'
for frame in container.decode(stream):
if count < length:
frames[start + count] = resize_frame(frame.to_ndarray(format='rgb24'), hw)
count += 1
frames.flush()
del frames
if count != length:
raise ValueError(f'{path}: {count} video frames, {length} data rows')
return length
def frame_cache(dataset: LeRobotDataset, lowdim: dict, cameras, hw, cache_root, workers=8) -> dict:
"""{camera: path of the (N, h, w, 3) uint8 memory map} for the episodes of `lowdim`, decoded once and reused."""
from multiprocessing import get_context
total = int(lowdim['lengths'].sum())
paths = {}
for camera in cameras:
key = json.dumps(dict(version=CACHE_VERSION, root=str(dataset.root), episodes=lowdim['episodes'],
lengths=lowdim['lengths'].tolist(), camera=camera, hw=list(hw)), sort_keys=True)
digest = hashlib.sha256(key.encode()).hexdigest()[:16]
folder = Path(cache_root).expanduser() / f'{dataset.name}_{camera}_{hw[0]}x{hw[1]}_{digest}'
folder.mkdir(parents=True, exist_ok=True)
out, meta = folder / 'frames.u8', folder / 'meta.json'
if meta.exists() and json.loads(meta.read_text()).get('complete'):
paths[camera] = out
continue
np.memmap(out, dtype=np.uint8, mode='w+', shape=(total, hw[0], hw[1], 3)).flush()
jobs = [(dataset.video(e, camera), out, total, int(s), int(n), tuple(hw))
for e, s, n in zip(lowdim['episodes'], lowdim['starts'], lowdim['lengths'])]
with get_context('spawn').Pool(max(1, int(workers))) as pool: # the caller may hold torch threads
for done, _ in enumerate(pool.imap_unordered(_decode_into, jobs), 1):
if done % 20 == 0 or done == len(jobs):
print(json.dumps(dict(event='frame_cache', camera=camera, episodes=done, of=len(jobs))),
flush=True)
meta.write_text(json.dumps(dict(json.loads(key), frames=total, complete=True), indent=1))
paths[camera] = out
return paths
class ChunkDataset:
"""torch-style dataset of (images (n_cameras, h, w, 3) uint8, state (state_dim,), action (horizon, action_dim))
over the frames of the selected episodes (positions into lowdim['episodes']). Memory maps open lazily per
process (DataLoader workers).
`body_dims` > 0 turns on the training augmentation (docs/augmentation_v2.md, evaluation/augmentation): per item,
the cached frames of all cameras get one hb_aug_v2.augment call (one time step, no future targets) and the first
`body_dims` state values the per-sample joint bias + noise; the action rows are never touched. 0 (evaluation,
offline checks) returns the cached frames and the recorded state unchanged, deterministically."""
def __init__(self, lowdim: dict, frames: dict, cameras, hw, horizon: int, positions, body_dims: int = 0):
self.lowdim, self.paths, self.cameras, self.hw, self.horizon = lowdim, frames, list(cameras), tuple(hw), horizon
self.positions, self.body_dims = list(positions), int(body_dims)
if self.body_dims:
augmentation_module() # asserts the spec version before any worker starts
self.total = int(lowdim['lengths'].sum())
index, end = [], np.zeros(self.total, np.int64)
for p in positions:
s, n = int(lowdim['starts'][p]), int(lowdim['lengths'][p])
index.append(np.arange(s, s + n))
end[s:s + n] = s + n
self.index, self.end = np.concatenate(index), end
self._maps, self._pid = None, None
def __len__(self):
return len(self.index)
def _frames(self):
if self._maps is None or self._pid != os.getpid():
self._maps = {c: np.memmap(self.paths[c], dtype=np.uint8, mode='r', shape=(self.total, *self.hw, 3))
for c in self.cameras}
self._pid = os.getpid()
return self._maps
def rows(self, i: int) -> np.ndarray:
return np.minimum(np.arange(i, i + self.horizon), self.end[i] - 1)
def __getitem__(self, k):
import torch
i = int(self.index[k])
maps = self._frames()
images = torch.from_numpy(np.stack([maps[c][i] for c in self.cameras]))
state = self.lowdim['state'][i]
if self.body_dims:
augmentation = augmentation_module()
images, _ = augmentation.augment_frames_uint8(images)
state = augmentation.perturb_body(state, self.body_dims)
return images, torch.from_numpy(state), torch.from_numpy(self.lowdim['action'][self.rows(i)])