File size: 6,467 Bytes
ab621ab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """Humanoid Benchmark data augmentation spec v2 (docs/augmentation_v2.md): the training-time entry every model uses.
`hb_aug_v2.py` is an unmodified copy of HumanWAM `hb_training/hb_groot/hb_aug_v2.py` (branch bai, commit af94cde,
git blob 75542085), the implementation GR00T, pi0.5, Psi-Zero, ACT, Delta Flow and DiT4DiT train with. Do not edit
it here; a new version of the spec is a new file with a new VERSION.
How a training entry uses it (the FM-DiT / DP / ACT baselines in evaluation/baselines do exactly this):
- Visual, once per training sample: after the model's native deterministic resize / center crop and before its
normalization, stack every camera and every frame of the sample into x float [T, V, 3, H, W] in [0, 1] and call
`augment_images(x, obs_mask)` (obs_mask[t] False marks a future image target: it gets the colour, geometry, light
patch and white-balance effects of its camera but none of the sensor/optics effects). `augment_frames_uint8` does
the same for uint8 (V, H, W, 3) frames. Pixel-size effects scale with H / 224 inside hb_aug_v2.
- Proprio, once per training sample: `perturb_body(state, body_dims)` adds the spec's per-joint bias (U +-2 deg,
drawn per sample, shared by every frame of the sample) and N(0, 0.25 deg) noise clipped at 3 sigma to the leading
`body_dims` raw-radian joint angles (G1 29, H2 31 incl. the head), before normalization. Hands, gravity and every
action label stay bit-identical. No proprio delay, no history drop.
- The model's own image augmentation (random crop / shift, colour jitter, rotation, ...) is off.
- Evaluation, offline checks and rollouts call none of this: same resize, same input layout, same normalization.
- Randomness comes from `worker_rng()`, one numpy Generator per process (DataLoader worker) seeded from its pid and
os.urandom, never from the global np.random (which DataLoader seeds identically on every rank).
The interface state is raw and velocity-free: G1 44 = body 29 + measured hands 12 + projected gravity 3; H2 78 =
75 joints (body 31 incl. head, Sharpa hands 22 + 22) + projected gravity 3 (`STATE_DIMS`, `check_state_layout`).
"""
from __future__ import annotations
import math
import os
import re
import numpy as np
from . import hb_aug_v2
VERSION = 'hb-aug-v2-20261109'
if hb_aug_v2.VERSION != VERSION:
raise ImportError(f'evaluation/augmentation/hb_aug_v2.py is {hb_aug_v2.VERSION}, expected {VERSION}')
BODY_JOINTS = {'g1': 29, 'h2_sharpa': 31} # leading raw-radian body joint angles of the state (H2: incl. head)
STATE_DIMS = {'g1': 44, 'h2_sharpa': 78} # the benchmark state of each robot, no joint velocity
BIAS_DEG, NOISE_DEG = 2.0, 0.25 # = hb_aug_v2.perturb_joints
_RNG = {'pid': None, 'rng': None}
def worker_rng() -> np.random.Generator:
"""This process's generator, re-created after a fork (DataLoader workers) from its pid and os.urandom."""
pid = os.getpid()
if _RNG['pid'] != pid:
_RNG['rng'] = np.random.default_rng([pid, int.from_bytes(os.urandom(8), 'little')])
_RNG['pid'] = pid
return _RNG['rng']
def augment_images(x, obs_mask=None, rng=None):
"""hb_aug_v2.augment: x float [T, V, 3, H, W] in [0, 1], one call per training sample -> (augmented, params)."""
if x.ndim != 5 or x.shape[2] != 3:
raise ValueError(f'expected [T, V, 3, H, W], got {tuple(x.shape)}')
return hb_aug_v2.augment(x, rng if rng is not None else worker_rng(), obs_mask=obs_mask)
def augment_frames_uint8(frames, rng=None):
"""One sample's uint8 frames (V, H, W, 3) (one time step) -> augmented uint8 frames of the same shape (rounded to
the nearest level, as the GR00T and Psi-Zero integrations do), and the parameters."""
import torch
frames = torch.as_tensor(frames)
if frames.dtype != torch.uint8 or frames.ndim != 4 or frames.shape[-1] != 3:
raise ValueError(f'expected uint8 (V, H, W, 3) frames, got {frames.dtype} {tuple(frames.shape)}')
x = frames.permute(0, 3, 1, 2).float().div(255.0)[None] # [1, V, 3, H, W]
y, params = augment_images(x, rng=rng)
out = (y[0] * 255.0 + 0.5).clamp(0, 255).to(torch.uint8).permute(0, 2, 3, 1).contiguous()
return out, params
def perturb_body(state, body_dims: int, rng=None) -> np.ndarray:
"""A float32 copy of `state` (..., D) whose first `body_dims` values got hb_aug_v2.perturb_joints (one bias per
joint for the whole sample, noise per value); the other values are copied bit for bit."""
state = np.array(state, dtype=np.float32, copy=True)
if not 0 < body_dims <= state.shape[-1]:
raise ValueError(f'body_dims {body_dims} outside the state width {state.shape[-1]}')
state[..., :body_dims] = hb_aug_v2.perturb_joints(state[..., :body_dims], rng if rng is not None else worker_rng())
return state
def check_state_layout(robot: str, state_dim: int, columns=()) -> None:
"""The benchmark interface: G1 44 / H2 78 raw state values, no velocity column."""
if robot not in STATE_DIMS:
raise KeyError(f'no augmentation layout for robot {robot!r}; known: {sorted(STATE_DIMS)}')
if state_dim != STATE_DIMS[robot]:
raise ValueError(f'{robot} state must be {STATE_DIMS[robot]}-D (no joint velocity), got {state_dim}')
velocity = [c for c in columns if re.search(r'(velocit|(^|[._])(vel|qvel|dq)([._]|$))', str(c).lower())]
if velocity:
raise ValueError(f'joint velocity is not allowed in the model input: {velocity}')
def max_body_offset_rad() -> float:
"""The largest |perturbation| perturb_body can add to one joint: bias 2 deg + clipped noise 3 x 0.25 deg."""
return math.radians(BIAS_DEG + 3 * NOISE_DEG)
def describe(robot: str, body_dims: int) -> dict:
"""The record a training run stores (run.json / checkpoint config)."""
return dict(spec='docs/augmentation_v2.md', version=VERSION, implementation='evaluation/augmentation/hb_aug_v2.py',
visual=dict(effects=list(hb_aug_v2.EFFECTS), boosted_per_sample=1, boost=1.5,
pixel_scale='H / 224', future_targets='none (one observation frame)'),
proprio=dict(robot=robot, body_dims=int(body_dims), bias_deg=BIAS_DEG, bias_scope='per_sample',
noise_deg=NOISE_DEG, noise_clip_sigma=3, delay_probability=0.0,
history_drop_probability=0.0, velocity=False),
native_image_augmentation='off', evaluation='off')
|