Download code/hb_baselines_v1/baselines/runtime.py from humanoid-benchmark/act: direct link, hf CLI and curl.
- Browser
- Download file 6.64 kB
-
https://huggingface.co/humanoid-benchmark/act/resolve/main/code/hb_baselines_v1/baselines/runtime.py
- Command line
-
hf download hf://humanoid-benchmark/act/code/hb_baselines_v1/baselines/runtime.py
-
curl -L -o runtime.py https://huggingface.co/humanoid-benchmark/act/resolve/main/code/hb_baselines_v1/baselines/runtime.py
6.64 kB
| """Checkpoints and inference of the baselines. | |
| A checkpoint is a directory written by train.py (`<run>/checkpoints/step-NNNNNN/`, `final` links the last one): | |
| - `model.pt`: torch.save of {format, config (the config.json text), state_dict}: the EMA weights (or the raw ones when | |
| EMA is off) with the policy's state and action normalizers, loaded strictly with `weights_only=True`; | |
| - `config.json`: the policy configuration, the contract (robot, cameras, state and action columns), the image size, | |
| the training task text, dataset, episodes, recipe and statistics; | |
| - `protocol.json`: model.pt's size and SHA-256. | |
| `BaselinePolicy.predict` takes the live frames of the checkpoint's cameras (any size; resized with data.resize_frame, | |
| as the training cache), the physical state of the contract and a seed, and returns the physical (horizon, | |
| action_dim) chunk with its normalized form; row 0 is the action of the observation's frame. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| FORMAT = 'hb_baselines/1' | |
| def sha256(path) -> str: | |
| digest = hashlib.sha256() | |
| with Path(path).open('rb') as stream: | |
| for block in iter(lambda: stream.read(16 << 20), b''): | |
| digest.update(block) | |
| return digest.hexdigest() | |
| def save_checkpoint(folder, policy, config: dict): | |
| """Write model.pt, config.json and protocol.json of `policy` (an nn.Module) into `folder`.""" | |
| import torch | |
| folder = Path(folder) | |
| folder.mkdir(parents=True, exist_ok=True) | |
| text = json.dumps(dict(config, format=FORMAT), indent=1) | |
| state = {k: v.detach().cpu().contiguous() for k, v in policy.state_dict().items()} | |
| torch.save(dict(format=FORMAT, config=text, state_dict=state), folder / 'model.pt.tmp') | |
| (folder / 'model.pt.tmp').replace(folder / 'model.pt') | |
| (folder / 'config.json').write_text(text) | |
| (folder / 'protocol.json').write_text(json.dumps(dict( | |
| format=FORMAT, step=config.get('step'), policy=config['policy']['policy'], | |
| files={'model.pt': dict(bytes=(folder / 'model.pt').stat().st_size, sha256=sha256(folder / 'model.pt'))}), | |
| indent=1)) | |
| def load_checkpoint(folder, device='cpu', verify=True): | |
| """(policy in eval mode on `device`, config dict) of a checkpoint directory.""" | |
| import torch | |
| from .policies import build_policy | |
| folder = Path(folder) | |
| if verify: | |
| protocol = json.loads((folder / 'protocol.json').read_text()) | |
| entry = protocol['files']['model.pt'] | |
| if (folder / 'model.pt').stat().st_size != entry['bytes']: | |
| raise ValueError(f'{folder}/model.pt differs from protocol.json') | |
| payload = torch.load(folder / 'model.pt', map_location='cpu', weights_only=True) | |
| if payload.get('format') != FORMAT: | |
| raise ValueError(f"{folder}: format {payload.get('format')!r}, expected {FORMAT}") | |
| config = json.loads(payload['config']) | |
| if json.loads((folder / 'config.json').read_text()) != config: | |
| raise ValueError(f'{folder}: config.json differs from the one inside model.pt') | |
| policy = build_policy(config['policy'], pretrained=False) # every weight comes from model.pt | |
| policy.load_state_dict(payload['state_dict'], strict=True) | |
| return policy.to(device).eval(), config | |
| class BaselinePolicy: | |
| """A loaded checkpoint for closed-loop inference (one request at a time).""" | |
| def __init__(self, checkpoint, device='cuda:0', policy: str | None = None, robot: str | None = None): | |
| import torch | |
| self.torch = torch | |
| self.device = torch.device(device) | |
| self.model, self.config = load_checkpoint(checkpoint, device=self.device) | |
| self.checkpoint = str(Path(checkpoint).resolve()) | |
| name, contract = self.config['policy']['policy'], self.config['contract'] | |
| if policy is not None and name != policy: | |
| raise ValueError(f'{checkpoint} is a {name!r} checkpoint, not {policy!r}') | |
| if robot is not None and contract['robot'] != robot: | |
| raise ValueError(f"{checkpoint} was trained for {contract['robot']!r} ({contract['name']}), not {robot!r}") | |
| self.name, self.contract = name, contract | |
| self.cameras = tuple(contract['cameras']) | |
| self.image_hw = tuple(self.config['image_hw']) | |
| self.horizon = int(self.config['policy']['horizon']) | |
| self.state_dim, self.action_dim = int(contract['state_dim']), int(contract['action_dim']) | |
| self.sampling_steps = int(self.model.sampling_steps) | |
| self.task_text = self.config.get('task_text') | |
| def info(self) -> dict: | |
| return dict(model=self.name, checkpoint=self.checkpoint, step=self.config.get('step'), | |
| contract=self.contract['name'], views=list(self.cameras), image_hw=list(self.image_hw), | |
| action_chunk=self.horizon, action_dim=self.action_dim, state_dim=self.state_dim, | |
| sampling_steps=self.sampling_steps, task_text=self.task_text, | |
| training_data=self.config.get('dataset', {}).get('name')) | |
| def images_tensor(self, images: dict): | |
| from .data import resize_frame | |
| missing = [c for c in self.cameras if c not in images] | |
| if missing: | |
| raise KeyError(f'missing cameras {missing}; the checkpoint reads {list(self.cameras)}') | |
| frames = np.stack([resize_frame(np.asarray(images[c]), self.image_hw) for c in self.cameras]) | |
| x = self.torch.from_numpy(frames).to(self.device).permute(0, 3, 1, 2).float().div_(255.) | |
| return x.unsqueeze(0) # (1, n_cameras, 3, h, w) | |
| def predict(self, images: dict, state, seed: int = 0): | |
| """(action (horizon, action_dim) float32 physical, normalized (horizon, action_dim) float32).""" | |
| torch = self.torch | |
| state = np.asarray(state, np.float32).reshape(-1) | |
| if state.shape != (self.state_dim,) or not np.isfinite(state).all(): | |
| raise ValueError(f'expected a finite state of {self.state_dim} values, got {state.shape}') | |
| generator = torch.Generator(device=self.device) | |
| generator.manual_seed(int(seed) % (2 ** 63 - 1)) | |
| with torch.inference_mode(): | |
| action, raw = self.model.predict(self.images_tensor(images), | |
| torch.from_numpy(state)[None].to(self.device), generator=generator) | |
| action = action[0].float().cpu().numpy() | |
| if action.shape != (self.horizon, self.action_dim) or not np.isfinite(action).all(): | |
| raise ValueError(f'invalid prediction {action.shape}') | |
| return np.ascontiguousarray(action, np.float32), raw[0].float().cpu().numpy().astype(np.float32) | |