Download code/hb_baselines_v1/baselines/train.py from humanoid-benchmark/dp: direct link, hf CLI and curl.
- Browser
- Download file 23.7 kB
-
https://huggingface.co/humanoid-benchmark/dp/resolve/main/code/hb_baselines_v1/baselines/train.py
- Command line
-
hf download hf://humanoid-benchmark/dp/code/hb_baselines_v1/baselines/train.py
-
curl -L -o train.py https://huggingface.co/humanoid-benchmark/dp/resolve/main/code/hb_baselines_v1/baselines/train.py
23.7 kB
| """Train an FM-DiT, DP (DDPM-DiT) or ACT baseline on a task's LeRobot v2.1 expert dataset. | |
| <baselines env>/bin/python -I evaluation/baselines/train.py --policy fm_dit --contract h2_sharpa \\ | |
| --dataset data/h2_pour_water/h2_pour_water_r1/lerobot --out <run dir> --steps 60000 | |
| The recipe is DextraVerse's shared IL recipe (configs/algorithm/il/train.yaml; runners/train_dex_flow_matching.py): | |
| AdamW (lr 1e-4, betas 0.95/0.999, weight decay 1e-6), cosine decay (here after a 500-step linear warm-up, as | |
| Diffusion Policy), whole-policy EMA (Diffusion Policy's warm-up schedule, max decay 0.9999) whose weights are the | |
| saved checkpoint, batch 64, one observation frame and 16-row chunks. Added for the humanoid views: frames resized | |
| to 168x224 (the 480x640 aspect). bf16 autocast on CUDA. | |
| Augmentation (2026-11-16): the Humanoid Benchmark spec v2 (docs/augmentation_v2.md, evaluation/augmentation), the | |
| same for every model of the benchmark, is on by default (--augmentation hb-v2): per training item one hb_aug_v2 call | |
| on the cached 168x224 frames of all cameras (colour shared, geometry / light patches / white balance per camera, one | |
| of the 15 effects boosted 1.5 x) and the per-sample joint bias (+-2 deg) + noise (0.25 deg, clipped 3 sigma) on the | |
| body joint angles of the raw state (G1 29, H2 31), before the policy's normalizers; actions untouched. The native | |
| random shift is off (--shift 0; a shift with hb-v2 is refused). Before the first step a preflight reads training items | |
| through the augmented and the plain path and checks the state width (G1 44 / H2 78, no velocity), that only the body | |
| joints move and by at most 2.75 deg, that the action rows are bit-identical, that the images change, and that the | |
| offline-check items are deterministic; it is stored in run.json (--preflight-only stops after it). --augmentation | |
| none is an explicit exception to the spec, recorded in run.json and the checkpoints. The ResNet18 starts from torchvision's ImageNet-1k weights | |
| (--backbone-weights IMAGENET1K_V1, downloaded once to the torch hub cache) and is fine-tuned at 0.1 x the learning | |
| rate (--backbone-lr-scale, the DETR practice) with its BatchNorm frozen; --backbone-weights none trains it from | |
| scratch (DextraVerse's GroupNorm encoder for FM-DiT and DP). | |
| Writes into --out: run.json (arguments), train_log.jsonl, checkpoints/step-NNNNNN/ every --save-every steps and at the | |
| end (runtime.py format), checkpoints/final -> the last one, and train_state.pt (optimizer, schedule, EMA, step) that | |
| --resume continues from. Each save also measures the EMA policy offline on fixed frames (held-out episodes with | |
| --val-episodes, else training frames): normalized-action MSE overall and per action group. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import copy | |
| from datetime import datetime | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import time | |
| import numpy as np | |
| HERE = Path(__file__).resolve().parent | |
| REPO = HERE.parents[1] | |
| if str(HERE.parent) not in sys.path: | |
| sys.path.insert(0, str(HERE.parent)) | |
| from baselines.data import (CONTRACTS, ChunkDataset, LeRobotDataset, augmentation_module, frame_cache, # noqa: E402 | |
| load_lowdim, parse_episodes, statistics) | |
| from baselines.policies import POLICIES, PolicyConfig, build_policy # noqa: E402 | |
| from baselines.runtime import save_checkpoint # noqa: E402 | |
| DEFAULTS = dict(lr=1e-4, betas=(0.95, 0.999), weight_decay=1e-6, warmup=500, ema=0.9999, batch_size=64, | |
| backbone_weights='IMAGENET1K_V1', backbone_lr_scale=0.1) | |
| def git_commit(): | |
| try: | |
| return subprocess.run(['git', '-C', str(REPO), 'rev-parse', 'HEAD'], capture_output=True, text=True, | |
| check=True).stdout.strip() | |
| except Exception: # noqa: BLE001 - outside a checkout | |
| return None | |
| def parse_hw(text): | |
| h, w = text.lower().split('x') | |
| return int(h), int(w) | |
| def random_shift(images, pad): | |
| """DrQ random shift: replicate-pad by `pad` pixels and crop back at a random offset per sample. | |
| images (B, N, 3, H, W) float.""" | |
| import torch | |
| import torch.nn.functional as F | |
| if pad <= 0: | |
| return images | |
| B, N, C, H, W = images.shape | |
| x = F.pad(images.reshape(B * N, C, H, W), (pad,) * 4, mode='replicate') | |
| oy = torch.randint(0, 2 * pad + 1, (B * N,), device=images.device) | |
| ox = torch.randint(0, 2 * pad + 1, (B * N,), device=images.device) | |
| rows = (oy[:, None] + torch.arange(H, device=images.device)[None])[:, None, :, None].expand(-1, C, -1, W + 2 * pad) | |
| x = x.gather(2, rows) | |
| cols = (ox[:, None] + torch.arange(W, device=images.device)[None])[:, None, None, :].expand(-1, C, H, -1) | |
| return x.gather(3, cols).reshape(B, N, C, H, W) | |
| class EMA: | |
| """Diffusion Policy's EMAModel schedule: decay = min(max_decay, 1 - (1 + step) ** -0.75), parameters averaged, | |
| buffers (normalizers) copied.""" | |
| def __init__(self, model, max_decay): | |
| self.model = copy.deepcopy(model).eval().requires_grad_(False) | |
| self.max_decay = float(max_decay) | |
| def update(self, model, step): | |
| import torch | |
| decay = 0.0 if step <= 0 else min(self.max_decay, max(0.0, 1 - (1 + step) ** -0.75)) | |
| with torch.no_grad(): | |
| for e, p in zip(self.model.parameters(), model.parameters()): | |
| e.mul_(decay).add_(p.detach(), alpha=1 - decay) | |
| for e, b in zip(self.model.buffers(), model.buffers()): | |
| e.copy_(b) | |
| return decay | |
| def preflight(train_set, eval_set, contract, samples=64, seed=0) -> dict: | |
| """Training items of `train_set` (augmented unless its body_dims is 0) against the same frames read without | |
| augmentation, and two reads of the offline-check items; raises on any violation of docs/augmentation_v2.md.""" | |
| import torch | |
| plain = ChunkDataset(train_set.lowdim, train_set.paths, train_set.cameras, train_set.hw, train_set.horizon, | |
| train_set.positions) # the same frames without augmentation | |
| picks = np.random.default_rng(seed).choice(len(plain), size=min(samples, len(plain)), replace=False) | |
| nb, sd = train_set.body_dims, contract.state_dim | |
| limit = augmentation_module().max_body_offset_rad() * (1 + 1e-5) | |
| images_changed = body_moved = 0 | |
| worst = 0.0 | |
| for k in picks: | |
| (im, st, ac), (im0, st0, ac0) = train_set[int(k)], plain[int(k)] | |
| if st.shape != (sd,) or st0.shape != (sd,) or ac.shape != ac0.shape: | |
| raise SystemExit(f'preflight: state {tuple(st.shape)} / action {tuple(ac.shape)}, expected ({sd},)') | |
| if not torch.equal(ac, ac0): | |
| raise SystemExit('preflight: an action label differs from the recorded data') | |
| if not torch.equal(st[nb:], st0[nb:]): | |
| raise SystemExit(f'preflight: a state value past the {nb} body joints changed') | |
| delta = (st[:nb].double() - st0[:nb].double()).abs() | |
| if nb: | |
| worst = max(worst, float(delta.max())) | |
| if worst > limit: | |
| raise SystemExit(f'preflight: a body joint moved {math.degrees(worst):.3f} deg (> 2.75)') | |
| body_moved += int((delta > 0).sum()) | |
| images_changed += int(not torch.equal(im, im0)) | |
| n = len(picks) | |
| moved = body_moved / max(1, n * nb) | |
| if nb and (images_changed < 0.9 * n or moved < 0.95): | |
| raise SystemExit(f'preflight: augmentation inactive ({images_changed}/{n} images changed, {moved:.3f} of the ' | |
| 'body joint values moved)') | |
| if not nb and (images_changed or worst): | |
| raise SystemExit('preflight: items changed with the augmentation off') | |
| eval_picks = np.random.default_rng(seed + 1).choice(len(eval_set), size=min(8, len(eval_set)), replace=False) | |
| for k in eval_picks: | |
| a, b = eval_set[int(k)], eval_set[int(k)] | |
| if eval_set.body_dims or not all(torch.equal(x, y) for x, y in zip(a, b)): | |
| raise SystemExit('preflight: the offline-check items are not deterministic') | |
| return dict(samples=n, state_dim=sd, body_dims=nb, images_changed=images_changed, | |
| body_values_moved=round(moved, 4), | |
| max_body_offset_deg=round(math.degrees(worst), 4), actions_bit_identical=True, | |
| non_body_state_bit_identical=True, eval_items_deterministic=len(eval_picks)) | |
| def offline_eval(policy, dataset, samples, device, groups): | |
| """Normalized-action MSE of policy.predict on fixed dataset items (seeded), overall and per action group.""" | |
| import torch | |
| policy.eval() | |
| errs = [] | |
| generator = torch.Generator(device=device) | |
| generator.manual_seed(0) | |
| for start in range(0, len(samples), 32): | |
| batch = [dataset[int(k)] for k in samples[start:start + 32]] | |
| images = torch.stack([b[0] for b in batch]).to(device).permute(0, 1, 4, 2, 3).float().div_(255.) | |
| state = torch.stack([b[1] for b in batch]).to(device) | |
| target = torch.stack([b[2] for b in batch]).to(device) | |
| _, raw = policy.predict(images, state, generator=generator) | |
| errs.append((raw.float() - policy.action_norm.normalize(target)).pow(2).cpu().numpy()) | |
| err = np.concatenate(errs) # (S, horizon, action_dim) | |
| out = dict(mse=float(err.mean()), mse_row0=float(err[:, 0].mean())) | |
| offset = 0 | |
| for column, start, end in groups: | |
| out[f'mse_{column.split(".")[-1]}'] = float(err[..., offset:offset + end - start].mean()) | |
| offset += end - start | |
| return out | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| p.add_argument('--policy', choices=sorted(POLICIES), required=True) | |
| p.add_argument('--dataset', type=Path, required=True, help='LeRobot v2.1 dataset directory') | |
| p.add_argument('--contract', choices=sorted(CONTRACTS), required=True) | |
| p.add_argument('--cameras', help='comma-separated cameras (default: the contract\'s)') | |
| p.add_argument('--episodes', help="episodes to use, e.g. '0:100' (default: all)") | |
| p.add_argument('--val-episodes', type=int, default=0, help='hold out the last N of them for the offline check') | |
| p.add_argument('--out', type=Path, required=True) | |
| p.add_argument('--steps', type=int, default=60000) | |
| p.add_argument('--batch-size', type=int, default=DEFAULTS['batch_size']) | |
| p.add_argument('--lr', type=float, default=DEFAULTS['lr']) | |
| p.add_argument('--weight-decay', type=float, default=DEFAULTS['weight_decay']) | |
| p.add_argument('--warmup', type=int, default=DEFAULTS['warmup']) | |
| p.add_argument('--ema', type=float, default=DEFAULTS['ema'], help='max EMA decay; 0 saves the raw weights') | |
| p.add_argument('--horizon', type=int, default=16) | |
| p.add_argument('--image-size', default='168x224', help='HxW of the policy images') | |
| p.add_argument('--augmentation', choices=['hb-v2', 'none'], default='hb-v2', | |
| help='training augmentation: the benchmark spec v2 (default; docs/augmentation_v2.md) or none ' | |
| '(an explicit exception, recorded in run.json)') | |
| p.add_argument('--shift', type=int, default=0, | |
| help='native random-shift augmentation in pixels (default 0: off; only with --augmentation none)') | |
| p.add_argument('--preflight-only', action='store_true', help='run the data preflight, write run.json and stop') | |
| p.add_argument('--backbone-weights', default=DEFAULTS['backbone_weights'], | |
| help="torchvision weights the ResNet18 starts from (default %(default)s); 'none': from scratch") | |
| p.add_argument('--backbone-lr-scale', type=float, default=DEFAULTS['backbone_lr_scale'], | |
| help='learning rate of the image backbone relative to --lr (default %(default)s)') | |
| p.add_argument('--model', action='append', default=[], metavar='KEY=VALUE', | |
| help='override a PolicyConfig field (JSON value), e.g. num_layers=6; repeatable') | |
| p.add_argument('--cache-dir', type=Path, default=Path('~/hb_models/baselines/cache')) | |
| p.add_argument('--workers', type=int, default=32, | |
| help='DataLoader workers (and frame-cache decoders); the v2 augmentation costs ~17 ms of one CPU ' | |
| 'core per item, so 8 workers feed only ~6 steps/s at batch 64') | |
| p.add_argument('--prefetch', type=int, default=2, help='batches prefetched per worker') | |
| p.add_argument('--save-every', type=int, default=10000) | |
| p.add_argument('--log-every', type=int, default=100) | |
| p.add_argument('--eval-samples', type=int, default=256) | |
| p.add_argument('--seed', type=int, default=0) | |
| p.add_argument('--device', default='cuda:0') | |
| p.add_argument('--no-amp', action='store_true') | |
| p.add_argument('--resume', action='store_true', help='continue from <out>/train_state.pt') | |
| args = p.parse_args() | |
| out = args.out.expanduser().resolve() | |
| out.mkdir(parents=True, exist_ok=True) | |
| contract = CONTRACTS[args.contract] | |
| aug = augmentation_module() | |
| aug.check_state_layout(contract.robot, contract.state_dim, [column for column, _, _ in contract.state]) | |
| hb_aug = args.augmentation == 'hb-v2' | |
| if hb_aug and args.shift: | |
| raise SystemExit('--shift is the native random shift, which the augmentation spec v2 turns off ' | |
| '(docs/augmentation_v2.md); use it only with --augmentation none') | |
| if contract.body_dims != aug.BODY_JOINTS[contract.robot]: | |
| raise SystemExit(f'contract {contract.name}: body_dims {contract.body_dims}, the spec perturbs ' | |
| f'{aug.BODY_JOINTS[contract.robot]} body joints') | |
| augmentation = (aug.describe(contract.robot, contract.body_dims) if hb_aug else | |
| dict(version=None, exception='--augmentation none (not the benchmark default)')) | |
| cameras = tuple(args.cameras.split(',')) if args.cameras else contract.cameras | |
| hw = parse_hw(args.image_size) | |
| dataset = LeRobotDataset(args.dataset) | |
| dataset.check_contract(contract, cameras) | |
| episodes = parse_episodes(args.episodes, dataset.lengths) | |
| if not 0 <= args.val_episodes < len(episodes): | |
| raise SystemExit('--val-episodes must leave training episodes') | |
| lowdim = load_lowdim(dataset, contract, episodes) | |
| if len(lowdim['tasks']) != 1: | |
| raise SystemExit(f"the baselines take no language: one task per dataset, found {lowdim['tasks']}") | |
| n_train = len(episodes) - args.val_episodes | |
| train_pos, val_pos = list(range(n_train)), list(range(n_train, len(episodes))) | |
| train_rows = np.concatenate([np.arange(s, s + n) for s, n in | |
| zip(lowdim['starts'][:n_train], lowdim['lengths'][:n_train])]) | |
| stats = statistics(lowdim, train_rows) | |
| frames = frame_cache(dataset, lowdim, cameras, hw, args.cache_dir, workers=args.workers) | |
| train_set = ChunkDataset(lowdim, frames, cameras, hw, args.horizon, train_pos, | |
| body_dims=contract.body_dims if hb_aug else 0) | |
| eval_set = ChunkDataset(lowdim, frames, cameras, hw, args.horizon, val_pos or train_pos) # never augmented | |
| checks = preflight(train_set, eval_set, contract, seed=args.seed) | |
| print(json.dumps(dict(event='preflight', augmentation=augmentation.get('version'), **checks)), flush=True) | |
| if args.preflight_only: | |
| (out / 'run.json').write_text(json.dumps(dict( | |
| argv=sys.argv, started=datetime.now().isoformat(timespec='seconds'), host=os.uname().nodename, | |
| contract=contract.to_dict(cameras), image_hw=list(hw), augmentation=augmentation, preflight=checks, | |
| shift_px=args.shift, code_commit=git_commit()), indent=1, default=str)) | |
| return | |
| import torch | |
| torch.manual_seed(args.seed) | |
| np.random.seed(args.seed) | |
| device = torch.device(args.device) | |
| overrides = {k: json.loads(v) for k, v in (item.split('=', 1) for item in args.model)} | |
| weights = None if str(args.backbone_weights).lower() in ('none', '') else args.backbone_weights | |
| pcfg = PolicyConfig.from_dict({**dict(policy=args.policy, n_cameras=len(cameras), state_dim=contract.state_dim, | |
| action_dim=contract.action_dim, horizon=args.horizon, | |
| backbone_weights=weights), **overrides}) | |
| policy = build_policy(pcfg) | |
| policy.fit_normalizers(stats) | |
| policy.to(device) | |
| ema = EMA(policy, args.ema) if args.ema > 0 else None | |
| named = list(policy.named_parameters()) | |
| backbone = [p for n, p in named if n.startswith(policy.backbone_prefix)] | |
| rest = [p for n, p in named if not n.startswith(policy.backbone_prefix)] | |
| if not backbone: | |
| raise SystemExit(f'no backbone parameters under {policy.backbone_prefix!r}') | |
| optimizer = torch.optim.AdamW([dict(params=rest), dict(params=backbone, lr=args.lr * args.backbone_lr_scale)], | |
| lr=args.lr, betas=DEFAULTS['betas'], weight_decay=args.weight_decay) | |
| def lr_lambda(step): | |
| if step < args.warmup: | |
| return (step + 1) / max(1, args.warmup) | |
| return 0.5 * (1 + math.cos(math.pi * min(1.0, (step - args.warmup) / max(1, args.steps - args.warmup)))) | |
| scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) | |
| step = 0 | |
| state_path = out / 'train_state.pt' | |
| if args.resume and state_path.exists(): | |
| saved = torch.load(state_path, map_location='cpu', weights_only=False) | |
| if saved['config']['policy'] != pcfg.to_dict(): | |
| raise SystemExit('train_state.pt was written for another policy configuration') | |
| before = saved['config'].get('training', {}) | |
| if (before.get('augmentation') or {}).get('version') != augmentation.get('version') \ | |
| or before.get('shift_px') != args.shift: | |
| raise SystemExit('train_state.pt was trained with another augmentation (version / --shift); resuming ' | |
| 'would change it mid-run') | |
| policy.load_state_dict(saved['policy']) | |
| optimizer.load_state_dict(saved['optimizer']) | |
| scheduler.load_state_dict(saved['scheduler']) | |
| if ema is not None: | |
| ema.model.load_state_dict(saved['ema']) | |
| step = int(saved['step']) | |
| print(json.dumps(dict(event='resumed', step=step)), flush=True) | |
| samples = np.random.default_rng(1).choice(len(eval_set), size=min(args.eval_samples, len(eval_set)), replace=False) | |
| config = dict( | |
| policy=pcfg.to_dict(), contract=contract.to_dict(cameras), image_hw=list(hw), task_text=lowdim['tasks'][0], | |
| dataset=dict(name=dataset.name, | |
| root=str(dataset.root), robot_type=dataset.info.get('robot_type'), fps=dataset.fps, | |
| episodes=episodes, train_episodes=episodes[:n_train], val_episodes=episodes[n_train:], | |
| train_frames=int(len(train_set)), | |
| git_commit=dataset.info.get('script_config', {}).get('git_commit')), | |
| training=dict(steps=args.steps, batch_size=args.batch_size, lr=args.lr, betas=list(DEFAULTS['betas']), | |
| weight_decay=args.weight_decay, warmup=args.warmup, schedule='linear warm-up + cosine', | |
| backbone_lr_scale=args.backbone_lr_scale, | |
| ema=args.ema, shift_px=args.shift, amp=not args.no_amp, seed=args.seed, | |
| augmentation=augmentation, preflight=checks), | |
| stats=stats, code_commit=git_commit()) | |
| run = dict(argv=sys.argv, started=datetime.now().isoformat(timespec='seconds'), host=os.uname().nodename, | |
| torch=torch.__version__, **config) | |
| (out / 'run.json').write_text(json.dumps(run, indent=1, default=str)) | |
| loader = torch.utils.data.DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, | |
| num_workers=args.workers, pin_memory=True, persistent_workers=True, | |
| prefetch_factor=args.prefetch) | |
| params = sum(x.numel() for x in policy.parameters()) | |
| print(json.dumps(dict(event='start', policy=args.policy, params=params, | |
| backbone_params=sum(x.numel() for x in backbone), backbone_weights=pcfg.backbone_weights, | |
| backbone_norm=pcfg.backbone_norm, train_frames=len(train_set), steps=args.steps, | |
| from_step=step, cameras=list(cameras), hw=list(hw))), flush=True) | |
| log = (out / 'train_log.jsonl').open('a') | |
| amp = (not args.no_amp) and device.type == 'cuda' | |
| def save(step): | |
| folder = out / f'checkpoints/step-{step:06d}' | |
| model = ema.model if ema is not None else policy | |
| metrics = offline_eval(model, eval_set, samples, device, contract.action) | |
| save_checkpoint(folder, model, dict(config, step=step, weights='ema' if ema is not None else 'raw', | |
| offline_eval=dict(metrics, samples=len(samples), | |
| frames='val' if val_pos else 'train'))) | |
| final = out / 'checkpoints/final' | |
| if final.is_symlink() or final.exists(): | |
| final.unlink() | |
| final.symlink_to(folder.name) | |
| torch.save(dict(policy=policy.state_dict(), optimizer=optimizer.state_dict(), scheduler=scheduler.state_dict(), | |
| ema=None if ema is None else ema.model.state_dict(), step=step, config=config), | |
| out / 'train_state.pt.tmp') | |
| (out / 'train_state.pt.tmp').replace(state_path) | |
| record = dict(event='save', step=step, checkpoint=str(folder), **metrics) | |
| print(json.dumps(record), flush=True) | |
| log.write(json.dumps(record) + '\n') | |
| log.flush() | |
| policy.train() | |
| policy.train() | |
| t0, data_s, acc = time.monotonic(), 0.0, {} | |
| while step < args.steps: | |
| tick = time.monotonic() | |
| for images, state, action in loader: | |
| data_s += time.monotonic() - tick | |
| images = images.to(device, non_blocking=True).permute(0, 1, 4, 2, 3).float().div_(255.) | |
| images = random_shift(images, args.shift) | |
| state, action = state.to(device, non_blocking=True), action.to(device, non_blocking=True) | |
| with torch.autocast(device.type, dtype=torch.bfloat16, enabled=amp): | |
| loss, metrics = policy.compute_loss(images, state, action) | |
| optimizer.zero_grad(set_to_none=True) | |
| loss.backward() | |
| optimizer.step() | |
| scheduler.step() | |
| step += 1 | |
| decay = ema.update(policy, step) if ema is not None else 0.0 | |
| for k, v in dict(metrics, loss=float(loss.detach())).items(): | |
| acc[k] = acc.get(k, 0.0) + v | |
| if step % args.log_every == 0: | |
| elapsed = time.monotonic() - t0 | |
| record = dict(event='train', step=step, lr=scheduler.get_last_lr()[0], | |
| backbone_lr=scheduler.get_last_lr()[1], ema_decay=round(decay, 6), | |
| it_s=round(args.log_every / elapsed, 2), data_frac=round(data_s / elapsed, 3), | |
| **{k: v / args.log_every for k, v in acc.items()}) | |
| print(json.dumps(record), flush=True) | |
| log.write(json.dumps(record) + '\n') | |
| log.flush() | |
| t0, data_s, acc = time.monotonic(), 0.0, {} | |
| if step % args.save_every == 0 or step == args.steps: | |
| save(step) | |
| if step >= args.steps: | |
| break | |
| tick = time.monotonic() | |
| log.close() | |
| print(json.dumps(dict(event='done', step=step, checkpoint=str(out / 'checkpoints/final'))), flush=True) | |
| if __name__ == '__main__': | |
| main() | |