| """Build the FACT offline caches for a whole LeRobot-v2 T-Rex dataset. |
| |
| * ``--what latents``: Wan2.2 VAE latents of every frame of every episode |
| (``<out>/latents/episode_XXXXXX.npy``, float16 [N,48,10,20]). Shardable |
| across devices with ``--shard i --num-shards n``; existing files are skipped. |
| * ``--what text``: umT5-XXL token embeddings of every task string |
| (``<out>/text_cache.npz``). |
| |
| Observation preprocessing is identical to the baseline path (per-view resize |
| to 160x320, DreamZero 2x2 grid [head | right ; left | black], resize back to |
| 160x320, single-frame VAE encode). |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import time |
| from pathlib import Path |
|
|
| import numpy as np |
| import torch |
|
|
| from fact.data import latent_cache_path, load_episode_tasks |
|
|
| VIEW_DIRS = ( |
| "observation.images.head_left", |
| "observation.images.left_wrist", |
| "observation.images.right_wrist", |
| ) |
|
|
|
|
| def _load_component(module: torch.nn.Module, path: str, min_coverage: float) -> None: |
| state = torch.load(path, map_location="cpu") |
| if isinstance(state, dict): |
| for key in ("state_dict", "model", "module"): |
| if isinstance(state.get(key), dict): |
| state = state[key] |
| break |
| state = {k.removeprefix("module."): v for k, v in state.items() if torch.is_tensor(v)} |
| current = module.state_dict() |
| compatible = {k: v for k, v in state.items() if k in current and current[k].shape == v.shape} |
| coverage = sum(v.numel() for v in compatible.values()) / max(1, sum(v.numel() for v in current.values())) |
| if coverage < min_coverage: |
| raise ValueError(f"checkpoint coverage {coverage:.3f} < {min_coverage} for {path}") |
| module.load_state_dict(compatible, strict=False) |
|
|
|
|
| def load_vae(path: str, device: torch.device): |
| from fact.perception.wan_video_vae import WanVideoVAE38 |
|
|
| vae = WanVideoVAE38(z_dim=48, dim=160, vae_pretrained_path=path) |
| _load_component(vae.model, path, 0.99) |
| vae = vae.to(device).eval() |
| vae.mean = vae.mean.to(device) |
| vae.std = vae.std.to(device) |
| vae.scale = [vae.mean, vae.std.reciprocal()] |
| return vae |
|
|
|
|
| def read_all_frames(path: Path) -> np.ndarray: |
| import decord |
|
|
| reader = decord.VideoReader(str(path), num_threads=2) |
| return reader.get_batch(list(range(len(reader)))).asnumpy().astype(np.uint8) |
|
|
|
|
| def composite(views: list[np.ndarray], height: int, width: int) -> torch.Tensor: |
| from torchvision.transforms import InterpolationMode |
| from torchvision.transforms import functional as TF |
|
|
| resized = [] |
| for frames in views: |
| t = torch.from_numpy(frames).permute(0, 3, 1, 2) |
| t = TF.resize(t, [height, width], interpolation=InterpolationMode.BILINEAR, antialias=True) |
| resized.append(t) |
| head, left, right = resized |
| n = head.shape[0] |
| grid = torch.zeros(n, 3, 2 * height, 2 * width, dtype=torch.uint8) |
| grid[:, :, :height, :width] = head |
| grid[:, :, height:, :width] = left |
| grid[:, :, :height, width:] = right |
| grid = grid.float() / 255.0 * 2 - 1 |
| return torch.nn.functional.interpolate(grid, size=(height, width), mode="bilinear", align_corners=False) |
|
|
|
|
| def encode_episode(vae, root: Path, episode: int, device, height: int, width: int, batch: int) -> np.ndarray: |
| views = [] |
| for view in VIEW_DIRS: |
| path = root / "videos" / f"chunk-{episode // 1000:03d}" / view / f"episode_{episode:06d}.mp4" |
| views.append(read_all_frames(path)) |
| n = min(v.shape[0] for v in views) |
| images = composite([v[:n] for v in views], height, width) |
| latents = [] |
| with torch.no_grad(): |
| for i in range(0, n, batch): |
| chunk = images[i : i + batch].to(device).unsqueeze(2) |
| z = vae.encode(chunk, tiled=False) |
| latents.append(z[:, :, 0].float().cpu()) |
| return torch.cat(latents).numpy().astype(np.float16) |
|
|
|
|
| def build_latents(args, root: Path, out: Path) -> None: |
| device = torch.device("cuda") |
| tasks = load_episode_tasks(root) |
| episodes = sorted(tasks) if not args.episodes else [int(e) for e in args.episodes.split(",")] |
| episodes = [e for i, e in enumerate(episodes) if i % args.num_shards == args.shard] |
| (out / "latents").mkdir(parents=True, exist_ok=True) |
| todo = [e for e in episodes if not latent_cache_path(out, e).exists()] |
| print(f"shard {args.shard}/{args.num_shards}: {len(episodes)} episodes, {len(todo)} to encode", flush=True) |
| if not todo: |
| return |
| vae = load_vae(args.vae, device) |
| start = time.time() |
| for n, episode in enumerate(todo): |
| latents = encode_episode(vae, root, episode, device, args.height, args.width, args.batch) |
| if not np.isfinite(latents.astype(np.float32)).all(): |
| raise ValueError(f"non-finite latents for episode {episode}") |
| tmp = latent_cache_path(out, episode).with_suffix(".tmp.npy") |
| np.save(tmp, latents) |
| tmp.rename(latent_cache_path(out, episode)) |
| if n % 5 == 0: |
| print(f" [{n + 1}/{len(todo)}] episode {episode}: {latents.shape} ({time.time() - start:.0f}s)", flush=True) |
| print(f"shard {args.shard} done in {(time.time() - start) / 60:.1f} min", flush=True) |
|
|
|
|
| def build_text(args, root: Path, out: Path) -> None: |
| from fact.perception.wan_video_text_encoder import WanTextEncoder |
| from fact.perception.tokenizer import HuggingfaceTokenizer |
|
|
| device = torch.device("cuda") |
| tasks: dict[int, str] = {} |
| with (root / "meta" / "tasks.jsonl").open() as handle: |
| for line in handle: |
| if line.strip(): |
| rec = json.loads(line) |
| tasks[int(rec["task_index"])] = rec["task"] |
| tokenizer = HuggingfaceTokenizer(name=args.tokenizer, seq_len=args.max_text_tokens, clean="whitespace") |
| encoder = WanTextEncoder(text_encoder_pretrained_path=args.text_encoder) |
| _load_component(encoder, args.text_encoder, 0.99) |
| encoder = encoder.to(device=device, dtype=torch.bfloat16).eval() |
| indices = sorted(tasks) |
| tokens = np.zeros((len(indices), args.max_text_tokens, 4096), dtype=np.float16) |
| masks = np.zeros((len(indices), args.max_text_tokens), dtype=bool) |
| with torch.no_grad(): |
| for i in range(0, len(indices), args.batch): |
| chunk = indices[i : i + args.batch] |
| ids, mask = tokenizer([tasks[t] for t in chunk], return_mask=True) |
| emb = encoder(ids.to(device), mask.to(device)).float().cpu().numpy() |
| tokens[i : i + len(chunk)] = emb.astype(np.float16) |
| masks[i : i + len(chunk)] = mask.numpy().astype(bool) |
| out.mkdir(parents=True, exist_ok=True) |
| np.savez_compressed(out / "text_cache.npz", tokens=tokens, mask=masks, task_index=np.array(indices)) |
| print(f"text cache: {len(indices)} tasks x {args.max_text_tokens} tokens -> {out / 'text_cache.npz'}", flush=True) |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--dataset-root", required=True) |
| parser.add_argument("--out", required=True, help="cache directory") |
| parser.add_argument("--what", choices=["latents", "text"], required=True) |
| parser.add_argument("--vae", default=None) |
| parser.add_argument("--text-encoder", default=None) |
| parser.add_argument("--tokenizer", default=None) |
| parser.add_argument("--max-text-tokens", type=int, default=64) |
| parser.add_argument("--episodes", default="", help="comma list; default all") |
| parser.add_argument("--shard", type=int, default=0) |
| parser.add_argument("--num-shards", type=int, default=1) |
| parser.add_argument("--height", type=int, default=160) |
| parser.add_argument("--width", type=int, default=320) |
| parser.add_argument("--batch", type=int, default=16) |
| args = parser.parse_args() |
| root, out = Path(args.dataset_root), Path(args.out) |
| if args.what == "latents": |
| if not args.vae: |
| parser.error("--vae is required for latents") |
| build_latents(args, root, out) |
| else: |
| if not (args.text_encoder and args.tokenizer): |
| parser.error("--text-encoder and --tokenizer are required for text") |
| build_text(args, root, out) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
