scripts/build_cache.py

"""Encode every frame of one T-Rex episode with the frozen Wan2.2 VAE.

Replicates the baseline observation path exactly: each view is resized to
160x320 (bilinear, antialias), tiled into the DreamZero 2x2 grid
[head | right ; left | black] (320x640), resized back to 160x320 (bilinear,
as ``encode_video`` does) and encoded as a single-frame video into a
48x10x20 latent.  Output: ``latents`` [N,48,10,20] float16.
"""

from __future__ import annotations

import argparse
import time
from pathlib import Path

import numpy as np
import torch
from torchvision.transforms import InterpolationMode
from torchvision.transforms import functional as TF

from fact.perception.wan_video_vae import WanVideoVAE38

VIEW_DIRS = (
    "observation.images.head_left",
    "observation.images.left_wrist",
    "observation.images.right_wrist",
)


def load_vae(path: str, device: torch.device) -> WanVideoVAE38:
    vae = WanVideoVAE38(z_dim=48, dim=160, vae_pretrained_path=path)
    state = torch.load(path, map_location="cpu")
    if isinstance(state, dict):
        for key in ("state_dict", "model", "module"):
            if isinstance(state.get(key), dict):
                state = state[key]
                break
    state = {k.removeprefix("module."): v for k, v in state.items() if torch.is_tensor(v)}
    current = vae.model.state_dict()
    compatible = {k: v for k, v in state.items() if k in current and current[k].shape == v.shape}
    coverage = sum(v.numel() for v in compatible.values()) / sum(v.numel() for v in current.values())
    if coverage < 0.99:
        raise ValueError(f"VAE coverage {coverage:.3f} from {path}")
    vae.model.load_state_dict(compatible, strict=False)
    vae = vae.to(device).eval()
    vae.mean = vae.mean.to(device)
    vae.std = vae.std.to(device)
    vae.scale = [vae.mean, vae.std.reciprocal()]
    return vae


def read_all_frames(path: Path) -> np.ndarray:
    import decord

    reader = decord.VideoReader(str(path), num_threads=2)
    return reader.get_batch(list(range(len(reader)))).asnumpy().astype(np.uint8)


def composite(views: list[np.ndarray], height: int, width: int) -> torch.Tensor:
    """3 x [N,H,W,3] uint8 -> [N,3,H,W] float in [-1,1] at the model size."""

    resized = []
    for frames in views:
        t = torch.from_numpy(frames).permute(0, 3, 1, 2)  # N,3,H,W uint8
        t = TF.resize(t, [height, width], interpolation=InterpolationMode.BILINEAR, antialias=True)
        resized.append(t)
    head, left, right = resized
    n = head.shape[0]
    grid = torch.zeros(n, 3, 2 * height, 2 * width, dtype=torch.uint8)
    grid[:, :, :height, :width] = head
    grid[:, :, height:, :width] = left
    grid[:, :, :height, width:] = right
    grid = grid.float() / 255.0 * 2 - 1
    grid = torch.nn.functional.interpolate(grid, size=(height, width), mode="bilinear", align_corners=False)
    return grid


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--dataset-root", required=True)
    parser.add_argument("--episode", type=int, default=0)
    parser.add_argument("--vae", required=True)
    parser.add_argument("--out", required=True)
    parser.add_argument("--height", type=int, default=160)
    parser.add_argument("--width", type=int, default=320)
    parser.add_argument("--batch", type=int, default=16)
    args = parser.parse_args()

    device = torch.device("cuda")
    root = Path(args.dataset_root)
    views = []
    for view in VIEW_DIRS:
        path = root / "videos" / "chunk-000" / view / f"episode_{args.episode:06d}.mp4"
        frames = read_all_frames(path)
        print(f"{view}: {frames.shape}", flush=True)
        views.append(frames)
    n = min(v.shape[0] for v in views)
    views = [v[:n] for v in views]
    images = composite(views, args.height, args.width)  # N,3,H,W
    vae = load_vae(args.vae, device)
    latents = []
    start = time.time()
    with torch.no_grad():
        for i in range(0, n, args.batch):
            chunk = images[i : i + args.batch].to(device)
            video = chunk.unsqueeze(2)  # B,3,1,H,W (single-frame videos)
            z = vae.encode(video, tiled=False)  # B,48,1,h,w
            latents.append(z[:, :, 0].float().cpu())
            if (i // args.batch) % 10 == 0:
                print(f"  {i}/{n} ({time.time() - start:.1f}s)", flush=True)
    latents = torch.cat(latents).numpy().astype(np.float16)
    print("latents", latents.shape, "mean", float(latents.mean()), "std", float(latents.std()))
    np.savez_compressed(args.out, latents=latents, episode=args.episode, dataset_root=str(root))
    print("wrote", args.out)


if __name__ == "__main__":
    main()