Download scripts/infer_video.py from RyanL22/RoboTryOn: direct link, hf CLI and curl.
- Browser
- Download file 8.59 kB
-
https://huggingface.co/spaces/RyanL22/RoboTryOn/resolve/main/scripts/infer_video.py
- Command line
-
hf download hf://spaces/RyanL22/RoboTryOn/scripts/infer_video.py
-
curl -L -o infer_video.py https://huggingface.co/spaces/RyanL22/RoboTryOn/resolve/main/scripts/infer_video.py
8.59 kB
| """Synthesize the robot video of one human episode (paper Sec. 3.3, Eq. 3.4, App. B.2.4 / B.4). | |
| python scripts/infer_video.py --config configs/video/infer.yaml \ | |
| input.episode=path/to/human/ep0 input.keyframes_dir=path/to/keyframes/ep0 \ | |
| output.dir=path/to/video/ep0 two_stage.mode=same_res | |
| Inputs: | |
| <episode>/meta.json fps (must equal ``fps``), views, num_frames, caption | |
| <episode>/keypoints_<view>.npz retargeted hand keypoints at video resolution | |
| <episode>/video_<view>.mp4 the human video (its frame size; its pixels too with | |
| ``mask_human: false``) | |
| <keyframes_dir>/keyframes.json {"indices": [...], ...} written by scripts/infer_keyframes.py | |
| (falls back to <episode>/keyframes.json) | |
| <keyframes_dir>/keyframe_<view>_<index:06d>.png translated robot keyframe per index | |
| The skeleton flow S^h_{1:T} is rendered from the keypoints at ``skeleton_size`` on black | |
| (``mask_human: true``, paper default). With ``mask_human: false`` it is drawn on the unmasked human | |
| video instead; the keyframes always come from the masked image condition. The episode is | |
| cut at its keyframes into clips of at most ``frames_range[1]`` frames that share their boundary | |
| keyframe (``robotryon.video.chunks``); each clip is generated in two stages from its keyframes | |
| and skeleton flow, with both stereo views in one sequence by default. | |
| Outputs in ``output.dir``: | |
| video_<view>.mp4 the robot video, ``resolution`` per view, ``fps`` frames per second, | |
| frames keyframes[0] .. keyframes[-1] of the episode | |
| chunks.json the clip plan (start, length, keyframes of every clip) | |
| meta.json episode metadata in the shared format (for the IDM) | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from robotryon.common.config import parse_args, save_config | |
| from robotryon.skeleton import draw_skeleton_video, paint | |
| from robotryon.video.chunks import concat_chunks, drop_colliding_keyframes, plan_chunks | |
| from robotryon.video.dataset import caption_text | |
| from robotryon.video.io import read_image, write_video | |
| from robotryon.video.pipeline import build_pipeline, generate | |
| from robotryon.video.text import encode_for_inference | |
| def to_unit(a: np.ndarray) -> torch.Tensor: | |
| """uint8 ``(..., H, W, 3)`` -> float in [-1, 1] with channels moved before the frame axes.""" | |
| t = torch.from_numpy(np.ascontiguousarray(a)).float() / 127.5 - 1 | |
| return t.permute(2, 0, 1) if a.ndim == 3 else t.permute(3, 0, 1, 2) | |
| def skeleton_flow(ep: Path, view: str, total: int, size, draw_kw: dict) -> np.ndarray: | |
| """``(T, h, w, 3)`` uint8 skeleton flow of one view at ``size``.""" | |
| import cv2 | |
| cap = cv2.VideoCapture(str(ep / f"video_{view}.mp4")) | |
| W, H = cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT) | |
| cap.release() | |
| z = np.load(ep / f"keypoints_{view}.npz") | |
| kp = z["kp"][:total].astype(np.float32).copy() | |
| kp[..., 0] = (kp[..., 0] + 0.5) * size[0] / W - 0.5 | |
| kp[..., 1] = (kp[..., 1] + 0.5) * size[1] / H - 0.5 | |
| return draw_skeleton_video(kp, z["vis"][:total].astype(bool), size[0], size[1], **draw_kw) | |
| def human_frames(ep: Path, view: str, total: int, size) -> np.ndarray: | |
| """``(T, h, w, 3)`` uint8 RGB frames of the human video of one view, resized to ``size``.""" | |
| import cv2 | |
| cap = cv2.VideoCapture(str(ep / f"video_{view}.mp4")) | |
| frames = [] | |
| while len(frames) < total: | |
| ok, f = cap.read() | |
| if not ok: | |
| break | |
| frames.append(cv2.cvtColor(cv2.resize(f, tuple(size), interpolation=cv2.INTER_AREA), | |
| cv2.COLOR_BGR2RGB)) | |
| cap.release() | |
| if len(frames) < total: | |
| raise SystemExit(f"{ep / f'video_{view}.mp4'}: {len(frames)} frames, expected {total}") | |
| return np.stack(frames) | |
| def reference_flow(ep: Path, view: str, total: int, cfg) -> np.ndarray: | |
| """Reference video of one view: the skeleton flow, on black or on the human video.""" | |
| skel = skeleton_flow(ep, view, total, cfg.skeleton_size, cfg.skeleton.draw) | |
| if cfg.get("mask_human", True): | |
| return skel | |
| human = human_frames(ep, view, total, cfg.skeleton_size) | |
| return np.stack([paint(h, s) for h, s in zip(human, skel)]) | |
| def plan(cfg): | |
| """``(episode dir, views, view groups, frame count, keyframes, clips)`` of one episode.""" | |
| ep = Path(cfg.input.episode) | |
| meta = json.loads((ep / "meta.json").read_text()) | |
| if abs(float(meta["fps"]) - cfg.fps) > 1e-6: | |
| raise SystemExit(f"episode is {meta['fps']} fps, the model generates {cfg.fps}: " | |
| f"resample the episode first") | |
| views = list(cfg.input.views or meta["views"]) | |
| groups = [views] if cfg.input.stereo and len(views) == 2 else [[v] for v in views] | |
| total = int(meta["num_frames"]) | |
| kf_json = Path(cfg.input.keyframes_dir) / "keyframes.json" | |
| kf_json = kf_json if kf_json.exists() else ep / "keyframes.json" | |
| kf = sorted(set(int(i) for i in json.loads(kf_json.read_text())["indices"])) | |
| chunks = plan_chunks(kf, total, cfg.frames_range) | |
| if not chunks: | |
| raise SystemExit(f"no clip can be planned from keyframes {kf} in {total} frames") | |
| return ep, meta, views, groups, total, kf, chunks | |
| def episode_caption(cfg) -> str: | |
| meta = json.loads((Path(cfg.input.episode) / "meta.json").read_text()) | |
| return caption_text(cfg.text.template, meta, cfg.text.embodiment) | |
| def load_pipeline(cfg, views: int, device: str = "cuda"): | |
| return build_pipeline(cfg.model.lora, base=cfg.model.base, | |
| transformer_subfolder=cfg.model.transformer_subfolder, | |
| views=views, attention_backend=cfg.model.attention_backend, | |
| upsample=cfg.two_stage.mode == "upsample", device=device) | |
| def synthesize(cfg, pipe, text) -> Path: | |
| """Generate and write the robot video of ``cfg.input.episode`` with a built pipeline and the | |
| encoded caption ``text`` (``robotryon.video.text.encode_for_inference``).""" | |
| ep, meta, views, groups, total, kf, chunks = plan(cfg) | |
| kf_dir, out = Path(cfg.input.keyframes_dir), Path(cfg.output.dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| print(f"{ep.name}: {total} frames, keyframes {kf}, {len(chunks)} clips, views {groups}, " | |
| f"mode {cfg.two_stage.mode}, mask_human {cfg.get('mask_human', True)}") | |
| skel = {v: reference_flow(ep, v, total, cfg) for v in views} | |
| clips: dict[str, list[np.ndarray]] = {v: [] for v in views} | |
| for ci, ch in enumerate(chunks): | |
| keep = drop_colliding_keyframes(ch.kf_local, ch.frames) | |
| print(f" clip {ci}: frames {ch.start}..{ch.start + ch.frames - 1} ({ch.frames}f), " | |
| f"keyframes at {[ch.kf_local[k] for k in keep]}", flush=True) | |
| for group in groups: | |
| sk = torch.stack([to_unit(skel[v][ch.start:ch.start + ch.frames]) for v in group]) | |
| kfs = [(ch.kf_local[k], torch.stack([ | |
| to_unit(read_image(kf_dir / f"keyframe_{v}_{kf[ch.kf_slots[k]]:06d}.png")) | |
| for v in group])) for k in keep] | |
| video = generate(pipe, skeleton=sk, keyframes=kfs, text=text, | |
| num_frames=ch.frames, size=tuple(cfg.resolution), fps=cfg.fps, | |
| seed=cfg.seed + ci, mode=cfg.two_stage.mode) | |
| for vi, v in enumerate(group): | |
| clips[v].append(video[vi].numpy()) | |
| for v in views: | |
| full = concat_chunks(chunks, clips[v]) | |
| write_video(full, out / f"video_{v}.mp4", cfg.fps) | |
| (out / "chunks.json").write_text(json.dumps( | |
| {"keyframes": kf, "chunks": [c.to_dict() for c in chunks]}, indent=1)) | |
| (out / "meta.json").write_text(json.dumps( | |
| {"fps": cfg.fps, "views": views, "num_frames": int(full.shape[0]), | |
| "caption": meta.get("caption", ""), "embodiment": cfg.text.embodiment or "robot", | |
| "source_episode": ep.name, "first_frame": kf[0]}, indent=1)) | |
| save_config(cfg, out / "infer_config.yaml") | |
| print(f"wrote {out}: {len(views)} x {full.shape[0]} frames at " | |
| f"{cfg.resolution[0]}x{cfg.resolution[1]}, {cfg.fps} fps") | |
| return out | |
| def main() -> int: | |
| cfg = parse_args(__doc__) | |
| groups = plan(cfg)[3] | |
| caption = episode_caption(cfg) | |
| text = encode_for_inference([caption], cfg.model.base)[caption] | |
| synthesize(cfg, load_pipeline(cfg, len(groups[0])), text) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |