scripts/infer_humanoid_singleview_long_autoregressive.py

"""
Long autoregressive rollout for humanoid singleview DreamDojo using generate_vid2world_long.

Uses the built-in chunk_overlap mechanism for smoother transitions between chunks.
Only the first GT frame is used as conditioning — pure autoregressive after that.

For each episode:
  1. Take the first GT frame as conditioning.
  2. Collect the full action sequence.
  3. Call generate_vid2world_long with chunk_overlap for smooth long-horizon generation.
  4. Output: one video per episode matching the GT episode length.
"""

import argparse
import json
import sys
import time
from pathlib import Path

import mediapy
import numpy as np
import piq
import torch
import torchvision

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "models" / "DreamDojo"))


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--checkpoints-dir", type=str, required=True)
    parser.add_argument("--experiment", type=str, default="dreamdojo_2b_480_640_gr1")
    parser.add_argument("--dataset-path", type=str, required=True)
    parser.add_argument("--save-dir", type=str, required=True)
    parser.add_argument("--num-frames", type=int, default=49,
                        help="Model's native chunk size (frames per forward pass)")
    parser.add_argument("--chunk-overlap", type=int, default=4,
                        help="Number of overlapping frames between chunks for smooth transitions")
    parser.add_argument("--num-episodes", type=int, default=None)
    parser.add_argument("--save-fps", type=int, default=10)
    parser.add_argument("--output-dir", type=str, default=None)
    parser.add_argument("--guidance", type=float, default=0)
    parser.add_argument("--num-latent-conditional-frames", type=int, default=1,
                        help="Latent conditional frames (1=image2world, 2=video2world with 5 pixel frames)")
    parser.add_argument("--resolution", type=str, default="480,640")
    return parser.parse_args()


def build_model(args):
    from cosmos_predict2.action_conditioned_config import ActionConditionedSetupArguments
    from cosmos_predict2.config import MODEL_CHECKPOINTS
    from cosmos_predict2._src.predict2.inference.video2world import Video2WorldInference

    setup_args = ActionConditionedSetupArguments(
        model="2B/robot/action-cond",
        config_file="cosmos_predict2/_src/predict2/action/configs/action_conditioned/config.py",
        checkpoints_dir=args.checkpoints_dir,
        experiment=args.experiment,
        num_frames=args.num_frames,
        dataset_path=args.dataset_path,
        save_dir=args.save_dir,
        output_dir=args.output_dir or args.save_dir,
        num_samples=1,
        data_split="full",
        single_base_index=True,
    )

    checkpoints_dir = Path(args.checkpoints_dir)
    last_checkpoint_file = checkpoints_dir / "latest_checkpoint.txt"
    if not last_checkpoint_file.exists():
        parent_file = checkpoints_dir.parent / "latest_checkpoint.txt"
        if parent_file.exists():
            checkpoints_dir = checkpoints_dir.parent
            last_checkpoint_file = parent_file

    if not last_checkpoint_file.exists():
        raise FileNotFoundError(f"Could not find latest_checkpoint.txt in {args.checkpoints_dir} or its parent.")

    with open(last_checkpoint_file) as f:
        last_checkpoint = f.read().strip()
    checkpoint_iter_dir = checkpoints_dir / last_checkpoint

    from examples.action_conditioned import resolve_checkpoint_path
    checkpoint_path = resolve_checkpoint_path(checkpoint_iter_dir)

    checkpoint = MODEL_CHECKPOINTS[setup_args.model_key]
    experiment = setup_args.experiment or checkpoint.experiment

    video2world_cli = Video2WorldInference(
        experiment_name=experiment,
        ckpt_path=checkpoint_path,
        s3_credential_path="",
        context_parallel_size=setup_args.context_parallel_size,
        config_file=setup_args.config_file,
        experiment_opts=[],
        cache_config=None,
    )

    return video2world_cli, checkpoint_iter_dir.name


def build_dataset(args):
    from groot_dreams.dataloader import MultiVideoActionDataset

    dataset = MultiVideoActionDataset(
        num_frames=args.num_frames,
        dataset_path=args.dataset_path,
        data_split="full",
        single_base_index=False,
        restrict_len=None,
        deterministic_uniform_sampling=False,
    )
    return dataset


def get_episode_plan(dataset, num_frames):
    """Group dataset indices by episode. Return plan with non-overlapping segment indices."""
    episodes = {}

    global_offset = 0
    for ds_idx, ds in enumerate(dataset.datasets):
        lerobot_ds = ds.lerobot_dataset
        for local_idx, (traj_id, base_index) in enumerate(lerobot_ds.all_steps):
            key = (ds_idx, int(traj_id))
            if key not in episodes:
                episodes[key] = []
            episodes[key].append((global_offset + local_idx, int(base_index)))
        global_offset += len(ds)

    delta_indices = dataset.datasets[0].lerobot_dataset.modality_configs["video"].delta_indices
    timestep_interval = delta_indices[1] - delta_indices[0]
    stride_raw = (num_frames - 1) * timestep_interval

    plan = []
    for key, steps in episodes.items():
        ds_idx, traj_id = key
        steps_sorted = sorted(steps, key=lambda x: x[1])
        if not steps_sorted:
            continue

        segment_indices = []
        next_base = 0
        for global_id, base_idx in steps_sorted:
            if base_idx >= next_base:
                segment_indices.append(global_id)
                next_base = base_idx + stride_raw

        if segment_indices:
            traj_length = int(dataset.datasets[ds_idx].lerobot_dataset.trajectory_lengths[
                np.where(dataset.datasets[ds_idx].lerobot_dataset.trajectory_ids == traj_id)[0][0]
            ])
            plan.append({
                "ds_idx": ds_idx,
                "traj_id": int(traj_id),
                "traj_length": traj_length,
                "segment_data_ids": segment_indices,
            })

    return plan


def generate_long_autoregressive(video2world_cli, first_frame_tensor, all_actions,
                                  num_frames, chunk_overlap, guidance, resolution,
                                  num_latent_conditional_frames, lam_video=None):
    """
    Chunked autoregressive generation with overlap for smooth long-horizon videos.

    Manually implements chunk overlap because generate_autoregressive_from_batch
    does not slice actions per chunk (passes full action tensor → shape mismatch).

    Each chunk generates num_frames pixel frames using (num_frames - 1) actions.
    Chunks overlap by chunk_overlap frames: the last chunk_overlap frames of chunk N
    become the conditioning context for chunk N+1.

    Args:
        first_frame_tensor: (1, C, H, W) uint8 tensor of the first GT frame
        all_actions: numpy array of shape (total_actions, action_dim)
        num_frames: model's native capacity (pixel frames per forward pass)
        chunk_overlap: number of overlapping frames between chunks
        guidance: CFG scale
        resolution: "H,W" string
        num_latent_conditional_frames: 1 or 2
        lam_video: optional LAM video tensor
    """
    actions_per_chunk = num_frames - 1
    total_actions = len(all_actions)

    generated_chunks = []
    cond_frames = first_frame_tensor  # (1, C, H, W) for first chunk
    action_offset = 0
    chunk_idx = 0

    while action_offset < total_actions:
        remaining_actions = total_actions - action_offset
        chunk_actions_len = min(actions_per_chunk, remaining_actions)

        # Need at least chunk_overlap+1 actions for subsequent chunks to produce new frames,
        # and at least 2 actions for the first chunk
        if chunk_idx == 0 and chunk_actions_len < 2:
            break
        if chunk_idx > 0 and chunk_actions_len <= chunk_overlap:
            break

        actions_chunk = all_actions[action_offset: action_offset + chunk_actions_len]
        if isinstance(actions_chunk, np.ndarray):
            actions_chunk = torch.from_numpy(actions_chunk).float()

        num_video_frames = chunk_actions_len + 1

        # Build video input: conditioning frames + zeros
        if chunk_idx == 0:
            # First chunk: single GT frame
            vid_input = torch.cat(
                [cond_frames, torch.zeros_like(cond_frames).repeat(num_video_frames - 1, 1, 1, 1)],
                dim=0,
            )
        else:
            # Subsequent chunks: use last chunk_overlap frames from previous output
            num_cond = cond_frames.shape[0]  # chunk_overlap frames
            num_new = num_video_frames - num_cond
            if num_new <= 0:
                break
            vid_input = torch.cat(
                [cond_frames, torch.zeros(num_new, *cond_frames.shape[1:])],
                dim=0,
            )

        vid_input = vid_input.to(torch.uint8)
        vid_input = vid_input.unsqueeze(0).permute(0, 2, 1, 3, 4)  # (1, C, T, H, W)

        # LAM video slice if available
        current_lam = None
        if lam_video is not None:
            lam_start = action_offset * 2
            lam_end = lam_start + chunk_actions_len * 2
            if lam_end <= len(lam_video):
                current_lam = lam_video[lam_start:lam_end]

        # Determine num_latent_conditional_frames for this chunk
        if chunk_idx == 0:
            chunk_cond_frames = num_latent_conditional_frames
        else:
            # For subsequent chunks, we condition on chunk_overlap pixel frames.
            # Tokenizer compresses time 4x: chunk_overlap pixel frames → (chunk_overlap+3)//4 latent frames.
            # Model only accepts 1 or 2 for num_latent_conditional_frames.
            chunk_cond_frames = min(2, max(1, (chunk_overlap + 3) // 4))

        video = video2world_cli.generate_vid2world(
            prompt="",
            input_path=vid_input,
            action=actions_chunk,
            guidance=guidance,
            num_video_frames=num_video_frames,
            num_latent_conditional_frames=chunk_cond_frames,
            resolution=resolution,
            seed=chunk_idx,
            negative_prompt="The video captures a scene with low visual quality, blurring, jittering, or distortion.",
            lam_video=current_lam,
        )

        # Convert output from [-1, 1] to uint8 numpy
        video_normalized = (video - (-1)) / (1 - (-1))
        video_clamped = (
            (torch.clamp(video_normalized[0], 0, 1) * 255).to(torch.uint8).permute(1, 2, 3, 0).cpu().numpy()
        )

        if chunk_idx == 0:
            generated_chunks.append(video_clamped)
        else:
            # Remove overlap frames from beginning (they were conditioning context)
            generated_chunks.append(video_clamped[chunk_overlap:])

        # Prepare conditioning for next chunk: last chunk_overlap frames as pixel tensor
        tail_frames = video_clamped[-chunk_overlap:]  # (overlap, H, W, C) uint8 numpy
        cond_frames = torch.from_numpy(tail_frames).permute(0, 3, 1, 2).float()  # (overlap, C, H, W)

        # Advance: new frames produced = chunk_actions_len + 1 - chunk_overlap (for non-first)
        # Actions consumed for new (non-overlapping) output:
        #   chunk 0: all actions_per_chunk actions advance the timeline
        #   chunk N>0: we re-use chunk_overlap frames as context, so only advance by (chunk_actions_len - chunk_overlap)
        if chunk_idx == 0:
            action_offset += chunk_actions_len
        else:
            action_offset += chunk_actions_len - chunk_overlap

        chunk_idx += 1

    if not generated_chunks:
        return None

    return np.concatenate(generated_chunks, axis=0)


def main():
    args = parse_args()

    from cosmos_oss.init import init_environment, cleanup_environment
    init_environment()
    torch.enable_grad(False)

    print("Building model...")
    video2world_cli, iter_name = build_model(args)

    print("Building dataset...")
    dataset = build_dataset(args)

    print("Planning episodes...")
    plan = get_episode_plan(dataset, args.num_frames)
    total_episodes = len(plan)
    num_episodes = min(args.num_episodes or total_episodes, total_episodes)
    print(f"Total episodes: {total_episodes}, processing: {num_episodes}")

    save_root = Path(args.save_dir) / iter_name
    save_root.mkdir(parents=True, exist_ok=True)

    run_config = {
        "checkpoint": str(Path(args.checkpoints_dir).resolve()),
        "iter": iter_name,
        "experiment": args.experiment,
        "dataset_path": args.dataset_path,
        "num_frames": args.num_frames,
        "chunk_overlap": args.chunk_overlap,
        "num_latent_conditional_frames": args.num_latent_conditional_frames,
        "guidance": args.guidance,
        "resolution": args.resolution,
        "save_fps": args.save_fps,
        "num_episodes": num_episodes,
        "total_episodes": total_episodes,
        "mode": "long_autoregressive",
    }
    with open(save_root / "run_config.json", "w") as f:
        json.dump(run_config, f, indent=2)

    all_psnr, all_ssim, all_lpips = [], [], []
    all_gen_times = []

    for ep_idx in range(num_episodes):
        ep_info = plan[ep_idx]
        traj_id = ep_info["traj_id"]
        ep_save_dir = save_root / f"episode_{ep_idx:04d}"

        if (ep_save_dir / "full_pred.mp4").exists() and (ep_save_dir / "metrics.json").exists():
            print(f"[{ep_idx}/{num_episodes}] episode_{ep_idx:04d} already exists, loading metrics.")
            try:
                with open(ep_save_dir / "metrics.json") as f:
                    m = json.load(f)
                if m.get("psnr") is not None:
                    all_psnr.append(m["psnr"])
                    all_ssim.append(m["ssim"])
                    all_lpips.append(m["lpips"])
            except (json.JSONDecodeError, KeyError):
                pass
            continue

        num_segments = len(ep_info["segment_data_ids"])
        print(f"[{ep_idx}/{num_episodes}] traj_id={traj_id}, "
              f"length={ep_info['traj_length']}, segments={num_segments}")

        # Collect all actions and GT frames from all segments
        all_actions_parts = []
        all_lam_parts = []
        gt_segments = []

        for seg_idx, data_id in enumerate(ep_info["segment_data_ids"]):
            sample = dataset[data_id]
            actions = sample["action"][:args.num_frames - 1]
            if isinstance(actions, torch.Tensor):
                actions = actions.numpy()
            all_actions_parts.append(actions)

            lam = sample.get("lam_video", None)
            if lam is not None:
                all_lam_parts.append(lam)

            gt_seg = sample["video"].permute(1, 2, 3, 0).numpy()
            gt_segments.append(gt_seg)

        # First frame from the first segment
        first_sample = dataset[ep_info["segment_data_ids"][0]]
        first_frame = first_sample["video"].transpose(0, 1)[:1]  # (1, C, H, W)

        full_actions = np.concatenate(all_actions_parts, axis=0)

        full_lam = None
        if all_lam_parts:
            full_lam = torch.cat(all_lam_parts, dim=0) if isinstance(all_lam_parts[0], torch.Tensor) else None

        gen_start_time = time.time()

        pred_video = generate_long_autoregressive(
            video2world_cli,
            first_frame,
            full_actions,
            num_frames=args.num_frames,
            chunk_overlap=args.chunk_overlap,
            guidance=args.guidance,
            resolution=args.resolution,
            num_latent_conditional_frames=args.num_latent_conditional_frames,
            lam_video=full_lam,
        )

        gen_elapsed = time.time() - gen_start_time

        if pred_video is None or len(pred_video) == 0:
            print(f"  Skipping episode {traj_id}: could not generate any frames.")
            continue

        all_gen_times.append(gen_elapsed)

        # Build GT
        concat_gt = np.concatenate(gt_segments, axis=0)

        # Trim to same length
        min_len = min(len(pred_video), len(concat_gt))
        if len(pred_video) != len(concat_gt):
            print(f"  [Info] Frame mismatch: pred={len(pred_video)}, gt={len(concat_gt)}, using min={min_len}")
        pred_video = pred_video[:min_len]
        concat_gt = concat_gt[:min_len]

        ep_save_dir.mkdir(parents=True, exist_ok=True)
        mediapy.write_video(str(ep_save_dir / "full_pred.mp4"), pred_video, fps=args.save_fps)
        mediapy.write_video(str(ep_save_dir / "full_gt.mp4"), concat_gt, fps=args.save_fps)
        concat_merged = np.concatenate([concat_gt, pred_video], axis=2)
        mediapy.write_video(str(ep_save_dir / "full_merged.mp4"), concat_merged, fps=args.save_fps)

        # Compute metrics
        x_batch = torch.clamp(torch.from_numpy(pred_video.copy()) / 255.0, 0, 1).permute(0, 3, 1, 2)
        y_batch = torch.clamp(torch.from_numpy(concat_gt.copy()) / 255.0, 0, 1).permute(0, 3, 1, 2)
        try:
            psnr_val = piq.psnr(x_batch, y_batch).mean().item()
            ssim_val = piq.ssim(x_batch, y_batch).mean().item()
            lpips_val = piq.LPIPS()(x_batch, y_batch).mean().item()
        except (RuntimeError, ValueError) as e:
            print(f"  Metrics failed: {e}")
            psnr_val = ssim_val = lpips_val = None

        with open(ep_save_dir / "metrics.json", "w") as f:
            json.dump({
                "psnr": psnr_val, "ssim": ssim_val, "lpips": lpips_val,
                "gen_time_s": round(gen_elapsed, 2),
                "total_frames_pred": len(pred_video),
                "total_frames_gt": ep_info["traj_length"],
                "trajectory_id": traj_id,
                "chunk_overlap": args.chunk_overlap,
                "num_latent_conditional_frames": args.num_latent_conditional_frames,
                "mode": "long_autoregressive",
            }, f, indent=2)

        if psnr_val is not None:
            all_psnr.append(psnr_val)
            all_ssim.append(ssim_val)
            all_lpips.append(lpips_val)
        print(f"  -> {len(pred_video)} frames ({gen_elapsed:.1f}s), "
              f"PSNR={psnr_val:.2f}, SSIM={ssim_val:.4f}, LPIPS={lpips_val:.4f}")

    # Summary
    timing_summary = {}
    if all_gen_times:
        mean_gen_time = sum(all_gen_times) / len(all_gen_times)
        total_gen_time = sum(all_gen_times)
        print(f"\n[Timing] Generated {len(all_gen_times)} episodes in {total_gen_time:.2f}s total")
        print(f"[Timing] Average generation time per episode: {mean_gen_time:.2f}s")
        timing_summary = {
            "num_episodes_generated": len(all_gen_times),
            "total_gen_time_s": round(total_gen_time, 2),
            "avg_gen_time_s": round(mean_gen_time, 2),
        }

    if all_psnr:
        summary = {
            "mean_psnr": sum(all_psnr) / len(all_psnr),
            "mean_ssim": sum(all_ssim) / len(all_ssim),
            "mean_lpips": sum(all_lpips) / len(all_lpips),
            "num_episodes_processed": len(all_psnr),
            "chunk_overlap": args.chunk_overlap,
            "num_latent_conditional_frames": args.num_latent_conditional_frames,
            "mode": "long_autoregressive",
            **timing_summary,
        }
        with open(save_root / "all_summary.json", "w") as f:
            json.dump(summary, f, indent=2)
        print(f"\n=== Summary ({len(all_psnr)} episodes) ===")
        print(f"PSNR: {summary['mean_psnr']:.3f}")
        print(f"SSIM: {summary['mean_ssim']:.4f}")
        print(f"LPIPS: {summary['mean_lpips']:.4f}")
    else:
        with open(save_root / "all_summary.json", "w") as f:
            json.dump({"psnr": None, "ssim": None, "lpips": None, **timing_summary}, f, indent=2)

    cleanup_environment()


if __name__ == "__main__":
    main()