inference.py

import argparse
import os

os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import torch
from omegaconf import OmegaConf
from tqdm import tqdm
from torchvision import transforms
from torchvision.io import write_video
from einops import rearrange
import torch.distributed as dist
from torch.utils.data import DataLoader, SequentialSampler
from torch.utils.data.distributed import DistributedSampler

from pipeline import (
    CausalDiffusionInferencePipeline,
    CausalInferencePipeline,
)
from utils.dataset import TextDataset, TextImagePairDataset
from utils.misc import set_seed
from hydra import initialize, compose
from hydra.core.global_hydra import GlobalHydra

from demo_utils.memory import gpu, get_cuda_free_memory_gb, DynamicSwapInstaller

from pathlib import Path

config_name = "self_forcing_dmd_vsink"
output_chunk_number = 21
output_latent_frame_number = 21
# output_latent_frame_number = 81
seed = 42
import sys

sys.argv.extend(
    [
        "--output_folder",
        f"outputs/{output_latent_frame_number}-{config_name}-seed{seed}",
        # f"outputs-test/{output_latent_frame_number}-{config_name}-seed{seed}",
        "--config_dir",
        "configs",
        "--config_name",
        config_name,
        "--num_output_frames",
        f"{output_latent_frame_number}",
        "--data_path",
        "prompts/MovieGenVideoBench_extended.txt",
        "--checkpoint_path",
        "./checkpoints/self_forcing_dmd.pt",
        "--use_ema",
        "--seed",
        f"{seed}",
    ]
)
print(f"{sys.argv = }")


parser = argparse.ArgumentParser()
parser.add_argument("--config_dir", type=str, help="Directory to the config file")
parser.add_argument("--config_name", type=str, help="Name to the config file")
parser.add_argument("--checkpoint_path", type=str, help="Path to the checkpoint folder")
parser.add_argument("--data_path", type=str, help="Path to the dataset")
parser.add_argument("--extended_prompt_path", type=str, help="Path to the extended prompt")
parser.add_argument("--output_folder", type=str, help="Output folder")
parser.add_argument(
    "--num_output_frames",
    type=int,
    default=21,
    help="Number of overlap frames between sliding windows",
)
parser.add_argument(
    "--i2v",
    action="store_true",
    help="Whether to perform I2V (or T2V by default)",
)
parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA parameters")
parser.add_argument("--seed", type=int, default=0, help="Random seed")
parser.add_argument(
    "--num_samples",
    type=int,
    default=1,
    help="Number of samples to generate per prompt",
)
args = parser.parse_args()

# Initialize distributed inference
if "LOCAL_RANK" in os.environ:
    dist.init_process_group(backend="nccl")
    local_rank = int(os.environ["LOCAL_RANK"])
    torch.cuda.set_device(local_rank)
    device = torch.device(f"cuda:{local_rank}")
    world_size = dist.get_world_size()
    set_seed(args.seed + local_rank)
else:
    device = torch.device("cuda")
    local_rank = 0
    world_size = 1
    set_seed(args.seed)

print(f"Free VRAM {get_cuda_free_memory_gb(gpu)} GB")
low_memory = get_cuda_free_memory_gb(gpu) < 40

torch.set_grad_enabled(False)

if GlobalHydra.instance().is_initialized():
    GlobalHydra.instance().clear()

with initialize(version_base=None, config_path=args.config_dir):
    config = compose(config_name=args.config_name)
print(f"{config = }")

# Initialize pipeline
if hasattr(config, "denoising_step_list"):
    # Few-step inference
    pipeline = CausalInferencePipeline(config, device=device)
else:
    # Multi-step diffusion inference
    pipeline = CausalDiffusionInferencePipeline(config, device=device)

if args.checkpoint_path:
    state_dict = torch.load(args.checkpoint_path, map_location="cpu")
    pipeline.generator.load_state_dict(
        state_dict["generator" if not args.use_ema else "generator_ema"]
    )

pipeline = pipeline.to(dtype=torch.bfloat16)
if low_memory:
    DynamicSwapInstaller.install_model(pipeline.text_encoder, device=gpu)
else:
    pipeline.text_encoder.to(device=gpu)
pipeline.generator.to(device=gpu)
pipeline.vae.to(device=gpu)


# Create dataset
if args.i2v:
    assert not dist.is_initialized(), "I2V does not support distributed inference yet"
    transform = transforms.Compose(
        [
            transforms.Resize((480, 832)),
            transforms.ToTensor(),
            transforms.Normalize([0.5], [0.5]),
        ]
    )
    dataset = TextImagePairDataset(args.data_path, transform=transform)
else:
    dataset = TextDataset(
        prompt_path=args.data_path,
        extended_prompt_path=args.extended_prompt_path,
    )
num_prompts = len(dataset)
print(f"Number of prompts: {num_prompts}")

if dist.is_initialized():
    sampler = DistributedSampler(dataset, shuffle=False, drop_last=True)
else:
    sampler = SequentialSampler(dataset)
dataloader = DataLoader(dataset, batch_size=1, sampler=sampler, num_workers=0, drop_last=False)

# Create output directory (only on main process to avoid race conditions)
if local_rank == 0:
    os.makedirs(args.output_folder, exist_ok=True)

if dist.is_initialized():
    dist.barrier()


def encode(self, videos: torch.Tensor) -> torch.Tensor:
    device, dtype = videos[0].device, videos[0].dtype
    scale = [
        self.mean.to(device=device, dtype=dtype),
        1.0 / self.std.to(device=device, dtype=dtype),
    ]
    output = [self.model.encode(u.unsqueeze(0), scale).float().squeeze(0) for u in videos]

    output = torch.stack(output, dim=0)
    return output


for i, batch_data in tqdm(enumerate(dataloader), disable=(local_rank != 0)):
    idx = batch_data["idx"].item()

    # For DataLoader batch_size=1, the batch_data is already a single item, but in a batch container
    # Unpack the batch data for convenience
    if isinstance(batch_data, dict):
        batch = batch_data
    elif isinstance(batch_data, list):
        batch = batch_data[0]  # First (and only) item in the batch

    all_video = []
    num_generated_frames = 0  # Number of generated (latent) frames

    set_seed(args.seed)
    if args.i2v:
        # For image-to-video, batch contains image and caption
        prompt = batch["prompts"][0]  # Get caption from batch
        prompts = [prompt] * args.num_samples

        # Process the image
        image = (
            batch["image"]
            .squeeze(0)
            .unsqueeze(0)
            .unsqueeze(2)
            .to(device=device, dtype=torch.bfloat16)
        )

        # Encode the input image as the first latent
        initial_latent = pipeline.vae.encode_to_latent(image).to(
            device=device, dtype=torch.bfloat16
        )
        initial_latent = initial_latent.repeat(args.num_samples, 1, 1, 1, 1)

        sampled_noise = torch.randn(
            [args.num_samples, args.num_output_frames - 1, 16, 60, 104],
            device=device,
            dtype=torch.bfloat16,
        )
    else:
        # For text-to-video, batch is just the text prompt
        prompt = batch["prompts"][0]
        extended_prompt = batch["extended_prompts"][0] if "extended_prompts" in batch else None
        if extended_prompt is not None:
            prompts = [extended_prompt] * args.num_samples
        else:
            prompts = [prompt] * args.num_samples
        initial_latent = None

        sampled_noise = torch.randn(
            [args.num_samples, args.num_output_frames, 16, 60, 104],
            device=device,
            dtype=torch.bfloat16,
        )

    set_seed(args.seed)
    # Generate 81 frames
    video, latents = pipeline.inference(
        noise=sampled_noise,
        text_prompts=prompts,
        return_latents=True,
        initial_latent=initial_latent,
        low_memory=low_memory,
    )
    current_video = rearrange(video, "b t c h w -> b t h w c").cpu()
    all_video.append(current_video)
    num_generated_frames += latents.shape[1]

    # Final output video
    video = 255.0 * torch.cat(all_video, dim=1)

    # Clear VAE cache
    pipeline.vae.model.clear_cache()

    # Save the video if the current prompt is not a dummy prompt
    if idx < num_prompts:
        model = "regular" if not args.use_ema else "ema"
        for seed_idx in range(args.num_samples):
            # All processes save their videos
            output_path = os.path.join(
                args.output_folder,
                f"{idx}-{prompt[:50].replace(' ', '_')}-{seed_idx}_{model}.mp4",
            )
            write_video(output_path, video[seed_idx], fps=16)