stream_inference.py

import os

os.environ["CUDA_VISIBLE_DEVICES"] = "1"

from tqdm import tqdm
from torchvision.io import write_video
from torch.utils.data import DataLoader, SequentialSampler

from stream_inference_wrapper import StreamInferenceWrapper

# from stream_drag_inference_wrapper import StreamDragInferenceWrapper
from utils.dataset import TextDataset
from utils.misc import set_seed
from hydra import initialize, compose
from hydra.core.global_hydra import GlobalHydra

from demo_utils.memory import gpu, get_cuda_free_memory_gb


def main():

    output_block_number = 27

    config_dir = "configs"
    stream_config_name = "self_forcing_dmd_vsink_stream"
    # stream_config_name = "self_forcing_dmd_vsink_stream_drag"

    data_path = "prompts/MovieGenVideoBench_extended.txt"

    seed = 42
    set_seed(seed)

    output_folder = "outputs-stream"
    output_folder = f"{output_folder}/blk{output_block_number}-{stream_config_name}-seed{seed}"

    print(f"Free VRAM {get_cuda_free_memory_gb(gpu)} GB")
    # low_memory = get_cuda_free_memory_gb(gpu) < 40

    # Create dataset
    dataset = TextDataset(prompt_path=data_path)
    num_prompts = len(dataset)
    print(f"Number of prompts: {num_prompts}")

    sampler = SequentialSampler(dataset)
    dataloader = DataLoader(dataset, batch_size=1, sampler=sampler, num_workers=0, drop_last=False)

    os.makedirs(output_folder, exist_ok=True)

    if GlobalHydra.instance().is_initialized():
        GlobalHydra.instance().clear()

    with initialize(version_base=None, config_path=config_dir):
        stream_config = compose(config_name=stream_config_name)
    print(f"{stream_config = }")

    stream_inference = StreamInferenceWrapper(
        stream_model_config=stream_config,
        checkpoint_path="./checkpoints/self_forcing_dmd.pt",
        total_generate_block_number=output_block_number,
        use_ema=True,
        seed=seed,
    )

    for i, batch_data in tqdm(enumerate(dataloader)):
        idx = batch_data["idx"].item()
        print(f"{idx = }")

        # For DataLoader batch_size=1, the batch_data is already a single item, but in a batch container
        # Unpack the batch data for convenience
        if isinstance(batch_data, dict):
            batch = batch_data
        elif isinstance(batch_data, list):
            batch = batch_data[0]  # First (and only) item in the batch

        # For text-to-video, batch is just the text prompt
        prompt = batch["prompts"][0]
        print(f"{prompt = }")
        extended_prompt = batch["extended_prompts"][0] if "extended_prompts" in batch else None
        print(f"{extended_prompt = }")

        set_seed(seed)
        stream_inference.reset()

        current_block_index = 0
        block_step = 3
        while current_block_index < output_block_number:
            end_block_index = current_block_index + block_step
            all_video, current_video = stream_inference.inference(
                start_block_index=current_block_index,
                end_block_index=end_block_index,
                prompt=prompt,
            )

            # Save the video if the current prompt is not a dummy prompt
            if idx < num_prompts:
                current_video_output_path = os.path.join(
                    output_folder,
                    f"{idx:04d}-{prompt[:50].replace(' ', '_')}-{current_block_index:02d}-{end_block_index:02d}.mp4",
                )
                write_video(current_video_output_path, current_video, fps=16)
                all_video_output_path = os.path.join(
                    output_folder,
                    f"{idx:04d}-{prompt[:50].replace(' ', '_')}-{0:02d}-{end_block_index:02d}.mp4",
                )
                write_video(all_video_output_path, all_video, fps=16)

            current_block_index = end_block_index


if __name__ == "__main__":
    main()