src/infer.py

import os
import argparse
import numpy as np
import torch
import imageio
from diffusers import FlowMatchEulerDiscreteScheduler
from omegaconf import OmegaConf
from PIL import Image
from transformers import AutoTokenizer
import scipy
import cv2
from glob import glob
import torch.distributed as dist
from videox_fun.dist import set_multi_gpus_devices
from videox_fun.models import AutoencoderKLWan, WanT5EncoderModel, VaceWanModel, load_lightx2v_vae
from videox_fun.data.remove_dataset import orientation_aware_size
from videox_fun.pipeline import RemovePipeline
from videox_fun.utils.fp8_optimization import (
    convert_model_weight_to_float8,
    replace_parameters_by_name,
    convert_weight_dtype_wrapper,
)
from videox_fun.utils.lora_utils import merge_lora
from videox_fun.utils.utils import save_videos_grid, filter_kwargs


def load_patch_safetensors(path):
    list_tensors = glob(path + "/*.safetensors")
    all = {}
    for x in list_tensors:
        from safetensors.torch import load_file

        tmp = load_file(x)
        all.update(tmp)
    return all


def parse_args():
    parser = argparse.ArgumentParser(description="WanFun Video Editing Script")

    # GPU and memory configuration
    parser.add_argument(
        "--gpu_memory_mode",
        type=str,
        default="model_full_load",
        choices=["model_full_load", "model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"],
        help="GPU memory optimization mode",
    )
    parser.add_argument("--ulysses_degree", type=int, default=1, help="Ulysses degree for multi-GPU configuration")
    parser.add_argument("--ring_degree", type=int, default=1, help="Ring degree for multi-GPU configuration")

    # Model paths
    parser.add_argument(
        "--config_path", type=str, default="config/wan2.1/wan_civitai.yaml", help="Path to model configuration file"
    )
    parser.add_argument("--model_name", type=str, default="models/Wan2.1-VACE-1.3B", help="Path to pretrained model")
    parser.add_argument(
        "--common_model_name",
        type=str,
        default=None,
        help="Optional directory containing the shared VAE, text encoder, and tokenizer. Defaults to --model_name.",
    )

    # VAE configuration. The Remove LoRAs are trained against the pruned LightVAE latent
    # distribution, so inference must use the same LightVAE by default. Using the full
    # Wan VAE both mismatches the latents and blows up decode-time GPU memory (OOM).
    parser.add_argument(
        "--lightvae_path",
        type=str,
        default="/data1/yfu/1_CVPR2025_Remove/10_transsion/6_0713/lightvaew2_1.pth",
        help="Path to the LightVAE checkpoint used during training. Loaded by default.",
    )
    parser.add_argument("--lightvae_pruning_rate", type=float, default=0.75, help="LightVAE channel pruning rate")
    parser.add_argument("--lightvae_dim", type=int, default=96, help="LightVAE base model dim")
    parser.add_argument(
        "--use_full_vae",
        action="store_true",
        help="Force the full official Wan VAE instead of LightVAE (higher VRAM; only for debugging).",
    )

    # Generation parameters
    parser.add_argument(
        "--sample_size",
        type=str,
        default="480,832",
        help="Orientation-aware canvas as 'height,width' for landscape; swapped for portrait. Matches training.",
    )
    parser.add_argument("--video_length", type=int, default=81, help="Length of generated video in frames")
    parser.add_argument("--fps", type=int, default=16, help="Frames per second for output video")
    parser.add_argument(
        "--weight_dtype",
        type=str,
        default="bfloat16",
        choices=["float16", "bfloat16"],
        help="Data type for model weights",
    )

    # Prompt and generation settings
    parser.add_argument(
        "--prompt",
        type=str,
        default="Remove the target and fill the content appropriately",
        help="Text prompt for generation",
    )
    parser.add_argument(
        "--negative_prompt",
        type=str,
        default="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
        help="Negative text prompt",
    )
    parser.add_argument(
        "--guidance_scale",
        type=float,
        default=1.0,
        help="Guidance scale. 1.0 disables CFG (single forward per step); >1.0 enables CFG (2x forwards).",
    )
    parser.add_argument("--seed", type=int, default=43, help="Random seed for reproducibility")
    parser.add_argument("--context_scale", type=float, default=1.0, help="Context scale for vace control")
    parser.add_argument("--dilation", type=int, default=6, help="Dilation for inp mask (only for inpaint mode)")

    # Parameters for Remove
    parser.add_argument(
        "--lora_path",
        type=str,
        default=["models/remove_model_stage1.safetensors", "models/remove_model_stage2.safetensors"],
        nargs="+",
        help="Optional path to LoRA checkpoint",
    )
    parser.add_argument(
        "--lora_weight", type=float, default=[1.0, 1.0], nargs="+", help="Weight for LoRA model if used"
    )
    parser.add_argument(
        "--skip_lora",
        action="store_true",
        help="Do not load external LoRAs. Required when --model_name already contains merged LoRAs.",
    )
    # Single-pair inference
    parser.add_argument(
        "--input_video",
        type=str,
        default=None,
        help="Path to a single input video for editing",
    )
    parser.add_argument(
        "--input_mask_video",
        type=str,
        default=None,
        help="Path to a single mask video for editing",
    )

    # Directory batch inference
    parser.add_argument(
        "--input_dir",
        type=str,
        default=None,
        help="Directory containing input videos",
    )
    parser.add_argument(
        "--input_mask_dir",
        type=str,
        default=None,
        help="Directory containing mask videos with exactly matching filenames",
    )
    parser.add_argument("--num_inference_steps", type=int, default=4, help="Number of inference steps")
    parser.add_argument("--dmd_steps", type=int, choices=[1, 2], default=None)
    parser.add_argument("--shard_index", type=int, default=0)
    parser.add_argument("--num_shards", type=int, default=1)
    parser.add_argument("--save_dir", type=str, default="samples/Remove", help="Directory to save generated videos")

    return parser.parse_args()


def process_video(
    input_video_path,
    input_mask_video_path,
    video_length,
    sample_size,
    dilation=0,
):
    """Process input video and mask for editing"""
    if input_video_path is not None:
        cap = cv2.VideoCapture(input_video_path)
        frames = []
        while cap.isOpened():
            ret, frame = cap.read()
            if not ret:
                break
            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            frames.append(Image.fromarray(frame))
        cap.release()

        frames = frames[:video_length]
        if len(frames) < video_length:
            frames += [frames[-1]] * (video_length - len(frames))

        resized_frames = [frame.resize([sample_size[1], sample_size[0]]) for frame in frames]

        # Keep the raw RGB frames (uint8, [T,H,W,3]) for the cat/overlay visualization.
        raw_frames = np.stack([np.array(frame) for frame in resized_frames]).astype(np.uint8)

        input_video = (
            torch.stack([torch.from_numpy(np.array(frame)).permute(2, 0, 1) for frame in resized_frames])
            .permute(1, 0, 2, 3)
            .unsqueeze(0)
        )  # [1, C, T, H, W]

    else:
        input_video = torch.zeros((1, 3, video_length, sample_size[0], sample_size[1])).float()
        raw_frames = np.zeros((video_length, sample_size[0], sample_size[1], 3), dtype=np.uint8)
    if input_mask_video_path is not None:
        mask_cap = cv2.VideoCapture(input_mask_video_path)
        mask_frames = []
        while mask_cap.isOpened():
            ret, frame = mask_cap.read()
            if not ret:
                break
            if len(frame.shape) == 3:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            _, mask = cv2.threshold(frame, 127, 255, cv2.THRESH_BINARY)
            if dilation > 0:
                mask_np = (mask > 0).astype(np.uint8)
                mask = scipy.ndimage.binary_dilation(mask_np, iterations=dilation).astype(np.uint8) * 255
            mask_frames.append(mask)
        mask_cap.release()

        mask_frames = mask_frames[:video_length]
        if len(mask_frames) < video_length:
            mask_frames += [mask_frames[-1]] * (video_length - len(mask_frames))

        resized_masks = [Image.fromarray(mask).resize([sample_size[1], sample_size[0]]) for mask in mask_frames]

        # Keep the binary mask (uint8, [T,H,W], 0/255) for the cat/overlay visualization.
        raw_masks = np.stack([np.array(mask) for mask in resized_masks]).astype(np.uint8)

        input_video_mask = (
            torch.stack([torch.from_numpy(np.array(mask)) for mask in resized_masks]).unsqueeze(0).unsqueeze(0) / 255.0
        )  # [1, 1, T, H, W]

    else:
        input_video_mask = torch.ones((1, 1, video_length, sample_size[0], sample_size[1])).float()
        raw_masks = np.full((video_length, sample_size[0], sample_size[1]), 255, dtype=np.uint8)

    if input_video_path is not None and input_video is not None:
        input_video = input_video * (torch.tile(input_video_mask, [1, 3, 1, 1, 1]) < 0.5) + (128.0) * (
            torch.tile(input_video_mask, [1, 3, 1, 1, 1]) >= 0.5
        )
    input_video = input_video.div_(127.5).sub_(1.0)

    return input_video, input_video_mask, raw_frames, raw_masks


def process_single_task(
    pipeline,
    args,
    input_video_path,
    input_mask_video_path,
    prompt,
):
    """Process a single video editing task"""
    if input_video_path is not None:
        # Get video resolution
        cap = cv2.VideoCapture(input_video_path)
        width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
        cap.release()

        # Match training: resize to a fixed orientation-aware canvas rather than an
        # aspect-ratio-preserving max-area box. Training always feeds landscape
        # (height,width) or its swap for portrait, so inference must do the same or
        # the model sees an out-of-distribution resolution.
        base_height, base_width = [int(value) for value in args.sample_size.split(",")]
        new_height, new_width = orientation_aware_size(width, height, base_height, base_width)
        sample_size = [new_height, new_width]
    else:
        sample_size = [int(args.sample_size.split(",")[0]), int(args.sample_size.split(",")[1])]

    generator = torch.Generator(device=pipeline.device).manual_seed(args.seed)

    with torch.no_grad():
        video_length = (
            int(
                (args.video_length - 1)
                // pipeline.vae.config.temporal_compression_ratio
                * pipeline.vae.config.temporal_compression_ratio
            )
            + 1
            if args.video_length != 1
            else 1
        )

        # Process video and mask
        (
            input_video,
            input_video_mask,
            raw_frames,
            raw_masks,
        ) = process_video(
            input_video_path,
            input_mask_video_path,
            video_length=video_length,
            sample_size=sample_size,
            dilation=args.dilation,
        )

        # Generate edited video
        sample = pipeline(
            prompt,
            negative_prompt=args.negative_prompt,
            height=sample_size[0],
            width=sample_size[1],
            generator=generator,
            guidance_scale=args.guidance_scale,
            num_inference_steps=args.num_inference_steps,
            video=input_video,
            mask_video=input_video_mask,
            context_scale=args.context_scale,
        ).videos
        if not torch.isfinite(sample).all():
            raise FloatingPointError('Inference generated NaN or infinite pixel values')
        timing = pipeline.last_timing
        print(
            "[Timing] "
            f"vae_encode={timing.get('vae_encode_seconds', float('nan')):.3f}s, "
            f"condition_prepare={timing.get('condition_prepare_seconds', float('nan')):.3f}s, "
            f"denoise={timing.get('denoise_seconds', float('nan')):.3f}s, "
            f"vae_decode={timing.get('vae_decode_seconds', float('nan')):.3f}s, "
            f"decode_postprocess={timing.get('decode_postprocess_seconds', float('nan')):.3f}s, "
            f"vae_total={timing.get('vae_total_seconds', float('nan')):.3f}s, "
            f"pipeline_total={timing.get('pipeline_total_seconds', float('nan')):.3f}s"
        )

    return sample, video_length, raw_frames, raw_masks


def sample_to_uint8_frames(sample):
    """Convert a pipeline sample [1,3,T,H,W] in [0,1] to uint8 RGB frames [T,H,W,3]."""
    video = sample[0].detach().float().clamp(0, 1).cpu()  # [3,T,H,W]
    frames = video.permute(1, 2, 3, 0).mul(255).round().clamp(0, 255).to(torch.uint8).numpy()
    return frames  # [T,H,W,3]


def overlay_mask_on_frames(
    frames,
    masks,
    fill_color=(255, 255, 0),
    edge_color=(255, 0, 0),
    fill_alpha=0.3,
    edge_thickness=2,
):
    """Annotate the masked region on each frame.

    The region is filled with a highly transparent yellow (low alpha) and outlined
    with an opaque red edge, so the target area is clearly marked without hiding
    the underlying content.

    Args:
        frames: uint8 RGB array [T,H,W,3].
        masks: uint8 array [T,H,W] with 0/255 (or any >0 as foreground).
        fill_color: RGB of the transparent fill (default yellow).
        edge_color: RGB of the opaque outline (default red).
        fill_alpha: opacity of the fill in [0,1]; small means highly transparent.
        edge_thickness: outline thickness in pixels.
    """
    fill = np.array(fill_color, dtype=np.float32)
    edge = np.array(edge_color, dtype=np.uint8)
    erode_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    out = []
    for frame, mask in zip(frames, masks):
        frame = frame.astype(np.float32)
        binary = (mask > 127).astype(np.uint8)
        if binary.any():
            # Highly transparent yellow fill inside the region.
            selected = binary.astype(bool)
            frame[selected] = (1.0 - fill_alpha) * frame[selected] + fill_alpha * fill
            # Opaque red outline along the region boundary.
            eroded = cv2.erode(binary, erode_kernel, iterations=edge_thickness)
            border = (binary - eroded).astype(bool)
            frame_uint8 = frame.clip(0, 255).astype(np.uint8)
            frame_uint8[border] = edge
        else:
            frame_uint8 = frame.clip(0, 255).astype(np.uint8)
        out.append(frame_uint8)
    return np.stack(out)


def save_video_frames(frames, path, fps):
    """Write uint8 RGB frames [T,H,W,3] to an mp4 via imageio."""
    os.makedirs(os.path.dirname(path), exist_ok=True)
    imageio.mimsave(path, list(frames), fps=fps)


def save_results(sample, args, video_length, fps, task_name=None, raw_frames=None, raw_masks=None):
    """Save the generated results.

    Produces two artifacts:
      - meta/<name>.mp4 : the generated result on its own.
      - cat/<name>.mp4  : the input video with the mask region annotated (highly
        transparent yellow fill + red edge), horizontally concatenated with the
        generated result.
    """
    prefix = task_name
    meta_dir = os.path.join(args.save_dir, "meta")
    cat_dir = os.path.join(args.save_dir, "cat")
    os.makedirs(meta_dir, exist_ok=True)

    if video_length == 1:
        # Single-frame (image) output.
        meta_path = os.path.join(meta_dir, prefix + ".png")
        image = sample[0, :, 0]
        image = image.transpose(0, 1).transpose(1, 2)
        gen_image = (image * 255).numpy().astype(np.uint8)
        Image.fromarray(gen_image).save(meta_path)
        if raw_frames is not None and raw_masks is not None:
            os.makedirs(cat_dir, exist_ok=True)
            overlay = overlay_mask_on_frames(raw_frames[:1], raw_masks[:1])[0]
            cat_image = np.concatenate([overlay, gen_image], axis=1)
            Image.fromarray(cat_image).save(os.path.join(cat_dir, prefix + ".png"))
        return

    # Video output.
    gen_frames = sample_to_uint8_frames(sample)  # [T,H,W,3]
    meta_path = os.path.join(meta_dir, prefix + ".mp4")
    save_video_frames(gen_frames, meta_path, fps)

    if raw_frames is not None and raw_masks is not None:
        os.makedirs(cat_dir, exist_ok=True)
        frame_count = min(len(raw_frames), len(raw_masks), len(gen_frames))
        overlay = overlay_mask_on_frames(raw_frames[:frame_count], raw_masks[:frame_count])
        cat_frames = np.concatenate([overlay, gen_frames[:frame_count]], axis=2)  # concat along width
        save_video_frames(cat_frames, os.path.join(cat_dir, prefix + ".mp4"), fps)


def main():
    args = parse_args()
    if not 0 <= args.shard_index < args.num_shards:
        raise ValueError('Require 0 <= shard_index < num_shards')
    if args.dmd_steps is not None:
        if args.guidance_scale != 1.0:
            raise ValueError('DMD student is distilled at CFG=1; inference must use CFG=1')
        args.num_inference_steps = args.dmd_steps

    # Validate arguments
    single_mode = args.input_video is not None or args.input_mask_video is not None
    dir_mode = args.input_dir is not None or args.input_mask_dir is not None

    if single_mode and dir_mode:
        raise ValueError(
            "Do not mix single-file mode and directory mode. "
            "Use either --input_video + --input_mask_video OR --input_dir + --input_mask_dir."
        )

    if single_mode:
        if args.input_video is None or args.input_mask_video is None:
            raise ValueError("Single-file mode requires both --input_video and --input_mask_video")
        if not os.path.isfile(args.input_video):
            raise FileNotFoundError(f"Input video not found: {args.input_video}")
        if not os.path.isfile(args.input_mask_video):
            raise FileNotFoundError(f"Input mask video not found: {args.input_mask_video}")
    elif dir_mode:
        if args.input_dir is None or args.input_mask_dir is None:
            raise ValueError("Directory mode requires both --input_dir and --input_mask_dir")
        if not os.path.isdir(args.input_dir):
            raise NotADirectoryError(f"Input directory not found: {args.input_dir}")
        if not os.path.isdir(args.input_mask_dir):
            raise NotADirectoryError(f"Mask directory not found: {args.input_mask_dir}")
    else:
        raise ValueError(
            "Please provide either --input_video + --input_mask_video "
            "or --input_dir + --input_mask_dir"
        )

    # Convert weight dtype
    weight_dtype = torch.bfloat16 if args.weight_dtype == "bfloat16" else torch.float16
    device = set_multi_gpus_devices(args.ulysses_degree, args.ring_degree)
    config = OmegaConf.load(args.config_path)
    common_model_name = args.common_model_name or args.model_name

    # Initialize transformer
    from load_vace import load_vace, load_text_encoder
    transformer = load_vace(
        os.path.join(
            args.model_name, config["transformer_additional_kwargs"].get("transformer_subpath", "transformer")
        ),
        OmegaConf.to_container(config["transformer_additional_kwargs"]),
        weight_dtype,
    )

    # Get Vae. Default to the pruned LightVAE the Remove LoRAs were trained with; only
    # fall back to the full Wan VAE when explicitly requested via --use_full_vae.
    if args.use_full_vae:
        print("[INFO] Using full official Wan VAE (--use_full_vae). This needs more GPU memory.")
        vae = AutoencoderKLWan.from_pretrained(
            os.path.join(common_model_name, config["vae_kwargs"].get("vae_subpath", "vae")),
            additional_kwargs=OmegaConf.to_container(config["vae_kwargs"]),
        ).to(weight_dtype)
    else:
        if not args.lightvae_path or not os.path.isfile(args.lightvae_path):
            raise FileNotFoundError(
                f"LightVAE checkpoint not found: {args.lightvae_path}. "
                "Pass a valid --lightvae_path, or use --use_full_vae to force the full VAE."
            )
        print(
            f"[INFO] Loading LightVAE from {args.lightvae_path} "
            f"(pruning_rate={args.lightvae_pruning_rate}, dim={args.lightvae_dim})"
        )
        vae = load_lightx2v_vae(
            args.lightvae_path,
            pruning_rate=args.lightvae_pruning_rate,
            model_dim=args.lightvae_dim,
            torch_dtype=weight_dtype,
            device="cpu",
        )

    # Get Tokenizer
    tokenizer = AutoTokenizer.from_pretrained(
        os.path.join(common_model_name, config["text_encoder_kwargs"].get("tokenizer_subpath", "tokenizer")),
    )

    # Get Text encoder
    text_encoder = load_text_encoder(
        os.path.join(common_model_name, config["text_encoder_kwargs"].get("text_encoder_subpath", "text_encoder")),
        OmegaConf.to_container(config["text_encoder_kwargs"]),weight_dtype,
    )
    text_encoder = text_encoder.eval()

    # Get Scheduler
    Choosen_Scheduler = {
        "Flow": FlowMatchEulerDiscreteScheduler,
    }["Flow"]
    scheduler = Choosen_Scheduler(
        **filter_kwargs(Choosen_Scheduler, OmegaConf.to_container(config["scheduler_kwargs"]))
    )
    if args.dmd_steps is not None:
        from dmd_scheduler import DMDFlowScheduler
        scheduler = DMDFlowScheduler.from_config(scheduler.config)
        scheduler.dmd_steps = args.dmd_steps

    # Get Pipeline
    pipeline = RemovePipeline(
        transformer=transformer,
        vae=vae,
        tokenizer=tokenizer,
        text_encoder=text_encoder,
        scheduler=scheduler,
    )

    if args.ulysses_degree > 1 or args.ring_degree > 1:
        transformer.enable_multi_gpus_inference()

    if args.gpu_memory_mode == "sequential_cpu_offload":
        replace_parameters_by_name(
            transformer,
            [
                "modulation",
            ],
            device=device,
        )
        transformer.freqs = transformer.freqs.to(device=device)
        pipeline.enable_sequential_cpu_offload(device=device)
    elif args.gpu_memory_mode == "model_cpu_offload_and_qfloat8":
        convert_model_weight_to_float8(
            transformer,
            exclude_module_name=[
                "modulation",
            ],
        )
        convert_weight_dtype_wrapper(transformer, weight_dtype)
        pipeline.enable_model_cpu_offload(device=device)
    elif args.gpu_memory_mode == "model_cpu_offload":
        pipeline.enable_model_cpu_offload(device=device)
    else:
        pipeline.to(device=device)

    if args.skip_lora:
        print("[INFO] Skipping external LoRA loading; using transformer weights from --model_name directly.")
    elif args.lora_path is not None:
        if len(args.lora_weight) != len(args.lora_path):
            args.lora_weight = [args.lora_weight[0]] * len(args.lora_path)
        for lora_path, lora_weight in zip(args.lora_path, args.lora_weight):
            print(f"[INFO] Loading LoRA: {lora_path}, weight: {lora_weight}")
            pipeline = merge_lora(pipeline, lora_path, lora_weight)

    if not os.path.exists(args.save_dir):
        os.makedirs(args.save_dir, exist_ok=True)

    # Inference
    if args.input_dir is not None:
        # Batch directory inference. Model/pipeline is loaded only once above.
        valid_exts = {".mp4", ".avi", ".mov", ".mkv", ".webm"}

        input_filenames = sorted(
            filename
            for filename in os.listdir(args.input_dir)
            if os.path.isfile(os.path.join(args.input_dir, filename))
            and os.path.splitext(filename)[1].lower() in valid_exts
        )

        pairs = []
        for filename in input_filenames:
            input_video_path = os.path.join(args.input_dir, filename)
            input_mask_video_path = os.path.join(args.input_mask_dir, filename)

            # Exact filename matching: abc.mp4 <-> abc.mp4
            if os.path.isfile(input_mask_video_path):
                pairs.append((filename, input_video_path, input_mask_video_path))
            else:
                if not dist.is_initialized() or dist.get_rank() == 0:
                    print(f"[WARNING] No matching mask for {filename}; skipping.")

        if len(pairs) == 0:
            raise RuntimeError(
                f"No valid video-mask pairs found. "
                f"input_dir={args.input_dir}, input_mask_dir={args.input_mask_dir}"
            )

        pairs = pairs[args.shard_index::args.num_shards]

        if not dist.is_initialized() or dist.get_rank() == 0:
            print(f"[INFO] Found {len(pairs)} valid video-mask pairs.")

        for idx, (filename, input_video_path, input_mask_video_path) in enumerate(pairs, start=1):
            if not dist.is_initialized() or dist.get_rank() == 0:
                print("\n" + "=" * 80)
                print(f"[INFO] Processing {idx}/{len(pairs)}: {filename}")
                print(f"[INFO] Video: {input_video_path}")
                print(f"[INFO] Mask : {input_mask_video_path}")
                print("=" * 80)

            sample, video_length, raw_frames, raw_masks = process_single_task(
                pipeline,
                args,
                input_video_path,
                input_mask_video_path,
                args.prompt,
            )

            video_basename = os.path.splitext(filename)[0]
            if not dist.is_initialized() or dist.get_rank() == 0:
                save_results(
                    sample,
                    args,
                    video_length,
                    args.fps,
                    task_name=video_basename,
                    raw_frames=raw_frames,
                    raw_masks=raw_masks,
                )
                print(
                    f"[INFO] Saved: meta={os.path.join(args.save_dir, 'meta', video_basename + '.mp4')}, "
                    f"cat={os.path.join(args.save_dir, 'cat', video_basename + '.mp4')}"
                )

            # Release per-sample tensors before the next pair.
            del sample
            if torch.cuda.is_available():
                torch.cuda.empty_cache()

        if not dist.is_initialized() or dist.get_rank() == 0:
            print(f"\n[INFO] Finished all {len(pairs)} pairs. Results: {args.save_dir}")

    else:
        # Original single-pair inference
        sample, video_length, raw_frames, raw_masks = process_single_task(
            pipeline,
            args,
            args.input_video,
            args.input_mask_video,
            args.prompt,
        )

        video_basename = os.path.splitext(os.path.basename(args.input_video))[0]
        if not dist.is_initialized() or dist.get_rank() == 0:
            save_results(
                sample,
                args,
                video_length,
                args.fps,
                task_name=video_basename,
                raw_frames=raw_frames,
                raw_masks=raw_masks,
            )


if __name__ == "__main__":
    main()