cosmos-framework/cosmos_framework/inference/vision.py

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: OpenMDW-1.1

import math
from pathlib import Path
from typing import Literal

import numpy as np
import torch
import torchvision.io
import torchvision.transforms.functional as TF
from PIL import Image

from cosmos_framework.data.vfm.sequence_packing import SequencePlan
from cosmos_framework.data.vfm.utils import VIDEO_RES_SIZE_INFO


def resize_pil_image(image: Image.Image, max_size: int, padding_constant: int) -> Image.Image:
    """Resize a PIL image so the max side length is at most *max_size* and both
    dimensions are divisible by *padding_constant*.

    Args:
        image: Input PIL image.
        max_size: Maximum allowed side length (longest edge will be at most this).
        padding_constant: Both height and width are rounded down to the nearest
            multiple of this value.

    Returns:
        Resized PIL image.
    """
    orig_w, orig_h = image.size
    scale = max_size / max(orig_w, orig_h)
    new_w = int(orig_w * scale)
    new_h = int(orig_h * scale)
    new_w = (new_w // padding_constant) * padding_constant
    new_h = (new_h // padding_constant) * padding_constant
    new_w = max(new_w, padding_constant)
    new_h = max(new_h, padding_constant)
    return image.resize(
        (new_w, new_h),
        Image.LANCZOS,  # type: ignore
    )


def _resize_and_center_crop(frames: torch.Tensor, target_h: int, target_w: int) -> torch.Tensor:
    """Aspect-ratio-preserving resize followed by center crop."""
    orig_h, orig_w = frames.shape[2], frames.shape[3]
    scaling_ratio = max(target_w / orig_w, target_h / orig_h)
    resize_h = int(math.ceil(scaling_ratio * orig_h))
    resize_w = int(math.ceil(scaling_ratio * orig_w))
    frames = TF.resize(frames, [resize_h, resize_w])  # [...,resize_h,resize_w]
    frames = TF.center_crop(frames, [target_h, target_w])  # [...,target_h,target_w]
    return frames


def load_conditioning_image_pixels(image_path: Path, target_h: int, target_w: int) -> torch.Tensor:
    """Load an image as resized/cropped uint8 pixels in ``[3, H, W]``."""
    with image_path.open("rb") as f:
        img = Image.open(f).convert("RGB")
    img_tensor = torch.from_numpy(np.array(img)).permute(2, 0, 1).float().unsqueeze(0)  # [1,3,H,W]
    img_tensor = _resize_and_center_crop(img_tensor, target_h, target_w)  # [1,3,target_h,target_w]
    return img_tensor.squeeze(0).round().clamp(0, 255).to(torch.uint8)  # [3,target_h,target_w]


def load_prompt_upsampling_image(image_path: Path, target_h: int, target_w: int) -> Image.Image:
    """Load an image as resized/cropped RGB PIL pixels for VLM prompt upsampling."""
    img_tensor = load_conditioning_image_pixels(image_path, target_h, target_w)  # [3,target_h,target_w]
    img_array = img_tensor.permute(1, 2, 0).contiguous().cpu().numpy()  # [target_h,target_w,3]
    return Image.fromarray(img_array, mode="RGB")


def load_conditioning_image(image_path: Path, target_h: int, target_w: int) -> torch.Tensor:
    """Load an image as conditioning frames from local or remote path; returns (3, 1, H, W) in [-1, 1]."""
    img_tensor = load_conditioning_image_pixels(image_path, target_h, target_w).float()  # [3,target_h,target_w]
    img_tensor = img_tensor / 127.5 - 1.0  # [3,target_h,target_w]
    return img_tensor.unsqueeze(1)  # [3,1,target_h,target_w]


def load_conditioning_video(
    video_path: Path,
    target_h: int,
    target_w: int,
    max_frames: int,
    *,
    keep: Literal["first", "last"] = "first",
) -> torch.Tensor:
    """Load video frames for conditioning; returns (3, T, H, W) in [-1, 1].

    ``keep`` selects which ``max_frames`` to take when the input is longer.
    """
    frames, _, _ = torchvision.io.read_video(str(video_path), pts_unit="sec")
    frames = frames[-max_frames:] if keep == "last" else frames[:max_frames]  # [T,H,W,3]
    frames_tchw = frames.permute(0, 3, 1, 2).float()  # [T,3,H,W]
    frames_resized = _resize_and_center_crop(frames_tchw, target_h, target_w)  # [T,3,target_h,target_w]
    frames_normalized = frames_resized / 127.5 - 1.0  # [T,3,target_h,target_w]
    return frames_normalized.permute(1, 0, 2, 3)  # [3,T,target_h,target_w]


def pil_to_conditioning_frames(pil_img: Image.Image) -> tuple[torch.Tensor, int, int]:
    """Convert a PIL image to a conditioning tensor in [-1, 1] and return (frames, h, w)."""
    w, h = pil_img.size
    img_tensor = torch.from_numpy(np.array(pil_img)).permute(2, 0, 1).float()  # [3,H,W]
    return (img_tensor / 127.5 - 1.0).unsqueeze(1), h, w  # [3,1,H,W]


def build_conditioned_video_batch(
    conditioning_frames: torch.Tensor,
    condition_frames_vision: list[int],
    w: int,
    h: int,
    num_frames: int,
    fps: float,
    batch_size: int = 1,
) -> dict:
    """Build a data batch with conditioning frames and sequence plans for generation."""
    t_cond = conditioning_frames.shape[1]
    video_data = torch.zeros(1, 3, num_frames, h, w, dtype=torch.bfloat16)  # [1,3,num_frames,h,w]
    t_fill = min(t_cond, num_frames)
    video_data[0, :, :t_fill, :, :] = conditioning_frames[:, :t_fill, :, :].to(dtype=torch.bfloat16)  # [3,t_fill,h,w]
    if t_fill < num_frames:
        video_data[0, :, t_fill:, :, :] = video_data[0, :, t_fill - 1 : t_fill, :, :].expand(
            -1, num_frames - t_fill, -1, -1
        )  # [3,num_frames-t_fill,h,w]
    video_list = [video_data.cuda() for _ in range(batch_size)]  # list of [1,3,num_frames,h,w]
    image_size = [torch.tensor([[h, w, h, w]], dtype=torch.float32).cuda() for _ in range(batch_size)]  # list of [1,4]
    sequence_plans = [
        SequencePlan(has_text=True, has_vision=True, condition_frame_indexes_vision=list(condition_frames_vision))
        for _ in range(batch_size)
    ]
    return {
        "dataset_name": "video_data",
        "video": video_list,
        "image_size": image_size,
        "t5_text_embeddings": torch.randn(batch_size, 512, 1024).cuda().to(dtype=torch.bfloat16),  # [B,512,1024]
        "fps": torch.full((batch_size,), float(fps)).cuda(),  # [B]
        "conditioning_fps": torch.full((batch_size,), float(fps)).cuda(),  # [B]
        "num_frames": torch.full((batch_size,), num_frames).cuda(),  # [B]
        "is_preprocessed": True,
        "sequence_plan": sequence_plans,
    }


def build_image_edit_batch(
    conditioning_frames: torch.Tensor,
    h: int,
    w: int,
    batch_size: int = 1,
) -> dict:
    """Build a data batch for image-to-image editing."""
    image = conditioning_frames.unsqueeze(0).cuda().to(dtype=torch.bfloat16)  # [1,3,1,h,w]
    sequence_plans = [
        SequencePlan(has_text=True, has_vision=True, condition_frame_indexes_vision=[]) for _ in range(batch_size)
    ]
    image_size = torch.tensor([[h, w, h, w]], dtype=torch.float32).cuda()  # [1,4]
    return {
        "dataset_name": "image_data",
        "images": [image, image] * batch_size,
        "image_size": [image_size, image_size] * batch_size,
        "num_frames": [torch.tensor([2], dtype=torch.int64).cuda() for _ in range(batch_size)],  # list of [1]
        "num_vision_items_per_sample": [2 for _ in range(batch_size)],
        "is_preprocessed": True,
        "sequence_plan": sequence_plans,
    }


_VIDEO_EXTENSIONS = {".mp4", ".avi", ".mov", ".mkv", ".webm"}
_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}


def detect_aspect_ratio(width: int, height: int) -> str:
    """Return the closest supported aspect-ratio key for a frame size."""
    aspect_ratios = np.array([16 / 9, 4 / 3, 1, 3 / 4, 9 / 16])
    aspect_ratio_keys = ["16,9", "4,3", "1,1", "3,4", "9,16"]
    current = width / height
    return aspect_ratio_keys[int(np.argmin((aspect_ratios - current) ** 2))]


def read_media_frames(path: Path, max_frames: int) -> tuple[torch.Tensor, float]:
    """Read an image or video into a uint8 tensor of shape (C, T, H, W)."""
    ext = path.suffix.lower()
    if ext in _IMAGE_EXTENSIONS:
        with path.open("rb") as f:
            image = Image.open(f).convert("RGB")
        frames = torch.from_numpy(np.array(image)).permute(2, 0, 1).unsqueeze(1)
        return frames, 1.0
    if ext not in _VIDEO_EXTENSIONS:
        raise ValueError(f"Unsupported media extension: {ext}")
    return _read_video_frames(path, max_frames=max_frames)


def _read_video_frames(path: Path, max_frames: int) -> tuple[torch.Tensor, float]:
    """Read video frames through PyAV to avoid torchvision.io.read_video removal."""

    import av

    try:
        container = av.open(str(path))
    except (OSError, av.error.FFmpegError) as exc:
        raise OSError(f"Failed to open video {path}: {exc}") from exc

    try:
        stream = container.streams.video[0]
        fps = float(stream.average_rate) if stream.average_rate is not None else 24.0
        frames = []
        for frame in container.decode(stream):
            frames.append(torch.from_numpy(frame.to_rgb().to_ndarray()).permute(2, 0, 1))
            if len(frames) >= max_frames:
                break
    except (OSError, av.error.FFmpegError) as exc:
        raise OSError(f"Failed to decode video {path}: {exc}") from exc
    finally:
        container.close()

    if not frames:
        raise ValueError(f"No frames decoded from video {path}")
    return torch.stack(frames, dim=1).to(torch.uint8), fps


def read_and_resize_media(
    path: Path,
    *,
    resolution: str,
    aspect_ratio: str | None,
    max_frames: int,
) -> tuple[torch.Tensor, float, str, tuple[int, int]]:
    """Read an image/video and resize it to the requested resolution bucket."""
    raw_frames, fps = read_media_frames(path, max_frames=max_frames)
    original_hw = (raw_frames.shape[2], raw_frames.shape[3])
    detected_aspect_ratio = detect_aspect_ratio(raw_frames.shape[3], raw_frames.shape[2])
    final_aspect_ratio = aspect_ratio or detected_aspect_ratio
    width, height = VIDEO_RES_SIZE_INFO[resolution][final_aspect_ratio]
    resized = _resize_and_center_crop(raw_frames.permute(1, 0, 2, 3), height, width)
    return resized.permute(1, 0, 2, 3), fps, final_aspect_ratio, original_hw


def uint8_to_normalized_float(tensor: torch.Tensor, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
    """Convert uint8 [0, 255] frames into normalized [-1, 1] frames."""
    return tensor.to(dtype=dtype) / 127.5 - 1.0


def pad_temporal_frames(frames: torch.Tensor, target_frames: int) -> torch.Tensor:
    """Pad a (C, T, H, W) tensor along time using reflection/repeat behavior."""
    num_frames = frames.shape[1]
    if num_frames >= target_frames:
        return frames
    if num_frames == 0:
        raise ValueError("Cannot pad an empty frame tensor.")
    padded = frames
    while padded.shape[1] < target_frames:
        pad_len = min(padded.shape[1] - 1, target_frames - padded.shape[1])
        if pad_len <= 0:
            pad_frame = padded[:, -1:].repeat(1, target_frames - padded.shape[1], 1, 1)
            padded = torch.cat([padded, pad_frame], dim=1)
            break
        padded = torch.cat([padded, padded.flip(dims=[1])[:, :pad_len]], dim=1)
    return padded