diffsynth/pipelines/i2v_vae_padding.py

"""
I2V VAE 未知帧像素填充策略（仅推理/实验 opt-in）

Wan 官方 image2video.py 对未知帧使用 ``torch.zeros``，张量值在 **[-1, 1]**
归一化空间里为 **0**，对应 RGB 约 **128 的中灰**，不是 uint8 意义的黑 (0→-1)。

本模块供 ``WanVideoUnit_ImageEmbedderVAE`` 在 ``pipe.i2v_vae_unknown_frame_fill``
被设置时选用；未设置时行为与上游完全一致。
"""
from __future__ import annotations

from typing import Optional, Tuple

import torch

# ImageNet 均值 (RGB, [0,1])，映射到 preprocess_image 的 [-1, 1]
_IMAGENET_MEAN_01: Tuple[float, float, float] = (0.485, 0.456, 0.406)

FILL_MODE_ALIASES = {
    None: "official_zero",
    "official": "official_zero",
    "zero": "official_zero",
    "official_zero": "official_zero",
    "black": "black",
    "gray": "official_zero",
    "mid_gray": "official_zero",
    "first_frame_mean": "first_frame_mean",
    "imagenet_mean": "imagenet_mean",
}

FILL_MODE_DESCRIPTIONS = {
    "official_zero": "官方 Wan: torch.zeros → 归一化空间 0 ≈ RGB128 中灰",
    "black": "归一化空间 -1 ≈ RGB0 真黑（常被误称为 zero pixel）",
    "first_frame_mean": "首帧逐通道均值铺满未知帧",
    "imagenet_mean": "ImageNet RGB 均值映射到 [-1,1]",
}


def normalize_fill_mode(mode: Optional[str]) -> str:
    if mode not in FILL_MODE_ALIASES:
        known = sorted({k for k in FILL_MODE_ALIASES if k is not None})
        raise ValueError(f"Unknown i2v_vae_unknown_frame_fill={mode!r}. Known: {known}")
    return FILL_MODE_ALIASES[mode]


def resolve_padding_mode(pipe) -> Optional[str]:
    return getattr(pipe, "i2v_vae_unknown_frame_fill", None)


def _rgb01_to_normalized(v: float) -> float:
    return v * 2.0 - 1.0


def make_vae_unknown_frames(
    num_pad_frames: int,
    height: int,
    width: int,
    *,
    device: torch.device,
    dtype: torch.dtype,
    mode: Optional[str] = None,
    first_frame_chw: Optional[torch.Tensor] = None,
) -> torch.Tensor:
    """
    构造 VAE encode 用的未知帧像素块 ``[3, T_pad, H, W]``（与首帧同 dtype/device）。

    Args:
        num_pad_frames: 待填充帧数 (通常 num_frames-1 或 num_frames-2)
        first_frame_chw: 已 preprocess 的首帧 ``[3, H, W]``，``first_frame_mean`` 需要
    """
    if num_pad_frames <= 0:
        return torch.empty(3, 0, height, width, device=device, dtype=dtype)

    key = normalize_fill_mode(mode)
    shape = (3, num_pad_frames, height, width)

    if key == "official_zero":
        return torch.zeros(shape, device=device, dtype=dtype)

    if key == "black":
        return torch.full(shape, -1.0, device=device, dtype=dtype)

    if key == "first_frame_mean":
        if first_frame_chw is None:
            raise ValueError("first_frame_mean requires first_frame_chw")
        fm = first_frame_chw
        if fm.dim() != 3 or fm.shape[0] != 3:
            raise ValueError(f"first_frame_chw must be [3,H,W], got {tuple(fm.shape)}")
        mean = fm.reshape(3, -1).mean(dim=1).view(3, 1, 1, 1)
        return mean.expand(shape).contiguous()

    if key == "imagenet_mean":
        vals = torch.tensor(
            [_rgb01_to_normalized(v) for v in _IMAGENET_MEAN_01],
            device=device,
            dtype=dtype,
        )
        return vals.view(3, 1, 1, 1).expand(shape).contiguous()

    raise RuntimeError(f"Unhandled fill mode: {key}")