diffusers/modular_pipelines/cosmos/decoders.py

import torch

from ...configuration_utils import FrozenDict
from ...models.autoencoders.autoencoder_cosmos3_audio import Cosmos3AVAEAudioTokenizer
from ...models.autoencoders.autoencoder_kl_wan import AutoencoderKLWan
from ...utils import logging
from ...video_processor import VideoProcessor
from ..modular_pipeline import ModularPipelineBlocks, PipelineState
from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam
from .modular_pipeline import Cosmos3OmniModularPipeline


logger = logging.get_logger(__name__)


class Cosmos3VideoDecodeStep(ModularPipelineBlocks):
    model_name = "cosmos3-omni"

    @property
    def description(self) -> str:
        return "Decodes denoised vision latents into video outputs."

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [
            ComponentSpec("vae", AutoencoderKLWan),
            ComponentSpec(
                "video_processor",
                VideoProcessor,
                config=FrozenDict({"vae_scale_factor": 16, "resample": "bilinear"}),
                default_creation_method="from_config",
            ),
        ]

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam.template("latents", required=True, description="Denoised vision latents to decode."),
            InputParam.template("output_type", default="pil"),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [OutputParam.template("videos")]

    @torch.no_grad()
    def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)

        device = components._execution_device

        if block_state.output_type == "latent":
            block_state.videos = block_state.latents
        else:
            in_dtype = block_state.latents.dtype
            vae_dtype = components.vae.dtype
            mean = components._vae_latents_mean.to(device=block_state.latents.device, dtype=vae_dtype)
            inv_std = components._vae_latents_inv_std.to(device=block_state.latents.device, dtype=vae_dtype)
            z_raw = block_state.latents.to(vae_dtype) / inv_std.view(1, -1, 1, 1, 1) + mean.view(1, -1, 1, 1, 1)
            decoded = components.vae.decode(z_raw).sample.to(in_dtype)
            block_state.videos = components.video_processor.postprocess_video(
                decoded, output_type=block_state.output_type
            )[0]

        if components.requires_safety_checker and block_state.output_type != "latent":
            if getattr(components, "safety_checker", None) is None:
                raise ValueError(
                    "Cosmos3 requires a safety checker by default. Call `pipe.enable_safety_checker()` to load it "
                    "(or pass your own), or opt out explicitly with `pipe.disable_safety_checker()`."
                )
            block_state.videos = components._apply_video_safety_check(
                block_state.videos, output_type=block_state.output_type, device=device
            )

        self.set_block_state(state, block_state)
        return components, state


class Cosmos3SoundDecodeStep(ModularPipelineBlocks):
    model_name = "cosmos3-omni"

    @property
    def description(self) -> str:
        return "Decodes sound latents into waveform output."

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [ComponentSpec("sound_tokenizer", Cosmos3AVAEAudioTokenizer)]

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(
                name="sound_latents",
                type_hint=torch.Tensor,
                required=True,
                description="Denoised sound latents to decode.",
            )
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [
            OutputParam("sound", type_hint=torch.Tensor, description="Generated waveform."),
            OutputParam("sampling_rate", type_hint=int, description="Sample rate of the generated waveform in Hz."),
        ]

    @torch.no_grad()
    def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)
        if components.sound_tokenizer is None:
            raise ValueError("Sound decoding requires a sound-capable checkpoint with a sound_tokenizer.")
        block_state.sound = components.decode_sound(block_state.sound_latents)
        block_state.sampling_rate = int(components.sound_tokenizer.config.sampling_rate)
        self.set_block_state(state, block_state)
        return components, state


class Cosmos3TransferDecodeChunkStep(ModularPipelineBlocks):
    model_name = "cosmos3-omni"

    @property
    def description(self) -> str:
        return (
            "Decodes one transfer chunk's latents to pixels (float32, clamped to [-1, 1]), records it as the "
            "autoregressive seed for the next chunk, and appends it to output_chunks (dropping the overlap that "
            "later chunks share with the previous chunk's conditioning frames)."
        )

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [ComponentSpec("vae", AutoencoderKLWan)]

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(
                name="latents",
                type_hint=torch.Tensor,
                required=True,
                description="Denoised target latents for this chunk.",
            ),
            InputParam(name="chunk_id", type_hint=int, default=0, description="Index of the current chunk."),
            InputParam(
                name="current_conditional_frames",
                type_hint=int,
                required=True,
                description="Number of pixel frames this chunk reused from the previous chunk.",
            ),
            InputParam(
                name="output_chunks",
                type_hint=list[torch.Tensor],
                required=True,
                description="Decoded pixel chunks accumulated so far.",
            ),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [
            OutputParam(
                "previous_output",
                type_hint=torch.Tensor,
                description="Decoded pixels of this chunk, used to seed the next chunk.",
            ),
            OutputParam(
                "output_chunks",
                type_hint=list[torch.Tensor],
                description="Decoded pixel chunks accumulated so far (with this chunk appended).",
            ),
        ]

    @torch.no_grad()
    def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)
        latents = block_state.latents
        vae_dtype = components.vae.dtype
        mean = components._vae_latents_mean.to(device=latents.device, dtype=vae_dtype)
        inv_std = components._vae_latents_inv_std.to(device=latents.device, dtype=vae_dtype)
        z_raw = latents.to(vae_dtype) / inv_std.view(1, -1, 1, 1, 1) + mean.view(1, -1, 1, 1, 1)
        output_video = components.vae.decode(z_raw).sample.to(torch.float32).clamp(-1, 1)
        block_state.previous_output = output_video
        chunk = (
            output_video if block_state.chunk_id == 0 else output_video[:, :, block_state.current_conditional_frames :]
        )
        block_state.output_chunks = [*block_state.output_chunks, chunk]
        self.set_block_state(state, block_state)
        return components, state


class Cosmos3TransferStitchStep(ModularPipelineBlocks):
    model_name = "cosmos3-omni"

    @property
    def description(self) -> str:
        return (
            "Concatenates the decoded transfer chunks along time, truncates to total_frames, and post-processes to "
            "the requested output type. Transfer produces no audio, so sound / sampling_rate are None."
        )

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [
            ComponentSpec("vae", AutoencoderKLWan),
            ComponentSpec(
                "video_processor",
                VideoProcessor,
                config=FrozenDict({"vae_scale_factor": 16, "resample": "bilinear"}),
                default_creation_method="from_config",
            ),
        ]

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(
                name="output_chunks",
                type_hint=list[torch.Tensor],
                required=True,
                description="Decoded pixel chunks to stitch together.",
            ),
            InputParam(
                name="total_frames", type_hint=int, required=True, description="Total number of output frames to keep."
            ),
            InputParam.template("output_type", default="pil"),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [
            OutputParam("videos", description="The generated transfer video."),
            OutputParam("sound", description="Always None for transfer (no audio)."),
            OutputParam("sampling_rate", description="Always None for transfer (no audio)."),
        ]

    @torch.no_grad()
    def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)
        device = components._execution_device

        decoded = torch.cat(block_state.output_chunks, dim=2)[:, :, : block_state.total_frames]
        block_state.videos = components.video_processor.postprocess_video(
            decoded, output_type=block_state.output_type
        )[0]

        if components.requires_safety_checker and block_state.output_type != "latent":
            if getattr(components, "safety_checker", None) is None:
                raise ValueError(
                    "Cosmos3 requires a safety checker by default. Call `pipe.enable_safety_checker()` to load it "
                    "(or pass your own), or opt out explicitly with `pipe.disable_safety_checker()`."
                )
            block_state.videos = components._apply_video_safety_check(
                block_state.videos, output_type=block_state.output_type, device=device
            )

        block_state.sound = None
        block_state.sampling_rate = None
        self.set_block_state(state, block_state)
        return components, state