diffusers/modular_pipelines/cosmos/after_decode.py

import torch

from ...utils import encode_video, export_to_video
from ..modular_pipeline import ModularPipelineBlocks, PipelineState
from ..modular_pipeline_utils import InputParam, OutputParam
from .modular_pipeline import Cosmos3OmniModularPipeline


class Cosmos3ActionOutputStep(ModularPipelineBlocks):
    model_name = "cosmos3-omni"

    @property
    def description(self) -> str:
        return "Post-processes action latents into action outputs."

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(
                name="action_latents",
                type_hint=torch.Tensor,
                default=None,
                description="Denoised action latents.",
            ),
            InputParam(
                name="action_mode", type_hint=str, default=None, description="Requested action-generation mode."
            ),
            InputParam(
                name="raw_action_dim_resolved",
                type_hint=int,
                default=None,
                description="Unpadded action-vector dimension.",
            ),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [OutputParam("action", type_hint=list[torch.Tensor], description="Generated action vectors.")]

    @torch.no_grad()
    def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)
        action_output = None
        if block_state.action_mode in {"inverse_dynamics", "policy"} and block_state.action_latents is not None:
            action_output = block_state.action_latents
            if block_state.raw_action_dim_resolved is not None:
                action_output = action_output[:, : block_state.raw_action_dim_resolved]
            action_output = [action_output.detach().cpu()]
        block_state.action = action_output
        self.set_block_state(state, block_state)
        return components, state


class Cosmos3ExportStep(ModularPipelineBlocks):
    model_name = "cosmos3-omni"

    @property
    def description(self) -> str:
        return (
            "Optional export block that writes decoded outputs to disk. Writes `videos` to `output_path` via "
            "`export_to_video`, or muxes `videos` with `sound` via `encode_video` when a waveform is present. "
            "Not wired into the default blocks; add it explicitly when you want the pipeline to produce a file."
        )

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(name="videos", required=True, description="Generated video frames to export."),
            InputParam(
                name="output_path",
                type_hint=str,
                required=True,
                description="Destination path for the exported video.",
            ),
            InputParam(name="fps", type_hint=float, default=24.0, description="Frame rate of the exported video."),
            InputParam(
                name="sound",
                type_hint=torch.Tensor,
                default=None,
                description="Generated waveform to mux into the video.",
            ),
            InputParam(
                name="sampling_rate",
                type_hint=int,
                default=None,
                description="Sample rate of the generated waveform in Hz.",
            ),
        ]

    @property
    def intermediate_outputs(self) -> list[OutputParam]:
        return [OutputParam("output_path", type_hint=str, description="Path of the exported video file.")]

    @torch.no_grad()
    def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)
        output_path = str(block_state.output_path)
        fps = int(round(block_state.fps))
        if block_state.sound is not None:
            if block_state.sampling_rate is None:
                raise ValueError("`sampling_rate` is required to export a video with sound.")
            encode_video(
                block_state.videos,
                fps=fps,
                audio=block_state.sound,
                audio_sample_rate=int(block_state.sampling_rate),
                output_path=output_path,
            )
        else:
            export_to_video(block_state.videos, output_path, fps=fps, macro_block_size=1)
        block_state.output_path = output_path
        self.set_block_state(state, block_state)
        return components, state