denoise.py

# Copyright 2026 OpenMOSS and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0. See the LICENSE file in this repository.

import torch

from diffusers import AutoModel
from diffusers.configuration_utils import FrozenDict
from diffusers.guiders import ClassifierFreeGuidance
from diffusers.modular_pipelines import (
    BlockState,
    LoopSequentialPipelineBlocks,
    ModularPipelineBlocks,
    PipelineState,
)
from diffusers.modular_pipelines.modular_pipeline_utils import ComponentSpec, InputParam
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler


class MOSSImageLoopBeforeDenoiser(ModularPipelineBlocks):
    model_name = "moss-image"

    @property
    def description(self) -> str:
        return "Step within the denoising loop that casts the float32 latents to the transformer dtype."

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(
                "latents",
                required=True,
                type_hint=torch.Tensor,
                description="The latents being denoised. Can be generated in the prepare latents step.",
            ),
            InputParam(
                "dtype",
                required=True,
                type_hint=torch.dtype,
                description="Dtype of the transformer inputs. Can be generated in the input step.",
            ),
        ]

    @torch.no_grad()
    def __call__(self, components, block_state: BlockState, i: int, t: torch.Tensor):
        block_state.latent_model_input = block_state.latents.to(block_state.dtype)
        return components, block_state


class MOSSImageLoopDenoiser(ModularPipelineBlocks):
    model_name = "moss-image"

    # transformer argument -> (conditional, unconditional) block_state fields
    guider_input_fields = {"encoder_hidden_states": ("prompt_embeds", "negative_prompt_embeds")}

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [
            ComponentSpec(
                "guider",
                ClassifierFreeGuidance,
                config=FrozenDict({"guidance_scale": 3.0}),
                default_creation_method="from_config",
            ),
            ComponentSpec("transformer", AutoModel),
        ]

    @property
    def description(self) -> str:
        return (
            "Step within the denoising loop that runs the transformer for each guidance batch and combines the "
            "predictions with the guider. The unconditional prediction skips the transformer blocks in "
            "`uncond_skip_layers`."
        )

    @property
    def inputs(self) -> list[InputParam]:
        return [
            InputParam(
                "num_inference_steps",
                required=True,
                type_hint=int,
                description="The number of denoising steps. Can be generated in the set timesteps step.",
            ),
            InputParam(
                "uncond_skip_layers",
                type_hint=list[int],
                default=[22],
                description="Zero-based transformer blocks skipped in the unconditional prediction. Pass `[]` to "
                "run the full model for it.",
            ),
            InputParam.template("attention_kwargs"),
            InputParam("prompt_embeds", required=True, type_hint=list[torch.Tensor]),
            InputParam("negative_prompt_embeds", type_hint=list[torch.Tensor]),
        ]

    @torch.no_grad()
    def __call__(self, components, block_state: BlockState, i: int, t: torch.Tensor):
        guider = components.guider
        guider.set_state(step=i, num_inference_steps=block_state.num_inference_steps, timestep=t)
        guider_state = guider.prepare_inputs_from_block_state(block_state, self.guider_input_fields)

        for guider_state_batch in guider_state:
            guider.prepare_models(components.transformer)
            is_uncond = getattr(guider_state_batch, guider._identifier_key) == "pred_uncond"
            skip_layers = block_state.uncond_skip_layers if is_uncond and block_state.uncond_skip_layers else None
            model_out = components.transformer(
                hidden_states=block_state.latent_model_input,
                timestep=t,
                encoder_hidden_states=[e.to(block_state.dtype) for e in guider_state_batch.encoder_hidden_states],
                skip_layers=skip_layers,
                attention_kwargs=block_state.attention_kwargs,
                return_dict=False,
            )[0]
            # guidance and the Euler update run in float32
            guider_state_batch.noise_pred = model_out.float()
            guider.cleanup_models(components.transformer)

        block_state.noise_pred = guider(guider_state)[0]
        return components, block_state


class MOSSImageLoopAfterDenoiser(ModularPipelineBlocks):
    model_name = "moss-image"

    @property
    def expected_components(self) -> list[ComponentSpec]:
        return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler)]

    @property
    def description(self) -> str:
        return "Step within the denoising loop that updates the float32 latents with the scheduler."

    @torch.no_grad()
    def __call__(self, components, block_state: BlockState, i: int, t: torch.Tensor):
        block_state.latents = components.scheduler.step(
            block_state.noise_pred, t, block_state.latents, return_dict=False
        )[0]
        return components, block_state


class MOSSImageDenoiseLoopWrapper(LoopSequentialPipelineBlocks):
    model_name = "moss-image"

    @property
    def description(self) -> str:
        return (
            "Pipeline block that iteratively denoises the latents over `timesteps`. "
            "The steps of each iteration are defined by `sub_blocks`."
        )

    @property
    def loop_expected_components(self) -> list[ComponentSpec]:
        return [ComponentSpec("scheduler", FlowMatchEulerDiscreteScheduler)]

    @property
    def loop_inputs(self) -> list[InputParam]:
        return [
            InputParam(
                "timesteps",
                required=True,
                type_hint=torch.Tensor,
                description="The timesteps of the denoising loop. Can be generated in the set timesteps step.",
            ),
            InputParam(
                "num_inference_steps",
                required=True,
                type_hint=int,
                description="The number of denoising steps. Can be generated in the set timesteps step.",
            ),
        ]

    @torch.no_grad()
    def __call__(self, components, state: PipelineState) -> PipelineState:
        block_state = self.get_block_state(state)
        with self.progress_bar(total=block_state.num_inference_steps) as progress_bar:
            for i, t in enumerate(block_state.timesteps):
                components, block_state = self.loop_step(components, block_state, i=i, t=t)
                progress_bar.update()
        self.set_block_state(state, block_state)
        return components, state


class MOSSImageDenoiseStep(MOSSImageDenoiseLoopWrapper):
    block_classes = [MOSSImageLoopBeforeDenoiser, MOSSImageLoopDenoiser, MOSSImageLoopAfterDenoiser]
    block_names = ["before_denoiser", "denoiser", "after_denoiser"]

    @property
    def description(self) -> str:
        return (
            "Denoise step that iteratively denoises the latents. At each iteration it runs, in order: "
            "`MOSSImageLoopBeforeDenoiser`, `MOSSImageLoopDenoiser`, `MOSSImageLoopAfterDenoiser`."
        )