| """Shared generator runtime used by distribution matching objectives. |
| |
| This module contains rollout mechanics only. Distribution objectives belong in |
| their concrete model classes; frozen-encoder losses and statistics are not part |
| of the generator runtime. |
| """ |
|
|
| from typing import Optional |
|
|
| import torch |
| import torch.distributed as dist |
|
|
| from model.base import SelfForcingModel |
|
|
|
|
| class GeneratorRuntimeBase(SelfForcingModel): |
| """Initialize the generator and provide common differentiable rollouts.""" |
|
|
| uses_critic = False |
| uses_vae_for_loss = True |
|
|
| def __init__(self, args, device): |
| super().__init__(args, device) |
| self.causal_generator = bool(getattr(args, "causal", True)) |
| self.num_frame_per_block = getattr(args, "num_frame_per_block", 1) |
| self.same_step_across_blocks = getattr( |
| args, "same_step_across_blocks", True) |
| self.num_training_frames = getattr(args, "num_training_frames", 21) |
|
|
| if self.causal_generator and self.num_frame_per_block > 1: |
| self.generator.model.num_frame_per_block = self.num_frame_per_block |
|
|
| self.independent_first_frame = getattr( |
| args, "independent_first_frame", False) |
| if self.causal_generator and self.independent_first_frame: |
| self.generator.model.independent_first_frame = True |
| if args.gradient_checkpointing: |
| self.generator.enable_gradient_checkpointing() |
|
|
| self.inference_pipeline = None |
| self.generation_guidance_scale = float( |
| getattr(args, "guidance_scale", 1.0)) |
|
|
| if not hasattr(self, "denoising_step_list"): |
| raise ValueError( |
| "few-step generation requires denoising_step_list") |
| denoising_step_list = self.denoising_step_list.to(device=device) |
| if ( |
| denoising_step_list.numel() > 0 |
| and denoising_step_list[-1].item() == 0 |
| ): |
| denoising_step_list = denoising_step_list[:-1] |
| self.generation_denoising_step_list = denoising_step_list |
|
|
| @staticmethod |
| def _apply_gradient_mask( |
| latents: torch.Tensor, |
| gradient_mask: Optional[torch.Tensor], |
| ) -> torch.Tensor: |
| if gradient_mask is None: |
| return latents |
| frame_mask = gradient_mask.flatten(2).any(dim=2).view( |
| latents.shape[0], latents.shape[1], 1, 1, 1) |
| return torch.where(frame_mask, latents, latents.detach()) |
|
|
| def _predict_x0_with_cfg( |
| self, |
| noisy_latent: torch.Tensor, |
| conditional_dict: dict, |
| unconditional_dict: dict, |
| timestep: torch.Tensor, |
| ) -> torch.Tensor: |
| _, x0_cond = self.generator( |
| noisy_image_or_video=noisy_latent, |
| conditional_dict=conditional_dict, |
| timestep=timestep, |
| ) |
| if self.generation_guidance_scale == 1.0: |
| return x0_cond |
|
|
| _, x0_uncond = self.generator( |
| noisy_image_or_video=noisy_latent, |
| conditional_dict=unconditional_dict, |
| timestep=timestep, |
| ) |
| return x0_uncond + self.generation_guidance_scale * ( |
| x0_cond - x0_uncond) |
|
|
| def _run_few_step_generator( |
| self, |
| image_or_video_shape, |
| conditional_dict: dict, |
| unconditional_dict: dict, |
| exit_index: Optional[int] = None, |
| ) -> torch.Tensor: |
| """Roll out detached steps, then differentiate one selected exit step.""" |
| latent = torch.randn( |
| image_or_video_shape, |
| device=self.device, |
| dtype=self.dtype, |
| ) |
| batch_size, num_frames = latent.shape[:2] |
| denoising_steps = self.generation_denoising_step_list.to( |
| device=self.device) |
| num_steps = len(denoising_steps) |
|
|
| if exit_index is None: |
| if num_steps <= 1: |
| exit_index = 0 |
| else: |
| index_tensor = torch.randint( |
| 0, num_steps, (1,), device=self.device) |
| if dist.is_available() and dist.is_initialized(): |
| dist.broadcast(index_tensor, src=0) |
| exit_index = int(index_tensor.item()) |
|
|
| for index in range(num_steps): |
| current_timestep = denoising_steps[index] |
| timestep = torch.ones( |
| [batch_size, num_frames], |
| device=self.device, |
| dtype=current_timestep.dtype, |
| ) * current_timestep |
| if index < exit_index: |
| with torch.no_grad(): |
| x0_pred = self._predict_x0_with_cfg( |
| noisy_latent=latent, |
| conditional_dict=conditional_dict, |
| unconditional_dict=unconditional_dict, |
| timestep=timestep, |
| ) |
| next_timestep = denoising_steps[index + 1] |
| latent = self.scheduler.add_noise( |
| x0_pred.flatten(0, 1), |
| torch.randn_like(x0_pred.flatten(0, 1)), |
| next_timestep * torch.ones( |
| [batch_size * num_frames], |
| device=self.device, |
| dtype=next_timestep.dtype, |
| ), |
| ).unflatten(0, x0_pred.shape[:2]).to(self.dtype) |
| else: |
| return self._predict_x0_with_cfg( |
| noisy_latent=latent, |
| conditional_dict=conditional_dict, |
| unconditional_dict=unconditional_dict, |
| timestep=timestep, |
| ) |
| return latent |
|
|
