sd-webui-animatediff/scripts/animatediff_infv2v.py

from typing import List

import numpy as np
import torch

from modules import prompt_parser, devices, sd_samplers_common, shared
from modules.shared import opts, state
from modules.script_callbacks import CFGDenoiserParams, cfg_denoiser_callback
from modules.script_callbacks import CFGDenoisedParams, cfg_denoised_callback
from modules.script_callbacks import AfterCFGCallbackParams, cfg_after_cfg_callback
from modules.sd_samplers_cfg_denoiser import CFGDenoiser, catenate_conds, subscript_cond, pad_cond

from scripts.animatediff_logger import logger_animatediff as logger
from scripts.animatediff_ui import AnimateDiffProcess
from scripts.animatediff_prompt import AnimateDiffPromptSchedule


class AnimateDiffInfV2V:
    cfg_original_forward = None

    def __init__(self, p, prompt_scheduler: AnimateDiffPromptSchedule):
        try:
            from scripts.external_code import find_cn_script
            self.cn_script = find_cn_script(p.scripts)
        except:
            self.cn_script = None
        self.prompt_scheduler = prompt_scheduler


    # Returns fraction that has denominator that is a power of 2
    @staticmethod
    def ordered_halving(val):
        # get binary value, padded with 0s for 64 bits
        bin_str = f"{val:064b}"
        # flip binary value, padding included
        bin_flip = bin_str[::-1]
        # convert binary to int
        as_int = int(bin_flip, 2)
        # divide by 1 << 64, equivalent to 2**64, or 18446744073709551616,
        # or b10000000000000000000000000000000000000000000000000000000000000000 (1 with 64 zero's)
        final = as_int / (1 << 64)
        return final


    # Generator that returns lists of latent indeces to diffuse on
    @staticmethod
    def uniform(
        step: int = ...,
        video_length: int = 0,
        batch_size: int = 16,
        stride: int = 1,
        overlap: int = 4,
        loop_setting: str = 'R-P',
    ):
        if video_length <= batch_size:
            yield list(range(batch_size))
            return

        closed_loop = (loop_setting == 'A')
        stride = min(stride, int(np.ceil(np.log2(video_length / batch_size))) + 1)

        for context_step in 1 << np.arange(stride):
            pad = int(round(video_length * AnimateDiffInfV2V.ordered_halving(step)))
            both_close_loop = False
            for j in range(
                int(AnimateDiffInfV2V.ordered_halving(step) * context_step) + pad,
                video_length + pad + (0 if closed_loop else -overlap),
                (batch_size * context_step - overlap),
            ):
                if loop_setting == 'N' and context_step == 1:
                    current_context = [e % video_length for e in range(j, j + batch_size * context_step, context_step)]
                    first_context = [e % video_length for e in range(0, batch_size * context_step, context_step)]
                    last_context = [e % video_length for e in range(video_length - batch_size * context_step, video_length, context_step)]
                    def get_unsorted_index(lst):
                        for i in range(1, len(lst)):
                            if lst[i] < lst[i-1]:
                                return i
                        return None
                    unsorted_index = get_unsorted_index(current_context)
                    if unsorted_index is None:
                        yield current_context
                    elif both_close_loop: # last and this context are close loop
                        both_close_loop = False
                        yield first_context
                    elif unsorted_index < batch_size - overlap: # only this context is close loop
                        yield last_context
                        yield first_context
                    else: # this and next context are close loop
                        both_close_loop = True
                        yield last_context
                else:
                    yield [e % video_length for e in range(j, j + batch_size * context_step, context_step)]


    def hack(self, params: AnimateDiffProcess):
        if AnimateDiffInfV2V.cfg_original_forward is not None:
            logger.info("CFGDenoiser already hacked")
            return

        logger.info(f"Hacking CFGDenoiser forward function.")
        AnimateDiffInfV2V.cfg_original_forward = CFGDenoiser.forward
        cn_script = self.cn_script
        prompt_scheduler = self.prompt_scheduler

        def mm_cn_select(context: List[int]):
            # take control images for current context.
            if cn_script and cn_script.latest_network:
                from scripts.hook import ControlModelType
                for control in cn_script.latest_network.control_params:
                    if control.control_model_type not in [ControlModelType.IPAdapter, ControlModelType.Controlllite]:
                        if control.hint_cond.shape[0] > len(context):
                            control.hint_cond_backup = control.hint_cond
                            control.hint_cond = control.hint_cond[context]
                        control.hint_cond = control.hint_cond.to(device=devices.get_device_for("controlnet"))
                        if control.hr_hint_cond is not None:
                            if control.hr_hint_cond.shape[0] > len(context):
                                control.hr_hint_cond_backup = control.hr_hint_cond
                                control.hr_hint_cond = control.hr_hint_cond[context]
                            control.hr_hint_cond = control.hr_hint_cond.to(device=devices.get_device_for("controlnet"))
                    # IPAdapter and Controlllite are always on CPU.
                    elif control.control_model_type == ControlModelType.IPAdapter and control.control_model.image_emb.shape[0] > len(context):
                        control.control_model.image_emb_backup = control.control_model.image_emb
                        control.control_model.image_emb = control.control_model.image_emb[context]
                        control.control_model.uncond_image_emb_backup = control.control_model.uncond_image_emb
                        control.control_model.uncond_image_emb = control.control_model.uncond_image_emb[context]
                    elif control.control_model_type == ControlModelType.Controlllite:
                        for module in control.control_model.modules.values():
                            if module.cond_image.shape[0] > len(context):
                                module.cond_image_backup = module.cond_image
                                module.set_cond_image(module.cond_image[context])
        
        def mm_cn_restore(context: List[int]):
            # restore control images for next context
            if cn_script and cn_script.latest_network:
                from scripts.hook import ControlModelType
                for control in cn_script.latest_network.control_params:
                    if control.control_model_type not in [ControlModelType.IPAdapter, ControlModelType.Controlllite]:
                        if getattr(control, "hint_cond_backup", None) is not None:
                            control.hint_cond_backup[context] = control.hint_cond.to(device="cpu")
                            control.hint_cond = control.hint_cond_backup
                        if control.hr_hint_cond is not None and getattr(control, "hr_hint_cond_backup", None) is not None:
                            control.hr_hint_cond_backup[context] = control.hr_hint_cond.to(device="cpu")
                            control.hr_hint_cond = control.hr_hint_cond_backup
                    elif control.control_model_type == ControlModelType.IPAdapter and getattr(control.control_model, "image_emb_backup", None) is not None:
                        control.control_model.image_emb = control.control_model.image_emb_backup
                        control.control_model.uncond_image_emb = control.control_model.uncond_image_emb_backup
                    elif control.control_model_type == ControlModelType.Controlllite:
                        for module in control.control_model.modules.values():
                            if getattr(module, "cond_image_backup", None) is not None:
                                module.set_cond_image(module.cond_image_backup)

        def mm_sd_forward(self, x_in, sigma_in, cond_in, image_cond_in, make_condition_dict):
            x_out = torch.zeros_like(x_in)
            for context in AnimateDiffInfV2V.uniform(self.step, params.video_length, params.batch_size, params.stride, params.overlap, params.closed_loop):
                if shared.opts.batch_cond_uncond:
                    _context = context + [c + params.video_length for c in context]
                else:
                    _context = context
                mm_cn_select(_context)
                out = self.inner_model(
                    x_in[_context], sigma_in[_context],
                    cond=make_condition_dict(
                        cond_in[_context] if not isinstance(cond_in, dict) else {k: v[_context] for k, v in cond_in.items()},
                        image_cond_in[_context]))
                x_out = x_out.to(dtype=out.dtype)
                x_out[_context] = out
                mm_cn_restore(_context)
            return x_out

        def mm_cfg_forward(self, x, sigma, uncond, cond, cond_scale, s_min_uncond, image_cond):
            if state.interrupted or state.skipped:
                raise sd_samplers_common.InterruptedException

            if sd_samplers_common.apply_refiner(self):
                cond = self.sampler.sampler_extra_args['cond']
                uncond = self.sampler.sampler_extra_args['uncond']

            # at self.image_cfg_scale == 1.0 produced results for edit model are the same as with normal sampling,
            # so is_edit_model is set to False to support AND composition.
            is_edit_model = shared.sd_model.cond_stage_key == "edit" and self.image_cfg_scale is not None and self.image_cfg_scale != 1.0

            conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step)
            uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step)

            assert not is_edit_model or all(len(conds) == 1 for conds in conds_list), "AND is not supported for InstructPix2Pix checkpoint (unless using Image CFG scale = 1.0)"

            if self.mask_before_denoising and self.mask is not None:
                x = self.init_latent * self.mask + self.nmask * x

            batch_size = len(conds_list)
            repeats = [len(conds_list[i]) for i in range(batch_size)]

            if shared.sd_model.model.conditioning_key == "crossattn-adm":
                image_uncond = torch.zeros_like(image_cond) # this should not be supported.
                make_condition_dict = lambda c_crossattn, c_adm: {"c_crossattn": [c_crossattn], "c_adm": c_adm}
            else:
                image_uncond = image_cond
                if isinstance(uncond, dict):
                    make_condition_dict = lambda c_crossattn, c_concat: {**c_crossattn, "c_concat": [c_concat]}
                else:
                    make_condition_dict = lambda c_crossattn, c_concat: {"c_crossattn": [c_crossattn], "c_concat": [c_concat]}

            if not is_edit_model:
                x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x])
                sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma])
                image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_uncond])
            else:
                x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x])
                sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma])
                image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_uncond] + [torch.zeros_like(self.init_latent)])

            denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps, tensor, uncond)
            cfg_denoiser_callback(denoiser_params)
            x_in = denoiser_params.x
            image_cond_in = denoiser_params.image_cond
            sigma_in = denoiser_params.sigma
            tensor = denoiser_params.text_cond
            uncond = denoiser_params.text_uncond
            skip_uncond = False

            # alternating uncond allows for higher thresholds without the quality loss normally expected from raising it
            if self.step % 2 and s_min_uncond > 0 and sigma[0] < s_min_uncond and not is_edit_model:
                skip_uncond = True
                x_in = x_in[:-batch_size]
                sigma_in = sigma_in[:-batch_size]

            self.padded_cond_uncond = False
            if shared.opts.pad_cond_uncond and tensor.shape[1] != uncond.shape[1]:
                empty = shared.sd_model.cond_stage_model_empty_prompt
                num_repeats = (tensor.shape[1] - uncond.shape[1]) // empty.shape[1]

                if num_repeats < 0:
                    tensor = pad_cond(tensor, -num_repeats, empty)
                    self.padded_cond_uncond = True
                elif num_repeats > 0:
                    uncond = pad_cond(uncond, num_repeats, empty)
                    self.padded_cond_uncond = True

            if tensor.shape[1] == uncond.shape[1] or skip_uncond:
                prompt_closed_loop = (params.video_length > params.batch_size) and (params.closed_loop in ['R+P', 'A']) # hook
                tensor = prompt_scheduler.multi_cond(tensor, prompt_closed_loop) # hook
                if is_edit_model:
                    cond_in = catenate_conds([tensor, uncond, uncond])
                elif skip_uncond:
                    cond_in = tensor
                else:
                    cond_in = catenate_conds([tensor, uncond])

                if shared.opts.batch_cond_uncond:
                    x_out = mm_sd_forward(self, x_in, sigma_in, cond_in, image_cond_in, make_condition_dict) # hook
                else:
                    x_out = torch.zeros_like(x_in)
                    for batch_offset in range(0, x_out.shape[0], batch_size):
                        a = batch_offset
                        b = a + batch_size
                        x_out[a:b] = mm_sd_forward(self, x_in[a:b], sigma_in[a:b], subscript_cond(cond_in, a, b), subscript_cond(image_cond_in, a, b), make_condition_dict) # hook
            else:
                x_out = torch.zeros_like(x_in)
                batch_size = batch_size*2 if shared.opts.batch_cond_uncond else batch_size
                for batch_offset in range(0, tensor.shape[0], batch_size):
                    a = batch_offset
                    b = min(a + batch_size, tensor.shape[0])

                    if not is_edit_model:
                        c_crossattn = subscript_cond(tensor, a, b)
                    else:
                        c_crossattn = torch.cat([tensor[a:b]], uncond)

                    x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(c_crossattn, image_cond_in[a:b]))

                if not skip_uncond:
                    x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict(uncond, image_cond_in[-uncond.shape[0]:]))

            denoised_image_indexes = [x[0][0] for x in conds_list]
            if skip_uncond:
                fake_uncond = torch.cat([x_out[i:i+1] for i in denoised_image_indexes])
                x_out = torch.cat([x_out, fake_uncond])  # we skipped uncond denoising, so we put cond-denoised image to where the uncond-denoised image should be

            denoised_params = CFGDenoisedParams(x_out, state.sampling_step, state.sampling_steps, self.inner_model)
            cfg_denoised_callback(denoised_params)

            devices.test_for_nans(x_out, "unet")

            if is_edit_model:
                denoised = self.combine_denoised_for_edit_model(x_out, cond_scale)
            elif skip_uncond:
                denoised = self.combine_denoised(x_out, conds_list, uncond, 1.0)
            else:
                denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale)

            if not self.mask_before_denoising and self.mask is not None:
                denoised = self.init_latent * self.mask + self.nmask * denoised

            self.sampler.last_latent = self.get_pred_x0(torch.cat([x_in[i:i + 1] for i in denoised_image_indexes]), torch.cat([x_out[i:i + 1] for i in denoised_image_indexes]), sigma)

            if opts.live_preview_content == "Prompt":
                preview = self.sampler.last_latent
            elif opts.live_preview_content == "Negative prompt":
                preview = self.get_pred_x0(x_in[-uncond.shape[0]:], x_out[-uncond.shape[0]:], sigma)
            else:
                preview = self.get_pred_x0(torch.cat([x_in[i:i+1] for i in denoised_image_indexes]), torch.cat([denoised[i:i+1] for i in denoised_image_indexes]), sigma)

            sd_samplers_common.store_latent(preview)

            after_cfg_callback_params = AfterCFGCallbackParams(denoised, state.sampling_step, state.sampling_steps)
            cfg_after_cfg_callback(after_cfg_callback_params)
            denoised = after_cfg_callback_params.x

            self.step += 1
            return denoised

        CFGDenoiser.forward = mm_cfg_forward


    def restore(self):
        if AnimateDiffInfV2V.cfg_original_forward is None:
            logger.info("CFGDenoiser already restored.")
            return

        logger.info(f"Restoring CFGDenoiser forward function.")
        CFGDenoiser.forward = AnimateDiffInfV2V.cfg_original_forward
        AnimateDiffInfV2V.cfg_original_forward = None