generation_sdxl.py

import torch
import copy
import random
import numpy as np


# Diffusion utils
# ------------------------------------------------------------------------
def encode_prompt(prompt_batch, text_encoders, tokenizers, proportion_empty_prompts, is_train=True):
    prompt_embeds_list = []

    captions = []
    for caption in prompt_batch:
        if random.random() < proportion_empty_prompts:
            captions.append("")
        elif isinstance(caption, str):
            captions.append(caption)
        elif isinstance(caption, (list, np.ndarray)):
            # take a random caption if there are multiple
            captions.append(random.choice(caption) if is_train else caption[0])

    with torch.no_grad():
        for tokenizer, text_encoder in zip(tokenizers, text_encoders):
            text_inputs = tokenizer(
                captions,
                padding="max_length",
                max_length=tokenizer.model_max_length,
                truncation=True,
                return_tensors="pt",
            )
            text_input_ids = text_inputs.input_ids
            prompt_embeds = text_encoder(
                text_input_ids.to(text_encoder.device),
                output_hidden_states=True,
            )

            # We are only ALWAYS interested in the pooled output of the final text encoder
            pooled_prompt_embeds = prompt_embeds[0]
            prompt_embeds = prompt_embeds.hidden_states[-2]
            bs_embed, seq_len, _ = prompt_embeds.shape
            prompt_embeds = prompt_embeds.view(bs_embed, seq_len, -1)
            prompt_embeds_list.append(prompt_embeds)

    prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
    pooled_prompt_embeds = pooled_prompt_embeds.view(bs_embed, -1)
    return prompt_embeds, pooled_prompt_embeds


def compute_embeddings(
    prompt_batch, original_sizes, crop_coords, proportion_empty_prompts, text_encoders, tokenizers, is_train=True,
    device='cuda'
):
    target_size = (1024, 1024)
    original_sizes = original_sizes #list(map(list, zip(*original_sizes)))
    crops_coords_top_left = crop_coords #list(map(list, zip(*crop_coords)))

    original_sizes = torch.tensor(original_sizes, dtype=torch.long)
    crops_coords_top_left = torch.tensor(crops_coords_top_left, dtype=torch.long)

    prompt_embeds, pooled_prompt_embeds = encode_prompt(
        prompt_batch, text_encoders, tokenizers, proportion_empty_prompts, is_train
    )
    add_text_embeds = pooled_prompt_embeds

    # Adapted from pipeline.StableDiffusionXLPipeline._get_add_time_ids
    add_time_ids = list(target_size)
    add_time_ids = torch.tensor([add_time_ids])
    add_time_ids = add_time_ids.repeat(len(prompt_batch), 1)
    add_time_ids = torch.cat([original_sizes, crops_coords_top_left, add_time_ids], dim=-1)
    add_time_ids = add_time_ids.to(device, dtype=prompt_embeds.dtype)

    prompt_embeds = prompt_embeds.to(device)
    add_text_embeds = add_text_embeds.to(device)
    unet_added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}

    return {"prompt_embeds": prompt_embeds, **unet_added_cond_kwargs}

def extract_into_tensor(a, t, x_shape):
    b, *_ = t.shape
    out = a.gather(-1, t)
    return out.reshape(b, *((1,) * (len(x_shape) - 1)))


def guidance_scale_embedding(w, embedding_dim=512, dtype=torch.float32):
    """
    See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298

    Args:
        timesteps (`torch.Tensor`):
            generate embedding vectors at these timesteps
        embedding_dim (`int`, *optional*, defaults to 512):
            dimension of the embeddings to generate
        dtype:
            data type of the generated embeddings

    Returns:
        `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)`
    """
    assert len(w.shape) == 1
    w = w * 1000.0

    half_dim = embedding_dim // 2
    emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
    emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
    emb = w.to(dtype)[:, None] * emb[None, :]
    emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
    if embedding_dim % 2 == 1:  # zero pad
        emb = torch.nn.functional.pad(emb, (0, 1))
    assert emb.shape == (w.shape[0], embedding_dim)
    return emb

def predicted_origin(model_output, timesteps, boundary_timesteps, sample, prediction_type, alphas, sigmas):
    sigmas_s = extract_into_tensor(sigmas, boundary_timesteps, sample.shape)
    alphas_s = extract_into_tensor(alphas, boundary_timesteps, sample.shape)

    sigmas = extract_into_tensor(sigmas, timesteps, sample.shape)
    alphas = extract_into_tensor(alphas, timesteps, sample.shape)

    # Set hard boundaries to ensure equivalence with forward (direct) CD
    alphas_s[boundary_timesteps == 0] = 1.0
    sigmas_s[boundary_timesteps == 0] = 0.0

    if prediction_type == "epsilon":
        pred_x_0 = (sample - sigmas * model_output) / alphas # x0 prediction
        pred_x_0 = alphas_s * pred_x_0 + sigmas_s * model_output # Euler step to the boundary step
    elif prediction_type == "v_prediction":
        assert boundary_timesteps == 0, "v_prediction does not support multiple endpoints at the moment"
        pred_x_0 = alphas * sample - sigmas * model_output
    else:
        raise ValueError(f"Prediction type {prediction_type} currently not supported.")

    return pred_x_0


class DDIMSolver:
    def __init__(
            self, alpha_cumprods, timesteps=1000, ddim_timesteps=50,
            num_endpoints=1, num_inverse_endpoints=1,
            max_inverse_timestep_index=49,
            endpoints=None, inverse_endpoints=None
    ):
        # DDIM sampling parameters
        step_ratio = timesteps // ddim_timesteps
        self.ddim_timesteps = (np.arange(1, ddim_timesteps + 1) * step_ratio).round().astype(
            np.int64) - 1  # [19, ..., 999]
        self.ddim_alpha_cumprods = alpha_cumprods[self.ddim_timesteps]
        self.ddim_alpha_cumprods_prev = np.asarray(
            [alpha_cumprods[0]] + alpha_cumprods[self.ddim_timesteps[:-1]].tolist()
        )
        self.ddim_alpha_cumprods_next = np.asarray(
            alpha_cumprods[self.ddim_timesteps[1:]].tolist() + [0.0]
        )
        # convert to torch tensors
        self.ddim_timesteps = torch.from_numpy(self.ddim_timesteps).long()
        self.ddim_alpha_cumprods = torch.from_numpy(self.ddim_alpha_cumprods)
        self.ddim_alpha_cumprods_prev = torch.from_numpy(self.ddim_alpha_cumprods_prev)
        self.ddim_alpha_cumprods_next = torch.from_numpy(self.ddim_alpha_cumprods_next)

        # Set endpoints for direct CTM
        if endpoints is None:
            timestep_interval = ddim_timesteps // num_endpoints + int(ddim_timesteps % num_endpoints > 0)
            endpoint_idxs = torch.arange(timestep_interval, ddim_timesteps, timestep_interval) - 1
            self.endpoints = torch.tensor([0] + self.ddim_timesteps[endpoint_idxs].tolist())
        else:
            self.endpoints = torch.tensor([int(endpoint) for endpoint in endpoints.split(',')])
            assert len(self.endpoints) == num_endpoints

        # Set endpoints for inverse CTM
        if inverse_endpoints is None:
            timestep_interval = ddim_timesteps // num_inverse_endpoints + int(
                ddim_timesteps % num_inverse_endpoints > 0)
            inverse_endpoint_idxs = torch.arange(timestep_interval, ddim_timesteps, timestep_interval) - 1
            inverse_endpoint_idxs = torch.tensor(inverse_endpoint_idxs.tolist() + [max_inverse_timestep_index])
            self.inverse_endpoints = self.ddim_timesteps[inverse_endpoint_idxs]
        else:
            self.inverse_endpoints = torch.tensor([int(endpoint) for endpoint in inverse_endpoints.split(',')])
            assert len(self.inverse_endpoints) == num_inverse_endpoints

    def to(self, device):
        self.endpoints = self.endpoints.to(device)
        self.inverse_endpoints = self.inverse_endpoints.to(device)

        self.ddim_timesteps = self.ddim_timesteps.to(device)
        self.ddim_alpha_cumprods = self.ddim_alpha_cumprods.to(device)
        self.ddim_alpha_cumprods_prev = self.ddim_alpha_cumprods_prev.to(device)
        self.ddim_alpha_cumprods_next = self.ddim_alpha_cumprods_next.to(device)
        return self

    def ddim_step(self, pred_x0, pred_noise, timestep_index):
        alpha_cumprod_prev = extract_into_tensor(self.ddim_alpha_cumprods_prev, timestep_index, pred_x0.shape)
        dir_xt = (1.0 - alpha_cumprod_prev).sqrt() * pred_noise
        x_prev = alpha_cumprod_prev.sqrt() * pred_x0 + dir_xt
        return x_prev

    def inverse_ddim_step(self, pred_x0, pred_noise, timestep_index):
        alpha_cumprod_next = extract_into_tensor(self.ddim_alpha_cumprods_next, timestep_index, pred_x0.shape)
        dir_xt = (1.0 - alpha_cumprod_next).sqrt() * pred_noise
        x_next = alpha_cumprod_next.sqrt() * pred_x0 + dir_xt
        return x_next
# ------------------------------------------------------------------------

# Distillation specific
# ------------------------------------------------------------------------
def inverse_sample_deterministic(
        pipe,
        images,
        prompt,
        generator=None,
        num_scales=50,
        num_inference_steps=1,
        timesteps=None,
        start_timestep=19,
        max_inverse_timestep_index=49,
        return_start_latent=False,
        guidance_scale=None,  # Used only if the student has w_embedding
        compute_embeddings_fn=None,
        is_sdxl=False,
        inverse_endpoints=None,
        seed=0,
):
    # assert isinstance(pipe, StableDiffusionImg2ImgPipeline), f"Does not support the pipeline {type(pipe)}"

    if prompt is not None and isinstance(prompt, str):
        batch_size = 1
    elif prompt is not None and isinstance(prompt, list):
        batch_size = len(prompt)

    device = pipe._execution_device

    # Prepare text embeddings
    if compute_embeddings_fn is not None:
        if is_sdxl:
            orig_size = [(1024, 1024)] * len(prompt)
            crop_coords = [(0, 0)] * len(prompt)
            encoded_text = compute_embeddings_fn(prompt, orig_size, crop_coords)
            prompt_embeds = encoded_text.pop("prompt_embeds")
        else:
            prompt_embeds = compute_embeddings_fn(prompt)["prompt_embeds"]
            encoded_text = {}
        prompt_embeds = prompt_embeds.to(pipe.unet.dtype)
    else:
        prompt_embeds = pipe.encode_prompt(prompt, device, 1, False)[0]
        encoded_text = {}
    assert prompt_embeds.dtype == pipe.unet.dtype

    # Prepare the DDIM solver
    endpoints = ','.join(['0'] + inverse_endpoints.split(',')[:-1]) if inverse_endpoints is not None else None
    solver = DDIMSolver(
        pipe.scheduler.alphas_cumprod.cpu().numpy(),
        timesteps=pipe.scheduler.num_train_timesteps,
        ddim_timesteps=num_scales,
        num_endpoints=num_inference_steps,
        num_inverse_endpoints=num_inference_steps,
        max_inverse_timestep_index=max_inverse_timestep_index,
        endpoints=endpoints,
        inverse_endpoints=inverse_endpoints
    ).to(device)

    if timesteps is None:
        timesteps = solver.inverse_endpoints.flip(0)
        boundary_timesteps = solver.endpoints.flip(0)
    else:
        timesteps, boundary_timesteps = timesteps, timesteps
        boundary_timesteps = boundary_timesteps[1:] + [boundary_timesteps[0]]
        boundary_timesteps[-1] = 999
        timesteps, boundary_timesteps = torch.tensor(timesteps), torch.tensor(boundary_timesteps)

    alpha_schedule = torch.sqrt(pipe.scheduler.alphas_cumprod).to(device)
    sigma_schedule = torch.sqrt(1 - pipe.scheduler.alphas_cumprod).to(device)

    # 5. Prepare latent variables
    num_channels_latents = pipe.unet.config.in_channels
    start_latents = pipe.prepare_latents(
        images, timesteps[0], batch_size, 1, prompt_embeds.dtype, device,
        generator=torch.Generator().manual_seed(seed),
    )
    latents = start_latents.clone()

    if guidance_scale is not None:
        w = torch.ones(batch_size) * guidance_scale
        w_embedding = guidance_scale_embedding(w, embedding_dim=512)
        w_embedding = w_embedding.to(device=latents.device, dtype=latents.dtype)
    else:
        w_embedding = None

    for i, (t, s) in enumerate(zip(timesteps, boundary_timesteps)):
        # predict the noise residual
        noise_pred = pipe.unet(
            latents.to(prompt_embeds.dtype),
            t,
            encoder_hidden_states=prompt_embeds,
            return_dict=False,
            timestep_cond=w_embedding,
            added_cond_kwargs=encoded_text,
        )[0]

        latents = predicted_origin(
            noise_pred,
            torch.tensor([t] * len(latents), device=device),
            torch.tensor([s] * len(latents), device=device),
            latents,
            pipe.scheduler.config.prediction_type,
            alpha_schedule,
            sigma_schedule,
        ).to(prompt_embeds.dtype)

    if return_start_latent:
        return latents, start_latents
    else:
        return latents


def linear_schedule_old(t, guidance_scale, tau1, tau2):
    t = t / 1000
    if t <= tau1:
        gamma = 1.0
    elif t >= tau2:
        gamma = 0.0
    else:
        gamma = (tau2 - t) / (tau2 - tau1)
    return gamma * guidance_scale


@torch.no_grad()
def sample_deterministic(
        pipe,
        prompt,
        latents=None,
        generator=None,
        num_scales=50,
        num_inference_steps=1,
        timesteps=None,
        start_timestep=19,
        max_inverse_timestep_index=49,
        return_latent=False,
        guidance_scale=None,  # Used only if the student has w_embedding
        compute_embeddings_fn=None,
        is_sdxl=False,
        endpoints=None,
        use_dynamic_guidance=False,
        tau1=0.7,
        tau2=0.7,
        amplify_prompt=None,
):
    # assert isinstance(pipe, StableDiffusionPipeline), f"Does not support the pipeline {type(pipe)}"
    height = pipe.unet.config.sample_size * pipe.vae_scale_factor
    width = pipe.unet.config.sample_size * pipe.vae_scale_factor

    # 1. Define call parameters
    if prompt is not None and isinstance(prompt, str):
        batch_size = 1
    elif prompt is not None and isinstance(prompt, list):
        batch_size = len(prompt)

    device = pipe._execution_device

    # Prepare text embeddings
    if compute_embeddings_fn is not None:
        if is_sdxl:
            orig_size = [(1024, 1024)] * len(prompt)
            crop_coords = [(0, 0)] * len(prompt)
            encoded_text = compute_embeddings_fn(prompt, orig_size, crop_coords)
            prompt_embeds = encoded_text.pop("prompt_embeds")
            if amplify_prompt is not None:
                orig_size = [(1024, 1024)] * len(amplify_prompt)
                crop_coords = [(0, 0)] * len(amplify_prompt)
                encoded_text_old = compute_embeddings_fn(amplify_prompt, orig_size, crop_coords)
                amplify_prompt_embeds = encoded_text_old.pop("prompt_embeds")
        else:
            prompt_embeds = compute_embeddings_fn(prompt)["prompt_embeds"]
            encoded_text = {}
        prompt_embeds = prompt_embeds.to(pipe.unet.dtype)
    else:
        prompt_embeds = pipe.encode_prompt(prompt, device, 1, False)[0]
        encoded_text = {}
    assert prompt_embeds.dtype == pipe.unet.dtype

    # Prepare the DDIM solver
    inverse_endpoints = ','.join(endpoints.split(',')[1:] + ['999']) if endpoints is not None else None
    solver = DDIMSolver(
        pipe.scheduler.alphas_cumprod.numpy(),
        timesteps=pipe.scheduler.num_train_timesteps,
        ddim_timesteps=num_scales,
        num_endpoints=num_inference_steps,
        num_inverse_endpoints=num_inference_steps,
        max_inverse_timestep_index=max_inverse_timestep_index,
        endpoints=endpoints,
        inverse_endpoints=inverse_endpoints
    ).to(device)

    prompt_embeds_init = copy.deepcopy(prompt_embeds)

    if timesteps is None:
        timesteps = solver.inverse_endpoints.flip(0)
        boundary_timesteps = solver.endpoints.flip(0)
    else:
        timesteps, boundary_timesteps = copy.deepcopy(timesteps), copy.deepcopy(timesteps)
        timesteps.reverse()
        boundary_timesteps.reverse()
        boundary_timesteps = boundary_timesteps[1:] + [boundary_timesteps[0]]
        boundary_timesteps[-1] = 0
        timesteps, boundary_timesteps = torch.tensor(timesteps), torch.tensor(boundary_timesteps)

    alpha_schedule = torch.sqrt(pipe.scheduler.alphas_cumprod).to(device)
    sigma_schedule = torch.sqrt(1 - pipe.scheduler.alphas_cumprod).to(device)

    # 5. Prepare latent variables
    if latents is None:
        num_channels_latents = pipe.unet.config.in_channels
        latents = pipe.prepare_latents(
            batch_size,
            num_channels_latents,
            height,
            width,
            prompt_embeds.dtype,
            device,
            generator,
            None,
        )
        assert latents.dtype == pipe.unet.dtype
    else:
        latents = latents.to(prompt_embeds.dtype)

    if guidance_scale is not None:
        w = torch.ones(batch_size) * guidance_scale
        w_embedding = guidance_scale_embedding(w, embedding_dim=512)
        w_embedding = w_embedding.to(device=latents.device, dtype=latents.dtype)
    else:
        w_embedding = None

    for i, (t, s) in enumerate(zip(timesteps, boundary_timesteps)):
        if use_dynamic_guidance:
            if not isinstance(t, int):
                t_item = t.item()
            if t_item > tau1 * 1000 and amplify_prompt is not None:
                prompt_embeds = amplify_prompt_embeds
            else:
                prompt_embeds = prompt_embeds_init
            guidance_scale = linear_schedule_old(t_item, w, tau1=tau1, tau2=tau2)
            guidance_scale_tensor = torch.tensor([guidance_scale] * len(latents))
            w_embedding = guidance_scale_embedding(guidance_scale_tensor, embedding_dim=512)
            w_embedding = w_embedding.to(device=latents.device, dtype=latents.dtype)

        # predict the noise residual
        noise_pred = pipe.unet(
            latents,
            t,
            encoder_hidden_states=prompt_embeds,
            cross_attention_kwargs=None,
            return_dict=False,
            timestep_cond=w_embedding,
            added_cond_kwargs=encoded_text,
        )[0]

        latents = predicted_origin(
            noise_pred,
            torch.tensor([t] * len(noise_pred)).to(device),
            torch.tensor([s] * len(noise_pred)).to(device),
            latents,
            pipe.scheduler.config.prediction_type,
            alpha_schedule,
            sigma_schedule,
        ).to(pipe.unet.dtype)

    pipe.vae.to(torch.float32)
    image = pipe.vae.decode(latents.to(torch.float32) / pipe.vae.config.scaling_factor, return_dict=False)[0]
    do_denormalize = [True] * image.shape[0]
    image = pipe.image_processor.postprocess(image, output_type="pil", do_denormalize=do_denormalize)

    if return_latent:
        return image, latents
    else:
        return image
# ------------------------------------------------------------------------