diffcve-2607 / diffcve /models /diffusion.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
d2c5d46 verified
Raw History Blame Contribute Delete
9.51 kB
from functools import partial
import numpy as np
import torch
from torch import nn
from peft import LoraConfig, get_peft_model
from tqdm import tqdm
from diffcve.utils import default, exists, instantiate_from_config
from diffcve.modules.diffusion.utils import make_beta_schedule, extract_into_tensor
from diffcve.modules.distributions import DiagonalGaussianDistribution
from diffcve.modules.text_encoder import load_text_conditions
class ConditionedDenoiser(nn.Module):
def __init__(self, network_config, conditioning_key="crossattn"):
super().__init__()
if conditioning_key != "crossattn":
raise ValueError("DiffCVE requires cross-attention conditioning")
self.conditioning_key = conditioning_key
self.network = instantiate_from_config(network_config)
def forward(self, x, t, c_crossattn, struct_cond=None, prior_cond=None):
return self.network(
x, t, context=torch.cat(c_crossattn, 1),
struct_cond=struct_cond, prior_cond=prior_cond,
)
class VideoDiffusion(nn.Module):
def __init__(self, denoiser_config, prior_encoder_config, prior_condition_config,
structure_encoder_config, timesteps=1000, linear_start=0.00085,
linear_end=0.012, num_frames=5, scale_factor=0.18215,
conditioning_key="crossattn", parameterization="eps",
text_conditions_path="checkpoints/text_conditions.pt",
cond_stage_config=None, text_prompts=None, text_condition_qps=None):
super().__init__()
if parameterization != "eps":
raise ValueError("DiffCVE requires noise prediction (eps)")
self.parameterization = parameterization
self.num_frames = num_frames
self.scale_factor = scale_factor
self.v_posterior = 0.0
self.denoiser = ConditionedDenoiser(denoiser_config, conditioning_key)
self.prior_encoder = instantiate_from_config(prior_encoder_config)
self.prior_condition_encoder = instantiate_from_config(prior_condition_config)
self.structure_encoder = instantiate_from_config(structure_encoder_config)
self.add_lora_adapters()
self.text_conditions = load_text_conditions(
text_conditions_path, cond_stage_config, text_prompts, text_condition_qps,
)
self.register_schedule(timesteps=timesteps, linear_start=linear_start, linear_end=linear_end)
@property
def device(self):
return self.betas.device
def add_lora_adapters(self):
config = LoraConfig(
r=32, init_lora_weights="gaussian",
target_modules=["attn1.to_k", "attn1.to_q", "attn1.to_v", "attn1.to_out.0",
"attn2.to_k", "attn2.to_q", "attn2.to_v", "attn2.to_out.0",
"conv", "conv1", "conv2", "conv_shortcut", "conv_out",
"proj_in", "proj_out", "ff.net.2", "ff.net.0.proj"],
)
self.denoiser = get_peft_model(self.denoiser, config)
def register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000,
linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3):
if exists(given_betas):
betas = given_betas
else:
betas = make_beta_schedule(beta_schedule, timesteps, linear_start=linear_start, linear_end=linear_end,
cosine_s=cosine_s)
alphas = 1. - betas
alphas_cumprod = np.cumprod(alphas, axis=0)
alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1])
timesteps, = betas.shape
self.num_timesteps = int(timesteps)
self.linear_start = linear_start
self.linear_end = linear_end
assert alphas_cumprod.shape[0] == self.num_timesteps, 'alphas have to be defined for each timestep'
to_torch = partial(torch.tensor, dtype=torch.float32)
self.register_buffer('betas', to_torch(betas))
self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod))
self.register_buffer('alphas_cumprod_prev', to_torch(alphas_cumprod_prev))
self.register_buffer('sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod)))
self.register_buffer('sqrt_one_minus_alphas_cumprod', to_torch(np.sqrt(1. - alphas_cumprod)))
self.register_buffer('log_one_minus_alphas_cumprod', to_torch(np.log(1. - alphas_cumprod)))
self.register_buffer('sqrt_recip_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod)))
self.register_buffer('sqrt_recipm1_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod - 1)))
posterior_variance = (1 - self.v_posterior) * betas * (1. - alphas_cumprod_prev) / (
1. - alphas_cumprod) + self.v_posterior * betas
self.register_buffer('posterior_variance', to_torch(posterior_variance))
self.register_buffer('posterior_log_variance_clipped', to_torch(np.log(np.maximum(posterior_variance, 1e-20))))
self.register_buffer('posterior_mean_coef1', to_torch(
betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)))
self.register_buffer('posterior_mean_coef2', to_torch(
(1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod)))
def predict_start_from_noise(self, x_t, t, noise):
return (
extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t -
extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise
)
def q_posterior(self, x_start, x_t, t):
posterior_mean = (
extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) * x_start +
extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t
)
posterior_variance = extract_into_tensor(self.posterior_variance, t, x_t.shape)
posterior_log_variance_clipped = extract_into_tensor(self.posterior_log_variance_clipped, t, x_t.shape)
return posterior_mean, posterior_variance, posterior_log_variance_clipped
def q_sample(self, x_start, t, noise=None):
noise = default(noise, lambda: torch.randn_like(x_start))
return (extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise)
def add_noise_with_schedule(self, x_start, t, sqrt_alphas_cumprod, sqrt_one_minus_alphas_cumprod, noise=None):
noise = default(noise, lambda: torch.randn_like(x_start))
return (extract_into_tensor(sqrt_alphas_cumprod.to(noise.device), t, x_start.shape) * x_start +
extract_into_tensor(sqrt_one_minus_alphas_cumprod.to(noise.device), t, x_start.shape) * noise)
def sample_latent(self, encoder_posterior):
if isinstance(encoder_posterior, DiagonalGaussianDistribution):
z = encoder_posterior.sample()
elif isinstance(encoder_posterior, torch.Tensor):
z = encoder_posterior
else:
raise TypeError(type(encoder_posterior))
return self.scale_factor * z
def apply_model(self, x_noisy, t, cond, struct_cond, diff_prior_c):
if not isinstance(cond, dict):
cond = {"c_crossattn": cond if isinstance(cond, list) else [cond]}
return self.denoiser(x_noisy, t, **cond, struct_cond=struct_cond, prior_cond=diff_prior_c)
def predict_noise(self, x, t, cond, struct_cond, prior):
cond_prior, diffusion_prior = self.prior_condition_encoder(prior, t)
struct_features = self.structure_encoder(struct_cond, t, cond_prior)
return self.apply_model(x, t, cond, struct_features, diffusion_prior)
class MultiStepVideoEnhancer(VideoDiffusion):
@torch.no_grad()
def sample(self, cond, struct_cond, prior, x_T, timesteps=50, verbose=True):
if not 2 <= timesteps <= len(self.betas):
raise ValueError("Multi-step inference requires at least two respaced timesteps")
img = x_T
cond = cond[:img.shape[0]]
iterator = reversed(range(timesteps))
if verbose:
iterator = tqdm(iterator, desc="Sampling t", total=timesteps)
for i in iterator:
ts = torch.full((img.shape[0],), i, device=self.device, dtype=torch.long)
original_t = torch.full_like(ts, self.original_timesteps[i])
noise_pred = self.predict_noise(img, original_t, cond, struct_cond, prior)
x_recon = self.predict_start_from_noise(img, ts, noise_pred)
mean, _, log_variance = self.q_posterior(x_recon, img, ts)
noise = torch.randn(img.shape, device=img.device)
nonzero_mask = (1 - (ts == 0).float()).reshape(-1, 1, 1, 1)
img = mean + nonzero_mask * (0.5 * log_variance).exp() * noise
return img
class SingleStepVideoEnhancer(VideoDiffusion):
def __init__(self, vae_config, **kwargs):
super().__init__(**kwargs)
self.vae = instantiate_from_config(vae_config)
@torch.no_grad()
def sample(self, cond, struct_cond, prior, x_T, timesteps=1, verbose=True):
if timesteps != 1:
raise ValueError("The distilled checkpoint supports exactly one sampling step")
ts = torch.full((x_T.shape[0],), 99, device=self.device, dtype=torch.long)
noise_pred = self.predict_noise(x_T, ts, cond[:x_T.shape[0]], struct_cond, prior)
return self.predict_start_from_noise(x_T, ts, noise_pred)