Spaces:
Running on Zero
Running on Zero
Download diffcve/models/diffusion.py from hugging-apps/diffcve-2607: direct link, hf CLI and curl.
- Browser
- Download file 9.51 kB
-
https://huggingface.co/spaces/hugging-apps/diffcve-2607/resolve/main/diffcve/models/diffusion.py
- Command line
-
hf download hf://spaces/hugging-apps/diffcve-2607/diffcve/models/diffusion.py
-
curl -L -o diffusion.py https://huggingface.co/spaces/hugging-apps/diffcve-2607/resolve/main/diffcve/models/diffusion.py
9.51 kB
| from functools import partial | |
| import numpy as np | |
| import torch | |
| from torch import nn | |
| from peft import LoraConfig, get_peft_model | |
| from tqdm import tqdm | |
| from diffcve.utils import default, exists, instantiate_from_config | |
| from diffcve.modules.diffusion.utils import make_beta_schedule, extract_into_tensor | |
| from diffcve.modules.distributions import DiagonalGaussianDistribution | |
| from diffcve.modules.text_encoder import load_text_conditions | |
| class ConditionedDenoiser(nn.Module): | |
| def __init__(self, network_config, conditioning_key="crossattn"): | |
| super().__init__() | |
| if conditioning_key != "crossattn": | |
| raise ValueError("DiffCVE requires cross-attention conditioning") | |
| self.conditioning_key = conditioning_key | |
| self.network = instantiate_from_config(network_config) | |
| def forward(self, x, t, c_crossattn, struct_cond=None, prior_cond=None): | |
| return self.network( | |
| x, t, context=torch.cat(c_crossattn, 1), | |
| struct_cond=struct_cond, prior_cond=prior_cond, | |
| ) | |
| class VideoDiffusion(nn.Module): | |
| def __init__(self, denoiser_config, prior_encoder_config, prior_condition_config, | |
| structure_encoder_config, timesteps=1000, linear_start=0.00085, | |
| linear_end=0.012, num_frames=5, scale_factor=0.18215, | |
| conditioning_key="crossattn", parameterization="eps", | |
| text_conditions_path="checkpoints/text_conditions.pt", | |
| cond_stage_config=None, text_prompts=None, text_condition_qps=None): | |
| super().__init__() | |
| if parameterization != "eps": | |
| raise ValueError("DiffCVE requires noise prediction (eps)") | |
| self.parameterization = parameterization | |
| self.num_frames = num_frames | |
| self.scale_factor = scale_factor | |
| self.v_posterior = 0.0 | |
| self.denoiser = ConditionedDenoiser(denoiser_config, conditioning_key) | |
| self.prior_encoder = instantiate_from_config(prior_encoder_config) | |
| self.prior_condition_encoder = instantiate_from_config(prior_condition_config) | |
| self.structure_encoder = instantiate_from_config(structure_encoder_config) | |
| self.add_lora_adapters() | |
| self.text_conditions = load_text_conditions( | |
| text_conditions_path, cond_stage_config, text_prompts, text_condition_qps, | |
| ) | |
| self.register_schedule(timesteps=timesteps, linear_start=linear_start, linear_end=linear_end) | |
| def device(self): | |
| return self.betas.device | |
| def add_lora_adapters(self): | |
| config = LoraConfig( | |
| r=32, init_lora_weights="gaussian", | |
| target_modules=["attn1.to_k", "attn1.to_q", "attn1.to_v", "attn1.to_out.0", | |
| "attn2.to_k", "attn2.to_q", "attn2.to_v", "attn2.to_out.0", | |
| "conv", "conv1", "conv2", "conv_shortcut", "conv_out", | |
| "proj_in", "proj_out", "ff.net.2", "ff.net.0.proj"], | |
| ) | |
| self.denoiser = get_peft_model(self.denoiser, config) | |
| def register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000, | |
| linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3): | |
| if exists(given_betas): | |
| betas = given_betas | |
| else: | |
| betas = make_beta_schedule(beta_schedule, timesteps, linear_start=linear_start, linear_end=linear_end, | |
| cosine_s=cosine_s) | |
| alphas = 1. - betas | |
| alphas_cumprod = np.cumprod(alphas, axis=0) | |
| alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1]) | |
| timesteps, = betas.shape | |
| self.num_timesteps = int(timesteps) | |
| self.linear_start = linear_start | |
| self.linear_end = linear_end | |
| assert alphas_cumprod.shape[0] == self.num_timesteps, 'alphas have to be defined for each timestep' | |
| to_torch = partial(torch.tensor, dtype=torch.float32) | |
| self.register_buffer('betas', to_torch(betas)) | |
| self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) | |
| self.register_buffer('alphas_cumprod_prev', to_torch(alphas_cumprod_prev)) | |
| self.register_buffer('sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod))) | |
| self.register_buffer('sqrt_one_minus_alphas_cumprod', to_torch(np.sqrt(1. - alphas_cumprod))) | |
| self.register_buffer('log_one_minus_alphas_cumprod', to_torch(np.log(1. - alphas_cumprod))) | |
| self.register_buffer('sqrt_recip_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod))) | |
| self.register_buffer('sqrt_recipm1_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod - 1))) | |
| posterior_variance = (1 - self.v_posterior) * betas * (1. - alphas_cumprod_prev) / ( | |
| 1. - alphas_cumprod) + self.v_posterior * betas | |
| self.register_buffer('posterior_variance', to_torch(posterior_variance)) | |
| self.register_buffer('posterior_log_variance_clipped', to_torch(np.log(np.maximum(posterior_variance, 1e-20)))) | |
| self.register_buffer('posterior_mean_coef1', to_torch( | |
| betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod))) | |
| self.register_buffer('posterior_mean_coef2', to_torch( | |
| (1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod))) | |
| def predict_start_from_noise(self, x_t, t, noise): | |
| return ( | |
| extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - | |
| extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise | |
| ) | |
| def q_posterior(self, x_start, x_t, t): | |
| posterior_mean = ( | |
| extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) * x_start + | |
| extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t | |
| ) | |
| posterior_variance = extract_into_tensor(self.posterior_variance, t, x_t.shape) | |
| posterior_log_variance_clipped = extract_into_tensor(self.posterior_log_variance_clipped, t, x_t.shape) | |
| return posterior_mean, posterior_variance, posterior_log_variance_clipped | |
| def q_sample(self, x_start, t, noise=None): | |
| noise = default(noise, lambda: torch.randn_like(x_start)) | |
| return (extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + | |
| extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise) | |
| def add_noise_with_schedule(self, x_start, t, sqrt_alphas_cumprod, sqrt_one_minus_alphas_cumprod, noise=None): | |
| noise = default(noise, lambda: torch.randn_like(x_start)) | |
| return (extract_into_tensor(sqrt_alphas_cumprod.to(noise.device), t, x_start.shape) * x_start + | |
| extract_into_tensor(sqrt_one_minus_alphas_cumprod.to(noise.device), t, x_start.shape) * noise) | |
| def sample_latent(self, encoder_posterior): | |
| if isinstance(encoder_posterior, DiagonalGaussianDistribution): | |
| z = encoder_posterior.sample() | |
| elif isinstance(encoder_posterior, torch.Tensor): | |
| z = encoder_posterior | |
| else: | |
| raise TypeError(type(encoder_posterior)) | |
| return self.scale_factor * z | |
| def apply_model(self, x_noisy, t, cond, struct_cond, diff_prior_c): | |
| if not isinstance(cond, dict): | |
| cond = {"c_crossattn": cond if isinstance(cond, list) else [cond]} | |
| return self.denoiser(x_noisy, t, **cond, struct_cond=struct_cond, prior_cond=diff_prior_c) | |
| def predict_noise(self, x, t, cond, struct_cond, prior): | |
| cond_prior, diffusion_prior = self.prior_condition_encoder(prior, t) | |
| struct_features = self.structure_encoder(struct_cond, t, cond_prior) | |
| return self.apply_model(x, t, cond, struct_features, diffusion_prior) | |
| class MultiStepVideoEnhancer(VideoDiffusion): | |
| def sample(self, cond, struct_cond, prior, x_T, timesteps=50, verbose=True): | |
| if not 2 <= timesteps <= len(self.betas): | |
| raise ValueError("Multi-step inference requires at least two respaced timesteps") | |
| img = x_T | |
| cond = cond[:img.shape[0]] | |
| iterator = reversed(range(timesteps)) | |
| if verbose: | |
| iterator = tqdm(iterator, desc="Sampling t", total=timesteps) | |
| for i in iterator: | |
| ts = torch.full((img.shape[0],), i, device=self.device, dtype=torch.long) | |
| original_t = torch.full_like(ts, self.original_timesteps[i]) | |
| noise_pred = self.predict_noise(img, original_t, cond, struct_cond, prior) | |
| x_recon = self.predict_start_from_noise(img, ts, noise_pred) | |
| mean, _, log_variance = self.q_posterior(x_recon, img, ts) | |
| noise = torch.randn(img.shape, device=img.device) | |
| nonzero_mask = (1 - (ts == 0).float()).reshape(-1, 1, 1, 1) | |
| img = mean + nonzero_mask * (0.5 * log_variance).exp() * noise | |
| return img | |
| class SingleStepVideoEnhancer(VideoDiffusion): | |
| def __init__(self, vae_config, **kwargs): | |
| super().__init__(**kwargs) | |
| self.vae = instantiate_from_config(vae_config) | |
| def sample(self, cond, struct_cond, prior, x_T, timesteps=1, verbose=True): | |
| if timesteps != 1: | |
| raise ValueError("The distilled checkpoint supports exactly one sampling step") | |
| ts = torch.full((x_T.shape[0],), 99, device=self.device, dtype=torch.long) | |
| noise_pred = self.predict_noise(x_T, ts, cond[:x_T.shape[0]], struct_cond, prior) | |
| return self.predict_start_from_noise(x_T, ts, noise_pred) | |