Spaces:
Sleeping
Sleeping
File size: 11,297 Bytes
e6d357a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 | import math
from collections import namedtuple
import torch
from torch import nn
import torch.nn.functional as F
from tqdm.auto import tqdm
ModelPrediction = namedtuple("ModelPrediction", ["pred_noise", "pred_x_start"])
def identity(t, *args, **kwargs):
return t
def exists(x):
return x is not None
def default(val, d):
if exists(val):
return val
return d() if callable(d) else d
def extract(a, t, x_shape):
b, *_ = t.shape
out = a.gather(-1, t)
return out.reshape(1, b, *((1,) * (len(x_shape) - 2)))
def cosine_beta_schedule(timesteps, s=0.008):
"""cosine schedule as proposed in https://openreview.net/forum?id=-NEXDKk8gZ"""
steps = timesteps + 1
x = torch.linspace(0, timesteps, steps, dtype=torch.float64)
alphas_cumprod = torch.cos(((x / timesteps) + s) / (1 + s) * math.pi * 0.5) ** 2
alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
return torch.clip(betas, 0, 0.999)
class LatentDiffusion(nn.Module):
"""Gaussian diffusion in the latent space of the autoencoder (one latent vector per graph node), predicting x_0.
Subclasses define the forward process (q_sample, q_posterior, predict_noise_from_start, loss_funct)."""
def __init__(self, model: torch.nn.Module, latent_size=96, diffusion_timesteps=10, diffusion_objective="pred_x0",
sampling_timesteps=None, diffusion_activation="identity", diffusion_conditioning=False,
beta_schedule="cosine", ddim_sampling_eta=0.0, **kwargs):
super().__init__()
assert diffusion_objective == "pred_x0", "only the pred_x0 objective is implemented"
assert beta_schedule == "cosine", "only the cosine beta schedule is implemented"
self.enable_masking = kwargs.get("enable_masking", False)
self.activation = torch.nn.Tanh() if diffusion_activation == "tanh" else torch.nn.Identity()
self.silent = True
self.condition = diffusion_conditioning
self.model = model
self.channels = self.model.channels
self.self_condition = self.model.self_condition
self.seq_length = latent_size
self.objective = diffusion_objective
betas = cosine_beta_schedule(diffusion_timesteps)
alphas = 1.0 - betas
alphas_cumprod = torch.cumprod(alphas, dim=0)
alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.0)
self.num_timesteps = int(betas.shape[0])
self.sampling_timesteps = default(sampling_timesteps, self.num_timesteps)
assert self.sampling_timesteps <= self.num_timesteps
self.is_ddim_sampling = self.sampling_timesteps < self.num_timesteps
self.ddim_sampling_eta = ddim_sampling_eta
register_buffer = lambda name, val: self.register_buffer(name, val.to(torch.float32))
register_buffer("betas", betas)
register_buffer("alphas_cumprod", alphas_cumprod)
register_buffer("alphas_cumprod_prev", alphas_cumprod_prev)
register_buffer("sqrt_alphas_cumprod", torch.sqrt(alphas_cumprod))
print("Diffusion is_ddim_sampling: ", self.is_ddim_sampling)
def set_eval_dataidx(self, *args, **kwargs):
pass
# --- noise ---------------------------------------------------------------------------------------
def get_white_noise(self, x, *args, **kwargs):
return self.get_noise(x, *args, **kwargs)
def get_start_noise(self, x, *args, **kwargs):
return self.get_white_noise(x, *args, **kwargs)
def get_noise(self, x, *args, **kwargs):
"""x is either a tensor or a shape"""
if torch.is_tensor(x):
return torch.randn_like(x)
return torch.randn(*x, *args, **kwargs)
# --- to be defined by the diffusion process ---------------------------------------------------------
def q_sample(self, x_start, t, noise=None):
raise NotImplementedError
def q_posterior(self, x_start, x_t, t):
raise NotImplementedError
def predict_noise_from_start(self, x_t, t, x0):
raise NotImplementedError
def p_combine_mean_var_noise(self, model_mean, model_log_variance, noise):
raise NotImplementedError
def loss_funct(self, model_out, target, *args, **kwargs):
raise NotImplementedError
# --- denoiser -----------------------------------------------------------------------------------
def feed_model(self, x, t, x_self_cond=None, x_cond=None, **kwargs):
if self.condition:
assert x_cond is not None
assert len(x_cond.shape) == len(x.shape), f"x_cond must have same number of dimensions as x, but got {x_cond.shape} vs {x.shape}"
if x.shape[0] > x_cond.shape[0]:
x_cond = x_cond.expand(x.shape[0], -1, -1, -1)
model_output = self.model(x, t, x_self_cond, x_cond=x_cond, **kwargs)
return self.activation(model_output)
def model_predictions(self, x, t, x_self_cond=None, x_cond=None, clip_x_start=False, rederive_pred_noise=False,
**network_kwargs):
x_start = self.feed_model(x, t, x_self_cond=x_self_cond, x_cond=x_cond, **network_kwargs)
if clip_x_start:
x_start = torch.clamp(x_start, min=-1.0, max=1.0)
pred_noise = self.predict_noise_from_start(x, t, x_start)
return ModelPrediction(pred_noise, x_start)
# --- training -----------------------------------------------------------------------------------
def p_losses(self, x_start, t, noise=None, x_cond=None, num_samples=1, difftrain_kwargs={}, **network_kwargs):
"""Loss of num_samples noisy versions of x_start [B, N, D] -> ([num_samples, B], loss weights, model output)."""
b, c, n = x_start.shape
if x_cond is not None:
x_cond = x_cond.unsqueeze(0)
noise = default(noise, self.get_white_noise(x_start, num_samples=num_samples))
x_start = x_start.unsqueeze(0)
x = self.q_sample(x_start=x_start, t=t, noise=noise, **difftrain_kwargs)
model_out = self.feed_model(x, t, x_self_cond=None, x_cond=x_cond, **network_kwargs)
loss = self.loss_funct(model_out, x_start, t, difftrain_kwargs=difftrain_kwargs)
if self.enable_masking:
mask = network_kwargs["mask"].squeeze(-1)
num_joints = mask.sum(-1)
loss = loss.mean(-1).sum(-1) / num_joints # mean over the real (non-padded) nodes
assert len(loss.shape) == 2, f"loss shape {loss.shape} != 2"
else:
loss = loss.mean(dim=tuple(range(2, loss.dim()))) # [num_samples, B]
return loss, extract(self.loss_weight, t.view(b, -1)[:, 0], loss.shape[0:1]), model_out
def forward(self, x, *args, x_cond=None, **kwargs):
b, c, n, device, seq_length, = *x.shape, x.device, self.seq_length
assert n == seq_length, f"seq length must be {seq_length}"
t = torch.randint(0, self.num_timesteps, (b,), device=device).long()
return self.p_losses(x, t, *args, x_cond=x_cond, **kwargs)
# --- sampling -----------------------------------------------------------------------------------
def p_mean_variance(self, x, t, x_self_cond=None, x_cond=None, clip_denoised=True, **network_kwargs):
preds = self.model_predictions(x, t, x_self_cond, x_cond=x_cond, **network_kwargs)
x_start = preds.pred_x_start
if clip_denoised:
x_start.clamp_(-1.0, 1.0)
model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start=x_start, x_t=x, t=t)
return model_mean, posterior_variance, posterior_log_variance, x_start
@torch.no_grad()
def p_sample(self, x, t: int, x_self_cond=None, clip_denoised=True, *args, perm=None, **kwargs):
s, b, *_, device = *x.shape, x.device
batched_times = torch.full((b,), t, device=x.device, dtype=torch.long)
model_mean, _, model_log_variance, x_start = self.p_mean_variance(
x=x, t=batched_times, x_self_cond=x_self_cond, clip_denoised=clip_denoised, *args, **kwargs)
noise = self.get_white_noise(x, perm=perm) if t > 0 else 0.0 # no noise at t == 0
pred_img = self.p_combine_mean_var_noise(model_mean, model_log_variance, noise)
return pred_img, x_start, noise, model_mean
@torch.no_grad()
def p_sample_loop(self, shape, x_cond=None, start_noise=None, perm=None, **kwargs):
device = self.betas.device
if start_noise is not None:
assert start_noise.shape == shape, f"Shape mismatch: {start_noise.shape} != {shape}"
img = start_noise
else:
img = self.get_start_noise(shape, device=device, perm=perm)
noise = img.clone()
x_start = None
for t in reversed(range(0, self.num_timesteps)):
self_cond = x_start if self.self_condition else None
img, x_start, _, _ = self.p_sample(img, t, self_cond, x_cond=x_cond, perm=perm, **kwargs)
return img, noise
def ddim_sample(self, shape, clip_denoised = True, x_cond=None, start_noise=None, **network_kwargs):
batch, device, total_timesteps, sampling_timesteps, eta, objective = shape[0], self.betas.device, self.num_timesteps, self.sampling_timesteps, self.ddim_sampling_eta, self.objective
times = torch.linspace(0, total_timesteps-1, steps=sampling_timesteps + 1) # [-1, 0, 1, 2, ..., T-1] when sampling_timesteps == total_timesteps
times = list(reversed(times.int().tolist()))
time_pairs = list(zip(times[:-1], times[1:])) # [(T-1, T-2), (T-2, T-3), ..., (1, 0), (0, -1)]
if start_noise is not None:
assert start_noise.shape == shape
img = start_noise
noise = start_noise.clone()
else:
img = torch.randn(shape, device=device)
noise = img.clone()
x_start = None
if not self.silent:
print(f"Evaluation with {len(time_pairs)} diffusion steps")
for time, time_next in tqdm(time_pairs, desc = 'sampling loop time step'):
time_cond = torch.full((batch,), time, device=device, dtype=torch.long)
self_cond = x_start if self.self_condition else None
pred_noise, x_start, *_ = self.model_predictions(img, time_cond, self_cond, x_cond=x_cond, clip_x_start = clip_denoised, **network_kwargs)
if time_next < 0:
img = x_start
continue
alpha = self.alphas_cumprod[time]
alpha_next = self.alphas_cumprod[time_next]
sqrt_alpha_next = self.sqrt_alphas_cumprod[time_next]
sigma = eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt() #eta*beta_t_tilde
c = (1 - alpha_next - sigma ** 2).sqrt()
noise = torch.randn_like(img)
img = x_start * sqrt_alpha_next + \
c * pred_noise + \
sigma * noise
return img, noise
@torch.no_grad()
def sample(self, batch_size=16, num_samples=50, *args, **kwargs):
sample_fn = self.p_sample_loop if not self.is_ddim_sampling else self.ddim_sample
shape = (num_samples, batch_size, self.channels, self.seq_length)
return sample_fn(shape, *args, **kwargs)
|