Spaces:
Sleeping
Sleeping
File size: 7,164 Bytes
e6d357a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | from typing import Dict
import torch
from torch.amp import autocast
from .base import LatentDiffusion, extract, default
def extract_node_scale(scale: torch.Tensor, t: torch.Tensor, perm_idxs: torch.Tensor = None):
"""scale: [T, N] (all samples from one skeleton) or [B, T, N] (per-sample skeletons) -> [1, B, N, 1]
at timesteps t, with the nodes permuted like the input sample if perm_idxs [B, N] is given."""
if scale.dim() == 2:
out = scale[t] # [B, N]
else:
out = scale[torch.arange(len(t), device=t.device), t] # [B, N]
if perm_idxs is not None:
out = torch.gather(out, -1, perm_idxs)
return out.unsqueeze(0).unsqueeze(-1)
def apply_permutation_on_vec(vec, perm):
return (vec.transpose(-1, -2) @ perm.mT).transpose(-2, -1)
class PermutableGaussianDiffusion(LatentDiffusion):
"""Isotropic Gaussian latent diffusion over the nodes of skeleton graphs of different sizes.
Differs from SkeletonDiffusion's NonisotropicGaussianDiffusion: the noise is isotropic (i.i.d. per node),
so the diffusion process is equivariant to permutations of the graph nodes. Skeletons with fewer nodes
than `num_nodes` are zero-padded. All coefficients are per-node scales [T, N]; they are equal to the
diagonals of the (identity-covariance) matrices of the released models' implementation, and nodes
beyond a dataset's skeleton size keep the padding value of that implementation (they carry zeros).
"""
def __init__(self, num_nodes_per_dataset: Dict[str, int], num_nodes: int, loss_reduction_type: str = "l1",
**kwargs):
super().__init__(**kwargs)
assert self.objective == "pred_x0", "only the pred_x0 objective is implemented"
self.enable_masking = True
self.loss_reduction_type = loss_reduction_type
self.dataset_names = sorted(num_nodes_per_dataset.keys())
self.data_name2idx = {name: i for i, name in enumerate(self.dataset_names)}
# Variance schedule of one node (float32 arithmetic of the original implementation)
alphas = 1.0 - self.betas
var_t = 1 - alphas # variance added at step t
var_bar_t = 1 - self.alphas_cumprod # variance of x_t given x_0
var_bar_t_prev = torch.cat([torch.zeros(1), var_bar_t[:-1]])
posterior_var = var_t * var_bar_t_prev * (1 / var_bar_t)
per_step = { # name: (scale per timestep, value on padded nodes)
"sqrt_var_bar": (torch.sqrt(var_bar_t), 1.0), # x_t = sqrt(ac) x0 + sqrt(var_bar) eps
"recip_sqrt_var_bar": (1 / torch.sqrt(var_bar_t), 1.0), # eps from (x_t, x0)
"sqrt_ac_recip_sqrt_var_bar": ((1 / torch.sqrt(var_bar_t)) * self.sqrt_alphas_cumprod, 1.0),
"mahalanobis": (torch.sqrt(1.0 / var_bar_t), 1.0), # loss weighting of the difference
"posterior_coef1": (torch.sqrt(self.alphas_cumprod_prev) * ((1 / var_bar_t) * var_t), 1.0),
"posterior_coef2": (torch.sqrt(alphas) * ((1 / var_bar_t) * var_bar_t_prev), 1.0),
"posterior_log_variance_clipped": (torch.log(posterior_var.clamp(min=1e-20)), 0.0),
}
for name, (per_t, pad_value) in per_step.items():
scales = []
for dname in self.dataset_names:
s = torch.full((len(per_t), num_nodes), pad_value, dtype=torch.float32)
s[:, : num_nodes_per_dataset[dname]] = per_t.unsqueeze(-1)
scales.append(s)
self.register_buffer(name, torch.stack(scales), persistent=False) # [D, T, N]
self.register_buffer("loss_weight", self.alphas_cumprod.clone(), persistent=False) # objective pred_x0
self.eval_idx = 0
def set_eval_dataidx(self, name: str, device="cuda"):
"""At inference, all samples of a batch come from the same dataset (skeleton)."""
self.eval_idx = self.data_name2idx[name]
def _scale(self, name, t, dataidx=None, perm=None):
scale = getattr(self, name)
if dataidx is None:
scale = scale[self.eval_idx] # [T, N]
else:
scale = scale[torch.tensor([self.data_name2idx[d] for d in dataidx], device=t.device)] # [B, T, N]
perm_idxs = None if perm is None else perm.argmax(-1) # permutation matrix -> index vector
return extract_node_scale(scale, t, perm_idxs)
# --- noise -------------------------------------------------------------------------------------
def get_white_noise(self, x_start, *args, num_samples=None, perm=None, **kwargs):
mask = x_start.sum(-1) == 0 # padded nodes stay zero
if num_samples is not None:
noise = self.get_noise((num_samples, *x_start.shape), *args, device=x_start.device)
noise[:, mask] = 0
else:
noise = self.get_noise(x_start.shape, *args, device=x_start.device)
noise[mask] = 0
if perm is not None:
noise = apply_permutation_on_vec(noise, perm)
return noise
def get_start_noise(self, shape, mask=None, *args, perm=None, **kwargs):
noise = self.get_noise(shape, *args, **kwargs)
if perm is not None:
noise = apply_permutation_on_vec(noise, perm)
if mask is not None:
noise = noise * mask
return noise
# --- forward process ---------------------------------------------------------------------------
@autocast("cuda", enabled=False)
def q_sample(self, x_start, t, noise=None, perm=None, dataidx=None):
noise = default(noise, lambda: self.get_white_noise(x_start))
return extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + \
self._scale("sqrt_var_bar", t, dataidx, perm) * noise
def predict_noise_from_start(self, x_t, t, x0):
return self._scale("recip_sqrt_var_bar", t) * x_t - self._scale("sqrt_ac_recip_sqrt_var_bar", t) * x0
def loss_funct(self, model_out, target, t, difftrain_kwargs={}):
loss = (self._scale("mahalanobis", t, **difftrain_kwargs) * (model_out - target)).abs()
if self.loss_reduction_type == "mse":
loss = loss ** 2
return loss
# --- reverse process ---------------------------------------------------------------------------
def q_posterior(self, x_start, x_t, t):
mean = self._scale("posterior_coef1", t) * x_start + self._scale("posterior_coef2", t) * x_t
log_variance = self._scale("posterior_log_variance_clipped", t)
return mean, log_variance.exp(), log_variance
def p_combine_mean_var_noise(self, model_mean, posterior_log_variance, noise):
return model_mean + (0.5 * posterior_log_variance).exp() * noise
@torch.no_grad()
def sample(self, batch_size=16, num_samples=50, *args, mask=None, perm=None, **kwargs):
shape = (num_samples, batch_size, self.channels, self.seq_length)
start_noise = self.get_start_noise(shape, mask=mask, device=self.betas.device, perm=perm)
return self.p_sample_loop(shape, start_noise=start_noise, *args, mask=mask, perm=perm, **kwargs)
|