File size: 11,297 Bytes
e6d357a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
import math
from collections import namedtuple

import torch
from torch import nn
import torch.nn.functional as F
from tqdm.auto import tqdm

ModelPrediction = namedtuple("ModelPrediction", ["pred_noise", "pred_x_start"])


def identity(t, *args, **kwargs):
    return t


def exists(x):
    return x is not None


def default(val, d):
    if exists(val):
        return val
    return d() if callable(d) else d


def extract(a, t, x_shape):
    b, *_ = t.shape
    out = a.gather(-1, t)
    return out.reshape(1, b, *((1,) * (len(x_shape) - 2)))


def cosine_beta_schedule(timesteps, s=0.008):
    """cosine schedule as proposed in https://openreview.net/forum?id=-NEXDKk8gZ"""
    steps = timesteps + 1
    x = torch.linspace(0, timesteps, steps, dtype=torch.float64)
    alphas_cumprod = torch.cos(((x / timesteps) + s) / (1 + s) * math.pi * 0.5) ** 2
    alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
    betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
    return torch.clip(betas, 0, 0.999)


class LatentDiffusion(nn.Module):
    """Gaussian diffusion in the latent space of the autoencoder (one latent vector per graph node), predicting x_0.
    Subclasses define the forward process (q_sample, q_posterior, predict_noise_from_start, loss_funct)."""

    def __init__(self, model: torch.nn.Module, latent_size=96, diffusion_timesteps=10, diffusion_objective="pred_x0",
                 sampling_timesteps=None, diffusion_activation="identity", diffusion_conditioning=False,
                 beta_schedule="cosine", ddim_sampling_eta=0.0, **kwargs):
        super().__init__()
        assert diffusion_objective == "pred_x0", "only the pred_x0 objective is implemented"
        assert beta_schedule == "cosine", "only the cosine beta schedule is implemented"
        self.enable_masking = kwargs.get("enable_masking", False)
        self.activation = torch.nn.Tanh() if diffusion_activation == "tanh" else torch.nn.Identity()
        self.silent = True
        self.condition = diffusion_conditioning
        self.model = model
        self.channels = self.model.channels
        self.self_condition = self.model.self_condition
        self.seq_length = latent_size
        self.objective = diffusion_objective

        betas = cosine_beta_schedule(diffusion_timesteps)
        alphas = 1.0 - betas
        alphas_cumprod = torch.cumprod(alphas, dim=0)
        alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.0)
        self.num_timesteps = int(betas.shape[0])
        self.sampling_timesteps = default(sampling_timesteps, self.num_timesteps)
        assert self.sampling_timesteps <= self.num_timesteps
        self.is_ddim_sampling = self.sampling_timesteps < self.num_timesteps
        self.ddim_sampling_eta = ddim_sampling_eta
        register_buffer = lambda name, val: self.register_buffer(name, val.to(torch.float32))
        register_buffer("betas", betas)
        register_buffer("alphas_cumprod", alphas_cumprod)
        register_buffer("alphas_cumprod_prev", alphas_cumprod_prev)
        register_buffer("sqrt_alphas_cumprod", torch.sqrt(alphas_cumprod))
        print("Diffusion is_ddim_sampling: ", self.is_ddim_sampling)

    def set_eval_dataidx(self, *args, **kwargs):
        pass

    # --- noise ---------------------------------------------------------------------------------------
    def get_white_noise(self, x, *args, **kwargs):
        return self.get_noise(x, *args, **kwargs)

    def get_start_noise(self, x, *args, **kwargs):
        return self.get_white_noise(x, *args, **kwargs)

    def get_noise(self, x, *args, **kwargs):
        """x is either a tensor or a shape"""
        if torch.is_tensor(x):
            return torch.randn_like(x)
        return torch.randn(*x, *args, **kwargs)

    # --- to be defined by the diffusion process ---------------------------------------------------------
    def q_sample(self, x_start, t, noise=None):
        raise NotImplementedError

    def q_posterior(self, x_start, x_t, t):
        raise NotImplementedError

    def predict_noise_from_start(self, x_t, t, x0):
        raise NotImplementedError

    def p_combine_mean_var_noise(self, model_mean, model_log_variance, noise):
        raise NotImplementedError

    def loss_funct(self, model_out, target, *args, **kwargs):
        raise NotImplementedError

    # --- denoiser -----------------------------------------------------------------------------------
    def feed_model(self, x, t, x_self_cond=None, x_cond=None, **kwargs):
        if self.condition:
            assert x_cond is not None
            assert len(x_cond.shape) == len(x.shape), f"x_cond must have same number of dimensions as x, but got {x_cond.shape} vs {x.shape}"
            if x.shape[0] > x_cond.shape[0]:
                x_cond = x_cond.expand(x.shape[0], -1, -1, -1)
        model_output = self.model(x, t, x_self_cond, x_cond=x_cond, **kwargs)
        return self.activation(model_output)

    def model_predictions(self, x, t, x_self_cond=None, x_cond=None, clip_x_start=False, rederive_pred_noise=False,
                          **network_kwargs):
        x_start = self.feed_model(x, t, x_self_cond=x_self_cond, x_cond=x_cond, **network_kwargs)
        if clip_x_start:
            x_start = torch.clamp(x_start, min=-1.0, max=1.0)
        pred_noise = self.predict_noise_from_start(x, t, x_start)
        return ModelPrediction(pred_noise, x_start)

    # --- training -----------------------------------------------------------------------------------
    def p_losses(self, x_start, t, noise=None, x_cond=None, num_samples=1, difftrain_kwargs={}, **network_kwargs):
        """Loss of num_samples noisy versions of x_start [B, N, D] -> ([num_samples, B], loss weights, model output)."""
        b, c, n = x_start.shape
        if x_cond is not None:
            x_cond = x_cond.unsqueeze(0)
        noise = default(noise, self.get_white_noise(x_start, num_samples=num_samples))
        x_start = x_start.unsqueeze(0)
        x = self.q_sample(x_start=x_start, t=t, noise=noise, **difftrain_kwargs)
        model_out = self.feed_model(x, t, x_self_cond=None, x_cond=x_cond, **network_kwargs)
        loss = self.loss_funct(model_out, x_start, t, difftrain_kwargs=difftrain_kwargs)
        if self.enable_masking:
            mask = network_kwargs["mask"].squeeze(-1)
            num_joints = mask.sum(-1)
            loss = loss.mean(-1).sum(-1) / num_joints  # mean over the real (non-padded) nodes
            assert len(loss.shape) == 2, f"loss shape {loss.shape} != 2"
        else:
            loss = loss.mean(dim=tuple(range(2, loss.dim())))  # [num_samples, B]
        return loss, extract(self.loss_weight, t.view(b, -1)[:, 0], loss.shape[0:1]), model_out

    def forward(self, x, *args, x_cond=None, **kwargs):
        b, c, n, device, seq_length, = *x.shape, x.device, self.seq_length
        assert n == seq_length, f"seq length must be {seq_length}"
        t = torch.randint(0, self.num_timesteps, (b,), device=device).long()
        return self.p_losses(x, t, *args, x_cond=x_cond, **kwargs)

    # --- sampling -----------------------------------------------------------------------------------
    def p_mean_variance(self, x, t, x_self_cond=None, x_cond=None, clip_denoised=True, **network_kwargs):
        preds = self.model_predictions(x, t, x_self_cond, x_cond=x_cond, **network_kwargs)
        x_start = preds.pred_x_start
        if clip_denoised:
            x_start.clamp_(-1.0, 1.0)
        model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start=x_start, x_t=x, t=t)
        return model_mean, posterior_variance, posterior_log_variance, x_start

    @torch.no_grad()
    def p_sample(self, x, t: int, x_self_cond=None, clip_denoised=True, *args, perm=None, **kwargs):
        s, b, *_, device = *x.shape, x.device
        batched_times = torch.full((b,), t, device=x.device, dtype=torch.long)
        model_mean, _, model_log_variance, x_start = self.p_mean_variance(
            x=x, t=batched_times, x_self_cond=x_self_cond, clip_denoised=clip_denoised, *args, **kwargs)
        noise = self.get_white_noise(x, perm=perm) if t > 0 else 0.0  # no noise at t == 0
        pred_img = self.p_combine_mean_var_noise(model_mean, model_log_variance, noise)
        return pred_img, x_start, noise, model_mean

    @torch.no_grad()
    def p_sample_loop(self, shape, x_cond=None, start_noise=None, perm=None, **kwargs):
        device = self.betas.device
        if start_noise is not None:
            assert start_noise.shape == shape, f"Shape mismatch: {start_noise.shape} != {shape}"
            img = start_noise
        else:
            img = self.get_start_noise(shape, device=device, perm=perm)
        noise = img.clone()
        x_start = None
        for t in reversed(range(0, self.num_timesteps)):
            self_cond = x_start if self.self_condition else None
            img, x_start, _, _ = self.p_sample(img, t, self_cond, x_cond=x_cond, perm=perm, **kwargs)
        return img, noise

    def ddim_sample(self, shape, clip_denoised = True, x_cond=None, start_noise=None, **network_kwargs):
        batch, device, total_timesteps, sampling_timesteps, eta, objective = shape[0], self.betas.device, self.num_timesteps, self.sampling_timesteps, self.ddim_sampling_eta, self.objective

        times = torch.linspace(0, total_timesteps-1, steps=sampling_timesteps + 1)   # [-1, 0, 1, 2, ..., T-1] when sampling_timesteps == total_timesteps
        times = list(reversed(times.int().tolist()))
        time_pairs = list(zip(times[:-1], times[1:])) # [(T-1, T-2), (T-2, T-3), ..., (1, 0), (0, -1)]
        
        if start_noise is not None:
            assert start_noise.shape == shape
            img = start_noise
            noise = start_noise.clone()
        else:
            img = torch.randn(shape, device=device)
            noise = img.clone()

        x_start = None
        if not self.silent:
            print(f"Evaluation with {len(time_pairs)} diffusion steps")
        for time, time_next in tqdm(time_pairs, desc = 'sampling loop time step'):
            time_cond = torch.full((batch,), time, device=device, dtype=torch.long)
            self_cond = x_start if self.self_condition else None
            pred_noise, x_start, *_ = self.model_predictions(img, time_cond, self_cond, x_cond=x_cond, clip_x_start = clip_denoised, **network_kwargs)

            if time_next < 0:
                img = x_start
                continue

            alpha = self.alphas_cumprod[time]
            alpha_next = self.alphas_cumprod[time_next]
            sqrt_alpha_next = self.sqrt_alphas_cumprod[time_next]

            sigma = eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt() #eta*beta_t_tilde
            c = (1 - alpha_next - sigma ** 2).sqrt()

            noise = torch.randn_like(img)

            img = x_start * sqrt_alpha_next + \
                  c * pred_noise + \
                  sigma * noise
        return img, noise

    @torch.no_grad()
    def sample(self, batch_size=16, num_samples=50, *args, **kwargs):
        sample_fn = self.p_sample_loop if not self.is_ddim_sampling else self.ddim_sample
        shape = (num_samples, batch_size, self.channels, self.seq_length)
        return sample_fn(shape, *args, **kwargs)