File size: 4,660 Bytes
d2c5d46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75889bd
 
 
 
d2c5d46
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# https://github.com/openai/improved-diffusion/blob/main/improved_diffusion/gaussian_diffusion.py

# https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py

# https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py


import math

import torch

import torch.nn as nn

import numpy as np

from einops import repeat, rearrange

def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3):
    if schedule == "linear":
        betas = (
                torch.linspace(linear_start ** 0.5, linear_end ** 0.5, n_timestep, dtype=torch.float64) ** 2
        )

    elif schedule == "cosine":
        timesteps = (
                torch.arange(n_timestep + 1, dtype=torch.float64) / n_timestep + cosine_s
        )
        alphas = timesteps / (1 + cosine_s) * np.pi / 2
        alphas = torch.cos(alphas).pow(2)
        alphas = alphas / alphas[0]
        betas = 1 - alphas[1:] / alphas[:-1]
        betas = np.clip(betas, a_min=0, a_max=0.999)

    elif schedule == "sqrt_linear":
        betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64)
    elif schedule == "sqrt":
        betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) ** 0.5
    else:
        raise ValueError(f"schedule '{schedule}' unknown.")
    return betas.numpy()

def extract_into_tensor(a, t, x_shape):
    b, *_ = t.shape
    out = a.gather(-1, t)
    return out.reshape(b, *((1,) * (len(x_shape) - 1)))

def checkpoint(func, inputs, params, flag):
    return func(*inputs)

def timestep_embedding(timesteps, dim, max_period=10000, repeat_only=False):
    if not repeat_only:
        half = dim // 2
        freqs = torch.exp(
            -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
        ).to(device=timesteps.device)
        args = timesteps[:, None].float() * freqs[None]
        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
        if dim % 2:
            embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
    else:
        embedding = repeat(timesteps, 'b -> b d', d=dim)
    return embedding

def zero_module(module):
    for p in module.parameters():
        p.detach().zero_()
    return module

def normalization(channels, norm_channel=32):
    return GroupNorm32(norm_channel, channels)

class GroupNorm32(nn.GroupNorm):
    def forward(self, x):
        return super().forward(x.float()).type(x.dtype)

def conv_nd(dims, *args, **kwargs):
    if dims == 1:
        return nn.Conv1d(*args, **kwargs)
    elif dims == 2:
        return nn.Conv2d(*args, **kwargs)
    elif dims == 3:
        return nn.Conv3d(*args, **kwargs)
    raise ValueError(f"unsupported dimensions: {dims}")

def linear(*args, **kwargs):
    return nn.Linear(*args, **kwargs)

def avg_pool_nd(dims, *args, **kwargs):
    if dims == 1:
        return nn.AvgPool1d(*args, **kwargs)
    elif dims == 2:
        return nn.AvgPool2d(*args, **kwargs)
    elif dims == 3:
        return nn.AvgPool3d(*args, **kwargs)
    raise ValueError(f"unsupported dimensions: {dims}")

def from_4d_to_5d(inp, b, c, t, h, w):
    out = rearrange(inp, '(b t) c h w -> b c t h w', b=b, c=c, t=t, h=h, w=w)
    return out

def from_5d_to_4d(inp, b, c, t, h, w):
    out = rearrange(inp, 'b c t h w -> (b t) c h w')
    return out

def from_4d_to_3d(inp, b, c, t, h, w):
    out = rearrange(inp, '(b t) c h w -> (b h w) t c', b=b, c=c, t=t, h=h, w=w)
    return out

def from_3d_to_4d(inp, b, c, t, h, w):
    out = rearrange(inp, '(b h w) t c -> (b t) c h w', b=b, c=c, t=t, h=h, w=w)
    return out

class SpatialTemporalConv(nn.Module):
    def __init__(self, num_feat, num_frames=1):
        super().__init__()

        self.num_frames = num_frames


        self.temporal_conv = conv_nd(3, num_feat, num_feat, (3, 1, 1), padding=(1, 0, 0))

        self.temporal_alpha_logit = nn.Parameter(torch.zeros(1))

    def forward(self, inp, t=None):
        bt, c, h, w = inp.shape
        # einops >= 0.4 raises when an explicit axis length is None (the authors'
        # einops 0.3.0 inferred it from the shape); use the effective frame count.
        t = t or self.num_frames
        b = bt // t
        ori = inp
        inp = from_4d_to_5d(inp, b, c, t, h, w)
        res = self.temporal_conv(inp)
        res = from_5d_to_4d(res, b, c, t, h, w)
        alpha = torch.sigmoid(self.temporal_alpha_logit).to(inp.dtype)
        out = alpha * res + (1-alpha) *ori


        return out