diffcve-2607 / diffcve /modules /vae_blocks.py
multimodalart's picture
multimodalart HF Staff
Upload diffcve/modules/vae_blocks.py with huggingface_hub
b1625c2 verified
Raw History Blame Contribute Delete
20.6 kB
import torch
import torch.nn as nn
import numpy as np
from diffcve.modules.attention import LinearAttention
from diffcve.modules.diffusion.utils import SpatialTemporalConv
from diffcve.modules.initialization import make_layer
from diffcve.modules.residual import ResidualDenseBlock
try:
import xformers
import xformers.ops
XFORMERS_AVAILABLE = True
except:
XFORMERS_AVAILABLE = False
def nonlinearity(x):
return x*torch.sigmoid(x)
def Normalize(in_channels, num_groups=32):
return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
class Upsample(nn.Module):
def __init__(self, in_channels, with_conv):
super().__init__()
self.with_conv = with_conv
if self.with_conv:
self.conv = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=3,
stride=1,
padding=1)
def forward(self, x):
x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
if self.with_conv:
x = self.conv(x)
return x
class Downsample(nn.Module):
def __init__(self, in_channels, with_conv):
super().__init__()
self.with_conv = with_conv
if self.with_conv:
self.conv = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=3,
stride=2,
padding=0)
def forward(self, x):
if self.with_conv:
pad = (0,1,0,1)
x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
x = self.conv(x)
else:
x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
return x
class ResnetBlock(nn.Module):
def __init__(self, *, in_channels, out_channels=None, conv_shortcut=False,
dropout, temb_channels=512):
super().__init__()
self.in_channels = in_channels
out_channels = in_channels if out_channels is None else out_channels
self.out_channels = out_channels
self.use_conv_shortcut = conv_shortcut
self.norm1 = Normalize(in_channels)
self.conv1 = torch.nn.Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=1,
padding=1)
if temb_channels > 0:
self.temb_proj = torch.nn.Linear(temb_channels,
out_channels)
self.norm2 = Normalize(out_channels)
self.dropout = torch.nn.Dropout(dropout)
self.conv2 = torch.nn.Conv2d(out_channels,
out_channels,
kernel_size=3,
stride=1,
padding=1)
if self.in_channels != self.out_channels:
if self.use_conv_shortcut:
self.conv_shortcut = torch.nn.Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=1,
padding=1)
else:
self.nin_shortcut = torch.nn.Conv2d(in_channels,
out_channels,
kernel_size=1,
stride=1,
padding=0)
def forward(self, x, temb):
h = x
h = self.norm1(h)
h = nonlinearity(h)
h = self.conv1(h)
if temb is not None:
h = h + self.temb_proj(nonlinearity(temb))[:,:,None,None]
h = self.norm2(h)
h = nonlinearity(h)
h = self.dropout(h)
h = self.conv2(h)
if self.in_channels != self.out_channels:
if self.use_conv_shortcut:
x = self.conv_shortcut(x)
else:
x = self.nin_shortcut(x)
return x+h
class LinAttnBlock(LinearAttention):
def __init__(self, in_channels):
super().__init__(dim=in_channels, heads=1, dim_head=in_channels)
class AttnBlock(nn.Module):
def __init__(self, in_channels):
super().__init__()
self.in_channels = in_channels
self.norm = Normalize(in_channels)
self.q = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.k = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.v = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.proj_out = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
def forward(self, x):
h_ = x
h_ = self.norm(h_)
q = self.q(h_)
k = self.k(h_)
v = self.v(h_)
b,c,h,w = q.shape
q = q.reshape(b,c,h*w)
q = q.permute(0,2,1)
k = k.reshape(b,c,h*w)
w_ = torch.bmm(q,k)
w_ = w_ * (int(c)**(-0.5))
w_ = torch.nn.functional.softmax(w_, dim=2)
v = v.reshape(b,c,h*w)
w_ = w_.permute(0,2,1)
h_ = torch.bmm(v,w_)
h_ = h_.reshape(b,c,h,w)
h_ = self.proj_out(h_)
return x+h_
class MemoryEfficientAttnBlock(nn.Module):
def __init__(self, in_channels):
super().__init__()
self.in_channels = in_channels
self.norm = Normalize(in_channels)
self.q = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.k = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.v = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.proj_out = torch.nn.Conv2d(in_channels,
in_channels,
kernel_size=1,
stride=1,
padding=0)
self.attention_op: Optional[Any] = None
def forward(self, x):
h_ = x
h_ = self.norm(h_)
q = self.q(h_)
k = self.k(h_)
v = self.v(h_)
b,c,h,w = q.shape
q = q.reshape(b, c, h * w).permute(0, 2, 1).contiguous()
k = k.reshape(b, c, h * w).permute(0, 2, 1).contiguous()
v = v.reshape(b, c, h * w).permute(0, 2, 1).contiguous()
if torch.onnx.is_in_onnx_export():
weights = torch.bmm(q, k.transpose(1, 2)) * (int(c) ** -0.5)
weights = torch.softmax(weights, dim=2)
out = torch.bmm(weights, v)
else:
try:
out = xformers.ops.memory_efficient_attention(
q, k, v, attn_bias=None, scale=(int(c) ** (-0.5)), op=self.attention_op
)
except NotImplementedError:
# No xformers kernel for this shape on this GPU (e.g. the VAE
# mid-block's single 512-dim head on Blackwell, where the
# torch-backed FA2 path caps head_dim at 256). Fall back to the
# mathematically-identical dense attention.
weights = torch.bmm(q, k.transpose(1, 2)) * (int(c) ** -0.5)
weights = torch.softmax(weights, dim=2)
out = torch.bmm(weights, v)
h_ = (
out.permute(0,2,1)
.unsqueeze(3)
.reshape(b, c, h, w)
)
h_ = self.proj_out(h_)
return x+h_
def make_attn(in_channels, attn_type="vanilla"):
assert attn_type in ["vanilla", "linear", "none"], f'attn_type {attn_type} unknown'
print(f"making attention of type '{attn_type}' with {in_channels} in_channels")
if attn_type == "vanilla":
if XFORMERS_AVAILABLE:
return MemoryEfficientAttnBlock(in_channels)
else:
return AttnBlock(in_channels)
elif attn_type == "none":
return nn.Identity(in_channels)
else:
return LinAttnBlock(in_channels)
class Encoder(nn.Module):
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
resolution, z_channels, double_z=True, use_linear_attn=False, attn_type="vanilla",
**ignore_kwargs):
super().__init__()
if use_linear_attn: attn_type = "linear"
self.ch = ch
self.temb_ch = 0
self.num_resolutions = len(ch_mult)
self.num_res_blocks = num_res_blocks
self.resolution = resolution
self.in_channels = in_channels
self.conv_in = torch.nn.Conv2d(in_channels,
self.ch,
kernel_size=3,
stride=1,
padding=1)
curr_res = resolution
in_ch_mult = (1,)+tuple(ch_mult)
self.in_ch_mult = in_ch_mult
self.down = nn.ModuleList()
for i_level in range(self.num_resolutions):
block = nn.ModuleList()
attn = nn.ModuleList()
block_in = ch*in_ch_mult[i_level]
block_out = ch*ch_mult[i_level]
for i_block in range(self.num_res_blocks):
block.append(ResnetBlock(in_channels=block_in,
out_channels=block_out,
temb_channels=self.temb_ch,
dropout=dropout))
block_in = block_out
if curr_res in attn_resolutions:
attn.append(make_attn(block_in, attn_type=attn_type))
down = nn.Module()
down.block = block
down.attn = attn
if i_level != self.num_resolutions-1:
down.downsample = Downsample(block_in, resamp_with_conv)
curr_res = curr_res // 2
self.down.append(down)
self.mid = nn.Module()
self.mid.block_1 = ResnetBlock(in_channels=block_in,
out_channels=block_in,
temb_channels=self.temb_ch,
dropout=dropout)
self.mid.attn_1 = make_attn(block_in, attn_type=attn_type)
self.mid.block_2 = ResnetBlock(in_channels=block_in,
out_channels=block_in,
temb_channels=self.temb_ch,
dropout=dropout)
self.norm_out = Normalize(block_in)
self.conv_out = torch.nn.Conv2d(block_in,
2*z_channels if double_z else z_channels,
kernel_size=3,
stride=1,
padding=1)
def forward(self, x, return_features=False):
temb = None
hs = [self.conv_in(x)]
fea_list = []
for i_level in range(self.num_resolutions):
for i_block in range(self.num_res_blocks):
h = self.down[i_level].block[i_block](hs[-1], temb)
if len(self.down[i_level].attn) > 0:
h = self.down[i_level].attn[i_block](h)
hs.append(h)
if return_features:
if i_level==1 or i_level==2:
fea_list.append(h)
if i_level != self.num_resolutions-1:
hs.append(self.down[i_level].downsample(hs[-1]))
h = hs[-1]
h = self.mid.block_1(h, temb)
h = self.mid.attn_1(h)
h = self.mid.block_2(h, temb)
h = self.norm_out(h)
h = nonlinearity(h)
h = self.conv_out(h)
if return_features:
return h, fea_list
return h
class PriorConditionedVideoDecoder(nn.Module):
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
attn_resolutions, num_frames=1, dropout=0.0, resamp_with_conv=True, in_channels,
resolution, z_channels, give_pre_end=False, tanh_out=False, use_linear_attn=False,
attn_type="vanilla", num_fuse_block=2, fusion_w=1.0, prior_ch_list = [64, 128], **ignorekwargs):
super().__init__()
if use_linear_attn: attn_type = "linear"
self.ch = ch
self.temb_ch = 0
self.num_resolutions = len(ch_mult)
self.num_res_blocks = num_res_blocks
self.resolution = resolution
self.in_channels = in_channels
self.num_frames = num_frames
self.give_pre_end = give_pre_end
self.tanh_out = tanh_out
in_ch_mult = (1,)+tuple(ch_mult)
block_in = ch*ch_mult[self.num_resolutions-1]
curr_res = resolution // 2**(self.num_resolutions-1)
self.z_shape = (1,z_channels,curr_res,curr_res)
print("Working with z of shape {} = {} dimensions.".format(
self.z_shape, np.prod(self.z_shape)))
self.conv_in = torch.nn.Conv2d(z_channels,
block_in,
kernel_size=3,
stride=1,
padding=1)
self.mid = nn.Module()
self.mid.block_1 = ResnetBlock(in_channels=block_in,
out_channels=block_in,
temb_channels=self.temb_ch,
dropout=dropout)
self.mid.attn_1 = make_attn(block_in, attn_type=attn_type)
self.mid.block_2 = ResnetBlock(in_channels=block_in,
out_channels=block_in,
temb_channels=self.temb_ch,
dropout=dropout)
self.up = nn.ModuleList()
for i_level in reversed(range(self.num_resolutions)):
block = nn.ModuleList()
attn = nn.ModuleList()
temporal_mixing = nn.ModuleList()
block_out = ch*ch_mult[i_level]
if i_level != self.num_resolutions-1:
if i_level != 0:
prior_ch = prior_ch_list[i_level - 1]
fuse_layer = PriorFeatureFusion(in_ch=block_out, prior_ch=prior_ch, out_ch=block_out, num_block=num_fuse_block)
setattr(self, 'fusion_layer_{}'.format(i_level), fuse_layer)
for i_block in range(self.num_res_blocks+1):
block.append(ResnetBlock(in_channels=block_in,
out_channels=block_out,
temb_channels=self.temb_ch,
dropout=dropout))
block_in = block_out
if i_level in [0, self.num_resolutions-1]:
if i_block == self.num_res_blocks // 2:
temporal_mixing.append(SpatialTemporalConv(num_feat=block_in, num_frames=num_frames))
else:
temporal_mixing.append(nn.Identity())
else:
temporal_mixing.append(SpatialTemporalConv(num_feat=block_in, num_frames=num_frames))
if curr_res in attn_resolutions:
attn.append(make_attn(block_in, attn_type=attn_type))
up = nn.Module()
up.block = block
up.temporal_mixing = temporal_mixing
up.attn = attn
if i_level != 0:
up.upsample = Upsample(block_in, resamp_with_conv)
curr_res = curr_res * 2
self.up.insert(0, up)
self.norm_out = Normalize(block_in)
self.conv_out = torch.nn.Conv2d(block_in,
out_ch,
kernel_size=3,
stride=1,
padding=1)
def forward(self, z, encoder_features, prior_features, qp):
self.last_z_shape = z.shape
temb = None
h = self.conv_in(z)
h = self.mid.block_1(h, temb)
h = self.mid.attn_1(h)
h = self.mid.block_2(h, temb)
for i_level in reversed(range(self.num_resolutions)):
for i_block in range(self.num_res_blocks+1):
h = self.up[i_level].block[i_block](h, temb)
h = self.up[i_level].temporal_mixing[i_block](h)
if len(self.up[i_level].attn) > 0:
h = self.up[i_level].attn[i_block](h)
if i_level != self.num_resolutions-1 and i_level != 0:
cur_fuse_layer = getattr(self, 'fusion_layer_{}'.format(i_level))
h = cur_fuse_layer(encoder_features[i_level-1], h, prior_features[i_level-1], qp)
if i_level != 0:
h = self.up[i_level].upsample(h)
if self.give_pre_end:
return h
h = self.norm_out(h)
h = nonlinearity(h)
h = self.conv_out(h)
if self.tanh_out:
h = torch.tanh(h)
return h
class ResBlock(nn.Module):
def __init__(self, in_channels, out_channels=None):
super(ResBlock, self).__init__()
self.in_channels = in_channels
self.out_channels = in_channels if out_channels is None else out_channels
self.norm1 = Normalize(in_channels)
self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
self.norm2 = Normalize(out_channels)
self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
if self.in_channels != self.out_channels:
self.conv_out = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
def forward(self, x_in):
x = x_in
x = self.norm1(x)
x = nonlinearity(x)
x = self.conv1(x)
x = self.norm2(x)
x = nonlinearity(x)
x = self.conv2(x)
if self.in_channels != self.out_channels:
x_in = self.conv_out(x_in)
return x + x_in
class PriorFeatureFusion(nn.Module):
def __init__(self, in_ch, prior_ch, out_ch, num_block=1, num_grow_ch=32):
super().__init__()
self.encode_enc_1 = ResBlock(2*in_ch+prior_ch, in_ch)
self.encode_enc_2 = make_layer(ResidualDenseBlock, num_block, num_feat=in_ch, num_grow_ch=num_grow_ch)
self.encode_enc_3 = ResBlock(in_ch, out_ch)
self.qp_mlp = nn.Sequential(
nn.Linear(1, 32),
nn.SiLU(),
nn.Linear(32, 1),
nn.Sigmoid()
)
def forward(self, encoder_features, decoder_features, prior_features, qp):
encoder_features = self.encode_enc_1(torch.cat([encoder_features, decoder_features, prior_features], dim=1))
encoder_features = self.encode_enc_2(encoder_features)
encoder_features = self.encode_enc_3(encoder_features)
qp = qp.view(qp.shape[0], 1).to(dtype=encoder_features.dtype)
w = self.qp_mlp(qp)
w = w.view(-1, 1, 1, 1)
residual = w * encoder_features
out = decoder_features + residual
return out