diffcve-2607 / diffcve /models /prior_encoder.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
d2c5d46 verified
Raw History Blame Contribute Delete
2.52 kB
import torch
import torch.nn as nn
class DownBlock(nn.Module):
def __init__(self, in_ch, out_ch):
super().__init__()
self.conv = nn.Conv2d(
in_ch, out_ch,
kernel_size=3,
stride=2,
padding=1
)
def forward(self, x):
return self.conv(x)
class ResidualConvBlock(nn.Module):
def __init__(self, ch, use_residual=True):
super().__init__()
self.use_residual = use_residual
self.conv1 = nn.Conv2d(ch, ch, 3, padding=1)
self.norm1 = nn.GroupNorm(max(1, ch//8), ch)
self.act = nn.SiLU()
def forward(self, x):
identity = x
out = self.norm1(x)
out = self.act(out)
out = self.conv1(out)
if self.use_residual:
out = out + identity
return out
class ResidualMotionEncoder(nn.Module):
def __init__(
self,
res_in_channels=1,
mv_in_channels=2,
res_initial_ch=12,
mv_initial_ch=20,
output_ch=256,
use_residual=True,
):
super().__init__()
self.use_residual = use_residual
self.res_conv = nn.Conv2d(res_in_channels, res_initial_ch, 3, 1, 1)
self.mv_conv = nn.Conv2d(mv_in_channels, mv_initial_ch, 3, 1, 1)
self.act = nn.SiLU()
merged_ch = res_initial_ch + mv_initial_ch
self.merge_conv = nn.Conv2d(merged_ch, merged_ch, 3, 1, 1)
self.merge_norm = nn.GroupNorm(max(1, merged_ch//8), merged_ch)
self.down1 = DownBlock(merged_ch, merged_ch * 2)
self.conv1 = ResidualConvBlock(merged_ch * 2, use_residual)
self.down2 = DownBlock(merged_ch * 2, merged_ch * 4)
self.conv2 = ResidualConvBlock(merged_ch * 4, use_residual)
self.down3 = DownBlock(merged_ch * 4, merged_ch * 8)
self.conv3 = ResidualConvBlock(output_ch, use_residual)
def forward(self, res, mv, return_features=False):
h_res = self.res_conv(res)
h_mv = self.mv_conv(mv)
h = torch.cat([h_res, h_mv], dim=1)
h = self.merge_norm(h)
h = self.act(h)
h = self.merge_conv(h)
fea_list = []
h = self.down1(h)
h = self.conv1(h)
if return_features:
fea_list.append(h)
h = self.down2(h)
h = self.conv2(h)
if return_features:
fea_list.append(h)
h = self.down3(h)
h = self.conv3(h)
if return_features:
return h, fea_list
return h