File size: 2,523 Bytes
d2c5d46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import torch

import torch.nn as nn

class DownBlock(nn.Module):
    def __init__(self, in_ch, out_ch):
        super().__init__()
        self.conv = nn.Conv2d(
            in_ch, out_ch,
            kernel_size=3,
            stride=2,
            padding=1
        )

    def forward(self, x):
        return self.conv(x)

class ResidualConvBlock(nn.Module):
    def __init__(self, ch, use_residual=True):
        super().__init__()
        self.use_residual = use_residual

        self.conv1 = nn.Conv2d(ch, ch, 3, padding=1)
        self.norm1 = nn.GroupNorm(max(1, ch//8), ch)
        self.act = nn.SiLU()

    def forward(self, x):
        identity = x

        out = self.norm1(x)
        out = self.act(out)
        out = self.conv1(out)

        if self.use_residual:
            out = out + identity

        return out

class ResidualMotionEncoder(nn.Module):
    def __init__(
        self,
        res_in_channels=1,
        mv_in_channels=2,
        res_initial_ch=12,
        mv_initial_ch=20,
        output_ch=256,
        use_residual=True,
    ):
        super().__init__()
        self.use_residual = use_residual

        self.res_conv = nn.Conv2d(res_in_channels, res_initial_ch, 3, 1, 1)
        self.mv_conv = nn.Conv2d(mv_in_channels, mv_initial_ch, 3, 1, 1)

        self.act = nn.SiLU()

        merged_ch = res_initial_ch + mv_initial_ch

        self.merge_conv = nn.Conv2d(merged_ch, merged_ch, 3, 1, 1)
        self.merge_norm = nn.GroupNorm(max(1, merged_ch//8), merged_ch)

        self.down1 = DownBlock(merged_ch, merged_ch * 2)
        self.conv1 = ResidualConvBlock(merged_ch * 2, use_residual)

        self.down2 = DownBlock(merged_ch * 2, merged_ch * 4)
        self.conv2 = ResidualConvBlock(merged_ch * 4, use_residual)

        self.down3 = DownBlock(merged_ch * 4, merged_ch * 8)
        self.conv3 = ResidualConvBlock(output_ch, use_residual)


    def forward(self, res, mv, return_features=False):
        h_res = self.res_conv(res)
        h_mv = self.mv_conv(mv)

        h = torch.cat([h_res, h_mv], dim=1)
        h = self.merge_norm(h)
        h = self.act(h)
        h = self.merge_conv(h)

        fea_list = []

        h = self.down1(h)
        h = self.conv1(h)
        if return_features:
            fea_list.append(h)

        h = self.down2(h)
        h = self.conv2(h)
        if return_features:
            fea_list.append(h)

        h = self.down3(h)
        h = self.conv3(h)

        if return_features:
            return h, fea_list

        return h