lib/net/HGFilters.py

# -*- coding: utf-8 -*-

# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
# holder of all proprietary rights on this computer program.
# You can only use this computer program if you have closed
# a license agreement with MPG or you get the right to use the computer
# program from someone who is authorized to grant you that right.
# Any use of the computer program without a valid license is prohibited and
# liable to prosecution.
#
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
# for Intelligent Systems. All rights reserved.
#
# Contact: ps-license@tuebingen.mpg.de

from lib.net.net_util import *
import torch.nn as nn
import torch.nn.functional as F


class HourGlass(nn.Module):

    def __init__(self, num_modules, depth, num_features, opt):
        super(HourGlass, self).__init__()
        self.num_modules = num_modules
        self.depth = depth
        self.features = num_features
        self.opt = opt

        self._generate_network(self.depth)

    def _generate_network(self, level):
        self.add_module('b1_' + str(level),
                        ConvBlock(self.features, self.features, self.opt))

        self.add_module('b2_' + str(level),
                        ConvBlock(self.features, self.features, self.opt))

        if level > 1:
            self._generate_network(level - 1)
        else:
            self.add_module('b2_plus_' + str(level),
                            ConvBlock(self.features, self.features, self.opt))

        self.add_module('b3_' + str(level),
                        ConvBlock(self.features, self.features, self.opt))

    def _forward(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._modules['b1_' + str(level)](up1)

        # Lower branch
        low1 = F.avg_pool2d(inp, 2, stride=2)
        low1 = self._modules['b2_' + str(level)](low1)

        if level > 1:
            low2 = self._forward(level - 1, low1)
        else:
            low2 = low1
            low2 = self._modules['b2_plus_' + str(level)](low2)

        low3 = low2
        low3 = self._modules['b3_' + str(level)](low3)

        # NOTE: for newer PyTorch (1.3~), it seems that training results are degraded due to implementation diff in F.grid_sample
        # if the pretrained model behaves weirdly, switch with the commented line.
        # NOTE: I also found that "bicubic" works better.
        up2 = F.interpolate(low3,
                            scale_factor=2,
                            mode='bicubic',
                            align_corners=True)
        # up2 = F.interpolate(low3, scale_factor=2, mode='nearest)

        return up1 + up2

    def forward(self, x):
        return self._forward(self.depth, x)


class HGFilter(nn.Module):

    def __init__(self, opt, num_modules, in_dim):
        super(HGFilter, self).__init__()
        self.num_modules = num_modules

        self.opt = opt
        [k, s, d, p] = self.opt.conv1

        # self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3)
        self.conv1 = nn.Conv2d(in_dim,
                               64,
                               kernel_size=k,
                               stride=s,
                               dilation=d,
                               padding=p)

        if self.opt.norm == 'batch':
            self.bn1 = nn.BatchNorm2d(64)
        elif self.opt.norm == 'group':
            self.bn1 = nn.GroupNorm(32, 64)

        if self.opt.hg_down == 'conv64':
            self.conv2 = ConvBlock(64, 64, self.opt)
            self.down_conv2 = nn.Conv2d(64,
                                        128,
                                        kernel_size=3,
                                        stride=2,
                                        padding=1)
        elif self.opt.hg_down == 'conv128':
            self.conv2 = ConvBlock(64, 128, self.opt)
            self.down_conv2 = nn.Conv2d(128,
                                        128,
                                        kernel_size=3,
                                        stride=2,
                                        padding=1)
        elif self.opt.hg_down == 'ave_pool':
            self.conv2 = ConvBlock(64, 128, self.opt)
        else:
            raise NameError('Unknown Fan Filter setting!')

        self.conv3 = ConvBlock(128, 128, self.opt)
        self.conv4 = ConvBlock(128, 256, self.opt)

        # Stacking part
        for hg_module in range(self.num_modules):
            self.add_module('m' + str(hg_module),
                            HourGlass(1, opt.num_hourglass, 256, self.opt))

            self.add_module('top_m_' + str(hg_module),
                            ConvBlock(256, 256, self.opt))
            self.add_module(
                'conv_last' + str(hg_module),
                nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
            if self.opt.norm == 'batch':
                self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))
            elif self.opt.norm == 'group':
                self.add_module('bn_end' + str(hg_module),
                                nn.GroupNorm(32, 256))

            self.add_module(
                'l' + str(hg_module),
                nn.Conv2d(256,
                          opt.hourglass_dim,
                          kernel_size=1,
                          stride=1,
                          padding=0))

            if hg_module < self.num_modules - 1:
                self.add_module(
                    'bl' + str(hg_module),
                    nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
                self.add_module(
                    'al' + str(hg_module),
                    nn.Conv2d(opt.hourglass_dim,
                              256,
                              kernel_size=1,
                              stride=1,
                              padding=0))

    def forward(self, x):
        x = F.relu(self.bn1(self.conv1(x)), True)
        tmpx = x
        if self.opt.hg_down == 'ave_pool':
            x = F.avg_pool2d(self.conv2(x), 2, stride=2)
        elif self.opt.hg_down in ['conv64', 'conv128']:
            x = self.conv2(x)
            x = self.down_conv2(x)
        else:
            raise NameError('Unknown Fan Filter setting!')

        x = self.conv3(x)
        x = self.conv4(x)

        previous = x

        outputs = []
        for i in range(self.num_modules):
            hg = self._modules['m' + str(i)](previous)

            ll = hg
            ll = self._modules['top_m_' + str(i)](ll)

            ll = F.relu(
                self._modules['bn_end' + str(i)](
                    self._modules['conv_last' + str(i)](ll)), True)

            # Predict heatmaps
            tmp_out = self._modules['l' + str(i)](ll)
            outputs.append(tmp_out)

            if i < self.num_modules - 1:
                ll = self._modules['bl' + str(i)](ll)
                tmp_out_ = self._modules['al' + str(i)](tmp_out)
                previous = previous + ll + tmp_out_

        return outputs
    






class FuseHGFilter(nn.Module):

    def __init__(self, opt, num_modules, in_dim):
        super(FuseHGFilter, self).__init__()
        self.num_modules = num_modules

        self.opt = opt
        [k, s, d, p] = self.opt.conv1

        # self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3)
        self.conv1 = nn.Conv2d(in_dim,
                               64,
                               kernel_size=k,
                               stride=s,
                               dilation=d,
                               padding=p)

        if self.opt.norm == 'batch':
            self.bn1 = nn.BatchNorm2d(64)
        elif self.opt.norm == 'group':
            self.bn1 = nn.GroupNorm(32, 64)

        
        self.conv2 = ConvBlock(64, 128, self.opt)
        self.down_conv2 = nn.Conv2d(128,
                                    96,
                                    kernel_size=3,
                                    stride=2,
                                    padding=1)
        # elif self.opt.hg_down == 'conv128':
        #     self.conv2 = ConvBlock(64, 128, self.opt)
        #     self.down_conv2 = nn.Conv2d(128,
        #                                 128,
        #                                 kernel_size=3,
        #                                 stride=2,
        #                                 padding=1)
        
        dim=96+32
        self.conv3 = ConvBlock(dim, dim, self.opt)
        self.conv4 = ConvBlock(dim, 256, self.opt)

        # Stacking part
        for hg_module in range(self.num_modules):
            self.add_module('m' + str(hg_module),
                            HourGlass(1, opt.num_hourglass, 256, self.opt))

            self.add_module('top_m_' + str(hg_module),
                            ConvBlock(256, 256, self.opt))
            self.add_module(
                'conv_last' + str(hg_module),
                nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
            if self.opt.norm == 'batch':
                self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))
            elif self.opt.norm == 'group':
                self.add_module('bn_end' + str(hg_module),
                                nn.GroupNorm(32, 256))

            hourglass_dim=256
            self.add_module(
                'l' + str(hg_module),
                nn.Conv2d(256,
                          hourglass_dim,
                          kernel_size=1,
                          stride=1,
                          padding=0))

            if hg_module < self.num_modules - 1:
                self.add_module(
                    'bl' + str(hg_module),
                    nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
                self.add_module(
                    'al' + str(hg_module),
                    nn.Conv2d(hourglass_dim,
                              256,
                              kernel_size=1,
                              stride=1,
                              padding=0))
                
        self.up_conv=nn.ConvTranspose2d(hourglass_dim,64,kernel_size=2,stride=2)

    def forward(self, x,plane):
        x = F.relu(self.bn1(self.conv1(x)), True)  # 64*256*256
        tmpx = x
        
        x = self.conv2(x)
        x = self.down_conv2(x)

        x=torch.cat([x,plane],1)   # 128*128*128


        x = self.conv3(x)
        x = self.conv4(x)

        previous = x

        outputs = []
        for i in range(self.num_modules):
            hg = self._modules['m' + str(i)](previous)

            ll = hg
            ll = self._modules['top_m_' + str(i)](ll)

            ll = F.relu(
                self._modules['bn_end' + str(i)](
                    self._modules['conv_last' + str(i)](ll)), True)

            # Predict heatmaps
            tmp_out = self._modules['l' + str(i)](ll)
            outputs.append(tmp_out)

            if i < self.num_modules - 1:
                ll = self._modules['bl' + str(i)](ll)
                tmp_out_ = self._modules['al' + str(i)](tmp_out)
                previous = previous + ll + tmp_out_

        out=self.up_conv(outputs[-1])

        return out