model.py

import torch
import torch.nn as nn
from resnet import resnet50
import numpy as np
import cv2


def save_feats_mean(x, size=(256, 256)):
    b, c, h, w = x.shape
    with torch.no_grad():
        x = x.detach().cpu().numpy()
        x = np.transpose(x[0], (1, 2, 0))
        x = np.mean(x, axis=-1)
        x = x/np.max(x)
        x = x * 255.0
        x = x.astype(np.uint8)
        if h != size[1]:
            x = cv2.resize(x, size)
        x = cv2.applyColorMap(x, cv2.COLORMAP_JET)
        x = np.array(x, dtype=np.uint8)
        return x


def get_mean_attention_map(x):
    x = torch.mean(x, axis=1)
    x = torch.unsqueeze(x, 1)
    x = x / torch.max(x)
    return x


class ResidualBlock(nn.Module):
    def __init__(self, in_c, out_c):
        super().__init__()
        self.relu = nn.ReLU()
        self.conv = nn.Sequential(
            nn.Conv2d(in_c, out_c, kernel_size=3, padding=1),
            nn.BatchNorm2d(out_c),
            nn.ReLU(),
            nn.Conv2d(out_c, out_c, kernel_size=3, padding=1),
            nn.BatchNorm2d(out_c)
        )
        self.shortcut = nn.Sequential(
            nn.Conv2d(in_c, out_c, kernel_size=1, padding=0),
            nn.BatchNorm2d(out_c)
        )

    def forward(self, inputs):
        x1 = self.conv(inputs)
        x2 = self.shortcut(inputs)
        x = self.relu(x1 + x2)
        return x


class DilatedConv(nn.Module):
    def __init__(self, in_c, out_c):
        super().__init__()
        self.c1 = nn.Sequential(
            nn.Conv2d(in_c, out_c, kernel_size=3, padding=1, dilation=1),
            nn.BatchNorm2d(out_c),
            nn.ReLU()
        )
        self.c2 = nn.Sequential(
            nn.Conv2d(in_c, out_c, kernel_size=3, padding=3, dilation=3),
            nn.BatchNorm2d(out_c),
            nn.ReLU()
        )
        self.c3 = nn.Sequential(
            nn.Conv2d(in_c, out_c, kernel_size=3, padding=6, dilation=6),
            nn.BatchNorm2d(out_c),
            nn.ReLU()
        )
        self.c4 = nn.Sequential(
            nn.Conv2d(in_c, out_c, kernel_size=3, padding=9, dilation=9),
            nn.BatchNorm2d(out_c),
            nn.ReLU()
        )
        self.c5 = nn.Sequential(
            nn.Conv2d(out_c*4, out_c, kernel_size=1, padding=0),
            nn.BatchNorm2d(out_c),
            nn.ReLU()
        )

    def forward(self, inputs):
        x1 = self.c1(inputs)
        x2 = self.c2(inputs)
        x3 = self.c3(inputs)
        x4 = self.c4(inputs)
        x = torch.cat([x1, x2, x3, x4], axis=1)
        x = self.c5(x)
        return x


class ChannelAttention(nn.Module):
    def __init__(self, in_planes, ratio=16):
        super(ChannelAttention, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.max_pool = nn.AdaptiveMaxPool2d(1)
        self.fc1   = nn.Conv2d(in_planes, in_planes // 16, 1, bias=False)
        self.relu1 = nn.ReLU()
        self.fc2   = nn.Conv2d(in_planes // 16, in_planes, 1, bias=False)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x0 = x
        avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x))))
        max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x))))
        out = avg_out + max_out
        return x0 * self.sigmoid(out)


class SpatialAttention(nn.Module):
    def __init__(self, kernel_size=7):
        super(SpatialAttention, self).__init__()
        assert kernel_size in (3, 7), 'kernel size must be 3 or 7'
        padding = 3 if kernel_size == 7 else 1
        self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x0 = x
        avg_out = torch.mean(x, dim=1, keepdim=True)
        max_out, _ = torch.max(x, dim=1, keepdim=True)
        x = torch.cat([avg_out, max_out], dim=1)
        x = self.conv1(x)
        return x0 * self.sigmoid(x)


class DecoderBlock(nn.Module):
    def __init__(self, in_c, out_c):
        super().__init__()
        self.up = nn.Upsample(scale_factor=2, mode="bilinear", align_corners=True)
        self.r1 = ResidualBlock(in_c[0]+in_c[1], out_c)
        self.r2 = ResidualBlock(out_c, out_c)
        self.ca = ChannelAttention(out_c)
        self.sa = SpatialAttention()

    def forward(self, x, s):
        x = self.up(x)
        x = torch.cat([x, s], axis=1)
        x = self.r1(x)
        x = self.r2(x)
        x = self.ca(x)
        x = self.sa(x)
        return x


class RUPNet(nn.Module):
    def __init__(self):
        super().__init__()
        backbone = resnet50(pretrained=False)
        self.layer0 = nn.Sequential(backbone.conv1, backbone.bn1, backbone.relu)
        self.layer1 = nn.Sequential(backbone.maxpool, backbone.layer1)
        self.layer2 = backbone.layer2
        self.layer3 = backbone.layer3
        self.r1 = nn.Sequential(DilatedConv(64, 64), nn.MaxPool2d((8, 8)))
        self.r2 = nn.Sequential(DilatedConv(256, 64), nn.MaxPool2d((4, 4)))
        self.r3 = nn.Sequential(DilatedConv(512, 64), nn.MaxPool2d((2, 2)))
        self.r4 = DilatedConv(1024, 64)
        self.d1 = DecoderBlock([256, 512], 256)
        self.d2 = DecoderBlock([256, 256], 128)
        self.d3 = DecoderBlock([128, 64], 64)
        self.d4 = DecoderBlock([64, 3], 32)
        self.y = nn.Conv2d(32, 1, kernel_size=1, padding=0)

    def forward(self, x, heatmap=None):
        s0 = x
        s1 = self.layer0(s0)
        s2 = self.layer1(s1)
        s3 = self.layer2(s2)
        s4 = self.layer3(s3)
        r1 = self.r1(s1)
        r2 = self.r2(s2)
        r3 = self.r3(s3)
        r4 = self.r4(s4)
        rx = torch.cat([r1, r2, r3, r4], axis=1)
        d1 = self.d1(rx, s3)
        d2 = self.d2(d1, s2)
        d3 = self.d3(d2, s1)
        d4 = self.d4(d3, s0)
        y = self.y(d4)
        if heatmap is not None:
            hmap = save_feats_mean(d4)
            return hmap, y
        else:
            return y