model/networks/zencoder.py

import torch
import torch.nn as nn
import torch.nn.functional as F
from .base_network import BaseNetwork
import random
from .blocks import Conv2dBlock

def inst_id_to_label_id(inst_id: int) -> int:
    return int(inst_id) % 120

class DeeperZencoder(BaseNetwork):
    def __init__(self,cfg, input_nc = 3, output_nc = 512, ngf=32, n_downsampling=2, norm_layer=nn.InstanceNorm2d):
        super().__init__()
        self.cfg = cfg
        self.n_downsampling = 6
        self.non_spade_norm_layer = norm_layer
        self.output_nc = cfg["style_length"]
        input_nc = input_nc +1 #RGB + Valid

        self.gamma = nn.Linear(1, 1)
        self.beta = nn.Linear(1, 1)
        ### downsample
        input_nc = cfg['input_nc']
        ngf = cfg['ngf']
        output_nc = cfg['style_length']
        lab_nc = cfg['lab_dim'] + 1
        g_norm = cfg['G_norm_type']
        self.enc1 = Conv2dBlock(4, 16, kernel_size=3, stride=1, padding=1, norm=g_norm, activation='lrelu') # 16, 256, 256
        self.enc2 = Conv2dBlock(16, 32, kernel_size=3, stride=2, padding=1, norm=g_norm, activation='lrelu') # 32, 128, 128
        self.enc3 = Conv2dBlock(32, 64, kernel_size=3, stride=2, padding=1, norm=g_norm, activation='lrelu') # 64, 64, 64
        self.enc4 = Conv2dBlock(64, 128, kernel_size=3, stride=2, padding=1, norm=g_norm, activation='lrelu') # 128, 32, 32
        self.enc5 = Conv2dBlock(128, 256, kernel_size=3, stride=2, padding=1, norm=g_norm, activation='lrelu') # 256, 16, 16
        self.enc6 = Conv2dBlock(256, 512, kernel_size=3, stride=1, padding=1, dilation=1, norm=g_norm, activation='lrelu') # 512, 16, 16

        # Decoder layers
        self.dec5 = Conv2dBlock(512+128, 512, kernel_size=3, stride=1, padding=1, norm=g_norm,
                                   activation='lrelu')
        self.dec4 = Conv2dBlock(512+64, 256, kernel_size=3, stride=1, padding=1, norm=g_norm,
                                   activation='lrelu')
        self.dec3 = Conv2dBlock(256+32, 256, kernel_size=3, stride=1, padding=1, norm=g_norm,
                                   activation='lrelu')
        self.dec2 = Conv2dBlock(256+16, 256, kernel_size=3, stride=1, padding=1, norm=g_norm,
                                   activation='lrelu')
        self.dec1 = Conv2dBlock(256, output_nc, kernel_size=3, stride=1, padding=1, norm='none', activation='tanh')

    def forward(self, input, segmap, valids, instance_map=None, cached_codes=None):
        masked_input = input*valids
        masked_input = torch.cat((masked_input, valids), 1)
        codes = self._forward_layers(masked_input)

        original_input = torch.cat((input, torch.ones(valids.shape, device=valids.device, dtype=valids.dtype)), 1)
        unmasked_codes = self._forward_layers(original_input)

        if instance_map is None:
            styles_map = segmap
        else:
            styles_map = instance_map
        styles_map = F.interpolate(styles_map, size=codes.size()[2:], mode='nearest')
        valids = F.interpolate(valids, size=codes.size()[2:], mode='nearest')
        # instance-wise average pooling
        style_codes = codes.clone()
        for b in range(input.size()[0]):
            inst_list = torch.unique(styles_map[b]).to(torch.long)
            for i in inst_list:
                indices = (styles_map[b:b+1] == int(i)).nonzero() # n x 4 
                valid_ins = valids[indices[:,0] + b, :, indices[:,2], indices[:,3]]
                if valid_ins.sum() > 0:
                    output_ins = codes[indices[:,0] + b, :, indices[:,2], indices[:,3]]
                    mean_feat = output_ins.mean(dim=0).expand_as(output_ins)
                    valid_ratio = valid_ins.mean().unsqueeze(0)
                    mean_feat = mean_feat * torch.sigmoid(self.gamma(valid_ratio)) + torch.sigmoid(self.beta(valid_ratio))
                elif self.cfg["is_train"]:
                    unmasked_output_ins = unmasked_codes[indices[:,0] + b, :, indices[:,2], indices[:,3]]
                    mean_feat = unmasked_output_ins.mean(dim=0).expand_as(unmasked_output_ins)
                    valid_ratio = torch.ones((1,))
                    mean_feat = mean_feat * torch.sigmoid(self.gamma(valid_ratio)) + torch.sigmoid(self.beta(valid_ratio))
                else:
                    code_list = cached_codes[inst_id_to_label_id(i)]
                    random_code = random.choice(code_list)
                    mean_feat = random_code.expand_as(codes[indices[:,0] + b, :, indices[:,2], indices[:,3]])
                    valid_ratio = torch.ones((1,))
                    mean_feat = mean_feat * torch.sigmoid(self.gamma(valid_ratio)) + torch.sigmoid(self.beta(valid_ratio))
                style_codes[indices[:,0] + b, :, indices[:,2], indices[:,3]] = mean_feat
        return style_codes

    def _forward_layers(self, input):
        # Encoder
        e1 = self.enc1(input)
        e2 = self.enc2(e1)
        e3 = self.enc3(e2)
        e4 = self.enc4(e3)
        e5 = self.enc5(e4)
        x = self.enc6(e5)
        
        x = F.interpolate(x, scale_factor=2, mode='bilinear')
        x = torch.cat((x, e4), dim=1)
        x = self.dec5(x)
        
        x = F.interpolate(x, scale_factor=2, mode='bilinear')
        x = torch.cat((x, e3), dim=1)
        x = self.dec4(x)
        
        x = F.interpolate(x, scale_factor=2, mode='bilinear')
        x = torch.cat((x, e2), dim=1)
        x = self.dec3(x)
        
        x = F.interpolate(x, scale_factor=2, mode='bilinear')
        x = torch.cat((x, e1), dim=1)
        x = self.dec2(x)
        
        x = self.dec1(x)

        return x