import torch import torch.nn.functional as F from torch import nn def _make_scratch(in_shape, out_shape, groups=1, expand=False): scratch = nn.Module() output_shapes = [out_shape, out_shape, out_shape, out_shape] if expand: output_shapes = [out_shape, out_shape * 2, out_shape * 4, out_shape * 8] for index, (input_channels, output_channels) in enumerate( zip(in_shape, output_shapes), start=1 ): setattr( scratch, f"layer{index}_rn", nn.Conv2d( input_channels, output_channels, kernel_size=3, stride=1, padding=1, bias=False, groups=groups, ), ) return scratch class ResidualConvUnit(nn.Module): def __init__(self, features, activation, use_bn): super().__init__() self.bn = use_bn self.groups = 1 self.conv1 = nn.Conv2d(features, features, 3, padding=1, bias=True) self.conv2 = nn.Conv2d(features, features, 3, padding=1, bias=True) if use_bn: self.bn1 = nn.BatchNorm2d(features) self.bn2 = nn.BatchNorm2d(features) self.activation = activation self.skip_add = nn.quantized.FloatFunctional() def forward(self, x): output = self.conv1(self.activation(x)) if self.bn: output = self.bn1(output) output = self.conv2(self.activation(output)) if self.bn: output = self.bn2(output) return self.skip_add.add(output, x) class FeatureFusionBlock(nn.Module): def __init__(self, features, activation, use_bn=False, size=None): super().__init__() self.deconv = False self.align_corners = True self.groups = 1 self.expand = False self.out_conv = nn.Conv2d(features, features, kernel_size=1, bias=True) self.resConfUnit1 = ResidualConvUnit(features, activation, use_bn) self.resConfUnit2 = ResidualConvUnit(features, activation, use_bn) self.skip_add = nn.quantized.FloatFunctional() self.size = size def forward(self, *inputs, size=None): output = inputs[0] if len(inputs) == 2: output = self.skip_add.add(output, self.resConfUnit1(inputs[1])) output = self.resConfUnit2(output) if size is None and self.size is None: modifier = {"scale_factor": 2} else: modifier = {"size": self.size if size is None else size} output = F.interpolate( output, **modifier, mode="bilinear", align_corners=self.align_corners ) return self.out_conv(output) def _make_fusion_block(features, use_bn, size=None): return FeatureFusionBlock(features, nn.ReLU(False), use_bn, size) class DPTHead(nn.Module): def __init__( self, in_channels, features=256, use_bn=False, out_channels=(256, 512, 1024, 1024), use_clstoken=False, ): super().__init__() self.use_clstoken = use_clstoken self.projects = nn.ModuleList( [nn.Conv2d(in_channels, channel, kernel_size=1) for channel in out_channels] ) self.resize_layers = nn.ModuleList( [ nn.ConvTranspose2d(out_channels[0], out_channels[0], 4, stride=4), nn.ConvTranspose2d(out_channels[1], out_channels[1], 2, stride=2), nn.Identity(), nn.Conv2d(out_channels[3], out_channels[3], 3, stride=2, padding=1), ] ) if use_clstoken: self.readout_projects = nn.ModuleList( [ nn.Sequential(nn.Linear(2 * in_channels, in_channels), nn.GELU()) for _ in self.projects ] ) self.scratch = _make_scratch(out_channels, features) self.scratch.stem_transpose = None self.scratch.refinenet1 = _make_fusion_block(features, use_bn) self.scratch.refinenet2 = _make_fusion_block(features, use_bn) self.scratch.refinenet3 = _make_fusion_block(features, use_bn) self.scratch.refinenet4 = _make_fusion_block(features, use_bn) self.scratch.output_conv1 = nn.Conv2d( features, features // 2, kernel_size=3, padding=1 ) self.scratch.output_conv2 = nn.Sequential( nn.Conv2d(features // 2, 32, kernel_size=3, padding=1), nn.ReLU(True), nn.Conv2d(32, 1, kernel_size=1), ) def forward(self, out_features, patch_h, patch_w): outputs = [] for index, feature in enumerate(out_features): if self.use_clstoken: feature, class_token = feature readout = class_token.unsqueeze(1).expand_as(feature) feature = self.readout_projects[index]( torch.cat((feature, readout), dim=-1) ) else: feature = feature[0] outputs.append(self.resize_layers[index](self.projects[index](feature))) layer_1, layer_2, layer_3, layer_4 = outputs layer_1 = self.scratch.layer1_rn(layer_1) layer_2 = self.scratch.layer2_rn(layer_2) layer_3 = self.scratch.layer3_rn(layer_3) layer_4 = self.scratch.layer4_rn(layer_4) path_4 = self.scratch.refinenet4(layer_4, size=layer_3.shape[2:]) path_3 = self.scratch.refinenet3(path_4, layer_3, size=layer_2.shape[2:]) path_2 = self.scratch.refinenet2(path_3, layer_2, size=layer_1.shape[2:]) path_1 = self.scratch.refinenet1(path_2, layer_1) output = self.scratch.output_conv1(path_1) output = F.interpolate( output, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True, ) return self.scratch.output_conv2(output)