utils/layers.py

import torch
import torch.nn as nn

from functools import partial
from timm.models.vision_transformer import Block

# 3D positional encoding, from https://github.com/bmild/nerf.
class Embedder:
    def __init__(self, **kwargs):
        self.kwargs = kwargs
        self.create_embedding_fn()

    def create_embedding_fn(self):
        embed_fns = []
        d = self.kwargs['input_dims']
        out_dim = 0
        if self.kwargs['include_input']:
            embed_fns.append(lambda x: x)
            out_dim += d

        max_freq = self.kwargs['max_freq_log2']
        N_freqs = self.kwargs['num_freqs']

        if self.kwargs['log_sampling']:
            freq_bands = 2. ** torch.linspace(0., max_freq, N_freqs)
        else:
            freq_bands = torch.linspace(2.**0., 2.**max_freq, N_freqs)

        for freq in freq_bands:
            for p_fn in self.kwargs['periodic_fns']:
                embed_fns.append(lambda x, p_fn=p_fn,
                                 freq=freq: p_fn(x * freq))
                out_dim += d

        self.embed_fns = embed_fns
        self.out_dim = out_dim

    def embed(self, inputs):
        return torch.cat([fn(inputs) for fn in self.embed_fns], -1)

def get_embedder(posenc_res, input_dims=3):
    embed_kwargs = {
        'include_input': True,
        'input_dims': input_dims,
        'max_freq_log2': posenc_res-1,
        'num_freqs': posenc_res,
        'log_sampling': True,
        'periodic_fns': [torch.sin, torch.cos],
    }

    embedder_obj = Embedder(**embed_kwargs)
    def embed(x, eo=embedder_obj): return eo.embed(x)
    return embed, embedder_obj.out_dim

class LayerScale(nn.Module):
    def __init__(self, dim, init_values=1e-5, inplace=False):
        super().__init__()
        self.inplace = inplace
        self.gamma = nn.Parameter(init_values * torch.ones(dim))

    def forward(self, x):
        return x.mul_(self.gamma) if self.inplace else x * self.gamma

class Bottleneck_Linear(nn.Module):
    def __init__(self, n_channels):
        super().__init__()
        self.linear1 = nn.Linear(n_channels, n_channels)
        self.norm = nn.LayerNorm(n_channels)
        self.linear2 = nn.Linear(n_channels, n_channels)
        self.gelu = nn.GELU()
        
    def forward(self, x):
        x = x + self.linear2(self.gelu(self.linear1(self.norm(x))))
        return x

class Bottleneck_Conv(nn.Module):
    def __init__(self, n_channels, kernel_size=1):
        super().__init__()
        self.linear1 = nn.Conv2d(n_channels, n_channels, kernel_size=kernel_size, padding=kernel_size//2, bias=False)
        self.bn1 = nn.BatchNorm2d(n_channels)
        self.linear2 = nn.Conv2d(n_channels, n_channels, kernel_size=kernel_size, padding=kernel_size//2, bias=False)
        self.bn2 = nn.BatchNorm2d(n_channels)
        self.relu = nn.ReLU(inplace=True)
        
    def forward(self, x):
        assert len(x.shape) in [2, 4]
        input_dims = len(x.shape)
        if input_dims == 2:
            x = x.unsqueeze(-1).unsqueeze(-1)
        residual = x
        out = self.linear1(x)
        out = self.bn1(out)
        out = self.relu(out)
        out = self.linear2(out)
        out = self.bn2(out)
        out += residual
        out = self.relu(out)
        if input_dims == 2:
            out = out.squeeze(-1).squeeze(-1)
        return out

class CLIPFusionBlock_Concat(nn.Module):
    """ 
    Fuse clip and rgb embeddings via concat-proj
    """
    def __init__(self, n_channels=512, n_layers=1, act=True):
        super().__init__()
        proj = [Bottleneck_Linear(2 * n_channels) for _ in range(n_layers)]
        proj.append(nn.Linear(2 * n_channels, n_channels))
        if act: proj.append(nn.GELU())
        self.proj = nn.Sequential(*proj)
    
    def forward(self, sem_latent, clip_latent):
        """ 
        sem_latent: [B, N, C]
        clip_latent: [B, C]
        """
        # [B, N, 2C]
        latent_concat = torch.cat([sem_latent, clip_latent.unsqueeze(1).expand_as(sem_latent)], dim=-1)
        # [B, N, C]
        latent = self.proj(latent_concat)
        return latent

class CLIPFusionBlock_Attn(nn.Module):
    """ 
    Fuse geometric and semantic embeddings via multi-layer MHA blocks
    """
    def __init__(self, n_channels=512, n_layers=1, act=True):
        super().__init__()
        self.attn_blocks = nn.ModuleList(
            [Block(
                n_channels, 8, 4, qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), drop_path=0.1
            ) for _ in range(n_layers)]
        )
        if act: self.attn_blocks.append(nn.GELU())
    
    def forward(self, sem_latent, clip_latent):
        """ 
        sem_latent: [B, N, C]
        clip_latent: [B, C]
        """
        # [B, 1+N, C], clip first
        latent = torch.cat([clip_latent.unsqueeze(1), sem_latent], dim=1)
        for attn_block in self.attn_blocks:
            latent = attn_block(latent)
        # [B, N, C]
        return latent[:, 1:, :]