models/global_local_memory_module.py

import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import time
# FFN
def FeedForward(dim, mult=4):
    inner_dim = int(dim * mult)
    return nn.Sequential(
        nn.LayerNorm(dim),
        nn.Linear(dim, inner_dim, bias=False),
        nn.GELU(),
        nn.Linear(inner_dim, dim, bias=False),
    )


def reshape_tensor(x, heads):
    bs, length, width = x.shape
    # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
    x = x.view(bs, length, heads, -1)
    # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
    x = x.transpose(1, 2)
    # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
    x = x.reshape(bs, heads, length, -1)
    return x


class PerceiverAttentionCA(nn.Module):
    def __init__(self, *, dim=3072, dim_head=128, heads=16, kv_dim=2048):
        super().__init__()
        self.scale = dim_head ** -0.5
        self.dim_head = dim_head
        self.heads = heads
        inner_dim = dim_head * heads

        self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim)
        self.norm2 = nn.LayerNorm(dim)

        self.to_q = nn.Linear(dim, inner_dim, bias=False)
        self.to_kv = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim * 2, bias=False)
        self.to_out = nn.Linear(inner_dim, dim, bias=False)

    def forward(self, x, latents):
        """
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D)
        """
        x = self.norm1(x)
        latents = self.norm2(latents)

        b, seq_len, _ = latents.shape

        q = self.to_q(latents)
        k, v = self.to_kv(x).chunk(2, dim=-1)

        q = reshape_tensor(q, self.heads)
        k = reshape_tensor(k, self.heads)
        v = reshape_tensor(v, self.heads)

        # attention
        scale = 1 / math.sqrt(math.sqrt(self.dim_head))
        weight = (q * scale) @ (k * scale).transpose(-2, -1)  # More stable with f16 than dividing afterwards
        weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
        out = weight @ v

        out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1)

        return self.to_out(out)


class PerceiverAttention(nn.Module):
    def __init__(self, *, dim, dim_head=64, heads=8, kv_dim=None):
        super().__init__()
        self.scale = dim_head ** -0.5
        self.dim_head = dim_head
        self.heads = heads
        inner_dim = dim_head * heads

        self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim)
        self.norm2 = nn.LayerNorm(dim)

        self.to_q = nn.Linear(dim, inner_dim, bias=False)
        self.to_k = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim , bias=False)
        self.to_v = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim , bias=False)
        self.to_out = nn.Linear(inner_dim, dim, bias=False)

    def forward(self, gl_key,gl_value, latents):
        """
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D)
        """
        gl_key=self.norm1(gl_key)
        gl_value=self.norm1(gl_value)
        latents = self.norm2(latents)

        b, seq_len, _ = latents.shape

        q = self.to_q(latents)
        
        k=self.to_k(gl_key)
        v=self.to_v(gl_value)
        

        q = reshape_tensor(q, self.heads)
        k = reshape_tensor(k, self.heads)
        v = reshape_tensor(v, self.heads)
        
        out = F.scaled_dot_product_attention(
            q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False
        )
        
        # attention
        '''
        scale = 1 / math.sqrt(math.sqrt(self.dim_head))
        weight = (q * scale) @ (k * scale).transpose(-2, -1)  # More stable with f16 than dividing afterwards
        weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
        out = weight @ v
        '''

        out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1)
        
        return self.to_out(out)


class global_local_memory(nn.Module):
    """
    - perceiver resampler like arch (compared with previous MLP-like arch)
    - we concat id embedding (generated by arcface) and query tokens as latents
    - latents will attend each other and interact with vit features through cross-attention
    - vit features are multi-scaled and inserted into IDFormer in order, currently, each scale corresponds to two IDFormer layers
    """
    def __init__(
            self,
            dim=3072,
            depth=3,
            dim_head=96,
            heads=32,
            num_queries=32,
            output_dim=3072,
            ff_mult=4,
    ):
        super().__init__()
        
        self.dim = dim
        self.num_queries = num_queries
        assert depth % 3 == 0
        scale = dim ** -0.5

        self.proj_out = nn.Parameter( torch.zeros(dim, output_dim))

        self.layers = nn.ModuleList([])
        for _ in range(depth):
            self.layers.append(
                nn.ModuleList(
                    [
                        PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
                        FeedForward(dim=dim, mult=ff_mult),
                    ]
                )
            )

        for i in range(3):
            
            setattr(
                self,
                f'key_mapping_{i}',
                nn.Sequential(
                    nn.Linear(512, 1024),
                    nn.LayerNorm(1024),
                    nn.LeakyReLU(),
                    nn.Linear(1024, 2048),
                    nn.LayerNorm(2048),
                    nn.LeakyReLU(),
                    nn.Linear(2048, dim),
                ),
            )
            setattr(
                self,
                f'value_mapping_{i}',
                nn.Sequential(
                    nn.Linear(512, 1024),
                    nn.LayerNorm(1024),
                    nn.LeakyReLU(),
                    nn.Linear(1024, 2048),
                    nn.LayerNorm(2048),
                    nn.LeakyReLU(),
                    nn.Linear(2048, dim),
                ),
            )
    def forward(self, global_memory, local_memory,latents):
        
        latents = latents.repeat(global_memory.size(0), 1,1)

        global_local_memory_key=torch.cat((global_memory[:,0],local_memory[:,0]),dim=2)  #torch.Size([5, 5110, 512])
        global_local_memory_value=torch.cat((global_memory[:,1],local_memory[:,1]),dim=2) #torch.Size([5, 5110, 512])


        for i in range(3):
            global_local_key = getattr(self, f'key_mapping_{i}')(global_local_memory_key[:,i])
            global_local_value  = getattr(self, f'value_mapping_{i}')(global_local_memory_value[:,i])
            attn,ff=self.layers[i]
            
            latents = attn(global_local_key.repeat_interleave(latents.shape[0], dim=0), global_local_value.repeat_interleave(latents.shape[0], dim=0), latents) + latents
            latents = ff(latents) + latents    
            
        latents = latents @ self.proj_out
       
        return latents


if __name__=="__main__":
    global_memory=torch.randn(1,2, 3, 3626, 512)
    local_memory=torch.randn(1,2,3,1484,512)
    latents=torch.randn(1,25344,3072)
    glm=global_local_memory()
    total_params = sum(p.numel() for p in glm.parameters())
    print(total_params)
    
    start_time=time.time()
    latents=glm(global_memory,local_memory,latents)
    end_time=time.time()
    print(f"Elapsed time (process_time): {end_time-start_time} seconds")