handler

encoder/model.py

@@ -0,0 +1,129 @@
+import sys
+from .transformer import ViT
+sys.path.append("/".join(__file__.split('/')[:-2]))
+from params_model import *
+from params_data import *
+
+from collections import OrderedDict
+from torch import nn
+import torch
+
+class ConvBlock(nn.Sequential):
+    def __init__(self, in_channels, out_channels, kernel_size, padding=0):
+        super().__init__(OrderedDict([
+            ('conv', nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding, bias=False)),
+            ('bn', nn.BatchNorm2d(out_channels)),
+            ('relu', nn.ReLU(inplace=True)),
+        ]))
+
+class SqueezeExcitation(nn.Module):
+    def __init__(self, channels, ratio):
+        super().__init__()
+
+        self.pool = nn.AdaptiveAvgPool2d(1)
+        # tiny nn
+        self.lin1 = nn.Linear(channels, channels // ratio)
+        self.relu = nn.ReLU(inplace=True)
+        self.lin2 = nn.Linear(channels // ratio, 2 * channels)
+
+    def forward(self, x):
+        n, c, h, w = x.size()
+        x_in = x
+
+        x = self.pool(x).view(n, c)
+        x = self.lin1(x)
+        x = self.relu(x)
+        x = self.lin2(x)
+
+        x = x.view(n, 2 * c, 1, 1)
+        scale, shift = x.chunk(2, dim=1)
+
+        x = scale.sigmoid() * x_in + shift
+        return x
+
+class ResidualBlock(nn.Module):
+    def __init__(self, channels, se_ratio):
+        super().__init__()
+        self.layers = nn.Sequential(OrderedDict([
+            ('conv1', nn.Conv2d(channels, channels, 3, padding=1, bias=False)),
+            ('bn1', nn.BatchNorm2d(channels)),
+            ('relu', nn.ReLU(inplace=True)),
+
+            ('conv2', nn.Conv2d(channels, channels, 3, padding=1, bias=False)),
+            ('bn2', nn.BatchNorm2d(channels)),
+
+            ('se', SqueezeExcitation(channels, se_ratio)),
+        ]))
+        self.relu2 = nn.ReLU(inplace=True)
+
+    def forward(self, x):
+        x_in = x
+
+        x = self.layers(x)
+
+        x = x + x_in
+        x = self.relu2(x)
+        return x
+
+class Encoder(nn.Module):
+
+    def __init__(self, loss_device, loss_method = "softmax"):
+        super().__init__()
+        self.loss_device = loss_device
+        
+        channels = residual_channels
+
+        self.conv_block = ConvBlock(34, channels, 3, padding=1)
+        blocks = [(f'block{i+1}', ResidualBlock(channels, se_ratio)) for i in range(residual_blocks)]
+        self.residual_stack = nn.Sequential(OrderedDict(blocks))
+
+        self.conv_block2 = ConvBlock(channels, channels, 3, padding=1)
+        self.final_feature = ConvBlock(channels, vit_input_channels, 3, padding=1)
+        self.global_avgpool = nn.AvgPool2d(kernel_size=8)
+
+        self.cnn = nn.Sequential(*[
+            self.conv_block,
+            self.residual_stack,
+            self.conv_block2,
+            self.final_feature,
+            self.global_avgpool,
+            torch.nn.Flatten()
+        ])
+
+        self.transformer = ViT(input_dim=vit_input_channels, 
+                               output_dim=model_embedding_size, 
+                               dim=transformer_input_dim, 
+                               depth=transformer_depth, 
+                               heads=attention_heads, 
+                               mlp_dim=mlp_dim, 
+                               pool='mean',
+                               dim_head = dim_head,
+                               dropout=dropout, 
+                               emb_dropout=emb_dropout)
+        
+        # Cosine similarity scaling (with fixed initial parameter values)
+        self.similarity_weight = nn.Parameter(torch.tensor([similarity_weight_init])) 
+        self.similarity_bias = nn.Parameter(torch.tensor([similarity_bias_init]))
+    
+    def forward(self, games):
+
+        batch_size, n_frames, feature_shape = games.shape[0], games.shape[1], games.shape[2:]
+        
+        #  (batch_size, n_frames, 34, 8, 8) -> (batch_size*n_frames, 34, 8, 8)
+        games = torch.reshape(games, (batch_size*n_frames, *feature_shape))
+
+        # (batch_size*n_frames, cnn_out_features)
+        game_features = self.cnn(games)
+
+        # (batch_size*n_frames, cnn_out_features) -> (batch_size, n_frames, cnn_out_features)
+        game_features = torch.reshape(game_features, (batch_size, n_frames, game_features.shape[-1]))
+
+        # Pass the input into transformer
+        # (batch_size, n_frames, n_features) 
+        embeds_raw = self.transformer(game_features)
+        # self.lstm.flatten_parameters()
+
+        # L2-normalize it
+        embeds = embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True)
+        
+        return embeds

encoder/transformer.py

@@ -0,0 +1,143 @@
+# original vision transformer from https://github.com/lucidrains/vit-pytorch/blob/main/vit_pytorch/vit.py
+import numpy as np
+import torch
+from torch import nn, einsum
+import torch.nn.functional as F
+
+# https://einops.rocks/pytorch-examples.html
+from einops import rearrange
+
+class PreNorm(nn.Module):
+    def __init__(self, dim, fn):
+        super().__init__()
+        self.norm = nn.LayerNorm(dim)
+        self.fn = fn
+    def forward(self, x):
+        return self.fn(self.norm(x))
+
+class FeedForward(nn.Module):
+    def __init__(self, dim, hidden_dim, dropout = 0.):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(dim, hidden_dim),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim, dim),
+            nn.Dropout(dropout)
+        )
+    def forward(self, x):
+        return self.net(x)
+
+class Attention(nn.Module):
+    def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):
+        super().__init__()
+        inner_dim = dim_head *  heads
+        project_out = not (heads == 1 and dim_head == dim)
+
+        self.heads = heads
+        self.scale = dim_head ** -0.5
+
+        self.attend = nn.Softmax(dim = -1)
+        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)
+
+        self.to_out = nn.Sequential(
+            nn.Linear(inner_dim, dim),
+            nn.Dropout(dropout)
+        ) if project_out else nn.Identity()
+
+    def forward(self, x):
+        b, n, _, h = *x.shape, self.heads
+        qkv = self.to_qkv(x).chunk(3, dim = -1)
+        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv)
+
+        # for each batch and each head, multiply each query position (i) with each key position (j), summing over the embedding dimension (d), etc
+        dots = einsum('b h i d, b h j d -> b h i j', q, k) * self.scale
+
+        attn = self.attend(dots)
+
+        out = einsum('b h i j, b h j d -> b h i d', attn, v)
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        return self.to_out(out)
+
+class Transformer(nn.Module):
+    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.):
+        super().__init__()
+        self.layers = nn.ModuleList([])
+        for _ in range(depth):
+            self.layers.append(nn.ModuleList([
+                PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)),
+                PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))
+            ]))
+    def forward(self, x):
+        for attn, ff in self.layers:
+            x = attn(x) + x
+            x = ff(x) + x
+        return x
+
+class PositionalEncoding(nn.Module):
+    # https://discuss.pytorch.org/t/positional-encoding/175953
+    def __init__(self, d_model, max_len=500):
+        super().__init__()
+
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        # alternatively adding sign and cos waves of increasing frequency
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-np.log(10000.0) / d_model))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x):
+        # not x = x + self.pe[:x.size(0), :] since
+        # x.size(0): batch size whereas x.size(1): length of sequence
+        x = x + self.pe[:, :x.size(1), :]
+        return x
+
+class ViT(nn.Module):
+    """
+    input_size: number of inputs
+    input_dim: number of channels in input
+    dim: Last dimension of output tensor after linear transformation nn.Linear(..., dim).
+    depth: Number of Transformer blocks.
+    heads: Number of heads in Multi-head Attention layer.
+    mlp_dim: Dimension of the MLP (FeedForward) layer.
+    dropout: Dropout rate.
+    emb_dropout: Embedding dropout rate.
+    pool: either cls token pooling or mean pooling
+    """
+    # * to force keyword-only args
+    def __init__(self, *, input_dim, output_dim, dim, depth, heads, mlp_dim, pool = 'mean', dim_head = 64, dropout, emb_dropout):
+        super().__init__()
+
+        self.project = nn.Linear(input_dim, dim)
+
+        self.pos_encoder = PositionalEncoding(dim)
+
+        self.dropout = nn.Dropout(emb_dropout)
+
+        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)
+
+        self.pool = pool
+
+        self.mlp_head = nn.Sequential(
+            nn.LayerNorm(dim),
+            nn.Linear(dim, output_dim)
+        )
+
+        self.tanh = torch.nn.Tanh()
+
+    def forward(self, x):
+
+        x = self.project(x)
+        b, n, _ = x.shape
+
+        x = self.pos_encoder(x)
+
+        x = self.dropout(x)
+
+        x = self.transformer(x)
+
+        x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]
+
+        return self.tanh(self.mlp_head(x))

handler.py

@@ -0,0 +1,29 @@
+import torch
+from encoder.model import Encoder
+ 
+class EndpointHandler():
+    def __init__(self, path="6.pt"):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        checkpoint = torch.load(path, self.device, weights_only=True)
+        self.model = Encoder(self.device)
+        state_dict = checkpoint['model_state']
+        self.model.load_state_dict(state_dict)
+        self.model = self.model.to(self.device)
+        self.model.eval()
+ 
+    def __call__(self, data):
+        tensor = torch.from_numpy(data.tensor).float().to(self.device)
+        if len(data) == 1:
+            with torch.no_grad():
+                embed = self.model(tensor)
+                embed = embed / torch.norm(embed)
+            return {"reply": embed.cpu().numpy()}
+        else:
+            with torch.no_grad():
+                embeds = self.model(tensor)
+                embeds = embeds.view((1, data.num_games, -1)).to(self.device)
+                centroids_incl = torch.mean(embeds, dim=1, keepdim=True)
+                centroids_incl = centroids_incl.clone() / torch.norm(centroids_incl, dim=2, keepdim=True)
+            centroids_incl = centroids_incl.cpu().squeeze(1)
+            final_embeds = centroids_incl[0].numpy().tolist()
+            return {"reply": final_embeds}

params_data.py

@@ -0,0 +1,7 @@
+# can add variability in frames per batch
+random_partial_low = 32
+random_partial_high = 32
+game_start = 0
+
+# 32 moves as a window
+partials_n_frames = 32

params_model.py

@@ -0,0 +1,24 @@
+## Model parameters
+residual_channels = 64
+residual_blocks = 6
+se_ratio = 8
+vit_input_channels = 320 # input dimension to ViT
+transformer_input_dim = 1024
+model_embedding_size = 512
+transformer_depth = 12
+attention_heads = 8
+mlp_dim = 2048
+dim_head = 64 # k_q_v dims, risky to tune?
+dropout = 0.
+emb_dropout = 0.
+similarity_weight_init = 10.
+similarity_bias_init = -5.
+
+## Training parameters
+learning_rate_init = 0.005
+players_per_batch = 36
+games_per_player = 10
+
+v_players_per_batch = 40
+v_games_per_player = 10
+num_validate = 10

requirements.txt

@@ -0,0 +1,2 @@
+torch
+numpy