Delete models

models/__init__.py

@@ -1,14 +0,0 @@
-from .bert import BERTModel
-from .dae import DAEModel
-from .vae import VAEModel
-
-MODELS = {
-    BERTModel.code(): BERTModel,
-    DAEModel.code(): DAEModel,
-    VAEModel.code(): VAEModel
-}
-
-
-def model_factory(args):
-    model = MODELS[args.model_code]
-    return model(args)

models/base.py

@@ -1,15 +0,0 @@
-import torch.nn as nn
-
-from abc import *
-
-
-class BaseModel(nn.Module, metaclass=ABCMeta):
-    def __init__(self, args):
-        super().__init__()
-        self.args = args
-
-    @classmethod
-    @abstractmethod
-    def code(cls):
-        pass
-

models/bert.py

@@ -1,19 +0,0 @@
-from .base import BaseModel
-from .bert_modules.bert import BERT
-
-import torch.nn as nn
-
-
-class BERTModel(BaseModel):
-    def __init__(self, args):
-        super().__init__(args)
-        self.bert = BERT(args)
-        self.out = nn.Linear(self.bert.hidden, args.num_items + 1)
-
-    @classmethod
-    def code(cls):
-        return 'bert'
-
-    def forward(self, x):
-        x = self.bert(x)
-        return self.out(x)

models/bert_modules/__init__.py

@@ -1 +0,0 @@
-

models/bert_modules/attention/__init__.py

@@ -1,2 +0,0 @@
-from .multi_head import MultiHeadedAttention
-from .single import Attention

models/bert_modules/attention/multi_head.py

@@ -1,37 +0,0 @@
-import torch.nn as nn
-from .single import Attention
-
-
-class MultiHeadedAttention(nn.Module):
-    """
-    Take in model size and number of heads.
-    """
-
-    def __init__(self, h, d_model, dropout=0.1):
-        super().__init__()
-        assert d_model % h == 0
-
-        # We assume d_v always equals d_k
-        self.d_k = d_model // h
-        self.h = h
-
-        self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)])
-        self.output_linear = nn.Linear(d_model, d_model)
-        self.attention = Attention()
-
-        self.dropout = nn.Dropout(p=dropout)
-
-    def forward(self, query, key, value, mask=None):
-        batch_size = query.size(0)
-
-        # 1) Do all the linear projections in batch from d_model => h x d_k
-        query, key, value = [l(x).view(batch_size, -1, self.h, self.d_k).transpose(1, 2)
-                             for l, x in zip(self.linear_layers, (query, key, value))]
-
-        # 2) Apply attention on all the projected vectors in batch.
-        x, attn = self.attention(query, key, value, mask=mask, dropout=self.dropout)
-
-        # 3) "Concat" using a view and apply a final linear.
-        x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)
-
-        return self.output_linear(x)

models/bert_modules/attention/single.py

@@ -1,25 +0,0 @@
-import torch.nn as nn
-import torch.nn.functional as F
-import torch
-
-import math
-
-
-class Attention(nn.Module):
-    """
-    Compute 'Scaled Dot Product Attention
-    """
-
-    def forward(self, query, key, value, mask=None, dropout=None):
-        scores = torch.matmul(query, key.transpose(-2, -1)) \
-                 / math.sqrt(query.size(-1))
-
-        if mask is not None:
-            scores = scores.masked_fill(mask == 0, -1e9)
-
-        p_attn = F.softmax(scores, dim=-1)
-
-        if dropout is not None:
-            p_attn = dropout(p_attn)
-
-        return torch.matmul(p_attn, value), p_attn

models/bert_modules/bert.py

@@ -1,44 +0,0 @@
-from torch import nn as nn
-
-from models.bert_modules.embedding import BERTEmbedding
-from models.bert_modules.transformer import TransformerBlock
-from utils import fix_random_seed_as
-
-
-class BERT(nn.Module):
-    def __init__(self, args):
-        super().__init__()
-
-        fix_random_seed_as(args.model_init_seed)
-        # self.init_weights()
-
-        max_len = args.bert_max_len
-        num_items = args.num_items
-        n_layers = args.bert_num_blocks
-        heads = args.bert_num_heads
-        vocab_size = num_items + 2
-        hidden = args.bert_hidden_units
-        self.hidden = hidden
-        dropout = args.bert_dropout
-
-        # embedding for BERT, sum of positional, segment, token embeddings
-        self.embedding = BERTEmbedding(vocab_size=vocab_size, embed_size=self.hidden, max_len=max_len, dropout=dropout)
-
-        # multi-layers transformer blocks, deep network
-        self.transformer_blocks = nn.ModuleList(
-            [TransformerBlock(hidden, heads, hidden * 4, dropout) for _ in range(n_layers)])
-
-    def forward(self, x):
-        mask = (x > 0).unsqueeze(1).repeat(1, x.size(1), 1).unsqueeze(1)
-
-        # embedding the indexed sequence to sequence of vectors
-        x = self.embedding(x)
-
-        # running over multiple transformer blocks
-        for transformer in self.transformer_blocks:
-            x = transformer.forward(x, mask)
-
-        return x
-
-    def init_weights(self):
-        pass

models/bert_modules/embedding/__init__.py

@@ -1 +0,0 @@
-from .bert import BERTEmbedding

models/bert_modules/embedding/bert.py

@@ -1,31 +0,0 @@
-import torch.nn as nn
-from .token import TokenEmbedding
-from .position import PositionalEmbedding
-
-
-class BERTEmbedding(nn.Module):
-    """
-    BERT Embedding which is consisted with under features
-        1. TokenEmbedding : normal embedding matrix
-        2. PositionalEmbedding : adding positional information using sin, cos
-        2. SegmentEmbedding : adding sentence segment info, (sent_A:1, sent_B:2)
-
-        sum of all these features are output of BERTEmbedding
-    """
-
-    def __init__(self, vocab_size, embed_size, max_len, dropout=0.1):
-        """
-        :param vocab_size: total vocab size
-        :param embed_size: embedding size of token embedding
-        :param dropout: dropout rate
-        """
-        super().__init__()
-        self.token = TokenEmbedding(vocab_size=vocab_size, embed_size=embed_size)
-        self.position = PositionalEmbedding(max_len=max_len, d_model=embed_size)
-        # self.segment = SegmentEmbedding(embed_size=self.token.embedding_dim)
-        self.dropout = nn.Dropout(p=dropout)
-        self.embed_size = embed_size
-
-    def forward(self, sequence):
-        x = self.token(sequence) # + self.position(sequence)  # + self.segment(segment_label)
-        return self.dropout(x)

models/bert_modules/embedding/position.py

@@ -1,16 +0,0 @@
-import torch.nn as nn
-import torch
-import math
-
-
-class PositionalEmbedding(nn.Module):
-
-    def __init__(self, max_len, d_model):
-        super().__init__()
-
-        # Compute the positional encodings once in log space.
-        self.pe = nn.Embedding(max_len, d_model)
-
-    def forward(self, x):
-        batch_size = x.size(0)
-        return self.pe.weight.unsqueeze(0).repeat(batch_size, 1, 1)

models/bert_modules/embedding/segment.py

@@ -1,6 +0,0 @@
-import torch.nn as nn
-
-
-class SegmentEmbedding(nn.Embedding):
-    def __init__(self, embed_size=512):
-        super().__init__(3, embed_size, padding_idx=0)

models/bert_modules/embedding/token.py

@@ -1,6 +0,0 @@
-import torch.nn as nn
-
-
-class TokenEmbedding(nn.Embedding):
-    def __init__(self, vocab_size, embed_size=512):
-        super().__init__(vocab_size, embed_size, padding_idx=0)

models/bert_modules/transformer.py

@@ -1,31 +0,0 @@
-import torch.nn as nn
-
-from .attention import MultiHeadedAttention
-from .utils import SublayerConnection, PositionwiseFeedForward
-
-
-class TransformerBlock(nn.Module):
-    """
-    Bidirectional Encoder = Transformer (self-attention)
-    Transformer = MultiHead_Attention + Feed_Forward with sublayer connection
-    """
-
-    def __init__(self, hidden, attn_heads, feed_forward_hidden, dropout):
-        """
-        :param hidden: hidden size of transformer
-        :param attn_heads: head sizes of multi-head attention
-        :param feed_forward_hidden: feed_forward_hidden, usually 4*hidden_size
-        :param dropout: dropout rate
-        """
-
-        super().__init__()
-        self.attention = MultiHeadedAttention(h=attn_heads, d_model=hidden, dropout=dropout)
-        self.feed_forward = PositionwiseFeedForward(d_model=hidden, d_ff=feed_forward_hidden, dropout=dropout)
-        self.input_sublayer = SublayerConnection(size=hidden, dropout=dropout)
-        self.output_sublayer = SublayerConnection(size=hidden, dropout=dropout)
-        self.dropout = nn.Dropout(p=dropout)
-
-    def forward(self, x, mask):
-        x = self.input_sublayer(x, lambda _x: self.attention.forward(_x, _x, _x, mask=mask))
-        x = self.output_sublayer(x, self.feed_forward)
-        return self.dropout(x)

models/bert_modules/utils/__init__.py

@@ -1,4 +0,0 @@
-from .feed_forward import PositionwiseFeedForward
-from .layer_norm import LayerNorm
-from .sublayer import SublayerConnection
-from .gelu import GELU

models/bert_modules/utils/feed_forward.py

@@ -1,16 +0,0 @@
-import torch.nn as nn
-from .gelu import GELU
-
-
-class PositionwiseFeedForward(nn.Module):
-    "Implements FFN equation."
-
-    def __init__(self, d_model, d_ff, dropout=0.1):
-        super(PositionwiseFeedForward, self).__init__()
-        self.w_1 = nn.Linear(d_model, d_ff)
-        self.w_2 = nn.Linear(d_ff, d_model)
-        self.dropout = nn.Dropout(dropout)
-        self.activation = GELU()
-
-    def forward(self, x):
-        return self.w_2(self.dropout(self.activation(self.w_1(x))))

models/bert_modules/utils/gelu.py

@@ -1,12 +0,0 @@
-import torch.nn as nn
-import torch
-import math
-
-
-class GELU(nn.Module):
-    """
-    Paper Section 3.4, last paragraph notice that BERT used the GELU instead of RELU
-    """
-
-    def forward(self, x):
-        return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))

models/bert_modules/utils/layer_norm.py

@@ -1,17 +0,0 @@
-import torch.nn as nn
-import torch
-
-
-class LayerNorm(nn.Module):
-    "Construct a layernorm module (See citation for details)."
-
-    def __init__(self, features, eps=1e-6):
-        super(LayerNorm, self).__init__()
-        self.a_2 = nn.Parameter(torch.ones(features))
-        self.b_2 = nn.Parameter(torch.zeros(features))
-        self.eps = eps
-
-    def forward(self, x):
-        mean = x.mean(-1, keepdim=True)
-        std = x.std(-1, keepdim=True)
-        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2

models/bert_modules/utils/sublayer.py

@@ -1,18 +0,0 @@
-import torch.nn as nn
-from .layer_norm import LayerNorm
-
-
-class SublayerConnection(nn.Module):
-    """
-    A residual connection followed by a layer norm.
-    Note for code simplicity the norm is first as opposed to last.
-    """
-
-    def __init__(self, size, dropout):
-        super(SublayerConnection, self).__init__()
-        self.norm = LayerNorm(size)
-        self.dropout = nn.Dropout(dropout)
-
-    def forward(self, x, sublayer):
-        "Apply residual connection to any sublayer with the same size."
-        return x + self.dropout(sublayer(self.norm(x)))

models/dae.py

@@ -1,54 +0,0 @@
-from .base import BaseModel
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class DAEModel(BaseModel):
-    def __init__(self, args):
-        super().__init__(args)
-
-        # Input dropout
-        self.input_dropout = nn.Dropout(p=args.dae_dropout)
-
-        # Construct a list of dimensions for the encoder and the decoder
-        dims = [args.dae_hidden_dim] * 2 * args.dae_num_hidden
-        dims = [args.num_items] + dims + [args.dae_latent_dim]
-
-        # Stack encoders and decoders
-        encoder_modules, decoder_modules = [], []
-        for i in range(len(dims)//2):
-            encoder_modules.append(nn.Linear(dims[2*i], dims[2*i+1]))
-            decoder_modules.append(nn.Linear(dims[-2*i-1], dims[-2*i-2]))
-        self.encoder = nn.ModuleList(encoder_modules)
-        self.decoder = nn.ModuleList(decoder_modules)
-
-        # Initialize weights
-        self.encoder.apply(self.weight_init)
-        self.decoder.apply(self.weight_init)
-
-    def weight_init(self, m):
-        if isinstance(m, nn.Linear):
-            nn.init.kaiming_normal_(m.weight)
-            m.bias.data.normal_(0.0, 0.001)
-
-    @classmethod
-    def code(cls):
-        return 'dae'
-
-    def forward(self, x):
-        x = F.normalize(x)
-        x = self.input_dropout(x)
-        
-        for i, layer in enumerate(self.encoder):
-            x = layer(x)
-            x = torch.tanh(x)
-        
-        for i, layer in enumerate(self.decoder):
-            x = layer(x)
-            if i != len(self.decoder)-1:
-                x = torch.tanh(x)
-
-        return x
-

models/vae.py

@@ -1,69 +0,0 @@
-from .base import BaseModel
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class VAEModel(BaseModel):
-    def __init__(self, args):
-        super().__init__(args)
-        self.latent_dim = args.vae_latent_dim
-
-        # Input dropout
-        self.input_dropout = nn.Dropout(p=args.vae_dropout)
-
-        # Construct a list of dimensions for the encoder and the decoder
-        dims = [args.vae_hidden_dim] * 2 * args.vae_num_hidden
-        dims = [args.num_items] + dims + [args.vae_latent_dim * 2]
-
-        # Stack encoders and decoders
-        encoder_modules, decoder_modules = [], []
-        for i in range(len(dims)//2):
-            encoder_modules.append(nn.Linear(dims[2*i], dims[2*i+1]))
-            if i == 0:
-                decoder_modules.append(nn.Linear(dims[-1]//2, dims[-2]))
-            else:
-                decoder_modules.append(nn.Linear(dims[-2*i-1], dims[-2*i-2]))
-        self.encoder = nn.ModuleList(encoder_modules)
-        self.decoder = nn.ModuleList(decoder_modules)
-
-        # Initialize weights
-        self.encoder.apply(self.weight_init)
-        self.decoder.apply(self.weight_init)
-
-    def weight_init(self, m):
-        if isinstance(m, nn.Linear):
-            nn.init.kaiming_normal_(m.weight)
-            m.bias.data.zero_()
-
-    @classmethod
-    def code(cls):
-        return 'vae'
-
-    def forward(self, x):
-        x = F.normalize(x)
-        x = self.input_dropout(x)
-        
-        for i, layer in enumerate(self.encoder):
-            x = layer(x)
-            if i != len(self.encoder) - 1:
-                x = torch.tanh(x)
-        
-        mu, logvar = x[:, :self.latent_dim], x[:, self.latent_dim:]
-
-        if self.training:
-            # since log(var) = log(sigma^2) = 2*log(sigma)
-            sigma = torch.exp(0.5 * logvar)
-            eps = torch.randn_like(sigma)
-            x = mu + eps * sigma
-        else:
-            x = mu
-
-        for i, layer in enumerate(self.decoder):
-            x = layer(x)
-            if i != len(self.decoder) - 1:
-                x = torch.tanh(x)
-                
-        return x, mu, logvar
-