add model

config.json

@@ -9,15 +9,17 @@
     "AutoModelForMaskedLM": "modeling_lddbert.LddBertForMaskedLM",
     "AutoModelForSequenceClassification": "modeling_lddbert.LddBertForSequenceClassification"
   },
+  "cnn_kernel_size": 5,
   "dim": 768,
   "dropout": 0.1,
   "hidden_dim": 3072,
   "initializer_range": 0.02,
   "max_position_embeddings": 512,
   "model_type": "lddbert",
-  "n_gru_layers": 1,
+  "n_cnn_layers": 6,
+  "n_gru_layers": 6,
   "n_heads": 12,
-  "n_layers": 12,
+  "n_layers": 6,
   "pad_token_id": 0,
   "qa_dropout": 0.1,
   "seq_classif_dropout": 0.2,

configuration_lddbert.py

@@ -87,7 +87,7 @@ class LddBertConfig(PretrainedConfig):
 
     def __init__(
         self,
-        n_layers=12,
+        n_layers=6,
         n_heads=12,
         dim=768,
         hidden_dim=4*768,
@@ -102,7 +102,9 @@ class LddBertConfig(PretrainedConfig):
         attention_dropout=0.1,
         qa_dropout=0.1,
         seq_classif_dropout=0.2,
-        n_gru_layers=1,
+        n_gru_layers=6,
+        n_cnn_layers=6,
+        cnn_kernel_size=5,
         **kwargs
     ):
         self.vocab_size = vocab_size
@@ -110,6 +112,8 @@ class LddBertConfig(PretrainedConfig):
         self.sinusoidal_pos_embds = sinusoidal_pos_embds
         self.n_layers = n_layers
         self.n_gru_layers = n_gru_layers
+        self.n_cnn_layers = n_cnn_layers
+        self.cnn_kernel_size = cnn_kernel_size
         self.n_heads = n_heads
         self.dim = dim
         self.hidden_dim = hidden_dim

modeling_lddbert.py

@@ -378,9 +378,15 @@ LDDBERT_INPUTS_DOCSTRING = DISTILBERT_INPUTS_DOCSTRING
 class LddBertModel(LddBertPreTrainedModel):
     def __init__(self, config: PretrainedConfig):
         super().__init__(config)
+        assert config.cnn_kernel_size%2 == 1
 
         self.embeddings = Embeddings(config)  # Embeddings
         self.transformer = Transformer(config)  # Encoder
+        self.gru = nn.GRU(config.dim , config.dim//2, config.n_gru_layers, batch_first=True, bidirectional=True)
+        self.cnn = nn.Sequential(*(
+            nn.Conv1d(config.max_position_embeddings, config.max_position_embeddings, config.cnn_kernel_size, padding=(config.cnn_kernel_size-1)//2)
+            for _ in range(config.n_cnn_layers)
+        ))
 
         # Initialize weights and apply final processing
         self.post_init()
@@ -494,7 +500,7 @@ class LddBertModel(LddBertPreTrainedModel):
                 token_type_ids=token_type_ids,
             )  # (bs, seq_length, dim)
         
-        return self.transformer(
+        bert_output = self.transformer(
             x=inputs_embeds,
             attn_mask=attention_mask,
             head_mask=head_mask,
@@ -503,6 +509,22 @@ class LddBertModel(LddBertPreTrainedModel):
             return_dict=return_dict,
         )
 
+        gru_output, _ = self.gru(bert_output[0])
+
+        cnn_output = self.cnn(bert_output[0])
+
+        output = gru_output + cnn_output
+        if not return_dict:
+            return (output, ) + bert_output[1:]
+
+        return BaseModelOutput(
+            last_hidden_state=output,
+            hidden_states=bert_output.hidden_states,
+            attentions=bert_output.attentions,
+        )
+
+        
+
 
 @add_start_docstrings(
     """LddBert Model with a `masked language modeling` head on top.""",
@@ -622,15 +644,12 @@ class LddBertForSequenceClassification(LddBertPreTrainedModel):
         self.num_labels = config.num_labels
         self.config = config
 
-        assert config.dim % 2 == 0
-
-        self.activation = get_activation(config.activation)
 
         self.lddbert = LddBertModel(config)
-        self.gru = nn.GRU(config.dim , 32, config.n_gru_layers, batch_first=True, bidirectional=True)
-        self.layer_norm = nn.LayerNorm(config.dim + 32*2*config.n_gru_layers, eps=1e-12)
+        self.pre_classifier = nn.Linear(config.dim, config.dim)
+        self.activation = get_activation(config.activation)
         self.dropout = nn.Dropout(config.seq_classif_dropout)
-        self.classifier = nn.Linear(config.dim + 32*2*config.n_gru_layers, config.num_labels)
+        self.classifier = nn.Linear(config.dim, config.num_labels)
 
         # Initialize weights and apply final processing
         self.post_init()
@@ -693,15 +712,11 @@ class LddBertForSequenceClassification(LddBertPreTrainedModel):
         )
         hidden_state = lddbert_output[0]  # (bs, seq_len, dim)
 
-        cls_output= hidden_state[:, 0]  # (bs, dim)
-        gru_output, _ = self.gru(hidden_state[:, 1:])  # (bs, seq_len, dim)
-        gru_output = gru_output[:, -1]
-
-        concat_output = torch.cat((cls_output, gru_output), dim=-1)
-        concat_output = self.activation(concat_output)  # (bs, dim)
-        concat_output = self.layer_norm(concat_output)  # (bs, dim)
-        concat_output = self.dropout(concat_output) # (bs, dim)
-        logits = self.classifier(concat_output)  # (bs, num_labels)
+        pooled_output = hidden_state[:, 0]  # (bs, dim)
+        pooled_output = self.pre_classifier(pooled_output)  # (bs, dim)
+        pooled_output = self.activation(pooled_output)  # (bs, dim)
+        pooled_output = self.dropout(pooled_output)  # (bs, dim)
+        logits = self.classifier(pooled_output)  # (bs, num_labels)
 
         loss = None
         if labels is not None:

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bfcefa51ca80f85ff8c117271997b66f92e10f8638237d14f908605703c7419
-size 438121645
+oid sha256:151f439844ff10c523e93c90fbce4a543ab1bcce6f660822748eae4bd2e9c94c
+size 363280885