Upload DistilBertClassifier

classifier.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel, AutoModel, AutoConfig, PretrainedConfig
+import transformers
+
+
+class DistilBertClassifier(PreTrainedModel):
+
+    def __init__(self, bert_config, model_name='distilbert-base-uncased', tokenizer_len=30528, freeze_bert=False):
+
+
+        super().__init__(bert_config)
+        D_in, H, D_out = 256, 50, 71
+
+        self.bert = AutoModel.from_pretrained(model_name)
+        self.bert.resize_token_embeddings(tokenizer_len)
+        self.classifier = nn.Sequential(
+            nn.GELU(),
+            nn.Linear(self.bert.config.hidden_size, 300),
+            nn.GELU(),
+            nn.Dropout(0.05),
+            nn.Linear(300, 71)
+        )
+
+        if freeze_bert:
+            for param in self.bert.parameters():
+                param.requires_grad = False
+
+    def forward(self, input_ids, attention_mask):
+
+        outputs = self.bert(input_ids=input_ids,
+                            attention_mask=attention_mask)
+
+        last_hidden_state_cls = outputs[0][:, 0, :]
+        logits = self.classifier(last_hidden_state_cls)
+        return logits

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "distilbert-base-uncased",
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertClassifier"
+  ],
+  "attention_dropout": 0.1,
+  "auto_map": {
+    "AutoModelForSequenceClassification": "classifier.DistilBertClassifier"
+  },
+  "dim": 800,
+  "dropout": 0.1,
+  "hidden_dim": 3072,
+  "hidden_dropout_prob": 0.1,
+  "initializer_range": 2,
+  "intermediate_size": 500,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 270,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 3,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.1",
+  "vocab_size": 30528
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df5a46dbe135b0e83312597b1e11f174fda4db4ac797a248c367d876e2ccad
+size 266511597