Upload BERT_Arch

config.json

@@ -3,8 +3,8 @@
     "BERT_Arch"
   ],
   "auto_map": {
-    "AutoConfig": "__main__.PragFormerConfig",
-    "AutoModel": "__main__.BERT_Arch"
+    "AutoConfig": "model_config.PragFormerConfig",
+    "AutoModel": "model.BERT_Arch"
   },
   "bert": {
     "_commit_hash": "43cf2d48e8c75d255dccab2a19e40d4774fd8853",

model.py

@@ -1,11 +1,19 @@
+from transformers import AutoModel, AutoConfig
+import torch.nn as nn
 from transformers import BertPreTrainedModel, AutoModel, PretrainedConfig
-import sys
-sys.path.append("..")
 
-import torch.nn as nn
-from Classifier.pragformer_config import PragFormerConfig
 
 
+class PragFormerConfig(PretrainedConfig):
+    model_type = "pragformer"
+
+    def __init__(self, bert=None, dropout=0.2, fc1=512, fc2=2, softmax_dim=1, **kwargs):
+        self.bert = bert
+        self.dropout = dropout
+        self.fc1 = fc1
+        self.fc2 = fc2
+        self.softmax_dim = softmax_dim
+        super().__init__(**kwargs)
 
 
 class BERT_Arch(BertPreTrainedModel):
@@ -49,99 +57,3 @@ class BERT_Arch(BertPreTrainedModel):
         # apply softmax activation
         x = self.softmax(x)
         return x
-
-
-
-
-
-# class BERT_Arch_new(BertPreTrainedModel):
-#     def __init__(self, config):
-#         super().__init__(config)
-#         self.bert = AutoModel.from_pretrained('/home/talkad/Desktop/pragformer/PragFormer/DeepSCC-RoBERTa')
-
-#         # dropout layer
-#         self.dropout = nn.Dropout(0.2)
-
-#         # relu activation function
-#         self.relu = nn.ReLU()
-
-#         # dense layer 1
-#         self.fc1 = nn.Linear(self.config.hidden_size, 512)
-#         # self.fc1 = nn.Linear(768, 512)
-
-#         # dense layer 2 (Output layer)
-#         self.fc2 = nn.Linear(512, 2)
-
-#         # softmax activation function
-#         self.softmax = nn.LogSoftmax(dim = 1)
-
-#     # define the forward pass
-#     def forward(self, input_ids, attention_mask):
-#         # pass the inputs to the model
-#         _, cls_hs = self.bert(input_ids, attention_mask = attention_mask, return_dict=False)
-
-#         x = self.fc1(cls_hs)
-
-#         x = self.relu(x)
-
-#         x = self.dropout(x)
-
-#         # output layer
-#         x = self.fc2(x)
-
-#         # apply softmax activation
-#         x = self.softmax(x)
-#         return x
-
-
-
-
-
-
-
-
-# class BERT_Arch(nn.Module):
-#     def __init__(self, bert):
-#         super(BERT_Arch, self).__init__()
-#         self.bert = bert
-
-#         # dropout layer
-#         self.dropout = nn.Dropout(0.2)
-
-#         # relu activation function
-#         self.relu = nn.ReLU()
-
-#         # dense layer 1
-#         self.fc1 = nn.Linear(768, 512)
-
-
-#         # dense layer 2 (Output layer)
-#         self.fc2 = nn.Linear(512, 2)
-
-#         # softmax activation function
-#         self.softmax = nn.LogSoftmax(dim = 1)
-
-#     # define the forward pass
-#     def forward(self, input_ids, attention_mask):
-#         # pass the inputs to the model
-#         _, cls_hs = self.bert(input_ids, attention_mask = attention_mask, return_dict=False)
-
-#         x = self.fc1(cls_hs)
-
-#         x = self.relu(x)
-
-#         x = self.dropout(x)
-
-#         # output layer
-#         x = self.fc2(x)
-
-#         # apply softmax activation
-#         x = self.softmax(x)
-#         return x
-
-#     def save_pretrained_model(self, path="", push=False, repo_name=""):
-#         if not push:
-#             self.bert.save_pretrained(path, repo_url=repo_name)
-#         else:
-#             self.bert.push_to_hub(repo_name)
-

model_config.py

@@ -0,0 +1,13 @@
+from transformers import PretrainedConfig
+
+
+class PragFormerConfig(PretrainedConfig):
+    model_type = "pragformer"
+
+    def __init__(self, bert=None, dropout=0.2, fc1=512, fc2=2, softmax_dim=1, **kwargs):
+        self.bert = bert
+        self.dropout = dropout
+        self.fc1 = fc1
+        self.fc2 = fc2
+        self.softmax_dim = softmax_dim
+        super().__init__(**kwargs)