add model

config.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "BertSentiment"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "configuration_cbert.BertCustomConfig",
+    "AutoModelForSequenceClassification": "model_cbert.BertSentiment"
+  },
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "hyperparams": {
+    "batch_size": 32,
+    "lr_bert": 1.3381477872420105e-05,
+    "lr_dense": 1.1619234627185892e-05,
+    "max_length": 512,
+    "model_name": "Transformer",
+    "n_epochs": 50,
+    "num_labels": 3,
+    "w_decay": 0.15190379301303872,
+    "warmup": 0.017012007455465432
+  },
+  "id2label": {
+    "0": "Neutral",
+    "1": "Hawkish",
+    "2": "Dovish"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "negative": 2,
+    "neutral": 0,
+    "positive": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.17.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30873
+}

configuration_cbert.py

@@ -0,0 +1,59 @@
+import torch
+import torch.nn as nn
+
+import numpy as np
+import numpy as np
+import pandas as pd
+
+import torch.nn.functional as F
+
+from transformers import PretrainedConfig
+import torch.optim as optim
+
+class BertCustomConfig(PretrainedConfig):
+    model_type = "bert"
+
+    def __init__(
+            self,
+            vocab_size=30873,
+            hidden_size=768,
+            num_hidden_layers=12,
+            num_attention_heads=12,
+            intermediate_size=3072,
+            hidden_act="gelu",
+            hidden_dropout_prob=0.1,
+            attention_probs_dropout_prob=0.1,
+            max_position_embeddings=512,
+            type_vocab_size=2,
+            initializer_range=0.02,
+            layer_norm_eps=1e-12,
+            pad_token_id=0,
+            position_embedding_type="absolute",
+            use_cache=True,
+            classifier_dropout=None,
+            max_length=512,
+            id2label={"0": "Neutral", "1": "Hawkish", "2": "Dovish"},
+            label2id={"positive": 1, "negative": 2, "neutral": 0},
+            hyperparams=None,
+            **kwargs
+    ):
+        super().__init__(pad_token_id=pad_token_id, **kwargs)
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.position_embedding_type = position_embedding_type
+        self.use_cache = use_cache
+        self.classifier_dropout = classifier_dropout
+        self.max_length = max_length
+        self.id2label = id2label
+        self.label2id = label2id
+        self.hyperparams = hyperparams

model_cbert.py

@@ -0,0 +1,49 @@
+import torch
+import torch.nn as nn
+
+import random
+import numpy as np
+
+import numpy as np
+
+import pandas as pd
+
+import torch.nn.functional as F
+
+from transformers import BertModel, PreTrainedModel
+from configuration_cbert import BertCustomConfig
+
+import torch.optim as optim
+
+class BertSentiment(PreTrainedModel):
+    config_class = BertCustomConfig
+    def __init__(self, config, weight_path=None):
+        super().__init__(config)
+        self.config = config
+        self.num_labels = self.config.hyperparams["num_labels"]
+        # self.bert = BertModel.from_pretrained('yiyanghkust/finbert-tone')
+        if weight_path:
+            self.bert = BertModel.from_pretrained(weight_path)
+        else:
+            self.bert = BertModel(self.config)
+
+        self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
+        self.hidden = nn.Linear(self.config.hidden_size, self.config.hidden_size)
+        self.classifier = nn.Linear(self.config.hidden_size, self.config.hyperparams["num_labels"])
+        # self.classifier2 = nn.Linear(dense_size + meta_size, num_labels)
+        nn.init.xavier_normal_(self.hidden.weight)
+        nn.init.xavier_normal_(self.classifier.weight)
+
+    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, graphEmbeddings=None):
+        # _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, return_dict=False)
+        output, ctoken = self.bert(input_ids, token_type_ids, attention_mask, return_dict=False)
+        pooled_output = torch.mean(output, 1)
+        pooled_output = self.hidden(pooled_output)
+        pooled_output = self.dropout(pooled_output)
+        pooled_output = F.relu(pooled_output)
+        logits = self.classifier(pooled_output)
+        # dense1 = self.classifier(pooled_output)
+        # concatl = torch.cat((dense1, meta_data.float()), 1)
+        # logits = self.classifier2(concatl)
+
+        return logits

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2d9417dfaf5df54bb639af002b149b1eef7986cf9ed2309778907a78decc6c1
+size 441461673