initial upload

config.json

@@ -22,5 +22,6 @@
     "num_hidden_layers": 12,
     "position_bucket_size": 32,
     "torch_dtype": "float32",
-    "vocab_size": 8192
+    "vocab_size": 16384,
+    "temperature": 2.5
 }

configuration_ltgbert.py

@@ -17,6 +17,7 @@ class LtgbertConfig(PretrainedConfig):
         num_hidden_layers=12,
         layer_norm_eps=1.0e-7,
         output_all_encoded_layers=True,
+        temperature=1.0,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -32,3 +33,4 @@ class LtgbertConfig(PretrainedConfig):
         self.output_all_encoded_layers = output_all_encoded_layers
         self.position_bucket_size = position_bucket_size
         self.layer_norm_eps = layer_norm_eps
+        self.temperature = temperature

modeling_ltgbert.py

@@ -443,6 +443,7 @@ class Classifier(nn.Module):
     def __init__(self, config, num_labels: int):
         super().__init__()
 
+        self.temperature = config.temperature
         drop_out = getattr(config, "cls_dropout", None)
         drop_out = config.hidden_dropout_prob if drop_out is None else drop_out
 
@@ -456,7 +457,7 @@ class Classifier(nn.Module):
         )
 
     def forward(self, x):
-        x = self.nonlinearity(x)
+        x = self.nonlinearity(x) / self.temperature
         return x
 
 

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e1e5fbe6e83f2268d4f30334df9e64bbb4cc5a2e2986d60594e2c9f27cbf60a
-size 130639876
+oid sha256:e5aaa5fba47a98d324a6af79ec8a03fe32c6abf1a483c0274075d5faa9e5d6a6
+size 503044925

spacial_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "␂", "eos_token": "␃", "unk_token": "␦", "sep_token": "␃", "pad_token": "␢", "cls_token": "␂", "mask_token": "␥"}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<oad>", "cls_token": "<s>", "mask_token": "<mask>"}

tokenizer_config.json

@@ -1,10 +1,10 @@
 {
     "tokenizer_class": "PreTrainedTokenizerFast",
-    "bos_token": "␂",
-    "eos_token": "␃",
-    "unk_token": "␦",
-    "sep_token": "␃",
-    "pad_token": "␢",
-    "cls_token": "␂",
-    "mask_token": "␥"
+    "bos_token": "<s>",
+    "eos_token": "</s>",
+    "unk_token": "<unk>",
+    "sep_token": "</s>",
+    "pad_token": "<pad>",
+    "cls_token": "<s>",
+    "mask_token": "<mask>"
 }