Upload DisamBert

DisamBert.py

@@ -47,9 +47,9 @@ class DisamBert(PreTrainedModel):
         else:
             self.BaseModel = ModernBertModel(config)
             self.classifier_head = nn.Parameter(
-                torch.empty((config.vocab_size, config.hidden_size))
+                torch.empty((config.ontology_size, config.hidden_size))
             )
-            self.bias = nn.Parameter(torch.empty((config.vocab_size, 1)))
+            self.bias = nn.Parameter(torch.empty((config.ontology_size, 1)))
             self.__entities = pd.Series(config.entities)
         config.init_basemodel = False
         self.tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_path)
@@ -87,11 +87,11 @@ class DisamBert(PreTrainedModel):
 
             self.__entities = pd.Series(entity_ids)
             self.config.entities = entity_ids
-            self.config.vocab_size = len(entity_ids)
+            self.config.ontology_size = len(entity_ids)
             self.classifier_head = nn.Parameter(torch.cat(vectors, dim=0))
             self.bias = nn.Parameter(
                 torch.nn.init.normal_(
-                    torch.empty((self.config.vocab_size, 1)), std=self.classifier_head.std().item()
+                    torch.empty((self.config.ontology_size, 1)), std=self.classifier_head.std().item()
                 )
             )
 
@@ -183,7 +183,7 @@ class DisamBert(PreTrainedModel):
                 torch.cat(
                     [
                         sentence,
-                        torch.zeros((self.__entities.shape[0], maxlength - length)),
+                        torch.zeros((self.config.ontology_size, maxlength - length)),
                     ],
                     dim=1,
                 )

config.json

@@ -117722,6 +117722,7 @@
   "norm_eps": 1e-05,
   "num_attention_heads": 12,
   "num_hidden_layers": 22,
+  "ontology_size": 117660,
   "pad_token_id": 50283,
   "position_embedding_type": "absolute",
   "repad_logits_with_grad": false,
@@ -117742,5 +117743,5 @@
   "tokenizer_path": "answerdotai/ModernBERT-base",
   "transformers_version": "5.0.0",
   "use_cache": false,
-  "vocab_size": 117660
+  "vocab_size": 50368
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff4e9bebae857919d9ca236d04b7bb8aae63f405f9cd624bc7ee5ac59f2bd54f
+oid sha256:3cad83795b87cb440ec0f169e2264c4b5072c8b762307706c2c04db26fbced65
 size 957993808