Upload DisamBert

DisamBert.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from enum import StrEnum
 from itertools import chain
 
+import numpy as np
 import pandas as pd
 import torch
 import torch.nn as nn
@@ -91,7 +92,8 @@ class DisamBert(PreTrainedModel):
             self.classifier_head = nn.Parameter(torch.cat(vectors, dim=0))
             self.bias = nn.Parameter(
                 torch.nn.init.normal_(
-                    torch.empty((self.config.ontology_size, 1)), std=self.classifier_head.std().item()
+                    torch.empty((self.config.ontology_size, 1)), 
+                    std=self.classifier_head.std().item() * np.sqrt(self.config.hidden_size)
                 )
             )
 

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cad83795b87cb440ec0f169e2264c4b5072c8b762307706c2c04db26fbced65
+oid sha256:ba02b2a39f89f4b20322e713d0882548617ed5657678db44d8ef6af92839f9f0
 size 957993808