Update modeling_gptbert.py

modeling_gptbert.py

@@ -688,7 +688,17 @@ class GptBertPreTrainedModel(PreTrainedModel):
     _supports_flex_attn = False
 
     def _init_weights(self, module):
-        pass
+        std = math.sqrt(2.0 / (5.0 * self.hidden_size))
+
+        if isinstance(module, nn.Linear):
+            nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
 
 
 class GptBertModel(GptBertPreTrainedModel):
@@ -879,17 +889,11 @@ class Classifier(nn.Module):
         self.emb2vocab.bias.zero_()
 
     def forward(self, x: torch.Tensor):
-        print(x)
         x = self.pre_norm(x)
-        print(x)
         x = self.dropout(x)
-        print(x)
         x = self.projection(x)
-        print(x)
         x = gelu_new(x)
-        print(x)
         x = self.post_norm(x)
-        print(x)
         return self.emb2vocab(x)
 
 
@@ -1043,7 +1047,6 @@ class GptBertForSequenceClassification(GptBertModel):
 
         sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
         logits = self.head(sequence_output[:, 0, :])
-        print(logits)
 
         loss = None
         if labels is not None: