Upload modeling_modern_llm.py with huggingface_hub

modeling_modern_llm.py

@@ -107,6 +107,15 @@ class ModernLLMForCausalLM(PreTrainedModel):
         self.rotary_emb = RotaryEmbedding(config.hidden_size // config.num_attention_heads, config.max_position_embeddings, config.rope_theta)
         self.post_init()
 
+    def _init_weights(self, module):
+        std = 0.02
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+
     def forward(self, input_ids, labels=None, **kwargs):
         _, seq_len = input_ids.shape
         x = self.embed_tokens(input_ids)