Create configuration_momo.py

configuration_momo.py

@@ -1,26 +1,47 @@
+# configuration_momo.py
+# 🌸 Momo-336M — HuggingFace compatible config
+# Upload this file to your HF repo alongside modeling_momo.py and config.json
+
 from transformers import PretrainedConfig
 
+
 class MomoConfig(PretrainedConfig):
     model_type = "momo"
+
     def __init__(
         self,
-        vocab_size=32000, hidden_size=1024, intermediate_size=2752,
-        num_hidden_layers=24, num_attention_heads=16, num_key_value_heads=8,
-        max_position_embeddings=512, rope_theta=10000.0, rms_norm_eps=1e-5,
+        vocab_size=32000,
+        hidden_size=1024,
+        intermediate_size=2752,
+        num_hidden_layers=24,
+        num_attention_heads=16,
+        num_key_value_heads=8,
+        max_position_embeddings=512,
+        rope_theta=10000.0,
+        rms_norm_eps=1e-5,
         use_gradient_checkpointing=False,
-        model_name='Momo-336M', model_version='1.0',
-        **kwargs
+        model_name="Momo-336M",
+        model_version="1.0",
+        pad_token_id=3,
+        bos_token_id=1,
+        eos_token_id=0,
+        **kwargs,
     ):
-        super().__init__(**kwargs)
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads
-        self.max_position_embeddings = max_position_embeddings
-        self.rope_theta = rope_theta
-        self.rms_norm_eps = rms_norm_eps
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs,
+        )
+        self.vocab_size               = vocab_size
+        self.hidden_size              = hidden_size
+        self.intermediate_size        = intermediate_size
+        self.num_hidden_layers        = num_hidden_layers
+        self.num_attention_heads      = num_attention_heads
+        self.num_key_value_heads      = num_key_value_heads
+        self.max_position_embeddings  = max_position_embeddings
+        self.rope_theta               = rope_theta
+        self.rms_norm_eps             = rms_norm_eps
         self.use_gradient_checkpointing = use_gradient_checkpointing
-        self.model_name = model_name
-        self.model_version = model_version
+        self.model_name               = model_name
+        self.model_version            = model_version