Create configuration_momo.py

configuration_momo.py

@@ -0,0 +1,26 @@
+from transformers import PretrainedConfig
+
+class MomoConfig(PretrainedConfig):
+    model_type = "momo"
+    def __init__(
+        self,
+        vocab_size=32000, hidden_size=1024, intermediate_size=2752,
+        num_hidden_layers=24, num_attention_heads=16, num_key_value_heads=8,
+        max_position_embeddings=512, rope_theta=10000.0, rms_norm_eps=1e-5,
+        use_gradient_checkpointing=False,
+        model_name='Momo-336M', model_version='1.0',
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.max_position_embeddings = max_position_embeddings
+        self.rope_theta = rope_theta
+        self.rms_norm_eps = rms_norm_eps
+        self.use_gradient_checkpointing = use_gradient_checkpointing
+        self.model_name = model_name
+        self.model_version = model_version