Update configuration_Llamoe.py

configuration_Llamoe.py

@@ -13,25 +13,24 @@ LLAMOE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 class LlamoeConfig(PretrainedConfig):
     model_type = "Llamoe"
     keys_to_ignore_at_inference = ["past_key_values"]
-
     def __init__(
         self,
         vocab_size=32000,
-        hidden_size=3072,
-        intermediate_size=24576,
-        num_hidden_layers=28,
-        num_attention_heads=16,
-        num_key_value_heads=16,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
         head_dim=256,
-        hidden_act="gelu",
-        max_position_embeddings=8192,
+        hidden_act="silu",
+        max_position_embeddings=4096,
         initializer_range=0.02,
-        rms_norm_eps=1e-6,
+        rms_norm_eps=1e-05,
         use_cache=True,
         pad_token_id=0,
         eos_token_id=1,
         bos_token_id=2,
-        tie_word_embeddings=True,
+        tie_word_embeddings=false,
         rope_theta=10000.0,
         attention_bias=False,
         attention_dropout=0.0,