Upload configuration_eve.py with huggingface_hub

configuration_eve.py

@@ -0,0 +1,37 @@
+
+from transformers import PretrainedConfig
+
+class EveConfig(PretrainedConfig):
+    model_type = "eve_moe"
+    
+    def __init__(
+        self,
+        vocab_size=50304,
+        n_layer=12,
+        n_embd=512,
+        n_head=8,
+        head_dim=64,
+        block_size=2048,
+        num_experts=8,
+        top_k=2,
+        expert_intermediate_size=1408,
+        shared_expert_intermediate_size=1408,
+        router_aux_loss_coef=0.01,
+        use_checkpointing=False,
+        rope_theta=10000.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.n_layer = n_layer
+        self.n_embd = n_embd
+        self.n_head = n_head
+        self.head_dim = head_dim
+        self.block_size = block_size
+        self.num_experts = num_experts
+        self.top_k = top_k
+        self.expert_intermediate_size = expert_intermediate_size
+        self.shared_expert_intermediate_size = shared_expert_intermediate_size
+        self.router_aux_loss_coef = router_aux_loss_coef
+        self.use_checkpointing = use_checkpointing
+        self.rope_theta = rope_theta
+        super().__init__(**kwargs)