Upload configuration_openthaiwilai.py with huggingface_hub

configuration_openthaiwilai.py

@@ -1,5 +1,5 @@
-from transformers import PretrainedConfig
 
+from transformers import PretrainedConfig
 
 class OpenThaiWilaiConfig(PretrainedConfig):
     model_type = "OpenThaiWilai"
@@ -7,7 +7,8 @@ class OpenThaiWilaiConfig(PretrainedConfig):
     attribute_map = {
         "num_experts": "num_experts",
         "top_k": "top_k",
-        "num_hidden_layers": "num_layers"
+        "num_hidden_layers": "num_layers",
+        "num_key_value_heads": "num_key_value_heads" # Added this line
     }
 
     def __init__(
@@ -16,10 +17,13 @@ class OpenThaiWilaiConfig(PretrainedConfig):
         hidden_size=768,
         num_layers=6,
         num_heads=8,
+        num_key_value_heads=None, # Added this line
         num_experts=4,
         top_k=2,
-        max_position_embeddings=512,
+        max_position_embeddings=2048, # Updated max_position_embeddings to match modeling file
         intermediate_size=3072,
+        rope=True, # Added rope and use_flashattn to match modeling file
+        use_flashattn=True, # Added rope and use_flashattn to match modeling file
         eos_token_id=None,
         bos_token_id=None,
         pad_token_id=None,
@@ -41,10 +45,13 @@ class OpenThaiWilaiConfig(PretrainedConfig):
         self.hidden_size = hidden_size
         self.num_layers = num_layers
         self.num_heads = num_heads
+        self.num_key_value_heads = num_key_value_heads or num_heads # Added this line
         self.num_experts = num_experts
         self.top_k = top_k
         self.max_position_embeddings = max_position_embeddings
         self.intermediate_size = intermediate_size
+        self.rope = rope # Added rope and use_flashattn
+        self.use_flashattn = use_flashattn # Added rope and use_flashattn
 
         self.num_hidden_layers = num_layers
 
@@ -53,4 +60,8 @@ class OpenThaiWilaiConfig(PretrainedConfig):
         output["num_experts"] = self.num_experts
         output["top_k"] = self.top_k
         output["num_hidden_layers"] = self.num_layers
-        return output
+        output["num_key_value_heads"] = self.num_key_value_heads # Added this line
+        output["max_position_embeddings"] = self.max_position_embeddings # Ensure this is included
+        output["rope"] = self.rope # Ensure this is included
+        output["use_flashattn"] = self.use_flashattn # Ensure this is included
+        return output