Create configuration_openthaiwilai.py

configuration_openthaiwilai.py

@@ -0,0 +1,56 @@
+from transformers import PretrainedConfig
+
+
+class OpenThaiWilaiConfig(PretrainedConfig):
+    model_type = "OpenThaiWilai"
+
+    attribute_map = {
+        "num_experts": "num_experts",
+        "top_k": "top_k",
+        "num_hidden_layers": "num_layers"
+    }
+
+    def __init__(
+        self,
+        vocab_size=50000,
+        hidden_size=768,
+        num_layers=6,
+        num_heads=8,
+        num_experts=4,
+        top_k=2,
+        max_position_embeddings=512,
+        intermediate_size=3072,
+        eos_token_id=None,
+        bos_token_id=None,
+        pad_token_id=None,
+        **kwargs
+    ):
+        if top_k > num_experts:
+            raise ValueError(
+                f"top_k ({top_k}) cannot be greater than num_experts ({num_experts})"
+            )
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs
+        )
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.num_experts = num_experts
+        self.top_k = top_k
+        self.max_position_embeddings = max_position_embeddings
+        self.intermediate_size = intermediate_size
+
+        self.num_hidden_layers = num_layers
+
+    def to_dict(self):
+        output = super().to_dict()
+        output["num_experts"] = self.num_experts
+        output["top_k"] = self.top_k
+        output["num_hidden_layers"] = self.num_layers
+        return output