| from transformers import PretrainedConfig |
|
|
| class MoEGPTConfig(PretrainedConfig): |
| model_type = "moegpt" |
|
|
| def __init__( |
| self, |
| vocab_size=50304, |
| n_embd=1152, |
| n_layer=24, |
| n_head=16, |
| sequence_length=1024, |
| moe=False, |
| moe_routing="standard_gating", |
| moe_num_experts=4, |
| moe_num_experts_per_tok=2, |
| moe_softmax_order="softmax_topk", |
| moe_router_loss="load_balancing_z_loss", |
| moe_aux_loss_factor=0.01, |
| moe_z_loss_factor=1.0, |
| mlp_dim_exp_factor=1.0, |
| dropout=0.0, |
| bias=False, |
| architectures=["MoEGPTForCausalLM"], |
| auto_map={ |
| "AutoConfig": "configuration.MoEGPTConfig", |
| "AutoModelForCausalLM": "modeling.MoEGPTForCausalLM", |
| "AutoTokenizer": "GPT2TokenizerFast" |
| }, |
| **kwargs, |
| ): |
| super().__init__(**kwargs) |
| self.vocab_size = vocab_size |
| self.n_embd = n_embd |
| self.n_layer = n_layer |
| self.n_head = n_head |
| self.sequence_length = sequence_length |
| self.moe = moe |
| self.moe_routing = moe_routing |
| self.moe_num_experts = moe_num_experts |
| self.moe_num_experts_per_tok = moe_num_experts_per_tok |
| self.moe_softmax_order = moe_softmax_order |
| self.moe_router_loss = moe_router_loss |
| self.moe_aux_loss_factor = moe_aux_loss_factor |
| self.moe_z_loss_factor = moe_z_loss_factor |
| self.mlp_dim_exp_factor = mlp_dim_exp_factor |
| self.dropout = dropout |
| self.bias = bias |
| self.architectures = architectures |
| self.auto_map = auto_map |
| |
