| |
| |
| |
| |
|
|
| """Configuration for the MiniMax M2 architecture.""" |
|
|
| from __future__ import annotations |
|
|
| from typing import List, Optional, Union |
|
|
| from transformers.configuration_utils import PretrainedConfig |
|
|
|
|
| class MiniMaxM2Config(PretrainedConfig): |
| model_type = "minimax" |
|
|
| def __init__( |
| self, |
| vocab_size: int = 200_064, |
| hidden_size: int = 3_072, |
| intermediate_size: int = 1_536, |
| mlp_intermediate_size: int = 8_192, |
| num_hidden_layers: int = 62, |
| num_attention_heads: int = 48, |
| num_key_value_heads: int = 8, |
| head_dim: Optional[int] = 128, |
| num_local_experts: int = 256, |
| num_experts_per_tok: int = 8, |
| attn_type_list: Optional[List[int]] = None, |
| attention_dropout: float = 0.0, |
| hidden_act: str = "silu", |
| rms_norm_eps: float = 1e-6, |
| max_position_embeddings: int = 196_608, |
| rope_theta: float = 5_000_000.0, |
| rotary_dim: int = 64, |
| rope_scaling: Optional[dict] = None, |
| use_qk_norm: bool = True, |
| qk_norm_type: str = "per_layer", |
| use_routing_bias: bool = True, |
| scoring_func: str = "sigmoid", |
| router_aux_loss_coef: float = 0.001, |
| router_jitter_noise: float = 0.0, |
| output_router_logits: bool = False, |
| use_grouped_topk: bool = True, |
| num_expert_group: Optional[int] = None, |
| topk_group: Optional[int] = None, |
| routed_scaling_factor: float = 1.0, |
| layernorm_full_attention_beta: float = 1.0, |
| layernorm_linear_attention_beta: float = 1.0, |
| layernorm_mlp_beta: float = 1.0, |
| shared_intermediate_size: int = 0, |
| shared_moe_mode: str = "sigmoid", |
| use_mtp: bool = True, |
| num_mtp_modules: int = 3, |
| mtp_transformer_layers: int = 1, |
| attn_window_size: Optional[Union[int, List[int]]] = None, |
| swa_rope_theta: float = -1.0, |
| sliding_window: Optional[int] = None, |
| initializer_range: float = 0.02, |
| tie_word_embeddings: bool = False, |
| max_model_len: Optional[int] = None, |
| bos_token_id: Optional[int] = None, |
| eos_token_id: Optional[int] = None, |
| pad_token_id: Optional[int] = None, |
| use_cache: bool = True, |
| **kwargs, |
| ) -> None: |
| quantization_config = kwargs.pop("quantization_config", None) |
| transformers_version = kwargs.pop("transformers_version", None) |
|
|
| super().__init__( |
| bos_token_id=bos_token_id, |
| eos_token_id=eos_token_id, |
| tie_word_embeddings=tie_word_embeddings, |
| pad_token_id=pad_token_id, |
| **kwargs, |
| ) |
|
|
| self.vocab_size = vocab_size |
| self.hidden_size = hidden_size |
| self.intermediate_size = intermediate_size |
| self.mlp_intermediate_size = mlp_intermediate_size |
| self.num_hidden_layers = num_hidden_layers |
| self.num_attention_heads = num_attention_heads |
| self.num_key_value_heads = num_key_value_heads |
| self.head_dim = head_dim or hidden_size // num_attention_heads |
| self.num_local_experts = num_local_experts |
| self.num_experts_per_tok = num_experts_per_tok |
| self.attn_type_list = attn_type_list or [1] * num_hidden_layers |
| self.attention_dropout = attention_dropout |
| self.hidden_act = hidden_act |
| self.rms_norm_eps = rms_norm_eps |
| self.max_position_embeddings = max_position_embeddings |
| self.rope_theta = rope_theta |
| self.rotary_dim = rotary_dim |
| self.rope_scaling = rope_scaling |
| self.use_qk_norm = use_qk_norm |
| self.qk_norm_type = qk_norm_type |
| self.use_routing_bias = use_routing_bias |
| self.scoring_func = scoring_func |
| self.router_aux_loss_coef = router_aux_loss_coef |
| self.router_jitter_noise = router_jitter_noise |
| self.output_router_logits = output_router_logits |
| self.use_grouped_topk = use_grouped_topk |
| self.num_expert_group = num_expert_group |
| self.topk_group = topk_group |
| self.routed_scaling_factor = routed_scaling_factor |
| self.layernorm_full_attention_beta = layernorm_full_attention_beta |
| self.layernorm_linear_attention_beta = layernorm_linear_attention_beta |
| self.layernorm_mlp_beta = layernorm_mlp_beta |
| self.shared_intermediate_size = shared_intermediate_size |
| self.shared_moe_mode = shared_moe_mode |
| self.use_mtp = use_mtp |
| self.num_mtp_modules = num_mtp_modules |
| self.mtp_transformer_layers = mtp_transformer_layers |
| self.attn_window_size = attn_window_size |
| self.swa_rope_theta = swa_rope_theta |
| self.sliding_window = sliding_window |
| self.initializer_range = initializer_range |
| self.max_model_len = max_model_len |
| self.use_cache = use_cache |
|
|
| |
| self.partial_rotary_factor = float(self.rotary_dim) / float(self.head_dim) |
| if quantization_config is not None: |
| self.quantization_config = quantization_config |
| self.transformers_version = transformers_version |
|
|
|
|
| __all__ = ["MiniMaxM2Config"] |
|
|
