Upload configuration_gravity_moe.py with huggingface_hub

configuration_gravity_moe.py

@@ -11,95 +11,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""GravityMoE model configuration"""
+"""GravityMoE model configuration — inherits from DeepSeek V3."""
 
-try:
-    from transformers.configuration_utils import PreTrainedConfig
-except ImportError:
-    from transformers.configuration_utils import PretrainedConfig as PreTrainedConfig
+from transformers import DeepseekV3Config
 
 
-class GravityMoEConfig(PreTrainedConfig):
+class GravityMoEConfig(DeepseekV3Config):
     r"""
-    Configuration class for the GravityMoE model.
+    Configuration class for the GravityMoE model, inheriting from
+    [`DeepseekV3Config`]. GravityMoE shares the same architecture as
+    DeepSeek V3 (sparse MoE with MLA) but uses different hyperparameters.
 
-    Args:
-        vocab_size (`int`, *optional*, defaults to 151552):
-            Vocabulary size of the model.
-        hidden_size (`int`, *optional*, defaults to 2048):
-            Dimensionality of the hidden representations.
-        intermediate_size (`int`, *optional*, defaults to 8192):
-            Dimensionality of the dense MLP intermediate representations.
-        moe_intermediate_size (`int`, *optional*, defaults to 1408):
-            Dimensionality of the MoE expert intermediate representations.
-        num_hidden_layers (`int`, *optional*, defaults to 28):
-            Number of hidden layers in the model.
-        num_attention_heads (`int`, *optional*, defaults to 16):
-            Number of attention heads.
-        num_key_value_heads (`int`, *optional*, defaults to 16):
-            Number of key-value heads.
-        n_shared_experts (`int`, *optional*, defaults to 1):
-            Number of shared experts.
-        n_routed_experts (`int`, *optional*, defaults to 64):
-            Number of routed experts.
-        routed_scaling_factor (`float`, *optional*, defaults to 2.446):
-            Scaling factor for routed expert weights.
-        kv_lora_rank (`int`, *optional*, defaults to 512):
-            Rank of the low-rank KV projection.
-        q_lora_rank (`int`, *optional*, defaults to `None`):
-            Rank of the low-rank Q projection. If `None`, uses standard Q projection.
-        qk_rope_head_dim (`int`, *optional*, defaults to 64):
-            Dimensionality of the RoPE component of QK heads.
-        v_head_dim (`int`, *optional*, defaults to 128):
-            Dimensionality of value heads.
-        qk_nope_head_dim (`int`, *optional*, defaults to 128):
-            Dimensionality of the non-RoPE component of QK heads.
-        n_group (`int`, *optional*, defaults to 1):
-            Number of groups for routed experts.
-        topk_group (`int`, *optional*, defaults to 1):
-            Number of top-k groups.
-        num_experts_per_tok (`int`, *optional*, defaults to 8):
-            Number of experts activated per token.
-        first_k_dense_replace (`int`, *optional*, defaults to 1):
-            Number of initial dense layers before MoE layers.
-        norm_topk_prob (`bool`, *optional*, defaults to `True`):
-            Whether to normalize top-k probabilities.
-        hidden_act (`str`, *optional*, defaults to `"silu"`):
-            Activation function.
-        max_position_embeddings (`int`, *optional*, defaults to 65536):
-            Maximum sequence length.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            Standard deviation for weight initialization.
-        rms_norm_eps (`float`, *optional*, defaults to 1e-6):
-            Epsilon for RMS normalization.
-        use_cache (`bool`, *optional*, defaults to `True`):
-            Whether to use KV cache.
-        rope_theta (`float`, *optional*, defaults to 1000000.0):
-            Base frequency for RoPE.
-        rope_scaling (`dict`, *optional*):
-            RoPE scaling configuration.
-        rope_interleave (`bool`, *optional*, defaults to `True`):
-            Whether to interleave the rotary position embeddings.
-        attention_bias (`bool`, *optional*, defaults to `False`):
-            Whether to use bias in attention projections.
-        attention_dropout (`float`, *optional*, defaults to 0.0):
-            Dropout rate for attention weights.
+    Only default values that differ from DeepSeek V3 are overridden here.
+    See [`DeepseekV3Config`] for full documentation of all parameters.
 
     Example:
 
     ```python
     >>> from configuration_gravity_moe import GravityMoEConfig
 
-    >>> # Initializing a GravityMoE style configuration
     >>> configuration = GravityMoEConfig()
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
+    >>> configuration.model_type
+    'gravity_moe'
     ```
     """
 
     model_type = "gravity_moe"
-    keys_to_ignore_at_inference = ["past_key_values"]
 
     def __init__(
         self,
@@ -139,45 +76,41 @@ class GravityMoEConfig(PreTrainedConfig):
         attention_dropout=0.0,
         **kwargs,
     ):
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.moe_intermediate_size = moe_intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads if num_key_value_heads is not None else num_attention_heads
-        self.n_shared_experts = n_shared_experts
-        self.n_routed_experts = n_routed_experts
-        self.num_local_experts = n_routed_experts
-        self.routed_scaling_factor = routed_scaling_factor
-        self.kv_lora_rank = kv_lora_rank
-        self.q_lora_rank = q_lora_rank
-        self.qk_rope_head_dim = qk_rope_head_dim
-        self.v_head_dim = v_head_dim
-        self.qk_nope_head_dim = qk_nope_head_dim
-        self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
-        self.head_dim = qk_rope_head_dim
-        self.n_group = n_group
-        self.topk_group = topk_group
-        self.num_experts_per_tok = num_experts_per_tok
-        self.first_k_dense_replace = first_k_dense_replace
-        self.norm_topk_prob = norm_topk_prob
-        self.hidden_act = hidden_act
-        self.max_position_embeddings = max_position_embeddings
-        self.initializer_range = initializer_range
-        self.rms_norm_eps = rms_norm_eps
-        self.use_cache = use_cache
-        self.rope_theta = rope_theta
-        self.rope_scaling = rope_scaling
-        self.rope_interleave = rope_interleave
-        self.attention_bias = attention_bias
-        self.attention_dropout = attention_dropout
-
         super().__init__(
+            vocab_size=vocab_size,
+            hidden_size=hidden_size,
+            intermediate_size=intermediate_size,
+            moe_intermediate_size=moe_intermediate_size,
+            num_hidden_layers=num_hidden_layers,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            n_shared_experts=n_shared_experts,
+            n_routed_experts=n_routed_experts,
+            routed_scaling_factor=routed_scaling_factor,
+            kv_lora_rank=kv_lora_rank,
+            q_lora_rank=q_lora_rank,
+            qk_rope_head_dim=qk_rope_head_dim,
+            v_head_dim=v_head_dim,
+            qk_nope_head_dim=qk_nope_head_dim,
+            n_group=n_group,
+            topk_group=topk_group,
+            num_experts_per_tok=num_experts_per_tok,
+            first_k_dense_replace=first_k_dense_replace,
+            norm_topk_prob=norm_topk_prob,
+            hidden_act=hidden_act,
+            max_position_embeddings=max_position_embeddings,
+            initializer_range=initializer_range,
+            rms_norm_eps=rms_norm_eps,
+            use_cache=use_cache,
             pad_token_id=pad_token_id,
             bos_token_id=bos_token_id,
             eos_token_id=eos_token_id,
             tie_word_embeddings=tie_word_embeddings,
+            rope_theta=rope_theta,
+            rope_scaling=rope_scaling,
+            rope_interleave=rope_interleave,
+            attention_bias=attention_bias,
+            attention_dropout=attention_dropout,
             **kwargs,
         )