Upload configuration_openpangu_dense.py with huggingface_hub

configuration_openpangu_dense.py

@@ -0,0 +1,92 @@
+# coding=utf-8
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+
+from transformers.utils import logging
+from transformers.configuration_utils import PretrainedConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class PanguEmbeddedConfig(PretrainedConfig):
+    model_type = "pangu_embedded"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=153376,
+        hidden_size=4096,
+        intermediate_size=16384,
+        num_hidden_layers=28,
+        num_attention_heads=32,
+        num_key_value_heads=4,
+        head_dim=128,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        rope_theta=16000000.0,
+        sliding_window=127,
+        attention_dropout=0.0,
+        bias=True,
+        layer_types=None,
+        param_sink_number=128,
+        attn_groupnorm=True,
+        attn_elementwise_gate=True,
+        router_sliding_window=3,
+        router_win_decay=0.5,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.head_dim = head_dim
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.sliding_window = sliding_window
+        self.attention_dropout = attention_dropout
+        self.bias = bias
+
+        # Custom arguments not standard in most HF models
+        self.param_sink_number = param_sink_number
+        self.attn_groupnorm = attn_groupnorm
+        self.attn_elementwise_gate = attn_elementwise_gate
+        self.router_sliding_window = router_sliding_window
+        self.router_win_decay = router_win_decay
+
+        if layer_types is None:
+            # Default layer types based on Megatron's swa_layers: 1,3,5,...,27
+            # In 0-based indexing, this corresponds to layers 0, 2, 4, ..., 26
+            swa_hf_layers = {i for i in range(0, num_hidden_layers, 2)}
+            self.layer_types = [
+                "sliding_attention" if i in swa_hf_layers else "full_attention"
+                for i in range(num_hidden_layers)
+            ]
+        else:
+            self.layer_types = layer_types
+        
+        if len(self.layer_types) != self.num_hidden_layers:
+            raise ValueError(
+                f"`layer_types` must have a length equal to `num_hidden_layers` ({self.num_hidden_layers}), "
+                f"but has length {len(self.layer_types)}."
+            )
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )