Update configuration_internlm.py

configuration_internlm.py

@@ -19,9 +19,8 @@
 # limitations under the License.
 """ InternLM model configuration"""
 
-from transformers.utils import logging
 from transformers.configuration_utils import PretrainedConfig
-
+from transformers.utils import logging
 
 logger = logging.get_logger(__name__)
 
@@ -30,14 +29,11 @@ INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
 
 class InternLMConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate an InternLM
-    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
-    defaults will yield a similar configuration to that of the InternLM-7B.
-
+    This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate
+    an InternLM model according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the InternLM-7B.
     Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
     documentation from [`PretrainedConfig`] for more information.
-
-
     Args:
         vocab_size (`int`, *optional*, defaults to 32000):
             Vocabulary size of the InternLM model. Defines the number of different tokens that can be represented by the
@@ -65,23 +61,19 @@ class InternLMConfig(PretrainedConfig):
         tie_word_embeddings(`bool`, *optional*, defaults to `False`):
             Whether to tie weight embeddings
         Example:
-
     ```python
     >>> from transformers import InternLMModel, InternLMConfig
-
     >>> # Initializing a InternLM internlm-7b style configuration
     >>> configuration = InternLMConfig()
-
     >>> # Initializing a model from the internlm-7b style configuration
     >>> model = InternLMModel(configuration)
-
     >>> # Accessing the model configuration
     >>> configuration = model.config
     ```"""
     model_type = "internlm"
     _auto_class = "AutoConfig"
 
-    def __init__(
+    def __init__(  # pylint: disable=W0102
         self,
         vocab_size=103168,
         hidden_size=4096,
@@ -98,6 +90,7 @@ class InternLMConfig(PretrainedConfig):
         eos_token_id=2,
         tie_word_embeddings=False,
         bias=True,
+        rotary={"base": 10000, "type": "dynamic"},  # pylint: disable=W0102
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -111,10 +104,11 @@ class InternLMConfig(PretrainedConfig):
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache
         self.bias = bias
+        self.rotary = rotary
         super().__init__(
             pad_token_id=pad_token_id,
             bos_token_id=bos_token_id,
             eos_token_id=eos_token_id,
             tie_word_embeddings=tie_word_embeddings,
             **kwargs,
-        )
+        )