Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "<EMAIL>": 110521,
+  "<KEY>": 110522,
+  "<NAME>": 110520,
+  "<PASSWORD>": 110523,
+  "<code_to_intermediate>": 110502,
+  "<empty_output>": 110501,
+  "<file_sep>": 110492,
+  "<intermediate_to_code>": 110503,
+  "<issue_closed>": 110495,
+  "<issue_comment>": 110494,
+  "<issue_start>": 110493,
+  "<jupyter_code>": 110498,
+  "<jupyter_output>": 110499,
+  "<jupyter_script>": 110500,
+  "<jupyter_start>": 110496,
+  "<jupyter_text>": 110497,
+  "<pr>": 110504,
+  "<pr_base>": 110507,
+  "<pr_base_code>": 110509,
+  "<pr_comment>": 110512,
+  "<pr_diff>": 110510,
+  "<pr_diff_hunk>": 110511,
+  "<pr_diff_hunk_comment_line>": 110519,
+  "<pr_event_id>": 110513,
+  "<pr_file>": 110508,
+  "<pr_in_reply_to_comment_id>": 110518,
+  "<pr_in_reply_to_review_id>": 110517,
+  "<pr_is_merged>": 110506,
+  "<pr_review>": 110514,
+  "<pr_review_comment>": 110516,
+  "<pr_review_state>": 110515,
+  "<pr_status>": 110505,
+  "<repo_name>": 110491
+}

chat_template.jinja

@@ -0,0 +1,100 @@
+{% if tools is not defined or tools is none %}
+    {{- '<|im_start|>tool_list\n<|im_end|>\n' }}
+{%- else %}
+    {{- '<|im_start|>tool_list\n[' }}
+    {%- for tool in tools %}
+        {{- '{"name": "' }}
+        {{- tool.function.name }}
+        {{- '", ' }}
+        {{- '"description": "' }}
+        {{- tool.function.description }}
+        {{- '"' }}
+        {%- if tool.function.parameters is defined %}
+            {{- ', "parameters": ' }}
+            {{- tool.function.parameters | tojson }}
+        {%- endif %}
+        {{- '}' }}
+        {%- if not loop.last %}
+            {{- ', ' }}
+        {%- endif %}
+    {%- endfor %}
+{{- ']<|im_end|>\n' }}
+{%- endif %}
+
+{%- set ns = namespace(is_searching=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.is_searching and (message.role == 'user' or message.role == 'tool') %}
+        {%- set ns.last_query_index = index %}
+        {%- set ns.is_searching = false %}
+    {%- endif %}
+{%- endfor %}
+
+{%- for message in messages %}
+    {%- if loop.index0 == 0 and message.role != 'system' %}
+        {{- '<|im_start|>system\n<|im_end|>\n' }}
+    {%- endif %}
+
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+
+    {%- set reasoning_content = '' %}
+    {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+        {%- set reasoning_content = message.reasoning_content %}    
+    {%- endif %}
+    {%- if message.role == "assistant" %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if reasoning_content %}
+                {{- '<|im_start|>assistant/think\n' + reasoning_content.strip('\n') + '<|im_end|>\n' }}
+            {%- endif %}
+        {%- endif %}
+
+        {%- if content %}
+            {{- '<|im_start|>assistant\n' + content.strip('\n') + '<|im_end|>' }}
+            {%- if message.tool_calls %}
+                {{- '\n' }}
+            {%- else %}
+                {{- '<|endofturn|>\n' }}
+            {%- endif %}
+        {%- endif %}
+
+        {%- if message.tool_calls %}
+            {{- '<|im_start|>assistant -> tool/function_call\n[' }}
+            {%- for tool_call in message.tool_calls %}
+                {%- if not loop.first %}
+                    {{- ', ' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}' }}
+                {%- endfor %}
+            {{- ']<|im_end|><|stop|>\n' }}
+
+        {%- endif %}
+    {%- elif message.role == "tool" %}
+        {{- '<|im_start|>tool/function_call\n' + content + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {%- if force_reasoning is defined and force_reasoning is true %}
+        {{- '<|im_start|>assistant/think\n' }}
+    {%- elif skip_reasoning is defined and skip_reasoning is true %}
+        {{- '<|im_start|>assistant\n' }}
+    {%- else %}
+        {{- '<|im_start|>assistant' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,69 @@
+{
+  "architectures": [
+    "HyperCLOVAXForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_multiplier": 0.0078125,
+  "attn_pdrop": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_hyperclovax.HyperCLOVAXConfig",
+    "AutoModel": "modeling_hyperclovax.HyperCLOVAXModel",
+    "AutoModelForCausalLM": "modeling_hyperclovax.HyperCLOVAXForCausalLM"
+  },
+  "bos_token_id": 100257,
+  "embd_pdrop": 0.0,
+  "embedding_multiplier": 10.0,
+  "end_token_id": 100257,
+  "eos_token_id": 100257,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.012727922061357854,
+  "intermediate_size": 14336,
+  "logits_scaling": 0.125,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "hyperclovax",
+  "num_attention_heads": 48,
+  "num_hidden_layers": 38,
+  "num_key_value_heads": 8,
+  "pad_token_id": 100257,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "group_size": 128,
+    "hyb_act": false,
+    "lm_head": false,
+    "meta": {
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "mse": 0.0,
+      "quantizer": [
+        "gptqmodel:4.0.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "v2": false,
+      "v2_alpha": 0.25
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "resid_pdrop": 0.0,
+  "residual_multiplier": 1.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 100000000,
+  "summary_first_dropout": 0.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": false,
+  "use_post_norm": true,
+  "vocab_size": 110592
+}

configuration_hyperclovax.py

@@ -0,0 +1,235 @@
+# coding=utf-8
+# This file was created for the HyperCLOVA X SEED 14B Think architecture.
+# partially copied and modified from https://github.com/huggingface/transformers
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""HyperCLOVAX model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+
+class HyperCLOVAXConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`HyperCLOVAXModel`]. It is used to instantiate an HyperCLOVAX
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the HyperCLOVAX.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the HyperCLOVAX model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`HyperCLOVAXModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        pretraining_tp (`int`, *optional*, defaults to 1):
+            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+            document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to
+            understand more about it. This value is necessary to ensure exact reproducibility of the pretraining
+            results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232).
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+            accordingly.
+            Expected contents:
+                `rope_type` (`str`):
+                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+                    'llama3'], with 'default' being the original RoPE implementation.
+                `factor` (`float`, *optional*):
+                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+                    original maximum pre-trained length.
+                `original_max_position_embeddings` (`int`, *optional*):
+                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+                    pretraining.
+                `attention_factor` (`float`, *optional*):
+                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+                    computation. If unspecified, it defaults to value recommended by the implementation, using the
+                    `factor` field to infer the suggested value.
+                `beta_fast` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 32.
+                `beta_slow` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 1.
+                `short_factor` (`List[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `long_factor` (`List[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `low_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+                `high_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+        attention_bias (`bool`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        mlp_bias (`bool`, *optional*, defaults to `False`):
+            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
+        head_dim (`int`, *optional*):
+            The attention head dimension. If None, it will default to hidden_size // num_heads
+        embedding_multiplier (`float, *optional*, defaults to `None`):
+            Multiplier applied to the embedding weights. If `None`, it is equivalent to `1.0`.
+        logits_scaling (`float, *optional*, defaults to `None`):
+            Scaling factor for logits. If `None`, it is equivalent to `1.0`.
+        attention_multiplier (`float, *optional*, defaults to `None`):
+            Multiplier applied to the attention weights. If `None`, it is equivalent to `self.head_dim ** -0.5`.
+        residual_multiplier (`float, *optional*, defaults to `None`):
+            Scaling factor for residual connections. If `None`, it is equivalent to `1.0`.
+        use_post_norm (`bool`, *optional*, defaults to `False`):
+            Determines whether to apply Peri-Layer Normalization. Set to True to enable this feature.
+
+    ```python
+    >>> from transformers import HyperCLOVAXModel, HyperCLOVAXConfig
+
+    >>> # Initializing a HyperCLOVAX HyperCLOVAX style configuration
+    >>> configuration = HyperCLOVAXConfig()
+
+    >>> # Initializing a model from the HyperCLOVAX style configuration
+    >>> model = HyperCLOVAXModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "hyperclovax"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        mlp_bias=False,
+        head_dim=None,
+        embedding_multiplier=None, # MuP
+        logits_scaling=None, # MuP
+        attention_multiplier=None, # MuP
+        residual_multiplier=None, # MuP
+        use_post_norm=False, # Peri-LN (post-norm)
+        auto_map={
+            "AutoConfig": "configuration_hyperclovax.HyperCLOVAXConfig",
+            "AutoModel": "modeling_hyperclovax.HyperCLOVAXModel",
+            "AutoModelForCausalLM": "modeling_hyperclovax.HyperCLOVAXForCausalLM"
+        },
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        self.mlp_bias = mlp_bias
+        self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads
+        # Validate the correctness of rotary position embeddings parameters
+        # BC: if there is a 'type' field, copy it it to 'rope_type'.
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        # rope_config_validation(self)
+
+        # MuP
+        self.embedding_multiplier = embedding_multiplier if embedding_multiplier is not None else 1.0
+        self.logits_scaling = logits_scaling if logits_scaling is not None else 1.0
+        self.attention_multiplier = attention_multiplier if attention_multiplier is not None else self.head_dim ** -0.5
+        self.residual_multiplier = residual_multiplier if residual_multiplier is not None else 1.0
+
+        # Peri-LN (post-norm)
+        self.use_post_norm = use_post_norm
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            auto_map=auto_map,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": 100257,
+  "pad_token_id": 100257,
+  "transformers_version": "4.52.4",
+  "use_cache": false
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:339fd315e55f86378c9f9fef6c828176af3ab4fc63ef245f0d422e7b54f39c3e
+size 3971009288

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111860a296339fbcb5d321a7e5fa90b16802ba6c81344edadac847682a26883a
+size 3986730616

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8774d840eb0385397fe18e993a36c5341a4aa413c10fe9d9aa2c4c532ed72db
+size 1725566128

modeling_hyperclovax.py

@@ -0,0 +1,979 @@
+# coding=utf-8
+# This file was created for the HyperCLOVA X SEED 14B Think architecture.
+# partially copied and modified from https://github.com/huggingface/transformers
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Callable, Optional, Union
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.generation import GenerationMixin
+from transformers.modeling_attn_mask_utils import AttentionMaskConverter
+from transformers.integrations import use_kernel_forward_from_hub
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutputWithPast,
+    TokenClassifierOutput,
+)
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
+from transformers.utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging
+from .configuration_hyperclovax import HyperCLOVAXConfig
+if is_torch_flex_attn_available():
+    from torch.nn.attention.flex_attention import BlockMask
+
+    from transformers.integrations.flex_attention import make_flex_block_causal_mask
+
+logger = logging.get_logger(__name__)
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class HyperCLOVAXRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        HyperCLOVAXRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+ALL_LAYERNORM_LAYERS.append(HyperCLOVAXRMSNorm)
+class HyperCLOVAXRotaryEmbedding(nn.Module):
+    def __init__(self, config: HyperCLOVAXConfig, device=None):
+        super().__init__()
+        # BC: "rope_type" was originally "type"
+        if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
+            self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
+        else:
+            self.rope_type = "default"
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+
+        self.config = config
+        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+
+        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.original_inv_freq = self.inv_freq
+
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        position_ids (`torch.Tensor`, *optional*):
+            Deprecated and unused.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class HyperCLOVAXMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias)
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        return down_proj
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: Optional[torch.Tensor],
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs,
+):
+    key_states = repeat_kv(key, module.num_key_value_groups)
+    value_states = repeat_kv(value, module.num_key_value_groups)
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+        attn_weights = attn_weights + causal_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class HyperCLOVAXAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: HyperCLOVAXConfig, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.scaling = getattr(config, "attention_multiplier", self.head_dim**-0.5) # MuP
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+
+        self.q_proj = nn.Linear(
+            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.k_proj = nn.Linear(
+            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.v_proj = nn.Linear(
+            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_value: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            # sin and cos are specific to RoPE models; cache_position needed for the static cache
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        attention_interface: Callable = eager_attention_forward
+
+        if self.config._attn_implementation != "eager":
+            if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False):
+                logger.warning_once(
+                    "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to "
+                    'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+                )
+            else:
+                attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class HyperCLOVAXDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: HyperCLOVAXConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+
+        self.self_attn = HyperCLOVAXAttention(config=config, layer_idx=layer_idx)
+
+        self.mlp = HyperCLOVAXMLP(config)
+        self.input_layernorm = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.use_post_norm = getattr(config, "use_post_norm", False)
+
+        # Peri-LN (post-norm)
+        if self.use_post_norm:
+            self.post_norm1 = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+            self.post_norm2 = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.residual_multiplier = getattr(config, "residual_multiplier", 1.0) # MuP
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            position_embeddings=position_embeddings,
+            **kwargs,
+        )
+
+        if self.use_post_norm: # Peri-LN
+            hidden_states = self.post_norm1(hidden_states)
+
+        hidden_states = residual + hidden_states * self.residual_multiplier # MuP
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+
+        if self.use_post_norm: # Peri-LN
+            hidden_states = self.post_norm2(hidden_states)
+
+        hidden_states = residual + hidden_states * self.residual_multiplier # MuP
+
+        outputs = (hidden_states,)
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        return outputs
+
+
+@auto_docstring
+class HyperCLOVAXPreTrainedModel(PreTrainedModel):
+    config_class = HyperCLOVAXConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["HyperCLOVAXDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+    _supports_cache_class = True
+    _supports_quantized_cache = True
+    _supports_static_cache = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, HyperCLOVAXRMSNorm):
+            module.weight.data.fill_(1.0)
+
+
+@auto_docstring
+class HyperCLOVAXModel(HyperCLOVAXPreTrainedModel):
+    def __init__(self, config: HyperCLOVAXConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [HyperCLOVAXDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = HyperCLOVAXRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+        # MuP
+        self.embedding_multiplier = getattr(config, "embedding_multiplier", 1.0)
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
+    ) -> BaseModelOutputWithPast:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if self.gradient_checkpointing and self.training and use_cache:
+            logger.warning_once(
+                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+            )
+            use_cache = False
+
+        # TODO (joao): remove this exception in v4.56 -- it exists for users that try to pass a legacy cache
+        if not isinstance(past_key_values, (type(None), Cache)):
+            raise ValueError("The `past_key_values` should be either a `Cache` object or `None`.")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        inputs_embeds = inputs_embeds * self.embedding_multiplier # MuP
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache()
+
+        if cache_position is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            cache_position = torch.arange(
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+            )
+
+        if position_ids is None:
+            position_ids = cache_position.unsqueeze(0)
+
+        causal_mask = self._update_causal_mask(
+            attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
+        )
+
+        hidden_states = inputs_embeds
+
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            layer_outputs = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_values,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+                cache_position=cache_position,
+                position_embeddings=position_embeddings,
+                **flash_attn_kwargs,
+            )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.norm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values if use_cache else None,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+    def _update_causal_mask(
+        self,
+        attention_mask: Union[torch.Tensor, "BlockMask"],
+        input_tensor: torch.Tensor,
+        cache_position: torch.Tensor,
+        past_key_values: Cache,
+        output_attentions: bool = False,
+    ):
+        if self.config._attn_implementation == "flash_attention_2":
+            if attention_mask is not None and (attention_mask == 0.0).any():
+                return attention_mask
+            return None
+        if self.config._attn_implementation == "flex_attention":
+            if isinstance(attention_mask, torch.Tensor):
+                attention_mask = make_flex_block_causal_mask(attention_mask)
+            return attention_mask
+
+        # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
+        # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
+        # to infer the attention mask.
+        past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+        using_compilable_cache = past_key_values.is_compileable if past_key_values is not None else False
+
+        # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
+        if self.config._attn_implementation == "sdpa" and not using_compilable_cache and not output_attentions:
+            if AttentionMaskConverter._ignore_causal_mask_sdpa(
+                attention_mask,
+                inputs_embeds=input_tensor,
+                past_key_values_length=past_seen_tokens,
+                is_training=self.training,
+            ):
+                return None
+
+        dtype = input_tensor.dtype
+        sequence_length = input_tensor.shape[1]
+        if using_compilable_cache:
+            target_length = past_key_values.get_max_cache_shape()
+        else:
+            target_length = (
+                attention_mask.shape[-1]
+                if isinstance(attention_mask, torch.Tensor)
+                else past_seen_tokens + sequence_length + 1
+            )
+
+        # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
+        causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
+            attention_mask,
+            sequence_length=sequence_length,
+            target_length=target_length,
+            dtype=dtype,
+            cache_position=cache_position,
+            batch_size=input_tensor.shape[0],
+        )
+
+        if (
+            self.config._attn_implementation == "sdpa"
+            and attention_mask is not None
+            and attention_mask.device.type in ["cuda", "xpu", "npu"]
+            and not output_attentions
+        ):
+            # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
+            # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
+            # Details: https://github.com/pytorch/pytorch/issues/110213
+            min_dtype = torch.finfo(dtype).min
+            causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)
+
+        return causal_mask
+
+    @staticmethod
+    def _prepare_4d_causal_attention_mask_with_cache_position(
+        attention_mask: torch.Tensor,
+        sequence_length: int,
+        target_length: int,
+        dtype: torch.dtype,
+        cache_position: torch.Tensor,
+        batch_size: int,
+        **kwargs,
+    ):
+        """
+        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
+        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
+
+        Args:
+            attention_mask (`torch.Tensor`):
+                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
+                `(batch_size, 1, query_length, key_value_length)`.
+            sequence_length (`int`):
+                The sequence length being processed.
+            target_length (`int`):
+                The target length: when generating with static cache, the mask should be as long as the static cache,
+                to account for the 0 padding, the part of the cache that is not filled yet.
+            dtype (`torch.dtype`):
+                The dtype to use for the 4D attention mask.
+            cache_position (`torch.Tensor`):
+                Indices depicting the position of the input sequence tokens in the sequence.
+            batch_size (`torch.Tensor`):
+                Batch size.
+        """
+        if attention_mask is not None and attention_mask.dim() == 4:
+            # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
+            causal_mask = attention_mask
+        else:
+            min_dtype = torch.finfo(dtype).min
+            causal_mask = torch.full(
+                (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=cache_position.device
+            )
+            if sequence_length != 1:
+                causal_mask = torch.triu(causal_mask, diagonal=1)
+            causal_mask *= torch.arange(target_length, device=cache_position.device) > cache_position.reshape(-1, 1)
+            causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+            if attention_mask is not None:
+                causal_mask = causal_mask.clone()  # copy to contiguous memory for in-place edit
+                mask_length = attention_mask.shape[-1]
+                padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(
+                    causal_mask.device
+                )
+                padding_mask = padding_mask == 0
+                causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
+                    padding_mask, min_dtype
+                )
+
+        return causal_mask
+
+
+class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ...
+
+
+@auto_docstring
+class HyperCLOVAXForCausalLM(HyperCLOVAXPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = ["lm_head.weight"]
+    _tp_plan = {"lm_head": "colwise_rep"}
+    _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = HyperCLOVAXModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.logits_scaling = getattr(config, "logits_scaling", 1.0)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        logits_to_keep: Union[int, torch.Tensor] = 0,
+        **kwargs: Unpack[KwargsForCausalLM],
+    ) -> CausalLMOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM
+
+        >>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/{model_name}")
+        >>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/{model_name}")
+
+        >>> prompt = "Hey, are you conscious? Can you talk to me?"
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
+        ```"""
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        # MuP
+        logits = self.lm_head(hidden_states[:, slice_indices, :]) * self.logits_scaling
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    The HyperCLOVAX Model transformer with a sequence classification head on top (linear layer).
+
+    [`HyperCLOVAXForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-2) do.
+
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """
+)
+class HyperCLOVAXForSequenceClassification(HyperCLOVAXPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = HyperCLOVAXModel(config)
+        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+    ) -> SequenceClassifierOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        transformer_outputs: BaseModelOutputWithPast = self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+        hidden_states = transformer_outputs.last_hidden_state
+        logits = self.score(hidden_states)
+
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            last_non_pad_token = -1
+        elif input_ids is not None:
+            # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id
+            non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32)
+            token_indices = torch.arange(input_ids.shape[-1], device=logits.device, dtype=torch.int32)
+            last_non_pad_token = (token_indices * non_pad_mask).argmax(-1)
+        else:
+            last_non_pad_token = -1
+            logger.warning_once(
+                f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "
+                "unexpected if using padding tokens in conjunction with `inputs_embeds.`"
+            )
+
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token]
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config)
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+
+@auto_docstring
+class HyperCLOVAXForQuestionAnswering(HyperCLOVAXPreTrainedModel):
+    base_model_prefix = "transformer"
+
+    # Copied from transformers.models.bloom.modeling_bloom.BloomForQuestionAnswering.__init__ with Bloom->HyperCLOVAX
+    def __init__(self, config):
+        super().__init__(config)
+        self.transformer = HyperCLOVAXModel(config)
+        self.qa_outputs = nn.Linear(config.hidden_size, 2)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.transformer.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.transformer.embed_tokens = value
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        start_positions: Optional[torch.LongTensor] = None,
+        end_positions: Optional[torch.LongTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        **kwargs,
+    ) -> QuestionAnsweringModelOutput:
+        outputs: BaseModelOutputWithPast = self.transformer(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+
+        sequence_output = outputs.last_hidden_state
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1).contiguous()
+        end_logits = end_logits.squeeze(-1).contiguous()
+
+        loss = None
+        if start_positions is not None and end_positions is not None:
+            loss = self.loss_function(start_logits, end_logits, start_positions, end_positions, **kwargs)
+
+        return QuestionAnsweringModelOutput(
+            loss=loss,
+            start_logits=start_logits,
+            end_logits=end_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class HyperCLOVAXForTokenClassification(HyperCLOVAXPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = HyperCLOVAXModel(config)
+        if getattr(config, "classifier_dropout", None) is not None:
+            classifier_dropout = config.classifier_dropout
+        elif getattr(config, "hidden_dropout", None) is not None:
+            classifier_dropout = config.hidden_dropout
+        else:
+            classifier_dropout = 0.1
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.score = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+    ) -> TokenClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        outputs: BaseModelOutputWithPast = self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+        sequence_output = outputs.last_hidden_state
+        sequence_output = self.dropout(sequence_output)
+        logits = self.score(sequence_output)
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits, labels, self.config)
+
+        return TokenClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "HyperCLOVAXForCausalLM",
+    "HyperCLOVAXModel",
+    "HyperCLOVAXPreTrainedModel",
+    "HyperCLOVAXForSequenceClassification",
+    "HyperCLOVAXForQuestionAnswering",
+    "HyperCLOVAXForTokenClassification",
+]

quant_log.csv

@@ -0,0 +1,267 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.0000059439,0.05000,1.523
+0,self_attn.v_proj,0.0000039075,0.05000,1.012
+0,self_attn.q_proj,0.0000238143,0.05000,1.021
+0,self_attn.o_proj,0.0000220391,0.05000,1.315
+0,mlp.up_proj,0.0000110815,0.05000,1.442
+0,mlp.gate_proj,0.0000113717,0.05000,1.044
+0,mlp.down_proj,0.0000151834,0.05000,3.415
+1,self_attn.k_proj,0.0000059126,0.05000,1.041
+1,self_attn.v_proj,0.0000043126,0.05000,1.004
+1,self_attn.q_proj,0.0000232026,0.05000,1.020
+1,self_attn.o_proj,0.0000089596,0.05000,1.041
+1,mlp.up_proj,0.0000111797,0.05000,1.065
+1,mlp.gate_proj,0.0000114342,0.05000,1.035
+1,mlp.down_proj,0.0000064124,0.05000,2.654
+2,self_attn.k_proj,0.0000052098,0.05000,1.045
+2,self_attn.v_proj,0.0000036584,0.05000,1.010
+2,self_attn.q_proj,0.0000192775,0.05000,1.018
+2,self_attn.o_proj,0.0000115804,0.05000,1.037
+2,mlp.up_proj,0.0000130055,0.05000,1.068
+2,mlp.gate_proj,0.0000134072,0.05000,1.041
+2,mlp.down_proj,0.0000102456,0.05000,2.650
+3,self_attn.k_proj,0.0000079650,0.05000,1.039
+3,self_attn.v_proj,0.0000062979,0.05000,1.005
+3,self_attn.q_proj,0.0000344130,0.05000,1.018
+3,self_attn.o_proj,0.0000148093,0.05000,1.042
+3,mlp.up_proj,0.0000074303,0.05000,1.066
+3,mlp.gate_proj,0.0000075460,0.05000,1.039
+3,mlp.down_proj,0.0000036906,0.05000,2.652
+4,self_attn.k_proj,0.0000072926,0.05000,1.035
+4,self_attn.v_proj,0.0000054965,0.05000,1.004
+4,self_attn.q_proj,0.0000324719,0.05000,1.015
+4,self_attn.o_proj,0.0000135363,0.05000,1.042
+4,mlp.up_proj,0.0000077935,0.05000,1.064
+4,mlp.gate_proj,0.0000079466,0.05000,1.039
+4,mlp.down_proj,0.0000032122,0.05000,2.651
+5,self_attn.k_proj,0.0000092984,0.05000,1.038
+5,self_attn.v_proj,0.0000075066,0.05000,1.011
+5,self_attn.q_proj,0.0000400782,0.05000,1.019
+5,self_attn.o_proj,0.0000137737,0.05000,1.039
+5,mlp.up_proj,0.0000084687,0.05000,1.066
+5,mlp.gate_proj,0.0000086387,0.05000,1.037
+5,mlp.down_proj,0.0000029938,0.05000,2.648
+6,self_attn.k_proj,0.0000097146,0.05000,1.043
+6,self_attn.v_proj,0.0000078300,0.05000,1.001
+6,self_attn.q_proj,0.0000425935,0.05000,1.016
+6,self_attn.o_proj,0.0000130678,0.05000,1.040
+6,mlp.up_proj,0.0000066432,0.05000,1.066
+6,mlp.gate_proj,0.0000067693,0.05000,1.037
+6,mlp.down_proj,0.0000019580,0.05000,2.641
+7,self_attn.k_proj,0.0000110695,0.05000,1.034
+7,self_attn.v_proj,0.0000098961,0.05000,1.006
+7,self_attn.q_proj,0.0000535851,0.05000,1.012
+7,self_attn.o_proj,0.0000108217,0.05000,1.037
+7,mlp.up_proj,0.0000063509,0.05000,1.063
+7,mlp.gate_proj,0.0000064191,0.05000,1.035
+7,mlp.down_proj,0.0000018483,0.05000,2.653
+8,self_attn.k_proj,0.0000095001,0.05000,1.041
+8,self_attn.v_proj,0.0000075776,0.05000,1.002
+8,self_attn.q_proj,0.0000411649,0.05000,1.013
+8,self_attn.o_proj,0.0000126457,0.05000,1.040
+8,mlp.up_proj,0.0000065683,0.05000,1.063
+8,mlp.gate_proj,0.0000067366,0.05000,1.034
+8,mlp.down_proj,0.0000015336,0.05000,2.636
+9,self_attn.k_proj,0.0000075909,0.05000,1.044
+9,self_attn.v_proj,0.0000057550,0.05000,1.013
+9,self_attn.q_proj,0.0000344832,0.05000,1.025
+9,self_attn.o_proj,0.0000085166,0.05000,1.038
+9,mlp.up_proj,0.0000043794,0.05000,1.061
+9,mlp.gate_proj,0.0000045057,0.05000,1.035
+9,mlp.down_proj,0.0000009460,0.05000,2.641
+10,self_attn.k_proj,0.0000111547,0.05000,1.033
+10,self_attn.v_proj,0.0000092689,0.05000,1.000
+10,self_attn.q_proj,0.0000519109,0.05000,1.011
+10,self_attn.o_proj,0.0000135746,0.05000,1.032
+10,mlp.up_proj,0.0000039040,0.05000,1.061
+10,mlp.gate_proj,0.0000039544,0.05000,1.033
+10,mlp.down_proj,0.0000007761,0.05000,2.628
+11,self_attn.k_proj,0.0000108629,0.05000,1.035
+11,self_attn.v_proj,0.0000084592,0.05000,1.006
+11,self_attn.q_proj,0.0000500227,0.05000,1.008
+11,self_attn.o_proj,0.0000132639,0.05000,1.035
+11,mlp.up_proj,0.0000032214,0.05000,1.060
+11,mlp.gate_proj,0.0000033015,0.05000,1.030
+11,mlp.down_proj,0.0000004664,0.05000,2.627
+12,self_attn.k_proj,0.0000125110,0.05000,1.037
+12,self_attn.v_proj,0.0000097656,0.05000,0.995
+12,self_attn.q_proj,0.0000552543,0.05000,1.007
+12,self_attn.o_proj,0.0000134279,0.05000,1.036
+12,mlp.up_proj,0.0000031708,0.05000,1.058
+12,mlp.gate_proj,0.0000032374,0.05000,1.032
+12,mlp.down_proj,0.0000003898,0.05000,2.639
+13,self_attn.k_proj,0.0000102793,0.05000,1.040
+13,self_attn.v_proj,0.0000081455,0.05000,1.008
+13,self_attn.q_proj,0.0000480064,0.05000,1.011
+13,self_attn.o_proj,0.0000093675,0.05000,1.033
+13,mlp.up_proj,0.0000021026,0.05000,1.063
+13,mlp.gate_proj,0.0000021487,0.05000,1.033
+13,mlp.down_proj,0.0000001983,0.05000,2.641
+14,self_attn.k_proj,0.0000103936,0.05000,1.041
+14,self_attn.v_proj,0.0000081330,0.05000,1.002
+14,self_attn.q_proj,0.0000488184,0.05000,1.011
+14,self_attn.o_proj,0.0000089386,0.05000,1.037
+14,mlp.up_proj,0.0000025068,0.05000,1.062
+14,mlp.gate_proj,0.0000025513,0.05000,1.036
+14,mlp.down_proj,0.0000002944,0.05000,2.642
+15,self_attn.k_proj,0.0000134852,0.05000,1.045
+15,self_attn.v_proj,0.0000107378,0.05000,1.014
+15,self_attn.q_proj,0.0000620740,0.05000,1.016
+15,self_attn.o_proj,0.0000136202,0.05000,1.043
+15,mlp.up_proj,0.0000032474,0.05000,1.064
+15,mlp.gate_proj,0.0000033245,0.05000,1.040
+15,mlp.down_proj,0.0000004164,0.05000,2.635
+16,self_attn.k_proj,0.0000150497,0.05000,1.043
+16,self_attn.v_proj,0.0000123104,0.05000,1.008
+16,self_attn.q_proj,0.0000706003,0.05000,1.011
+16,self_attn.o_proj,0.0000116576,0.05000,1.038
+16,mlp.up_proj,0.0000042218,0.05000,1.061
+16,mlp.gate_proj,0.0000043296,0.05000,1.033
+16,mlp.down_proj,0.0000004880,0.05000,2.634
+17,self_attn.k_proj,0.0000164214,0.05000,1.038
+17,self_attn.v_proj,0.0000140974,0.05000,1.007
+17,self_attn.q_proj,0.0000774397,0.05000,1.009
+17,self_attn.o_proj,0.0000219263,0.05000,1.039
+17,mlp.up_proj,0.0000038117,0.05000,1.065
+17,mlp.gate_proj,0.0000039213,0.05000,1.041
+17,mlp.down_proj,0.0000004193,0.05000,2.653
+18,self_attn.k_proj,0.0000172823,0.05000,1.033
+18,self_attn.v_proj,0.0000144956,0.05000,0.998
+18,self_attn.q_proj,0.0000809578,0.05000,1.004
+18,self_attn.o_proj,0.0000178630,0.05000,1.035
+18,mlp.up_proj,0.0000034930,0.05000,1.061
+18,mlp.gate_proj,0.0000035984,0.05000,1.031
+18,mlp.down_proj,0.0000003581,0.05000,2.634
+19,self_attn.k_proj,0.0000138637,0.05000,1.038
+19,self_attn.v_proj,0.0000106994,0.05000,1.005
+19,self_attn.q_proj,0.0000632666,0.05000,1.005
+19,self_attn.o_proj,0.0000055669,0.05000,1.032
+19,mlp.up_proj,0.0000037013,0.05000,1.070
+19,mlp.gate_proj,0.0000038086,0.05000,1.035
+19,mlp.down_proj,0.0000003597,0.05000,2.646
+20,self_attn.k_proj,0.0000187151,0.05000,1.037
+20,self_attn.v_proj,0.0000151013,0.05000,1.000
+20,self_attn.q_proj,0.0000877593,0.05000,1.008
+20,self_attn.o_proj,0.0000173331,0.05000,1.031
+20,mlp.up_proj,0.0000035404,0.05000,1.058
+20,mlp.gate_proj,0.0000036535,0.05000,1.030
+20,mlp.down_proj,0.0000003419,0.05000,2.647
+21,self_attn.k_proj,0.0000161697,0.05000,1.037
+21,self_attn.v_proj,0.0000122697,0.05000,1.005
+21,self_attn.q_proj,0.0000721811,0.05000,1.011
+21,self_attn.o_proj,0.0000143799,0.05000,1.041
+21,mlp.up_proj,0.0000040757,0.05000,1.062
+21,mlp.gate_proj,0.0000042371,0.05000,1.039
+21,mlp.down_proj,0.0000004525,0.05000,2.664
+22,self_attn.k_proj,0.0000169354,0.05000,1.033
+22,self_attn.v_proj,0.0000132405,0.05000,1.002
+22,self_attn.q_proj,0.0000890276,0.05000,1.006
+22,self_attn.o_proj,0.0000041558,0.05000,1.037
+22,mlp.up_proj,0.0000036900,0.05000,1.056
+22,mlp.gate_proj,0.0000038310,0.05000,1.025
+22,mlp.down_proj,0.0000004213,0.05000,2.630
+23,self_attn.k_proj,0.0000182787,0.05000,1.039
+23,self_attn.v_proj,0.0000157284,0.05000,1.006
+23,self_attn.q_proj,0.0000960312,0.05000,1.006
+23,self_attn.o_proj,0.0000131840,0.05000,1.034
+23,mlp.up_proj,0.0000030392,0.05000,1.069
+23,mlp.gate_proj,0.0000031354,0.05000,1.040
+23,mlp.down_proj,0.0000002823,0.05000,2.631
+24,self_attn.k_proj,0.0000211280,0.05000,1.030
+24,self_attn.v_proj,0.0000192252,0.05000,1.001
+24,self_attn.q_proj,0.0001013641,0.05000,1.004
+24,self_attn.o_proj,0.0000243574,0.05000,1.039
+24,mlp.up_proj,0.0000033583,0.05000,1.065
+24,mlp.gate_proj,0.0000034760,0.05000,1.034
+24,mlp.down_proj,0.0000003517,0.05000,2.634
+25,self_attn.k_proj,0.0000196986,0.05000,1.048
+25,self_attn.v_proj,0.0000153495,0.05000,1.019
+25,self_attn.q_proj,0.0000933140,0.05000,1.016
+25,self_attn.o_proj,0.0000164252,0.05000,1.037
+25,mlp.up_proj,0.0000034673,0.05000,1.066
+25,mlp.gate_proj,0.0000035728,0.05000,1.042
+25,mlp.down_proj,0.0000003741,0.05000,2.647
+26,self_attn.k_proj,0.0000196134,0.05000,1.039
+26,self_attn.v_proj,0.0000150768,0.05000,1.004
+26,self_attn.q_proj,0.0000988552,0.05000,1.005
+26,self_attn.o_proj,0.0000089032,0.05000,1.038
+26,mlp.up_proj,0.0000036113,0.05000,1.064
+26,mlp.gate_proj,0.0000037079,0.05000,1.033
+26,mlp.down_proj,0.0000003865,0.05000,2.638
+27,self_attn.k_proj,0.0000185194,0.05000,1.038
+27,self_attn.v_proj,0.0000147775,0.05000,1.005
+27,self_attn.q_proj,0.0000884268,0.05000,1.006
+27,self_attn.o_proj,0.0000162331,0.05000,1.038
+27,mlp.up_proj,0.0000031586,0.05000,1.061
+27,mlp.gate_proj,0.0000032463,0.05000,1.036
+27,mlp.down_proj,0.0000003164,0.05000,2.633
+28,self_attn.k_proj,0.0000182955,0.05000,1.043
+28,self_attn.v_proj,0.0000151388,0.05000,1.002
+28,self_attn.q_proj,0.0000885868,0.05000,1.005
+28,self_attn.o_proj,0.0000136137,0.05000,1.032
+28,mlp.up_proj,0.0000028749,0.05000,1.064
+28,mlp.gate_proj,0.0000029219,0.05000,1.033
+28,mlp.down_proj,0.0000003112,0.05000,2.635
+29,self_attn.k_proj,0.0000185079,0.05000,1.043
+29,self_attn.v_proj,0.0000122247,0.05000,1.010
+29,self_attn.q_proj,0.0000848096,0.05000,1.009
+29,self_attn.o_proj,0.0000079501,0.05000,1.033
+29,mlp.up_proj,0.0000034258,0.05000,1.060
+29,mlp.gate_proj,0.0000034560,0.05000,1.033
+29,mlp.down_proj,0.0000004112,0.05000,2.624
+30,self_attn.k_proj,0.0000200373,0.05000,1.034
+30,self_attn.v_proj,0.0000163101,0.05000,1.004
+30,self_attn.q_proj,0.0001012875,0.05000,1.004
+30,self_attn.o_proj,0.0000071170,0.05000,1.033
+30,mlp.up_proj,0.0000035733,0.05000,1.057
+30,mlp.gate_proj,0.0000035858,0.05000,1.030
+30,mlp.down_proj,0.0000004136,0.05000,2.630
+31,self_attn.k_proj,0.0000211797,0.05000,1.036
+31,self_attn.v_proj,0.0000139414,0.05000,1.005
+31,self_attn.q_proj,0.0000860812,0.05000,1.006
+31,self_attn.o_proj,0.0000048044,0.05000,1.033
+31,mlp.up_proj,0.0000037375,0.05000,1.060
+31,mlp.gate_proj,0.0000037139,0.05000,1.033
+31,mlp.down_proj,0.0000004492,0.05000,2.622
+32,self_attn.k_proj,0.0000207835,0.05000,1.035
+32,self_attn.v_proj,0.0000152161,0.05000,1.001
+32,self_attn.q_proj,0.0000913564,0.05000,1.007
+32,self_attn.o_proj,0.0000079106,0.05000,1.034
+32,mlp.up_proj,0.0000024507,0.05000,1.059
+32,mlp.gate_proj,0.0000024351,0.05000,1.033
+32,mlp.down_proj,0.0000001740,0.05000,2.624
+33,self_attn.k_proj,0.0000182657,0.05000,1.039
+33,self_attn.v_proj,0.0000128059,0.05000,1.006
+33,self_attn.q_proj,0.0000866449,0.05000,1.007
+33,self_attn.o_proj,0.0000072791,0.05000,1.032
+33,mlp.up_proj,0.0000031301,0.05000,1.068
+33,mlp.gate_proj,0.0000031329,0.05000,1.031
+33,mlp.down_proj,0.0000004381,0.05000,2.626
+34,self_attn.k_proj,0.0000166431,0.05000,1.037
+34,self_attn.v_proj,0.0000133107,0.05000,1.001
+34,self_attn.q_proj,0.0000836072,0.05000,1.005
+34,self_attn.o_proj,0.0000067067,0.05000,1.039
+34,mlp.up_proj,0.0000042955,0.05000,1.059
+34,mlp.gate_proj,0.0000043195,0.05000,1.031
+34,mlp.down_proj,0.0000005294,0.05000,2.631
+35,self_attn.k_proj,0.0000127095,0.05000,1.038
+35,self_attn.v_proj,0.0000130456,0.05000,1.007
+35,self_attn.q_proj,0.0000706689,0.05000,1.006
+35,self_attn.o_proj,0.0000047317,0.05000,1.031
+35,mlp.up_proj,0.0000050128,0.05000,1.061
+35,mlp.gate_proj,0.0000050194,0.05000,1.029
+35,mlp.down_proj,0.0000007617,0.05000,2.616
+36,self_attn.k_proj,0.0000122905,0.05000,1.035
+36,self_attn.v_proj,0.0000143251,0.05000,1.002
+36,self_attn.q_proj,0.0000760232,0.05000,1.004
+36,self_attn.o_proj,0.0000038522,0.05000,1.032
+36,mlp.up_proj,0.0000282895,0.05000,1.061
+36,mlp.gate_proj,0.0000269556,0.05000,1.028
+36,mlp.down_proj,0.0000333966,0.05000,2.631
+37,self_attn.k_proj,0.0000136531,0.05000,1.034
+37,self_attn.v_proj,0.0000129941,0.05000,1.002
+37,self_attn.q_proj,0.0000815826,0.05000,1.001
+37,self_attn.o_proj,0.0000139014,0.05000,1.027
+37,mlp.up_proj,0.0000297700,0.05000,1.056
+37,mlp.gate_proj,0.0000275959,0.05000,1.030
+37,mlp.down_proj,0.0000330723,0.05000,2.627

quantize_config.json

@@ -0,0 +1,24 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "hyb_act": false,
+  "sym": true,
+  "lm_head": false,
+  "quant_method": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:4.0.0"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.05,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "v2": false,
+    "v2_alpha": 0.25
+  }
+}

special_tokens_map.json

@@ -0,0 +1,86 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,501 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|_unuse_missing_100256|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "<|_unuse_missing_100261|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100262": {
+      "content": "<|_unuse_missing_100262|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|_unuse_missing_100263|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|_unuse_missing_100264|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|_unuse_missing_100265|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|_unuse_missing_100266|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|_unuse_missing_100267|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|_unuse_missing_100268|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|_unuse_missing_100269|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|_unuse_missing_100270|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|_unuse_missing_100271|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|stop|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|endofturn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110491": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110492": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110493": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110494": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110495": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110496": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110497": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110498": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110499": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110500": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110501": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110502": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110503": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110504": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110505": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110506": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110507": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110508": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110509": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110510": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110511": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110512": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110513": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110514": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110515": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110516": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110517": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110518": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110519": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110520": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110521": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110522": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110523": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}