Upload model

config.json

@@ -0,0 +1,60 @@
+{
+  "_name_or_path": "Qwen-14B-8bit-wikitext-ex512-len4096-hf-safetensors",
+  "architectures": [
+    "QWenLMHeadModel"
+  ],
+  "attn_dropout_prob": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_qwen.QWenConfig",
+    "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"
+  },
+  "bf16": false,
+  "emb_dropout_prob": 0.0,
+  "fp16": true,
+  "fp32": false,
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 27392,
+  "kv_channels": 128,
+  "layer_norm_epsilon": 1e-06,
+  "max_position_embeddings": 8192,
+  "model_type": "qwen",
+  "no_bias": true,
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "onnx_safe": null,
+  "quantization_config": {
+    "batch_size": 1,
+    "bits": 8,
+    "block_name_to_quantize": null,
+    "damp_percent": 0.01,
+    "dataset": null,
+    "desc_act": false,
+    "disable_exllama": false,
+    "group_size": 128,
+    "model_seqlen": null,
+    "module_name_preceding_first_block": null,
+    "pad_token_id": null,
+    "quant_method": "gptq",
+    "sym": true,
+    "tokenizer": null,
+    "true_sequential": true,
+    "use_cuda_fp16": false
+  },
+  "rotary_emb_base": 10000,
+  "rotary_pct": 1.0,
+  "scale_attn_weights": true,
+  "seq_length": 2048,
+  "softmax_in_fp32": false,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "QWenTokenizer",
+  "torch_dtype": "float16",
+  "transformers_version": "4.32.0",
+  "use_cache": true,
+  "use_cache_kernel": false,
+  "use_cache_quantization": false,
+  "use_dynamic_ntk": true,
+  "use_flash_attn": true,
+  "use_logn_attn": true,
+  "vocab_size": 152064
+}

configuration_qwen.py

@@ -0,0 +1,71 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from transformers import PretrainedConfig
+
+
+class QWenConfig(PretrainedConfig):
+    model_type = "qwen"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        emb_dropout_prob=0.0,
+        attn_dropout_prob=0.0,
+        layer_norm_epsilon=1e-6,
+        initializer_range=0.02,
+        max_position_embeddings=8192,
+        scale_attn_weights=True,
+        use_cache=True,
+        bf16=False,
+        fp16=False,
+        fp32=False,
+        kv_channels=128,
+        rotary_pct=1.0,
+        rotary_emb_base=10000,
+        use_dynamic_ntk=True,
+        use_logn_attn=True,
+        use_flash_attn="auto",
+        intermediate_size=22016,
+        no_bias=True,
+        tie_word_embeddings=False,
+        use_cache_quantization=False,
+        use_cache_kernel=False,
+        softmax_in_fp32=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.emb_dropout_prob = emb_dropout_prob
+        self.attn_dropout_prob = attn_dropout_prob
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+        self.scale_attn_weights = scale_attn_weights
+        self.use_cache = use_cache
+        self.max_position_embeddings = max_position_embeddings
+        self.bf16 = bf16
+        self.fp16 = fp16
+        self.fp32 = fp32
+        self.kv_channels = kv_channels
+        self.rotary_pct = rotary_pct
+        self.rotary_emb_base = rotary_emb_base
+        self.use_dynamic_ntk = use_dynamic_ntk
+        self.use_logn_attn = use_logn_attn
+        self.use_flash_attn = use_flash_attn
+        self.no_bias = no_bias
+        self.use_cache_quantization = use_cache_quantization
+        self.use_cache_kernel = use_cache_kernel
+        self.softmax_in_fp32 = softmax_in_fp32
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs
+        )

cpp_kernels.py

@@ -0,0 +1,55 @@
+from torch.utils import cpp_extension
+import pathlib
+import os
+import subprocess
+
+def _get_cuda_bare_metal_version(cuda_dir):
+    raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
+                                         universal_newlines=True)
+    output = raw_output.split()
+    release_idx = output.index("release") + 1
+    release = output[release_idx].split(".")
+    bare_metal_major = release[0]
+    bare_metal_minor = release[1][0]
+
+    return raw_output, bare_metal_major, bare_metal_minor
+
+def _create_build_dir(buildpath):
+    try:
+        os.mkdir(buildpath)
+    except OSError:
+        if not os.path.isdir(buildpath):
+            print(f"Creation of the build directory {buildpath} failed")
+
+# Check if cuda 11 is installed for compute capability 8.0
+cc_flag = []
+_, bare_metal_major, bare_metal_minor = _get_cuda_bare_metal_version(cpp_extension.CUDA_HOME)
+if int(bare_metal_major) >= 11:
+    cc_flag.append('-gencode')
+    cc_flag.append('arch=compute_80,code=sm_80')
+    if int(bare_metal_minor) >= 7:
+        cc_flag.append('-gencode')
+        cc_flag.append('arch=compute_90,code=sm_90')
+
+# Build path
+srcpath = pathlib.Path(__file__).parent.absolute()
+buildpath = srcpath / 'build'
+_create_build_dir(buildpath)
+
+def _cpp_extention_load_helper(name, sources, extra_cuda_flags):
+    return cpp_extension.load(
+        name=name,
+        sources=sources,
+        build_directory=buildpath,
+        extra_cflags=['-O3', ],
+        extra_cuda_cflags=['-O3',
+                           '-gencode', 'arch=compute_70,code=sm_70',
+                           '--use_fast_math'] + extra_cuda_flags + cc_flag,
+        verbose=1
+    )
+
+extra_flags = []
+
+cache_autogptq_cuda_256_sources = ["./cache_autogptq_cuda_256.cpp",
+           "./cache_autogptq_cuda_kernel_256.cu"]
+cache_autogptq_cuda_256 = _cpp_extention_load_helper("cache_autogptq_cuda_256", cache_autogptq_cuda_256_sources, extra_flags)

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.32.0"
+}

modeling_qwen.py

@@ -0,0 +1,1428 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import importlib
+import math
+from typing import TYPE_CHECKING, Optional, Tuple, Union, Callable, List, Any, Generator
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch.cuda.amp import autocast
+
+from torch.nn import CrossEntropyLoss
+from transformers import PreTrainedTokenizer, GenerationConfig, StoppingCriteriaList
+from transformers.generation.logits_process import LogitsProcessorList
+
+if TYPE_CHECKING:
+    from transformers.generation.streamers import BaseStreamer
+from transformers.generation.utils import GenerateOutput
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+try:
+    from einops import rearrange
+except ImportError:
+    rearrange = None
+from torch import nn
+
+SUPPORT_CUDA = torch.cuda.is_available()
+SUPPORT_BF16 = SUPPORT_CUDA and torch.cuda.is_bf16_supported()
+SUPPORT_FP16 = SUPPORT_CUDA and torch.cuda.get_device_capability(0)[0] >= 7
+SUPPORT_TORCH2 = hasattr(torch, '__version__') and int(torch.__version__.split(".")[0]) >= 2
+
+
+from .configuration_qwen import QWenConfig
+from .qwen_generation_utils import (
+    HistoryType,
+    make_context,
+    decode_tokens,
+    get_stop_words_ids,
+    StopWordsLogitsProcessor,
+)
+
+
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "qwen"
+_CONFIG_FOR_DOC = "QWenConfig"
+
+QWen_PRETRAINED_MODEL_ARCHIVE_LIST = ["qwen-7b"]
+
+_ERROR_BAD_CHAT_FORMAT = """\
+We detect you are probably using the pretrained model (rather than chat model) for chatting, since the chat_format in generation_config is not "chatml".
+If you are directly using the model downloaded from Huggingface, please make sure you are using our "Qwen/Qwen-7B-Chat" Huggingface model (rather than "Qwen/Qwen-7B") when you call model.chat().
+我们检测到您可能在使用预训练模型（而非chat模型）进行多轮chat，因为您当前在generation_config指定的chat_format，并未设置为我们在对话中所支持的"chatml"格式。
+如果您在直接使用我们从Huggingface提供的模型，请确保您在调用model.chat()时，使用的是"Qwen/Qwen-7B-Chat"模型（而非"Qwen/Qwen-7B"预训练模型）。
+"""
+
+_SENTINEL = object()
+_ERROR_STREAM_IN_CHAT = """\
+Pass argument `stream` to model.chat() is buggy, deprecated, and marked for removal. Please use model.chat_stream(...) instead of model.chat(..., stream=True).
+向model.chat()传入参数stream的用法可能存在Bug，该用法已被废弃，将在未来被移除。请使用model.chat_stream(...)代替model.chat(..., stream=True)。
+"""
+
+_ERROR_INPUT_CPU_QUERY_WITH_FLASH_ATTN_ACTIVATED = """\
+We detect you have activated flash attention support, but running model computation on CPU. Please make sure that your input data has been placed on GPU. If you actually want to run CPU computation, please following the readme and set device_map="cpu" to disable flash attention when loading the model (calling AutoModelForCausalLM.from_pretrained).
+检测到您的模型已激活了flash attention支持，但正在执行CPU运算任务。如使用flash attention，请您确认模型输入已经传到GPU上。如果您确认要执行CPU运算，请您在载入模型（调用AutoModelForCausalLM.from_pretrained）时，按照readme说法，指定device_map="cpu"以禁用flash attention。
+"""
+
+apply_rotary_emb_func = None
+rms_norm = None
+flash_attn_unpadded_func = None
+
+def _import_flash_attn():
+    global apply_rotary_emb_func, rms_norm, flash_attn_unpadded_func
+    try:
+        from flash_attn.layers.rotary import apply_rotary_emb_func as __apply_rotary_emb_func
+        apply_rotary_emb_func = __apply_rotary_emb_func
+    except ImportError:
+        logger.warn(
+            "Warning: import flash_attn rotary fail, please install FlashAttention rotary to get higher efficiency "
+            "https://github.com/Dao-AILab/flash-attention/tree/main/csrc/rotary"
+        )
+
+    try:
+        from flash_attn.ops.rms_norm import rms_norm as __rms_norm
+        rms_norm = __rms_norm
+    except ImportError:
+        logger.warn(
+            "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency "
+            "https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm"
+        )
+
+    try:
+        import flash_attn
+        if not hasattr(flash_attn, '__version__'):
+            from flash_attn.flash_attn_interface import flash_attn_unpadded_func as __flash_attn_unpadded_func
+        else:
+            if int(flash_attn.__version__.split(".")[0]) >= 2:
+                from flash_attn.flash_attn_interface import flash_attn_varlen_func as __flash_attn_unpadded_func
+            else:
+                from flash_attn.flash_attn_interface import flash_attn_unpadded_func as __flash_attn_unpadded_func
+        flash_attn_unpadded_func = __flash_attn_unpadded_func
+    except ImportError:
+        logger.warn(
+            "Warning: import flash_attn fail, please install FlashAttention to get higher efficiency "
+            "https://github.com/Dao-AILab/flash-attention"
+        )
+
+def quantize_cache_v(fdata, bits, qmax, qmin):
+    # b, s, head, h-dim->b, head, s, h-dim
+    qtype = torch.uint8
+    device = fdata.device
+    shape = fdata.shape
+
+    fdata_cal = torch.flatten(fdata, 2)
+    fmax = torch.amax(fdata_cal, dim=-1, keepdim=True)
+    fmin = torch.amin(fdata_cal, dim=-1, keepdim=True)
+    # Compute params
+    if qmax.device != fmax.device:
+        qmax = qmax.to(device)
+        qmin = qmin.to(device)
+    scale = (fmax - fmin) / (qmax - qmin)
+    zero = qmin - fmin / scale
+    scale = scale.unsqueeze(-1).repeat(1,1,shape[2],1).contiguous()
+    zero = zero.unsqueeze(-1).repeat(1,1,shape[2],1).contiguous()
+    # Quantize
+    res_data = fdata / scale + zero
+    qdata = torch.clamp(res_data, qmin, qmax).to(qtype)
+    return qdata.contiguous(), scale, zero
+
+def dequantize_cache_torch(qdata, scale, zero):
+    data = scale * (qdata - zero)
+    return data
+
+class FlashSelfAttention(torch.nn.Module):
+    def __init__(
+        self,
+        causal=False,
+        softmax_scale=None,
+        attention_dropout=0.0,
+    ):
+        super().__init__()
+        assert flash_attn_unpadded_func is not None, (
+            "Please install FlashAttention first, " "e.g., with pip install flash-attn"
+        )
+        assert (
+            rearrange is not None
+        ), "Please install einops first, e.g., with pip install einops"
+        self.causal = causal
+        self.softmax_scale = softmax_scale
+        self.dropout_p = attention_dropout
+
+    def unpad_input(self, hidden_states, attention_mask):
+        valid_mask = attention_mask.squeeze(1).squeeze(1).eq(0)
+        seqlens_in_batch = valid_mask.sum(dim=-1, dtype=torch.int32)
+        indices = torch.nonzero(valid_mask.flatten(), as_tuple=False).flatten()
+        max_seqlen_in_batch = seqlens_in_batch.max().item()
+        cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+        hidden_states = hidden_states[indices]
+        return hidden_states, indices, cu_seqlens, max_seqlen_in_batch
+
+    def pad_input(self, hidden_states, indices, batch, seqlen):
+        output = torch.zeros(batch * seqlen, *hidden_states.shape[1:], device=hidden_states.device,
+                             dtype=hidden_states.dtype)
+        output[indices] = hidden_states
+        return rearrange(output, '(b s) ... -> b s ...', b=batch)
+
+    def forward(self, q, k, v, attention_mask=None):
+        assert all((i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v)))
+        assert all((i.is_cuda for i in (q, k, v)))
+        batch_size, seqlen_q = q.shape[0], q.shape[1]
+        seqlen_k = k.shape[1]
+        seqlen_out = seqlen_q
+
+        q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
+        cu_seqlens_q = torch.arange(
+            0,
+            (batch_size + 1) * seqlen_q,
+            step=seqlen_q,
+            dtype=torch.int32,
+            device=q.device,
+        )
+
+        if batch_size > 1 and attention_mask is not None:
+            k, indices_k, cu_seqlens_k, seqlen_k = self.unpad_input(k, attention_mask)
+            if q.size(0) == v.size(0):
+                q = q[indices_k]
+                cu_seqlens_q = cu_seqlens_k
+                seqlen_q = seqlen_k
+            v = v[indices_k]
+        else:
+            cu_seqlens_k = torch.arange(
+                0,
+                (batch_size + 1) * seqlen_k,
+                step=seqlen_k,
+                dtype=torch.int32,
+                device=q.device,
+            )
+
+        if self.training:
+            assert seqlen_k == seqlen_q
+            is_causal = self.causal
+            dropout_p = self.dropout_p
+        else:
+            is_causal = seqlen_q == seqlen_k
+            dropout_p = 0
+
+        output = flash_attn_unpadded_func(
+            q,
+            k,
+            v,
+            cu_seqlens_q,
+            cu_seqlens_k,
+            seqlen_q,
+            seqlen_k,
+            dropout_p,
+            softmax_scale=self.softmax_scale,
+            causal=is_causal,
+        )
+        if batch_size > 1 and attention_mask is not None and seqlen_q == seqlen_k:
+            output = self.pad_input(output, indices_k, batch_size, seqlen_out)
+        else:
+            new_shape = (batch_size, output.shape[0] // batch_size) + output.shape[1:]
+            output = output.view(new_shape)
+        return output
+
+
+class QWenAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        self.register_buffer("masked_bias", torch.tensor(-1e4), persistent=False)
+        self.seq_length = config.seq_length
+
+        self.hidden_size = config.hidden_size
+        self.split_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+
+        self.use_flash_attn = config.use_flash_attn
+        self.scale_attn_weights = True
+
+        self.projection_size = config.kv_channels * config.num_attention_heads
+
+        assert self.projection_size % config.num_attention_heads == 0
+        self.hidden_size_per_attention_head = (
+            self.projection_size // config.num_attention_heads
+        )
+
+        self.c_attn = nn.Linear(config.hidden_size, 3 * self.projection_size)
+
+        self.c_proj = nn.Linear(
+            config.hidden_size, self.projection_size, bias=not config.no_bias
+        )
+
+        self.is_fp32 = not (config.bf16 or config.fp16)
+        if (
+            self.use_flash_attn
+            and flash_attn_unpadded_func is not None
+            and not self.is_fp32
+        ):
+            self.core_attention_flash = FlashSelfAttention(
+                causal=True, attention_dropout=config.attn_dropout_prob
+            )
+        self.bf16 = config.bf16
+
+        self.use_dynamic_ntk = config.use_dynamic_ntk
+        self.use_logn_attn = config.use_logn_attn
+
+        logn_list = [
+            math.log(i, self.seq_length) if i > self.seq_length else 1
+            for i in range(1, 32768)
+        ]
+        logn_tensor = torch.tensor(logn_list)[None, :, None, None]
+        self.register_buffer("logn_tensor", logn_tensor, persistent=False)
+
+        self.attn_dropout = nn.Dropout(config.attn_dropout_prob)
+        self.softmax_in_fp32 = config.softmax_in_fp32 if hasattr(config, 'softmax_in_fp32') else False
+        self.use_cache_quantization = config.use_cache_quantization if hasattr(config, 'use_cache_quantization') else False
+        self.use_cache_kernel = config.use_cache_kernel if hasattr(config,'use_cache_kernel') else False
+        cache_dtype = torch.float
+        if self.bf16:
+            cache_dtype=torch.bfloat16
+        elif config.fp16:
+            cache_dtype = torch.float16
+        self.cache_qmax = torch.tensor(torch.iinfo(torch.uint8).max, dtype=cache_dtype)
+        self.cache_qmin = torch.tensor(torch.iinfo(torch.uint8).min, dtype=cache_dtype)
+
+        if config.use_cache_quantization and config.use_cache_kernel:
+            from .cpp_kernels import cache_autogptq_cuda_256
+            try:
+                self.cache_kernels = cache_autogptq_cuda_256
+            except ImportError:
+                self.cache_kernels = None
+
+    def _attn(self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None):
+        device = query.device
+        if self.use_cache_quantization:
+            qk, qk_scale, qk_zero = key
+            if self.use_cache_kernel and self.cache_kernels is not None:
+                shape = query.shape[:-1] + (qk.shape[-2],)
+                attn_weights = torch.zeros(shape, dtype=torch.float16, device=device)
+                self.cache_kernels.vecquant8matmul_batched_faster_old(
+                    query.contiguous() if query.dtype == torch.float16 else query.to(torch.float16).contiguous(),
+                    qk.transpose(-1, -2).contiguous(),
+                    attn_weights,
+                    qk_scale.contiguous() if qk_scale.dtype == torch.float16 else qk_scale.to(torch.float16).contiguous(),
+                    qk_zero.contiguous()if qk_zero.dtype == torch.float16 else qk_zero.to(torch.float16).contiguous())
+                # attn_weights = attn_weights.to(query.dtype).contiguous()
+            else:
+                key = dequantize_cache_torch(qk, qk_scale, qk_zero)
+                attn_weights = torch.matmul(query, key.transpose(-1, -2))
+        else:
+            attn_weights = torch.matmul(query, key.transpose(-1, -2))
+
+        if self.scale_attn_weights:
+            if self.use_cache_quantization:
+                size_temp = value[0].size(-1)
+            else:
+                size_temp = value.size(-1)
+            attn_weights = attn_weights / torch.full(
+                [],
+                size_temp ** 0.5,
+                dtype=attn_weights.dtype,
+                device=attn_weights.device,
+            )
+        if self.use_cache_quantization:
+            query_length, key_length = query.size(-2), key[0].size(-2)
+        else:
+            query_length, key_length = query.size(-2), key.size(-2)
+        causal_mask = registered_causal_mask[
+            :, :, key_length - query_length : key_length, :key_length
+        ]
+        mask_value = torch.finfo(attn_weights.dtype).min
+        mask_value = torch.full([], mask_value, dtype=attn_weights.dtype).to(
+            attn_weights.device
+        )
+        attn_weights = torch.where(
+            causal_mask, attn_weights.to(attn_weights.dtype), mask_value
+        )
+
+        if attention_mask is not None:
+            attn_weights = attn_weights + attention_mask
+
+        if self.softmax_in_fp32:
+            attn_weights = nn.functional.softmax(attn_weights.float(), dim=-1)
+        else:
+            attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        attn_weights = attn_weights.type(query.dtype)
+        attn_weights = self.attn_dropout(attn_weights)
+
+        if head_mask is not None:
+            attn_weights = attn_weights * head_mask
+
+        if self.use_cache_quantization:
+            qv, qv_scale, qv_zero = value
+            if self.use_cache_kernel and self.cache_kernels is not None:
+                shape = attn_weights.shape[:-1] + (query.shape[-1],)
+                attn_output = torch.zeros(shape, dtype=torch.float16, device=device)
+                self.cache_kernels.vecquant8matmul_batched_column_compression_faster_old(
+                    attn_weights.contiguous() if attn_weights.dtype == torch.float16 else attn_weights.to(torch.float16).contiguous(),
+                    qv.contiguous(),  # dtype: int32
+                    attn_output,
+                    qv_scale.contiguous() if qv_scale.dtype == torch.float16 else qv_scale.to(torch.float16).contiguous(),
+                    qv_zero.contiguous() if qv_zero.dtype == torch.float16 else qv_zero.to(torch.float16).contiguous())
+                if attn_output.dtype != query.dtype:
+                    attn_output = attn_output.to(query.dtype)
+                    attn_weights = attn_weights.to(query.dtype)
+            else:
+                value = dequantize_cache_torch(qv, qv_scale, qv_zero)
+                attn_output = torch.matmul(attn_weights, value)
+        else:
+            attn_output = torch.matmul(attn_weights, value)
+
+        attn_output = attn_output.transpose(1, 2)
+
+        return attn_output, attn_weights
+
+    def _upcast_and_reordered_attn(
+        self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None
+    ):
+        bsz, num_heads, q_seq_len, dk = query.size()
+        _, _, k_seq_len, _ = key.size()
+
+        attn_weights = torch.empty(
+            bsz * num_heads,
+            q_seq_len,
+            k_seq_len,
+            dtype=torch.float32,
+            device=query.device,
+        )
+
+        scale_factor = 1.0
+        if self.scale_attn_weights:
+            scale_factor /= float(value.size(-1)) ** 0.5
+
+        with autocast(enabled=False):
+            q, k = query.reshape(-1, q_seq_len, dk), key.transpose(-1, -2).reshape(
+                -1, dk, k_seq_len
+            )
+            attn_weights = torch.baddbmm(
+                attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor
+            )
+            attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, k_seq_len)
+
+        query_length, key_length = query.size(-2), key.size(-2)
+        causal_mask = registered_causal_mask[
+            :, :, key_length - query_length : key_length, :key_length
+        ]
+        mask_value = torch.finfo(attn_weights.dtype).min
+        mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(
+            attn_weights.device
+        )
+        attn_weights = torch.where(causal_mask, attn_weights, mask_value)
+
+        if attention_mask is not None:
+            attn_weights = attn_weights + attention_mask
+
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        if attn_weights.dtype != torch.float32:
+            raise RuntimeError(
+                "Error with upcasting, attn_weights does not have dtype torch.float32"
+            )
+        attn_weights = attn_weights.type(value.dtype)
+        attn_weights = self.attn_dropout(attn_weights)
+
+        if head_mask is not None:
+            attn_weights = attn_weights * head_mask
+
+        attn_output = torch.matmul(attn_weights, value)
+
+        return attn_output, attn_weights
+
+    def _split_heads(self, tensor, num_heads, attn_head_size):
+        new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
+        tensor = tensor.view(new_shape)
+        return tensor
+
+    def _merge_heads(self, tensor, num_heads, attn_head_size):
+        tensor = tensor.contiguous()
+        new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
+        return tensor.view(new_shape)
+
+    def forward(
+        self,
+        hidden_states: Optional[Tuple[torch.FloatTensor]],
+        rotary_pos_emb_list: Optional[List[List[torch.Tensor]]] = None,
+        registered_causal_mask: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+    ):
+        mixed_x_layer = self.c_attn(hidden_states)
+
+        query, key, value = mixed_x_layer.split(self.split_size, dim=2)
+
+        query = self._split_heads(query, self.num_heads, self.head_dim)
+        key = self._split_heads(key, self.num_heads, self.head_dim)
+        value = self._split_heads(value, self.num_heads, self.head_dim)
+
+        if rotary_pos_emb_list is not None:
+            cur_len = query.shape[1]
+            if len(rotary_pos_emb_list) == 1:
+                rotary_pos_emb = rotary_pos_emb_list[0]
+                rotary_pos_emb = [i[:, -cur_len:, :, :] for i in rotary_pos_emb]
+                rotary_pos_emb = (rotary_pos_emb,) * 2
+                q_pos_emb, k_pos_emb = rotary_pos_emb
+                # Slice the pos emb for current inference
+                query = apply_rotary_pos_emb(query, q_pos_emb)
+                key = apply_rotary_pos_emb(key, k_pos_emb)
+            else:
+                query_list = []
+                key_list = []
+                for i, rotary_pos_emb in enumerate(rotary_pos_emb_list):
+                    rotary_pos_emb = [i[:, -cur_len:, :, :] for i in rotary_pos_emb]
+                    rotary_pos_emb = (rotary_pos_emb,) * 2
+                    q_pos_emb, k_pos_emb = rotary_pos_emb
+                    # Slice the pos emb for current inference
+                    query_list += [apply_rotary_pos_emb(query[i:i+1, :, :], q_pos_emb)]
+                    key_list += [apply_rotary_pos_emb(key[i:i+1, :, :], k_pos_emb)]
+                query = torch.cat(query_list, dim=0)
+                key = torch.cat(key_list, dim=0)
+
+        if self.use_cache_quantization:
+            key = quantize_cache_v(key.permute(0, 2, 1, 3),
+                                       bits=8,
+                                       qmin=self.cache_qmin,
+                                       qmax=self.cache_qmax)
+            value = quantize_cache_v(value.permute(0, 2, 1, 3),
+                                         bits=8,
+                                         qmin=self.cache_qmin,
+                                         qmax=self.cache_qmax)
+
+
+        if layer_past is not None:
+            past_key, past_value = layer_past[0], layer_past[1]
+            if self.use_cache_quantization:
+                # use_cache_quantization:
+                # present=((q_key,key_scale,key_zero_point),
+                #          (q_value,value_scale,value_zero_point))
+                key = (torch.cat((past_key[0], key[0]), dim=2),
+                       torch.cat((past_key[1], key[1]), dim=2),
+                       torch.cat((past_key[2], key[2]), dim=2))
+                value = (torch.cat((past_value[0], value[0]), dim=2),
+                         torch.cat((past_value[1], value[1]), dim=2),
+                         torch.cat((past_value[2], value[2]), dim=2))
+            else:
+                # not use_cache_quantization:
+                # present=(key,value)
+                key = torch.cat((past_key, key), dim=1)
+                value = torch.cat((past_value, value), dim=1)
+
+        if use_cache:
+            present = (key, value)
+        else:
+            present = None
+
+        if self.use_logn_attn and not self.training:
+            if self.use_cache_quantization:
+                seq_start = key[0].size(2) - query.size(1)
+                seq_end = key[0].size(2)
+            else:
+                seq_start = key.size(1) - query.size(1)
+                seq_end = key.size(1)
+            logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :].type_as(query)
+            query = query * logn_tensor.expand_as(query)
+
+        if (
+            self.use_flash_attn
+            and flash_attn_unpadded_func is not None
+            and not self.is_fp32
+            and query.is_cuda
+        ):
+            q, k, v = query, key, value
+            attn_output = self.core_attention_flash(q, k, v, attention_mask=attention_mask)
+        else:
+            query = query.permute(0, 2, 1, 3)
+            if not self.use_cache_quantization:
+                key = key.permute(0, 2, 1, 3)
+                value = value.permute(0, 2, 1, 3)
+            if (
+                registered_causal_mask is None
+                and self.use_flash_attn
+                and flash_attn_unpadded_func is not None
+                and not self.is_fp32
+                and not query.is_cuda
+            ):
+                raise Exception(_ERROR_INPUT_CPU_QUERY_WITH_FLASH_ATTN_ACTIVATED)
+
+            if not self.use_cache_quantization and SUPPORT_TORCH2:
+                causal_mask = registered_causal_mask[
+                    :, :, key.size(-2) - query.size(-2): key.size(-2), :key.size(-2)
+                ]
+                if attention_mask is not None:
+                    attention_mask = attention_mask.expand(
+                        -1, -1, causal_mask.size(2), -1
+                    ).masked_fill(~causal_mask, torch.finfo(query.dtype).min)
+                else:
+                    attention_mask = causal_mask
+                attn_output = F.scaled_dot_product_attention(
+                    query, key, value, attn_mask=attention_mask
+                ).transpose(1, 2)
+                attn_weight = None
+            else:
+                attn_output, attn_weight = self._attn(
+                    query, key, value, registered_causal_mask, attention_mask, head_mask
+                )
+        context_layer = self._merge_heads(
+            attn_output, self.num_heads, self.head_dim
+        )
+
+        attn_output = self.c_proj(context_layer)
+
+        outputs = (attn_output, present)
+        if output_attentions:
+            if (
+                self.use_flash_attn
+                and flash_attn_unpadded_func is not None
+                and not self.is_fp32
+            ):
+                raise ValueError("Cannot output attentions while using flash-attn")
+            else:
+                outputs += (attn_weight,)
+
+        return outputs
+
+
+class QWenMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.w1 = nn.Linear(
+            config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
+        )
+        self.w2 = nn.Linear(
+            config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
+        )
+        ff_dim_in = config.intermediate_size // 2
+        self.c_proj = nn.Linear(ff_dim_in, config.hidden_size, bias=not config.no_bias)
+
+    def forward(self, hidden_states):
+        a1 = self.w1(hidden_states)
+        a2 = self.w2(hidden_states)
+        intermediate_parallel = a1 * F.silu(a2)
+        output = self.c_proj(intermediate_parallel)
+        return output
+
+class QWenBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        hidden_size = config.hidden_size
+        self.bf16 = config.bf16
+
+        self.ln_1 = RMSNorm(
+            hidden_size,
+            eps=config.layer_norm_epsilon,
+        )
+        self.attn = QWenAttention(config)
+        self.ln_2 = RMSNorm(
+            hidden_size,
+            eps=config.layer_norm_epsilon,
+        )
+
+        self.mlp = QWenMLP(config)
+
+    def forward(
+        self,
+        hidden_states: Optional[Tuple[torch.FloatTensor]],
+        rotary_pos_emb_list: Optional[List[List[torch.Tensor]]] = None,
+        registered_causal_mask: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = False,
+        output_attentions: Optional[bool] = False,
+    ):
+        layernorm_output = self.ln_1(hidden_states)
+
+        attn_outputs = self.attn(
+            layernorm_output,
+            rotary_pos_emb_list,
+            registered_causal_mask=registered_causal_mask,
+            layer_past=layer_past,
+            attention_mask=attention_mask,
+            head_mask=head_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+        attn_output = attn_outputs[0]
+
+        outputs = attn_outputs[1:]
+
+        residual = hidden_states
+        layernorm_input = attn_output + residual
+
+        layernorm_output = self.ln_2(layernorm_input)
+
+        residual = layernorm_input
+        mlp_output = self.mlp(layernorm_output)
+        hidden_states = residual + mlp_output
+
+        if use_cache:
+            outputs = (hidden_states,) + outputs
+        else:
+            outputs = (hidden_states,) + outputs[1:]
+
+        return outputs
+
+
+class QWenPreTrainedModel(PreTrainedModel):
+    config_class = QWenConfig
+    base_model_prefix = "transformer"
+    is_parallelizable = False
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["QWenBlock"]
+
+    def __init__(self, *inputs, **kwargs):
+        super().__init__(*inputs, **kwargs)
+
+    def _init_weights(self, module):
+        """Initialize the weights."""
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, RMSNorm):
+            module.weight.data.fill_(1.0)
+
+        for name, p in module.named_parameters():
+            if name == "c_proj.weight":
+                p.data.normal_(
+                    mean=0.0,
+                    std=(
+                        self.config.initializer_range
+                        / math.sqrt(2 * self.config.num_hidden_layers)
+                    ),
+                )
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, QWenModel):
+            module.gradient_checkpointing = value
+
+
+class QWenModel(QWenPreTrainedModel):
+    _keys_to_ignore_on_load_missing = ["attn.masked_bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.vocab_size = config.vocab_size
+        self.num_hidden_layers = config.num_hidden_layers
+        self.embed_dim = config.hidden_size
+        self.use_cache_quantization = self.config.use_cache_quantization if hasattr(self.config, 'use_cache_quantization') else False
+
+        self.gradient_checkpointing = False
+        self.use_dynamic_ntk = config.use_dynamic_ntk
+        self.seq_length = config.seq_length
+
+        self.wte = nn.Embedding(self.vocab_size, self.embed_dim)
+
+        self.drop = nn.Dropout(config.emb_dropout_prob)
+
+        if config.rotary_pct == 1.0:
+            self.rotary_ndims = None
+        else:
+            assert config.rotary_pct < 1
+            self.rotary_ndims = int(
+                config.kv_channels * config.rotary_pct
+            )
+        dim = (
+            self.rotary_ndims
+            if self.rotary_ndims is not None
+            else config.kv_channels
+        )
+        self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base)
+
+        self.use_flash_attn = config.use_flash_attn
+        self.is_fp32 = not (config.bf16 or config.fp16)
+        if (
+            self.use_flash_attn
+            and flash_attn_unpadded_func is not None
+            and not self.is_fp32
+        ):
+            self.registered_causal_mask = None
+        else:
+            max_positions = config.max_position_embeddings
+            self.register_buffer(
+                "registered_causal_mask",
+                torch.tril(
+                    torch.ones((max_positions, max_positions), dtype=torch.bool)
+                ).view(1, 1, max_positions, max_positions),
+                persistent=False,
+            )
+
+        self.h = nn.ModuleList(
+            [
+                QWenBlock(
+                    config
+                )
+                for i in range(config.num_hidden_layers)
+            ]
+        )
+        self.ln_f = RMSNorm(
+            self.embed_dim,
+            eps=config.layer_norm_epsilon,
+        )
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+
+    def get_ntk_alpha(self, true_seq_len):
+        context_value = math.log(true_seq_len / self.seq_length, 2) + 1
+        ntk_alpha = 2 ** math.ceil(context_value) - 1
+        ntk_alpha = max(ntk_alpha, 1)
+        return ntk_alpha
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ):
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+            batch_size = input_ids.shape[0]
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size = inputs_embeds.shape[0]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if token_type_ids is not None:
+            token_type_ids = token_type_ids.view(-1, input_shape[-1])
+        if position_ids is not None:
+            position_ids = position_ids.view(-1, input_shape[-1])
+
+        if past_key_values is None:
+            past_length = 0
+            past_key_values = tuple([None] * len(self.h))
+        else:
+            if self.use_cache_quantization:
+                past_length = past_key_values[0][0][0].size(2)
+            else:
+                past_length = past_key_values[0][0].size(-2)
+        if position_ids is None:
+            position_ids = torch.arange(
+                past_length,
+                input_shape[-1] + past_length,
+                dtype=torch.long,
+                device=device,
+            )
+            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
+
+        if attention_mask is not None:
+            if batch_size <= 0:
+                raise ValueError("batch_size has to be defined and > 0")
+            attention_mask = attention_mask.view(batch_size, -1)
+            attention_mask = attention_mask[:, None, None, :]
+            attention_mask = attention_mask.to(dtype=self.dtype)
+            attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
+
+        encoder_attention_mask = None
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
+        hidden_states = inputs_embeds
+
+        kv_seq_len = hidden_states.size()[1]
+        if past_key_values[0] is not None:
+            # past key values[0][0] shape: bs * seq_len * head_num * dim
+            if self.use_cache_quantization:
+                kv_seq_len += past_key_values[0][0][0].shape[2]
+            else:
+                kv_seq_len += past_key_values[0][0].shape[1]
+
+        if self.training or not self.use_dynamic_ntk:
+            ntk_alpha_list = [1.0]
+        elif kv_seq_len != hidden_states.size()[1]:
+            ntk_alpha_list = self.rotary_emb._ntk_alpha_cached_list
+        else:
+            ntk_alpha_list = []
+            if attention_mask is not None and kv_seq_len > self.seq_length:
+                true_seq_lens = attention_mask.squeeze(1).squeeze(1).eq(0).sum(dim=-1, dtype=torch.int32)
+                for i in range(hidden_states.size()[0]):
+                    true_seq_len = true_seq_lens[i].item()
+                    ntk_alpha = self.get_ntk_alpha(true_seq_len)
+                    ntk_alpha_list.append(ntk_alpha)
+            else:
+                ntk_alpha = self.get_ntk_alpha(kv_seq_len)
+                ntk_alpha_list.append(ntk_alpha)
+        self.rotary_emb._ntk_alpha_cached_list = ntk_alpha_list
+        rotary_pos_emb_list = [
+            self.rotary_emb(kv_seq_len, ntk_alpha=ntk_alpha) for ntk_alpha in ntk_alpha_list
+        ]
+
+        hidden_states = self.drop(hidden_states)
+        output_shape = input_shape + (hidden_states.size(-1),)
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        presents = () if use_cache else None
+        all_self_attentions = () if output_attentions else None
+        all_hidden_states = () if output_hidden_states else None
+        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
+
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        # None for past_key_value
+                        return module(*inputs, use_cache, output_attentions)
+
+                    return custom_forward
+
+                outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(block),
+                    hidden_states,
+                    rotary_pos_emb_list,
+                    self.registered_causal_mask,
+                    None,
+                    attention_mask,
+                    head_mask[i],
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                )
+            else:
+                outputs = block(
+                    hidden_states,
+                    layer_past=layer_past,
+                    rotary_pos_emb_list=rotary_pos_emb_list,
+                    registered_causal_mask=self.registered_causal_mask,
+                    attention_mask=attention_mask,
+                    head_mask=head_mask[i],
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_attention_mask,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = outputs[0]
+            if use_cache is True:
+                presents = presents + (outputs[1],)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
+
+        hidden_states = self.ln_f(hidden_states)
+        hidden_states = hidden_states.view(output_shape)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v for v in [hidden_states, presents, all_hidden_states] if v is not None
+            )
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+class QWenLMHeadModel(QWenPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.rotary_emb\.inv_freq"]
+    _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.attn\.masked_bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        assert (
+            config.bf16 + config.fp16 + config.fp32 <= 1
+        ), "Only one of \"bf16\", \"fp16\", \"fp32\" can be true"
+        logger.warn(
+            "Warning: please make sure that you are using the latest codes and checkpoints, "
+            "especially if you used Qwen-7B before 09.25.2023."
+            "请使用最新模型和代码，尤其如果你在9月25日前已经开始使用Qwen-7B，千万注意不要使用错误代码和模型。"
+        )
+
+        autoset_precision = config.bf16 + config.fp16 + config.fp32 == 0
+
+        if autoset_precision:
+            if SUPPORT_BF16:
+                logger.warn(
+                    "The model is automatically converting to bf16 for faster inference. "
+                    "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
+                )
+                config.bf16 = True
+            elif SUPPORT_FP16:
+                logger.warn(
+                    "The model is automatically converting to fp16 for faster inference. "
+                    "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
+                )
+                config.fp16 = True
+            else:
+                config.fp32 = True
+
+        if config.bf16 and SUPPORT_CUDA and not SUPPORT_BF16:
+            logger.warn("Your device does NOT seem to support bf16, you can switch to fp16 or fp32 by by passing fp16/fp32=True in \"AutoModelForCausalLM.from_pretrained\".")
+        if config.fp16 and SUPPORT_CUDA and not SUPPORT_FP16:
+            logger.warn("Your device does NOT support faster inference with fp16, please switch to fp32 which is likely to be faster")
+        if config.fp32:
+            if SUPPORT_BF16:
+                logger.warn("Your device support faster inference by passing bf16=True in \"AutoModelForCausalLM.from_pretrained\".")
+            elif SUPPORT_FP16:
+                logger.warn("Your device support faster inference by passing fp16=True in \"AutoModelForCausalLM.from_pretrained\".")
+
+        if config.use_flash_attn == "auto":
+            if config.bf16 or config.fp16:
+                logger.warn("Try importing flash-attention for faster inference...")
+                config.use_flash_attn = True
+            else:
+                config.use_flash_attn = False
+        if config.use_flash_attn and config.fp32:
+            logger.warn("Flash attention will be disabled because it does NOT support fp32.")
+
+        if config.use_flash_attn:
+            _import_flash_attn()
+
+        self.transformer = QWenModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        if config.bf16:
+            self.transformer.bfloat16()
+            self.lm_head.bfloat16()
+        if config.fp16:
+            self.transformer.half()
+            self.lm_head.half()
+        self.post_init()
+
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs
+    ):
+        token_type_ids = kwargs.get("token_type_ids", None)
+        if past_key_values:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+            if token_type_ids is not None:
+                token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
+
+        attention_mask = kwargs.get("attention_mask", None)
+        position_ids = kwargs.get("position_ids", None)
+
+        if attention_mask is not None and position_ids is None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+        else:
+            position_ids = None
+
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "position_ids": position_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+        return model_inputs
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+
+        lm_logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(lm_logits.device)
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
+            )
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+        past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
+    ) -> Tuple[Tuple[torch.Tensor]]:
+
+        return tuple(
+            tuple(
+                past_state.index_select(0, beam_idx.to(past_state.device))
+                for past_state in layer_past
+            )
+            for layer_past in past_key_values
+        )
+
+    def chat(
+        self,
+        tokenizer: PreTrainedTokenizer,
+        query: str,
+        history: Optional[HistoryType],
+        system: str = "You are a helpful assistant.",
+        append_history: bool = True,
+        stream: Optional[bool] = _SENTINEL,
+        stop_words_ids: Optional[List[List[int]]] = None,
+        generation_config: Optional[GenerationConfig] = None,
+        **kwargs,
+    ) -> Tuple[str, HistoryType]:
+        generation_config = generation_config if generation_config is not None else self.generation_config
+
+        assert stream is _SENTINEL, _ERROR_STREAM_IN_CHAT
+        assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
+        if history is None:
+            history = []
+        if stop_words_ids is None:
+            stop_words_ids = []
+
+        max_window_size = kwargs.get('max_window_size', None)
+        if max_window_size is None:
+            max_window_size = generation_config.max_window_size
+        raw_text, context_tokens = make_context(
+            tokenizer,
+            query,
+            history=history,
+            system=system,
+            max_window_size=max_window_size,
+            chat_format=generation_config.chat_format,
+        )
+
+        stop_words_ids.extend(get_stop_words_ids(
+            generation_config.chat_format, tokenizer
+        ))
+        input_ids = torch.tensor([context_tokens]).to(self.device)
+        outputs = self.generate(
+                    input_ids,
+                    stop_words_ids=stop_words_ids,
+                    return_dict_in_generate=False,
+                    generation_config=generation_config,
+                    **kwargs,
+                )
+
+        response = decode_tokens(
+            outputs[0],
+            tokenizer,
+            raw_text_len=len(raw_text),
+            context_length=len(context_tokens),
+            chat_format=generation_config.chat_format,
+            verbose=False,
+            errors='replace'
+        )
+
+        if append_history:
+            history.append((query, response))
+
+        return response, history
+
+    def chat_stream(
+            self,
+            tokenizer: PreTrainedTokenizer,
+            query: str,
+            history: Optional[HistoryType],
+            system: str = "You are a helpful assistant.",
+            stop_words_ids: Optional[List[List[int]]] = None,
+            logits_processor: Optional[LogitsProcessorList] = None,
+            generation_config: Optional[GenerationConfig] = None,
+            **kwargs,
+    ) -> Generator[str, Any, None]:
+        generation_config = generation_config if generation_config is not None else self.generation_config
+        assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
+        if history is None:
+            history = []
+        if stop_words_ids is None:
+            stop_words_ids = []
+
+        max_window_size = kwargs.get('max_window_size', None)
+        if max_window_size is None:
+            max_window_size = generation_config.max_window_size
+        raw_text, context_tokens = make_context(
+            tokenizer,
+            query,
+            history=history,
+            system=system,
+            max_window_size=max_window_size,
+            chat_format=generation_config.chat_format,
+        )
+
+        stop_words_ids.extend(get_stop_words_ids(
+            generation_config.chat_format, tokenizer
+        ))
+        if stop_words_ids is not None:
+            stop_words_logits_processor = StopWordsLogitsProcessor(
+                stop_words_ids=stop_words_ids,
+                eos_token_id=generation_config.eos_token_id,
+            )
+            if logits_processor is None:
+                logits_processor = LogitsProcessorList([stop_words_logits_processor])
+            else:
+                logits_processor.append(stop_words_logits_processor)
+        input_ids = torch.tensor([context_tokens]).to(self.device)
+
+        from transformers_stream_generator.main import NewGenerationMixin, StreamGenerationConfig
+        self.__class__.generate_stream = NewGenerationMixin.generate
+        self.__class__.sample_stream = NewGenerationMixin.sample_stream
+        stream_config = StreamGenerationConfig(**generation_config.to_dict(), do_stream=True)
+
+        def stream_generator():
+            outputs = []
+            for token in self.generate_stream(
+                    input_ids,
+                    return_dict_in_generate=False,
+                    generation_config=stream_config,
+                    logits_processor=logits_processor,
+                    seed=-1,
+                    **kwargs):
+                outputs.append(token.item())
+                yield tokenizer.decode(outputs, skip_special_tokens=True, errors='ignore')
+
+        return stream_generator()
+
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        generation_config: Optional[GenerationConfig] = None,
+        logits_processor: Optional[LogitsProcessorList] = None,
+        stopping_criteria: Optional[StoppingCriteriaList] = None,
+        prefix_allowed_tokens_fn: Optional[
+            Callable[[int, torch.Tensor], List[int]]
+        ] = None,
+        synced_gpus: Optional[bool] = None,
+        assistant_model: Optional["PreTrainedModel"] = None,
+        streamer: Optional["BaseStreamer"] = None,
+        **kwargs,
+    ) -> Union[GenerateOutput, torch.LongTensor]:
+        generation_config = generation_config if generation_config is not None else self.generation_config
+
+        # Process stop_words_ids.
+        stop_words_ids = kwargs.pop("stop_words_ids", None)
+        if stop_words_ids is None and generation_config is not None:
+            stop_words_ids = getattr(generation_config, "stop_words_ids", None)
+        if stop_words_ids is None:
+            stop_words_ids = getattr(generation_config, "stop_words_ids", None)
+
+        if stop_words_ids is not None:
+            stop_words_logits_processor = StopWordsLogitsProcessor(
+                stop_words_ids=stop_words_ids,
+                eos_token_id=generation_config.eos_token_id,
+            )
+            if logits_processor is None:
+                logits_processor = LogitsProcessorList([stop_words_logits_processor])
+            else:
+                logits_processor.append(stop_words_logits_processor)
+
+        return super().generate(
+            inputs,
+            generation_config=generation_config,
+            logits_processor=logits_processor,
+            stopping_criteria=stopping_criteria,
+            prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
+            synced_gpus=synced_gpus,
+            assistant_model=assistant_model,
+            streamer=streamer,
+            **kwargs,
+        )
+
+
+class RotaryEmbedding(torch.nn.Module):
+    def __init__(self, dim, base=10000):
+        super().__init__()
+        self.dim = dim
+        self.base = base
+        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        if importlib.util.find_spec("einops") is None:
+            raise RuntimeError("einops is required for Rotary Embedding")
+
+        self._rotary_pos_emb_cache = None
+        self._seq_len_cached = 0
+        self._ntk_alpha_cached = 1.0
+        self._ntk_alpha_cached_list = [1.0]
+
+    def update_rotary_pos_emb_cache(self, max_seq_len, offset=0, ntk_alpha=1.0):
+        seqlen = max_seq_len + offset
+        if seqlen > self._seq_len_cached or ntk_alpha != self._ntk_alpha_cached:
+            base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
+            self.inv_freq = 1.0 / (
+                base
+                ** (
+                    torch.arange(0, self.dim, 2, device=self.inv_freq.device).float()
+                    / self.dim
+                )
+            )
+            self._seq_len_cached = max(2 * seqlen, 16)
+            self._ntk_alpha_cached = ntk_alpha
+            seq = torch.arange(self._seq_len_cached, device=self.inv_freq.device)
+            freqs = torch.outer(seq.type_as(self.inv_freq), self.inv_freq)
+
+            emb = torch.cat((freqs, freqs), dim=-1)
+            from einops import rearrange
+
+            emb = rearrange(emb, "n d -> 1 n 1 d")
+
+            cos, sin = emb.cos(), emb.sin()
+            self._rotary_pos_emb_cache = [cos, sin]
+
+    def forward(self, max_seq_len, offset=0, ntk_alpha=1.0):
+        self.update_rotary_pos_emb_cache(max_seq_len, offset, ntk_alpha)
+        cos, sin = self._rotary_pos_emb_cache
+        return [cos[:, offset : offset + max_seq_len], sin[:, offset : offset + max_seq_len]]
+
+
+def _rotate_half(x):
+    from einops import rearrange
+
+    x = rearrange(x, "... (j d) -> ... j d", j=2)
+    x1, x2 = x.unbind(dim=-2)
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(t, freqs):
+    cos, sin = freqs
+    if apply_rotary_emb_func is not None and t.is_cuda:
+        t_ = t.float()
+        cos = cos.squeeze(0).squeeze(1)[:, : cos.shape[-1] // 2]
+        sin = sin.squeeze(0).squeeze(1)[:, : sin.shape[-1] // 2]
+        output = apply_rotary_emb_func(t_, cos, sin).type_as(t)
+        return output
+    else:
+        rot_dim = freqs[0].shape[-1]
+        cos, sin = freqs
+        t_, t_pass_ = t[..., :rot_dim], t[..., rot_dim:]
+        t_ = t_.float()
+        t_pass_ = t_pass_.float()
+        t_ = (t_ * cos) + (_rotate_half(t_) * sin)
+        return torch.cat((t_, t_pass_), dim=-1).type_as(t)
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def _norm(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        if rms_norm is not None and x.is_cuda:
+            return rms_norm(x, self.weight, self.eps)
+        else:
+            output = self._norm(x.float()).type_as(x)
+            return output * self.weight

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d42dde5f24387cc72dd7320162aacbdde4376a2f01995d3c37e2f37c8ba8968
+size 9952260427

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b21d77e3223feb963f89263d4339f36ed7281e45f3bfe5c2aeb6c442d682dee
+size 6077579472

pytorch_model.bin.index.json

@@ -0,0 +1,1090 @@
+{
+  "metadata": {
+    "total_size": 16029460480
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.0.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.22.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.23.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.24.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.25.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.26.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.26.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.3.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.30.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.32.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.33.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.34.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.35.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.36.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.37.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.38.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_attn.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_attn.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_attn.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_attn.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.attn.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.c_proj.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.c_proj.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.c_proj.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.c_proj.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w1.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w1.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w1.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w1.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w2.g_idx": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w2.qweight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w2.qzeros": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.39.mlp.w2.scales": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.4.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.c_proj.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.c_proj.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.c_proj.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.c_proj.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w1.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w1.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w1.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w1.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w1.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w2.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w2.g_idx": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w2.qweight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w2.qzeros": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w2.scales": "pytorch_model-00001-of-00002.bin",
+    "transformer.ln_f.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.wte.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}

qwen_generation_utils.py

@@ -0,0 +1,416 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Generation support."""
+
+from typing import Tuple, List, Union, Iterable
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from transformers import PreTrainedTokenizer
+from transformers import logging
+from transformers.generation import LogitsProcessor
+
+logger = logging.get_logger(__name__)
+
+# Types.
+HistoryType = List[Tuple[str, str]]
+TokensType = List[int]
+BatchTokensType = List[List[int]]
+
+
+def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType:
+    for tokens in batch:
+        context_length = len(tokens)
+        if context_length < seq_length:
+            tokens.extend([pad_id] * (seq_length - context_length))
+    return batch
+
+
+def get_ltor_masks_and_position_ids(
+    data,
+    eod_token,
+    reset_position_ids,
+    reset_attention_mask,
+    eod_mask_loss,
+):
+    """Build masks and position id for left to right model."""
+
+    # Extract batch size and sequence length.
+    micro_batch_size, seq_length = data.size()
+
+    # Attention mask (lower triangular).
+    if reset_attention_mask:
+        att_mask_batch = micro_batch_size
+    else:
+        att_mask_batch = 1
+    attention_mask = torch.tril(
+        torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)
+    ).view(att_mask_batch, 1, seq_length, seq_length)
+
+    # Loss mask.
+    loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device)
+    if eod_mask_loss:
+        loss_mask[data == eod_token] = 0.0
+
+    # Position ids.
+    position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
+    position_ids = position_ids.unsqueeze(0).expand_as(data)
+    # We need to clone as the ids will be modifed based on batch index.
+    if reset_position_ids:
+        position_ids = position_ids.clone()
+
+    if reset_position_ids or reset_attention_mask:
+        # Loop through the batches:
+        for b in range(micro_batch_size):
+
+            # Find indecies where EOD token is.
+            eod_index = position_ids[b, data[b] == eod_token]
+            # Detach indecies from positions if going to modify positions.
+            if reset_position_ids:
+                eod_index = eod_index.clone()
+
+            # Loop through EOD indecies:
+            prev_index = 0
+            for j in range(eod_index.size()[0]):
+                i = eod_index[j]
+                # Mask attention loss.
+                if reset_attention_mask:
+                    attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0
+                # Reset positions.
+                if reset_position_ids:
+                    position_ids[b, (i + 1) :] -= i + 1 - prev_index
+                    prev_index = i + 1
+
+    # Convert attention mask to binary:
+    attention_mask = attention_mask < 0.5
+
+    return attention_mask, loss_mask, position_ids
+
+
+def get_batch(context_tokens: torch.LongTensor, eod_id: int):
+    """Generate batch from context tokens."""
+    # Move to GPU.
+    tokens = context_tokens.contiguous().to(context_tokens.device)
+    # Get the attention mask and postition ids.
+    attention_mask, _, position_ids = get_ltor_masks_and_position_ids(
+        tokens,
+        eod_id,
+        reset_position_ids=False,
+        reset_attention_mask=False,
+        eod_mask_loss=False,
+    )
+    return tokens, attention_mask, position_ids
+
+
+def get_stop_words_ids(chat_format, tokenizer):
+    if chat_format == "raw":
+        stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]]
+    elif chat_format == "chatml":
+        stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]]
+    else:
+        raise NotImplementedError(f"Unknown chat format {chat_format!r}")
+    return stop_words_ids
+
+
+def make_context(
+    tokenizer: PreTrainedTokenizer,
+    query: str,
+    history: List[Tuple[str, str]] = None,
+    system: str = "",
+    max_window_size: int = 6144,
+    chat_format: str = "chatml",
+):
+    if history is None:
+        history = []
+
+    if chat_format == "chatml":
+        im_start, im_end = "<|im_start|>", "<|im_end|>"
+        im_start_tokens = [tokenizer.im_start_id]
+        im_end_tokens = [tokenizer.im_end_id]
+        nl_tokens = tokenizer.encode("\n")
+
+        def _tokenize_str(role, content):
+            return f"{role}\n{content}", tokenizer.encode(
+                role, allowed_special=set()
+            ) + nl_tokens + tokenizer.encode(content, allowed_special=set())
+
+        system_text, system_tokens_part = _tokenize_str("system", system)
+        system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
+
+        raw_text = ""
+        context_tokens = []
+
+        for turn_query, turn_response in reversed(history):
+            query_text, query_tokens_part = _tokenize_str("user", turn_query)
+            query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
+            response_text, response_tokens_part = _tokenize_str(
+                "assistant", turn_response
+            )
+            response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
+
+            next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
+            prev_chat = (
+                f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
+            )
+
+            current_context_size = (
+                len(system_tokens) + len(next_context_tokens) + len(context_tokens)
+            )
+            if current_context_size < max_window_size:
+                context_tokens = next_context_tokens + context_tokens
+                raw_text = prev_chat + raw_text
+            else:
+                break
+
+        context_tokens = system_tokens + context_tokens
+        raw_text = f"{im_start}{system_text}{im_end}" + raw_text
+        context_tokens += (
+            nl_tokens
+            + im_start_tokens
+            + _tokenize_str("user", query)[1]
+            + im_end_tokens
+            + nl_tokens
+            + im_start_tokens
+            + tokenizer.encode("assistant")
+            + nl_tokens
+        )
+        raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
+
+    elif chat_format == "raw":
+        raw_text = query
+        context_tokens = tokenizer.encode(raw_text)
+    else:
+        raise NotImplementedError(f"Unknown chat format {chat_format!r}")
+
+    return raw_text, context_tokens
+
+
+def _decode_default(
+    tokens: List[int],
+    *,
+    stop_words: List[str],
+    eod_words: List[str],
+    tokenizer: PreTrainedTokenizer,
+    raw_text_len: int,
+    verbose: bool = False,
+    return_end_reason: bool = False,
+    errors: str='replace',
+):
+    trim_decode_tokens = tokenizer.decode(tokens, errors=errors)[raw_text_len:]
+    if verbose:
+        print("\nRaw Generate: ", trim_decode_tokens)
+
+    end_reason = f"Gen length {len(tokens)}"
+    for stop_word in stop_words:
+        trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
+    for eod_word in eod_words:
+        if eod_word in trim_decode_tokens:
+            end_reason = f"Gen {eod_word!r}"
+        trim_decode_tokens = trim_decode_tokens.split(eod_word)[0]
+    trim_decode_tokens = trim_decode_tokens.strip()
+    if verbose:
+        print("\nEnd Reason:", end_reason)
+        print("\nGenerate: ", trim_decode_tokens)
+
+    if return_end_reason:
+        return trim_decode_tokens, end_reason
+    else:
+        return trim_decode_tokens
+
+
+def _decode_chatml(
+    tokens: List[int],
+    *,
+    stop_words: List[str],
+    eod_token_ids: List[int],
+    tokenizer: PreTrainedTokenizer,
+    raw_text_len: int,
+    context_length: int,
+    verbose: bool = False,
+    return_end_reason: bool = False,
+    errors: str='replace'
+):
+    end_reason = f"Gen length {len(tokens)}"
+    eod_token_idx = context_length
+    for eod_token_idx in range(context_length, len(tokens)):
+        if tokens[eod_token_idx] in eod_token_ids:
+            end_reason = f"Gen {tokenizer.decode([tokens[eod_token_idx]])!r}"
+            break
+
+    trim_decode_tokens = tokenizer.decode(tokens[:eod_token_idx], errors=errors)[raw_text_len:]
+    if verbose:
+        print("\nRaw Generate w/o EOD:", tokenizer.decode(tokens, errors=errors)[raw_text_len:])
+        print("\nRaw Generate:", trim_decode_tokens)
+        print("\nEnd Reason:", end_reason)
+    for stop_word in stop_words:
+        trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
+    trim_decode_tokens = trim_decode_tokens.strip()
+    if verbose:
+        print("\nGenerate:", trim_decode_tokens)
+
+    if return_end_reason:
+        return trim_decode_tokens, end_reason
+    else:
+        return trim_decode_tokens
+
+
+def decode_tokens(
+    tokens: Union[torch.LongTensor, TokensType],
+    tokenizer: PreTrainedTokenizer,
+    raw_text_len: int,
+    context_length: int,
+    chat_format: str,
+    verbose: bool = False,
+    return_end_reason: bool = False,
+    errors: str="replace",
+) -> str:
+    if torch.is_tensor(tokens):
+        tokens = tokens.cpu().numpy().tolist()
+
+    if chat_format == "chatml":
+        return _decode_chatml(
+            tokens,
+            stop_words=[],
+            eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id],
+            tokenizer=tokenizer,
+            raw_text_len=raw_text_len,
+            context_length=context_length,
+            verbose=verbose,
+            return_end_reason=return_end_reason,
+            errors=errors,
+        )
+    elif chat_format == "raw":
+        return _decode_default(
+            tokens,
+            stop_words=["<|endoftext|>"],
+            eod_words=["<|endoftext|>"],
+            tokenizer=tokenizer,
+            raw_text_len=raw_text_len,
+            verbose=verbose,
+            return_end_reason=return_end_reason,
+            errors=errors,
+        )
+    else:
+        raise NotImplementedError(f"Unknown chat format {chat_format!r}")
+
+
+class StopWordsLogitsProcessor(LogitsProcessor):
+    """
+    :class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration.
+
+    Args:
+        stop_words_ids (:obj:`List[List[int]]`):
+            List of list of token ids of stop ids. In order to get the tokens of the words
+            that should not appear in the generated text, use :obj:`tokenizer(bad_word,
+            add_prefix_space=True).input_ids`.
+        eos_token_id (:obj:`int`):
+            The id of the `end-of-sequence` token.
+    """
+
+    def __init__(self, stop_words_ids: Iterable[Iterable[int]], eos_token_id: int):
+
+        if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0:
+            raise ValueError(
+                f"`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}."
+            )
+        if any(not isinstance(bad_word_ids, list) for bad_word_ids in stop_words_ids):
+            raise ValueError(
+                f"`stop_words_ids` has to be a list of lists, but is {stop_words_ids}."
+            )
+        if any(
+            any(
+                (not isinstance(token_id, (int, np.integer)) or token_id < 0)
+                for token_id in stop_word_ids
+            )
+            for stop_word_ids in stop_words_ids
+        ):
+            raise ValueError(
+                f"Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}."
+            )
+
+        self.stop_words_ids = list(
+            filter(
+                lambda bad_token_seq: bad_token_seq != [eos_token_id], stop_words_ids
+            )
+        )
+        self.eos_token_id = eos_token_id
+        for stop_token_seq in self.stop_words_ids:
+            assert (
+                len(stop_token_seq) > 0
+            ), "Stop words token sequences {} cannot have an empty list".format(
+                stop_words_ids
+            )
+
+    def __call__(
+        self, input_ids: torch.LongTensor, scores: torch.FloatTensor
+    ) -> torch.FloatTensor:
+        stopped_samples = self._calc_stopped_samples(input_ids)
+        for i, should_stop in enumerate(stopped_samples):
+            if should_stop:
+                scores[i, self.eos_token_id] = float(2**15)
+        return scores
+
+    def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool:
+        if len(tokens) == 0:
+            # if bad word tokens is just one token always ban it
+            return True
+        elif len(tokens) > len(prev_tokens):
+            # if bad word tokens are longer then prev input_ids they can't be equal
+            return False
+        elif prev_tokens[-len(tokens) :].tolist() == tokens:
+            # if tokens match
+            return True
+        else:
+            return False
+
+    def _calc_stopped_samples(self, prev_input_ids: Iterable[int]) -> Iterable[int]:
+        stopped_samples = []
+        for prev_input_ids_slice in prev_input_ids:
+            match = False
+            for stop_token_seq in self.stop_words_ids:
+                if self._tokens_match(prev_input_ids_slice, stop_token_seq):
+                    # if tokens do not match continue
+                    match = True
+                    break
+            stopped_samples.append(match)
+
+        return stopped_samples
+
+
+def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float("Inf")):
+    """This function has been mostly taken from huggingface conversational
+    ai code at
+        https://medium.com/huggingface/how-to-build-a-state-of-the-art-
+             conversational-ai-with-transfer-learning-2d818ac26313"""
+
+    if top_k > 0:
+        # Remove all tokens with a probability less than the
+        # last token of the top-k
+        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
+        logits[indices_to_remove] = filter_value
+
+    if top_p > 0.0:
+        # Cconvert to 1D
+        sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
+        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
+
+        # Remove tokens with cumulative probability above the threshold
+        sorted_indices_to_remove = cumulative_probs > top_p
+        # Shift the indices to the right to keep also the first token
+        # above the threshold
+        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+        sorted_indices_to_remove[..., 0] = 0
+        for i in range(sorted_indices.size(0)):
+            indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]]
+            logits[i][indices_to_remove] = filter_value
+
+    return logits
+
+
+def switch(val1, val2, boolean):
+    boolean = boolean.type_as(val1)
+    return (1 - boolean) * val1 + boolean * val2