diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..41478957aca7a04b7321022e7d1f73de5badd995 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,103 @@ +[gMASK] +{%- if tools -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{{ tool | tojson(ensure_ascii=False) }} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name} +{arg-key-1} +{arg-value-1} +{arg-key-2} +{arg-value-2} +... +{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) %} +{%- for m in messages %} + {%- if m.role == 'user' %} + {% set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{% for m in messages %} +{%- if m.role == 'user' -%}<|user|> +{{ visible_text(m.content) }} +{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set reasoning_content = '' %} +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} +{%- endif %} +{%- if loop.index0 > ns.last_user_index and reasoning_content -%} +{{ '\n' + reasoning_content.strip() + ''}} +{%- else -%} +{{ '\n' }} +{%- endif -%} +{%- if content.strip() -%} +{{ '\n' + content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{ '\n' + tc.name }} +{% set _args = tc.arguments %} +{% for k, v in _args.items() %} +{{ k }} +{{ v | tojson(ensure_ascii=False) if v is not string else v }} +{% endfor %} +{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if m.content is string -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' }} +{%- endif %} +{{- '\n\n' }} +{{- m.content }} +{{- '\n' }} +{%- else -%} +<|observation|>{% for tr in m.content %} + + +{{ tr.output if tr.output is defined else tr }} +{% endfor -%} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|> +{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '\n' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e2d5f6a4f545da3e3ac5fafb233d75ccc1acf73 --- /dev/null +++ b/config.json @@ -0,0 +1,48 @@ +{ + "architectures": [ + "Glm4SharedMoeForCausalLM" + ], + "attention_bias": true, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_glm4_shared_moe.Glm4SharedMoeConfig", + "AutoModel": "modeling_glm4_shared_moe.Glm4SharedMoeModel", + "AutoModelForCausalLM": "modeling_glm4_shared_moe.Glm4SharedMoeForCausalLM" + }, + "eos_token_id": [ + 151329, + 151336, + 151338 + ], + "first_k_dense_replace": 3, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 131072, + "model_type": "glm4_shared_moe", + "moe_intermediate_size": 1536, + "n_group": 1, + "n_routed_experts": 160, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 96, + "num_experts_per_tok": 8, + "num_hidden_layers": 92, + "num_key_value_heads": 8, + "num_nextn_predict_layers": 1, + "pad_token_id": 151329, + "partial_rotary_factor": 0.5, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 1000000, + "routed_scaling_factor": 2.5, + "tie_word_embeddings": false, + "topk_group": 1, + "torch_dtype": "bfloat16", + "transformers_version": "4.55.2", + "use_cache": true, + "use_qk_norm": true, + "vocab_size": 151552 +} diff --git a/configuration_glm4_shared_moe.py b/configuration_glm4_shared_moe.py new file mode 100644 index 0000000000000000000000000000000000000000..c273c214275486b5e9cbf02c99e771f8642e679a --- /dev/null +++ b/configuration_glm4_shared_moe.py @@ -0,0 +1,236 @@ +# coding=utf-8 +# Copyright 2025 The ZhipuAI Inc. team and HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_rope_utils import rope_config_validation + + +class Glm4SharedMoeConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Glm4MoeModel`]. It is used to instantiate a + Glm4Moe model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of [THUDM/GLM-4-100B-A10B](https://huggingface.co/THUDM/GLM-4-100B-A10B). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 151552): + Vocabulary size of the Glm4Moe model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Glm4MoeModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 10944): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 46): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 96): + Number of attention heads for each attention layer in the Transformer encoder. + partial_rotary_factor (`float`, *optional*, defaults to 0.5): + The factor of the partial rotary position. + num_key_value_heads (`int`, *optional*, defaults to 8): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details, check out [this + paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`. + + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 131072): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (`float`, *optional*): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (`int`, *optional*): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (`float`, *optional*): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (`list[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (`list[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + moe_intermediate_size (`int`, *optional*, defaults to 1408): + Intermediate size of the routed expert. + num_experts_per_tok (`int`, *optional*, defaults to 8): + number of experts per token. + n_shared_experts (`int`, *optional*, defaults to 1): + Number of shared experts. + n_routed_experts (`int`, *optional*, defaults to 128): + Number of routed experts. + routed_scaling_factor (`float`, *optional*, defaults to 1.0): + Scaling factor or routed experts. + n_group (`int`, *optional*, defaults to 1): + Number of groups for routed experts. + topk_group (`int`, *optional*, defaults to 1): + Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups). + first_k_dense_replace (`int`, *optional*, defaults to 1): + Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head). + \--k dense layers--/ + norm_topk_prob (`bool`, *optional*, defaults to `True`): + Whether to normalize the topk probabilities. + use_qk_norm (`bool`, *optional*, defaults to `False`): + Whether to use query-key normalization in the attention + ```python + >>> from transformers import Glm4MoeModel, Glm4MoeConfig + + >>> # Initializing a Glm4Moe style configuration + >>> configuration = Glm4MoeConfig() + + >>> # Initializing a model from the GLM-4-MOE-100B-A10B style configuration + >>> model = Glm4MoeModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "glm4_shared_moe" + keys_to_ignore_at_inference = ["past_key_values"] + + # Default tensor parallel plan for base model `Glm4Moe` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.experts.*.gate_proj": "colwise", + "layers.*.mlp.experts.*.up_proj": "colwise", + "layers.*.mlp.experts.*.down_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + def __init__( + self, + vocab_size=151552, + hidden_size=4096, + intermediate_size=10944, + num_hidden_layers=46, + num_attention_heads=96, + partial_rotary_factor=0.5, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=131072, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + moe_intermediate_size=1408, + num_experts_per_tok=8, + n_shared_experts=1, + n_routed_experts=128, + routed_scaling_factor=1.0, + n_group=1, + topk_group=1, + first_k_dense_replace=1, + norm_topk_prob=True, + use_qk_norm=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.partial_rotary_factor = partial_rotary_factor + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + # Validate the correctness of rotary position embeddings parameters + # BC: if there is a 'type' field, move it to 'rope_type'. + if self.rope_scaling is not None and "type" in self.rope_scaling: + self.rope_scaling["rope_type"] = self.rope_scaling["type"] + rope_config_validation(self) + + # MoE arguments + self.moe_intermediate_size = moe_intermediate_size + self.num_experts_per_tok = num_experts_per_tok + self.n_group = n_group + self.topk_group = topk_group + self.n_shared_experts = n_shared_experts + self.n_routed_experts = n_routed_experts + self.routed_scaling_factor = routed_scaling_factor + self.first_k_dense_replace = first_k_dense_replace + self.norm_topk_prob = norm_topk_prob + self.use_qk_norm = use_qk_norm + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +__all__ = ["Glm4SharedMoeConfig"] diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..93ede1955db44f7c7ff5d43756ed093d5263004c --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 151329, + 151336, + 151338 + ], + "pad_token_id": 151329, + "transformers_version": "4.55.2" +} diff --git a/model-00001-of-00179.safetensors b/model-00001-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7f074a753b908c28a697de32947ae3960b4c2121 --- /dev/null +++ b/model-00001-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:342c1f2d267f073e34ebbf09b7aba09266b874fa762995d0b393e0550c851e97 +size 5033164960 diff --git a/model-00002-of-00179.safetensors b/model-00002-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..82f6b49742eb56d8106d80660903569cc9820f6b --- /dev/null +++ b/model-00002-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d34e655aef934f37bce12187fb973d90d655a6f135eddbf71e3d503d22f8bef +size 3775057728 diff --git a/model-00003-of-00179.safetensors b/model-00003-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..845ef9a045864189dbab7b3a97e481a666cf27a7 --- /dev/null +++ b/model-00003-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48b6aa05cb17f13981845f088c8e2e6b6daa89e345078cf30fbb5905e5b2456 +size 5033164960 diff --git a/model-00004-of-00179.safetensors b/model-00004-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8f233faef1e459ce5f60f63218ae30ad62e6be1 --- /dev/null +++ b/model-00004-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5b44902125fa0294aaa35905832ac4decbaa116898133c80e9b1f87caa1c3ad +size 2838088784 diff --git a/model-00005-of-00179.safetensors b/model-00005-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..743c10bd7be76ee0c02ace5233713fb448e7c420 --- /dev/null +++ b/model-00005-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d78d534b34a2ef488302e9b91e6f8cdc6e2f96dcb3b2dfd050ddf7d3e587a9c +size 5033164960 diff --git a/model-00006-of-00179.safetensors b/model-00006-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0fb4cafb799b17308d9c64cb4b148a996d47fdf7 --- /dev/null +++ b/model-00006-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a28a0a80c918b089ae5e4873102d97682d90c779f39c9fdbf8433c3f8991e88 +size 2838088784 diff --git a/model-00007-of-00179.safetensors b/model-00007-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b26b43d2d5c449b2e55cdad86975feac4a8fa67f --- /dev/null +++ b/model-00007-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48963fecb1cded9bd1023247a6b48551885bc50853af9a1f0e6ea91b59597e3 +size 5033164960 diff --git a/model-00008-of-00179.safetensors b/model-00008-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e8d99a40aa70f7cec5d19d47625d22e65de408e3 --- /dev/null +++ b/model-00008-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c85509b5f3bdf6993d933b6304d0b0c94fc7ddc9d27655f59fcd95e8a5241fc +size 2838088784 diff --git a/model-00009-of-00179.safetensors b/model-00009-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..52aa4eae827863f8944a9df3e86ff14d4eddbffb --- /dev/null +++ b/model-00009-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4ad233d52314a4c8b254a1c4147029c46fc027bc337c82e69f8affa6b4a931f +size 5033164960 diff --git a/model-00010-of-00179.safetensors b/model-00010-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..51c5125c667b0f61409f6be813c8e69e340b45df --- /dev/null +++ b/model-00010-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d18544ef286c16b2e4a48d1a6ec19f98629b7e081cb50097d7911f1221440d14 +size 2838088784 diff --git a/model-00011-of-00179.safetensors b/model-00011-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aa79ae4b663921762a8390af8744e30ddeb64b72 --- /dev/null +++ b/model-00011-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:def92abe9e6b8476a1ecf925d31667ca88b0380d3c4f346c1d3dfbbebc17a6c3 +size 5033164960 diff --git a/model-00012-of-00179.safetensors b/model-00012-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0f68885e42672796ec004330bd1552ba597d7dc --- /dev/null +++ b/model-00012-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9acfd8bbb947385c869106c35979e6a4836130586a80c3cb56450da79d25490 +size 2838088784 diff --git a/model-00013-of-00179.safetensors b/model-00013-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..547ca2b04795d4acde54f8b70b0ebea80c06daf2 --- /dev/null +++ b/model-00013-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a906b5acd2be14b538376d409b41985a5622afd4721b164b66df788d56c5d4b6 +size 5033164960 diff --git a/model-00014-of-00179.safetensors b/model-00014-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..195abff82d840e7d50639d3b8a22109376e2f439 --- /dev/null +++ b/model-00014-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d2acf8f5ff4906da6ed1fe151b40245402fce23391aee75a2b5c222b1367a95 +size 2838088784 diff --git a/model-00015-of-00179.safetensors b/model-00015-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..408bbe1199e190ac3d6a76db41431919696c08d8 --- /dev/null +++ b/model-00015-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96ff7e7d03e4eed163c41fb8150ce7d8f246893bd17d1130b8569a6f1fe733d +size 5033164960 diff --git a/model-00016-of-00179.safetensors b/model-00016-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a1f4648cf6ecfebefaa535bf502b45d593793c1 --- /dev/null +++ b/model-00016-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6a00e5528cbed1ec503c948a5c7e23ab78e04a05f0c8be211654bb5568cc9e4 +size 2838088768 diff --git a/model-00017-of-00179.safetensors b/model-00017-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c81541672be134ab47b8cceafa85d8fd8053dcd0 --- /dev/null +++ b/model-00017-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a52b193546cacdd4aee9305b94f3aeabc0555997840afaf4e94ca8d0f388333 +size 5033164960 diff --git a/model-00018-of-00179.safetensors b/model-00018-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..248717e9b01b4bf74502b58fd556b856ccf3d4c8 --- /dev/null +++ b/model-00018-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42f13a6bca1095ab22bc9ee7f93355f7bf5dc5a1b759ff1c9019a7152d82a406 +size 2838088800 diff --git a/model-00019-of-00179.safetensors b/model-00019-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f34e456d44de0f1ce8252e52b49e7d994b293d1f --- /dev/null +++ b/model-00019-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a05b71e3293d5295f65a04ee45e6651d42dad82b6c483455c818834c117d8a4c +size 5033164960 diff --git a/model-00020-of-00179.safetensors b/model-00020-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4a377cc593726a4b2021ca1f9bca9be8d6c512f --- /dev/null +++ b/model-00020-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2851d8eecfc487e90f83edf3ead87cb516eee5a605e8934995c432657126ab34 +size 2838088800 diff --git a/model-00021-of-00179.safetensors b/model-00021-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2740c29d58f836ca02a6e9c578a5f5e3b8c6de89 --- /dev/null +++ b/model-00021-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09a10f3b0533a607eda2505d0746da51ad5d7c22a01c5da483bdd0964acdc1c1 +size 5033164960 diff --git a/model-00022-of-00179.safetensors b/model-00022-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e22a72deffa48c7c95c8a0d8a0377b02cb83b4a --- /dev/null +++ b/model-00022-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3bdfcbc2894acc38ea7d1de7caa6bbdb69e7536ad53e21f5f54925090c8093e +size 2838088800 diff --git a/model-00023-of-00179.safetensors b/model-00023-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a4ac5a94cb43f9546d9de96c144b9b5356963b3 --- /dev/null +++ b/model-00023-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd4b52077979cc5e7c9f2cadfd97ddc96e524bbc587e81a4ee3952ce88b1bee7 +size 5033164960 diff --git a/model-00024-of-00179.safetensors b/model-00024-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d372e2974ea9667dd6c4997125275838b9dc201 --- /dev/null +++ b/model-00024-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:717dc4b7873bb50dd157c50ab9342919b465db35d8a5257f49565d74d76d7476 +size 2838088800 diff --git a/model-00025-of-00179.safetensors b/model-00025-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b249908598132efbf037ab34cc45c20a4aab80ab --- /dev/null +++ b/model-00025-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7de8206df555915203d5482c5cd3dec81024710f8a5444df1ff5d0cc4e1a2d68 +size 5033164960 diff --git a/model-00026-of-00179.safetensors b/model-00026-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8469eddec63cbd13bad939db02b68ba06c2093d3 --- /dev/null +++ b/model-00026-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71a355a140b2be0deb38c80c6fc6cf70b2e546af709b42cc6d30c311c4bb657b +size 2838088800 diff --git a/model-00027-of-00179.safetensors b/model-00027-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..952767daeda293792e55cdf05e8c156769ff3fb0 --- /dev/null +++ b/model-00027-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2c835ff3acf37e41e609f10f1b79cb4d97b0847c956bc41aae3009f208bb02f +size 5033164960 diff --git a/model-00028-of-00179.safetensors b/model-00028-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9f116484851decb630f92c23905ad198300e8e4a --- /dev/null +++ b/model-00028-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92f61a7dc39a9e54361daa9fd96cc6edc87cf079b28f410f9a90c3fc361a852 +size 2838088800 diff --git a/model-00029-of-00179.safetensors b/model-00029-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..35078d61466a50178589d438eb4a104661362d53 --- /dev/null +++ b/model-00029-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeec0f43399352e8266f80fcc295f80dd10f8eab73682cc711b002c78193df5b +size 5033164960 diff --git a/model-00030-of-00179.safetensors b/model-00030-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..24d5e6845fab801d7dcdc6b619e7c1cd485642e3 --- /dev/null +++ b/model-00030-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55aabbc5ed5fc460a60d7d37588381ea99e74a371ad15b685d253986725f69e1 +size 2838088800 diff --git a/model-00031-of-00179.safetensors b/model-00031-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd55c1d9b100ab4c05ae9f9f87c3e8250cff5e45 --- /dev/null +++ b/model-00031-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e155356a500bbdfaf1790305ca471e31a9ab9d4ece9b2c6b50158d47d67797 +size 5033164960 diff --git a/model-00032-of-00179.safetensors b/model-00032-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..202a2845ef1156dd63213e1e4dbbd9fc128e5548 --- /dev/null +++ b/model-00032-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98732295e88022f13a42698b347e1a254fd6791d838bf1a4229506163486490d +size 2838088800 diff --git a/model-00033-of-00179.safetensors b/model-00033-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..876dfdfd015fb2ac5b03be9586c6233f0ec9e472 --- /dev/null +++ b/model-00033-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b53f771c9351a628502f635d20c08725734758ab06703da6991b065e3576ac +size 5033164960 diff --git a/model-00034-of-00179.safetensors b/model-00034-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8857d3717161d4af550a499b2e3b48d2174e78c5 --- /dev/null +++ b/model-00034-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43103ebd7877d8752ac42c585712904d19ecdb74b7ee363f4f9dbbfdc957388 +size 2838088800 diff --git a/model-00035-of-00179.safetensors b/model-00035-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a6b69e06fc0c515798d1332b7825af8a4273626 --- /dev/null +++ b/model-00035-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3fc22f152ca7393ef6d45e25052638baf0fda4134eec342ace8a6a65de8cb8 +size 5033164960 diff --git a/model-00036-of-00179.safetensors b/model-00036-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c70097582761816e5d9bde29ad82ca81eb57f403 --- /dev/null +++ b/model-00036-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57c1de44fe42779ddf79a2a24ca91373610a7a8e6ab96527d00f13ddf95657a1 +size 2838088800 diff --git a/model-00037-of-00179.safetensors b/model-00037-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b4720ab3a2a90fa5e8d27cd1b5905a0d65fae9f4 --- /dev/null +++ b/model-00037-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f2801226d0fc28026e053e0aea3c2dc2c199c0aad15528f9a5c37762cbaa95a +size 5033164960 diff --git a/model-00038-of-00179.safetensors b/model-00038-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8870bcb9164183cf7e4e74a8295bade3c26247ff --- /dev/null +++ b/model-00038-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28be52e6665fb5fd9913bffa54ad4b8af05a4cb503fdc076a29eacd25979faad +size 2838088800 diff --git a/model-00039-of-00179.safetensors b/model-00039-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..860bd53582ce731e03322aa6812454d04b1ac939 --- /dev/null +++ b/model-00039-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2cbb9f9cddd9e22b284ee670f58510809b0be528a3e709b9b65cef4a23b0c2 +size 5033164960 diff --git a/model-00040-of-00179.safetensors b/model-00040-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..448129243cccc65dd026732eefc9bc1e206382b8 --- /dev/null +++ b/model-00040-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd9031b29a6c7f320369203728eda17b9a8f706f03cde60a31ffcbacc73e8f9 +size 2838088800 diff --git a/model-00041-of-00179.safetensors b/model-00041-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b6c41e377e6895dfce2afa9a5831f3b4c295cb77 --- /dev/null +++ b/model-00041-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d339c1177b1ee3e609f2537cf86e71f7499f06c0a1846c7a821c3ccc50fe6500 +size 5033164960 diff --git a/model-00042-of-00179.safetensors b/model-00042-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6e58d8f330bdcd0630553b63edf7e30b441d3348 --- /dev/null +++ b/model-00042-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b157d807302adc3b82e71703a143885af70c6e66b8dc2c752d772714a2adaad8 +size 2838088800 diff --git a/model-00043-of-00179.safetensors b/model-00043-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4ef34d2545f0689a4a6ccc6b3c156d140281810a --- /dev/null +++ b/model-00043-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ff5e89421ec82f7fff115eea0c8b21209d28dc0e8b04356254ef3a99f2ad766 +size 5033164960 diff --git a/model-00044-of-00179.safetensors b/model-00044-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e1c406d1f4b9c80db1f893c80c5901115ab2d54 --- /dev/null +++ b/model-00044-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:582cd7cbf32daba2d55cc1d50811c8c56f229eac0745503fa5291fbf75a6010c +size 2838088800 diff --git a/model-00045-of-00179.safetensors b/model-00045-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0dfc14b1dbb4c6b7b39d6f64c7df65b9bd79582 --- /dev/null +++ b/model-00045-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f16a2a6936976c2b95409a8829f9ccab45057f142bac0c184c66ceb4e50b69f9 +size 5033164960 diff --git a/model-00046-of-00179.safetensors b/model-00046-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc5f9954215c94ee451a3400e695df4d9a55df6a --- /dev/null +++ b/model-00046-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6365df302a5856c5cbeae14daf058f0076467870696491e01a5600560fdab5a6 +size 2838088800 diff --git a/model-00047-of-00179.safetensors b/model-00047-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c814260bab4c41c27df921597a613dbef4080160 --- /dev/null +++ b/model-00047-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9550a17ba17796b677e1dfd1fb2382443bf5c8639864f7c1c8a625a63135e91 +size 5033164960 diff --git a/model-00048-of-00179.safetensors b/model-00048-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7bd439d705e555fb9eca3a17f6515c4bfc18130e --- /dev/null +++ b/model-00048-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b346c6196a9cdaae56d83e797fc78407d1b34845b0446c0f128a9dddec6c956 +size 2838088800 diff --git a/model-00049-of-00179.safetensors b/model-00049-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d4877b9b69019630e68c4e9f14306d4c595073eb --- /dev/null +++ b/model-00049-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:759407a355b64a3929a33a38f1b201436db444d4ddbf037b17a2f9d19ca4f059 +size 5033164960 diff --git a/model-00050-of-00179.safetensors b/model-00050-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e3d83d2dc1455c3e71a09e93ec5aa35d21c35710 --- /dev/null +++ b/model-00050-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f28b7bc6710e06f4d09e0ad1db32cff9089be4c5083cdfa05a3e088d1575f7c +size 2838088800 diff --git a/model-00051-of-00179.safetensors b/model-00051-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7ee71a047556f7a54f5cf49619ca1cfbf8b528e0 --- /dev/null +++ b/model-00051-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ca1910db4a63c0b8e861333fc71bab4375f0f6c5f717464e578797b5cd32f5 +size 5033164960 diff --git a/model-00052-of-00179.safetensors b/model-00052-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..611fa5e04f870569bb7ff7594ae031cbde491139 --- /dev/null +++ b/model-00052-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:619a7319305dd404c78e265e1ebf3ad8b5ff0a27d795ce06a384dea04d4a1784 +size 2838088800 diff --git a/model-00053-of-00179.safetensors b/model-00053-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f430e97ff8b5db8cc1741d4f8637a78de5a130fe --- /dev/null +++ b/model-00053-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c6869c5757634c796af3bc5d49f452cd038be4f1f13d5054660587b802b6915 +size 5033164960 diff --git a/model-00054-of-00179.safetensors b/model-00054-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab8225646711459e47afab7f49fcec88f41e2c60 --- /dev/null +++ b/model-00054-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af7a3ac6f3148e33383f9ec6108c0a3220ef56a92d50e60c76a448c096a208ae +size 2838088800 diff --git a/model-00055-of-00179.safetensors b/model-00055-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b3909d49e1c0887c40262cfa17e406bba63728f --- /dev/null +++ b/model-00055-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d274e4c627eaa92f56a219f7c96e3836897387eb148d64f578125461b365a34 +size 5033164960 diff --git a/model-00056-of-00179.safetensors b/model-00056-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..29770fdb8cd55954df58574cb5a3ab36008b4015 --- /dev/null +++ b/model-00056-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5167f1dcc4182e6eadf749736efae9d8e87a0b1fa1d7f279aa3f0ae6eef4737 +size 2838088800 diff --git a/model-00057-of-00179.safetensors b/model-00057-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ea9e03d3ce1260fbd1d8b1a072bb9c917f2470ba --- /dev/null +++ b/model-00057-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1df3a94c5d801235e9f8cf320c1d407824d7b20aaefd900cc79d3b5717f812a9 +size 5033164960 diff --git a/model-00058-of-00179.safetensors b/model-00058-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..45dce519b286f828effc4b1d4819501ed54a510b --- /dev/null +++ b/model-00058-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa76d0dce65406c66bfde373d9de8d39893b61b86995fbdc380c4097f99b00e6 +size 2838088800 diff --git a/model-00059-of-00179.safetensors b/model-00059-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3f64c478a25415e82ddce2075391f211cc7160f2 --- /dev/null +++ b/model-00059-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceb9409ce8193529a841cefc79194ee379bc8b4dd15596c0f9dd5b69895c47fa +size 5033164960 diff --git a/model-00060-of-00179.safetensors b/model-00060-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..02ba2b9f0af3938683a2e63b1a447490b76dfe6d --- /dev/null +++ b/model-00060-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:099e14af3e5a3c41c04480bce0836c4bbe2892b23a922adcae70b277268271af +size 2838088800 diff --git a/model-00061-of-00179.safetensors b/model-00061-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce98e45c434b20707f9661136de263ccbd674eb9 --- /dev/null +++ b/model-00061-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59cd5ecfd5c4f382912d8db9a082a89c71f83acb037eb7c5246b9a936a9cc7a6 +size 5033164960 diff --git a/model-00062-of-00179.safetensors b/model-00062-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7ed95ee034f54b90ee7d309da701e2edf2c3126a --- /dev/null +++ b/model-00062-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6c57e88ae155de8941f286e65809b912373cd532d02a27e50c9e3781eabb226 +size 2838088800 diff --git a/model-00063-of-00179.safetensors b/model-00063-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..91f4eea2fed6f19e5b7c0334ac9e7c6daa4d77dc --- /dev/null +++ b/model-00063-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f41ed224442c55399735a8b646689797e83c6c5b8c978c6314971232bb4269e6 +size 5033164960 diff --git a/model-00064-of-00179.safetensors b/model-00064-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a33116fea117fb7fdc205dc9c89c02af37defe81 --- /dev/null +++ b/model-00064-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0422e2f69fa2824a7db9a8e5f9c6ffbe2d09b6b33fb64f5c700100085207baaa +size 2838088800 diff --git a/model-00065-of-00179.safetensors b/model-00065-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e3a1bebaddbd5bba6eb047ed882193a21539a572 --- /dev/null +++ b/model-00065-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b2ddc7a6ce19fb5eaed6880e1c04f59f82352da6d874590685a346e6163cf4c +size 5033164960 diff --git a/model-00066-of-00179.safetensors b/model-00066-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ffda72c2c90436a973c6f30d4888bc196f14cc4c --- /dev/null +++ b/model-00066-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e043f01a6a96c89da843c08e158e206f6aa90eb1652713cd9f7f0a31864f40 +size 2838088800 diff --git a/model-00067-of-00179.safetensors b/model-00067-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4075342dc4936430a5202bf04843d961d34c0a5a --- /dev/null +++ b/model-00067-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb4e3c9e10b824d9b94781ac48d90afb94ab5a9cd7c1e24dafda106ca6b4ca1 +size 5033164960 diff --git a/model-00068-of-00179.safetensors b/model-00068-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..37f7003509aaae7c5206129d3c0e81f963914251 --- /dev/null +++ b/model-00068-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64405b2f5479bca0a7929c68d8c61434b4e6783c663075ffe14b5565c1eb6f51 +size 2838088800 diff --git a/model-00069-of-00179.safetensors b/model-00069-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1dba3cdf03036f6cd247de2a36f73fe9493327c8 --- /dev/null +++ b/model-00069-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e451bf32ef72b0b6d152ca7c7e6898a89c9247f8feea5d025af76380351396fc +size 5033164960 diff --git a/model-00070-of-00179.safetensors b/model-00070-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70daafba0ee510d52cdabfc5e1a1323775313eb3 --- /dev/null +++ b/model-00070-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa6569a28bc1bc347a2ff3df0761b4283f196f553c6a96a61327777ee680f8b8 +size 2838088800 diff --git a/model-00071-of-00179.safetensors b/model-00071-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2820785825b26bcdb110bdd32f61fb1bf5baf545 --- /dev/null +++ b/model-00071-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc28591ddc0034e441df9e60f1c828a26d332694029f1962505e652e65342e8 +size 5033164960 diff --git a/model-00072-of-00179.safetensors b/model-00072-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..efa28f0badf0376f50b319bd52175672f2c4a56c --- /dev/null +++ b/model-00072-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3c847aa28e3935e246c074e8d2256a7e1cb1142e70e01cbf192f34e6ce3ab19 +size 2838088800 diff --git a/model-00073-of-00179.safetensors b/model-00073-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..54d562cd05327e53eaa646ecb3307d1dd73b5b3a --- /dev/null +++ b/model-00073-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b37fe1a9e8736a4d5c67a6666429d5845044da6dfc2edcb9e76c167bb48522 +size 5033164960 diff --git a/model-00074-of-00179.safetensors b/model-00074-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..179527659c6105dfcbf632e670f0c80358258d87 --- /dev/null +++ b/model-00074-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3e9904041ec3a5e974c696e6b9475eb077409a4adcf0083afebb3c3502c060 +size 2838088800 diff --git a/model-00075-of-00179.safetensors b/model-00075-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d1d5e506e796b88c5f35dd1f3639c7e5ca875d91 --- /dev/null +++ b/model-00075-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f3ded91c74ca84f0e9c7d884e49cf18c613640ce981166b363aa03b64fdb35 +size 5033164960 diff --git a/model-00076-of-00179.safetensors b/model-00076-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e8ce60e3109b0e4f08c4192644fb0dc44d87611 --- /dev/null +++ b/model-00076-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09b054cfb5090c31bf00b84f4be80ed034ee42f45a873f0dcfe56dbdfd3459d1 +size 2838088800 diff --git a/model-00077-of-00179.safetensors b/model-00077-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da42f7019339a2c9c3932a0c0bf840db77788e62 --- /dev/null +++ b/model-00077-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5964c552ddf2e56f1747b55990c9061b960f2e4ff4bb11e57ed05255431a419d +size 5033164960 diff --git a/model-00078-of-00179.safetensors b/model-00078-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4cde4af8e878b41adf6eb580d1caedcf25393ef2 --- /dev/null +++ b/model-00078-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3681e090d6c7943aefc7aff934d7a0efe051e58f6457b0e97c3fb9be8f52ff7 +size 2838088800 diff --git a/model-00079-of-00179.safetensors b/model-00079-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d733a6f68d107d425a7200535b4c632dcc231524 --- /dev/null +++ b/model-00079-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ac9df76591aed6d070f0178f2dfee2f0ba9bc0c42a11d2d524ba468173a5866 +size 5033164960 diff --git a/model-00080-of-00179.safetensors b/model-00080-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..067b48034fd73d4c2b3161fafcb6903e79d59f84 --- /dev/null +++ b/model-00080-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74861c61c580d720183e79ccddd360af0137da2579819434a9192d5d1338a08c +size 2838088800 diff --git a/model-00081-of-00179.safetensors b/model-00081-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b96caac4fcc2aaab8a48bcba6ef1628104bc358 --- /dev/null +++ b/model-00081-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f51b84ca4dc95415a40541b98f1257dc25120d9084c12a5ec4ae56b283c0e3f +size 5033164960 diff --git a/model-00082-of-00179.safetensors b/model-00082-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..65403bf4e3d28722001321e43c9d9a602bf55a73 --- /dev/null +++ b/model-00082-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06cbdcba86b509e38c8692af7892321e15c4a5dd23bca192c562eac01ef48022 +size 2838088800 diff --git a/model-00083-of-00179.safetensors b/model-00083-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d5468f2ef9a7ad1f8c7db194eaa21255a9bbc546 --- /dev/null +++ b/model-00083-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35f76eddcc5a7a64317c56076e0d3db23bf1f04dd4b83a807d5591defebb4185 +size 5033164960 diff --git a/model-00084-of-00179.safetensors b/model-00084-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d514cf0b793df40219e00af8a5cc6ae772636e32 --- /dev/null +++ b/model-00084-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b08715eb92b85c8dbbb5df389ac5e9d22ff252f5579728191075996e9ea05545 +size 2838088800 diff --git a/model-00085-of-00179.safetensors b/model-00085-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..323e1c3b43f07de039864ebca9905ba0bad08e7b --- /dev/null +++ b/model-00085-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eafac758d43ab2b70aad9f4273cb6dfb0c6a2256c2a7374afb61050c514ef7f +size 5033164960 diff --git a/model-00086-of-00179.safetensors b/model-00086-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..288591a69ce653a6dc01b344068514a6c34ecc46 --- /dev/null +++ b/model-00086-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f331599c20ac0653e7fe1f408cf14535274b5e45616fdc051f379f430d7f545a +size 2838088800 diff --git a/model-00087-of-00179.safetensors b/model-00087-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..94ac2006926aee84b510e40a00fbfcd00ca58fee --- /dev/null +++ b/model-00087-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:185230397b9a0dde6e57db69ad88329990d1a3746fe790dfe068e32040f6074a +size 5033164960 diff --git a/model-00088-of-00179.safetensors b/model-00088-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f240cfc8baaf8ac88fa9d74a0530f703b9a68844 --- /dev/null +++ b/model-00088-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:881d94d18dbf4abf09aa9fef4a9d15460de02f572748667be66ccf6b9edae3f1 +size 2838088800 diff --git a/model-00089-of-00179.safetensors b/model-00089-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fd82e955b9d2e47eba9c994ec17f5eda93019a72 --- /dev/null +++ b/model-00089-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e60e865c9b17188d71758f5a99fce5967c1e5c8005fa883ae26b524436aa936 +size 5033164960 diff --git a/model-00090-of-00179.safetensors b/model-00090-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ab79c93c9ff4e752645371e81dd990ec6e8bac3 --- /dev/null +++ b/model-00090-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a5d0ed3ac8369885db0f23ab32c75d8ca4b93965158ee232edec2a33eada4d5 +size 2838088800 diff --git a/model-00091-of-00179.safetensors b/model-00091-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4a1267254a686e375ed258c4ec747414b5fbec2c --- /dev/null +++ b/model-00091-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83f8f00bf96ddf4ef51959f8764c00b92a6fcf24d0949bb8812e8fbd773c78f9 +size 5033164960 diff --git a/model-00092-of-00179.safetensors b/model-00092-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..63fbd44b61a45fc29efaa5391559663926cc4034 --- /dev/null +++ b/model-00092-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bc1077a82113cbdb4fd5601bce915be95976fe1d6140a4a856b4166239a7360 +size 2838088800 diff --git a/model-00093-of-00179.safetensors b/model-00093-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9854f0aa30acb14246ae92c11e23b760dd72dd0b --- /dev/null +++ b/model-00093-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f064e320718427811fb4d573970a871f543892228dc6b969de8b7fb2e07158dd +size 5033164960 diff --git a/model-00094-of-00179.safetensors b/model-00094-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4bd66288865c003ea6c7ccb3af27c8a3f3f63bcc --- /dev/null +++ b/model-00094-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44cfac66b71ef487cf5f2a7c39ec8cc0ebcdf6c4f8bab55d9ada3e66f6898f79 +size 2838088800 diff --git a/model-00095-of-00179.safetensors b/model-00095-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..48930d8f4ac6fea35ef7ef13d8e335089aa9dd42 --- /dev/null +++ b/model-00095-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecd16b02c5faea24f4d69e67c010c70ecfb877316631269600e4329705a8b011 +size 5033164960 diff --git a/model-00096-of-00179.safetensors b/model-00096-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d1327b503e790dbf78c2a69fbe7d670cb4ca8c0a --- /dev/null +++ b/model-00096-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd6b2546506c5861e1f9b9041920d806c7ea7a190bbd3d73ad766ac4354cd30a +size 2838088800 diff --git a/model-00097-of-00179.safetensors b/model-00097-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a305849a3c57b12b80daf78c6742fc5318f19278 --- /dev/null +++ b/model-00097-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7c86890694a7e2faed2f5a95bff24224081fb55429f4952e6da1f9c18b2fffc +size 5033164960 diff --git a/model-00098-of-00179.safetensors b/model-00098-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e66bcc0b8dc7c4e186f0c67996b6c9bd3cc3a4f9 --- /dev/null +++ b/model-00098-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc91b6c5d8b65cff2ab40240f39a35ab53396514b8c724e20ce98a7381725307 +size 2838088800 diff --git a/model-00099-of-00179.safetensors b/model-00099-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ac8936ef30d719c0c1901ced41786073f0ab613 --- /dev/null +++ b/model-00099-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe72ad02efd8956abf7a088cfc8a1562fc2732a257988d142469c6f4da1edd9b +size 5033164960 diff --git a/model-00100-of-00179.safetensors b/model-00100-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..652901c0f77f22c027b2858651d57b22b8ed4620 --- /dev/null +++ b/model-00100-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02fcc70a894ef38b88ac1fddf0713290b43e035de3d5fe063631398a0e136f10 +size 2838088800 diff --git a/model-00101-of-00179.safetensors b/model-00101-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7396526701bc1c943b04bb9b868ccf79a0b87f84 --- /dev/null +++ b/model-00101-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:896b29a28e85b7e64b3220c7cb3197aafd059c130f503ef355d181f00d43b0a2 +size 5033164960 diff --git a/model-00102-of-00179.safetensors b/model-00102-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f20499fa0de3c649172a42320b42ea1c9655b6b2 --- /dev/null +++ b/model-00102-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8aba64f0804e8befede916b2538a2a339ecede6de6a727d37aecd41bddef96 +size 2838088800 diff --git a/model-00103-of-00179.safetensors b/model-00103-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..73d3a6907c20e8ad1cad6516d497d72629a7fd8b --- /dev/null +++ b/model-00103-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb9f00b5b71e1ff9b8e1211bd5b1a7fc47121b790d2b1bc813975821882a4e26 +size 5033164960 diff --git a/model-00104-of-00179.safetensors b/model-00104-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a785ad063c271987e57a9ea92511d4f2dd0ad664 --- /dev/null +++ b/model-00104-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f066c9c7bfee41cdc128afaf3b4e68972499a1667bdefdd3a0749924cba51022 +size 2838088800 diff --git a/model-00105-of-00179.safetensors b/model-00105-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9fe75645b5bce76bd94756d4ad668f1d86386823 --- /dev/null +++ b/model-00105-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34995d1c40d595af96eee369c5abb3f42c14681f67008b5cb497a5dccce72e72 +size 5033164960 diff --git a/model-00106-of-00179.safetensors b/model-00106-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d1226b69a04969c942aaecc55bff0df1c36c0286 --- /dev/null +++ b/model-00106-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61013ad9cbbe510319e35d61754bfa1c3a893dba3b0d7e8d249ec55b07dd60e9 +size 2838088800 diff --git a/model-00107-of-00179.safetensors b/model-00107-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39b0e1891c63d6687d2e0d9a3a71533da9b75b16 --- /dev/null +++ b/model-00107-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d49b68d5cb2d0ed9b26505d329a28d087010448ede421fb7d730567f6a8c8c6 +size 5033164960 diff --git a/model-00108-of-00179.safetensors b/model-00108-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d194f896feea681bf67a56fd2892b2962030970e --- /dev/null +++ b/model-00108-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497607e497ddace5a47aba966d498dede770a5a2fc0899415e7eed854bd8b852 +size 2838088800 diff --git a/model-00109-of-00179.safetensors b/model-00109-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a113ca1bdf9121a6f9984946186aa51fcde8d6c0 --- /dev/null +++ b/model-00109-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c900a2c2361fcc918518f91cb8dd3cb2d46c21cc0711478c510e791237b62b1c +size 5033164960 diff --git a/model-00110-of-00179.safetensors b/model-00110-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e06aa2ad4d608a26a60795d21d53b01dcbee8ec5 --- /dev/null +++ b/model-00110-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09681f35cb2e4b4db8206579eebdb4119638d33619f9df91cce1c1f8abbc745d +size 2838088800 diff --git a/model-00111-of-00179.safetensors b/model-00111-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a12f8e2d177b3a38de65da6786be76ea26a4f540 --- /dev/null +++ b/model-00111-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68a60e7b1d9f4314953580d5cadf3f71bc6234a217bcf3167be88c3d1e7022db +size 5033164960 diff --git a/model-00112-of-00179.safetensors b/model-00112-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..08cc039eaa5715f66774adf0fb41c75f407bff6b --- /dev/null +++ b/model-00112-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca12d80ec7e25869e6c09ad7c73523bff4467a39b81be2bb61f652541b38ec4 +size 2838088800 diff --git a/model-00113-of-00179.safetensors b/model-00113-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9ad2bd7df5587ba88095b1fac6e88f48e8ee7735 --- /dev/null +++ b/model-00113-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d552e23696d43fa87d307369e7af58ea5550448a0377041e25f1da21e2b7cc9 +size 5033164960 diff --git a/model-00114-of-00179.safetensors b/model-00114-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0d41e8318ab091843bbba4fafe59babfd4aad67f --- /dev/null +++ b/model-00114-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571bb871bcbdab859f02118ab4744ac3488a2cc0d427c61cba3a3dfb12fa52cf +size 2838088800 diff --git a/model-00115-of-00179.safetensors b/model-00115-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2ca4b80565689b6b2f330f81278ee6acf4b0912b --- /dev/null +++ b/model-00115-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7bcaa48cb29f214a0ad956f64bf021d3651865e28d976ef2097b05d066f054c +size 5033164960 diff --git a/model-00116-of-00179.safetensors b/model-00116-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..64af301e89f9dd08c5a05f62eeada95e59259dab --- /dev/null +++ b/model-00116-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dfb88cec3d85780fe0af18abb1320cdc3427f754c1760c0019b994c9df9849a +size 2838088800 diff --git a/model-00117-of-00179.safetensors b/model-00117-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e6961673d07dcf252209026b718c3cb73ef7687f --- /dev/null +++ b/model-00117-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f4fcd315ae1a5296c0d30a6f70a7ddcb8e85a26eeee5e9b97b1b422774272a5 +size 5033164960 diff --git a/model-00118-of-00179.safetensors b/model-00118-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bd317a41659897a10b2fb683529f13e107b2b8c3 --- /dev/null +++ b/model-00118-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23110d4b8a5d6285474f1d2f246d216db892332c7a25e35b40577caa8ccd76c1 +size 2838088800 diff --git a/model-00119-of-00179.safetensors b/model-00119-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c6847f9cde19c44325d04fa631e15ebcd074f8cd --- /dev/null +++ b/model-00119-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98e455cfa475c4f5fd95f2dec628c3c5867d2327e1c17f80b827494d82fd2ab0 +size 5033164960 diff --git a/model-00120-of-00179.safetensors b/model-00120-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..521ba344299dfa1ac4f1644c17b0f8188dcfc655 --- /dev/null +++ b/model-00120-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ffcc6c6aadfbc0d2d37ce6d43e7cf36cbae51ac86f90c94e3d7f1e26a2d5b9 +size 2838088800 diff --git a/model-00121-of-00179.safetensors b/model-00121-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4bfdc6449e457438c0a1df865f7a3c940f43db4c --- /dev/null +++ b/model-00121-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ed821f198898e3129b4609f17ef9f53b13a5fcde5b36f8e4ed2aaeb483c93fe +size 5033164960 diff --git a/model-00122-of-00179.safetensors b/model-00122-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f8e97fd9893ea14639ef09175421f5ec4533946f --- /dev/null +++ b/model-00122-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cefaf14aa20745274dab7911b3c3837152e8e6483fb580557e75aa8f0eb584 +size 2838088800 diff --git a/model-00123-of-00179.safetensors b/model-00123-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..918798de47f941b701354796386ff9bb4f6fc590 --- /dev/null +++ b/model-00123-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09cb2cec87dd6563494f45f31cc945a7d6288b822e7dbafd1a59975f94d6f1b5 +size 5033164960 diff --git a/model-00124-of-00179.safetensors b/model-00124-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7054686b79eb40ad7bcddf6971ba27658443ac79 --- /dev/null +++ b/model-00124-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd736b9ebcef745a97a3eaf4d4adf304338c544700ab212cc3e431ca1dbd9d4b +size 2838088800 diff --git a/model-00125-of-00179.safetensors b/model-00125-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ea66d00a61b58e7421dff54480d85c6e26e7b593 --- /dev/null +++ b/model-00125-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c090e4f5df2dc13c711bb128dfd9c0c5bc7ef51a88a95307cd2eb23392b515c7 +size 5033164960 diff --git a/model-00126-of-00179.safetensors b/model-00126-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5123a3628aa734500bb205f6ac9cb63aa186692f --- /dev/null +++ b/model-00126-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bbc9d2bea6195c76f45ff2866ba9051dfd234bce0e36a437b74beeae0ef2a9c +size 2838088800 diff --git a/model-00127-of-00179.safetensors b/model-00127-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef1d9f9a6333eb1ae1ff366a9614ebe3773ec297 --- /dev/null +++ b/model-00127-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e659ecf95cbc4ec500c76fa1c2c84fdea5f921b2916325f66eac2255a918d3a9 +size 5033164960 diff --git a/model-00128-of-00179.safetensors b/model-00128-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..97eaa4922d97501fc2dfc02f37e2a27eb7cae79b --- /dev/null +++ b/model-00128-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:649e2f8f3ee1218daf735ddd75ead0fade4d59e5f716dd2b2933d893b05d90e3 +size 2838088800 diff --git a/model-00129-of-00179.safetensors b/model-00129-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d9659cfaf16f472521d91daf29dbba8ac6895de --- /dev/null +++ b/model-00129-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704a5926b5e1f59e02a3a465212f923a05fb0ef9488ddd44a44adfc196785bdf +size 5033164960 diff --git a/model-00130-of-00179.safetensors b/model-00130-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..25804674587e3784118eba2c72e8648cdb815e4e --- /dev/null +++ b/model-00130-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97da7d2db08dbb4439f5e805ef27b5f541fcc8d45cb329f8b9f616d5421a6609 +size 2838088800 diff --git a/model-00131-of-00179.safetensors b/model-00131-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..06248252b8d716b9819192f3283879fcef43b07c --- /dev/null +++ b/model-00131-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb29ac78f59f63b99928b39a95436d8120001c1728b28cca5371d79aafa1fffb +size 5033164960 diff --git a/model-00132-of-00179.safetensors b/model-00132-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c95cd73732b8834baa5dc9d4a1970b9913afc86c --- /dev/null +++ b/model-00132-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d358861810dbdcf4494c3fa45d0d8714a854d18222a11fde967bd7713afc4dc +size 2838088800 diff --git a/model-00133-of-00179.safetensors b/model-00133-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..75f69ebc4a9ce81554511b0b986939d9d2420d61 --- /dev/null +++ b/model-00133-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b29cee8cfa5a97063412dd5ea0583a3d3f87d9e3fe5d17f6719d7df2fb6601b +size 5033164960 diff --git a/model-00134-of-00179.safetensors b/model-00134-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e14efc22db3e9cf7ea06aceb4eacee7c4c3f921d --- /dev/null +++ b/model-00134-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:664d1278bbde4d25ce2d6b8bf693201ece26cdb31767e1cdacc75d37a0ec32c4 +size 2838088800 diff --git a/model-00135-of-00179.safetensors b/model-00135-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85b388257896d373e3e4c096f9e3b79dd2dc275b --- /dev/null +++ b/model-00135-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:486e9c5e7b185e0b57b30ce30d13b6cea11b3cd3866fac19f7bdbe93d93341aa +size 5033164960 diff --git a/model-00136-of-00179.safetensors b/model-00136-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..be64b452d1ec2c4eedabf4cbdc35ffc60320af53 --- /dev/null +++ b/model-00136-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a0639c77c177fa7c9ccc96ad8cc0864f02b9476c0d7144f523e92082e8fbe4d +size 2838088800 diff --git a/model-00137-of-00179.safetensors b/model-00137-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..542170c3fe6eac499d9e9436ffb45f01ab18e701 --- /dev/null +++ b/model-00137-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9499cddd86d20d72eea3a1c1f4b985b12deb68920a3adea049c85019486ad2 +size 5033164960 diff --git a/model-00138-of-00179.safetensors b/model-00138-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc590eb6db090d95b86bd38e59d84a72257d7551 --- /dev/null +++ b/model-00138-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4ac65eee5da70799e3c3c7a695e369f86d4d5d78ba358f110de29f25279200d +size 2838088800 diff --git a/model-00139-of-00179.safetensors b/model-00139-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1abf46a4ab79c8584d8b49135b81947687163c52 --- /dev/null +++ b/model-00139-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9768c67798b877b9f3c11686e2b5f998ab1c84405bd4e11fcb95ca3c8d8004b9 +size 5033164960 diff --git a/model-00140-of-00179.safetensors b/model-00140-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3618f519fd10496db27e4eeb8f8a21d05767ec18 --- /dev/null +++ b/model-00140-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cea7c48aa7071ca04530bfc8055ca252984e88aab0b1892813c5fe9aaf5f2587 +size 2838088800 diff --git a/model-00141-of-00179.safetensors b/model-00141-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..97fb229bc9ab74ae1581e553fa0641aeb713b71a --- /dev/null +++ b/model-00141-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e75458bcc8438c04f0b59118a69690cea0f0558bfaab2dcf6a23aabfee2b54d +size 5033164960 diff --git a/model-00142-of-00179.safetensors b/model-00142-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e31cf5da93b3b61df43f08ab700212158e6e464 --- /dev/null +++ b/model-00142-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d799ef1671780960b72b61f5c6bfd513c447c40b78fb297b3ad9cb76deb56633 +size 2838088800 diff --git a/model-00143-of-00179.safetensors b/model-00143-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e52d3b29015e8d5771b7aec8843153d0e267b969 --- /dev/null +++ b/model-00143-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b45ea18922624f291ab82ebc06efa2c4dfe18471647b3e407b661f17fa4c0732 +size 5033164960 diff --git a/model-00144-of-00179.safetensors b/model-00144-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88eabb20b8c056921fd5e73e82492eea0ee1042a --- /dev/null +++ b/model-00144-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9af79c6cec109606531653459652648c3f7db46daa8110be6017d4d32d59059 +size 2838088800 diff --git a/model-00145-of-00179.safetensors b/model-00145-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..57ebfe7035475c9956693b875cdb43edf5bbae59 --- /dev/null +++ b/model-00145-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9827f2cb7ed7da1426a9a17ed1eca1db8a6acb97faa2e95e1b3cdafbb07ac622 +size 5033164960 diff --git a/model-00146-of-00179.safetensors b/model-00146-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd0fdc772b58220a56850e7777698f6f04b4a9b8 --- /dev/null +++ b/model-00146-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc6ae79c2cb4b123a27819295f8fe7f2f91ac77dfbdef68fc1ee9bc17b992eb1 +size 2838088800 diff --git a/model-00147-of-00179.safetensors b/model-00147-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f4735c3756b27d6106b79137168f38a4f7fbd846 --- /dev/null +++ b/model-00147-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee1dce65c2ede8e5a7e97685f9d78c0199bb81047d1f0655211b278bfda69486 +size 5033164960 diff --git a/model-00148-of-00179.safetensors b/model-00148-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2120d1fb298c6ca76c78f334bec7556ff4239f73 --- /dev/null +++ b/model-00148-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b8c4581951c8dcafb4099954d44352421850845ec32f98a611f608fc3ea94a +size 2838088800 diff --git a/model-00149-of-00179.safetensors b/model-00149-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..866c6f97647d49b39d37daf724ed8044302acb8f --- /dev/null +++ b/model-00149-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32a0de6dbb06062376f60651739d692a9a91da27f28decd17bb05ed0a6857124 +size 5033164960 diff --git a/model-00150-of-00179.safetensors b/model-00150-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a3c2513c4be4da9e350999c6d8d880c32194e88 --- /dev/null +++ b/model-00150-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a289badf71631df5dcf196b14fa8ed7a7d2a3939517d9036f0450b4ee9c6d54 +size 2838088800 diff --git a/model-00151-of-00179.safetensors b/model-00151-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2f71f16c320c228dfc60c473657dc21db1887fb --- /dev/null +++ b/model-00151-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c83c9b5bb0a40e9d0a5a1072a4404a7abe78e51a5bab0278e3f66dddfb375c0 +size 5033164960 diff --git a/model-00152-of-00179.safetensors b/model-00152-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3d2b7768d661fce3a4f98d8d826059fb02c56774 --- /dev/null +++ b/model-00152-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019f6b71ae8fcc12bb8183e05f00a17e96523a0558a3fa9d783eec39935ee5be +size 2838088800 diff --git a/model-00153-of-00179.safetensors b/model-00153-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e52b2d6ae3ec0f26a72c469f20db3adfe878f392 --- /dev/null +++ b/model-00153-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2dc2bfed1d05a532d861877363a54fbb2b5c77bb6e1e6c53ca18d5bfa8e9006 +size 5033164960 diff --git a/model-00154-of-00179.safetensors b/model-00154-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dac24db4963a6aef19006e458bd9049694a4230e --- /dev/null +++ b/model-00154-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98fcf6d46d8214012c8de256ae16bbcb0eef51d5067148c36f86773a7dbf099f +size 2838088800 diff --git a/model-00155-of-00179.safetensors b/model-00155-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ad70ee1ef2e9fff98adfb0677abecadf4e006cd3 --- /dev/null +++ b/model-00155-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f15d50682fa513959fd603ce07bf3e7264ead9433a6a2f77e2dfd6496c42a2 +size 5033164960 diff --git a/model-00156-of-00179.safetensors b/model-00156-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ec9cdeabae1bfeb6216922fdfc42d137e564f3b --- /dev/null +++ b/model-00156-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bb70a7f770f2f4e34dc0613e779a98934e43b057bf2d0cd42e1bafc1bb64ce4 +size 2838088800 diff --git a/model-00157-of-00179.safetensors b/model-00157-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8ef2e5bbe97f5a7a5c1f1512eb6e1568956e3017 --- /dev/null +++ b/model-00157-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054f7387d6f8cfd7f312eef0a10f2cc6a9e98f02bb29fd1508508836c58feed4 +size 5033164960 diff --git a/model-00158-of-00179.safetensors b/model-00158-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e3c3b446a5d0c0a360f04cc5bd72fb117fe83fba --- /dev/null +++ b/model-00158-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:885b042a0378e5a548a56cc8965287bec0561b07945da6f08637705e803bb868 +size 2838088800 diff --git a/model-00159-of-00179.safetensors b/model-00159-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b002b3b1e3b286fdbc3af8597965b60203392009 --- /dev/null +++ b/model-00159-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e0e4c024d0cb562207fde36ea32bfc3d60c57f516f692cd5e52e5ef2ca672d +size 5033164960 diff --git a/model-00160-of-00179.safetensors b/model-00160-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..73a5b601d5869f70d46559389cec4e8bc38d9d3e --- /dev/null +++ b/model-00160-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab43e3baf8aef744cc96aa536b3a80a7a36e37780890a29dc6996661105b7fc0 +size 2838088800 diff --git a/model-00161-of-00179.safetensors b/model-00161-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8cd2e91239cacf946845b647e2a27e8547f32de9 --- /dev/null +++ b/model-00161-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:162de5402818d55af5d113269717a3efa9ca7e5cce21a11c4c4f33920ddf096f +size 5033164960 diff --git a/model-00162-of-00179.safetensors b/model-00162-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6b1811708e35b4417e4715ed4bcc43edd1e080d4 --- /dev/null +++ b/model-00162-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff3ec75fad8ed28e606753035be6e63147a641f778127749dbb8fe028d7a09b4 +size 2838088800 diff --git a/model-00163-of-00179.safetensors b/model-00163-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..42fcb640bd654238a85805191879dc196681613d --- /dev/null +++ b/model-00163-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b9a76bbd88116423e8288a7b0e6f57de703bee9d7b1f8e2e9356129d9acdce4 +size 5033164960 diff --git a/model-00164-of-00179.safetensors b/model-00164-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7387fbcfa034be4d07245e7e1f661b54e1c6f704 --- /dev/null +++ b/model-00164-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6edf77504392d5b7af1d87a4dc6dbb33e3fe3e1b4d8e722ae1000126dad8ba28 +size 2838088800 diff --git a/model-00165-of-00179.safetensors b/model-00165-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d13b358d262017c60418d2a8619649032d82fe9 --- /dev/null +++ b/model-00165-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:751611bef92030a75ac2b24d0592bf1c241ea662b2df10cd2e54c04e50f277b8 +size 5033164960 diff --git a/model-00166-of-00179.safetensors b/model-00166-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..03250d2fa86796f7f4cd5941db2f2ba28891c5ca --- /dev/null +++ b/model-00166-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87cb671147517401371e255b44b0280e4780867347676822189b0d6492dbca7 +size 2838088800 diff --git a/model-00167-of-00179.safetensors b/model-00167-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..75d91ecbe0122a17b59cada851e58b35f6881f2b --- /dev/null +++ b/model-00167-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1d0face86bc248d396a0f2f39354ed173aca5d299ea3cb2e0ecf249e85d9e54 +size 5033164960 diff --git a/model-00168-of-00179.safetensors b/model-00168-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ba40fd270e30e569acb67b901f2401a9454e202c --- /dev/null +++ b/model-00168-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925d77125c2fd37146f78c10431378dfd0820e616debf34548011e4b22f0a873 +size 2838088800 diff --git a/model-00169-of-00179.safetensors b/model-00169-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e9271b8ef43f4bec4795b7cf44362b7f812e0bb7 --- /dev/null +++ b/model-00169-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18865cc631603591d16e3ea99e501234bb7654fb3bbe7daba505c78ddf147d9f +size 5033164960 diff --git a/model-00170-of-00179.safetensors b/model-00170-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..309456d8b278394f148d12c517a1b7c284826f58 --- /dev/null +++ b/model-00170-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f04cd354136c07091ad833b969db001b90af15f52c6b62cc0f1146f392271f29 +size 2838088800 diff --git a/model-00171-of-00179.safetensors b/model-00171-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ec47830e7cedf9980dfa4f68afad2cfa5351ab84 --- /dev/null +++ b/model-00171-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb58c29bd3833a5fd96d72e5567e8a7cb771dd9a3114ddede5f80c5c089b268a +size 5033164960 diff --git a/model-00172-of-00179.safetensors b/model-00172-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..77952a3798301cbd7cb7190a8e86831a6793e538 --- /dev/null +++ b/model-00172-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9722861f8311b27ca12450970a6295cf29c1877041efc6e3f1f8ea6e0c49e4b +size 2838088800 diff --git a/model-00173-of-00179.safetensors b/model-00173-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27d04c49d03cc4b83abc3f9008bbfb05b12163e5 --- /dev/null +++ b/model-00173-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1603c7cd700554819feebafdf791b28c2e4089998e4370cadf49939712bff87e +size 5033164960 diff --git a/model-00174-of-00179.safetensors b/model-00174-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88315a91a3908640e891160c3453cab8bdccad92 --- /dev/null +++ b/model-00174-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3532f24456e57d6d8333fe6bc81c3350a04668c40e68eb6e9a1d6d275f92c15d +size 2838088800 diff --git a/model-00175-of-00179.safetensors b/model-00175-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2d7994eed3d0ce10f89b74d99c56c0a9b01cabe0 --- /dev/null +++ b/model-00175-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9de6e9bd9ea88e736ce41e4aecacde58455b2ad868420abdee124d92aa5d6a +size 5033164960 diff --git a/model-00176-of-00179.safetensors b/model-00176-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ccf0d281a9dc1afca46aa1b456a8f8b8a3949767 --- /dev/null +++ b/model-00176-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47ea763426cdc67a395baca169e16feb6eef90c1ac86f0ea6ac28f39111fa452 +size 2838088800 diff --git a/model-00177-of-00179.safetensors b/model-00177-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d9cc6dc4d9317a0e4f85522e55f976c3d863537 --- /dev/null +++ b/model-00177-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:461fe524194d2830408e4c69aec2c15d5462407ba22687069293201fd55d2fea +size 5033164960 diff --git a/model-00178-of-00179.safetensors b/model-00178-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..583ea50d55693e140b12aa4fac4b7bc9d985645f --- /dev/null +++ b/model-00178-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a9fa7ea3ccab7eb84d33d407a2eb0cd5edf92febcd2759cd27d127c70411ce8 +size 2838088800 diff --git a/model-00179-of-00179.safetensors b/model-00179-of-00179.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..73ae6629158886b301314b4b015f8176bfd2eb49 --- /dev/null +++ b/model-00179-of-00179.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e285825260b6f7a5fafc5e1cc395526587d20caa2e164f67e0ae414dbc80b8 +size 4117331752 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..bab08c43b43f1d8b02ca375305b8e343af5db553 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,1655 @@ +{ + "metadata": { + "total_parameters": 352797814784, + "total_size": 705595686528 + }, + "weight_map": { + "lm_head.weight": "model-00179-of-00179.safetensors", + "model.embed_tokens.weight": "model-00002-of-00179.safetensors", + "model.layers.0.input_layernorm.weight": "model-00002-of-00179.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.k_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.q_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.v_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.input_layernorm.weight": "model-00002-of-00179.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.k_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.q_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.v_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.10.input_layernorm.weight": "model-00018-of-00179.safetensors", + "model.layers.10.mlp.gate.e_score_correction_bias": "model-00018-of-00179.safetensors", + "model.layers.10.mlp.gate.weight": "model-00018-of-00179.safetensors", + "model.layers.10.mlp.moe_mlp.experts.weight": "model-00015-of-00179.safetensors", + "model.layers.10.mlp.moe_mlp.output_experts.weight": "model-00018-of-00179.safetensors", + "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.10.mlp.shared_experts.up_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00018-of-00179.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.k_proj.bias": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.q_proj.bias": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.v_proj.bias": "model-00016-of-00179.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.11.input_layernorm.weight": "model-00020-of-00179.safetensors", + "model.layers.11.mlp.gate.e_score_correction_bias": "model-00020-of-00179.safetensors", + "model.layers.11.mlp.gate.weight": "model-00020-of-00179.safetensors", + "model.layers.11.mlp.moe_mlp.experts.weight": "model-00017-of-00179.safetensors", + "model.layers.11.mlp.moe_mlp.output_experts.weight": "model-00020-of-00179.safetensors", + "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.11.mlp.shared_experts.up_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00020-of-00179.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.k_proj.bias": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.q_proj.bias": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.v_proj.bias": "model-00018-of-00179.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00018-of-00179.safetensors", + "model.layers.12.input_layernorm.weight": "model-00022-of-00179.safetensors", + "model.layers.12.mlp.gate.e_score_correction_bias": "model-00022-of-00179.safetensors", + "model.layers.12.mlp.gate.weight": "model-00022-of-00179.safetensors", + "model.layers.12.mlp.moe_mlp.experts.weight": "model-00019-of-00179.safetensors", + "model.layers.12.mlp.moe_mlp.output_experts.weight": "model-00022-of-00179.safetensors", + "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.12.mlp.shared_experts.up_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00022-of-00179.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.k_proj.bias": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.q_proj.bias": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.v_proj.bias": "model-00020-of-00179.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00020-of-00179.safetensors", + "model.layers.13.input_layernorm.weight": "model-00024-of-00179.safetensors", + "model.layers.13.mlp.gate.e_score_correction_bias": "model-00024-of-00179.safetensors", + "model.layers.13.mlp.gate.weight": "model-00024-of-00179.safetensors", + "model.layers.13.mlp.moe_mlp.experts.weight": "model-00021-of-00179.safetensors", + "model.layers.13.mlp.moe_mlp.output_experts.weight": "model-00024-of-00179.safetensors", + "model.layers.13.mlp.shared_experts.down_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00024-of-00179.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.k_proj.bias": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.q_proj.bias": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.v_proj.bias": "model-00022-of-00179.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00022-of-00179.safetensors", + "model.layers.14.input_layernorm.weight": "model-00026-of-00179.safetensors", + "model.layers.14.mlp.gate.e_score_correction_bias": "model-00026-of-00179.safetensors", + "model.layers.14.mlp.gate.weight": "model-00026-of-00179.safetensors", + "model.layers.14.mlp.moe_mlp.experts.weight": "model-00023-of-00179.safetensors", + "model.layers.14.mlp.moe_mlp.output_experts.weight": "model-00026-of-00179.safetensors", + "model.layers.14.mlp.shared_experts.down_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.14.mlp.shared_experts.up_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00026-of-00179.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.k_proj.bias": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.q_proj.bias": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.v_proj.bias": "model-00024-of-00179.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00024-of-00179.safetensors", + "model.layers.15.input_layernorm.weight": "model-00028-of-00179.safetensors", + "model.layers.15.mlp.gate.e_score_correction_bias": "model-00028-of-00179.safetensors", + "model.layers.15.mlp.gate.weight": "model-00028-of-00179.safetensors", + "model.layers.15.mlp.moe_mlp.experts.weight": "model-00025-of-00179.safetensors", + "model.layers.15.mlp.moe_mlp.output_experts.weight": "model-00028-of-00179.safetensors", + "model.layers.15.mlp.shared_experts.down_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00028-of-00179.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.k_proj.bias": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.q_proj.bias": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.v_proj.bias": "model-00026-of-00179.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00026-of-00179.safetensors", + "model.layers.16.input_layernorm.weight": "model-00030-of-00179.safetensors", + "model.layers.16.mlp.gate.e_score_correction_bias": "model-00030-of-00179.safetensors", + "model.layers.16.mlp.gate.weight": "model-00030-of-00179.safetensors", + "model.layers.16.mlp.moe_mlp.experts.weight": "model-00027-of-00179.safetensors", + "model.layers.16.mlp.moe_mlp.output_experts.weight": "model-00030-of-00179.safetensors", + "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.16.mlp.shared_experts.up_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00030-of-00179.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.k_proj.bias": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.q_proj.bias": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.v_proj.bias": "model-00028-of-00179.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00028-of-00179.safetensors", + "model.layers.17.input_layernorm.weight": "model-00032-of-00179.safetensors", + "model.layers.17.mlp.gate.e_score_correction_bias": "model-00032-of-00179.safetensors", + "model.layers.17.mlp.gate.weight": "model-00032-of-00179.safetensors", + "model.layers.17.mlp.moe_mlp.experts.weight": "model-00029-of-00179.safetensors", + "model.layers.17.mlp.moe_mlp.output_experts.weight": "model-00032-of-00179.safetensors", + "model.layers.17.mlp.shared_experts.down_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00032-of-00179.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.k_proj.bias": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.q_proj.bias": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.v_proj.bias": "model-00030-of-00179.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00030-of-00179.safetensors", + "model.layers.18.input_layernorm.weight": "model-00034-of-00179.safetensors", + "model.layers.18.mlp.gate.e_score_correction_bias": "model-00034-of-00179.safetensors", + "model.layers.18.mlp.gate.weight": "model-00034-of-00179.safetensors", + "model.layers.18.mlp.moe_mlp.experts.weight": "model-00031-of-00179.safetensors", + "model.layers.18.mlp.moe_mlp.output_experts.weight": "model-00034-of-00179.safetensors", + "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.18.mlp.shared_experts.up_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00034-of-00179.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.k_proj.bias": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.q_proj.bias": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.v_proj.bias": "model-00032-of-00179.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00032-of-00179.safetensors", + "model.layers.19.input_layernorm.weight": "model-00036-of-00179.safetensors", + "model.layers.19.mlp.gate.e_score_correction_bias": "model-00036-of-00179.safetensors", + "model.layers.19.mlp.gate.weight": "model-00036-of-00179.safetensors", + "model.layers.19.mlp.moe_mlp.experts.weight": "model-00033-of-00179.safetensors", + "model.layers.19.mlp.moe_mlp.output_experts.weight": "model-00036-of-00179.safetensors", + "model.layers.19.mlp.shared_experts.down_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00036-of-00179.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.k_proj.bias": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.q_proj.bias": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.v_proj.bias": "model-00034-of-00179.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00034-of-00179.safetensors", + "model.layers.2.input_layernorm.weight": "model-00002-of-00179.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.k_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.q_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.v_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.20.input_layernorm.weight": "model-00038-of-00179.safetensors", + "model.layers.20.mlp.gate.e_score_correction_bias": "model-00038-of-00179.safetensors", + "model.layers.20.mlp.gate.weight": "model-00038-of-00179.safetensors", + "model.layers.20.mlp.moe_mlp.experts.weight": "model-00035-of-00179.safetensors", + "model.layers.20.mlp.moe_mlp.output_experts.weight": "model-00038-of-00179.safetensors", + "model.layers.20.mlp.shared_experts.down_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.20.mlp.shared_experts.up_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00038-of-00179.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.k_proj.bias": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.q_proj.bias": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.v_proj.bias": "model-00036-of-00179.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00036-of-00179.safetensors", + "model.layers.21.input_layernorm.weight": "model-00040-of-00179.safetensors", + "model.layers.21.mlp.gate.e_score_correction_bias": "model-00040-of-00179.safetensors", + "model.layers.21.mlp.gate.weight": "model-00040-of-00179.safetensors", + "model.layers.21.mlp.moe_mlp.experts.weight": "model-00037-of-00179.safetensors", + "model.layers.21.mlp.moe_mlp.output_experts.weight": "model-00040-of-00179.safetensors", + "model.layers.21.mlp.shared_experts.down_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.21.mlp.shared_experts.up_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00040-of-00179.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.k_proj.bias": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.q_proj.bias": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.v_proj.bias": "model-00038-of-00179.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00038-of-00179.safetensors", + "model.layers.22.input_layernorm.weight": "model-00042-of-00179.safetensors", + "model.layers.22.mlp.gate.e_score_correction_bias": "model-00042-of-00179.safetensors", + "model.layers.22.mlp.gate.weight": "model-00042-of-00179.safetensors", + "model.layers.22.mlp.moe_mlp.experts.weight": "model-00039-of-00179.safetensors", + "model.layers.22.mlp.moe_mlp.output_experts.weight": "model-00042-of-00179.safetensors", + "model.layers.22.mlp.shared_experts.down_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.22.mlp.shared_experts.up_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00042-of-00179.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.k_proj.bias": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.q_proj.bias": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.v_proj.bias": "model-00040-of-00179.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00040-of-00179.safetensors", + "model.layers.23.input_layernorm.weight": "model-00044-of-00179.safetensors", + "model.layers.23.mlp.gate.e_score_correction_bias": "model-00044-of-00179.safetensors", + "model.layers.23.mlp.gate.weight": "model-00044-of-00179.safetensors", + "model.layers.23.mlp.moe_mlp.experts.weight": "model-00041-of-00179.safetensors", + "model.layers.23.mlp.moe_mlp.output_experts.weight": "model-00044-of-00179.safetensors", + "model.layers.23.mlp.shared_experts.down_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.23.mlp.shared_experts.up_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00044-of-00179.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.k_proj.bias": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.q_proj.bias": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.v_proj.bias": "model-00042-of-00179.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00042-of-00179.safetensors", + "model.layers.24.input_layernorm.weight": "model-00046-of-00179.safetensors", + "model.layers.24.mlp.gate.e_score_correction_bias": "model-00046-of-00179.safetensors", + "model.layers.24.mlp.gate.weight": "model-00046-of-00179.safetensors", + "model.layers.24.mlp.moe_mlp.experts.weight": "model-00043-of-00179.safetensors", + "model.layers.24.mlp.moe_mlp.output_experts.weight": "model-00046-of-00179.safetensors", + "model.layers.24.mlp.shared_experts.down_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.24.mlp.shared_experts.gate_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.24.mlp.shared_experts.up_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00046-of-00179.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.k_proj.bias": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.q_proj.bias": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.v_proj.bias": "model-00044-of-00179.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00044-of-00179.safetensors", + "model.layers.25.input_layernorm.weight": "model-00048-of-00179.safetensors", + "model.layers.25.mlp.gate.e_score_correction_bias": "model-00048-of-00179.safetensors", + "model.layers.25.mlp.gate.weight": "model-00048-of-00179.safetensors", + "model.layers.25.mlp.moe_mlp.experts.weight": "model-00045-of-00179.safetensors", + "model.layers.25.mlp.moe_mlp.output_experts.weight": "model-00048-of-00179.safetensors", + "model.layers.25.mlp.shared_experts.down_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.25.mlp.shared_experts.gate_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.25.mlp.shared_experts.up_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00048-of-00179.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.k_proj.bias": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.q_proj.bias": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.v_proj.bias": "model-00046-of-00179.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00046-of-00179.safetensors", + "model.layers.26.input_layernorm.weight": "model-00050-of-00179.safetensors", + "model.layers.26.mlp.gate.e_score_correction_bias": "model-00050-of-00179.safetensors", + "model.layers.26.mlp.gate.weight": "model-00050-of-00179.safetensors", + "model.layers.26.mlp.moe_mlp.experts.weight": "model-00047-of-00179.safetensors", + "model.layers.26.mlp.moe_mlp.output_experts.weight": "model-00050-of-00179.safetensors", + "model.layers.26.mlp.shared_experts.down_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.26.mlp.shared_experts.gate_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.26.mlp.shared_experts.up_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00050-of-00179.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.k_proj.bias": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.q_proj.bias": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.v_proj.bias": "model-00048-of-00179.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00048-of-00179.safetensors", + "model.layers.27.input_layernorm.weight": "model-00052-of-00179.safetensors", + "model.layers.27.mlp.gate.e_score_correction_bias": "model-00052-of-00179.safetensors", + "model.layers.27.mlp.gate.weight": "model-00052-of-00179.safetensors", + "model.layers.27.mlp.moe_mlp.experts.weight": "model-00049-of-00179.safetensors", + "model.layers.27.mlp.moe_mlp.output_experts.weight": "model-00052-of-00179.safetensors", + "model.layers.27.mlp.shared_experts.down_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.27.mlp.shared_experts.gate_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.27.mlp.shared_experts.up_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00052-of-00179.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.k_proj.bias": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.q_proj.bias": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.v_proj.bias": "model-00050-of-00179.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00050-of-00179.safetensors", + "model.layers.28.input_layernorm.weight": "model-00054-of-00179.safetensors", + "model.layers.28.mlp.gate.e_score_correction_bias": "model-00054-of-00179.safetensors", + "model.layers.28.mlp.gate.weight": "model-00054-of-00179.safetensors", + "model.layers.28.mlp.moe_mlp.experts.weight": "model-00051-of-00179.safetensors", + "model.layers.28.mlp.moe_mlp.output_experts.weight": "model-00054-of-00179.safetensors", + "model.layers.28.mlp.shared_experts.down_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.28.mlp.shared_experts.gate_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.28.mlp.shared_experts.up_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00054-of-00179.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.k_proj.bias": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.q_proj.bias": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.v_proj.bias": "model-00052-of-00179.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00052-of-00179.safetensors", + "model.layers.29.input_layernorm.weight": "model-00056-of-00179.safetensors", + "model.layers.29.mlp.gate.e_score_correction_bias": "model-00056-of-00179.safetensors", + "model.layers.29.mlp.gate.weight": "model-00056-of-00179.safetensors", + "model.layers.29.mlp.moe_mlp.experts.weight": "model-00053-of-00179.safetensors", + "model.layers.29.mlp.moe_mlp.output_experts.weight": "model-00056-of-00179.safetensors", + "model.layers.29.mlp.shared_experts.down_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.29.mlp.shared_experts.gate_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.29.mlp.shared_experts.up_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00056-of-00179.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.k_proj.bias": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.q_proj.bias": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.v_proj.bias": "model-00054-of-00179.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00054-of-00179.safetensors", + "model.layers.3.input_layernorm.weight": "model-00004-of-00179.safetensors", + "model.layers.3.mlp.gate.e_score_correction_bias": "model-00004-of-00179.safetensors", + "model.layers.3.mlp.gate.weight": "model-00004-of-00179.safetensors", + "model.layers.3.mlp.moe_mlp.experts.weight": "model-00001-of-00179.safetensors", + "model.layers.3.mlp.moe_mlp.output_experts.weight": "model-00004-of-00179.safetensors", + "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00004-of-00179.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00179.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00179.safetensors", + "model.layers.30.input_layernorm.weight": "model-00058-of-00179.safetensors", + "model.layers.30.mlp.gate.e_score_correction_bias": "model-00058-of-00179.safetensors", + "model.layers.30.mlp.gate.weight": "model-00058-of-00179.safetensors", + "model.layers.30.mlp.moe_mlp.experts.weight": "model-00055-of-00179.safetensors", + "model.layers.30.mlp.moe_mlp.output_experts.weight": "model-00058-of-00179.safetensors", + "model.layers.30.mlp.shared_experts.down_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.30.mlp.shared_experts.gate_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.30.mlp.shared_experts.up_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00058-of-00179.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.k_proj.bias": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.q_proj.bias": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.v_proj.bias": "model-00056-of-00179.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00056-of-00179.safetensors", + "model.layers.31.input_layernorm.weight": "model-00060-of-00179.safetensors", + "model.layers.31.mlp.gate.e_score_correction_bias": "model-00060-of-00179.safetensors", + "model.layers.31.mlp.gate.weight": "model-00060-of-00179.safetensors", + "model.layers.31.mlp.moe_mlp.experts.weight": "model-00057-of-00179.safetensors", + "model.layers.31.mlp.moe_mlp.output_experts.weight": "model-00060-of-00179.safetensors", + "model.layers.31.mlp.shared_experts.down_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.31.mlp.shared_experts.gate_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.31.mlp.shared_experts.up_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00060-of-00179.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.k_proj.bias": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.q_proj.bias": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.v_proj.bias": "model-00058-of-00179.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00058-of-00179.safetensors", + "model.layers.32.input_layernorm.weight": "model-00062-of-00179.safetensors", + "model.layers.32.mlp.gate.e_score_correction_bias": "model-00062-of-00179.safetensors", + "model.layers.32.mlp.gate.weight": "model-00062-of-00179.safetensors", + "model.layers.32.mlp.moe_mlp.experts.weight": "model-00059-of-00179.safetensors", + "model.layers.32.mlp.moe_mlp.output_experts.weight": "model-00062-of-00179.safetensors", + "model.layers.32.mlp.shared_experts.down_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.32.mlp.shared_experts.gate_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.32.mlp.shared_experts.up_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00062-of-00179.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.k_proj.bias": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.q_proj.bias": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.v_proj.bias": "model-00060-of-00179.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00060-of-00179.safetensors", + "model.layers.33.input_layernorm.weight": "model-00064-of-00179.safetensors", + "model.layers.33.mlp.gate.e_score_correction_bias": "model-00064-of-00179.safetensors", + "model.layers.33.mlp.gate.weight": "model-00064-of-00179.safetensors", + "model.layers.33.mlp.moe_mlp.experts.weight": "model-00061-of-00179.safetensors", + "model.layers.33.mlp.moe_mlp.output_experts.weight": "model-00064-of-00179.safetensors", + "model.layers.33.mlp.shared_experts.down_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.33.mlp.shared_experts.gate_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.33.mlp.shared_experts.up_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00064-of-00179.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.k_proj.bias": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.q_proj.bias": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.v_proj.bias": "model-00062-of-00179.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00062-of-00179.safetensors", + "model.layers.34.input_layernorm.weight": "model-00066-of-00179.safetensors", + "model.layers.34.mlp.gate.e_score_correction_bias": "model-00066-of-00179.safetensors", + "model.layers.34.mlp.gate.weight": "model-00066-of-00179.safetensors", + "model.layers.34.mlp.moe_mlp.experts.weight": "model-00063-of-00179.safetensors", + "model.layers.34.mlp.moe_mlp.output_experts.weight": "model-00066-of-00179.safetensors", + "model.layers.34.mlp.shared_experts.down_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.34.mlp.shared_experts.gate_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.34.mlp.shared_experts.up_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00066-of-00179.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.k_proj.bias": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.q_proj.bias": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.v_proj.bias": "model-00064-of-00179.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00064-of-00179.safetensors", + "model.layers.35.input_layernorm.weight": "model-00068-of-00179.safetensors", + "model.layers.35.mlp.gate.e_score_correction_bias": "model-00068-of-00179.safetensors", + "model.layers.35.mlp.gate.weight": "model-00068-of-00179.safetensors", + "model.layers.35.mlp.moe_mlp.experts.weight": "model-00065-of-00179.safetensors", + "model.layers.35.mlp.moe_mlp.output_experts.weight": "model-00068-of-00179.safetensors", + "model.layers.35.mlp.shared_experts.down_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.35.mlp.shared_experts.gate_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.35.mlp.shared_experts.up_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00068-of-00179.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.k_proj.bias": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.q_proj.bias": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.v_proj.bias": "model-00066-of-00179.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00066-of-00179.safetensors", + "model.layers.36.input_layernorm.weight": "model-00070-of-00179.safetensors", + "model.layers.36.mlp.gate.e_score_correction_bias": "model-00070-of-00179.safetensors", + "model.layers.36.mlp.gate.weight": "model-00070-of-00179.safetensors", + "model.layers.36.mlp.moe_mlp.experts.weight": "model-00067-of-00179.safetensors", + "model.layers.36.mlp.moe_mlp.output_experts.weight": "model-00070-of-00179.safetensors", + "model.layers.36.mlp.shared_experts.down_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.36.mlp.shared_experts.gate_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.36.mlp.shared_experts.up_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00070-of-00179.safetensors", + "model.layers.36.self_attn.k_norm.weight": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.k_proj.bias": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.q_norm.weight": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.q_proj.bias": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.v_proj.bias": "model-00068-of-00179.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00068-of-00179.safetensors", + "model.layers.37.input_layernorm.weight": "model-00072-of-00179.safetensors", + "model.layers.37.mlp.gate.e_score_correction_bias": "model-00072-of-00179.safetensors", + "model.layers.37.mlp.gate.weight": "model-00072-of-00179.safetensors", + "model.layers.37.mlp.moe_mlp.experts.weight": "model-00069-of-00179.safetensors", + "model.layers.37.mlp.moe_mlp.output_experts.weight": "model-00072-of-00179.safetensors", + "model.layers.37.mlp.shared_experts.down_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.37.mlp.shared_experts.gate_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.37.mlp.shared_experts.up_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00072-of-00179.safetensors", + "model.layers.37.self_attn.k_norm.weight": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.k_proj.bias": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.q_norm.weight": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.q_proj.bias": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.v_proj.bias": "model-00070-of-00179.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00070-of-00179.safetensors", + "model.layers.38.input_layernorm.weight": "model-00074-of-00179.safetensors", + "model.layers.38.mlp.gate.e_score_correction_bias": "model-00074-of-00179.safetensors", + "model.layers.38.mlp.gate.weight": "model-00074-of-00179.safetensors", + "model.layers.38.mlp.moe_mlp.experts.weight": "model-00071-of-00179.safetensors", + "model.layers.38.mlp.moe_mlp.output_experts.weight": "model-00074-of-00179.safetensors", + "model.layers.38.mlp.shared_experts.down_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.38.mlp.shared_experts.gate_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.38.mlp.shared_experts.up_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00074-of-00179.safetensors", + "model.layers.38.self_attn.k_norm.weight": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.k_proj.bias": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.q_norm.weight": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.q_proj.bias": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.v_proj.bias": "model-00072-of-00179.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00072-of-00179.safetensors", + "model.layers.39.input_layernorm.weight": "model-00076-of-00179.safetensors", + "model.layers.39.mlp.gate.e_score_correction_bias": "model-00076-of-00179.safetensors", + "model.layers.39.mlp.gate.weight": "model-00076-of-00179.safetensors", + "model.layers.39.mlp.moe_mlp.experts.weight": "model-00073-of-00179.safetensors", + "model.layers.39.mlp.moe_mlp.output_experts.weight": "model-00076-of-00179.safetensors", + "model.layers.39.mlp.shared_experts.down_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.39.mlp.shared_experts.gate_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.39.mlp.shared_experts.up_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00076-of-00179.safetensors", + "model.layers.39.self_attn.k_norm.weight": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.k_proj.bias": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.q_norm.weight": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.q_proj.bias": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.v_proj.bias": "model-00074-of-00179.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00074-of-00179.safetensors", + "model.layers.4.input_layernorm.weight": "model-00006-of-00179.safetensors", + "model.layers.4.mlp.gate.e_score_correction_bias": "model-00006-of-00179.safetensors", + "model.layers.4.mlp.gate.weight": "model-00006-of-00179.safetensors", + "model.layers.4.mlp.moe_mlp.experts.weight": "model-00003-of-00179.safetensors", + "model.layers.4.mlp.moe_mlp.output_experts.weight": "model-00006-of-00179.safetensors", + "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.4.mlp.shared_experts.up_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00006-of-00179.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.k_proj.bias": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.q_proj.bias": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.v_proj.bias": "model-00004-of-00179.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00004-of-00179.safetensors", + "model.layers.40.input_layernorm.weight": "model-00078-of-00179.safetensors", + "model.layers.40.mlp.gate.e_score_correction_bias": "model-00078-of-00179.safetensors", + "model.layers.40.mlp.gate.weight": "model-00078-of-00179.safetensors", + "model.layers.40.mlp.moe_mlp.experts.weight": "model-00075-of-00179.safetensors", + "model.layers.40.mlp.moe_mlp.output_experts.weight": "model-00078-of-00179.safetensors", + "model.layers.40.mlp.shared_experts.down_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.40.mlp.shared_experts.gate_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.40.mlp.shared_experts.up_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00078-of-00179.safetensors", + "model.layers.40.self_attn.k_norm.weight": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.k_proj.bias": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.q_norm.weight": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.q_proj.bias": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.v_proj.bias": "model-00076-of-00179.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00076-of-00179.safetensors", + "model.layers.41.input_layernorm.weight": "model-00080-of-00179.safetensors", + "model.layers.41.mlp.gate.e_score_correction_bias": "model-00080-of-00179.safetensors", + "model.layers.41.mlp.gate.weight": "model-00080-of-00179.safetensors", + "model.layers.41.mlp.moe_mlp.experts.weight": "model-00077-of-00179.safetensors", + "model.layers.41.mlp.moe_mlp.output_experts.weight": "model-00080-of-00179.safetensors", + "model.layers.41.mlp.shared_experts.down_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.41.mlp.shared_experts.gate_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.41.mlp.shared_experts.up_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00080-of-00179.safetensors", + "model.layers.41.self_attn.k_norm.weight": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.k_proj.bias": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.q_norm.weight": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.q_proj.bias": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.v_proj.bias": "model-00078-of-00179.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00078-of-00179.safetensors", + "model.layers.42.input_layernorm.weight": "model-00082-of-00179.safetensors", + "model.layers.42.mlp.gate.e_score_correction_bias": "model-00082-of-00179.safetensors", + "model.layers.42.mlp.gate.weight": "model-00082-of-00179.safetensors", + "model.layers.42.mlp.moe_mlp.experts.weight": "model-00079-of-00179.safetensors", + "model.layers.42.mlp.moe_mlp.output_experts.weight": "model-00082-of-00179.safetensors", + "model.layers.42.mlp.shared_experts.down_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.42.mlp.shared_experts.gate_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.42.mlp.shared_experts.up_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.42.post_attention_layernorm.weight": "model-00082-of-00179.safetensors", + "model.layers.42.self_attn.k_norm.weight": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.k_proj.bias": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.k_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.o_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.q_norm.weight": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.q_proj.bias": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.q_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.v_proj.bias": "model-00080-of-00179.safetensors", + "model.layers.42.self_attn.v_proj.weight": "model-00080-of-00179.safetensors", + "model.layers.43.input_layernorm.weight": "model-00084-of-00179.safetensors", + "model.layers.43.mlp.gate.e_score_correction_bias": "model-00084-of-00179.safetensors", + "model.layers.43.mlp.gate.weight": "model-00084-of-00179.safetensors", + "model.layers.43.mlp.moe_mlp.experts.weight": "model-00081-of-00179.safetensors", + "model.layers.43.mlp.moe_mlp.output_experts.weight": "model-00084-of-00179.safetensors", + "model.layers.43.mlp.shared_experts.down_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.43.mlp.shared_experts.gate_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.43.mlp.shared_experts.up_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.43.post_attention_layernorm.weight": "model-00084-of-00179.safetensors", + "model.layers.43.self_attn.k_norm.weight": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.k_proj.bias": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.k_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.o_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.q_norm.weight": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.q_proj.bias": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.q_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.v_proj.bias": "model-00082-of-00179.safetensors", + "model.layers.43.self_attn.v_proj.weight": "model-00082-of-00179.safetensors", + "model.layers.44.input_layernorm.weight": "model-00086-of-00179.safetensors", + "model.layers.44.mlp.gate.e_score_correction_bias": "model-00086-of-00179.safetensors", + "model.layers.44.mlp.gate.weight": "model-00086-of-00179.safetensors", + "model.layers.44.mlp.moe_mlp.experts.weight": "model-00083-of-00179.safetensors", + "model.layers.44.mlp.moe_mlp.output_experts.weight": "model-00086-of-00179.safetensors", + "model.layers.44.mlp.shared_experts.down_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.44.mlp.shared_experts.gate_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.44.mlp.shared_experts.up_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.44.post_attention_layernorm.weight": "model-00086-of-00179.safetensors", + "model.layers.44.self_attn.k_norm.weight": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.k_proj.bias": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.k_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.o_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.q_norm.weight": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.q_proj.bias": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.q_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.v_proj.bias": "model-00084-of-00179.safetensors", + "model.layers.44.self_attn.v_proj.weight": "model-00084-of-00179.safetensors", + "model.layers.45.input_layernorm.weight": "model-00088-of-00179.safetensors", + "model.layers.45.mlp.gate.e_score_correction_bias": "model-00088-of-00179.safetensors", + "model.layers.45.mlp.gate.weight": "model-00088-of-00179.safetensors", + "model.layers.45.mlp.moe_mlp.experts.weight": "model-00085-of-00179.safetensors", + "model.layers.45.mlp.moe_mlp.output_experts.weight": "model-00088-of-00179.safetensors", + "model.layers.45.mlp.shared_experts.down_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.45.mlp.shared_experts.gate_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.45.mlp.shared_experts.up_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.45.post_attention_layernorm.weight": "model-00088-of-00179.safetensors", + "model.layers.45.self_attn.k_norm.weight": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.k_proj.bias": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.k_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.o_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.q_norm.weight": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.q_proj.bias": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.q_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.v_proj.bias": "model-00086-of-00179.safetensors", + "model.layers.45.self_attn.v_proj.weight": "model-00086-of-00179.safetensors", + "model.layers.46.input_layernorm.weight": "model-00090-of-00179.safetensors", + "model.layers.46.mlp.gate.e_score_correction_bias": "model-00090-of-00179.safetensors", + "model.layers.46.mlp.gate.weight": "model-00090-of-00179.safetensors", + "model.layers.46.mlp.moe_mlp.experts.weight": "model-00087-of-00179.safetensors", + "model.layers.46.mlp.moe_mlp.output_experts.weight": "model-00090-of-00179.safetensors", + "model.layers.46.mlp.shared_experts.down_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.46.mlp.shared_experts.gate_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.46.mlp.shared_experts.up_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.46.post_attention_layernorm.weight": "model-00090-of-00179.safetensors", + "model.layers.46.self_attn.k_norm.weight": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.k_proj.bias": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.k_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.o_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.q_norm.weight": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.q_proj.bias": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.q_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.v_proj.bias": "model-00088-of-00179.safetensors", + "model.layers.46.self_attn.v_proj.weight": "model-00088-of-00179.safetensors", + "model.layers.47.input_layernorm.weight": "model-00092-of-00179.safetensors", + "model.layers.47.mlp.gate.e_score_correction_bias": "model-00092-of-00179.safetensors", + "model.layers.47.mlp.gate.weight": "model-00092-of-00179.safetensors", + "model.layers.47.mlp.moe_mlp.experts.weight": "model-00089-of-00179.safetensors", + "model.layers.47.mlp.moe_mlp.output_experts.weight": "model-00092-of-00179.safetensors", + "model.layers.47.mlp.shared_experts.down_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.47.mlp.shared_experts.gate_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.47.mlp.shared_experts.up_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.47.post_attention_layernorm.weight": "model-00092-of-00179.safetensors", + "model.layers.47.self_attn.k_norm.weight": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.k_proj.bias": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.k_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.o_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.q_norm.weight": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.q_proj.bias": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.q_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.v_proj.bias": "model-00090-of-00179.safetensors", + "model.layers.47.self_attn.v_proj.weight": "model-00090-of-00179.safetensors", + "model.layers.48.input_layernorm.weight": "model-00094-of-00179.safetensors", + "model.layers.48.mlp.gate.e_score_correction_bias": "model-00094-of-00179.safetensors", + "model.layers.48.mlp.gate.weight": "model-00094-of-00179.safetensors", + "model.layers.48.mlp.moe_mlp.experts.weight": "model-00091-of-00179.safetensors", + "model.layers.48.mlp.moe_mlp.output_experts.weight": "model-00094-of-00179.safetensors", + "model.layers.48.mlp.shared_experts.down_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.48.mlp.shared_experts.gate_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.48.mlp.shared_experts.up_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.48.post_attention_layernorm.weight": "model-00094-of-00179.safetensors", + "model.layers.48.self_attn.k_norm.weight": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.k_proj.bias": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.k_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.o_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.q_norm.weight": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.q_proj.bias": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.q_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.v_proj.bias": "model-00092-of-00179.safetensors", + "model.layers.48.self_attn.v_proj.weight": "model-00092-of-00179.safetensors", + "model.layers.49.input_layernorm.weight": "model-00096-of-00179.safetensors", + "model.layers.49.mlp.gate.e_score_correction_bias": "model-00096-of-00179.safetensors", + "model.layers.49.mlp.gate.weight": "model-00096-of-00179.safetensors", + "model.layers.49.mlp.moe_mlp.experts.weight": "model-00093-of-00179.safetensors", + "model.layers.49.mlp.moe_mlp.output_experts.weight": "model-00096-of-00179.safetensors", + "model.layers.49.mlp.shared_experts.down_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.49.mlp.shared_experts.gate_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.49.mlp.shared_experts.up_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.49.post_attention_layernorm.weight": "model-00096-of-00179.safetensors", + "model.layers.49.self_attn.k_norm.weight": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.k_proj.bias": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.k_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.o_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.q_norm.weight": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.q_proj.bias": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.q_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.v_proj.bias": "model-00094-of-00179.safetensors", + "model.layers.49.self_attn.v_proj.weight": "model-00094-of-00179.safetensors", + "model.layers.5.input_layernorm.weight": "model-00008-of-00179.safetensors", + "model.layers.5.mlp.gate.e_score_correction_bias": "model-00008-of-00179.safetensors", + "model.layers.5.mlp.gate.weight": "model-00008-of-00179.safetensors", + "model.layers.5.mlp.moe_mlp.experts.weight": "model-00005-of-00179.safetensors", + "model.layers.5.mlp.moe_mlp.output_experts.weight": "model-00008-of-00179.safetensors", + "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.5.mlp.shared_experts.up_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00008-of-00179.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.k_proj.bias": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.q_proj.bias": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.v_proj.bias": "model-00006-of-00179.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00006-of-00179.safetensors", + "model.layers.50.input_layernorm.weight": "model-00098-of-00179.safetensors", + "model.layers.50.mlp.gate.e_score_correction_bias": "model-00098-of-00179.safetensors", + "model.layers.50.mlp.gate.weight": "model-00098-of-00179.safetensors", + "model.layers.50.mlp.moe_mlp.experts.weight": "model-00095-of-00179.safetensors", + "model.layers.50.mlp.moe_mlp.output_experts.weight": "model-00098-of-00179.safetensors", + "model.layers.50.mlp.shared_experts.down_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.50.mlp.shared_experts.gate_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.50.mlp.shared_experts.up_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.50.post_attention_layernorm.weight": "model-00098-of-00179.safetensors", + "model.layers.50.self_attn.k_norm.weight": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.k_proj.bias": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.k_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.o_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.q_norm.weight": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.q_proj.bias": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.q_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.v_proj.bias": "model-00096-of-00179.safetensors", + "model.layers.50.self_attn.v_proj.weight": "model-00096-of-00179.safetensors", + "model.layers.51.input_layernorm.weight": "model-00100-of-00179.safetensors", + "model.layers.51.mlp.gate.e_score_correction_bias": "model-00100-of-00179.safetensors", + "model.layers.51.mlp.gate.weight": "model-00100-of-00179.safetensors", + "model.layers.51.mlp.moe_mlp.experts.weight": "model-00097-of-00179.safetensors", + "model.layers.51.mlp.moe_mlp.output_experts.weight": "model-00100-of-00179.safetensors", + "model.layers.51.mlp.shared_experts.down_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.51.mlp.shared_experts.gate_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.51.mlp.shared_experts.up_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.51.post_attention_layernorm.weight": "model-00100-of-00179.safetensors", + "model.layers.51.self_attn.k_norm.weight": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.k_proj.bias": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.k_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.o_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.q_norm.weight": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.q_proj.bias": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.q_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.v_proj.bias": "model-00098-of-00179.safetensors", + "model.layers.51.self_attn.v_proj.weight": "model-00098-of-00179.safetensors", + "model.layers.52.input_layernorm.weight": "model-00102-of-00179.safetensors", + "model.layers.52.mlp.gate.e_score_correction_bias": "model-00102-of-00179.safetensors", + "model.layers.52.mlp.gate.weight": "model-00102-of-00179.safetensors", + "model.layers.52.mlp.moe_mlp.experts.weight": "model-00099-of-00179.safetensors", + "model.layers.52.mlp.moe_mlp.output_experts.weight": "model-00102-of-00179.safetensors", + "model.layers.52.mlp.shared_experts.down_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.52.mlp.shared_experts.gate_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.52.mlp.shared_experts.up_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.52.post_attention_layernorm.weight": "model-00102-of-00179.safetensors", + "model.layers.52.self_attn.k_norm.weight": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.k_proj.bias": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.k_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.o_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.q_norm.weight": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.q_proj.bias": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.q_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.v_proj.bias": "model-00100-of-00179.safetensors", + "model.layers.52.self_attn.v_proj.weight": "model-00100-of-00179.safetensors", + "model.layers.53.input_layernorm.weight": "model-00104-of-00179.safetensors", + "model.layers.53.mlp.gate.e_score_correction_bias": "model-00104-of-00179.safetensors", + "model.layers.53.mlp.gate.weight": "model-00104-of-00179.safetensors", + "model.layers.53.mlp.moe_mlp.experts.weight": "model-00101-of-00179.safetensors", + "model.layers.53.mlp.moe_mlp.output_experts.weight": "model-00104-of-00179.safetensors", + "model.layers.53.mlp.shared_experts.down_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.53.mlp.shared_experts.gate_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.53.mlp.shared_experts.up_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.53.post_attention_layernorm.weight": "model-00104-of-00179.safetensors", + "model.layers.53.self_attn.k_norm.weight": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.k_proj.bias": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.k_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.o_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.q_norm.weight": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.q_proj.bias": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.q_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.v_proj.bias": "model-00102-of-00179.safetensors", + "model.layers.53.self_attn.v_proj.weight": "model-00102-of-00179.safetensors", + "model.layers.54.input_layernorm.weight": "model-00106-of-00179.safetensors", + "model.layers.54.mlp.gate.e_score_correction_bias": "model-00106-of-00179.safetensors", + "model.layers.54.mlp.gate.weight": "model-00106-of-00179.safetensors", + "model.layers.54.mlp.moe_mlp.experts.weight": "model-00103-of-00179.safetensors", + "model.layers.54.mlp.moe_mlp.output_experts.weight": "model-00106-of-00179.safetensors", + "model.layers.54.mlp.shared_experts.down_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.54.mlp.shared_experts.gate_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.54.mlp.shared_experts.up_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.54.post_attention_layernorm.weight": "model-00106-of-00179.safetensors", + "model.layers.54.self_attn.k_norm.weight": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.k_proj.bias": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.k_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.o_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.q_norm.weight": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.q_proj.bias": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.q_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.v_proj.bias": "model-00104-of-00179.safetensors", + "model.layers.54.self_attn.v_proj.weight": "model-00104-of-00179.safetensors", + "model.layers.55.input_layernorm.weight": "model-00108-of-00179.safetensors", + "model.layers.55.mlp.gate.e_score_correction_bias": "model-00108-of-00179.safetensors", + "model.layers.55.mlp.gate.weight": "model-00108-of-00179.safetensors", + "model.layers.55.mlp.moe_mlp.experts.weight": "model-00105-of-00179.safetensors", + "model.layers.55.mlp.moe_mlp.output_experts.weight": "model-00108-of-00179.safetensors", + "model.layers.55.mlp.shared_experts.down_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.55.mlp.shared_experts.gate_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.55.mlp.shared_experts.up_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.55.post_attention_layernorm.weight": "model-00108-of-00179.safetensors", + "model.layers.55.self_attn.k_norm.weight": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.k_proj.bias": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.k_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.o_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.q_norm.weight": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.q_proj.bias": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.q_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.v_proj.bias": "model-00106-of-00179.safetensors", + "model.layers.55.self_attn.v_proj.weight": "model-00106-of-00179.safetensors", + "model.layers.56.input_layernorm.weight": "model-00110-of-00179.safetensors", + "model.layers.56.mlp.gate.e_score_correction_bias": "model-00110-of-00179.safetensors", + "model.layers.56.mlp.gate.weight": "model-00110-of-00179.safetensors", + "model.layers.56.mlp.moe_mlp.experts.weight": "model-00107-of-00179.safetensors", + "model.layers.56.mlp.moe_mlp.output_experts.weight": "model-00110-of-00179.safetensors", + "model.layers.56.mlp.shared_experts.down_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.56.mlp.shared_experts.gate_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.56.mlp.shared_experts.up_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.56.post_attention_layernorm.weight": "model-00110-of-00179.safetensors", + "model.layers.56.self_attn.k_norm.weight": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.k_proj.bias": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.k_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.o_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.q_norm.weight": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.q_proj.bias": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.q_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.v_proj.bias": "model-00108-of-00179.safetensors", + "model.layers.56.self_attn.v_proj.weight": "model-00108-of-00179.safetensors", + "model.layers.57.input_layernorm.weight": "model-00112-of-00179.safetensors", + "model.layers.57.mlp.gate.e_score_correction_bias": "model-00112-of-00179.safetensors", + "model.layers.57.mlp.gate.weight": "model-00112-of-00179.safetensors", + "model.layers.57.mlp.moe_mlp.experts.weight": "model-00109-of-00179.safetensors", + "model.layers.57.mlp.moe_mlp.output_experts.weight": "model-00112-of-00179.safetensors", + "model.layers.57.mlp.shared_experts.down_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.57.mlp.shared_experts.gate_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.57.mlp.shared_experts.up_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.57.post_attention_layernorm.weight": "model-00112-of-00179.safetensors", + "model.layers.57.self_attn.k_norm.weight": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.k_proj.bias": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.k_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.o_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.q_norm.weight": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.q_proj.bias": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.q_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.v_proj.bias": "model-00110-of-00179.safetensors", + "model.layers.57.self_attn.v_proj.weight": "model-00110-of-00179.safetensors", + "model.layers.58.input_layernorm.weight": "model-00114-of-00179.safetensors", + "model.layers.58.mlp.gate.e_score_correction_bias": "model-00114-of-00179.safetensors", + "model.layers.58.mlp.gate.weight": "model-00114-of-00179.safetensors", + "model.layers.58.mlp.moe_mlp.experts.weight": "model-00111-of-00179.safetensors", + "model.layers.58.mlp.moe_mlp.output_experts.weight": "model-00114-of-00179.safetensors", + "model.layers.58.mlp.shared_experts.down_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.58.mlp.shared_experts.gate_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.58.mlp.shared_experts.up_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.58.post_attention_layernorm.weight": "model-00114-of-00179.safetensors", + "model.layers.58.self_attn.k_norm.weight": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.k_proj.bias": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.k_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.o_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.q_norm.weight": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.q_proj.bias": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.q_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.v_proj.bias": "model-00112-of-00179.safetensors", + "model.layers.58.self_attn.v_proj.weight": "model-00112-of-00179.safetensors", + "model.layers.59.input_layernorm.weight": "model-00116-of-00179.safetensors", + "model.layers.59.mlp.gate.e_score_correction_bias": "model-00116-of-00179.safetensors", + "model.layers.59.mlp.gate.weight": "model-00116-of-00179.safetensors", + "model.layers.59.mlp.moe_mlp.experts.weight": "model-00113-of-00179.safetensors", + "model.layers.59.mlp.moe_mlp.output_experts.weight": "model-00116-of-00179.safetensors", + "model.layers.59.mlp.shared_experts.down_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.59.mlp.shared_experts.gate_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.59.mlp.shared_experts.up_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.59.post_attention_layernorm.weight": "model-00116-of-00179.safetensors", + "model.layers.59.self_attn.k_norm.weight": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.k_proj.bias": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.k_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.o_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.q_norm.weight": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.q_proj.bias": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.q_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.v_proj.bias": "model-00114-of-00179.safetensors", + "model.layers.59.self_attn.v_proj.weight": "model-00114-of-00179.safetensors", + "model.layers.6.input_layernorm.weight": "model-00010-of-00179.safetensors", + "model.layers.6.mlp.gate.e_score_correction_bias": "model-00010-of-00179.safetensors", + "model.layers.6.mlp.gate.weight": "model-00010-of-00179.safetensors", + "model.layers.6.mlp.moe_mlp.experts.weight": "model-00007-of-00179.safetensors", + "model.layers.6.mlp.moe_mlp.output_experts.weight": "model-00010-of-00179.safetensors", + "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.6.mlp.shared_experts.up_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00010-of-00179.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.k_proj.bias": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.q_proj.bias": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.v_proj.bias": "model-00008-of-00179.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00008-of-00179.safetensors", + "model.layers.60.input_layernorm.weight": "model-00118-of-00179.safetensors", + "model.layers.60.mlp.gate.e_score_correction_bias": "model-00118-of-00179.safetensors", + "model.layers.60.mlp.gate.weight": "model-00118-of-00179.safetensors", + "model.layers.60.mlp.moe_mlp.experts.weight": "model-00115-of-00179.safetensors", + "model.layers.60.mlp.moe_mlp.output_experts.weight": "model-00118-of-00179.safetensors", + "model.layers.60.mlp.shared_experts.down_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.60.mlp.shared_experts.gate_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.60.mlp.shared_experts.up_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.60.post_attention_layernorm.weight": "model-00118-of-00179.safetensors", + "model.layers.60.self_attn.k_norm.weight": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.k_proj.bias": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.k_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.o_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.q_norm.weight": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.q_proj.bias": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.q_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.v_proj.bias": "model-00116-of-00179.safetensors", + "model.layers.60.self_attn.v_proj.weight": "model-00116-of-00179.safetensors", + "model.layers.61.input_layernorm.weight": "model-00120-of-00179.safetensors", + "model.layers.61.mlp.gate.e_score_correction_bias": "model-00120-of-00179.safetensors", + "model.layers.61.mlp.gate.weight": "model-00120-of-00179.safetensors", + "model.layers.61.mlp.moe_mlp.experts.weight": "model-00117-of-00179.safetensors", + "model.layers.61.mlp.moe_mlp.output_experts.weight": "model-00120-of-00179.safetensors", + "model.layers.61.mlp.shared_experts.down_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.61.mlp.shared_experts.gate_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.61.mlp.shared_experts.up_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.61.post_attention_layernorm.weight": "model-00120-of-00179.safetensors", + "model.layers.61.self_attn.k_norm.weight": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.k_proj.bias": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.k_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.o_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.q_norm.weight": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.q_proj.bias": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.q_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.v_proj.bias": "model-00118-of-00179.safetensors", + "model.layers.61.self_attn.v_proj.weight": "model-00118-of-00179.safetensors", + "model.layers.62.input_layernorm.weight": "model-00122-of-00179.safetensors", + "model.layers.62.mlp.gate.e_score_correction_bias": "model-00122-of-00179.safetensors", + "model.layers.62.mlp.gate.weight": "model-00122-of-00179.safetensors", + "model.layers.62.mlp.moe_mlp.experts.weight": "model-00119-of-00179.safetensors", + "model.layers.62.mlp.moe_mlp.output_experts.weight": "model-00122-of-00179.safetensors", + "model.layers.62.mlp.shared_experts.down_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.62.mlp.shared_experts.gate_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.62.mlp.shared_experts.up_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.62.post_attention_layernorm.weight": "model-00122-of-00179.safetensors", + "model.layers.62.self_attn.k_norm.weight": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.k_proj.bias": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.k_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.o_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.q_norm.weight": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.q_proj.bias": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.q_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.v_proj.bias": "model-00120-of-00179.safetensors", + "model.layers.62.self_attn.v_proj.weight": "model-00120-of-00179.safetensors", + "model.layers.63.input_layernorm.weight": "model-00124-of-00179.safetensors", + "model.layers.63.mlp.gate.e_score_correction_bias": "model-00124-of-00179.safetensors", + "model.layers.63.mlp.gate.weight": "model-00124-of-00179.safetensors", + "model.layers.63.mlp.moe_mlp.experts.weight": "model-00121-of-00179.safetensors", + "model.layers.63.mlp.moe_mlp.output_experts.weight": "model-00124-of-00179.safetensors", + "model.layers.63.mlp.shared_experts.down_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.63.mlp.shared_experts.gate_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.63.mlp.shared_experts.up_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.63.post_attention_layernorm.weight": "model-00124-of-00179.safetensors", + "model.layers.63.self_attn.k_norm.weight": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.k_proj.bias": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.k_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.o_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.q_norm.weight": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.q_proj.bias": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.q_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.v_proj.bias": "model-00122-of-00179.safetensors", + "model.layers.63.self_attn.v_proj.weight": "model-00122-of-00179.safetensors", + "model.layers.64.input_layernorm.weight": "model-00126-of-00179.safetensors", + "model.layers.64.mlp.gate.e_score_correction_bias": "model-00126-of-00179.safetensors", + "model.layers.64.mlp.gate.weight": "model-00126-of-00179.safetensors", + "model.layers.64.mlp.moe_mlp.experts.weight": "model-00123-of-00179.safetensors", + "model.layers.64.mlp.moe_mlp.output_experts.weight": "model-00126-of-00179.safetensors", + "model.layers.64.mlp.shared_experts.down_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.64.mlp.shared_experts.gate_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.64.mlp.shared_experts.up_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.64.post_attention_layernorm.weight": "model-00126-of-00179.safetensors", + "model.layers.64.self_attn.k_norm.weight": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.k_proj.bias": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.k_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.o_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.q_norm.weight": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.q_proj.bias": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.q_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.v_proj.bias": "model-00124-of-00179.safetensors", + "model.layers.64.self_attn.v_proj.weight": "model-00124-of-00179.safetensors", + "model.layers.65.input_layernorm.weight": "model-00128-of-00179.safetensors", + "model.layers.65.mlp.gate.e_score_correction_bias": "model-00128-of-00179.safetensors", + "model.layers.65.mlp.gate.weight": "model-00128-of-00179.safetensors", + "model.layers.65.mlp.moe_mlp.experts.weight": "model-00125-of-00179.safetensors", + "model.layers.65.mlp.moe_mlp.output_experts.weight": "model-00128-of-00179.safetensors", + "model.layers.65.mlp.shared_experts.down_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.65.mlp.shared_experts.gate_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.65.mlp.shared_experts.up_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.65.post_attention_layernorm.weight": "model-00128-of-00179.safetensors", + "model.layers.65.self_attn.k_norm.weight": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.k_proj.bias": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.k_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.o_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.q_norm.weight": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.q_proj.bias": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.q_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.v_proj.bias": "model-00126-of-00179.safetensors", + "model.layers.65.self_attn.v_proj.weight": "model-00126-of-00179.safetensors", + "model.layers.66.input_layernorm.weight": "model-00130-of-00179.safetensors", + "model.layers.66.mlp.gate.e_score_correction_bias": "model-00130-of-00179.safetensors", + "model.layers.66.mlp.gate.weight": "model-00130-of-00179.safetensors", + "model.layers.66.mlp.moe_mlp.experts.weight": "model-00127-of-00179.safetensors", + "model.layers.66.mlp.moe_mlp.output_experts.weight": "model-00130-of-00179.safetensors", + "model.layers.66.mlp.shared_experts.down_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.66.mlp.shared_experts.gate_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.66.mlp.shared_experts.up_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.66.post_attention_layernorm.weight": "model-00130-of-00179.safetensors", + "model.layers.66.self_attn.k_norm.weight": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.k_proj.bias": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.k_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.o_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.q_norm.weight": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.q_proj.bias": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.q_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.v_proj.bias": "model-00128-of-00179.safetensors", + "model.layers.66.self_attn.v_proj.weight": "model-00128-of-00179.safetensors", + "model.layers.67.input_layernorm.weight": "model-00132-of-00179.safetensors", + "model.layers.67.mlp.gate.e_score_correction_bias": "model-00132-of-00179.safetensors", + "model.layers.67.mlp.gate.weight": "model-00132-of-00179.safetensors", + "model.layers.67.mlp.moe_mlp.experts.weight": "model-00129-of-00179.safetensors", + "model.layers.67.mlp.moe_mlp.output_experts.weight": "model-00132-of-00179.safetensors", + "model.layers.67.mlp.shared_experts.down_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.67.mlp.shared_experts.gate_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.67.mlp.shared_experts.up_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.67.post_attention_layernorm.weight": "model-00132-of-00179.safetensors", + "model.layers.67.self_attn.k_norm.weight": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.k_proj.bias": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.k_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.o_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.q_norm.weight": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.q_proj.bias": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.q_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.v_proj.bias": "model-00130-of-00179.safetensors", + "model.layers.67.self_attn.v_proj.weight": "model-00130-of-00179.safetensors", + "model.layers.68.input_layernorm.weight": "model-00134-of-00179.safetensors", + "model.layers.68.mlp.gate.e_score_correction_bias": "model-00134-of-00179.safetensors", + "model.layers.68.mlp.gate.weight": "model-00134-of-00179.safetensors", + "model.layers.68.mlp.moe_mlp.experts.weight": "model-00131-of-00179.safetensors", + "model.layers.68.mlp.moe_mlp.output_experts.weight": "model-00134-of-00179.safetensors", + "model.layers.68.mlp.shared_experts.down_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.68.mlp.shared_experts.gate_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.68.mlp.shared_experts.up_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.68.post_attention_layernorm.weight": "model-00134-of-00179.safetensors", + "model.layers.68.self_attn.k_norm.weight": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.k_proj.bias": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.k_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.o_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.q_norm.weight": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.q_proj.bias": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.q_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.v_proj.bias": "model-00132-of-00179.safetensors", + "model.layers.68.self_attn.v_proj.weight": "model-00132-of-00179.safetensors", + "model.layers.69.input_layernorm.weight": "model-00136-of-00179.safetensors", + "model.layers.69.mlp.gate.e_score_correction_bias": "model-00136-of-00179.safetensors", + "model.layers.69.mlp.gate.weight": "model-00136-of-00179.safetensors", + "model.layers.69.mlp.moe_mlp.experts.weight": "model-00133-of-00179.safetensors", + "model.layers.69.mlp.moe_mlp.output_experts.weight": "model-00136-of-00179.safetensors", + "model.layers.69.mlp.shared_experts.down_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.69.mlp.shared_experts.gate_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.69.mlp.shared_experts.up_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.69.post_attention_layernorm.weight": "model-00136-of-00179.safetensors", + "model.layers.69.self_attn.k_norm.weight": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.k_proj.bias": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.k_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.o_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.q_norm.weight": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.q_proj.bias": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.q_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.v_proj.bias": "model-00134-of-00179.safetensors", + "model.layers.69.self_attn.v_proj.weight": "model-00134-of-00179.safetensors", + "model.layers.7.input_layernorm.weight": "model-00012-of-00179.safetensors", + "model.layers.7.mlp.gate.e_score_correction_bias": "model-00012-of-00179.safetensors", + "model.layers.7.mlp.gate.weight": "model-00012-of-00179.safetensors", + "model.layers.7.mlp.moe_mlp.experts.weight": "model-00009-of-00179.safetensors", + "model.layers.7.mlp.moe_mlp.output_experts.weight": "model-00012-of-00179.safetensors", + "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00012-of-00179.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.k_proj.bias": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.q_proj.bias": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.v_proj.bias": "model-00010-of-00179.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00010-of-00179.safetensors", + "model.layers.70.input_layernorm.weight": "model-00138-of-00179.safetensors", + "model.layers.70.mlp.gate.e_score_correction_bias": "model-00138-of-00179.safetensors", + "model.layers.70.mlp.gate.weight": "model-00138-of-00179.safetensors", + "model.layers.70.mlp.moe_mlp.experts.weight": "model-00135-of-00179.safetensors", + "model.layers.70.mlp.moe_mlp.output_experts.weight": "model-00138-of-00179.safetensors", + "model.layers.70.mlp.shared_experts.down_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.70.mlp.shared_experts.gate_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.70.mlp.shared_experts.up_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.70.post_attention_layernorm.weight": "model-00138-of-00179.safetensors", + "model.layers.70.self_attn.k_norm.weight": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.k_proj.bias": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.k_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.o_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.q_norm.weight": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.q_proj.bias": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.q_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.v_proj.bias": "model-00136-of-00179.safetensors", + "model.layers.70.self_attn.v_proj.weight": "model-00136-of-00179.safetensors", + "model.layers.71.input_layernorm.weight": "model-00140-of-00179.safetensors", + "model.layers.71.mlp.gate.e_score_correction_bias": "model-00140-of-00179.safetensors", + "model.layers.71.mlp.gate.weight": "model-00140-of-00179.safetensors", + "model.layers.71.mlp.moe_mlp.experts.weight": "model-00137-of-00179.safetensors", + "model.layers.71.mlp.moe_mlp.output_experts.weight": "model-00140-of-00179.safetensors", + "model.layers.71.mlp.shared_experts.down_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.71.mlp.shared_experts.gate_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.71.mlp.shared_experts.up_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.71.post_attention_layernorm.weight": "model-00140-of-00179.safetensors", + "model.layers.71.self_attn.k_norm.weight": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.k_proj.bias": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.k_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.o_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.q_norm.weight": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.q_proj.bias": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.q_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.v_proj.bias": "model-00138-of-00179.safetensors", + "model.layers.71.self_attn.v_proj.weight": "model-00138-of-00179.safetensors", + "model.layers.72.input_layernorm.weight": "model-00142-of-00179.safetensors", + "model.layers.72.mlp.gate.e_score_correction_bias": "model-00142-of-00179.safetensors", + "model.layers.72.mlp.gate.weight": "model-00142-of-00179.safetensors", + "model.layers.72.mlp.moe_mlp.experts.weight": "model-00139-of-00179.safetensors", + "model.layers.72.mlp.moe_mlp.output_experts.weight": "model-00142-of-00179.safetensors", + "model.layers.72.mlp.shared_experts.down_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.72.mlp.shared_experts.gate_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.72.mlp.shared_experts.up_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.72.post_attention_layernorm.weight": "model-00142-of-00179.safetensors", + "model.layers.72.self_attn.k_norm.weight": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.k_proj.bias": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.k_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.o_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.q_norm.weight": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.q_proj.bias": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.q_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.v_proj.bias": "model-00140-of-00179.safetensors", + "model.layers.72.self_attn.v_proj.weight": "model-00140-of-00179.safetensors", + "model.layers.73.input_layernorm.weight": "model-00144-of-00179.safetensors", + "model.layers.73.mlp.gate.e_score_correction_bias": "model-00144-of-00179.safetensors", + "model.layers.73.mlp.gate.weight": "model-00144-of-00179.safetensors", + "model.layers.73.mlp.moe_mlp.experts.weight": "model-00141-of-00179.safetensors", + "model.layers.73.mlp.moe_mlp.output_experts.weight": "model-00144-of-00179.safetensors", + "model.layers.73.mlp.shared_experts.down_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.73.mlp.shared_experts.gate_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.73.mlp.shared_experts.up_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.73.post_attention_layernorm.weight": "model-00144-of-00179.safetensors", + "model.layers.73.self_attn.k_norm.weight": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.k_proj.bias": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.k_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.o_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.q_norm.weight": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.q_proj.bias": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.q_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.v_proj.bias": "model-00142-of-00179.safetensors", + "model.layers.73.self_attn.v_proj.weight": "model-00142-of-00179.safetensors", + "model.layers.74.input_layernorm.weight": "model-00146-of-00179.safetensors", + "model.layers.74.mlp.gate.e_score_correction_bias": "model-00146-of-00179.safetensors", + "model.layers.74.mlp.gate.weight": "model-00146-of-00179.safetensors", + "model.layers.74.mlp.moe_mlp.experts.weight": "model-00143-of-00179.safetensors", + "model.layers.74.mlp.moe_mlp.output_experts.weight": "model-00146-of-00179.safetensors", + "model.layers.74.mlp.shared_experts.down_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.74.mlp.shared_experts.gate_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.74.mlp.shared_experts.up_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.74.post_attention_layernorm.weight": "model-00146-of-00179.safetensors", + "model.layers.74.self_attn.k_norm.weight": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.k_proj.bias": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.k_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.o_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.q_norm.weight": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.q_proj.bias": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.q_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.v_proj.bias": "model-00144-of-00179.safetensors", + "model.layers.74.self_attn.v_proj.weight": "model-00144-of-00179.safetensors", + "model.layers.75.input_layernorm.weight": "model-00148-of-00179.safetensors", + "model.layers.75.mlp.gate.e_score_correction_bias": "model-00148-of-00179.safetensors", + "model.layers.75.mlp.gate.weight": "model-00148-of-00179.safetensors", + "model.layers.75.mlp.moe_mlp.experts.weight": "model-00145-of-00179.safetensors", + "model.layers.75.mlp.moe_mlp.output_experts.weight": "model-00148-of-00179.safetensors", + "model.layers.75.mlp.shared_experts.down_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.75.mlp.shared_experts.gate_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.75.mlp.shared_experts.up_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.75.post_attention_layernorm.weight": "model-00148-of-00179.safetensors", + "model.layers.75.self_attn.k_norm.weight": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.k_proj.bias": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.k_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.o_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.q_norm.weight": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.q_proj.bias": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.q_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.v_proj.bias": "model-00146-of-00179.safetensors", + "model.layers.75.self_attn.v_proj.weight": "model-00146-of-00179.safetensors", + "model.layers.76.input_layernorm.weight": "model-00150-of-00179.safetensors", + "model.layers.76.mlp.gate.e_score_correction_bias": "model-00150-of-00179.safetensors", + "model.layers.76.mlp.gate.weight": "model-00150-of-00179.safetensors", + "model.layers.76.mlp.moe_mlp.experts.weight": "model-00147-of-00179.safetensors", + "model.layers.76.mlp.moe_mlp.output_experts.weight": "model-00150-of-00179.safetensors", + "model.layers.76.mlp.shared_experts.down_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.76.mlp.shared_experts.gate_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.76.mlp.shared_experts.up_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.76.post_attention_layernorm.weight": "model-00150-of-00179.safetensors", + "model.layers.76.self_attn.k_norm.weight": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.k_proj.bias": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.k_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.o_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.q_norm.weight": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.q_proj.bias": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.q_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.v_proj.bias": "model-00148-of-00179.safetensors", + "model.layers.76.self_attn.v_proj.weight": "model-00148-of-00179.safetensors", + "model.layers.77.input_layernorm.weight": "model-00152-of-00179.safetensors", + "model.layers.77.mlp.gate.e_score_correction_bias": "model-00152-of-00179.safetensors", + "model.layers.77.mlp.gate.weight": "model-00152-of-00179.safetensors", + "model.layers.77.mlp.moe_mlp.experts.weight": "model-00149-of-00179.safetensors", + "model.layers.77.mlp.moe_mlp.output_experts.weight": "model-00152-of-00179.safetensors", + "model.layers.77.mlp.shared_experts.down_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.77.mlp.shared_experts.gate_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.77.mlp.shared_experts.up_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.77.post_attention_layernorm.weight": "model-00152-of-00179.safetensors", + "model.layers.77.self_attn.k_norm.weight": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.k_proj.bias": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.k_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.o_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.q_norm.weight": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.q_proj.bias": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.q_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.v_proj.bias": "model-00150-of-00179.safetensors", + "model.layers.77.self_attn.v_proj.weight": "model-00150-of-00179.safetensors", + "model.layers.78.input_layernorm.weight": "model-00154-of-00179.safetensors", + "model.layers.78.mlp.gate.e_score_correction_bias": "model-00154-of-00179.safetensors", + "model.layers.78.mlp.gate.weight": "model-00154-of-00179.safetensors", + "model.layers.78.mlp.moe_mlp.experts.weight": "model-00151-of-00179.safetensors", + "model.layers.78.mlp.moe_mlp.output_experts.weight": "model-00154-of-00179.safetensors", + "model.layers.78.mlp.shared_experts.down_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.78.mlp.shared_experts.gate_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.78.mlp.shared_experts.up_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.78.post_attention_layernorm.weight": "model-00154-of-00179.safetensors", + "model.layers.78.self_attn.k_norm.weight": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.k_proj.bias": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.k_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.o_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.q_norm.weight": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.q_proj.bias": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.q_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.v_proj.bias": "model-00152-of-00179.safetensors", + "model.layers.78.self_attn.v_proj.weight": "model-00152-of-00179.safetensors", + "model.layers.79.input_layernorm.weight": "model-00156-of-00179.safetensors", + "model.layers.79.mlp.gate.e_score_correction_bias": "model-00156-of-00179.safetensors", + "model.layers.79.mlp.gate.weight": "model-00156-of-00179.safetensors", + "model.layers.79.mlp.moe_mlp.experts.weight": "model-00153-of-00179.safetensors", + "model.layers.79.mlp.moe_mlp.output_experts.weight": "model-00156-of-00179.safetensors", + "model.layers.79.mlp.shared_experts.down_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.79.mlp.shared_experts.gate_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.79.mlp.shared_experts.up_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.79.post_attention_layernorm.weight": "model-00156-of-00179.safetensors", + "model.layers.79.self_attn.k_norm.weight": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.k_proj.bias": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.k_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.o_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.q_norm.weight": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.q_proj.bias": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.q_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.v_proj.bias": "model-00154-of-00179.safetensors", + "model.layers.79.self_attn.v_proj.weight": "model-00154-of-00179.safetensors", + "model.layers.8.input_layernorm.weight": "model-00014-of-00179.safetensors", + "model.layers.8.mlp.gate.e_score_correction_bias": "model-00014-of-00179.safetensors", + "model.layers.8.mlp.gate.weight": "model-00014-of-00179.safetensors", + "model.layers.8.mlp.moe_mlp.experts.weight": "model-00011-of-00179.safetensors", + "model.layers.8.mlp.moe_mlp.output_experts.weight": "model-00014-of-00179.safetensors", + "model.layers.8.mlp.shared_experts.down_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.8.mlp.shared_experts.up_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00014-of-00179.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.k_proj.bias": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.q_proj.bias": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.v_proj.bias": "model-00012-of-00179.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00012-of-00179.safetensors", + "model.layers.80.input_layernorm.weight": "model-00158-of-00179.safetensors", + "model.layers.80.mlp.gate.e_score_correction_bias": "model-00158-of-00179.safetensors", + "model.layers.80.mlp.gate.weight": "model-00158-of-00179.safetensors", + "model.layers.80.mlp.moe_mlp.experts.weight": "model-00155-of-00179.safetensors", + "model.layers.80.mlp.moe_mlp.output_experts.weight": "model-00158-of-00179.safetensors", + "model.layers.80.mlp.shared_experts.down_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.80.mlp.shared_experts.gate_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.80.mlp.shared_experts.up_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.80.post_attention_layernorm.weight": "model-00158-of-00179.safetensors", + "model.layers.80.self_attn.k_norm.weight": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.k_proj.bias": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.k_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.o_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.q_norm.weight": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.q_proj.bias": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.q_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.v_proj.bias": "model-00156-of-00179.safetensors", + "model.layers.80.self_attn.v_proj.weight": "model-00156-of-00179.safetensors", + "model.layers.81.input_layernorm.weight": "model-00160-of-00179.safetensors", + "model.layers.81.mlp.gate.e_score_correction_bias": "model-00160-of-00179.safetensors", + "model.layers.81.mlp.gate.weight": "model-00160-of-00179.safetensors", + "model.layers.81.mlp.moe_mlp.experts.weight": "model-00157-of-00179.safetensors", + "model.layers.81.mlp.moe_mlp.output_experts.weight": "model-00160-of-00179.safetensors", + "model.layers.81.mlp.shared_experts.down_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.81.mlp.shared_experts.gate_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.81.mlp.shared_experts.up_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.81.post_attention_layernorm.weight": "model-00160-of-00179.safetensors", + "model.layers.81.self_attn.k_norm.weight": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.k_proj.bias": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.k_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.o_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.q_norm.weight": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.q_proj.bias": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.q_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.v_proj.bias": "model-00158-of-00179.safetensors", + "model.layers.81.self_attn.v_proj.weight": "model-00158-of-00179.safetensors", + "model.layers.82.input_layernorm.weight": "model-00162-of-00179.safetensors", + "model.layers.82.mlp.gate.e_score_correction_bias": "model-00162-of-00179.safetensors", + "model.layers.82.mlp.gate.weight": "model-00162-of-00179.safetensors", + "model.layers.82.mlp.moe_mlp.experts.weight": "model-00159-of-00179.safetensors", + "model.layers.82.mlp.moe_mlp.output_experts.weight": "model-00162-of-00179.safetensors", + "model.layers.82.mlp.shared_experts.down_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.82.mlp.shared_experts.gate_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.82.mlp.shared_experts.up_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.82.post_attention_layernorm.weight": "model-00162-of-00179.safetensors", + "model.layers.82.self_attn.k_norm.weight": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.k_proj.bias": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.k_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.o_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.q_norm.weight": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.q_proj.bias": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.q_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.v_proj.bias": "model-00160-of-00179.safetensors", + "model.layers.82.self_attn.v_proj.weight": "model-00160-of-00179.safetensors", + "model.layers.83.input_layernorm.weight": "model-00164-of-00179.safetensors", + "model.layers.83.mlp.gate.e_score_correction_bias": "model-00164-of-00179.safetensors", + "model.layers.83.mlp.gate.weight": "model-00164-of-00179.safetensors", + "model.layers.83.mlp.moe_mlp.experts.weight": "model-00161-of-00179.safetensors", + "model.layers.83.mlp.moe_mlp.output_experts.weight": "model-00164-of-00179.safetensors", + "model.layers.83.mlp.shared_experts.down_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.83.mlp.shared_experts.gate_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.83.mlp.shared_experts.up_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.83.post_attention_layernorm.weight": "model-00164-of-00179.safetensors", + "model.layers.83.self_attn.k_norm.weight": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.k_proj.bias": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.k_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.o_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.q_norm.weight": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.q_proj.bias": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.q_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.v_proj.bias": "model-00162-of-00179.safetensors", + "model.layers.83.self_attn.v_proj.weight": "model-00162-of-00179.safetensors", + "model.layers.84.input_layernorm.weight": "model-00166-of-00179.safetensors", + "model.layers.84.mlp.gate.e_score_correction_bias": "model-00166-of-00179.safetensors", + "model.layers.84.mlp.gate.weight": "model-00166-of-00179.safetensors", + "model.layers.84.mlp.moe_mlp.experts.weight": "model-00163-of-00179.safetensors", + "model.layers.84.mlp.moe_mlp.output_experts.weight": "model-00166-of-00179.safetensors", + "model.layers.84.mlp.shared_experts.down_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.84.mlp.shared_experts.gate_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.84.mlp.shared_experts.up_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.84.post_attention_layernorm.weight": "model-00166-of-00179.safetensors", + "model.layers.84.self_attn.k_norm.weight": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.k_proj.bias": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.k_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.o_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.q_norm.weight": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.q_proj.bias": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.q_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.v_proj.bias": "model-00164-of-00179.safetensors", + "model.layers.84.self_attn.v_proj.weight": "model-00164-of-00179.safetensors", + "model.layers.85.input_layernorm.weight": "model-00168-of-00179.safetensors", + "model.layers.85.mlp.gate.e_score_correction_bias": "model-00168-of-00179.safetensors", + "model.layers.85.mlp.gate.weight": "model-00168-of-00179.safetensors", + "model.layers.85.mlp.moe_mlp.experts.weight": "model-00165-of-00179.safetensors", + "model.layers.85.mlp.moe_mlp.output_experts.weight": "model-00168-of-00179.safetensors", + "model.layers.85.mlp.shared_experts.down_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.85.mlp.shared_experts.gate_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.85.mlp.shared_experts.up_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.85.post_attention_layernorm.weight": "model-00168-of-00179.safetensors", + "model.layers.85.self_attn.k_norm.weight": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.k_proj.bias": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.k_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.o_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.q_norm.weight": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.q_proj.bias": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.q_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.v_proj.bias": "model-00166-of-00179.safetensors", + "model.layers.85.self_attn.v_proj.weight": "model-00166-of-00179.safetensors", + "model.layers.86.input_layernorm.weight": "model-00170-of-00179.safetensors", + "model.layers.86.mlp.gate.e_score_correction_bias": "model-00170-of-00179.safetensors", + "model.layers.86.mlp.gate.weight": "model-00170-of-00179.safetensors", + "model.layers.86.mlp.moe_mlp.experts.weight": "model-00167-of-00179.safetensors", + "model.layers.86.mlp.moe_mlp.output_experts.weight": "model-00170-of-00179.safetensors", + "model.layers.86.mlp.shared_experts.down_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.86.mlp.shared_experts.gate_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.86.mlp.shared_experts.up_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.86.post_attention_layernorm.weight": "model-00170-of-00179.safetensors", + "model.layers.86.self_attn.k_norm.weight": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.k_proj.bias": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.k_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.o_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.q_norm.weight": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.q_proj.bias": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.q_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.v_proj.bias": "model-00168-of-00179.safetensors", + "model.layers.86.self_attn.v_proj.weight": "model-00168-of-00179.safetensors", + "model.layers.87.input_layernorm.weight": "model-00172-of-00179.safetensors", + "model.layers.87.mlp.gate.e_score_correction_bias": "model-00172-of-00179.safetensors", + "model.layers.87.mlp.gate.weight": "model-00172-of-00179.safetensors", + "model.layers.87.mlp.moe_mlp.experts.weight": "model-00169-of-00179.safetensors", + "model.layers.87.mlp.moe_mlp.output_experts.weight": "model-00172-of-00179.safetensors", + "model.layers.87.mlp.shared_experts.down_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.87.mlp.shared_experts.gate_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.87.mlp.shared_experts.up_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.87.post_attention_layernorm.weight": "model-00172-of-00179.safetensors", + "model.layers.87.self_attn.k_norm.weight": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.k_proj.bias": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.k_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.o_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.q_norm.weight": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.q_proj.bias": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.q_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.v_proj.bias": "model-00170-of-00179.safetensors", + "model.layers.87.self_attn.v_proj.weight": "model-00170-of-00179.safetensors", + "model.layers.88.input_layernorm.weight": "model-00174-of-00179.safetensors", + "model.layers.88.mlp.gate.e_score_correction_bias": "model-00174-of-00179.safetensors", + "model.layers.88.mlp.gate.weight": "model-00174-of-00179.safetensors", + "model.layers.88.mlp.moe_mlp.experts.weight": "model-00171-of-00179.safetensors", + "model.layers.88.mlp.moe_mlp.output_experts.weight": "model-00174-of-00179.safetensors", + "model.layers.88.mlp.shared_experts.down_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.88.mlp.shared_experts.gate_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.88.mlp.shared_experts.up_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.88.post_attention_layernorm.weight": "model-00174-of-00179.safetensors", + "model.layers.88.self_attn.k_norm.weight": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.k_proj.bias": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.k_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.o_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.q_norm.weight": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.q_proj.bias": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.q_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.v_proj.bias": "model-00172-of-00179.safetensors", + "model.layers.88.self_attn.v_proj.weight": "model-00172-of-00179.safetensors", + "model.layers.89.input_layernorm.weight": "model-00176-of-00179.safetensors", + "model.layers.89.mlp.gate.e_score_correction_bias": "model-00176-of-00179.safetensors", + "model.layers.89.mlp.gate.weight": "model-00176-of-00179.safetensors", + "model.layers.89.mlp.moe_mlp.experts.weight": "model-00173-of-00179.safetensors", + "model.layers.89.mlp.moe_mlp.output_experts.weight": "model-00176-of-00179.safetensors", + "model.layers.89.mlp.shared_experts.down_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.89.mlp.shared_experts.gate_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.89.mlp.shared_experts.up_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.89.post_attention_layernorm.weight": "model-00176-of-00179.safetensors", + "model.layers.89.self_attn.k_norm.weight": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.k_proj.bias": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.k_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.o_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.q_norm.weight": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.q_proj.bias": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.q_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.v_proj.bias": "model-00174-of-00179.safetensors", + "model.layers.89.self_attn.v_proj.weight": "model-00174-of-00179.safetensors", + "model.layers.9.input_layernorm.weight": "model-00016-of-00179.safetensors", + "model.layers.9.mlp.gate.e_score_correction_bias": "model-00016-of-00179.safetensors", + "model.layers.9.mlp.gate.weight": "model-00016-of-00179.safetensors", + "model.layers.9.mlp.moe_mlp.experts.weight": "model-00013-of-00179.safetensors", + "model.layers.9.mlp.moe_mlp.output_experts.weight": "model-00016-of-00179.safetensors", + "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00016-of-00179.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00016-of-00179.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.k_proj.bias": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.q_proj.bias": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.v_proj.bias": "model-00014-of-00179.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00014-of-00179.safetensors", + "model.layers.90.input_layernorm.weight": "model-00178-of-00179.safetensors", + "model.layers.90.mlp.gate.e_score_correction_bias": "model-00178-of-00179.safetensors", + "model.layers.90.mlp.gate.weight": "model-00178-of-00179.safetensors", + "model.layers.90.mlp.moe_mlp.experts.weight": "model-00175-of-00179.safetensors", + "model.layers.90.mlp.moe_mlp.output_experts.weight": "model-00178-of-00179.safetensors", + "model.layers.90.mlp.shared_experts.down_proj.weight": "model-00178-of-00179.safetensors", + "model.layers.90.mlp.shared_experts.gate_proj.weight": "model-00178-of-00179.safetensors", + "model.layers.90.mlp.shared_experts.up_proj.weight": "model-00178-of-00179.safetensors", + "model.layers.90.post_attention_layernorm.weight": "model-00178-of-00179.safetensors", + "model.layers.90.self_attn.k_norm.weight": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.k_proj.bias": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.k_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.o_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.q_norm.weight": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.q_proj.bias": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.q_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.v_proj.bias": "model-00176-of-00179.safetensors", + "model.layers.90.self_attn.v_proj.weight": "model-00176-of-00179.safetensors", + "model.layers.91.input_layernorm.weight": "model-00179-of-00179.safetensors", + "model.layers.91.mlp.gate.e_score_correction_bias": "model-00179-of-00179.safetensors", + "model.layers.91.mlp.gate.weight": "model-00179-of-00179.safetensors", + "model.layers.91.mlp.moe_mlp.experts.weight": "model-00177-of-00179.safetensors", + "model.layers.91.mlp.moe_mlp.output_experts.weight": "model-00179-of-00179.safetensors", + "model.layers.91.mlp.shared_experts.down_proj.weight": "model-00179-of-00179.safetensors", + "model.layers.91.mlp.shared_experts.gate_proj.weight": "model-00179-of-00179.safetensors", + "model.layers.91.mlp.shared_experts.up_proj.weight": "model-00179-of-00179.safetensors", + "model.layers.91.post_attention_layernorm.weight": "model-00179-of-00179.safetensors", + "model.layers.91.self_attn.k_norm.weight": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.k_proj.bias": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.k_proj.weight": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.o_proj.weight": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.q_norm.weight": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.q_proj.bias": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.q_proj.weight": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.v_proj.bias": "model-00178-of-00179.safetensors", + "model.layers.91.self_attn.v_proj.weight": "model-00178-of-00179.safetensors", + "model.norm.weight": "model-00179-of-00179.safetensors" + } +} diff --git a/modeling_glm4_shared_moe.py b/modeling_glm4_shared_moe.py new file mode 100644 index 0000000000000000000000000000000000000000..c2e59d0793e3655b71d24cb6ee3a677967167647 --- /dev/null +++ b/modeling_glm4_shared_moe.py @@ -0,0 +1,642 @@ +# coding=utf-8 +# Copyright 2025 The ZhipuAI Inc. team and HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Optional, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, DynamicCache +from transformers.generation import GenerationMixin +from transformers.integrations import use_kernel_forward_from_hub +from transformers.masking_utils import create_causal_mask +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.modeling_layers import GradientCheckpointingLayer +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from transformers.processing_utils import Unpack +from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple +from transformers.utils.deprecation import deprecate_kwarg +from transformers.utils.generic import check_model_inputs +from configuration_glm4_shared_moe import Glm4SharedMoeConfig + +import scattermoe + +# Liger Patch # +from liger_kernel.transformers.rms_norm import LigerRMSNorm as Glm4MoeRMSNorm +from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + + # Keep half or full tensor for later concatenation + rotary_dim = cos.shape[-1] + q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:] + k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:] + + # Apply rotary embeddings on the first half or full tensor + q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin) + k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin) + + # Concatenate back to full shape + q_embed = torch.cat([q_embed, q_pass], dim=-1) + k_embed = torch.cat([k_embed, k_pass], dim=-1) + return q_embed, k_embed + + +class Glm4MoeAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Glm4SharedMoeConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.rope_scaling = config.rope_scaling + self.attention_dropout = config.attention_dropout + self.is_causal = True + + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False) + self.use_qk_norm = config.use_qk_norm + if self.use_qk_norm: + self.q_norm = Glm4MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = Glm4MoeRMSNorm(self.head_dim, eps=config.rms_norm_eps) + + @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58") + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_values: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape) + key_states = self.k_proj(hidden_states).view(hidden_shape) + value_states = self.v_proj(hidden_states).view(hidden_shape) + + if self.use_qk_norm: # main diff from Llama + query_states = self.q_norm(query_states) + key_states = self.k_norm(key_states) + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + # sin and cos are specific to RoPE models; position_ids needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Glm4MoeMLP(nn.Module): + def __init__(self, config, hidden_size=None, intermediate_size=None): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size if hidden_size is None else hidden_size + self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size + + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +class Glm4MoeTopkRouter2(nn.Module): + def __init__(self, config: Glm4SharedMoeConfig): + super().__init__() + self.config = config + self.top_k = config.num_experts_per_tok + self.n_routed_experts = config.n_routed_experts + self.routed_scaling_factor = config.routed_scaling_factor + self.n_group = config.n_group + self.topk_group = config.topk_group + self.norm_topk_prob = config.norm_topk_prob + + self.weight = nn.Parameter(torch.empty((self.n_routed_experts, config.hidden_size))) + self.register_buffer("e_score_correction_bias", torch.zeros((self.n_routed_experts), dtype=torch.float32)) + + def forward(self, hidden_states): + hidden_states = hidden_states.view(-1, self.config.hidden_size) + router_logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32)) + scores = router_logits.sigmoid() + scores2 = scores + self.e_score_correction_bias.unsqueeze(0).to(scores.device) + _, selected_experts = torch.topk( + scores2, self.top_k, dim=-1 + ) + routing_weights = scores.gather(1, selected_experts) + if self.norm_topk_prob: + denominator = routing_weights.sum(dim=-1, keepdim=True) + 1e-20 + routing_weights /= denominator + routing_weights = routing_weights * self.routed_scaling_factor + return selected_experts, routing_weights + + +class Glm4MoeTopkRouter(nn.Module): + def __init__(self, config: Glm4SharedMoeConfig): + super().__init__() + self.config = config + self.top_k = config.num_experts_per_tok + self.n_routed_experts = config.n_routed_experts + self.routed_scaling_factor = config.routed_scaling_factor + self.n_group = config.n_group + self.topk_group = config.topk_group + self.norm_topk_prob = config.norm_topk_prob + + self.weight = nn.Parameter(torch.empty((self.n_routed_experts, config.hidden_size))) + self.register_buffer("e_score_correction_bias", torch.zeros((self.n_routed_experts), dtype=torch.float32)) + + @torch.no_grad() + def get_topk_indices(self, scores): + scores_for_choice = scores.view(-1, self.n_routed_experts) + self.e_score_correction_bias.unsqueeze(0).to(scores.device) + group_scores = ( + scores_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group) + .topk(2, dim=-1)[0] + .sum(dim=-1) + ) + group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = ( + group_mask.unsqueeze(-1) + .expand(-1, self.n_group, self.n_routed_experts // self.n_group) + .reshape(-1, self.n_routed_experts) + ) + scores_for_choice = scores_for_choice.masked_fill(~score_mask.bool(), 0.0) + topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] + return topk_indices + + def forward(self, hidden_states): + hidden_states = hidden_states.view(-1, self.config.hidden_size) + router_logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32)) + scores = router_logits.sigmoid() + topk_indices = self.get_topk_indices(scores) + topk_weights = scores.gather(1, topk_indices) + if self.norm_topk_prob: + denominator = topk_weights.sum(dim=-1, keepdim=True) + 1e-20 + topk_weights /= denominator + topk_weights = topk_weights * self.routed_scaling_factor + return topk_indices, topk_weights + + +@use_kernel_forward_from_hub("RMSNorm") +class Glm4MoeRMSNorm2(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Glm4MoeRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class Glm4MoeMoE(nn.Module): + """ + A mixed expert module containing shared experts. + """ + + def __init__(self, config): + super().__init__() + self.config = config + self.moe_mlp = scattermoe.mlp.GLUMLP( + input_size=self.config.hidden_size, + hidden_size=self.config.moe_intermediate_size, + num_experts=self.config.n_routed_experts, + top_k=self.config.num_experts_per_tok, + activation=ACT2FN[config.hidden_act], + ) + self.gate = Glm4MoeTopkRouter(config) + self.shared_experts = Glm4MoeMLP( + config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts + ) + + def forward(self, hidden_states): + residuals = hidden_states + orig_shape = hidden_states.shape + topk_indices, topk_weights = self.gate(hidden_states) + hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) + topk_weights = topk_weights.to(torch.bfloat16) + hs_0 = self.moe_mlp(hidden_states, topk_weights, topk_indices).view(*orig_shape) + res = hs_0 + self.shared_experts(residuals) + return res + + +class Glm4MoeDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: Glm4SharedMoeConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = Glm4MoeAttention(config=config, layer_idx=layer_idx) + + if layer_idx >= config.first_k_dense_replace: + self.mlp = Glm4MoeMoE(config) + else: + self.mlp = Glm4MoeMLP(config) + + self.input_layernorm = Glm4MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Glm4MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58") + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # necessary, but kept here for BC + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class Glm4SharedMoePreTrainedModel(PreTrainedModel): + config: Glm4SharedMoeConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Glm4MoeDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + _can_compile_fullgraph = False + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": Glm4MoeDecoderLayer, + "attentions": Glm4MoeAttention, + } + + def _init_weights(self, module): + super()._init_weights(module) + if isinstance(module, Glm4MoeTopkRouter): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + + +class Glm4MoeRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: Glm4SharedMoeConfig, device=None): + super().__init__() + # BC: "rope_type" was originally "type" + if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict): + self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = self.inv_freq + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class Glm4SharedMoeModel(Glm4SharedMoePreTrainedModel): + _keys_to_ignore_on_load_unexpected = [r"model\.layers\.92.*", r"model\.layers\.46.*"] + + def __init__(self, config: Glm4SharedMoeConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Glm4MoeDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Glm4MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Glm4MoeRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @check_model_inputs + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position: torch.Tensor = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + input_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring +class Glm4SharedMoeForCausalLM(Glm4SharedMoePreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + _tp_plan = {"lm_head": "colwise_rep"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config): + super().__init__(config) + self.model = Glm4SharedMoeModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, Glm4MoeForCausalLM + + >>> model = Glm4MoeForCausalLM.from_pretrained("meta-glm4_moe/Glm4Moe-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-glm4_moe/Glm4Moe-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + kept_hidden_states = hidden_states[:, slice_indices, :] + shift_labels = kwargs.pop("shift_labels", None) + logits = None + loss = None + + skip_logits = self.training and (labels is not None or shift_labels is not None) + if skip_logits: + loss = LigerForCausalLMLoss( + hidden_states=kept_hidden_states, + lm_head_weight=self.lm_head.weight, + labels=labels, + shift_labels=shift_labels, + hidden_size=self.config.hidden_size, + **kwargs, + ) + else: + logits = self.lm_head(hidden_states[:, slice_indices, :]) + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = ["Glm4SharedMoePreTrainedModel", "Glm4SharedMoeModel", "Glm4SharedMoeForCausalLM"] diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..43a71efad1ab3f7e4d66d1e48e7cc44f68f21f3a --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9340665016419c825c4bdabbcc9acc43b7ca2c68ce142724afa829abb1be5efd +size 19970699 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..053f12c9b03d5acbcc921042ea0c87a6baa5d3f7 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,325 @@ +{ + "added_tokens_decoder": { + "151329": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151330": { + "content": "[MASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151331": { + "content": "[gMASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151332": { + "content": "[sMASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151335": { + "content": "<|system|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151336": { + "content": "<|user|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151337": { + "content": "<|assistant|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151338": { + "content": "<|observation|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151339": { + "content": "<|begin_of_image|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151340": { + "content": "<|end_of_image|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151341": { + "content": "<|begin_of_video|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151342": { + "content": "<|end_of_video|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151343": { + "content": "<|begin_of_audio|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151344": { + "content": "<|end_of_audio|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151345": { + "content": "<|begin_of_transcription|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151346": { + "content": "<|end_of_transcription|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151347": { + "content": "<|code_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151348": { + "content": "<|code_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151349": { + "content": "<|code_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151350": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151351": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151352": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151353": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151354": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151355": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151356": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151357": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151358": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151359": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151360": { + "content": "/nothink", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151361": { + "content": "<|begin_of_box|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151362": { + "content": "<|end_of_box|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151363": { + "content": "<|image|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151364": { + "content": "<|video|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|endoftext|>", + "[MASK]", + "[gMASK]", + "[sMASK]", + "", + "", + "<|system|>", + "<|user|>", + "<|assistant|>", + "<|observation|>", + "<|begin_of_image|>", + "<|end_of_image|>", + "<|begin_of_video|>", + "<|end_of_video|>", + "<|begin_of_audio|>", + "<|end_of_audio|>", + "<|begin_of_transcription|>", + "<|end_of_transcription|>", + "<|code_prefix|>", + "<|code_middle|>", + "<|code_suffix|>", + "/nothink" + ], + "clean_up_tokenization_spaces": false, + "do_lower_case": false, + "eos_token": "<|endoftext|>", + "extra_special_tokens": {}, + "model_max_length": 128000, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "remove_space": false, + "tokenizer_class": "PreTrainedTokenizer" +}