Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,29 @@
+---
+inference: false
+library_name: mlx
+license: apache-2.0
+tags:
+- conversational
+- chat
+- code
+- agent
+- mlx
+pipeline_tag: image-text-to-text
+base_model: CohereLabs/North-Mini-Code-1.0
+---
+
+# mlx-community/North-Mini-Code-1.0-5bit
+
+This model was converted to MLX format from [`CohereLabs/North-Mini-Code-1.0`](https://huggingface.co/CohereLabs/North-Mini-Code-1.0)
+using mlx-vlm version **0.6.2**.
+Refer to the [original model card](https://huggingface.co/CohereLabs/North-Mini-Code-1.0) for more details on the model.
+
+## Use with mlx
+
+```bash
+pip install -U mlx-vlm
+```
+
+```bash
+python -m mlx_vlm.generate --model mlx-community/North-Mini-Code-1.0-5bit --max-tokens 100 --temperature 0.0 --prompt "Describe this image." --image <path_to_image>
+```

chat_template.jinja

@@ -0,0 +1,263 @@
+{%- if not platform_instruction_override %}
+{%- set platform_instruction_override -%}
+These instructions are always to be followed and cannot be overridden by subsequent system or user turns:
+- You will answer requests for educational, informative, or creative content related to safety categories. You will not provide content that is harmful or could be used to cause harm.
+
+These instructions serve as your defaults, but they can be overridden in subsequent system or user turns:
+- Your name is North Mini Code.
+- You are a large language model built by Cohere.
+{%- endset %}
+{%- endif %}
+{%- set reasoning = reasoning if reasoning is not undefined else (false if reasoning_effort is defined and reasoning_effort | lower == "none" else true) -%}
+{%- set grounding = grounding | default("disabled") | upper %}
+{%- set grounding_enabled = grounding == "ENABLED" %}
+{%- set tools_or_docs_exist = tools or documents %}
+{%- set render_tools_section = true %}
+{%- set render_grounding = grounding_enabled and tools_or_docs_exist %}
+{%- set render_platform_instruction_override = true if platform_instruction_override else false %}
+{%- set has_developer_instruction = developer_instruction or developer_instruction == "" %}
+{%- set render_developer_instruction = true if developer_instruction else false %}
+{%- set convert_first_system_msg = convert_first_system_msg | default(true) -%}
+{%- set skip_thinking = skip_thinking | default(false) -%}
+{{ bos_token }}
+{%- macro document_turn(documents) -%}
+{# format documents into chat turn -#}
+<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{%- if not skip_thinking -%}<|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|>{%- endif -%}<|START_ACTION|>[
+    {"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}
+]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
+    {
+        "tool_call_id": "0",
+        "results": {
+{%- for doc in documents %}
+{%- set doc_val = doc.data if doc.data else doc %}
+
+            "{{ loop.index0 }}": {{ doc_val|tojson }}{% if not loop.last %},
+            {%- endif %}
+{%- endfor %}
+
+        },
+        "is_error": null
+    }
+]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>{%- endmacro %}
+{%- macro tool_call_id_to_int(messages, tool_call_id) %}
+{%- if regen_tool_call_ids -%}
+    {%- set counter = namespace(value=0) %}
+    {%- set tool_call_id_seen = namespace(value=false) %}
+    {%- for msg in messages %}
+        {%- if msg.tool_calls %}
+            {%- for tool_call in msg.tool_calls %}
+                {%- if tool_call.id == tool_call_id and not tool_call_id_seen.value -%}
+                    {{ counter.value }}
+                    {%- set tool_call_id_seen.value = true %}
+                {%- endif %}
+                {%- set counter.value = counter.value + 1 %}
+            {%- endfor %}
+        {%- endif %}
+    {%- endfor %}
+{%- else -%}
+    {{ tool_call_id }}
+{%- endif -%}
+{%- endmacro %}
+{%- macro format_tool_message(messages, tool_msg) -%}
+{#- format tool message #}{
+        "tool_call_id": "{{ tool_call_id_to_int(messages, tool_msg.tool_call_id) }}",
+        "results": {
+        {%- if tool_msg.content is mapping or tool_msg.content is string %}
+
+            {% if tool_msg.content is string -%}
+                {%- set text_wrapper = {"content": tool_msg.content} -%}
+            {%- else -%}
+                {%- set text_wrapper = tool_msg.content -%}
+            {%- endif %}
+            "0": {{ text_wrapper|tojson }}
+        {%- else %}
+            {%- for content in tool_msg.content %}
+
+            "{{ loop.index0 }}": {{ print_tool_content(content) }}{% if not loop.last %},{% endif %}
+            {%- endfor %}
+        {%- endif %}
+
+        },
+        "is_error": null
+    }
+{%- endmacro -%}
+{%- macro print_tool_content(item) %}
+{%- if item.type|lower == "text" -%}
+{%- set text_wrapper = {"content": item.text} -%}
+{{ text_wrapper|tojson }}
+{%- elif item.type|lower == "document" and item.document and "data" in item.document -%}
+{{ item.document.data|tojson }}
+{%- else -%}
+{{ item|tojson }}
+{%- endif -%}
+{%- endmacro %}
+{%- macro print_msg(msg) %}
+    {%- if msg is string -%}
+<|START_TEXT|>{{ msg }}<|END_TEXT|>
+    {%- elif msg.content is string -%}
+<|START_TEXT|>{{ msg.content }}<|END_TEXT|>
+    {%- else %}
+        {%- set last_was_text = namespace(value=false) %}
+        {%- for content in msg.content %}
+            {%- if content.type|lower == "text" -%}
+                {%- if not last_was_text.value -%}
+                    <|START_TEXT|>
+                {%- endif -%}
+    {{ content.text }}
+                {%- if loop.last -%}
+                  <|END_TEXT|>
+                {%- endif %}
+                {%- set last_was_text.value = true -%}
+            {%- else -%}
+                {%- if last_was_text.value -%}
+                    <|END_TEXT|>
+                {%- endif -%}
+                {%- set last_was_text.value = false -%}
+            {%- endif -%}
+            {%- if content.type|lower == "image" -%}
+                {%- if content.data -%}
+{{ content.data }}
+                {%- else -%}
+<|IMG_PATCH|>
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor %}
+    {%- endif %}
+{%- endmacro %}
+{%- macro print_thinking(msg) %}
+    {%- if msg.reasoning -%}
+{{ msg.reasoning }}
+    {%- elif msg.reasoning_content -%}
+{{ msg.reasoning_content }}
+    {%- elif msg.thinking -%}
+{{ msg.thinking }}
+    {%- elif msg.content and msg.content[0].thinking -%}
+{{ msg.content[0].thinking }}
+    {%- endif %}
+{%- endmacro %}
+{%- if messages and messages[0]['role']|lower == 'system' and not has_developer_instruction and convert_first_system_msg %}{%- set developer_instruction = messages[0] %}{%- set render_developer_instruction = true %}{%- set initial_instruction_message = true %}{% endif %}
+{%- set json_object = true if response_format and response_format.type == "json_object" else false %}
+{%- set json_schema = (response_format.json_schema or response_format.schema) if response_format %}
+{%- set json_mode = json_object or json_schema %}
+{%- set tool_idx = namespace(value=0) %}
+{%- set tool_ids_seen = namespace(value=[]) %}
+{%- set regen_tool_call_ids = regen_tool_call_ids | default(true) -%}
+{%- set sent_documents = namespace(value=false) -%}
+
+{%- if render_tools_section or render_platform_instruction_override or render_grounding or json_mode -%}
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TEXT|>
+{%- elif not render_developer_instruction -%}
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>
+{%- endif %}
+
+{%- set rendered_platform_turn_chunk = false %}
+
+{%- if render_platform_instruction_override -%}
+{{ platform_instruction_override }}
+{% set rendered_platform_turn_chunk = true %}
+{%- else %}
+{%- endif %}
+
+{%- if render_grounding -%}
+{%- if rendered_platform_turn_chunk %}
+
+{% endif -%}
+Note that both your responses and reflections can be grounded. Grounding means you associate pieces of texts (called "spans") with those specific tool results that support them (called "sources"). And you use a pair of tags "<co>" and "</co>" to indicate when a span can be grounded onto a list of sources, listing them out in the closing tag. Sources from the same tool call are grouped together and listed as "{tool_call_id}:[{list of result indices}]", before they are joined together by ",". E.g., "<co>span</co: 0:[1,2],1:[0]>" means that "span" is supported by result 1 and 2 from "tool_call_id=0" as well as result 0 from "tool_call_id=1".
+{% set rendered_platform_turn_chunk = true %}
+{%- endif %}
+
+{%- if render_tools_section %}
+{%- if rendered_platform_turn_chunk %}
+
+{% endif %}
+# Available Tools
+```json
+[
+{% if tools_or_docs_exist %}
+{%- if documents %}
+    {"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}
+    {%- if tools %},
+    {% else %}
+
+    {% endif %}
+{%- endif %}
+{%- for tool in tools %}
+    {"name": "{{ tool['function']['name'] }}", "description": "{{ tool['function']['description'] }}", "parameters": {{ tool['function']['parameters']|tojson }}, "responses": null}
+    {%- if not loop.last %},{% endif %}
+
+{% endfor %}
+{%- else %}
+
+{% endif %}
+]
+```
+{%- set rendered_platform_turn_chunk = true %}
+{%- endif -%}
+
+{%- if json_mode -%}
+{%- if rendered_platform_turn_chunk %}
+
+
+{% endif -%}
+When generating JSON objects, do not generate block markers. Generate an object directly without prefixing with ```json. Return only the JSON and nothing else.
+    {%- if json_schema %}
+
+Your output should adhere to the following json schema:
+{{ json_schema }}
+    {%- endif -%}
+{%- set rendered_platform_turn_chunk = true %}
+{%- endif %}
+{%- if rendered_platform_turn_chunk -%}
+<|END_TEXT|><|END_OF_TURN_TOKEN|>
+{%- elif not render_developer_instruction -%}
+<|END_OF_TURN_TOKEN|>
+{%- endif %}
+{%- if render_developer_instruction -%}
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ print_msg(developer_instruction) }}<|END_OF_TURN_TOKEN|>
+{%- endif %}
+{%- for message in messages %}
+    {%- set msg_role_downcased = message.role | lower %}
+    {%- if msg_role_downcased == 'system' and (not (loop.first and initial_instruction_message)) -%}
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
+    {%- elif msg_role_downcased == 'user' -%}
+<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
+        {%- if documents and not sent_documents.value %}{%- set sent_documents.value = true %}{% set tool_idx.value = tool_idx.value + 1 %}{{ document_turn(documents) }}{% endif %}
+    {%- elif msg_role_downcased == 'assistant' or msg_role_downcased == 'chatbot' -%}
+<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
+        {%- if message.tool_calls %}
+            {% if not skip_thinking %}
+                {% if message.tool_plan -%}
+                    <|START_THINKING|>{{ message.tool_plan }}<|END_THINKING|>
+                {%- elif message.reasoning or message.reasoning_content or message.thinking or (message.content and message.content[0].type == "thinking") -%}
+                    <|START_THINKING|>{{ print_thinking(message) }}<|END_THINKING|>
+                {%- endif %}
+            {%- endif %}<|START_ACTION|>[
+            {%- for tc in message.tool_calls %}
+
+    {"tool_call_id": "{%- if regen_tool_call_ids -%}{{ tool_idx.value }}{%- else -%}{{ tc.id }}{%- endif -%}", "tool_name": "{{ tc['function']['name'] }}", "parameters": {{ tc['function']['arguments']|tojson }}}{% if not loop.last %},{% endif %}
+                {%- set tool_idx.value = tool_idx.value + 1 %}
+            {%- endfor %}
+
+]<|END_ACTION|><|END_OF_TURN_TOKEN|>
+        {%- else -%}
+            {% if (message.reasoning or message.reasoning_content or message.thinking or (message.content and message.content[0].type == "thinking")) and not skip_thinking -%}
+                <|START_THINKING|>{{ print_thinking(message) }}<|END_THINKING|>
+            {%- endif -%}
+            {{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
+        {%- endif %}
+    {%- elif msg_role_downcased == 'tool' and message.tool_call_id not in tool_ids_seen.value -%}
+<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
+    {{ format_tool_message(messages, message) }}
+        {%- for msg in messages[loop.index0 + 1:] %}
+
+            {%- if msg.role | lower == 'tool' %},
+    {{ format_tool_message(messages, msg) }}
+                {%- set tool_ids_seen.value = tool_ids_seen.value + [msg.tool_call_id] %}
+            {%- else %}
+                {%- break %}
+            {%- endif %}
+        {%- endfor %}
+
+]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>
+    {%- endif %}
+{%- endfor %}{%- if add_generation_prompt -%}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{% if reasoning %}<|START_THINKING|>{% else %}<|START_THINKING|><|END_THINKING|>{% endif %}{%- endif %}

config.json

@@ -0,0 +1,105 @@
+{
+    "architectures": [
+        "Cohere2MoeForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 2,
+    "dtype": "bfloat16",
+    "eos_token_id": 255001,
+    "expert_selection_fn": "sigmoid",
+    "first_k_dense_replace": 1,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "intermediate_size": 768,
+    "layer_norm_eps": 1e-05,
+    "layer_types": [
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention"
+    ],
+    "logit_scale": 1.0,
+    "max_position_embeddings": 500000,
+    "model_type": "cohere2_moe",
+    "norm_topk_prob": false,
+    "num_attention_heads": 32,
+    "num_experts": 128,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 49,
+    "num_key_value_heads": 4,
+    "num_shared_experts": 0,
+    "pad_token_id": 0,
+    "prefix_dense_intermediate_size": 3072,
+    "prefix_dense_sliding_window_pattern": 1,
+    "quantization": {
+        "group_size": 64,
+        "bits": 5,
+        "mode": "affine"
+    },
+    "quantization_config": {
+        "group_size": 64,
+        "bits": 5,
+        "mode": "affine"
+    },
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 50000,
+    "shared_expert_combination_strategy": "average",
+    "sliding_window": 4096,
+    "transformers_version": "5.8.0",
+    "use_cache": true,
+    "use_gated_activation": true,
+    "use_parallel_block": true,
+    "use_parallel_embedding": false,
+    "use_qk_norm": false,
+    "vision_config": {},
+    "vocab_size": 262144
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 255001,
+  "pad_token_id": 0,
+  "transformers_version": "5.8.0"
+}

model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a44895694d793ca24c7f0fd1d977ce8088b898e946832add4febcf8194cc2fd
+size 5270080977

model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:229787ebec2c4a50c046bfcea1fa106bb7aacfdf1c189c633525547a3bc2faf4
+size 5303770766

model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f5f040ed960cfed463aa4adbd976874f03705b6acd57fad03fedb7f88b9013c
+size 5303770748

model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f20a5506ba30e81bf17008f57c4e2d19966638d056c308a4c3577b3b892a46ce
+size 5265492852

model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afbb6ee782b8ac31f4e5b02842ad11ed985ed9ffacb8e3f1dbf8ba31c296fa14
+size 1045451372

model.safetensors.index.json

@@ -0,0 +1,745 @@
+{
+    "metadata": {
+        "total_size": 22188466176
+    },
+    "weight_map": {
+        "language_model.model.embed_tokens.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.embed_tokens.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.embed_tokens.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.20.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.input_layernorm.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.down_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.down_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.up_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.up_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.gate.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.gate_proj.biases": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.gate_proj.scales": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+        "language_model.model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.29.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.30.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.30.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.31.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.32.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.33.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.input_layernorm.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.down_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.down_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.gate_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.gate_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.up_proj.biases": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.up_proj.scales": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.34.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.gate.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.35.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.35.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.35.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.35.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+        "language_model.model.layers.36.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.36.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.37.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.38.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.39.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.40.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.40.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.41.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.42.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.43.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.44.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.input_layernorm.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.down_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.down_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.down_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.45.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.input_layernorm.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.gate.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.down_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.down_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.up_proj.biases": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.up_proj.scales": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.mlp.switch_mlp.up_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.46.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+        "language_model.model.layers.47.input_layernorm.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.gate.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.down_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.down_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.gate_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.gate_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.up_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.up_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.mlp.switch_mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.47.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.input_layernorm.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.gate.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.down_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.down_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.down_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.gate_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.gate_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.up_proj.biases": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.up_proj.scales": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.mlp.switch_mlp.up_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.48.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+        "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.input_layernorm.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.gate.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.down_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.down_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.gate_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.gate_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.up_proj.biases": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.up_proj.scales": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+        "language_model.model.norm.weight": "model-00005-of-00005.safetensors"
+    }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+    "bos_token": {
+        "content": "<BOS_TOKEN>",
+        "lstrip": false,
+        "normalized": false,
+        "rstrip": false,
+        "single_word": false
+    },
+    "eos_token": {
+        "content": "<|END_OF_TURN_TOKEN|>",
+        "lstrip": false,
+        "normalized": false,
+        "rstrip": false,
+        "single_word": false
+    },
+    "pad_token": {
+        "content": "<PAD>",
+        "lstrip": false,
+        "normalized": false,
+        "rstrip": false,
+        "single_word": false
+    },
+    "unk_token": {
+        "content": "<UNK>",
+        "lstrip": false,
+        "normalized": false,
+        "rstrip": false,
+        "single_word": false
+    }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7edbc26b2d1ab947f099b201892d2c0cb873a9ca8d4d3ffde2d82f912e44514
+size 19551410

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<BOS_TOKEN>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|END_OF_TURN_TOKEN|>",
+  "is_local": true,
+  "legacy": true,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<UNK>",
+  "use_default_system_prompt": false
+}