Upload folder using huggingface_hub

README.md

@@ -0,0 +1,82 @@
+---
+library_name: transformers
+pipeline_tag: text-generation
+---
+
+Random weights generated using script derived from
+`yujiepan/deepseek-v3-tiny-random`.
+
+```python
+import os
+from pathlib import Path
+
+import torch
+import transformers
+from huggingface_hub import create_repo, upload_folder
+from transformers import (AutoConfig, AutoModelForCausalLM, AutoTokenizer,
+                          GenerationConfig, enable_full_determinism, pipeline,
+                          set_seed)
+
+model_id = "deepseek-ai/DeepSeek-V3"
+repo_id = "modularai/deepseek-v3-small-random"
+save_path = f"/home/ubuntu/mock-models/{repo_id}"
+
+deepseek_config = AutoConfig.from_pretrained("deepseek-ai/DeepSeek-V3")
+
+config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+config.num_hidden_layers = 2
+config.first_k_dense_replace = 1
+
+# transformers has not supported the customized quantization config
+del config.quantization_config
+
+tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+tokenizer.save_pretrained(save_path)
+
+enable_full_determinism(seed=42)
+model = AutoModelForCausalLM.from_config(
+    config, torch_dtype=torch.bfloat16, trust_remote_code=True,
+)
+
+try:
+    model.generation_config = GenerationConfig.from_pretrained(
+        model_id, trust_remote_code=True)
+except:
+    print("No generation config found")
+
+# This fixes the NaN values
+model.model.layers[1].mlp.gate.e_score_correction_bias = torch.nn.Parameter(
+torch.randn_like(
+model.model.layers[1].mlp.gate.e_score_correction_bias) * 1e-2)
+
+num_params = 0
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        if 'experts' in name and 'experts.0.' not in name:  # avoid printing too much
+            pass
+        else:
+            print(name, p.shape)
+        # torch.nn.init.uniform_(p, -0.2, 0.2)
+        num_params += p.numel()
+print(f"Number of parameters: {num_params / 1e6:.2f}M")
+model.save_pretrained(save_path)
+
+# patch to use official modeling codes
+auto_map = config.auto_map
+import json
+with open(f"{save_path}/config.json", "r") as f:
+    config_json = json.load(f)
+    config_json['auto_map'] = auto_map
+with open(f"{save_path}/config.json", "w") as f:
+    json.dump(config_json, f, indent=2)
+
+! cat {save_path}/config.json
+
+del model
+del tokenizer
+for p in Path(save_path).glob("*.py"):
+    os.remove(p)
+
+os.system(f"ls -alh {save_path}")
+torch.use_deterministic_algorithms(False)
+```

chat_template.jinja

@@ -0,0 +1,11 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '
+
+' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'
+' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'
+<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}

config.json

@@ -0,0 +1,58 @@
+{
+  "architectures": [
+    "DeepseekV3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_deepseek.DeepseekV3Config",
+    "AutoModel": "modeling_deepseek.DeepseekV3Model",
+    "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
+  },
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "ep_size": 1,
+  "first_k_dense_replace": 1,
+  "hidden_act": "silu",
+  "hidden_size": 7168,
+  "initializer_range": 0.02,
+  "intermediate_size": 18432,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 163840,
+  "model_type": "deepseek_v3",
+  "moe_intermediate_size": 2048,
+  "moe_layer_freq": 1,
+  "n_group": 8,
+  "n_routed_experts": 256,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 128,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 128,
+  "num_nextn_predict_layers": 1,
+  "q_lora_rank": 1536,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 64,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "factor": 40,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 4096,
+    "type": "yarn"
+  },
+  "rope_theta": 10000,
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "tie_word_embeddings": false,
+  "topk_group": 4,
+  "topk_method": "noaux_tc",
+  "transformers_version": "4.56.2",
+  "use_cache": true,
+  "v_head_dim": 128,
+  "vocab_size": 129280
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "transformers_version": "4.56.2"
+}

model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34edf36a9e13b8309bd7634011236cb2996b719ef007351c698c62d444496006
+size 4979995520

model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b6ac13146850e41ba15e8ded1bb8c3eb43a271824bab85dc84961ba3000e4ce
+size 4991242856

model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e63676b3a751949e9da8722c4cbb6ed39e8cc7695d546ac3426fa990a7db60ea
+size 4991242952

model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312ceb79057621563a90c898b047f59e6b88b309cdf522851b38310ca72c370e
+size 4991243024

model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bca36f81f02aefc88988874db69811a19d67e93b1b1d08c0dcb2b0377f08fa3c
+size 4991243024

model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e11a6e55eace87a42bb72afff3d00ad0a61024d1c7edf2badeb65c9ca28fe8ed
+size 2943401680

model.safetensors.index.json

@@ -0,0 +1,805 @@
+{
+  "metadata": {
+    "total_parameters": 13944134912,
+    "total_size": 27888269824
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.kv_a_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.kv_b_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_a_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_a_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_b_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.0.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.0.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.0.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.1.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.1.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.1.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.10.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.10.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.10.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.100.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.100.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.100.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.101.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.101.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.101.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.102.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.102.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.102.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.103.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.103.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.103.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.104.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.104.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.104.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.105.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.105.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.105.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.106.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.106.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.106.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.107.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.107.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.107.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.108.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.108.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.108.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.109.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.109.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.109.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.11.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.11.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.11.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.110.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.110.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.110.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.111.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.111.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.111.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.112.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.112.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.112.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.113.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.113.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.113.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.114.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.114.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.114.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.115.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.115.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.115.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.116.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.116.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.116.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.117.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.117.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.117.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.118.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.118.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.118.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.119.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.119.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.119.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.12.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.12.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.12.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.120.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.120.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.120.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.121.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.121.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.121.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.122.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.122.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.122.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.123.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.123.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.123.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.124.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.124.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.124.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.125.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.125.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.125.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.126.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.126.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.126.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.127.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.127.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.127.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.128.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.128.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.128.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.129.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.129.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.129.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.13.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.13.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.13.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.130.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.130.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.130.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.131.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.131.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.131.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.132.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.132.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.132.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.133.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.133.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.133.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.134.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.134.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.134.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.135.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.135.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.135.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.136.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.136.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.136.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.137.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.137.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.137.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.138.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.138.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.138.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.139.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.139.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.139.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.14.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.14.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.14.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.140.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.140.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.140.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.141.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.141.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.141.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.142.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.142.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.142.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.143.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.143.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.143.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.144.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.144.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.144.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.145.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.145.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.145.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.146.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.146.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.146.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.147.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.147.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.147.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.148.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.148.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.148.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.149.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.149.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.149.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.15.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.15.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.15.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.150.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.150.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.150.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.151.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.151.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.151.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.152.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.152.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.152.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.153.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.153.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.153.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.154.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.154.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.154.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.155.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.155.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.155.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.156.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.156.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.156.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.157.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.157.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.157.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.158.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.158.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.158.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.159.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.159.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.159.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.16.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.16.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.16.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.160.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.160.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.160.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.161.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.161.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.161.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.162.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.162.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.162.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.163.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.163.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.163.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.164.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.164.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.164.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.165.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.165.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.165.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.166.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.166.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.166.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.167.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.167.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.167.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.168.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.168.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.168.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.169.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.169.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.169.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.17.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.17.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.17.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.170.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.170.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.170.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.171.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.171.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.171.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.172.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.172.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.172.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.173.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.173.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.173.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.174.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.174.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.174.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.175.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.175.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.175.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.176.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.176.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.176.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.177.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.177.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.177.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.178.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.178.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.178.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.179.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.179.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.179.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.18.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.18.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.18.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.180.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.180.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.180.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.181.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.181.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.181.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.182.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.182.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.182.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.183.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.183.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.183.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.184.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.184.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.184.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.185.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.185.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.185.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.186.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.186.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.186.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.187.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.187.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.187.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.1.mlp.experts.188.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.188.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.188.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.189.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.189.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.189.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.19.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.19.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.19.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.190.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.190.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.190.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.191.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.191.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.191.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.192.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.192.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.192.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.193.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.193.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.193.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.194.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.194.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.194.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.195.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.195.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.195.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.196.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.196.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.196.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.197.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.197.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.197.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.198.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.198.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.198.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.199.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.199.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.199.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.2.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.2.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.2.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.20.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.20.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.20.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.200.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.200.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.200.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.201.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.201.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.201.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.202.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.202.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.202.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.203.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.203.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.203.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.204.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.204.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.204.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.205.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.205.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.205.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.206.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.206.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.206.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.207.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.207.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.207.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.208.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.208.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.208.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.209.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.209.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.209.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.21.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.21.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.21.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.210.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.210.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.210.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.211.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.211.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.211.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.212.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.212.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.212.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.213.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.213.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.213.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.214.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.214.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.214.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.215.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.215.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.215.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.216.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.216.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.216.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.217.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.217.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.217.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.218.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.218.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.218.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.219.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.219.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.219.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.22.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.22.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.22.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.220.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.220.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.220.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.221.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.221.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.221.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.222.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.222.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.222.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.223.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.223.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.223.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.224.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.224.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.224.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.225.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.225.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.225.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.226.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.226.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.226.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.227.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.227.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.227.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.228.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.228.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.228.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.229.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.229.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.229.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.23.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.23.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.23.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.230.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.230.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.230.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.231.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.231.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.231.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.232.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.232.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.232.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.233.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.233.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.233.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.234.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.234.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.234.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.235.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.235.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.235.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.236.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.236.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.236.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.237.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.237.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.237.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.238.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.238.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.238.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.239.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.239.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.239.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.24.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.24.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.24.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.240.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.240.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.240.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.241.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.241.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.241.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.242.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.242.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.242.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.243.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.243.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.243.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.244.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.244.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.244.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.1.mlp.experts.245.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.245.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.245.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.246.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.246.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.246.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.247.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.247.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.247.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.248.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.248.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.248.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.249.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.249.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.249.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.25.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.25.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.25.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.250.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.250.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.250.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.251.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.251.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.251.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.252.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.252.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.252.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.253.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.253.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.253.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.254.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.254.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.254.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.255.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.255.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.255.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.experts.26.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.26.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.26.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.27.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.27.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.27.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.28.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.28.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.28.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.29.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.29.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.29.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.3.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.3.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.3.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.30.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.30.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.30.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.31.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.31.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.31.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.32.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.32.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.32.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.33.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.33.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.33.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.34.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.34.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.34.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.35.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.35.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.35.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.36.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.36.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.36.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.37.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.37.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.37.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.38.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.38.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.38.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.39.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.39.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.39.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.4.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.4.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.4.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.40.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.40.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.40.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.41.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.41.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.41.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.42.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.42.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.42.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.43.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.43.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.43.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.44.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.44.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.44.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.45.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.45.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.45.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.46.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.46.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.46.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.47.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.47.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.47.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.48.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.48.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.48.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.49.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.49.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.49.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.5.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.5.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.5.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.50.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.50.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.50.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.51.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.51.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.51.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.52.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.52.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.52.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.53.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.53.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.53.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.54.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.54.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.54.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.55.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.55.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.55.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.56.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.56.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.56.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.57.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.57.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.57.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.58.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.58.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.58.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.59.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.59.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.59.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.6.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.6.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.6.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.60.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.60.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.60.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.61.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.61.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.61.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.62.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.62.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.62.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.63.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.63.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.63.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.64.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.64.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.64.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.65.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.65.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.65.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.66.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.66.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.66.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.67.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.67.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.67.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.68.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.68.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.68.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.69.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.69.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.69.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.7.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.7.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.7.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.70.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.70.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.70.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.71.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.71.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.71.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.72.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.72.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.72.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.73.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.73.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.73.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.74.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.74.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.74.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.1.mlp.experts.75.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.75.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.75.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.76.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.76.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.76.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.77.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.77.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.77.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.78.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.78.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.78.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.79.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.79.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.79.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.8.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.8.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.8.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.80.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.80.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.80.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.81.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.81.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.81.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.82.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.82.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.82.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.83.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.83.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.83.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.84.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.84.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.84.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.85.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.85.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.85.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.86.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.86.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.86.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.87.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.87.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.87.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.88.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.88.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.88.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.89.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.89.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.89.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.9.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.9.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.9.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.experts.90.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.90.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.90.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.91.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.91.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.91.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.92.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.92.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.92.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.93.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.93.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.93.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.94.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.94.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.94.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.95.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.95.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.95.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.96.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.96.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.96.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.97.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.97.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.97.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.98.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.98.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.98.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.99.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.99.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.experts.99.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.1.mlp.gate.e_score_correction_bias": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.gate.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.shared_experts.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.mlp.shared_experts.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.1.self_attn.kv_a_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.kv_a_proj_with_mqa.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.kv_b_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_a_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_a_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_b_proj.weight": "model-00001-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}