Hugging Face's logo

bkhmsi
/
micro-llama-1b

Transformers
Safetensors
llama
text-generation-inference
Model card Files
xet
Community
bkhmsi commited on
Commit
cb4d4a5
·
verified ·
1 Parent(s): 18ddfb7

Upload MiCRoOLMo

Browse files
Files changed (7) hide show
  1. config.json +13 -22
  2. generation_config.json +2 -2
  3. model-00001-of-00004.safetensors +2 -2
  4. model-00002-of-00004.safetensors +2 -2
  5. model-00003-of-00004.safetensors +2 -2
  6. model-00004-of-00004.safetensors +2 -2
  7. model.safetensors.index.json +248 -120
config.json CHANGED
@@ -1,15 +1,13 @@
1
  {
2
  "ablate": [],
3
  "architectures": [
4
- "MiCRoLlama"
5
  ],
6
  "attention_bias": false,
7
  "attention_dropout": 0.0,
8
  "backbone_num_layers": 16,
9
- "bos_token_id": 128000,
10
- "config_path": "configs/config_micro_llama.yml",
11
- "eos_token_id": 128001,
12
- "head_dim": 64,
13
  "hidden_act": "silu",
14
  "hidden_size": 2048,
15
  "initializer_range": 0.02,
@@ -17,29 +15,22 @@
17
  "jitter_noise": 0,
18
  "loss_method": "all",
19
  "loss_type": "ForCausalLMLoss",
20
- "max_position_embeddings": 131072,
21
- "mlp_bias": false,
22
- "model_type": "llama",
23
- "num_attention_heads": 32,
24
  "num_experts": 4,
25
  "num_experts_per_tok": 1,
26
  "num_hidden_layers": 64,
27
- "num_key_value_heads": 8,
28
- "pretraining_tp": 1,
29
- "rms_norm_eps": 1e-05,
30
- "rope_scaling": {
31
- "factor": 32.0,
32
- "high_freq_factor": 4.0,
33
- "low_freq_factor": 1.0,
34
- "original_max_position_embeddings": 8192,
35
- "rope_type": "llama3"
36
- },
37
- "rope_theta": 500000.0,
38
- "tie_word_embeddings": true,
39
  "torch_dtype": "float32",
40
  "transformers_version": "4.51.3",
41
  "use_bfloat16": true,
42
  "use_cache": true,
43
  "use_router": true,
44
- "vocab_size": 128256
45
  }
 
1
  {
2
  "ablate": [],
3
  "architectures": [
4
+ "MiCRoOLMo"
5
  ],
6
  "attention_bias": false,
7
  "attention_dropout": 0.0,
8
  "backbone_num_layers": 16,
9
+ "config_path": "configs/config_micro_olmo.yml",
10
+ "eos_token_id": 100257,
 
 
11
  "hidden_act": "silu",
12
  "hidden_size": 2048,
13
  "initializer_range": 0.02,
 
15
  "jitter_noise": 0,
16
  "loss_method": "all",
17
  "loss_type": "ForCausalLMLoss",
18
+ "max_position_embeddings": 4096,
19
+ "model_type": "olmo2",
20
+ "num_attention_heads": 16,
 
21
  "num_experts": 4,
22
  "num_experts_per_tok": 1,
23
  "num_hidden_layers": 64,
24
+ "num_key_value_heads": 16,
25
+ "pad_token_id": 100277,
26
+ "rms_norm_eps": 1e-06,
27
+ "rope_scaling": null,
28
+ "rope_theta": 500000,
29
+ "tie_word_embeddings": false,
 
 
 
 
 
 
30
  "torch_dtype": "float32",
31
  "transformers_version": "4.51.3",
32
  "use_bfloat16": true,
33
  "use_cache": true,
34
  "use_router": true,
35
+ "vocab_size": 100279
36
  }
generation_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "_from_model_config": true,
3
- "bos_token_id": 128000,
4
- "eos_token_id": 128001,
5
  "transformers_version": "4.51.3"
6
  }
 
1
  {
2
  "_from_model_config": true,
3
+ "eos_token_id": 100257,
4
+ "pad_token_id": 100277,
5
  "transformers_version": "4.51.3"
6
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bcd3f2811acea4b4363c24c553e6885db0a7703c84ef280ddd78d926fb6422df
3
- size 4943381688
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:603f709b739e39df4832aaa72b278933d5ecd4ea48ec9b21dcd1f9b259f0672c
3
+ size 4982895344
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9fc1b82242c728cb167ba3deec8963c8e266a3213964ad48b6676fc978129fcb
3
- size 4949792456
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52e0c5cf1a70c39cfaf0d93182328f46c6bd9a19b5b56f6a1147347f7622c807
3
+ size 4983611400
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dc6eee178b3425cf08b6e539ba2bf43ce13ae9e5550b8a9d348c1e22795ab6ff
3
- size 4949792600
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6eee6c08e3b3b508079063fb57a239ee9d73510fcfcbe6226fd3ad4252a0087
3
+ size 4966801360
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2a8e0a2040aa71ee86f063ef96ffa36a4ba7a7e20e257004e5e99fc1211f67f2
3
- size 3097724152
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c8fd3b5096c6772fd02e2922abaf22d399ff38861a6e86989fef455138b0f42
3
+ size 4160684560
model.safetensors.index.json CHANGED
@@ -1,618 +1,746 @@
1
  {
2
  "metadata": {
3
- "total_size": 17940619264
4
  },
5
  "weight_map": {
6
  "embed_tokens.weight": "model-00001-of-00004.safetensors",
7
- "final_norm.weight": "model-00004-of-00004.safetensors",
8
- "layers.0.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
9
  "layers.0.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
10
  "layers.0.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
11
  "layers.0.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
12
  "layers.0.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
13
  "layers.0.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
14
  "layers.0.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
15
  "layers.0.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
16
  "layers.0.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
17
- "layers.0.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
18
  "layers.0.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
19
  "layers.0.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
20
  "layers.0.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
21
  "layers.0.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
22
  "layers.0.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
23
  "layers.0.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
24
  "layers.0.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
25
  "layers.0.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
26
- "layers.0.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
27
  "layers.0.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
28
  "layers.0.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
29
  "layers.0.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
30
  "layers.0.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
31
  "layers.0.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
32
  "layers.0.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
33
  "layers.0.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
34
  "layers.0.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
35
- "layers.0.experts.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
36
  "layers.0.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
37
  "layers.0.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
38
  "layers.0.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
39
  "layers.0.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
40
  "layers.0.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
41
  "layers.0.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
42
  "layers.0.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
43
  "layers.0.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
44
  "layers.0.gate.0.weight": "model-00001-of-00004.safetensors",
45
  "layers.0.gate.1.weight": "model-00001-of-00004.safetensors",
46
- "layers.1.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
47
  "layers.1.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
48
  "layers.1.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
49
  "layers.1.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
50
  "layers.1.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
51
  "layers.1.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
52
  "layers.1.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
53
  "layers.1.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
54
  "layers.1.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
55
- "layers.1.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
56
  "layers.1.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
57
  "layers.1.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
58
  "layers.1.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
59
  "layers.1.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
60
  "layers.1.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
61
  "layers.1.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
62
  "layers.1.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
63
  "layers.1.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
64
- "layers.1.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
65
  "layers.1.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
66
  "layers.1.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
67
  "layers.1.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
68
  "layers.1.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
69
  "layers.1.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
70
  "layers.1.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
71
  "layers.1.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
72
  "layers.1.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
73
- "layers.1.experts.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
74
  "layers.1.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
75
  "layers.1.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
76
  "layers.1.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
77
  "layers.1.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
78
  "layers.1.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
79
  "layers.1.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
80
  "layers.1.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
81
  "layers.1.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
82
  "layers.1.gate.0.weight": "model-00001-of-00004.safetensors",
83
  "layers.1.gate.1.weight": "model-00001-of-00004.safetensors",
84
- "layers.10.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
85
  "layers.10.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
86
  "layers.10.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
87
  "layers.10.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
88
  "layers.10.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
89
  "layers.10.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
90
  "layers.10.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
91
  "layers.10.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
92
  "layers.10.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
93
- "layers.10.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
94
  "layers.10.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
95
  "layers.10.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
96
  "layers.10.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
97
  "layers.10.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
98
  "layers.10.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
99
  "layers.10.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
100
  "layers.10.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
101
  "layers.10.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
102
- "layers.10.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
103
  "layers.10.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
104
  "layers.10.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
105
  "layers.10.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
106
  "layers.10.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
107
  "layers.10.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
108
  "layers.10.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
109
  "layers.10.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
110
  "layers.10.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
111
- "layers.10.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
112
  "layers.10.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
113
  "layers.10.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
114
  "layers.10.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
115
  "layers.10.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
116
  "layers.10.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
117
  "layers.10.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
118
  "layers.10.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
119
  "layers.10.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
120
  "layers.10.gate.0.weight": "model-00003-of-00004.safetensors",
121
  "layers.10.gate.1.weight": "model-00003-of-00004.safetensors",
122
- "layers.11.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
123
  "layers.11.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
124
  "layers.11.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
125
  "layers.11.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
126
  "layers.11.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
127
  "layers.11.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
128
  "layers.11.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
129
  "layers.11.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
130
  "layers.11.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
131
- "layers.11.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
132
  "layers.11.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
133
  "layers.11.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
134
  "layers.11.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
135
  "layers.11.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
136
  "layers.11.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
137
  "layers.11.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
138
  "layers.11.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
139
  "layers.11.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
140
- "layers.11.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
141
  "layers.11.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
142
  "layers.11.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
143
  "layers.11.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
144
  "layers.11.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
145
  "layers.11.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
146
  "layers.11.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
147
  "layers.11.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
148
  "layers.11.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
149
- "layers.11.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
150
  "layers.11.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
151
  "layers.11.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
152
  "layers.11.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
153
  "layers.11.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
154
  "layers.11.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
155
  "layers.11.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
156
  "layers.11.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
157
  "layers.11.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
158
  "layers.11.gate.0.weight": "model-00003-of-00004.safetensors",
159
  "layers.11.gate.1.weight": "model-00003-of-00004.safetensors",
160
- "layers.12.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
161
  "layers.12.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
162
  "layers.12.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
163
  "layers.12.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
164
  "layers.12.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
165
  "layers.12.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
166
  "layers.12.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
167
  "layers.12.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
168
  "layers.12.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
169
- "layers.12.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
170
  "layers.12.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
171
  "layers.12.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
172
  "layers.12.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
173
  "layers.12.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
174
  "layers.12.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
175
  "layers.12.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
176
  "layers.12.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
177
  "layers.12.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
178
- "layers.12.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
179
  "layers.12.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
180
  "layers.12.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
181
  "layers.12.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
182
  "layers.12.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
183
  "layers.12.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
  "layers.12.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
185
  "layers.12.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
186
  "layers.12.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
187
- "layers.12.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
188
- "layers.12.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
189
  "layers.12.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
190
  "layers.12.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
191
- "layers.12.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
192
  "layers.12.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
193
  "layers.12.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
194
  "layers.12.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
195
  "layers.12.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
196
  "layers.12.gate.0.weight": "model-00003-of-00004.safetensors",
197
  "layers.12.gate.1.weight": "model-00003-of-00004.safetensors",
198
- "layers.13.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
199
- "layers.13.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
200
- "layers.13.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
201
- "layers.13.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
202
- "layers.13.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
203
- "layers.13.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
204
- "layers.13.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
205
- "layers.13.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
206
- "layers.13.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
207
- "layers.13.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
208
- "layers.13.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
209
- "layers.13.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
210
- "layers.13.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
211
- "layers.13.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
212
- "layers.13.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
213
- "layers.13.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
214
- "layers.13.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
215
- "layers.13.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
216
- "layers.13.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
217
- "layers.13.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
218
- "layers.13.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
219
- "layers.13.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
220
- "layers.13.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
221
- "layers.13.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
222
- "layers.13.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
223
- "layers.13.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
224
- "layers.13.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
- "layers.13.experts.3.input_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
 
 
 
226
  "layers.13.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
227
- "layers.13.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
228
- "layers.13.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
229
  "layers.13.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
230
- "layers.13.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
231
- "layers.13.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
232
- "layers.13.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
233
- "layers.13.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
234
- "layers.13.gate.0.weight": "model-00003-of-00004.safetensors",
235
- "layers.13.gate.1.weight": "model-00003-of-00004.safetensors",
236
- "layers.14.experts.0.input_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
237
  "layers.14.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
238
  "layers.14.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
239
  "layers.14.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
240
  "layers.14.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
241
  "layers.14.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
242
  "layers.14.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
243
  "layers.14.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
244
  "layers.14.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
245
- "layers.14.experts.1.input_layernorm.weight": "model-00004-of-00004.safetensors",
246
  "layers.14.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
247
  "layers.14.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
248
  "layers.14.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
249
  "layers.14.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
250
  "layers.14.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
251
  "layers.14.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
252
  "layers.14.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
253
  "layers.14.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
254
- "layers.14.experts.2.input_layernorm.weight": "model-00004-of-00004.safetensors",
255
  "layers.14.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
256
  "layers.14.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
257
  "layers.14.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
258
  "layers.14.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
259
  "layers.14.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
260
  "layers.14.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
261
  "layers.14.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
262
  "layers.14.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
263
- "layers.14.experts.3.input_layernorm.weight": "model-00004-of-00004.safetensors",
264
  "layers.14.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
265
  "layers.14.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
266
  "layers.14.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
267
  "layers.14.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
268
  "layers.14.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
269
  "layers.14.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
270
  "layers.14.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
271
  "layers.14.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
272
  "layers.14.gate.0.weight": "model-00004-of-00004.safetensors",
273
  "layers.14.gate.1.weight": "model-00004-of-00004.safetensors",
274
- "layers.15.experts.0.input_layernorm.weight": "model-00004-of-00004.safetensors",
275
  "layers.15.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
276
  "layers.15.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
277
  "layers.15.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
278
  "layers.15.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
279
  "layers.15.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
280
  "layers.15.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
281
  "layers.15.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
282
  "layers.15.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
283
- "layers.15.experts.1.input_layernorm.weight": "model-00004-of-00004.safetensors",
284
  "layers.15.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
285
  "layers.15.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
286
  "layers.15.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
287
  "layers.15.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
288
  "layers.15.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
289
  "layers.15.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
290
  "layers.15.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
291
  "layers.15.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
292
- "layers.15.experts.2.input_layernorm.weight": "model-00004-of-00004.safetensors",
293
  "layers.15.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
294
  "layers.15.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
295
  "layers.15.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
296
  "layers.15.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
297
  "layers.15.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
298
  "layers.15.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
299
  "layers.15.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
300
  "layers.15.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
301
- "layers.15.experts.3.input_layernorm.weight": "model-00004-of-00004.safetensors",
302
  "layers.15.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
303
  "layers.15.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
304
  "layers.15.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
305
  "layers.15.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
306
  "layers.15.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
307
  "layers.15.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
308
  "layers.15.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
309
  "layers.15.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
310
  "layers.15.gate.0.weight": "model-00004-of-00004.safetensors",
311
  "layers.15.gate.1.weight": "model-00004-of-00004.safetensors",
312
- "layers.2.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
313
  "layers.2.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
314
  "layers.2.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
315
  "layers.2.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
316
  "layers.2.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
317
  "layers.2.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
318
  "layers.2.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
319
  "layers.2.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
320
  "layers.2.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
321
- "layers.2.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
322
  "layers.2.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
323
  "layers.2.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
324
  "layers.2.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
325
  "layers.2.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
326
  "layers.2.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
327
  "layers.2.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
328
  "layers.2.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
329
  "layers.2.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
330
- "layers.2.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
331
  "layers.2.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
332
  "layers.2.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
333
  "layers.2.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
334
  "layers.2.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
335
  "layers.2.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
336
  "layers.2.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
337
  "layers.2.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
338
  "layers.2.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
339
- "layers.2.experts.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
340
  "layers.2.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
341
  "layers.2.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
342
  "layers.2.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
343
  "layers.2.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
344
  "layers.2.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
345
  "layers.2.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
346
  "layers.2.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
347
  "layers.2.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
348
  "layers.2.gate.0.weight": "model-00001-of-00004.safetensors",
349
  "layers.2.gate.1.weight": "model-00001-of-00004.safetensors",
350
- "layers.3.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
351
  "layers.3.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
352
  "layers.3.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
353
  "layers.3.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
354
  "layers.3.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
355
  "layers.3.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
356
  "layers.3.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
357
  "layers.3.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
358
  "layers.3.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
359
- "layers.3.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
360
  "layers.3.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
361
  "layers.3.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
362
  "layers.3.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
363
  "layers.3.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
364
  "layers.3.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
365
  "layers.3.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
366
  "layers.3.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
367
  "layers.3.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
368
- "layers.3.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
369
  "layers.3.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
370
  "layers.3.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
371
  "layers.3.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
372
  "layers.3.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
373
  "layers.3.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
374
  "layers.3.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
375
  "layers.3.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
376
  "layers.3.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
377
- "layers.3.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
378
  "layers.3.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
379
- "layers.3.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
380
- "layers.3.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
381
  "layers.3.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
382
  "layers.3.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
383
  "layers.3.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
384
  "layers.3.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
385
  "layers.3.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
386
  "layers.3.gate.0.weight": "model-00001-of-00004.safetensors",
387
  "layers.3.gate.1.weight": "model-00001-of-00004.safetensors",
388
- "layers.4.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
389
  "layers.4.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
390
  "layers.4.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
391
  "layers.4.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
392
  "layers.4.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
393
  "layers.4.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
394
  "layers.4.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
395
  "layers.4.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
396
  "layers.4.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
397
- "layers.4.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
398
  "layers.4.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
399
  "layers.4.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
400
  "layers.4.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
401
  "layers.4.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
402
  "layers.4.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
403
  "layers.4.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
404
  "layers.4.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
405
  "layers.4.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
406
- "layers.4.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
407
  "layers.4.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
408
  "layers.4.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
409
  "layers.4.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
410
  "layers.4.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
411
  "layers.4.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
412
  "layers.4.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
413
  "layers.4.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
414
  "layers.4.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
415
- "layers.4.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
416
  "layers.4.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
417
  "layers.4.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
418
  "layers.4.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
419
  "layers.4.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
420
  "layers.4.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
421
  "layers.4.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
422
  "layers.4.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
423
  "layers.4.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
424
  "layers.4.gate.0.weight": "model-00002-of-00004.safetensors",
425
  "layers.4.gate.1.weight": "model-00002-of-00004.safetensors",
426
- "layers.5.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
427
  "layers.5.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
428
  "layers.5.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
429
  "layers.5.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
430
  "layers.5.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
431
  "layers.5.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
432
  "layers.5.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
433
  "layers.5.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
434
  "layers.5.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
435
- "layers.5.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
436
  "layers.5.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
437
  "layers.5.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
438
  "layers.5.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
439
  "layers.5.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
440
  "layers.5.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
441
  "layers.5.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
442
  "layers.5.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
443
  "layers.5.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
444
- "layers.5.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
445
  "layers.5.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
446
  "layers.5.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
447
  "layers.5.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
448
  "layers.5.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
449
  "layers.5.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
450
  "layers.5.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
451
  "layers.5.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
452
  "layers.5.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
453
- "layers.5.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
454
  "layers.5.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
455
  "layers.5.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
456
  "layers.5.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
457
  "layers.5.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
458
  "layers.5.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
459
  "layers.5.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
460
  "layers.5.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
461
  "layers.5.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
462
  "layers.5.gate.0.weight": "model-00002-of-00004.safetensors",
463
  "layers.5.gate.1.weight": "model-00002-of-00004.safetensors",
464
- "layers.6.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
465
  "layers.6.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
466
  "layers.6.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
467
  "layers.6.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
468
  "layers.6.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
469
  "layers.6.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
470
  "layers.6.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
471
  "layers.6.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
472
  "layers.6.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
473
- "layers.6.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
474
  "layers.6.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
475
  "layers.6.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
476
  "layers.6.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
477
  "layers.6.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
478
  "layers.6.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
479
  "layers.6.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
480
  "layers.6.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
481
  "layers.6.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
482
- "layers.6.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
483
  "layers.6.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
484
  "layers.6.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
485
  "layers.6.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
486
  "layers.6.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
487
  "layers.6.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
488
  "layers.6.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
489
  "layers.6.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
490
  "layers.6.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
491
- "layers.6.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
492
  "layers.6.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
493
  "layers.6.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
494
  "layers.6.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
495
  "layers.6.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
496
  "layers.6.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
497
  "layers.6.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
498
  "layers.6.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
499
  "layers.6.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
500
  "layers.6.gate.0.weight": "model-00002-of-00004.safetensors",
501
  "layers.6.gate.1.weight": "model-00002-of-00004.safetensors",
502
- "layers.7.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
503
  "layers.7.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
504
  "layers.7.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
505
  "layers.7.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
506
  "layers.7.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
507
  "layers.7.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
508
  "layers.7.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
509
  "layers.7.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
510
  "layers.7.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
511
- "layers.7.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
512
  "layers.7.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
513
  "layers.7.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
514
  "layers.7.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
515
  "layers.7.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
516
  "layers.7.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
517
  "layers.7.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
518
  "layers.7.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
519
  "layers.7.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
520
- "layers.7.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
521
  "layers.7.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
522
  "layers.7.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
523
  "layers.7.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
524
  "layers.7.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
525
  "layers.7.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
526
  "layers.7.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
527
  "layers.7.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
528
  "layers.7.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
529
- "layers.7.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
530
  "layers.7.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
531
  "layers.7.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
532
  "layers.7.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
533
  "layers.7.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
534
  "layers.7.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
535
  "layers.7.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
536
  "layers.7.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
537
  "layers.7.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
538
  "layers.7.gate.0.weight": "model-00002-of-00004.safetensors",
539
  "layers.7.gate.1.weight": "model-00002-of-00004.safetensors",
540
- "layers.8.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
541
  "layers.8.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
542
  "layers.8.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
543
  "layers.8.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
544
  "layers.8.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
545
  "layers.8.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
546
  "layers.8.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
547
  "layers.8.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
548
  "layers.8.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
549
- "layers.8.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
550
- "layers.8.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
551
  "layers.8.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
552
- "layers.8.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
553
- "layers.8.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
554
  "layers.8.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
555
  "layers.8.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
556
  "layers.8.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
557
  "layers.8.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
558
- "layers.8.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
559
- "layers.8.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
560
- "layers.8.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
561
- "layers.8.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
562
- "layers.8.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
563
- "layers.8.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
564
- "layers.8.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
565
- "layers.8.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
566
- "layers.8.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
567
- "layers.8.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
 
568
  "layers.8.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
569
- "layers.8.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
570
- "layers.8.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
571
  "layers.8.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
572
- "layers.8.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
573
- "layers.8.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
574
- "layers.8.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
575
- "layers.8.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
 
 
576
  "layers.8.gate.0.weight": "model-00002-of-00004.safetensors",
577
  "layers.8.gate.1.weight": "model-00002-of-00004.safetensors",
578
- "layers.9.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
579
  "layers.9.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
580
  "layers.9.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
581
  "layers.9.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
582
  "layers.9.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
583
  "layers.9.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
584
  "layers.9.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
585
  "layers.9.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
586
  "layers.9.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
587
- "layers.9.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
588
  "layers.9.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
589
  "layers.9.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
590
  "layers.9.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
591
  "layers.9.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
592
  "layers.9.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
593
  "layers.9.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
594
  "layers.9.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
595
  "layers.9.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
596
- "layers.9.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
597
  "layers.9.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
598
  "layers.9.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
599
  "layers.9.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
600
  "layers.9.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
601
  "layers.9.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
602
  "layers.9.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
603
  "layers.9.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
604
  "layers.9.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
605
- "layers.9.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
606
  "layers.9.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
607
  "layers.9.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
608
  "layers.9.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
609
  "layers.9.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
610
  "layers.9.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
611
  "layers.9.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
612
  "layers.9.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
613
  "layers.9.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
614
  "layers.9.gate.0.weight": "model-00003-of-00004.safetensors",
615
  "layers.9.gate.1.weight": "model-00003-of-00004.safetensors",
616
- "lm_head.weight": "model-00004-of-00004.safetensors"
 
617
  }
618
  }
 
1
  {
2
  "metadata": {
3
+ "total_size": 19093905408
4
  },
5
  "weight_map": {
6
  "embed_tokens.weight": "model-00001-of-00004.safetensors",
 
 
7
  "layers.0.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
8
  "layers.0.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
9
  "layers.0.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
10
  "layers.0.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
11
+ "layers.0.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
12
+ "layers.0.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
13
  "layers.0.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
14
  "layers.0.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
15
+ "layers.0.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
16
  "layers.0.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
17
  "layers.0.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
18
  "layers.0.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
19
  "layers.0.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
20
  "layers.0.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
21
  "layers.0.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
22
+ "layers.0.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
23
+ "layers.0.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
24
  "layers.0.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
25
  "layers.0.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
26
+ "layers.0.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
27
  "layers.0.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
28
  "layers.0.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
29
  "layers.0.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
30
  "layers.0.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
31
  "layers.0.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
32
  "layers.0.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
33
+ "layers.0.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
34
+ "layers.0.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
35
  "layers.0.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
36
  "layers.0.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
37
+ "layers.0.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
38
  "layers.0.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
39
  "layers.0.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
40
  "layers.0.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
41
  "layers.0.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
42
  "layers.0.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
43
  "layers.0.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
44
+ "layers.0.experts.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
45
+ "layers.0.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
46
  "layers.0.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
47
  "layers.0.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
48
+ "layers.0.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
49
  "layers.0.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
50
  "layers.0.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
51
  "layers.0.gate.0.weight": "model-00001-of-00004.safetensors",
52
  "layers.0.gate.1.weight": "model-00001-of-00004.safetensors",
 
53
  "layers.1.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
54
  "layers.1.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
55
  "layers.1.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
56
  "layers.1.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
57
+ "layers.1.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
58
+ "layers.1.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
59
  "layers.1.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
60
  "layers.1.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
61
+ "layers.1.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
62
  "layers.1.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
63
  "layers.1.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
64
  "layers.1.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
65
  "layers.1.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
66
  "layers.1.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
67
  "layers.1.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
68
+ "layers.1.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
69
+ "layers.1.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
70
  "layers.1.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
71
  "layers.1.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
72
+ "layers.1.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
73
  "layers.1.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
74
  "layers.1.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
75
  "layers.1.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
76
  "layers.1.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
77
  "layers.1.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
78
  "layers.1.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
79
+ "layers.1.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
80
+ "layers.1.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
81
  "layers.1.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
82
  "layers.1.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
83
+ "layers.1.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
84
  "layers.1.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
85
  "layers.1.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
86
  "layers.1.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
87
  "layers.1.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
88
  "layers.1.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
89
  "layers.1.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
90
+ "layers.1.experts.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
91
+ "layers.1.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
92
  "layers.1.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
93
  "layers.1.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
94
+ "layers.1.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
95
  "layers.1.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
96
  "layers.1.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
97
  "layers.1.gate.0.weight": "model-00001-of-00004.safetensors",
98
  "layers.1.gate.1.weight": "model-00001-of-00004.safetensors",
 
99
  "layers.10.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
100
  "layers.10.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
101
  "layers.10.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
102
  "layers.10.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
103
+ "layers.10.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
104
+ "layers.10.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
105
  "layers.10.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
106
  "layers.10.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
107
+ "layers.10.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
108
  "layers.10.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
109
  "layers.10.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
110
  "layers.10.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
111
  "layers.10.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
112
  "layers.10.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
113
  "layers.10.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
114
+ "layers.10.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
115
+ "layers.10.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
116
  "layers.10.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
117
  "layers.10.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
118
+ "layers.10.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
119
  "layers.10.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
120
  "layers.10.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
121
  "layers.10.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
122
  "layers.10.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
123
  "layers.10.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
124
  "layers.10.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
125
+ "layers.10.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
126
+ "layers.10.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
127
  "layers.10.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
128
  "layers.10.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
129
+ "layers.10.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
130
  "layers.10.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
131
  "layers.10.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
132
  "layers.10.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
133
  "layers.10.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
134
  "layers.10.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
135
  "layers.10.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
136
+ "layers.10.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
137
+ "layers.10.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
138
  "layers.10.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
139
  "layers.10.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
140
+ "layers.10.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
141
  "layers.10.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
142
  "layers.10.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
143
  "layers.10.gate.0.weight": "model-00003-of-00004.safetensors",
144
  "layers.10.gate.1.weight": "model-00003-of-00004.safetensors",
 
145
  "layers.11.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
146
  "layers.11.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
147
  "layers.11.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
148
  "layers.11.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
149
+ "layers.11.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
150
+ "layers.11.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
151
  "layers.11.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
152
  "layers.11.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
153
+ "layers.11.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
154
  "layers.11.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
155
  "layers.11.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
156
  "layers.11.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
157
  "layers.11.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
158
  "layers.11.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
159
  "layers.11.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
160
+ "layers.11.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
161
+ "layers.11.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
162
  "layers.11.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
163
  "layers.11.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
164
+ "layers.11.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
165
  "layers.11.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
166
  "layers.11.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
167
  "layers.11.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
168
  "layers.11.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
169
  "layers.11.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
170
  "layers.11.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
171
+ "layers.11.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
172
+ "layers.11.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
173
  "layers.11.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
174
  "layers.11.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
175
+ "layers.11.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
176
  "layers.11.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
177
  "layers.11.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
178
  "layers.11.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
179
  "layers.11.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
180
  "layers.11.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
181
  "layers.11.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
182
+ "layers.11.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
183
+ "layers.11.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
184
  "layers.11.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
185
  "layers.11.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
186
+ "layers.11.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
187
  "layers.11.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
188
  "layers.11.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
  "layers.11.gate.0.weight": "model-00003-of-00004.safetensors",
190
  "layers.11.gate.1.weight": "model-00003-of-00004.safetensors",
 
191
  "layers.12.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
192
  "layers.12.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
193
  "layers.12.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
194
  "layers.12.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
195
+ "layers.12.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
196
+ "layers.12.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
197
  "layers.12.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
198
  "layers.12.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
199
+ "layers.12.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
200
  "layers.12.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
201
  "layers.12.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
202
  "layers.12.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
203
  "layers.12.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
204
  "layers.12.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
205
  "layers.12.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
206
+ "layers.12.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
207
+ "layers.12.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
208
  "layers.12.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
209
  "layers.12.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
210
+ "layers.12.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
211
  "layers.12.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
212
  "layers.12.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
213
  "layers.12.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
214
  "layers.12.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
215
  "layers.12.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
216
  "layers.12.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
217
+ "layers.12.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
218
+ "layers.12.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
219
  "layers.12.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
220
  "layers.12.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
221
+ "layers.12.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
222
  "layers.12.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
223
  "layers.12.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
224
+ "layers.12.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
 
225
  "layers.12.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
226
  "layers.12.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
227
+ "layers.12.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
228
+ "layers.12.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
229
+ "layers.12.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
230
  "layers.12.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
231
  "layers.12.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
232
+ "layers.12.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
233
  "layers.12.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
234
  "layers.12.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
235
  "layers.12.gate.0.weight": "model-00003-of-00004.safetensors",
236
  "layers.12.gate.1.weight": "model-00003-of-00004.safetensors",
237
+ "layers.13.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
238
+ "layers.13.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
239
+ "layers.13.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
240
+ "layers.13.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
241
+ "layers.13.experts.0.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
242
+ "layers.13.experts.0.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
243
+ "layers.13.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
244
+ "layers.13.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
245
+ "layers.13.experts.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
246
+ "layers.13.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
247
+ "layers.13.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
248
+ "layers.13.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
249
+ "layers.13.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
250
+ "layers.13.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
251
+ "layers.13.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
252
+ "layers.13.experts.1.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
253
+ "layers.13.experts.1.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
254
+ "layers.13.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
255
+ "layers.13.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
256
+ "layers.13.experts.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
257
+ "layers.13.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
258
+ "layers.13.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
259
+ "layers.13.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
260
+ "layers.13.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
261
+ "layers.13.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
262
+ "layers.13.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
263
+ "layers.13.experts.2.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
264
+ "layers.13.experts.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
265
+ "layers.13.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
266
+ "layers.13.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
267
+ "layers.13.experts.2.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
268
+ "layers.13.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
269
+ "layers.13.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
270
  "layers.13.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
271
+ "layers.13.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
272
+ "layers.13.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
273
  "layers.13.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
274
+ "layers.13.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
275
+ "layers.13.experts.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
276
+ "layers.13.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
277
+ "layers.13.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
278
+ "layers.13.experts.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
279
+ "layers.13.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
280
+ "layers.13.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
281
+ "layers.13.gate.0.weight": "model-00004-of-00004.safetensors",
282
+ "layers.13.gate.1.weight": "model-00004-of-00004.safetensors",
283
  "layers.14.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
284
  "layers.14.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
285
  "layers.14.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
286
  "layers.14.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
287
+ "layers.14.experts.0.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
288
+ "layers.14.experts.0.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
289
  "layers.14.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
290
  "layers.14.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
291
+ "layers.14.experts.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
292
  "layers.14.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
293
  "layers.14.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
294
  "layers.14.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
295
  "layers.14.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
296
  "layers.14.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
297
  "layers.14.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
298
+ "layers.14.experts.1.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
299
+ "layers.14.experts.1.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
300
  "layers.14.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
301
  "layers.14.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
302
+ "layers.14.experts.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
303
  "layers.14.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
304
  "layers.14.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
305
  "layers.14.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
306
  "layers.14.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
307
  "layers.14.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
308
  "layers.14.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
309
+ "layers.14.experts.2.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
310
+ "layers.14.experts.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
311
  "layers.14.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
312
  "layers.14.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
313
+ "layers.14.experts.2.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
314
  "layers.14.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
315
  "layers.14.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
316
  "layers.14.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
317
  "layers.14.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
318
  "layers.14.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
319
  "layers.14.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
320
+ "layers.14.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
321
+ "layers.14.experts.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
322
  "layers.14.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
323
  "layers.14.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
324
+ "layers.14.experts.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
325
  "layers.14.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
326
  "layers.14.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
327
  "layers.14.gate.0.weight": "model-00004-of-00004.safetensors",
328
  "layers.14.gate.1.weight": "model-00004-of-00004.safetensors",
 
329
  "layers.15.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
330
  "layers.15.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
331
  "layers.15.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
332
  "layers.15.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
333
+ "layers.15.experts.0.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
334
+ "layers.15.experts.0.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
335
  "layers.15.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
336
  "layers.15.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
337
+ "layers.15.experts.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
338
  "layers.15.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
339
  "layers.15.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
340
  "layers.15.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
341
  "layers.15.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
342
  "layers.15.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
343
  "layers.15.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
344
+ "layers.15.experts.1.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
345
+ "layers.15.experts.1.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
346
  "layers.15.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
347
  "layers.15.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
348
+ "layers.15.experts.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
349
  "layers.15.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
350
  "layers.15.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
351
  "layers.15.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
352
  "layers.15.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
353
  "layers.15.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
354
  "layers.15.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
355
+ "layers.15.experts.2.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
356
+ "layers.15.experts.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
357
  "layers.15.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
358
  "layers.15.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
359
+ "layers.15.experts.2.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
360
  "layers.15.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
361
  "layers.15.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
362
  "layers.15.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
363
  "layers.15.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
364
  "layers.15.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
365
  "layers.15.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
366
+ "layers.15.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
367
+ "layers.15.experts.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
368
  "layers.15.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
369
  "layers.15.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
370
+ "layers.15.experts.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
371
  "layers.15.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
372
  "layers.15.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
373
  "layers.15.gate.0.weight": "model-00004-of-00004.safetensors",
374
  "layers.15.gate.1.weight": "model-00004-of-00004.safetensors",
 
375
  "layers.2.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
376
  "layers.2.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
377
  "layers.2.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
378
  "layers.2.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
379
+ "layers.2.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
380
+ "layers.2.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
381
  "layers.2.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
382
  "layers.2.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
383
+ "layers.2.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
384
  "layers.2.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
385
  "layers.2.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
386
  "layers.2.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
387
  "layers.2.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
388
  "layers.2.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
389
  "layers.2.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
390
+ "layers.2.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
391
+ "layers.2.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
392
  "layers.2.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
393
  "layers.2.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
394
+ "layers.2.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
395
  "layers.2.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
396
  "layers.2.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
397
  "layers.2.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
398
  "layers.2.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
399
  "layers.2.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
400
  "layers.2.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
401
+ "layers.2.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
402
+ "layers.2.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
403
  "layers.2.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
404
  "layers.2.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
405
+ "layers.2.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
406
  "layers.2.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
407
  "layers.2.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
408
  "layers.2.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
409
  "layers.2.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
410
  "layers.2.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
411
  "layers.2.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
412
+ "layers.2.experts.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
413
+ "layers.2.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
414
  "layers.2.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
415
  "layers.2.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
416
+ "layers.2.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
417
  "layers.2.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
418
  "layers.2.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
419
  "layers.2.gate.0.weight": "model-00001-of-00004.safetensors",
420
  "layers.2.gate.1.weight": "model-00001-of-00004.safetensors",
 
421
  "layers.3.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
422
  "layers.3.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
423
  "layers.3.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
424
  "layers.3.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
425
+ "layers.3.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
426
+ "layers.3.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
427
  "layers.3.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
428
  "layers.3.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
429
+ "layers.3.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
430
  "layers.3.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
431
  "layers.3.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
432
  "layers.3.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
433
  "layers.3.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
434
  "layers.3.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
435
  "layers.3.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
436
+ "layers.3.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
437
+ "layers.3.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
438
  "layers.3.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
439
  "layers.3.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
440
+ "layers.3.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
441
  "layers.3.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
442
  "layers.3.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
443
  "layers.3.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
444
  "layers.3.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
445
  "layers.3.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
446
  "layers.3.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
447
+ "layers.3.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
448
+ "layers.3.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
449
  "layers.3.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
450
  "layers.3.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
451
+ "layers.3.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
452
  "layers.3.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
453
  "layers.3.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
454
  "layers.3.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
455
+ "layers.3.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
456
+ "layers.3.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
457
  "layers.3.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
458
+ "layers.3.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
459
+ "layers.3.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
460
  "layers.3.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
461
  "layers.3.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
462
+ "layers.3.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
463
  "layers.3.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
464
  "layers.3.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
465
  "layers.3.gate.0.weight": "model-00001-of-00004.safetensors",
466
  "layers.3.gate.1.weight": "model-00001-of-00004.safetensors",
 
467
  "layers.4.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
468
  "layers.4.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
469
  "layers.4.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
470
  "layers.4.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
471
+ "layers.4.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
472
+ "layers.4.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
473
  "layers.4.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
474
  "layers.4.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
475
+ "layers.4.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
476
  "layers.4.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
477
  "layers.4.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
478
  "layers.4.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
479
  "layers.4.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
480
  "layers.4.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
481
  "layers.4.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
482
+ "layers.4.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
483
+ "layers.4.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
484
  "layers.4.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
485
  "layers.4.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
486
+ "layers.4.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
487
  "layers.4.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
488
  "layers.4.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
489
  "layers.4.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
490
  "layers.4.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
491
  "layers.4.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
492
  "layers.4.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
493
+ "layers.4.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
494
+ "layers.4.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
495
  "layers.4.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
496
  "layers.4.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
497
+ "layers.4.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
498
  "layers.4.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
499
  "layers.4.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
500
  "layers.4.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
501
  "layers.4.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
502
  "layers.4.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
503
  "layers.4.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
504
+ "layers.4.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
505
+ "layers.4.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
506
  "layers.4.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
507
  "layers.4.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
508
+ "layers.4.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
509
  "layers.4.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
510
  "layers.4.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
511
  "layers.4.gate.0.weight": "model-00002-of-00004.safetensors",
512
  "layers.4.gate.1.weight": "model-00002-of-00004.safetensors",
 
513
  "layers.5.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
514
  "layers.5.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
515
  "layers.5.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
516
  "layers.5.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
517
+ "layers.5.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
518
+ "layers.5.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
519
  "layers.5.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
520
  "layers.5.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
521
+ "layers.5.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
522
  "layers.5.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
523
  "layers.5.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
524
  "layers.5.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
525
  "layers.5.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
526
  "layers.5.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
527
  "layers.5.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
528
+ "layers.5.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
529
+ "layers.5.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
530
  "layers.5.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
531
  "layers.5.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
532
+ "layers.5.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
533
  "layers.5.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
534
  "layers.5.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
535
  "layers.5.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
536
  "layers.5.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
537
  "layers.5.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
538
  "layers.5.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
539
+ "layers.5.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
540
+ "layers.5.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
541
  "layers.5.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
542
  "layers.5.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
543
+ "layers.5.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
544
  "layers.5.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
545
  "layers.5.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
546
  "layers.5.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
547
  "layers.5.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
548
  "layers.5.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
549
  "layers.5.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
550
+ "layers.5.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
551
+ "layers.5.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
552
  "layers.5.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
553
  "layers.5.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
554
+ "layers.5.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
555
  "layers.5.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
556
  "layers.5.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
557
  "layers.5.gate.0.weight": "model-00002-of-00004.safetensors",
558
  "layers.5.gate.1.weight": "model-00002-of-00004.safetensors",
 
559
  "layers.6.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
560
  "layers.6.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
561
  "layers.6.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
562
  "layers.6.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
563
+ "layers.6.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
564
+ "layers.6.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
565
  "layers.6.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
566
  "layers.6.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
567
+ "layers.6.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
568
  "layers.6.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
569
  "layers.6.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
570
  "layers.6.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
571
  "layers.6.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
572
  "layers.6.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
573
  "layers.6.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
574
+ "layers.6.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
575
+ "layers.6.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
576
  "layers.6.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
577
  "layers.6.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
578
+ "layers.6.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
579
  "layers.6.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
580
  "layers.6.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
581
  "layers.6.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
582
  "layers.6.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
583
  "layers.6.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
584
  "layers.6.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
585
+ "layers.6.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
586
+ "layers.6.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
587
  "layers.6.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
588
  "layers.6.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
589
+ "layers.6.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
590
  "layers.6.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
591
  "layers.6.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
592
  "layers.6.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
593
  "layers.6.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
594
  "layers.6.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
595
  "layers.6.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
596
+ "layers.6.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
597
+ "layers.6.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
598
  "layers.6.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
599
  "layers.6.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
600
+ "layers.6.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
601
  "layers.6.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
602
  "layers.6.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
603
  "layers.6.gate.0.weight": "model-00002-of-00004.safetensors",
604
  "layers.6.gate.1.weight": "model-00002-of-00004.safetensors",
 
605
  "layers.7.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
606
  "layers.7.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
607
  "layers.7.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
608
  "layers.7.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
609
+ "layers.7.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
610
+ "layers.7.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
611
  "layers.7.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
612
  "layers.7.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
613
+ "layers.7.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
614
  "layers.7.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
615
  "layers.7.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
616
  "layers.7.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
617
  "layers.7.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
618
  "layers.7.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
619
  "layers.7.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
620
+ "layers.7.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
621
+ "layers.7.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
622
  "layers.7.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
623
  "layers.7.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
624
+ "layers.7.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
625
  "layers.7.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
626
  "layers.7.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
627
  "layers.7.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
628
  "layers.7.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
629
  "layers.7.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
630
  "layers.7.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
631
+ "layers.7.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
632
+ "layers.7.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
633
  "layers.7.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
634
  "layers.7.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
635
+ "layers.7.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
636
  "layers.7.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
637
  "layers.7.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
638
  "layers.7.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
639
  "layers.7.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
640
  "layers.7.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
641
  "layers.7.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
642
+ "layers.7.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
643
+ "layers.7.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
644
  "layers.7.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
645
  "layers.7.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
646
+ "layers.7.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
647
  "layers.7.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
648
  "layers.7.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
649
  "layers.7.gate.0.weight": "model-00002-of-00004.safetensors",
650
  "layers.7.gate.1.weight": "model-00002-of-00004.safetensors",
 
651
  "layers.8.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
652
  "layers.8.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
653
  "layers.8.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
654
  "layers.8.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
655
+ "layers.8.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
656
+ "layers.8.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
657
  "layers.8.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
658
  "layers.8.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
659
+ "layers.8.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
660
  "layers.8.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
661
  "layers.8.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
662
+ "layers.8.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
 
663
  "layers.8.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
664
+ "layers.8.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
665
+ "layers.8.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
666
+ "layers.8.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
667
+ "layers.8.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
668
  "layers.8.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
669
  "layers.8.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
670
+ "layers.8.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
671
  "layers.8.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
672
  "layers.8.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
673
+ "layers.8.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
674
+ "layers.8.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
675
+ "layers.8.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
676
+ "layers.8.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
677
+ "layers.8.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
678
+ "layers.8.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
679
+ "layers.8.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
680
+ "layers.8.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
681
+ "layers.8.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
682
+ "layers.8.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
683
+ "layers.8.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
684
  "layers.8.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
685
+ "layers.8.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
686
+ "layers.8.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
687
  "layers.8.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
688
+ "layers.8.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
689
+ "layers.8.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
690
+ "layers.8.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
691
+ "layers.8.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
692
+ "layers.8.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
693
+ "layers.8.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
694
+ "layers.8.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
695
  "layers.8.gate.0.weight": "model-00002-of-00004.safetensors",
696
  "layers.8.gate.1.weight": "model-00002-of-00004.safetensors",
 
697
  "layers.9.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
698
  "layers.9.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
699
  "layers.9.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
700
  "layers.9.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
701
+ "layers.9.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
702
+ "layers.9.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
703
  "layers.9.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
704
  "layers.9.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
705
+ "layers.9.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
706
  "layers.9.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
707
  "layers.9.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
708
  "layers.9.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
709
  "layers.9.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
710
  "layers.9.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
711
  "layers.9.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
712
+ "layers.9.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
713
+ "layers.9.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
714
  "layers.9.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
715
  "layers.9.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
716
+ "layers.9.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
717
  "layers.9.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
718
  "layers.9.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
719
  "layers.9.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
720
  "layers.9.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
721
  "layers.9.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
722
  "layers.9.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
723
+ "layers.9.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
724
+ "layers.9.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
725
  "layers.9.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
726
  "layers.9.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
727
+ "layers.9.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
728
  "layers.9.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
729
  "layers.9.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
730
  "layers.9.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
731
  "layers.9.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
732
  "layers.9.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
733
  "layers.9.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
734
+ "layers.9.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
735
+ "layers.9.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
736
  "layers.9.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
737
  "layers.9.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
738
+ "layers.9.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
739
  "layers.9.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
740
  "layers.9.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
741
  "layers.9.gate.0.weight": "model-00003-of-00004.safetensors",
742
  "layers.9.gate.1.weight": "model-00003-of-00004.safetensors",
743
+ "lm_head.weight": "model-00004-of-00004.safetensors",
744
+ "norm.weight": "model-00004-of-00004.safetensors"
745
  }
746
  }