Hugging Face's logo

bkhmsi
/
micro-llama-1b

Transformers
Safetensors
llama
text-generation-inference
Model card Files
xet
Community
bkhmsi commited on
Commit
b9ea46b
·
verified ·
1 Parent(s): cb4d4a5

Upload MiCRoLlama

Browse files
Files changed (7) hide show
  1. config.json +22 -13
  2. generation_config.json +2 -2
  3. model-00001-of-00004.safetensors +2 -2
  4. model-00002-of-00004.safetensors +2 -2
  5. model-00003-of-00004.safetensors +2 -2
  6. model-00004-of-00004.safetensors +2 -2
  7. model.safetensors.index.json +120 -248
config.json CHANGED
@@ -1,13 +1,15 @@
1
  {
2
  "ablate": [],
3
  "architectures": [
4
- "MiCRoOLMo"
5
  ],
6
  "attention_bias": false,
7
  "attention_dropout": 0.0,
8
  "backbone_num_layers": 16,
9
- "config_path": "configs/config_micro_olmo.yml",
10
- "eos_token_id": 100257,
 
 
11
  "hidden_act": "silu",
12
  "hidden_size": 2048,
13
  "initializer_range": 0.02,
@@ -15,22 +17,29 @@
15
  "jitter_noise": 0,
16
  "loss_method": "all",
17
  "loss_type": "ForCausalLMLoss",
18
- "max_position_embeddings": 4096,
19
- "model_type": "olmo2",
20
- "num_attention_heads": 16,
 
21
  "num_experts": 4,
22
  "num_experts_per_tok": 1,
23
  "num_hidden_layers": 64,
24
- "num_key_value_heads": 16,
25
- "pad_token_id": 100277,
26
- "rms_norm_eps": 1e-06,
27
- "rope_scaling": null,
28
- "rope_theta": 500000,
29
- "tie_word_embeddings": false,
 
 
 
 
 
 
30
  "torch_dtype": "float32",
31
  "transformers_version": "4.51.3",
32
  "use_bfloat16": true,
33
  "use_cache": true,
34
  "use_router": true,
35
- "vocab_size": 100279
36
  }
 
1
  {
2
  "ablate": [],
3
  "architectures": [
4
+ "MiCRoLlama"
5
  ],
6
  "attention_bias": false,
7
  "attention_dropout": 0.0,
8
  "backbone_num_layers": 16,
9
+ "bos_token_id": 128000,
10
+ "config_path": "configs/config_micro_llama.yml",
11
+ "eos_token_id": 128001,
12
+ "head_dim": 64,
13
  "hidden_act": "silu",
14
  "hidden_size": 2048,
15
  "initializer_range": 0.02,
 
17
  "jitter_noise": 0,
18
  "loss_method": "all",
19
  "loss_type": "ForCausalLMLoss",
20
+ "max_position_embeddings": 131072,
21
+ "mlp_bias": false,
22
+ "model_type": "llama",
23
+ "num_attention_heads": 32,
24
  "num_experts": 4,
25
  "num_experts_per_tok": 1,
26
  "num_hidden_layers": 64,
27
+ "num_key_value_heads": 8,
28
+ "pretraining_tp": 1,
29
+ "rms_norm_eps": 1e-05,
30
+ "rope_scaling": {
31
+ "factor": 32.0,
32
+ "high_freq_factor": 4.0,
33
+ "low_freq_factor": 1.0,
34
+ "original_max_position_embeddings": 8192,
35
+ "rope_type": "llama3"
36
+ },
37
+ "rope_theta": 500000.0,
38
+ "tie_word_embeddings": true,
39
  "torch_dtype": "float32",
40
  "transformers_version": "4.51.3",
41
  "use_bfloat16": true,
42
  "use_cache": true,
43
  "use_router": true,
44
+ "vocab_size": 128256
45
  }
generation_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "_from_model_config": true,
3
- "eos_token_id": 100257,
4
- "pad_token_id": 100277,
5
  "transformers_version": "4.51.3"
6
  }
 
1
  {
2
  "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "eos_token_id": 128001,
5
  "transformers_version": "4.51.3"
6
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:603f709b739e39df4832aaa72b278933d5ecd4ea48ec9b21dcd1f9b259f0672c
3
- size 4982895344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcd3f2811acea4b4363c24c553e6885db0a7703c84ef280ddd78d926fb6422df
3
+ size 4943381688
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:52e0c5cf1a70c39cfaf0d93182328f46c6bd9a19b5b56f6a1147347f7622c807
3
- size 4983611400
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fc1b82242c728cb167ba3deec8963c8e266a3213964ad48b6676fc978129fcb
3
+ size 4949792456
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a6eee6c08e3b3b508079063fb57a239ee9d73510fcfcbe6226fd3ad4252a0087
3
- size 4966801360
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc6eee178b3425cf08b6e539ba2bf43ce13ae9e5550b8a9d348c1e22795ab6ff
3
+ size 4949792600
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9c8fd3b5096c6772fd02e2922abaf22d399ff38861a6e86989fef455138b0f42
3
- size 4160684560
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a8e0a2040aa71ee86f063ef96ffa36a4ba7a7e20e257004e5e99fc1211f67f2
3
+ size 3097724152
model.safetensors.index.json CHANGED
@@ -1,746 +1,618 @@
1
  {
2
  "metadata": {
3
- "total_size": 19093905408
4
  },
5
  "weight_map": {
6
  "embed_tokens.weight": "model-00001-of-00004.safetensors",
 
 
7
  "layers.0.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
8
  "layers.0.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
9
  "layers.0.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
10
  "layers.0.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
11
- "layers.0.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
12
- "layers.0.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
13
  "layers.0.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
14
  "layers.0.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
15
- "layers.0.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
16
  "layers.0.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
17
  "layers.0.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
18
  "layers.0.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
19
  "layers.0.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
20
  "layers.0.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
21
  "layers.0.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
22
- "layers.0.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
23
- "layers.0.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
24
  "layers.0.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
25
  "layers.0.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
26
- "layers.0.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
27
  "layers.0.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
28
  "layers.0.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
29
  "layers.0.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
30
  "layers.0.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
31
  "layers.0.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
32
  "layers.0.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
33
- "layers.0.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
34
- "layers.0.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
35
  "layers.0.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
36
  "layers.0.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
37
- "layers.0.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
38
  "layers.0.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
39
  "layers.0.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
40
  "layers.0.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
41
  "layers.0.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
42
  "layers.0.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
43
  "layers.0.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
44
- "layers.0.experts.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
45
- "layers.0.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
46
  "layers.0.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
47
  "layers.0.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
48
- "layers.0.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
49
  "layers.0.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
50
  "layers.0.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
51
  "layers.0.gate.0.weight": "model-00001-of-00004.safetensors",
52
  "layers.0.gate.1.weight": "model-00001-of-00004.safetensors",
 
53
  "layers.1.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
54
  "layers.1.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
55
  "layers.1.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
56
  "layers.1.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
57
- "layers.1.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
58
- "layers.1.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
59
  "layers.1.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
60
  "layers.1.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
61
- "layers.1.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
62
  "layers.1.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
63
  "layers.1.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
64
  "layers.1.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
65
  "layers.1.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
66
  "layers.1.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
67
  "layers.1.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
68
- "layers.1.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
69
- "layers.1.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
70
  "layers.1.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
71
  "layers.1.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
72
- "layers.1.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
73
  "layers.1.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
74
  "layers.1.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
75
  "layers.1.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
76
  "layers.1.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
77
  "layers.1.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
78
  "layers.1.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
79
- "layers.1.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
80
- "layers.1.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
81
  "layers.1.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
82
  "layers.1.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
83
- "layers.1.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
84
  "layers.1.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
85
  "layers.1.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
86
  "layers.1.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
87
  "layers.1.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
88
  "layers.1.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
89
  "layers.1.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
90
- "layers.1.experts.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
91
- "layers.1.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
92
  "layers.1.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
93
  "layers.1.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
94
- "layers.1.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
95
  "layers.1.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
96
  "layers.1.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
97
  "layers.1.gate.0.weight": "model-00001-of-00004.safetensors",
98
  "layers.1.gate.1.weight": "model-00001-of-00004.safetensors",
 
99
  "layers.10.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
100
  "layers.10.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
101
  "layers.10.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
102
  "layers.10.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
103
- "layers.10.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
104
- "layers.10.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
105
  "layers.10.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
106
  "layers.10.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
107
- "layers.10.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
108
  "layers.10.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
109
  "layers.10.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
110
  "layers.10.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
111
  "layers.10.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
112
  "layers.10.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
113
  "layers.10.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
114
- "layers.10.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
115
- "layers.10.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
116
  "layers.10.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
117
  "layers.10.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
118
- "layers.10.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
119
  "layers.10.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
120
  "layers.10.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
121
  "layers.10.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
122
  "layers.10.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
123
  "layers.10.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
124
  "layers.10.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
125
- "layers.10.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
126
- "layers.10.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
127
  "layers.10.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
128
  "layers.10.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
129
- "layers.10.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
130
  "layers.10.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
131
  "layers.10.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
132
  "layers.10.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
133
  "layers.10.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
134
  "layers.10.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
135
  "layers.10.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
136
- "layers.10.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
137
- "layers.10.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
138
  "layers.10.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
139
  "layers.10.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
140
- "layers.10.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
141
  "layers.10.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
142
  "layers.10.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
143
  "layers.10.gate.0.weight": "model-00003-of-00004.safetensors",
144
  "layers.10.gate.1.weight": "model-00003-of-00004.safetensors",
 
145
  "layers.11.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
146
  "layers.11.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
147
  "layers.11.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
148
  "layers.11.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
149
- "layers.11.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
150
- "layers.11.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
151
  "layers.11.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
152
  "layers.11.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
153
- "layers.11.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
154
  "layers.11.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
155
  "layers.11.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
156
  "layers.11.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
157
  "layers.11.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
158
  "layers.11.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
159
  "layers.11.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
160
- "layers.11.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
161
- "layers.11.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
162
  "layers.11.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
163
  "layers.11.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
164
- "layers.11.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
165
  "layers.11.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
166
  "layers.11.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
167
  "layers.11.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
168
  "layers.11.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
169
  "layers.11.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
170
  "layers.11.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
171
- "layers.11.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
172
- "layers.11.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
173
  "layers.11.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
174
  "layers.11.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
175
- "layers.11.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
176
  "layers.11.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
177
  "layers.11.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
178
  "layers.11.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
179
  "layers.11.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
180
  "layers.11.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
181
  "layers.11.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
182
- "layers.11.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
183
- "layers.11.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
184
  "layers.11.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
185
  "layers.11.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
186
- "layers.11.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
187
  "layers.11.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
188
  "layers.11.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
  "layers.11.gate.0.weight": "model-00003-of-00004.safetensors",
190
  "layers.11.gate.1.weight": "model-00003-of-00004.safetensors",
 
191
  "layers.12.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
192
  "layers.12.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
193
  "layers.12.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
194
  "layers.12.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
195
- "layers.12.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
196
- "layers.12.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
197
  "layers.12.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
198
  "layers.12.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
199
- "layers.12.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
200
  "layers.12.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
201
  "layers.12.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
202
  "layers.12.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
203
  "layers.12.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
204
  "layers.12.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
205
  "layers.12.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
206
- "layers.12.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
207
- "layers.12.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
208
  "layers.12.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
209
  "layers.12.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
210
- "layers.12.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
211
  "layers.12.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
212
  "layers.12.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
213
  "layers.12.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
214
  "layers.12.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
215
  "layers.12.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
216
  "layers.12.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
217
- "layers.12.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
218
- "layers.12.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
219
  "layers.12.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
220
  "layers.12.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
221
- "layers.12.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
222
  "layers.12.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
223
  "layers.12.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
224
- "layers.12.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
 
225
  "layers.12.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
226
  "layers.12.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
227
- "layers.12.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
228
- "layers.12.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
229
- "layers.12.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
230
  "layers.12.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
231
  "layers.12.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
232
- "layers.12.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
233
  "layers.12.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
234
  "layers.12.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
235
  "layers.12.gate.0.weight": "model-00003-of-00004.safetensors",
236
  "layers.12.gate.1.weight": "model-00003-of-00004.safetensors",
237
- "layers.13.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
238
- "layers.13.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
239
- "layers.13.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
240
- "layers.13.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
241
- "layers.13.experts.0.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
242
- "layers.13.experts.0.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
243
- "layers.13.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
244
- "layers.13.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
245
- "layers.13.experts.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
246
- "layers.13.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
247
- "layers.13.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
248
- "layers.13.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
249
- "layers.13.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
250
- "layers.13.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
251
- "layers.13.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
252
- "layers.13.experts.1.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
253
- "layers.13.experts.1.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
254
- "layers.13.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
255
- "layers.13.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
256
- "layers.13.experts.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
257
- "layers.13.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
258
- "layers.13.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
259
- "layers.13.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
260
- "layers.13.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
261
- "layers.13.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
262
- "layers.13.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
263
- "layers.13.experts.2.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
264
- "layers.13.experts.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
265
- "layers.13.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
266
- "layers.13.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
267
- "layers.13.experts.2.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
268
- "layers.13.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
269
- "layers.13.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
270
  "layers.13.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
271
- "layers.13.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
272
- "layers.13.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
273
  "layers.13.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
274
- "layers.13.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
275
- "layers.13.experts.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
276
- "layers.13.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
277
- "layers.13.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
278
- "layers.13.experts.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
279
- "layers.13.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
280
- "layers.13.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
281
- "layers.13.gate.0.weight": "model-00004-of-00004.safetensors",
282
- "layers.13.gate.1.weight": "model-00004-of-00004.safetensors",
283
  "layers.14.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
284
  "layers.14.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
285
  "layers.14.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
286
  "layers.14.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
287
- "layers.14.experts.0.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
288
- "layers.14.experts.0.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
289
  "layers.14.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
290
  "layers.14.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
291
- "layers.14.experts.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
292
  "layers.14.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
293
  "layers.14.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
294
  "layers.14.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
295
  "layers.14.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
296
  "layers.14.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
297
  "layers.14.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
298
- "layers.14.experts.1.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
299
- "layers.14.experts.1.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
300
  "layers.14.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
301
  "layers.14.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
302
- "layers.14.experts.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
303
  "layers.14.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
304
  "layers.14.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
305
  "layers.14.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
306
  "layers.14.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
307
  "layers.14.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
308
  "layers.14.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
309
- "layers.14.experts.2.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
310
- "layers.14.experts.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
311
  "layers.14.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
312
  "layers.14.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
313
- "layers.14.experts.2.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
314
  "layers.14.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
315
  "layers.14.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
316
  "layers.14.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
317
  "layers.14.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
318
  "layers.14.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
319
  "layers.14.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
320
- "layers.14.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
321
- "layers.14.experts.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
322
  "layers.14.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
323
  "layers.14.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
324
- "layers.14.experts.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
325
  "layers.14.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
326
  "layers.14.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
327
  "layers.14.gate.0.weight": "model-00004-of-00004.safetensors",
328
  "layers.14.gate.1.weight": "model-00004-of-00004.safetensors",
 
329
  "layers.15.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
330
  "layers.15.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
331
  "layers.15.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
332
  "layers.15.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
333
- "layers.15.experts.0.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
334
- "layers.15.experts.0.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
335
  "layers.15.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
336
  "layers.15.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
337
- "layers.15.experts.0.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
338
  "layers.15.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
339
  "layers.15.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
340
  "layers.15.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
341
  "layers.15.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
342
  "layers.15.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
343
  "layers.15.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
344
- "layers.15.experts.1.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
345
- "layers.15.experts.1.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
346
  "layers.15.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
347
  "layers.15.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
348
- "layers.15.experts.1.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
349
  "layers.15.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
350
  "layers.15.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
351
  "layers.15.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
352
  "layers.15.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
353
  "layers.15.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
354
  "layers.15.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
355
- "layers.15.experts.2.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
356
- "layers.15.experts.2.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
357
  "layers.15.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
358
  "layers.15.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
359
- "layers.15.experts.2.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
360
  "layers.15.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
361
  "layers.15.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
 
362
  "layers.15.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
363
  "layers.15.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
364
  "layers.15.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
365
  "layers.15.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
366
- "layers.15.experts.3.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
367
- "layers.15.experts.3.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
368
  "layers.15.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
369
  "layers.15.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
370
- "layers.15.experts.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
371
  "layers.15.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
372
  "layers.15.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
373
  "layers.15.gate.0.weight": "model-00004-of-00004.safetensors",
374
  "layers.15.gate.1.weight": "model-00004-of-00004.safetensors",
 
375
  "layers.2.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
376
  "layers.2.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
377
  "layers.2.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
378
  "layers.2.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
379
- "layers.2.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
380
- "layers.2.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
381
  "layers.2.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
382
  "layers.2.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
383
- "layers.2.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
384
  "layers.2.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
385
  "layers.2.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
386
  "layers.2.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
387
  "layers.2.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
388
  "layers.2.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
389
  "layers.2.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
390
- "layers.2.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
391
- "layers.2.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
392
  "layers.2.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
393
  "layers.2.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
394
- "layers.2.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
395
  "layers.2.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
396
  "layers.2.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
397
  "layers.2.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
398
  "layers.2.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
399
  "layers.2.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
400
  "layers.2.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
401
- "layers.2.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
402
- "layers.2.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
403
  "layers.2.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
404
  "layers.2.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
405
- "layers.2.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
406
  "layers.2.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
407
  "layers.2.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
408
  "layers.2.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
409
  "layers.2.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
410
  "layers.2.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
411
  "layers.2.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
412
- "layers.2.experts.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
413
- "layers.2.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
414
  "layers.2.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
415
  "layers.2.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
416
- "layers.2.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
417
  "layers.2.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
418
  "layers.2.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
419
  "layers.2.gate.0.weight": "model-00001-of-00004.safetensors",
420
  "layers.2.gate.1.weight": "model-00001-of-00004.safetensors",
 
421
  "layers.3.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
422
  "layers.3.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
423
  "layers.3.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
424
  "layers.3.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
425
- "layers.3.experts.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
426
- "layers.3.experts.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
427
  "layers.3.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
428
  "layers.3.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
429
- "layers.3.experts.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
430
  "layers.3.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
431
  "layers.3.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
432
  "layers.3.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
433
  "layers.3.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
434
  "layers.3.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
435
  "layers.3.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
436
- "layers.3.experts.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
437
- "layers.3.experts.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
438
  "layers.3.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
439
  "layers.3.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
440
- "layers.3.experts.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
441
  "layers.3.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
442
  "layers.3.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
443
  "layers.3.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
444
  "layers.3.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
445
  "layers.3.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
446
  "layers.3.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
447
- "layers.3.experts.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
448
- "layers.3.experts.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
449
  "layers.3.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
450
  "layers.3.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
451
- "layers.3.experts.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
452
  "layers.3.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
453
  "layers.3.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
 
454
  "layers.3.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
455
- "layers.3.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
456
- "layers.3.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
457
  "layers.3.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
458
- "layers.3.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
459
- "layers.3.experts.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
460
  "layers.3.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
461
  "layers.3.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
462
- "layers.3.experts.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
463
  "layers.3.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
464
  "layers.3.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
465
  "layers.3.gate.0.weight": "model-00001-of-00004.safetensors",
466
  "layers.3.gate.1.weight": "model-00001-of-00004.safetensors",
 
467
  "layers.4.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
468
  "layers.4.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
469
  "layers.4.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
470
  "layers.4.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
471
- "layers.4.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
472
- "layers.4.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
473
  "layers.4.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
474
  "layers.4.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
475
- "layers.4.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
476
  "layers.4.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
477
  "layers.4.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
478
  "layers.4.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
479
  "layers.4.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
480
  "layers.4.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
481
  "layers.4.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
482
- "layers.4.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
483
- "layers.4.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
484
  "layers.4.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
485
  "layers.4.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
486
- "layers.4.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
487
  "layers.4.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
488
  "layers.4.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
489
  "layers.4.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
490
  "layers.4.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
491
  "layers.4.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
492
  "layers.4.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
493
- "layers.4.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
494
- "layers.4.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
495
  "layers.4.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
496
  "layers.4.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
497
- "layers.4.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
498
  "layers.4.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
499
  "layers.4.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
500
  "layers.4.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
501
  "layers.4.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
502
  "layers.4.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
503
  "layers.4.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
504
- "layers.4.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
505
- "layers.4.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
506
  "layers.4.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
507
  "layers.4.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
508
- "layers.4.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
509
  "layers.4.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
510
  "layers.4.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
511
  "layers.4.gate.0.weight": "model-00002-of-00004.safetensors",
512
  "layers.4.gate.1.weight": "model-00002-of-00004.safetensors",
 
513
  "layers.5.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
514
  "layers.5.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
515
  "layers.5.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
516
  "layers.5.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
517
- "layers.5.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
518
- "layers.5.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
519
  "layers.5.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
520
  "layers.5.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
521
- "layers.5.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
522
  "layers.5.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
523
  "layers.5.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
524
  "layers.5.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
525
  "layers.5.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
526
  "layers.5.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
527
  "layers.5.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
528
- "layers.5.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
529
- "layers.5.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
530
  "layers.5.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
531
  "layers.5.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
532
- "layers.5.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
533
  "layers.5.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
534
  "layers.5.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
535
  "layers.5.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
536
  "layers.5.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
537
  "layers.5.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
538
  "layers.5.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
539
- "layers.5.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
540
- "layers.5.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
541
  "layers.5.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
542
  "layers.5.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
543
- "layers.5.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
544
  "layers.5.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
545
  "layers.5.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
546
  "layers.5.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
547
  "layers.5.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
548
  "layers.5.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
549
  "layers.5.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
550
- "layers.5.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
551
- "layers.5.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
552
  "layers.5.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
553
  "layers.5.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
554
- "layers.5.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
555
  "layers.5.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
556
  "layers.5.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
557
  "layers.5.gate.0.weight": "model-00002-of-00004.safetensors",
558
  "layers.5.gate.1.weight": "model-00002-of-00004.safetensors",
 
559
  "layers.6.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
560
  "layers.6.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
561
  "layers.6.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
562
  "layers.6.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
563
- "layers.6.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
564
- "layers.6.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
565
  "layers.6.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
566
  "layers.6.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
567
- "layers.6.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
568
  "layers.6.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
569
  "layers.6.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
570
  "layers.6.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
571
  "layers.6.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
572
  "layers.6.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
573
  "layers.6.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
574
- "layers.6.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
575
- "layers.6.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
576
  "layers.6.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
577
  "layers.6.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
578
- "layers.6.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
579
  "layers.6.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
580
  "layers.6.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
581
  "layers.6.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
582
  "layers.6.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
583
  "layers.6.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
584
  "layers.6.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
585
- "layers.6.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
586
- "layers.6.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
587
  "layers.6.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
588
  "layers.6.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
589
- "layers.6.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
590
  "layers.6.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
591
  "layers.6.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
592
  "layers.6.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
593
  "layers.6.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
594
  "layers.6.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
595
  "layers.6.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
596
- "layers.6.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
597
- "layers.6.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
598
  "layers.6.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
599
  "layers.6.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
600
- "layers.6.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
601
  "layers.6.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
602
  "layers.6.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
603
  "layers.6.gate.0.weight": "model-00002-of-00004.safetensors",
604
  "layers.6.gate.1.weight": "model-00002-of-00004.safetensors",
 
605
  "layers.7.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
606
  "layers.7.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
607
  "layers.7.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
608
  "layers.7.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
609
- "layers.7.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
610
- "layers.7.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
611
  "layers.7.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
612
  "layers.7.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
613
- "layers.7.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
614
  "layers.7.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
615
  "layers.7.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
616
  "layers.7.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
617
  "layers.7.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
618
  "layers.7.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
619
  "layers.7.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
620
- "layers.7.experts.1.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
621
- "layers.7.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
622
  "layers.7.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
623
  "layers.7.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
624
- "layers.7.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
625
  "layers.7.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
626
  "layers.7.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
627
  "layers.7.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
628
  "layers.7.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
629
  "layers.7.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
630
  "layers.7.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
631
- "layers.7.experts.2.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
632
- "layers.7.experts.2.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
633
  "layers.7.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
634
  "layers.7.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
635
- "layers.7.experts.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
636
  "layers.7.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
637
  "layers.7.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
638
  "layers.7.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
639
  "layers.7.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
640
  "layers.7.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
641
  "layers.7.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
642
- "layers.7.experts.3.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
643
- "layers.7.experts.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
644
  "layers.7.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
645
  "layers.7.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
646
- "layers.7.experts.3.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
647
  "layers.7.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
648
  "layers.7.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
649
  "layers.7.gate.0.weight": "model-00002-of-00004.safetensors",
650
  "layers.7.gate.1.weight": "model-00002-of-00004.safetensors",
 
651
  "layers.8.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
652
  "layers.8.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
653
  "layers.8.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
654
  "layers.8.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
655
- "layers.8.experts.0.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
656
- "layers.8.experts.0.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
657
  "layers.8.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
658
  "layers.8.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
659
- "layers.8.experts.0.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
660
  "layers.8.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
661
  "layers.8.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
662
- "layers.8.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
 
663
  "layers.8.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
664
- "layers.8.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
665
- "layers.8.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
666
- "layers.8.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
667
- "layers.8.experts.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
668
  "layers.8.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
669
  "layers.8.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
670
- "layers.8.experts.1.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
671
  "layers.8.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
672
  "layers.8.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
673
- "layers.8.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
674
- "layers.8.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
675
- "layers.8.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
676
- "layers.8.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
677
- "layers.8.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
678
- "layers.8.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
679
- "layers.8.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
680
- "layers.8.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
681
- "layers.8.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
682
- "layers.8.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
683
- "layers.8.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
684
  "layers.8.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
685
- "layers.8.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
686
- "layers.8.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
687
  "layers.8.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
688
- "layers.8.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
689
- "layers.8.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
690
- "layers.8.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
691
- "layers.8.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
692
- "layers.8.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
693
- "layers.8.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
694
- "layers.8.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
695
  "layers.8.gate.0.weight": "model-00002-of-00004.safetensors",
696
  "layers.8.gate.1.weight": "model-00002-of-00004.safetensors",
 
697
  "layers.9.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
698
  "layers.9.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
699
  "layers.9.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
700
  "layers.9.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
701
- "layers.9.experts.0.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
702
- "layers.9.experts.0.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
703
  "layers.9.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
704
  "layers.9.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
705
- "layers.9.experts.0.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
706
  "layers.9.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
707
  "layers.9.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
708
  "layers.9.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
709
  "layers.9.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
710
  "layers.9.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
711
  "layers.9.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
712
- "layers.9.experts.1.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
713
- "layers.9.experts.1.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
714
  "layers.9.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
715
  "layers.9.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
716
- "layers.9.experts.1.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
717
  "layers.9.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
718
  "layers.9.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
719
  "layers.9.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
720
  "layers.9.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
721
  "layers.9.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
722
  "layers.9.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
723
- "layers.9.experts.2.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
724
- "layers.9.experts.2.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
725
  "layers.9.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
726
  "layers.9.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
727
- "layers.9.experts.2.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
728
  "layers.9.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
729
  "layers.9.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
 
730
  "layers.9.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
731
  "layers.9.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
732
  "layers.9.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
733
  "layers.9.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
734
- "layers.9.experts.3.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
735
- "layers.9.experts.3.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
736
  "layers.9.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
737
  "layers.9.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
738
- "layers.9.experts.3.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
739
  "layers.9.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
740
  "layers.9.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
741
  "layers.9.gate.0.weight": "model-00003-of-00004.safetensors",
742
  "layers.9.gate.1.weight": "model-00003-of-00004.safetensors",
743
- "lm_head.weight": "model-00004-of-00004.safetensors",
744
- "norm.weight": "model-00004-of-00004.safetensors"
745
  }
746
  }
 
1
  {
2
  "metadata": {
3
+ "total_size": 17940619264
4
  },
5
  "weight_map": {
6
  "embed_tokens.weight": "model-00001-of-00004.safetensors",
7
+ "final_norm.weight": "model-00004-of-00004.safetensors",
8
+ "layers.0.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
9
  "layers.0.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
10
  "layers.0.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
11
  "layers.0.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
12
  "layers.0.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
13
  "layers.0.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
14
  "layers.0.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
15
  "layers.0.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
16
  "layers.0.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
17
+ "layers.0.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
18
  "layers.0.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
19
  "layers.0.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
20
  "layers.0.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
21
  "layers.0.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
22
  "layers.0.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
23
  "layers.0.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
24
  "layers.0.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
25
  "layers.0.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
26
+ "layers.0.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
27
  "layers.0.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
28
  "layers.0.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
29
  "layers.0.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
30
  "layers.0.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
31
  "layers.0.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
32
  "layers.0.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
33
  "layers.0.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
34
  "layers.0.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
35
+ "layers.0.experts.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
36
  "layers.0.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
37
  "layers.0.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
38
  "layers.0.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
39
  "layers.0.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
40
  "layers.0.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
41
  "layers.0.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
42
  "layers.0.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
43
  "layers.0.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
44
  "layers.0.gate.0.weight": "model-00001-of-00004.safetensors",
45
  "layers.0.gate.1.weight": "model-00001-of-00004.safetensors",
46
+ "layers.1.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
47
  "layers.1.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
48
  "layers.1.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
49
  "layers.1.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
50
  "layers.1.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
51
  "layers.1.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
52
  "layers.1.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
53
  "layers.1.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
54
  "layers.1.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
55
+ "layers.1.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
56
  "layers.1.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
57
  "layers.1.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
58
  "layers.1.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
59
  "layers.1.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
60
  "layers.1.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
61
  "layers.1.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
62
  "layers.1.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
63
  "layers.1.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
64
+ "layers.1.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
65
  "layers.1.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
66
  "layers.1.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
67
  "layers.1.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
68
  "layers.1.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
69
  "layers.1.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
70
  "layers.1.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
71
  "layers.1.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
72
  "layers.1.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
73
+ "layers.1.experts.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
74
  "layers.1.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
75
  "layers.1.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
76
  "layers.1.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
77
  "layers.1.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
78
  "layers.1.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
79
  "layers.1.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
80
  "layers.1.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
81
  "layers.1.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
82
  "layers.1.gate.0.weight": "model-00001-of-00004.safetensors",
83
  "layers.1.gate.1.weight": "model-00001-of-00004.safetensors",
84
+ "layers.10.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
85
  "layers.10.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
86
  "layers.10.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
87
  "layers.10.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
88
  "layers.10.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
89
  "layers.10.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
90
  "layers.10.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
91
  "layers.10.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
92
  "layers.10.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
93
+ "layers.10.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
94
  "layers.10.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
95
  "layers.10.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
96
  "layers.10.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
97
  "layers.10.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
98
  "layers.10.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
99
  "layers.10.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
100
  "layers.10.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
101
  "layers.10.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
102
+ "layers.10.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
103
  "layers.10.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
104
  "layers.10.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
105
  "layers.10.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
106
  "layers.10.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
107
  "layers.10.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
108
  "layers.10.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
109
  "layers.10.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
110
  "layers.10.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
111
+ "layers.10.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
112
  "layers.10.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
113
  "layers.10.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
114
  "layers.10.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
115
  "layers.10.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
116
  "layers.10.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
117
  "layers.10.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
118
  "layers.10.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
119
  "layers.10.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
120
  "layers.10.gate.0.weight": "model-00003-of-00004.safetensors",
121
  "layers.10.gate.1.weight": "model-00003-of-00004.safetensors",
122
+ "layers.11.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
123
  "layers.11.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
124
  "layers.11.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
125
  "layers.11.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
126
  "layers.11.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
127
  "layers.11.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
128
  "layers.11.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
129
  "layers.11.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
130
  "layers.11.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
131
+ "layers.11.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
132
  "layers.11.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
133
  "layers.11.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
134
  "layers.11.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
135
  "layers.11.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
136
  "layers.11.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
137
  "layers.11.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
138
  "layers.11.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
139
  "layers.11.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
140
+ "layers.11.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
141
  "layers.11.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
142
  "layers.11.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
143
  "layers.11.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
144
  "layers.11.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
145
  "layers.11.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
146
  "layers.11.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
147
  "layers.11.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
148
  "layers.11.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
149
+ "layers.11.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
150
  "layers.11.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
151
  "layers.11.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
152
  "layers.11.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
153
  "layers.11.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
154
  "layers.11.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
155
  "layers.11.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
156
  "layers.11.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
157
  "layers.11.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
158
  "layers.11.gate.0.weight": "model-00003-of-00004.safetensors",
159
  "layers.11.gate.1.weight": "model-00003-of-00004.safetensors",
160
+ "layers.12.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
161
  "layers.12.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
162
  "layers.12.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
163
  "layers.12.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
164
  "layers.12.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
165
  "layers.12.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
166
  "layers.12.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
167
  "layers.12.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
168
  "layers.12.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
169
+ "layers.12.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
170
  "layers.12.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
171
  "layers.12.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
172
  "layers.12.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
173
  "layers.12.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
174
  "layers.12.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
175
  "layers.12.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
176
  "layers.12.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
177
  "layers.12.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
178
+ "layers.12.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
179
  "layers.12.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
180
  "layers.12.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
181
  "layers.12.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
182
  "layers.12.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
183
  "layers.12.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
  "layers.12.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
185
  "layers.12.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
186
  "layers.12.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
187
+ "layers.12.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
188
+ "layers.12.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
189
  "layers.12.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
190
  "layers.12.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
191
+ "layers.12.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
192
  "layers.12.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
193
  "layers.12.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
194
  "layers.12.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
195
  "layers.12.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
196
  "layers.12.gate.0.weight": "model-00003-of-00004.safetensors",
197
  "layers.12.gate.1.weight": "model-00003-of-00004.safetensors",
198
+ "layers.13.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
199
+ "layers.13.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
200
+ "layers.13.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
201
+ "layers.13.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
202
+ "layers.13.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
203
+ "layers.13.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
204
+ "layers.13.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
205
+ "layers.13.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
206
+ "layers.13.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
207
+ "layers.13.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
208
+ "layers.13.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
209
+ "layers.13.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
210
+ "layers.13.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
211
+ "layers.13.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
212
+ "layers.13.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
213
+ "layers.13.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
214
+ "layers.13.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
215
+ "layers.13.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
216
+ "layers.13.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
217
+ "layers.13.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
218
+ "layers.13.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
219
+ "layers.13.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
220
+ "layers.13.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
221
+ "layers.13.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
222
+ "layers.13.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
223
+ "layers.13.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
224
+ "layers.13.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
+ "layers.13.experts.3.input_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
 
 
 
226
  "layers.13.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
227
+ "layers.13.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
228
+ "layers.13.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
229
  "layers.13.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
230
+ "layers.13.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
231
+ "layers.13.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
232
+ "layers.13.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
233
+ "layers.13.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
234
+ "layers.13.gate.0.weight": "model-00003-of-00004.safetensors",
235
+ "layers.13.gate.1.weight": "model-00003-of-00004.safetensors",
236
+ "layers.14.experts.0.input_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
237
  "layers.14.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
238
  "layers.14.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
239
  "layers.14.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
240
  "layers.14.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
241
  "layers.14.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
242
  "layers.14.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
243
  "layers.14.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
244
  "layers.14.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
245
+ "layers.14.experts.1.input_layernorm.weight": "model-00004-of-00004.safetensors",
246
  "layers.14.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
247
  "layers.14.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
248
  "layers.14.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
249
  "layers.14.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
250
  "layers.14.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
251
  "layers.14.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
252
  "layers.14.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
253
  "layers.14.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
254
+ "layers.14.experts.2.input_layernorm.weight": "model-00004-of-00004.safetensors",
255
  "layers.14.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
256
  "layers.14.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
257
  "layers.14.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
258
  "layers.14.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
259
  "layers.14.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
260
  "layers.14.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
261
  "layers.14.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
262
  "layers.14.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
263
+ "layers.14.experts.3.input_layernorm.weight": "model-00004-of-00004.safetensors",
264
  "layers.14.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
265
  "layers.14.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
266
  "layers.14.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
267
  "layers.14.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
268
  "layers.14.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
269
  "layers.14.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
270
  "layers.14.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
271
  "layers.14.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
272
  "layers.14.gate.0.weight": "model-00004-of-00004.safetensors",
273
  "layers.14.gate.1.weight": "model-00004-of-00004.safetensors",
274
+ "layers.15.experts.0.input_layernorm.weight": "model-00004-of-00004.safetensors",
275
  "layers.15.experts.0.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
276
  "layers.15.experts.0.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
277
  "layers.15.experts.0.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
278
  "layers.15.experts.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
279
  "layers.15.experts.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
280
  "layers.15.experts.0.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
281
  "layers.15.experts.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
282
  "layers.15.experts.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
283
+ "layers.15.experts.1.input_layernorm.weight": "model-00004-of-00004.safetensors",
284
  "layers.15.experts.1.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
285
  "layers.15.experts.1.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
286
  "layers.15.experts.1.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
287
  "layers.15.experts.1.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
288
  "layers.15.experts.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
289
  "layers.15.experts.1.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
290
  "layers.15.experts.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
291
  "layers.15.experts.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
292
+ "layers.15.experts.2.input_layernorm.weight": "model-00004-of-00004.safetensors",
293
  "layers.15.experts.2.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
294
  "layers.15.experts.2.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
295
  "layers.15.experts.2.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
296
  "layers.15.experts.2.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
297
  "layers.15.experts.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
298
  "layers.15.experts.2.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
299
  "layers.15.experts.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
300
  "layers.15.experts.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
301
+ "layers.15.experts.3.input_layernorm.weight": "model-00004-of-00004.safetensors",
302
  "layers.15.experts.3.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
303
  "layers.15.experts.3.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
304
  "layers.15.experts.3.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
305
  "layers.15.experts.3.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
 
 
306
  "layers.15.experts.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
307
  "layers.15.experts.3.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
 
308
  "layers.15.experts.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
309
  "layers.15.experts.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
310
  "layers.15.gate.0.weight": "model-00004-of-00004.safetensors",
311
  "layers.15.gate.1.weight": "model-00004-of-00004.safetensors",
312
+ "layers.2.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
313
  "layers.2.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
314
  "layers.2.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
315
  "layers.2.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
316
  "layers.2.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
317
  "layers.2.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
318
  "layers.2.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
319
  "layers.2.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
320
  "layers.2.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
321
+ "layers.2.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
322
  "layers.2.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
323
  "layers.2.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
324
  "layers.2.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
325
  "layers.2.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
326
  "layers.2.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
327
  "layers.2.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
328
  "layers.2.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
329
  "layers.2.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
330
+ "layers.2.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
331
  "layers.2.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
332
  "layers.2.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
333
  "layers.2.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
334
  "layers.2.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
335
  "layers.2.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
336
  "layers.2.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
337
  "layers.2.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
338
  "layers.2.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
339
+ "layers.2.experts.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
340
  "layers.2.experts.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
341
  "layers.2.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
342
  "layers.2.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
343
  "layers.2.experts.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
344
  "layers.2.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
345
  "layers.2.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
346
  "layers.2.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
347
  "layers.2.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
348
  "layers.2.gate.0.weight": "model-00001-of-00004.safetensors",
349
  "layers.2.gate.1.weight": "model-00001-of-00004.safetensors",
350
+ "layers.3.experts.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
351
  "layers.3.experts.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
352
  "layers.3.experts.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
353
  "layers.3.experts.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
354
  "layers.3.experts.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
355
  "layers.3.experts.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
356
  "layers.3.experts.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
357
  "layers.3.experts.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
358
  "layers.3.experts.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
359
+ "layers.3.experts.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
360
  "layers.3.experts.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
361
  "layers.3.experts.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
362
  "layers.3.experts.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
363
  "layers.3.experts.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
364
  "layers.3.experts.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
365
  "layers.3.experts.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
366
  "layers.3.experts.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
367
  "layers.3.experts.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
368
+ "layers.3.experts.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
369
  "layers.3.experts.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
370
  "layers.3.experts.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
371
  "layers.3.experts.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
372
  "layers.3.experts.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
 
 
373
  "layers.3.experts.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
374
  "layers.3.experts.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
375
  "layers.3.experts.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
376
  "layers.3.experts.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
377
+ "layers.3.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
378
  "layers.3.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
379
+ "layers.3.experts.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
380
+ "layers.3.experts.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
381
  "layers.3.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
382
  "layers.3.experts.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
383
  "layers.3.experts.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
 
384
  "layers.3.experts.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
385
  "layers.3.experts.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
386
  "layers.3.gate.0.weight": "model-00001-of-00004.safetensors",
387
  "layers.3.gate.1.weight": "model-00001-of-00004.safetensors",
388
+ "layers.4.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
389
  "layers.4.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
390
  "layers.4.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
391
  "layers.4.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
392
  "layers.4.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
393
  "layers.4.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
394
  "layers.4.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
395
  "layers.4.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
396
  "layers.4.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
397
+ "layers.4.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
398
  "layers.4.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
399
  "layers.4.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
400
  "layers.4.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
401
  "layers.4.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
402
  "layers.4.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
403
  "layers.4.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
404
  "layers.4.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
405
  "layers.4.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
406
+ "layers.4.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
407
  "layers.4.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
408
  "layers.4.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
409
  "layers.4.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
410
  "layers.4.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
411
  "layers.4.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
412
  "layers.4.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
413
  "layers.4.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
414
  "layers.4.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
415
+ "layers.4.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
416
  "layers.4.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
417
  "layers.4.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
418
  "layers.4.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
419
  "layers.4.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
420
  "layers.4.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
421
  "layers.4.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
422
  "layers.4.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
423
  "layers.4.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
424
  "layers.4.gate.0.weight": "model-00002-of-00004.safetensors",
425
  "layers.4.gate.1.weight": "model-00002-of-00004.safetensors",
426
+ "layers.5.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
427
  "layers.5.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
428
  "layers.5.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
429
  "layers.5.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
430
  "layers.5.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
431
  "layers.5.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
432
  "layers.5.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
433
  "layers.5.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
434
  "layers.5.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
435
+ "layers.5.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
436
  "layers.5.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
437
  "layers.5.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
438
  "layers.5.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
439
  "layers.5.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
440
  "layers.5.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
441
  "layers.5.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
442
  "layers.5.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
443
  "layers.5.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
444
+ "layers.5.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
445
  "layers.5.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
446
  "layers.5.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
447
  "layers.5.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
448
  "layers.5.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
449
  "layers.5.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
450
  "layers.5.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
451
  "layers.5.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
452
  "layers.5.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
453
+ "layers.5.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
454
  "layers.5.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
455
  "layers.5.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
456
  "layers.5.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
457
  "layers.5.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
458
  "layers.5.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
459
  "layers.5.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
460
  "layers.5.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
461
  "layers.5.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
462
  "layers.5.gate.0.weight": "model-00002-of-00004.safetensors",
463
  "layers.5.gate.1.weight": "model-00002-of-00004.safetensors",
464
+ "layers.6.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
465
  "layers.6.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
466
  "layers.6.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
467
  "layers.6.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
468
  "layers.6.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
469
  "layers.6.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
470
  "layers.6.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
471
  "layers.6.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
472
  "layers.6.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
473
+ "layers.6.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
474
  "layers.6.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
475
  "layers.6.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
476
  "layers.6.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
477
  "layers.6.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
478
  "layers.6.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
479
  "layers.6.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
480
  "layers.6.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
481
  "layers.6.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
482
+ "layers.6.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
483
  "layers.6.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
484
  "layers.6.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
485
  "layers.6.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
486
  "layers.6.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
487
  "layers.6.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
488
  "layers.6.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
489
  "layers.6.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
490
  "layers.6.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
491
+ "layers.6.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
492
  "layers.6.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
493
  "layers.6.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
494
  "layers.6.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
495
  "layers.6.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
496
  "layers.6.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
497
  "layers.6.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
498
  "layers.6.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
499
  "layers.6.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
500
  "layers.6.gate.0.weight": "model-00002-of-00004.safetensors",
501
  "layers.6.gate.1.weight": "model-00002-of-00004.safetensors",
502
+ "layers.7.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
503
  "layers.7.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
504
  "layers.7.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
505
  "layers.7.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
506
  "layers.7.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
507
  "layers.7.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
508
  "layers.7.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
509
  "layers.7.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
510
  "layers.7.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
511
+ "layers.7.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
512
  "layers.7.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
513
  "layers.7.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
514
  "layers.7.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
515
  "layers.7.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
516
  "layers.7.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
517
  "layers.7.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
518
  "layers.7.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
519
  "layers.7.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
520
+ "layers.7.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
521
  "layers.7.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
522
  "layers.7.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
523
  "layers.7.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
524
  "layers.7.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
525
  "layers.7.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
526
  "layers.7.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
527
  "layers.7.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
528
  "layers.7.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
529
+ "layers.7.experts.3.input_layernorm.weight": "model-00002-of-00004.safetensors",
530
  "layers.7.experts.3.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
531
  "layers.7.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
532
  "layers.7.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
533
  "layers.7.experts.3.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
534
  "layers.7.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
535
  "layers.7.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
536
  "layers.7.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
537
  "layers.7.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
538
  "layers.7.gate.0.weight": "model-00002-of-00004.safetensors",
539
  "layers.7.gate.1.weight": "model-00002-of-00004.safetensors",
540
+ "layers.8.experts.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
541
  "layers.8.experts.0.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
542
  "layers.8.experts.0.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
543
  "layers.8.experts.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
544
  "layers.8.experts.0.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
545
  "layers.8.experts.0.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
546
  "layers.8.experts.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
547
  "layers.8.experts.0.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
548
  "layers.8.experts.0.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
549
+ "layers.8.experts.1.input_layernorm.weight": "model-00002-of-00004.safetensors",
550
+ "layers.8.experts.1.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
551
  "layers.8.experts.1.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
552
+ "layers.8.experts.1.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
553
+ "layers.8.experts.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
 
 
554
  "layers.8.experts.1.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
555
  "layers.8.experts.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
 
556
  "layers.8.experts.1.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
557
  "layers.8.experts.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
558
+ "layers.8.experts.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
559
+ "layers.8.experts.2.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
560
+ "layers.8.experts.2.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
561
+ "layers.8.experts.2.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
562
+ "layers.8.experts.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
563
+ "layers.8.experts.2.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
564
+ "layers.8.experts.2.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
565
+ "layers.8.experts.2.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
566
+ "layers.8.experts.2.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
567
+ "layers.8.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
 
568
  "layers.8.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
569
+ "layers.8.experts.3.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
570
+ "layers.8.experts.3.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
571
  "layers.8.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
572
+ "layers.8.experts.3.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
573
+ "layers.8.experts.3.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
574
+ "layers.8.experts.3.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
575
+ "layers.8.experts.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
 
 
 
576
  "layers.8.gate.0.weight": "model-00002-of-00004.safetensors",
577
  "layers.8.gate.1.weight": "model-00002-of-00004.safetensors",
578
+ "layers.9.experts.0.input_layernorm.weight": "model-00003-of-00004.safetensors",
579
  "layers.9.experts.0.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
580
  "layers.9.experts.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
581
  "layers.9.experts.0.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
582
  "layers.9.experts.0.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
583
  "layers.9.experts.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
584
  "layers.9.experts.0.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
585
  "layers.9.experts.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
586
  "layers.9.experts.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
587
+ "layers.9.experts.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
588
  "layers.9.experts.1.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
589
  "layers.9.experts.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
590
  "layers.9.experts.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
591
  "layers.9.experts.1.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
592
  "layers.9.experts.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
593
  "layers.9.experts.1.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
594
  "layers.9.experts.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
595
  "layers.9.experts.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
596
+ "layers.9.experts.2.input_layernorm.weight": "model-00003-of-00004.safetensors",
597
  "layers.9.experts.2.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
598
  "layers.9.experts.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
599
  "layers.9.experts.2.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
600
  "layers.9.experts.2.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
601
  "layers.9.experts.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
602
  "layers.9.experts.2.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
603
  "layers.9.experts.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
604
  "layers.9.experts.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
605
+ "layers.9.experts.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
606
  "layers.9.experts.3.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
607
  "layers.9.experts.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
608
  "layers.9.experts.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
609
  "layers.9.experts.3.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
 
 
610
  "layers.9.experts.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
611
  "layers.9.experts.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
 
612
  "layers.9.experts.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
613
  "layers.9.experts.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
614
  "layers.9.gate.0.weight": "model-00003-of-00004.safetensors",
615
  "layers.9.gate.1.weight": "model-00003-of-00004.safetensors",
616
+ "lm_head.weight": "model-00004-of-00004.safetensors"
 
617
  }
618
  }