Hugging Face's logo

AlexNG01
/
Affine_bee0

Text Generation
Transformers
Safetensors
qwen3_next
conversational
fp8
Model card Files
xet
Community
AlexNG01 commited on
Commit
76fe072
·
verified ·
1 Parent(s): 8071cd5

Upload Qwen3NextForCausalLM

Browse files
Files changed (21) hide show
  1. .gitattributes +1 -0
  2. config.json +511 -0
  3. generation_config.json +13 -0
  4. model-00001-of-00017.safetensors +3 -0
  5. model-00002-of-00017.safetensors +3 -0
  6. model-00003-of-00017.safetensors +3 -0
  7. model-00004-of-00017.safetensors +3 -0
  8. model-00005-of-00017.safetensors +3 -0
  9. model-00006-of-00017.safetensors +3 -0
  10. model-00007-of-00017.safetensors +3 -0
  11. model-00008-of-00017.safetensors +3 -0
  12. model-00009-of-00017.safetensors +3 -0
  13. model-00010-of-00017.safetensors +3 -0
  14. model-00011-of-00017.safetensors +3 -0
  15. model-00012-of-00017.safetensors +3 -0
  16. model-00013-of-00017.safetensors +3 -0
  17. model-00014-of-00017.safetensors +3 -0
  18. model-00015-of-00017.safetensors +3 -0
  19. model-00016-of-00017.safetensors +3 -0
  20. model-00017-of-00017.safetensors +3 -0
  21. model.safetensors.index.json +3 -0
.gitattributes CHANGED
@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,511 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3NextForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "decoder_sparse_step": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 151645,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2048,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 5120,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention",
50
+ "linear_attention",
51
+ "linear_attention",
52
+ "linear_attention",
53
+ "full_attention",
54
+ "linear_attention",
55
+ "linear_attention",
56
+ "linear_attention",
57
+ "full_attention",
58
+ "linear_attention",
59
+ "linear_attention",
60
+ "linear_attention",
61
+ "full_attention",
62
+ "linear_attention",
63
+ "linear_attention",
64
+ "linear_attention",
65
+ "full_attention"
66
+ ],
67
+ "linear_conv_kernel_dim": 4,
68
+ "linear_key_head_dim": 128,
69
+ "linear_num_key_heads": 16,
70
+ "linear_num_value_heads": 32,
71
+ "linear_value_head_dim": 128,
72
+ "max_position_embeddings": 262144,
73
+ "mlp_only_layers": [],
74
+ "model_type": "qwen3_next",
75
+ "moe_intermediate_size": 512,
76
+ "norm_topk_prob": true,
77
+ "num_attention_heads": 16,
78
+ "num_experts": 512,
79
+ "num_experts_per_tok": 10,
80
+ "num_hidden_layers": 48,
81
+ "num_key_value_heads": 2,
82
+ "output_router_logits": false,
83
+ "partial_rotary_factor": 0.25,
84
+ "quantization_config": {
85
+ "activation_scheme": "dynamic",
86
+ "modules_to_not_convert": [
87
+ "lm_head",
88
+ "model.layers.0.input_layernorm",
89
+ "model.layers.0.linear_attn.A_log",
90
+ "model.layers.0.linear_attn.conv1d",
91
+ "model.layers.0.linear_attn.dt_bias",
92
+ "model.layers.0.linear_attn.in_proj_ba",
93
+ "model.layers.0.linear_attn.norm",
94
+ "model.layers.0.mlp.gate",
95
+ "model.layers.0.mlp.shared_expert_gate",
96
+ "model.layers.0.post_attention_layernorm",
97
+ "model.layers.1.input_layernorm",
98
+ "model.layers.1.linear_attn.A_log",
99
+ "model.layers.1.linear_attn.conv1d",
100
+ "model.layers.1.linear_attn.dt_bias",
101
+ "model.layers.1.linear_attn.in_proj_ba",
102
+ "model.layers.1.linear_attn.norm",
103
+ "model.layers.1.mlp.gate",
104
+ "model.layers.1.mlp.shared_expert_gate",
105
+ "model.layers.1.post_attention_layernorm",
106
+ "model.layers.2.input_layernorm",
107
+ "model.layers.2.linear_attn.A_log",
108
+ "model.layers.2.linear_attn.conv1d",
109
+ "model.layers.2.linear_attn.dt_bias",
110
+ "model.layers.2.linear_attn.in_proj_ba",
111
+ "model.layers.2.linear_attn.norm",
112
+ "model.layers.2.mlp.gate",
113
+ "model.layers.2.mlp.shared_expert_gate",
114
+ "model.layers.2.post_attention_layernorm",
115
+ "model.layers.3.input_layernorm",
116
+ "model.layers.3.mlp.gate",
117
+ "model.layers.3.mlp.shared_expert_gate",
118
+ "model.layers.3.post_attention_layernorm",
119
+ "model.layers.3.self_attn.k_norm",
120
+ "model.layers.3.self_attn.q_norm",
121
+ "model.layers.4.input_layernorm",
122
+ "model.layers.4.linear_attn.A_log",
123
+ "model.layers.4.linear_attn.conv1d",
124
+ "model.layers.4.linear_attn.dt_bias",
125
+ "model.layers.4.linear_attn.in_proj_ba",
126
+ "model.layers.4.linear_attn.norm",
127
+ "model.layers.4.mlp.gate",
128
+ "model.layers.4.mlp.shared_expert_gate",
129
+ "model.layers.4.post_attention_layernorm",
130
+ "model.layers.5.input_layernorm",
131
+ "model.layers.5.linear_attn.A_log",
132
+ "model.layers.5.linear_attn.conv1d",
133
+ "model.layers.5.linear_attn.dt_bias",
134
+ "model.layers.5.linear_attn.in_proj_ba",
135
+ "model.layers.5.linear_attn.norm",
136
+ "model.layers.5.mlp.gate",
137
+ "model.layers.5.mlp.shared_expert_gate",
138
+ "model.layers.5.post_attention_layernorm",
139
+ "model.layers.6.input_layernorm",
140
+ "model.layers.6.linear_attn.A_log",
141
+ "model.layers.6.linear_attn.conv1d",
142
+ "model.layers.6.linear_attn.dt_bias",
143
+ "model.layers.6.linear_attn.in_proj_ba",
144
+ "model.layers.6.linear_attn.norm",
145
+ "model.layers.6.mlp.gate",
146
+ "model.layers.6.mlp.shared_expert_gate",
147
+ "model.layers.6.post_attention_layernorm",
148
+ "model.layers.7.input_layernorm",
149
+ "model.layers.7.mlp.gate",
150
+ "model.layers.7.mlp.shared_expert_gate",
151
+ "model.layers.7.post_attention_layernorm",
152
+ "model.layers.7.self_attn.k_norm",
153
+ "model.layers.7.self_attn.q_norm",
154
+ "model.layers.8.input_layernorm",
155
+ "model.layers.8.linear_attn.A_log",
156
+ "model.layers.8.linear_attn.conv1d",
157
+ "model.layers.8.linear_attn.dt_bias",
158
+ "model.layers.8.linear_attn.in_proj_ba",
159
+ "model.layers.8.linear_attn.norm",
160
+ "model.layers.8.mlp.gate",
161
+ "model.layers.8.mlp.shared_expert_gate",
162
+ "model.layers.8.post_attention_layernorm",
163
+ "model.layers.9.input_layernorm",
164
+ "model.layers.9.linear_attn.A_log",
165
+ "model.layers.9.linear_attn.conv1d",
166
+ "model.layers.9.linear_attn.dt_bias",
167
+ "model.layers.9.linear_attn.in_proj_ba",
168
+ "model.layers.9.linear_attn.norm",
169
+ "model.layers.9.mlp.gate",
170
+ "model.layers.9.mlp.shared_expert_gate",
171
+ "model.layers.9.post_attention_layernorm",
172
+ "model.layers.10.input_layernorm",
173
+ "model.layers.10.linear_attn.A_log",
174
+ "model.layers.10.linear_attn.conv1d",
175
+ "model.layers.10.linear_attn.dt_bias",
176
+ "model.layers.10.linear_attn.in_proj_ba",
177
+ "model.layers.10.linear_attn.norm",
178
+ "model.layers.10.mlp.gate",
179
+ "model.layers.10.mlp.shared_expert_gate",
180
+ "model.layers.10.post_attention_layernorm",
181
+ "model.layers.11.input_layernorm",
182
+ "model.layers.11.mlp.gate",
183
+ "model.layers.11.mlp.shared_expert_gate",
184
+ "model.layers.11.post_attention_layernorm",
185
+ "model.layers.11.self_attn.k_norm",
186
+ "model.layers.11.self_attn.q_norm",
187
+ "model.layers.12.input_layernorm",
188
+ "model.layers.12.linear_attn.A_log",
189
+ "model.layers.12.linear_attn.conv1d",
190
+ "model.layers.12.linear_attn.dt_bias",
191
+ "model.layers.12.linear_attn.in_proj_ba",
192
+ "model.layers.12.linear_attn.norm",
193
+ "model.layers.12.mlp.gate",
194
+ "model.layers.12.mlp.shared_expert_gate",
195
+ "model.layers.12.post_attention_layernorm",
196
+ "model.layers.13.input_layernorm",
197
+ "model.layers.13.linear_attn.A_log",
198
+ "model.layers.13.linear_attn.conv1d",
199
+ "model.layers.13.linear_attn.dt_bias",
200
+ "model.layers.13.linear_attn.in_proj_ba",
201
+ "model.layers.13.linear_attn.norm",
202
+ "model.layers.13.mlp.gate",
203
+ "model.layers.13.mlp.shared_expert_gate",
204
+ "model.layers.13.post_attention_layernorm",
205
+ "model.layers.14.input_layernorm",
206
+ "model.layers.14.linear_attn.A_log",
207
+ "model.layers.14.linear_attn.conv1d",
208
+ "model.layers.14.linear_attn.dt_bias",
209
+ "model.layers.14.linear_attn.in_proj_ba",
210
+ "model.layers.14.linear_attn.norm",
211
+ "model.layers.14.mlp.gate",
212
+ "model.layers.14.mlp.shared_expert_gate",
213
+ "model.layers.14.post_attention_layernorm",
214
+ "model.layers.15.input_layernorm",
215
+ "model.layers.15.mlp.gate",
216
+ "model.layers.15.mlp.shared_expert_gate",
217
+ "model.layers.15.post_attention_layernorm",
218
+ "model.layers.15.self_attn.k_norm",
219
+ "model.layers.15.self_attn.q_norm",
220
+ "model.layers.16.input_layernorm",
221
+ "model.layers.16.linear_attn.A_log",
222
+ "model.layers.16.linear_attn.conv1d",
223
+ "model.layers.16.linear_attn.dt_bias",
224
+ "model.layers.16.linear_attn.in_proj_ba",
225
+ "model.layers.16.linear_attn.norm",
226
+ "model.layers.16.mlp.gate",
227
+ "model.layers.16.mlp.shared_expert_gate",
228
+ "model.layers.16.post_attention_layernorm",
229
+ "model.layers.17.input_layernorm",
230
+ "model.layers.17.linear_attn.A_log",
231
+ "model.layers.17.linear_attn.conv1d",
232
+ "model.layers.17.linear_attn.dt_bias",
233
+ "model.layers.17.linear_attn.in_proj_ba",
234
+ "model.layers.17.linear_attn.norm",
235
+ "model.layers.17.mlp.gate",
236
+ "model.layers.17.mlp.shared_expert_gate",
237
+ "model.layers.17.post_attention_layernorm",
238
+ "model.layers.18.input_layernorm",
239
+ "model.layers.18.linear_attn.A_log",
240
+ "model.layers.18.linear_attn.conv1d",
241
+ "model.layers.18.linear_attn.dt_bias",
242
+ "model.layers.18.linear_attn.in_proj_ba",
243
+ "model.layers.18.linear_attn.norm",
244
+ "model.layers.18.mlp.gate",
245
+ "model.layers.18.mlp.shared_expert_gate",
246
+ "model.layers.18.post_attention_layernorm",
247
+ "model.layers.19.input_layernorm",
248
+ "model.layers.19.mlp.gate",
249
+ "model.layers.19.mlp.shared_expert_gate",
250
+ "model.layers.19.post_attention_layernorm",
251
+ "model.layers.19.self_attn.k_norm",
252
+ "model.layers.19.self_attn.q_norm",
253
+ "model.layers.20.input_layernorm",
254
+ "model.layers.20.linear_attn.A_log",
255
+ "model.layers.20.linear_attn.conv1d",
256
+ "model.layers.20.linear_attn.dt_bias",
257
+ "model.layers.20.linear_attn.in_proj_ba",
258
+ "model.layers.20.linear_attn.norm",
259
+ "model.layers.20.mlp.gate",
260
+ "model.layers.20.mlp.shared_expert_gate",
261
+ "model.layers.20.post_attention_layernorm",
262
+ "model.layers.21.input_layernorm",
263
+ "model.layers.21.linear_attn.A_log",
264
+ "model.layers.21.linear_attn.conv1d",
265
+ "model.layers.21.linear_attn.dt_bias",
266
+ "model.layers.21.linear_attn.in_proj_ba",
267
+ "model.layers.21.linear_attn.norm",
268
+ "model.layers.21.mlp.gate",
269
+ "model.layers.21.mlp.shared_expert_gate",
270
+ "model.layers.21.post_attention_layernorm",
271
+ "model.layers.22.input_layernorm",
272
+ "model.layers.22.linear_attn.A_log",
273
+ "model.layers.22.linear_attn.conv1d",
274
+ "model.layers.22.linear_attn.dt_bias",
275
+ "model.layers.22.linear_attn.in_proj_ba",
276
+ "model.layers.22.linear_attn.norm",
277
+ "model.layers.22.mlp.gate",
278
+ "model.layers.22.mlp.shared_expert_gate",
279
+ "model.layers.22.post_attention_layernorm",
280
+ "model.layers.23.input_layernorm",
281
+ "model.layers.23.mlp.gate",
282
+ "model.layers.23.mlp.shared_expert_gate",
283
+ "model.layers.23.post_attention_layernorm",
284
+ "model.layers.23.self_attn.k_norm",
285
+ "model.layers.23.self_attn.q_norm",
286
+ "model.layers.24.input_layernorm",
287
+ "model.layers.24.linear_attn.A_log",
288
+ "model.layers.24.linear_attn.conv1d",
289
+ "model.layers.24.linear_attn.dt_bias",
290
+ "model.layers.24.linear_attn.in_proj_ba",
291
+ "model.layers.24.linear_attn.norm",
292
+ "model.layers.24.mlp.gate",
293
+ "model.layers.24.mlp.shared_expert_gate",
294
+ "model.layers.24.post_attention_layernorm",
295
+ "model.layers.25.input_layernorm",
296
+ "model.layers.25.linear_attn.A_log",
297
+ "model.layers.25.linear_attn.conv1d",
298
+ "model.layers.25.linear_attn.dt_bias",
299
+ "model.layers.25.linear_attn.in_proj_ba",
300
+ "model.layers.25.linear_attn.norm",
301
+ "model.layers.25.mlp.gate",
302
+ "model.layers.25.mlp.shared_expert_gate",
303
+ "model.layers.25.post_attention_layernorm",
304
+ "model.layers.26.input_layernorm",
305
+ "model.layers.26.linear_attn.A_log",
306
+ "model.layers.26.linear_attn.conv1d",
307
+ "model.layers.26.linear_attn.dt_bias",
308
+ "model.layers.26.linear_attn.in_proj_ba",
309
+ "model.layers.26.linear_attn.norm",
310
+ "model.layers.26.mlp.gate",
311
+ "model.layers.26.mlp.shared_expert_gate",
312
+ "model.layers.26.post_attention_layernorm",
313
+ "model.layers.27.input_layernorm",
314
+ "model.layers.27.mlp.gate",
315
+ "model.layers.27.mlp.shared_expert_gate",
316
+ "model.layers.27.post_attention_layernorm",
317
+ "model.layers.27.self_attn.k_norm",
318
+ "model.layers.27.self_attn.q_norm",
319
+ "model.layers.28.input_layernorm",
320
+ "model.layers.28.linear_attn.A_log",
321
+ "model.layers.28.linear_attn.conv1d",
322
+ "model.layers.28.linear_attn.dt_bias",
323
+ "model.layers.28.linear_attn.in_proj_ba",
324
+ "model.layers.28.linear_attn.norm",
325
+ "model.layers.28.mlp.gate",
326
+ "model.layers.28.mlp.shared_expert_gate",
327
+ "model.layers.28.post_attention_layernorm",
328
+ "model.layers.29.input_layernorm",
329
+ "model.layers.29.linear_attn.A_log",
330
+ "model.layers.29.linear_attn.conv1d",
331
+ "model.layers.29.linear_attn.dt_bias",
332
+ "model.layers.29.linear_attn.in_proj_ba",
333
+ "model.layers.29.linear_attn.norm",
334
+ "model.layers.29.mlp.gate",
335
+ "model.layers.29.mlp.shared_expert_gate",
336
+ "model.layers.29.post_attention_layernorm",
337
+ "model.layers.30.input_layernorm",
338
+ "model.layers.30.linear_attn.A_log",
339
+ "model.layers.30.linear_attn.conv1d",
340
+ "model.layers.30.linear_attn.dt_bias",
341
+ "model.layers.30.linear_attn.in_proj_ba",
342
+ "model.layers.30.linear_attn.norm",
343
+ "model.layers.30.mlp.gate",
344
+ "model.layers.30.mlp.shared_expert_gate",
345
+ "model.layers.30.post_attention_layernorm",
346
+ "model.layers.31.input_layernorm",
347
+ "model.layers.31.mlp.gate",
348
+ "model.layers.31.mlp.shared_expert_gate",
349
+ "model.layers.31.post_attention_layernorm",
350
+ "model.layers.31.self_attn.k_norm",
351
+ "model.layers.31.self_attn.q_norm",
352
+ "model.layers.32.input_layernorm",
353
+ "model.layers.32.linear_attn.A_log",
354
+ "model.layers.32.linear_attn.conv1d",
355
+ "model.layers.32.linear_attn.dt_bias",
356
+ "model.layers.32.linear_attn.in_proj_ba",
357
+ "model.layers.32.linear_attn.norm",
358
+ "model.layers.32.mlp.gate",
359
+ "model.layers.32.mlp.shared_expert_gate",
360
+ "model.layers.32.post_attention_layernorm",
361
+ "model.layers.33.input_layernorm",
362
+ "model.layers.33.linear_attn.A_log",
363
+ "model.layers.33.linear_attn.conv1d",
364
+ "model.layers.33.linear_attn.dt_bias",
365
+ "model.layers.33.linear_attn.in_proj_ba",
366
+ "model.layers.33.linear_attn.norm",
367
+ "model.layers.33.mlp.gate",
368
+ "model.layers.33.mlp.shared_expert_gate",
369
+ "model.layers.33.post_attention_layernorm",
370
+ "model.layers.34.input_layernorm",
371
+ "model.layers.34.linear_attn.A_log",
372
+ "model.layers.34.linear_attn.conv1d",
373
+ "model.layers.34.linear_attn.dt_bias",
374
+ "model.layers.34.linear_attn.in_proj_ba",
375
+ "model.layers.34.linear_attn.norm",
376
+ "model.layers.34.mlp.gate",
377
+ "model.layers.34.mlp.shared_expert_gate",
378
+ "model.layers.34.post_attention_layernorm",
379
+ "model.layers.35.input_layernorm",
380
+ "model.layers.35.mlp.gate",
381
+ "model.layers.35.mlp.shared_expert_gate",
382
+ "model.layers.35.post_attention_layernorm",
383
+ "model.layers.35.self_attn.k_norm",
384
+ "model.layers.35.self_attn.q_norm",
385
+ "model.layers.36.input_layernorm",
386
+ "model.layers.36.linear_attn.A_log",
387
+ "model.layers.36.linear_attn.conv1d",
388
+ "model.layers.36.linear_attn.dt_bias",
389
+ "model.layers.36.linear_attn.in_proj_ba",
390
+ "model.layers.36.linear_attn.norm",
391
+ "model.layers.36.mlp.gate",
392
+ "model.layers.36.mlp.shared_expert_gate",
393
+ "model.layers.36.post_attention_layernorm",
394
+ "model.layers.37.input_layernorm",
395
+ "model.layers.37.linear_attn.A_log",
396
+ "model.layers.37.linear_attn.conv1d",
397
+ "model.layers.37.linear_attn.dt_bias",
398
+ "model.layers.37.linear_attn.in_proj_ba",
399
+ "model.layers.37.linear_attn.norm",
400
+ "model.layers.37.mlp.gate",
401
+ "model.layers.37.mlp.shared_expert_gate",
402
+ "model.layers.37.post_attention_layernorm",
403
+ "model.layers.38.input_layernorm",
404
+ "model.layers.38.linear_attn.A_log",
405
+ "model.layers.38.linear_attn.conv1d",
406
+ "model.layers.38.linear_attn.dt_bias",
407
+ "model.layers.38.linear_attn.in_proj_ba",
408
+ "model.layers.38.linear_attn.norm",
409
+ "model.layers.38.mlp.gate",
410
+ "model.layers.38.mlp.shared_expert_gate",
411
+ "model.layers.38.post_attention_layernorm",
412
+ "model.layers.39.input_layernorm",
413
+ "model.layers.39.mlp.gate",
414
+ "model.layers.39.mlp.shared_expert_gate",
415
+ "model.layers.39.post_attention_layernorm",
416
+ "model.layers.39.self_attn.k_norm",
417
+ "model.layers.39.self_attn.q_norm",
418
+ "model.layers.40.input_layernorm",
419
+ "model.layers.40.linear_attn.A_log",
420
+ "model.layers.40.linear_attn.conv1d",
421
+ "model.layers.40.linear_attn.dt_bias",
422
+ "model.layers.40.linear_attn.in_proj_ba",
423
+ "model.layers.40.linear_attn.norm",
424
+ "model.layers.40.mlp.gate",
425
+ "model.layers.40.mlp.shared_expert_gate",
426
+ "model.layers.40.post_attention_layernorm",
427
+ "model.layers.41.input_layernorm",
428
+ "model.layers.41.linear_attn.A_log",
429
+ "model.layers.41.linear_attn.conv1d",
430
+ "model.layers.41.linear_attn.dt_bias",
431
+ "model.layers.41.linear_attn.in_proj_ba",
432
+ "model.layers.41.linear_attn.norm",
433
+ "model.layers.41.mlp.gate",
434
+ "model.layers.41.mlp.shared_expert_gate",
435
+ "model.layers.41.post_attention_layernorm",
436
+ "model.layers.42.input_layernorm",
437
+ "model.layers.42.linear_attn.A_log",
438
+ "model.layers.42.linear_attn.conv1d",
439
+ "model.layers.42.linear_attn.dt_bias",
440
+ "model.layers.42.linear_attn.in_proj_ba",
441
+ "model.layers.42.linear_attn.norm",
442
+ "model.layers.42.mlp.gate",
443
+ "model.layers.42.mlp.shared_expert_gate",
444
+ "model.layers.42.post_attention_layernorm",
445
+ "model.layers.43.input_layernorm",
446
+ "model.layers.43.mlp.gate",
447
+ "model.layers.43.mlp.shared_expert_gate",
448
+ "model.layers.43.post_attention_layernorm",
449
+ "model.layers.43.self_attn.k_norm",
450
+ "model.layers.43.self_attn.q_norm",
451
+ "model.layers.44.input_layernorm",
452
+ "model.layers.44.linear_attn.A_log",
453
+ "model.layers.44.linear_attn.conv1d",
454
+ "model.layers.44.linear_attn.dt_bias",
455
+ "model.layers.44.linear_attn.in_proj_ba",
456
+ "model.layers.44.linear_attn.norm",
457
+ "model.layers.44.mlp.gate",
458
+ "model.layers.44.mlp.shared_expert_gate",
459
+ "model.layers.44.post_attention_layernorm",
460
+ "model.layers.45.input_layernorm",
461
+ "model.layers.45.linear_attn.A_log",
462
+ "model.layers.45.linear_attn.conv1d",
463
+ "model.layers.45.linear_attn.dt_bias",
464
+ "model.layers.45.linear_attn.in_proj_ba",
465
+ "model.layers.45.linear_attn.norm",
466
+ "model.layers.45.mlp.gate",
467
+ "model.layers.45.mlp.shared_expert_gate",
468
+ "model.layers.45.post_attention_layernorm",
469
+ "model.layers.46.input_layernorm",
470
+ "model.layers.46.linear_attn.A_log",
471
+ "model.layers.46.linear_attn.conv1d",
472
+ "model.layers.46.linear_attn.dt_bias",
473
+ "model.layers.46.linear_attn.in_proj_ba",
474
+ "model.layers.46.linear_attn.norm",
475
+ "model.layers.46.mlp.gate",
476
+ "model.layers.46.mlp.shared_expert_gate",
477
+ "model.layers.46.post_attention_layernorm",
478
+ "model.layers.47.input_layernorm",
479
+ "model.layers.47.mlp.gate",
480
+ "model.layers.47.mlp.shared_expert_gate",
481
+ "model.layers.47.post_attention_layernorm",
482
+ "model.layers.47.self_attn.k_norm",
483
+ "model.layers.47.self_attn.q_norm",
484
+ "mtp.fc",
485
+ "mtp.layers.0.input_layernorm",
486
+ "mtp.layers.0.mlp.gate",
487
+ "mtp.layers.0.mlp.shared_expert_gate",
488
+ "mtp.layers.0.post_attention_layernorm",
489
+ "mtp.layers.0.self_attn.k_norm",
490
+ "mtp.layers.0.self_attn.q_norm",
491
+ "mtp.norm",
492
+ "mtp.pre_fc_norm_embedding",
493
+ "mtp.pre_fc_norm_hidden"
494
+ ],
495
+ "quant_method": "fp8",
496
+ "weight_block_size": [
497
+ 128,
498
+ 128
499
+ ]
500
+ },
501
+ "rms_norm_eps": 1e-06,
502
+ "rope_scaling": null,
503
+ "rope_theta": 10000000,
504
+ "router_aux_loss_coef": 0.001,
505
+ "shared_expert_intermediate_size": 512,
506
+ "tie_word_embeddings": false,
507
+ "transformers_version": "4.57.3",
508
+ "use_cache": true,
509
+ "use_sliding_window": false,
510
+ "vocab_size": 151936
511
+ }
generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151645,
6
+ 151643
7
+ ],
8
+ "pad_token_id": 151643,
9
+ "temperature": 0.6,
10
+ "top_k": 20,
11
+ "top_p": 0.95,
12
+ "transformers_version": "4.57.3"
13
+ }
model-00001-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc903284a69307e207ea53073afddddabf2872a07b2638700bd641da39c49736
3
+ size 5000622184
model-00002-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ee0b44c2e380392012307d37c800c82d73a1ef5afedc7ca96d843ee67701949
3
+ size 5001035704
model-00003-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdf070a3ac1c5b8b83f9b873b06b7e9097ec31082973950555fd18120b33dce5
3
+ size 5001036184
model-00004-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:beb57480e907174766b31e8ad87892ae69484b514734661537b45bd5e8478866
3
+ size 5001043720
model-00005-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:919c1657593d4991a7cfad1ab70caca08f426360925c74567dc8e567c8837188
3
+ size 5000322432
model-00006-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec946ae149b135613d9f5eac22e1c6fe7933125b055f02d4db600dc6d71e081c
3
+ size 5001045504
model-00007-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44013d20ee41cd1ddfedf0f4bec0986bb21c0ecbeb86e6eae9e2131c1dd969ac
3
+ size 5001046400
model-00008-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07ad3341572acdc603e819b68dcd3a0dcc2199dc6c39f3d5572a018c1c6e2365
3
+ size 5001046616
model-00009-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d70540f14ca5b0aa9c7f5a5922abdfc98f6dc6126d6a2758530257993ec1ba7d
3
+ size 5000323272
model-00010-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0fa33a9b3cb77e746a3609cb6c8df6d0ad1a2b565b69cdb0eab9d35642d3ef7
3
+ size 5001045928
model-00011-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:625b3c1af8f125edd33c3664d5032cc335e6e9d0dd89d727c4f1424d30cbb083
3
+ size 5001047256
model-00012-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1baf1f6c9ec689106bad3af07f3fa3994f3ea6acdaff47b7dcc93b1e39028e13
3
+ size 4999283264
model-00013-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a00032199027eadd8ff38a05f17f7c65f8bc73ef05a07bfa1403e5fbbfb7e34c
3
+ size 5001045192
model-00014-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b58c50e935b865769f806e11086cd2a590cd1a089672010efe67e94c1f9d902b
3
+ size 5000319352
model-00015-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7116d171972ab6827eccdc62d914ba730aa471670cb5d65c0b52dec6a882173e
3
+ size 5001043008
model-00016-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc5ebb680ce3fb4c5b314f9f21723086020ac35a877b48d3fb6152be47952813
3
+ size 4747928624
model-00017-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:846e14111af57c5b9df10d32674a7b7636a5a6edd52465317c7c5c74e7a8cb3b
3
+ size 622329984
model.safetensors.index.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43b06c5a3d732d8ab94775c11b0f7abf0740b5fcf1ff7a33934dcddd777085d1
3
+ size 14225977