Hugging Face's logo

Tesleum
/
0xCoder

Text Generation
Transformers
Safetensors
gpt_oss
Claude
Fable
Antropic
Agent
Ollama
vLLM
mxfp4
quantization
Mixture of Experts
conversational
Eval Results (legacy)
8-bit precision
Model card Files
xet
 Community
stevelionheart commited on
Commit
e64831e
·
verified ·
1 Parent(s): f08f10a

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +20 -873
config.json CHANGED
@@ -4,7 +4,7 @@
4
  ],
5
  "attention_bias": true,
6
  "attention_dropout": 0.0,
7
- "bos_token_id": null,
8
  "dtype": "bfloat16",
9
  "eos_token_id": 200002,
10
  "experts_per_token": 4,
@@ -15,900 +15,47 @@
15
  "initializer_range": 0.02,
16
  "intermediate_size": 2880,
17
  "layer_types": [
18
- "sliding_attention",
19
- "full_attention",
20
- "sliding_attention",
21
- "full_attention",
22
- "sliding_attention",
23
- "full_attention",
24
- "sliding_attention",
25
- "full_attention",
26
- "sliding_attention",
27
- "full_attention",
28
- "sliding_attention",
29
- "full_attention",
30
- "sliding_attention",
31
- "full_attention",
32
- "sliding_attention",
33
- "full_attention",
34
- "sliding_attention",
35
- "full_attention",
36
- "sliding_attention",
37
- "full_attention",
38
- "sliding_attention",
39
- "full_attention",
40
- "sliding_attention",
41
- "full_attention"
42
  ],
43
  "max_position_embeddings": 131072,
44
- "model_name": "Tesleum/Claude-OSS",
45
  "model_type": "gpt_oss",
46
  "num_attention_heads": 64,
47
  "num_experts_per_tok": 4,
48
  "num_hidden_layers": 24,
49
  "num_key_value_heads": 8,
50
  "num_local_experts": 32,
51
- "output_router_logits": false,
52
  "pad_token_id": 199999,
53
  "quantization_config": {
54
  "quant_method": "mxfp4",
55
- "config_groups": {
56
- "group_0": {
57
- "format": "mxfp4-pack-quantized",
58
- "input_activations": {
59
- "actorder": null,
60
- "block_structure": null,
61
- "dynamic": true,
62
- "group_size": 32,
63
- "num_bits": 4,
64
- "observer": null,
65
- "observer_kwargs": {},
66
- "scale_dtype": "torch.uint8",
67
- "strategy": "group",
68
- "symmetric": true,
69
- "type": "float",
70
- "zp_dtype": null
71
- },
72
- "output_activations": null,
73
- "targets": [
74
- "Linear"
75
- ],
76
- "weights": {
77
- "actorder": null,
78
- "block_structure": null,
79
- "dynamic": false,
80
- "group_size": 32,
81
- "num_bits": 4,
82
- "observer": "memoryless_minmax",
83
- "observer_kwargs": {},
84
- "scale_dtype": "torch.uint8",
85
- "strategy": "group",
86
- "symmetric": true,
87
- "type": "float",
88
- "zp_dtype": null
89
- }
90
- }
91
- },
92
- "format": "mxfp4-pack-quantized",
93
- "global_compression_ratio": null,
94
- "ignore": [
95
- "model.layers.0.mlp.router",
96
- "model.layers.0.mlp.experts.0",
97
- "model.layers.0.mlp.experts.1",
98
- "model.layers.0.mlp.experts.2",
99
- "model.layers.0.mlp.experts.3",
100
- "model.layers.0.mlp.experts.4",
101
- "model.layers.0.mlp.experts.5",
102
- "model.layers.0.mlp.experts.6",
103
- "model.layers.0.mlp.experts.7",
104
- "model.layers.0.mlp.experts.8",
105
- "model.layers.0.mlp.experts.9",
106
- "model.layers.0.mlp.experts.10",
107
- "model.layers.0.mlp.experts.11",
108
- "model.layers.0.mlp.experts.12",
109
- "model.layers.0.mlp.experts.13",
110
- "model.layers.0.mlp.experts.14",
111
- "model.layers.0.mlp.experts.15",
112
- "model.layers.0.mlp.experts.16",
113
- "model.layers.0.mlp.experts.17",
114
- "model.layers.0.mlp.experts.18",
115
- "model.layers.0.mlp.experts.19",
116
- "model.layers.0.mlp.experts.20",
117
- "model.layers.0.mlp.experts.21",
118
- "model.layers.0.mlp.experts.22",
119
- "model.layers.0.mlp.experts.23",
120
- "model.layers.0.mlp.experts.24",
121
- "model.layers.0.mlp.experts.25",
122
- "model.layers.0.mlp.experts.26",
123
- "model.layers.0.mlp.experts.27",
124
- "model.layers.0.mlp.experts.28",
125
- "model.layers.0.mlp.experts.29",
126
- "model.layers.0.mlp.experts.30",
127
- "model.layers.0.mlp.experts.31",
128
- "model.layers.1.mlp.router",
129
- "model.layers.1.mlp.experts.0",
130
- "model.layers.1.mlp.experts.1",
131
- "model.layers.1.mlp.experts.2",
132
- "model.layers.1.mlp.experts.3",
133
- "model.layers.1.mlp.experts.4",
134
- "model.layers.1.mlp.experts.5",
135
- "model.layers.1.mlp.experts.6",
136
- "model.layers.1.mlp.experts.7",
137
- "model.layers.1.mlp.experts.8",
138
- "model.layers.1.mlp.experts.9",
139
- "model.layers.1.mlp.experts.10",
140
- "model.layers.1.mlp.experts.11",
141
- "model.layers.1.mlp.experts.12",
142
- "model.layers.1.mlp.experts.13",
143
- "model.layers.1.mlp.experts.14",
144
- "model.layers.1.mlp.experts.15",
145
- "model.layers.1.mlp.experts.16",
146
- "model.layers.1.mlp.experts.17",
147
- "model.layers.1.mlp.experts.18",
148
- "model.layers.1.mlp.experts.19",
149
- "model.layers.1.mlp.experts.20",
150
- "model.layers.1.mlp.experts.21",
151
- "model.layers.1.mlp.experts.22",
152
- "model.layers.1.mlp.experts.23",
153
- "model.layers.1.mlp.experts.24",
154
- "model.layers.1.mlp.experts.25",
155
- "model.layers.1.mlp.experts.26",
156
- "model.layers.1.mlp.experts.27",
157
- "model.layers.1.mlp.experts.28",
158
- "model.layers.1.mlp.experts.29",
159
- "model.layers.1.mlp.experts.30",
160
- "model.layers.1.mlp.experts.31",
161
- "model.layers.2.mlp.router",
162
- "model.layers.2.mlp.experts.0",
163
- "model.layers.2.mlp.experts.1",
164
- "model.layers.2.mlp.experts.2",
165
- "model.layers.2.mlp.experts.3",
166
- "model.layers.2.mlp.experts.4",
167
- "model.layers.2.mlp.experts.5",
168
- "model.layers.2.mlp.experts.6",
169
- "model.layers.2.mlp.experts.7",
170
- "model.layers.2.mlp.experts.8",
171
- "model.layers.2.mlp.experts.9",
172
- "model.layers.2.mlp.experts.10",
173
- "model.layers.2.mlp.experts.11",
174
- "model.layers.2.mlp.experts.12",
175
- "model.layers.2.mlp.experts.13",
176
- "model.layers.2.mlp.experts.14",
177
- "model.layers.2.mlp.experts.15",
178
- "model.layers.2.mlp.experts.16",
179
- "model.layers.2.mlp.experts.17",
180
- "model.layers.2.mlp.experts.18",
181
- "model.layers.2.mlp.experts.19",
182
- "model.layers.2.mlp.experts.20",
183
- "model.layers.2.mlp.experts.21",
184
- "model.layers.2.mlp.experts.22",
185
- "model.layers.2.mlp.experts.23",
186
- "model.layers.2.mlp.experts.24",
187
- "model.layers.2.mlp.experts.25",
188
- "model.layers.2.mlp.experts.26",
189
- "model.layers.2.mlp.experts.27",
190
- "model.layers.2.mlp.experts.28",
191
- "model.layers.2.mlp.experts.29",
192
- "model.layers.2.mlp.experts.30",
193
- "model.layers.2.mlp.experts.31",
194
- "model.layers.3.mlp.router",
195
- "model.layers.3.mlp.experts.0",
196
- "model.layers.3.mlp.experts.1",
197
- "model.layers.3.mlp.experts.2",
198
- "model.layers.3.mlp.experts.3",
199
- "model.layers.3.mlp.experts.4",
200
- "model.layers.3.mlp.experts.5",
201
- "model.layers.3.mlp.experts.6",
202
- "model.layers.3.mlp.experts.7",
203
- "model.layers.3.mlp.experts.8",
204
- "model.layers.3.mlp.experts.9",
205
- "model.layers.3.mlp.experts.10",
206
- "model.layers.3.mlp.experts.11",
207
- "model.layers.3.mlp.experts.12",
208
- "model.layers.3.mlp.experts.13",
209
- "model.layers.3.mlp.experts.14",
210
- "model.layers.3.mlp.experts.15",
211
- "model.layers.3.mlp.experts.16",
212
- "model.layers.3.mlp.experts.17",
213
- "model.layers.3.mlp.experts.18",
214
- "model.layers.3.mlp.experts.19",
215
- "model.layers.3.mlp.experts.20",
216
- "model.layers.3.mlp.experts.21",
217
- "model.layers.3.mlp.experts.22",
218
- "model.layers.3.mlp.experts.23",
219
- "model.layers.3.mlp.experts.24",
220
- "model.layers.3.mlp.experts.25",
221
- "model.layers.3.mlp.experts.26",
222
- "model.layers.3.mlp.experts.27",
223
- "model.layers.3.mlp.experts.28",
224
- "model.layers.3.mlp.experts.29",
225
- "model.layers.3.mlp.experts.30",
226
- "model.layers.3.mlp.experts.31",
227
- "model.layers.4.mlp.router",
228
- "model.layers.4.mlp.experts.0",
229
- "model.layers.4.mlp.experts.1",
230
- "model.layers.4.mlp.experts.2",
231
- "model.layers.4.mlp.experts.3",
232
- "model.layers.4.mlp.experts.4",
233
- "model.layers.4.mlp.experts.5",
234
- "model.layers.4.mlp.experts.6",
235
- "model.layers.4.mlp.experts.7",
236
- "model.layers.4.mlp.experts.8",
237
- "model.layers.4.mlp.experts.9",
238
- "model.layers.4.mlp.experts.10",
239
- "model.layers.4.mlp.experts.11",
240
- "model.layers.4.mlp.experts.12",
241
- "model.layers.4.mlp.experts.13",
242
- "model.layers.4.mlp.experts.14",
243
- "model.layers.4.mlp.experts.15",
244
- "model.layers.4.mlp.experts.16",
245
- "model.layers.4.mlp.experts.17",
246
- "model.layers.4.mlp.experts.18",
247
- "model.layers.4.mlp.experts.19",
248
- "model.layers.4.mlp.experts.20",
249
- "model.layers.4.mlp.experts.21",
250
- "model.layers.4.mlp.experts.22",
251
- "model.layers.4.mlp.experts.23",
252
- "model.layers.4.mlp.experts.24",
253
- "model.layers.4.mlp.experts.25",
254
- "model.layers.4.mlp.experts.26",
255
- "model.layers.4.mlp.experts.27",
256
- "model.layers.4.mlp.experts.28",
257
- "model.layers.4.mlp.experts.29",
258
- "model.layers.4.mlp.experts.30",
259
- "model.layers.4.mlp.experts.31",
260
- "model.layers.5.mlp.router",
261
- "model.layers.5.mlp.experts.0",
262
- "model.layers.5.mlp.experts.1",
263
- "model.layers.5.mlp.experts.2",
264
- "model.layers.5.mlp.experts.3",
265
- "model.layers.5.mlp.experts.4",
266
- "model.layers.5.mlp.experts.5",
267
- "model.layers.5.mlp.experts.6",
268
- "model.layers.5.mlp.experts.7",
269
- "model.layers.5.mlp.experts.8",
270
- "model.layers.5.mlp.experts.9",
271
- "model.layers.5.mlp.experts.10",
272
- "model.layers.5.mlp.experts.11",
273
- "model.layers.5.mlp.experts.12",
274
- "model.layers.5.mlp.experts.13",
275
- "model.layers.5.mlp.experts.14",
276
- "model.layers.5.mlp.experts.15",
277
- "model.layers.5.mlp.experts.16",
278
- "model.layers.5.mlp.experts.17",
279
- "model.layers.5.mlp.experts.18",
280
- "model.layers.5.mlp.experts.19",
281
- "model.layers.5.mlp.experts.20",
282
- "model.layers.5.mlp.experts.21",
283
- "model.layers.5.mlp.experts.22",
284
- "model.layers.5.mlp.experts.23",
285
- "model.layers.5.mlp.experts.24",
286
- "model.layers.5.mlp.experts.25",
287
- "model.layers.5.mlp.experts.26",
288
- "model.layers.5.mlp.experts.27",
289
- "model.layers.5.mlp.experts.28",
290
- "model.layers.5.mlp.experts.29",
291
- "model.layers.5.mlp.experts.30",
292
- "model.layers.5.mlp.experts.31",
293
- "model.layers.6.mlp.router",
294
- "model.layers.6.mlp.experts.0",
295
- "model.layers.6.mlp.experts.1",
296
- "model.layers.6.mlp.experts.2",
297
- "model.layers.6.mlp.experts.3",
298
- "model.layers.6.mlp.experts.4",
299
- "model.layers.6.mlp.experts.5",
300
- "model.layers.6.mlp.experts.6",
301
- "model.layers.6.mlp.experts.7",
302
- "model.layers.6.mlp.experts.8",
303
- "model.layers.6.mlp.experts.9",
304
- "model.layers.6.mlp.experts.10",
305
- "model.layers.6.mlp.experts.11",
306
- "model.layers.6.mlp.experts.12",
307
- "model.layers.6.mlp.experts.13",
308
- "model.layers.6.mlp.experts.14",
309
- "model.layers.6.mlp.experts.15",
310
- "model.layers.6.mlp.experts.16",
311
- "model.layers.6.mlp.experts.17",
312
- "model.layers.6.mlp.experts.18",
313
- "model.layers.6.mlp.experts.19",
314
- "model.layers.6.mlp.experts.20",
315
- "model.layers.6.mlp.experts.21",
316
- "model.layers.6.mlp.experts.22",
317
- "model.layers.6.mlp.experts.23",
318
- "model.layers.6.mlp.experts.24",
319
- "model.layers.6.mlp.experts.25",
320
- "model.layers.6.mlp.experts.26",
321
- "model.layers.6.mlp.experts.27",
322
- "model.layers.6.mlp.experts.28",
323
- "model.layers.6.mlp.experts.29",
324
- "model.layers.6.mlp.experts.30",
325
- "model.layers.6.mlp.experts.31",
326
- "model.layers.7.mlp.router",
327
- "model.layers.7.mlp.experts.0",
328
- "model.layers.7.mlp.experts.1",
329
- "model.layers.7.mlp.experts.2",
330
- "model.layers.7.mlp.experts.3",
331
- "model.layers.7.mlp.experts.4",
332
- "model.layers.7.mlp.experts.5",
333
- "model.layers.7.mlp.experts.6",
334
- "model.layers.7.mlp.experts.7",
335
- "model.layers.7.mlp.experts.8",
336
- "model.layers.7.mlp.experts.9",
337
- "model.layers.7.mlp.experts.10",
338
- "model.layers.7.mlp.experts.11",
339
- "model.layers.7.mlp.experts.12",
340
- "model.layers.7.mlp.experts.13",
341
- "model.layers.7.mlp.experts.14",
342
- "model.layers.7.mlp.experts.15",
343
- "model.layers.7.mlp.experts.16",
344
- "model.layers.7.mlp.experts.17",
345
- "model.layers.7.mlp.experts.18",
346
- "model.layers.7.mlp.experts.19",
347
- "model.layers.7.mlp.experts.20",
348
- "model.layers.7.mlp.experts.21",
349
- "model.layers.7.mlp.experts.22",
350
- "model.layers.7.mlp.experts.23",
351
- "model.layers.7.mlp.experts.24",
352
- "model.layers.7.mlp.experts.25",
353
- "model.layers.7.mlp.experts.26",
354
- "model.layers.7.mlp.experts.27",
355
- "model.layers.7.mlp.experts.28",
356
- "model.layers.7.mlp.experts.29",
357
- "model.layers.7.mlp.experts.30",
358
- "model.layers.7.mlp.experts.31",
359
- "model.layers.8.mlp.router",
360
- "model.layers.8.mlp.experts.0",
361
- "model.layers.8.mlp.experts.1",
362
- "model.layers.8.mlp.experts.2",
363
- "model.layers.8.mlp.experts.3",
364
- "model.layers.8.mlp.experts.4",
365
- "model.layers.8.mlp.experts.5",
366
- "model.layers.8.mlp.experts.6",
367
- "model.layers.8.mlp.experts.7",
368
- "model.layers.8.mlp.experts.8",
369
- "model.layers.8.mlp.experts.9",
370
- "model.layers.8.mlp.experts.10",
371
- "model.layers.8.mlp.experts.11",
372
- "model.layers.8.mlp.experts.12",
373
- "model.layers.8.mlp.experts.13",
374
- "model.layers.8.mlp.experts.14",
375
- "model.layers.8.mlp.experts.15",
376
- "model.layers.8.mlp.experts.16",
377
- "model.layers.8.mlp.experts.17",
378
- "model.layers.8.mlp.experts.18",
379
- "model.layers.8.mlp.experts.19",
380
- "model.layers.8.mlp.experts.20",
381
- "model.layers.8.mlp.experts.21",
382
- "model.layers.8.mlp.experts.22",
383
- "model.layers.8.mlp.experts.23",
384
- "model.layers.8.mlp.experts.24",
385
- "model.layers.8.mlp.experts.25",
386
- "model.layers.8.mlp.experts.26",
387
- "model.layers.8.mlp.experts.27",
388
- "model.layers.8.mlp.experts.28",
389
- "model.layers.8.mlp.experts.29",
390
- "model.layers.8.mlp.experts.30",
391
- "model.layers.8.mlp.experts.31",
392
- "model.layers.9.mlp.router",
393
- "model.layers.9.mlp.experts.0",
394
- "model.layers.9.mlp.experts.1",
395
- "model.layers.9.mlp.experts.2",
396
- "model.layers.9.mlp.experts.3",
397
- "model.layers.9.mlp.experts.4",
398
- "model.layers.9.mlp.experts.5",
399
- "model.layers.9.mlp.experts.6",
400
- "model.layers.9.mlp.experts.7",
401
- "model.layers.9.mlp.experts.8",
402
- "model.layers.9.mlp.experts.9",
403
- "model.layers.9.mlp.experts.10",
404
- "model.layers.9.mlp.experts.11",
405
- "model.layers.9.mlp.experts.12",
406
- "model.layers.9.mlp.experts.13",
407
- "model.layers.9.mlp.experts.14",
408
- "model.layers.9.mlp.experts.15",
409
- "model.layers.9.mlp.experts.16",
410
- "model.layers.9.mlp.experts.17",
411
- "model.layers.9.mlp.experts.18",
412
- "model.layers.9.mlp.experts.19",
413
- "model.layers.9.mlp.experts.20",
414
- "model.layers.9.mlp.experts.21",
415
- "model.layers.9.mlp.experts.22",
416
- "model.layers.9.mlp.experts.23",
417
- "model.layers.9.mlp.experts.24",
418
- "model.layers.9.mlp.experts.25",
419
- "model.layers.9.mlp.experts.26",
420
- "model.layers.9.mlp.experts.27",
421
- "model.layers.9.mlp.experts.28",
422
- "model.layers.9.mlp.experts.29",
423
- "model.layers.9.mlp.experts.30",
424
- "model.layers.9.mlp.experts.31",
425
- "model.layers.10.mlp.router",
426
- "model.layers.10.mlp.experts.0",
427
- "model.layers.10.mlp.experts.1",
428
- "model.layers.10.mlp.experts.2",
429
- "model.layers.10.mlp.experts.3",
430
- "model.layers.10.mlp.experts.4",
431
- "model.layers.10.mlp.experts.5",
432
- "model.layers.10.mlp.experts.6",
433
- "model.layers.10.mlp.experts.7",
434
- "model.layers.10.mlp.experts.8",
435
- "model.layers.10.mlp.experts.9",
436
- "model.layers.10.mlp.experts.10",
437
- "model.layers.10.mlp.experts.11",
438
- "model.layers.10.mlp.experts.12",
439
- "model.layers.10.mlp.experts.13",
440
- "model.layers.10.mlp.experts.14",
441
- "model.layers.10.mlp.experts.15",
442
- "model.layers.10.mlp.experts.16",
443
- "model.layers.10.mlp.experts.17",
444
- "model.layers.10.mlp.experts.18",
445
- "model.layers.10.mlp.experts.19",
446
- "model.layers.10.mlp.experts.20",
447
- "model.layers.10.mlp.experts.21",
448
- "model.layers.10.mlp.experts.22",
449
- "model.layers.10.mlp.experts.23",
450
- "model.layers.10.mlp.experts.24",
451
- "model.layers.10.mlp.experts.25",
452
- "model.layers.10.mlp.experts.26",
453
- "model.layers.10.mlp.experts.27",
454
- "model.layers.10.mlp.experts.28",
455
- "model.layers.10.mlp.experts.29",
456
- "model.layers.10.mlp.experts.30",
457
- "model.layers.10.mlp.experts.31",
458
- "model.layers.11.mlp.router",
459
- "model.layers.11.mlp.experts.0",
460
- "model.layers.11.mlp.experts.1",
461
- "model.layers.11.mlp.experts.2",
462
- "model.layers.11.mlp.experts.3",
463
- "model.layers.11.mlp.experts.4",
464
- "model.layers.11.mlp.experts.5",
465
- "model.layers.11.mlp.experts.6",
466
- "model.layers.11.mlp.experts.7",
467
- "model.layers.11.mlp.experts.8",
468
- "model.layers.11.mlp.experts.9",
469
- "model.layers.11.mlp.experts.10",
470
- "model.layers.11.mlp.experts.11",
471
- "model.layers.11.mlp.experts.12",
472
- "model.layers.11.mlp.experts.13",
473
- "model.layers.11.mlp.experts.14",
474
- "model.layers.11.mlp.experts.15",
475
- "model.layers.11.mlp.experts.16",
476
- "model.layers.11.mlp.experts.17",
477
- "model.layers.11.mlp.experts.18",
478
- "model.layers.11.mlp.experts.19",
479
- "model.layers.11.mlp.experts.20",
480
- "model.layers.11.mlp.experts.21",
481
- "model.layers.11.mlp.experts.22",
482
- "model.layers.11.mlp.experts.23",
483
- "model.layers.11.mlp.experts.24",
484
- "model.layers.11.mlp.experts.25",
485
- "model.layers.11.mlp.experts.26",
486
- "model.layers.11.mlp.experts.27",
487
- "model.layers.11.mlp.experts.28",
488
- "model.layers.11.mlp.experts.29",
489
- "model.layers.11.mlp.experts.30",
490
- "model.layers.11.mlp.experts.31",
491
- "model.layers.12.mlp.router",
492
- "model.layers.12.mlp.experts.0",
493
- "model.layers.12.mlp.experts.1",
494
- "model.layers.12.mlp.experts.2",
495
- "model.layers.12.mlp.experts.3",
496
- "model.layers.12.mlp.experts.4",
497
- "model.layers.12.mlp.experts.5",
498
- "model.layers.12.mlp.experts.6",
499
- "model.layers.12.mlp.experts.7",
500
- "model.layers.12.mlp.experts.8",
501
- "model.layers.12.mlp.experts.9",
502
- "model.layers.12.mlp.experts.10",
503
- "model.layers.12.mlp.experts.11",
504
- "model.layers.12.mlp.experts.12",
505
- "model.layers.12.mlp.experts.13",
506
- "model.layers.12.mlp.experts.14",
507
- "model.layers.12.mlp.experts.15",
508
- "model.layers.12.mlp.experts.16",
509
- "model.layers.12.mlp.experts.17",
510
- "model.layers.12.mlp.experts.18",
511
- "model.layers.12.mlp.experts.19",
512
- "model.layers.12.mlp.experts.20",
513
- "model.layers.12.mlp.experts.21",
514
- "model.layers.12.mlp.experts.22",
515
- "model.layers.12.mlp.experts.23",
516
- "model.layers.12.mlp.experts.24",
517
- "model.layers.12.mlp.experts.25",
518
- "model.layers.12.mlp.experts.26",
519
- "model.layers.12.mlp.experts.27",
520
- "model.layers.12.mlp.experts.28",
521
- "model.layers.12.mlp.experts.29",
522
- "model.layers.12.mlp.experts.30",
523
- "model.layers.12.mlp.experts.31",
524
- "model.layers.13.mlp.router",
525
- "model.layers.13.mlp.experts.0",
526
- "model.layers.13.mlp.experts.1",
527
- "model.layers.13.mlp.experts.2",
528
- "model.layers.13.mlp.experts.3",
529
- "model.layers.13.mlp.experts.4",
530
- "model.layers.13.mlp.experts.5",
531
- "model.layers.13.mlp.experts.6",
532
- "model.layers.13.mlp.experts.7",
533
- "model.layers.13.mlp.experts.8",
534
- "model.layers.13.mlp.experts.9",
535
- "model.layers.13.mlp.experts.10",
536
- "model.layers.13.mlp.experts.11",
537
- "model.layers.13.mlp.experts.12",
538
- "model.layers.13.mlp.experts.13",
539
- "model.layers.13.mlp.experts.14",
540
- "model.layers.13.mlp.experts.15",
541
- "model.layers.13.mlp.experts.16",
542
- "model.layers.13.mlp.experts.17",
543
- "model.layers.13.mlp.experts.18",
544
- "model.layers.13.mlp.experts.19",
545
- "model.layers.13.mlp.experts.20",
546
- "model.layers.13.mlp.experts.21",
547
- "model.layers.13.mlp.experts.22",
548
- "model.layers.13.mlp.experts.23",
549
- "model.layers.13.mlp.experts.24",
550
- "model.layers.13.mlp.experts.25",
551
- "model.layers.13.mlp.experts.26",
552
- "model.layers.13.mlp.experts.27",
553
- "model.layers.13.mlp.experts.28",
554
- "model.layers.13.mlp.experts.29",
555
- "model.layers.13.mlp.experts.30",
556
- "model.layers.13.mlp.experts.31",
557
- "model.layers.14.mlp.router",
558
- "model.layers.14.mlp.experts.0",
559
- "model.layers.14.mlp.experts.1",
560
- "model.layers.14.mlp.experts.2",
561
- "model.layers.14.mlp.experts.3",
562
- "model.layers.14.mlp.experts.4",
563
- "model.layers.14.mlp.experts.5",
564
- "model.layers.14.mlp.experts.6",
565
- "model.layers.14.mlp.experts.7",
566
- "model.layers.14.mlp.experts.8",
567
- "model.layers.14.mlp.experts.9",
568
- "model.layers.14.mlp.experts.10",
569
- "model.layers.14.mlp.experts.11",
570
- "model.layers.14.mlp.experts.12",
571
- "model.layers.14.mlp.experts.13",
572
- "model.layers.14.mlp.experts.14",
573
- "model.layers.14.mlp.experts.15",
574
- "model.layers.14.mlp.experts.16",
575
- "model.layers.14.mlp.experts.17",
576
- "model.layers.14.mlp.experts.18",
577
- "model.layers.14.mlp.experts.19",
578
- "model.layers.14.mlp.experts.20",
579
- "model.layers.14.mlp.experts.21",
580
- "model.layers.14.mlp.experts.22",
581
- "model.layers.14.mlp.experts.23",
582
- "model.layers.14.mlp.experts.24",
583
- "model.layers.14.mlp.experts.25",
584
- "model.layers.14.mlp.experts.26",
585
- "model.layers.14.mlp.experts.27",
586
- "model.layers.14.mlp.experts.28",
587
- "model.layers.14.mlp.experts.29",
588
- "model.layers.14.mlp.experts.30",
589
- "model.layers.14.mlp.experts.31",
590
- "model.layers.15.mlp.router",
591
- "model.layers.15.mlp.experts.0",
592
- "model.layers.15.mlp.experts.1",
593
- "model.layers.15.mlp.experts.2",
594
- "model.layers.15.mlp.experts.3",
595
- "model.layers.15.mlp.experts.4",
596
- "model.layers.15.mlp.experts.5",
597
- "model.layers.15.mlp.experts.6",
598
- "model.layers.15.mlp.experts.7",
599
- "model.layers.15.mlp.experts.8",
600
- "model.layers.15.mlp.experts.9",
601
- "model.layers.15.mlp.experts.10",
602
- "model.layers.15.mlp.experts.11",
603
- "model.layers.15.mlp.experts.12",
604
- "model.layers.15.mlp.experts.13",
605
- "model.layers.15.mlp.experts.14",
606
- "model.layers.15.mlp.experts.15",
607
- "model.layers.15.mlp.experts.16",
608
- "model.layers.15.mlp.experts.17",
609
- "model.layers.15.mlp.experts.18",
610
- "model.layers.15.mlp.experts.19",
611
- "model.layers.15.mlp.experts.20",
612
- "model.layers.15.mlp.experts.21",
613
- "model.layers.15.mlp.experts.22",
614
- "model.layers.15.mlp.experts.23",
615
- "model.layers.15.mlp.experts.24",
616
- "model.layers.15.mlp.experts.25",
617
- "model.layers.15.mlp.experts.26",
618
- "model.layers.15.mlp.experts.27",
619
- "model.layers.15.mlp.experts.28",
620
- "model.layers.15.mlp.experts.29",
621
- "model.layers.15.mlp.experts.30",
622
- "model.layers.15.mlp.experts.31",
623
- "model.layers.16.mlp.router",
624
- "model.layers.16.mlp.experts.0",
625
- "model.layers.16.mlp.experts.1",
626
- "model.layers.16.mlp.experts.2",
627
- "model.layers.16.mlp.experts.3",
628
- "model.layers.16.mlp.experts.4",
629
- "model.layers.16.mlp.experts.5",
630
- "model.layers.16.mlp.experts.6",
631
- "model.layers.16.mlp.experts.7",
632
- "model.layers.16.mlp.experts.8",
633
- "model.layers.16.mlp.experts.9",
634
- "model.layers.16.mlp.experts.10",
635
- "model.layers.16.mlp.experts.11",
636
- "model.layers.16.mlp.experts.12",
637
- "model.layers.16.mlp.experts.13",
638
- "model.layers.16.mlp.experts.14",
639
- "model.layers.16.mlp.experts.15",
640
- "model.layers.16.mlp.experts.16",
641
- "model.layers.16.mlp.experts.17",
642
- "model.layers.16.mlp.experts.18",
643
- "model.layers.16.mlp.experts.19",
644
- "model.layers.16.mlp.experts.20",
645
- "model.layers.16.mlp.experts.21",
646
- "model.layers.16.mlp.experts.22",
647
- "model.layers.16.mlp.experts.23",
648
- "model.layers.16.mlp.experts.24",
649
- "model.layers.16.mlp.experts.25",
650
- "model.layers.16.mlp.experts.26",
651
- "model.layers.16.mlp.experts.27",
652
- "model.layers.16.mlp.experts.28",
653
- "model.layers.16.mlp.experts.29",
654
- "model.layers.16.mlp.experts.30",
655
- "model.layers.16.mlp.experts.31",
656
- "model.layers.17.mlp.router",
657
- "model.layers.17.mlp.experts.0",
658
- "model.layers.17.mlp.experts.1",
659
- "model.layers.17.mlp.experts.2",
660
- "model.layers.17.mlp.experts.3",
661
- "model.layers.17.mlp.experts.4",
662
- "model.layers.17.mlp.experts.5",
663
- "model.layers.17.mlp.experts.6",
664
- "model.layers.17.mlp.experts.7",
665
- "model.layers.17.mlp.experts.8",
666
- "model.layers.17.mlp.experts.9",
667
- "model.layers.17.mlp.experts.10",
668
- "model.layers.17.mlp.experts.11",
669
- "model.layers.17.mlp.experts.12",
670
- "model.layers.17.mlp.experts.13",
671
- "model.layers.17.mlp.experts.14",
672
- "model.layers.17.mlp.experts.15",
673
- "model.layers.17.mlp.experts.16",
674
- "model.layers.17.mlp.experts.17",
675
- "model.layers.17.mlp.experts.18",
676
- "model.layers.17.mlp.experts.19",
677
- "model.layers.17.mlp.experts.20",
678
- "model.layers.17.mlp.experts.21",
679
- "model.layers.17.mlp.experts.22",
680
- "model.layers.17.mlp.experts.23",
681
- "model.layers.17.mlp.experts.24",
682
- "model.layers.17.mlp.experts.25",
683
- "model.layers.17.mlp.experts.26",
684
- "model.layers.17.mlp.experts.27",
685
- "model.layers.17.mlp.experts.28",
686
- "model.layers.17.mlp.experts.29",
687
- "model.layers.17.mlp.experts.30",
688
- "model.layers.17.mlp.experts.31",
689
- "model.layers.18.mlp.router",
690
- "model.layers.18.mlp.experts.0",
691
- "model.layers.18.mlp.experts.1",
692
- "model.layers.18.mlp.experts.2",
693
- "model.layers.18.mlp.experts.3",
694
- "model.layers.18.mlp.experts.4",
695
- "model.layers.18.mlp.experts.5",
696
- "model.layers.18.mlp.experts.6",
697
- "model.layers.18.mlp.experts.7",
698
- "model.layers.18.mlp.experts.8",
699
- "model.layers.18.mlp.experts.9",
700
- "model.layers.18.mlp.experts.10",
701
- "model.layers.18.mlp.experts.11",
702
- "model.layers.18.mlp.experts.12",
703
- "model.layers.18.mlp.experts.13",
704
- "model.layers.18.mlp.experts.14",
705
- "model.layers.18.mlp.experts.15",
706
- "model.layers.18.mlp.experts.16",
707
- "model.layers.18.mlp.experts.17",
708
- "model.layers.18.mlp.experts.18",
709
- "model.layers.18.mlp.experts.19",
710
- "model.layers.18.mlp.experts.20",
711
- "model.layers.18.mlp.experts.21",
712
- "model.layers.18.mlp.experts.22",
713
- "model.layers.18.mlp.experts.23",
714
- "model.layers.18.mlp.experts.24",
715
- "model.layers.18.mlp.experts.25",
716
- "model.layers.18.mlp.experts.26",
717
- "model.layers.18.mlp.experts.27",
718
- "model.layers.18.mlp.experts.28",
719
- "model.layers.18.mlp.experts.29",
720
- "model.layers.18.mlp.experts.30",
721
- "model.layers.18.mlp.experts.31",
722
- "model.layers.19.mlp.router",
723
- "model.layers.19.mlp.experts.0",
724
- "model.layers.19.mlp.experts.1",
725
- "model.layers.19.mlp.experts.2",
726
- "model.layers.19.mlp.experts.3",
727
- "model.layers.19.mlp.experts.4",
728
- "model.layers.19.mlp.experts.5",
729
- "model.layers.19.mlp.experts.6",
730
- "model.layers.19.mlp.experts.7",
731
- "model.layers.19.mlp.experts.8",
732
- "model.layers.19.mlp.experts.9",
733
- "model.layers.19.mlp.experts.10",
734
- "model.layers.19.mlp.experts.11",
735
- "model.layers.19.mlp.experts.12",
736
- "model.layers.19.mlp.experts.13",
737
- "model.layers.19.mlp.experts.14",
738
- "model.layers.19.mlp.experts.15",
739
- "model.layers.19.mlp.experts.16",
740
- "model.layers.19.mlp.experts.17",
741
- "model.layers.19.mlp.experts.18",
742
- "model.layers.19.mlp.experts.19",
743
- "model.layers.19.mlp.experts.20",
744
- "model.layers.19.mlp.experts.21",
745
- "model.layers.19.mlp.experts.22",
746
- "model.layers.19.mlp.experts.23",
747
- "model.layers.19.mlp.experts.24",
748
- "model.layers.19.mlp.experts.25",
749
- "model.layers.19.mlp.experts.26",
750
- "model.layers.19.mlp.experts.27",
751
- "model.layers.19.mlp.experts.28",
752
- "model.layers.19.mlp.experts.29",
753
- "model.layers.19.mlp.experts.30",
754
- "model.layers.19.mlp.experts.31",
755
- "model.layers.20.mlp.router",
756
- "model.layers.20.mlp.experts.0",
757
- "model.layers.20.mlp.experts.1",
758
- "model.layers.20.mlp.experts.2",
759
- "model.layers.20.mlp.experts.3",
760
- "model.layers.20.mlp.experts.4",
761
- "model.layers.20.mlp.experts.5",
762
- "model.layers.20.mlp.experts.6",
763
- "model.layers.20.mlp.experts.7",
764
- "model.layers.20.mlp.experts.8",
765
- "model.layers.20.mlp.experts.9",
766
- "model.layers.20.mlp.experts.10",
767
- "model.layers.20.mlp.experts.11",
768
- "model.layers.20.mlp.experts.12",
769
- "model.layers.20.mlp.experts.13",
770
- "model.layers.20.mlp.experts.14",
771
- "model.layers.20.mlp.experts.15",
772
- "model.layers.20.mlp.experts.16",
773
- "model.layers.20.mlp.experts.17",
774
- "model.layers.20.mlp.experts.18",
775
- "model.layers.20.mlp.experts.19",
776
- "model.layers.20.mlp.experts.20",
777
- "model.layers.20.mlp.experts.21",
778
- "model.layers.20.mlp.experts.22",
779
- "model.layers.20.mlp.experts.23",
780
- "model.layers.20.mlp.experts.24",
781
- "model.layers.20.mlp.experts.25",
782
- "model.layers.20.mlp.experts.26",
783
- "model.layers.20.mlp.experts.27",
784
- "model.layers.20.mlp.experts.28",
785
- "model.layers.20.mlp.experts.29",
786
- "model.layers.20.mlp.experts.30",
787
- "model.layers.20.mlp.experts.31",
788
- "model.layers.21.mlp.router",
789
- "model.layers.21.mlp.experts.0",
790
- "model.layers.21.mlp.experts.1",
791
- "model.layers.21.mlp.experts.2",
792
- "model.layers.21.mlp.experts.3",
793
- "model.layers.21.mlp.experts.4",
794
- "model.layers.21.mlp.experts.5",
795
- "model.layers.21.mlp.experts.6",
796
- "model.layers.21.mlp.experts.7",
797
- "model.layers.21.mlp.experts.8",
798
- "model.layers.21.mlp.experts.9",
799
- "model.layers.21.mlp.experts.10",
800
- "model.layers.21.mlp.experts.11",
801
- "model.layers.21.mlp.experts.12",
802
- "model.layers.21.mlp.experts.13",
803
- "model.layers.21.mlp.experts.14",
804
- "model.layers.21.mlp.experts.15",
805
- "model.layers.21.mlp.experts.16",
806
- "model.layers.21.mlp.experts.17",
807
- "model.layers.21.mlp.experts.18",
808
- "model.layers.21.mlp.experts.19",
809
- "model.layers.21.mlp.experts.20",
810
- "model.layers.21.mlp.experts.21",
811
- "model.layers.21.mlp.experts.22",
812
- "model.layers.21.mlp.experts.23",
813
- "model.layers.21.mlp.experts.24",
814
- "model.layers.21.mlp.experts.25",
815
- "model.layers.21.mlp.experts.26",
816
- "model.layers.21.mlp.experts.27",
817
- "model.layers.21.mlp.experts.28",
818
- "model.layers.21.mlp.experts.29",
819
- "model.layers.21.mlp.experts.30",
820
- "model.layers.21.mlp.experts.31",
821
- "model.layers.22.mlp.router",
822
- "model.layers.22.mlp.experts.0",
823
- "model.layers.22.mlp.experts.1",
824
- "model.layers.22.mlp.experts.2",
825
- "model.layers.22.mlp.experts.3",
826
- "model.layers.22.mlp.experts.4",
827
- "model.layers.22.mlp.experts.5",
828
- "model.layers.22.mlp.experts.6",
829
- "model.layers.22.mlp.experts.7",
830
- "model.layers.22.mlp.experts.8",
831
- "model.layers.22.mlp.experts.9",
832
- "model.layers.22.mlp.experts.10",
833
- "model.layers.22.mlp.experts.11",
834
- "model.layers.22.mlp.experts.12",
835
- "model.layers.22.mlp.experts.13",
836
- "model.layers.22.mlp.experts.14",
837
- "model.layers.22.mlp.experts.15",
838
- "model.layers.22.mlp.experts.16",
839
- "model.layers.22.mlp.experts.17",
840
- "model.layers.22.mlp.experts.18",
841
- "model.layers.22.mlp.experts.19",
842
- "model.layers.22.mlp.experts.20",
843
- "model.layers.22.mlp.experts.21",
844
- "model.layers.22.mlp.experts.22",
845
- "model.layers.22.mlp.experts.23",
846
- "model.layers.22.mlp.experts.24",
847
- "model.layers.22.mlp.experts.25",
848
- "model.layers.22.mlp.experts.26",
849
- "model.layers.22.mlp.experts.27",
850
- "model.layers.22.mlp.experts.28",
851
- "model.layers.22.mlp.experts.29",
852
- "model.layers.22.mlp.experts.30",
853
- "model.layers.22.mlp.experts.31",
854
- "model.layers.23.mlp.router",
855
- "model.layers.23.mlp.experts.0",
856
- "model.layers.23.mlp.experts.1",
857
- "model.layers.23.mlp.experts.2",
858
- "model.layers.23.mlp.experts.3",
859
- "model.layers.23.mlp.experts.4",
860
- "model.layers.23.mlp.experts.5",
861
- "model.layers.23.mlp.experts.6",
862
- "model.layers.23.mlp.experts.7",
863
- "model.layers.23.mlp.experts.8",
864
- "model.layers.23.mlp.experts.9",
865
- "model.layers.23.mlp.experts.10",
866
- "model.layers.23.mlp.experts.11",
867
- "model.layers.23.mlp.experts.12",
868
- "model.layers.23.mlp.experts.13",
869
- "model.layers.23.mlp.experts.14",
870
- "model.layers.23.mlp.experts.15",
871
- "model.layers.23.mlp.experts.16",
872
- "model.layers.23.mlp.experts.17",
873
- "model.layers.23.mlp.experts.18",
874
- "model.layers.23.mlp.experts.19",
875
- "model.layers.23.mlp.experts.20",
876
- "model.layers.23.mlp.experts.21",
877
- "model.layers.23.mlp.experts.22",
878
- "model.layers.23.mlp.experts.23",
879
- "model.layers.23.mlp.experts.24",
880
- "model.layers.23.mlp.experts.25",
881
- "model.layers.23.mlp.experts.26",
882
- "model.layers.23.mlp.experts.27",
883
- "model.layers.23.mlp.experts.28",
884
- "model.layers.23.mlp.experts.29",
885
- "model.layers.23.mlp.experts.30",
886
- "model.layers.23.mlp.experts.31",
887
  "lm_head"
888
- ],
889
- "kv_cache_scheme": null,
890
- "quant_method": "compressed-tensors",
891
- "quantization_status": "compressed",
892
- "sparsity_config": {},
893
- "transform_config": {},
894
- "version": "0.17.2.a20260702"
895
  },
896
  "rms_norm_eps": 1e-05,
897
- "rope_parameters": {
898
  "beta_fast": 32.0,
899
  "beta_slow": 1.0,
900
  "factor": 32.0,
901
  "original_max_position_embeddings": 4096,
902
- "rope_theta": 150000,
903
  "rope_type": "yarn",
904
  "truncate": false
905
  },
906
- "router_aux_loss_coef": 0.9,
 
907
  "sliding_window": 128,
908
  "swiglu_limit": 7.0,
909
  "tie_word_embeddings": false,
910
- "transformers_version": "5.10.2",
911
- "unsloth_version": "2026.7.3",
912
- "use_cache": false,
913
- "vocab_size": 201088
914
  }
 
4
  ],
5
  "attention_bias": true,
6
  "attention_dropout": 0.0,
7
+ "bos_token_id": 199998,
8
  "dtype": "bfloat16",
9
  "eos_token_id": 200002,
10
  "experts_per_token": 4,
 
15
  "initializer_range": 0.02,
16
  "intermediate_size": 2880,
17
  "layer_types": [
18
+ "sliding_attention", "full_attention", "sliding_attention", "full_attention",
19
+ "sliding_attention", "full_attention", "sliding_attention", "full_attention",
20
+ "sliding_attention", "full_attention", "sliding_attention", "full_attention",
21
+ "sliding_attention", "full_attention", "sliding_attention", "full_attention",
22
+ "sliding_attention", "full_attention", "sliding_attention", "full_attention",
23
+ "sliding_attention", "full_attention", "sliding_attention", "full_attention"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  ],
25
  "max_position_embeddings": 131072,
 
26
  "model_type": "gpt_oss",
27
  "num_attention_heads": 64,
28
  "num_experts_per_tok": 4,
29
  "num_hidden_layers": 24,
30
  "num_key_value_heads": 8,
31
  "num_local_experts": 32,
32
+ "output_router_logits": true,
33
  "pad_token_id": 199999,
34
  "quantization_config": {
35
  "quant_method": "mxfp4",
36
+ "dequantize": false,
37
+ "modules_to_not_convert": [
38
+ "model.layers.*.self_attn",
39
+ "model.layers.*.mlp.router",
40
+ "model.embed_tokens",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  "lm_head"
42
+ ]
 
 
 
 
 
 
43
  },
44
  "rms_norm_eps": 1e-05,
45
+ "rope_scaling": {
46
  "beta_fast": 32.0,
47
  "beta_slow": 1.0,
48
  "factor": 32.0,
49
  "original_max_position_embeddings": 4096,
 
50
  "rope_type": "yarn",
51
  "truncate": false
52
  },
53
+ "rope_theta": 150000,
54
+ "router_aux_loss_coef": 0.001,
55
  "sliding_window": 128,
56
  "swiglu_limit": 7.0,
57
  "tie_word_embeddings": false,
58
+ "transformers_version": "4.57.6",
59
+ "use_cache": true,
60
+ "vocab_size": 200024
 
61
  }