Hugging Face's logo

XiaomiMiMo
/
MiMo-V2-Flash

Text Generation
Transformers
Safetensors
mimo_v2_flash
conversational
custom_code
Eval Results
fp8
Model card Files
xet
 Community
43

Update MiMo-V2-Flash config.json for native Transformers compatibility

#41
by casinca - opened
base: refs/heads/main
←
from: refs/pr/41
Discussion Files changed
+115
-2
Files changed (1) hide show
  1. config.json +115 -2
config.json CHANGED
@@ -205,9 +205,122 @@
205
  "n_group": 1,
206
  "topk_group": 1,
207
  "topk_method": "noaux_tc",
208
- "routed_scaling_factor": null,
209
  "swa_num_attention_heads": 64,
210
  "swa_num_key_value_heads": 8,
211
  "swa_head_dim": 192,
212
- "swa_v_head_dim": 128
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
213
  }
 
205
  "n_group": 1,
206
  "topk_group": 1,
207
  "topk_method": "noaux_tc",
208
+ "routed_scaling_factor": 1.0,
209
  "swa_num_attention_heads": 64,
210
  "swa_num_key_value_heads": 8,
211
  "swa_head_dim": 192,
212
+ "swa_v_head_dim": 128,
213
+ "rms_norm_eps": 1e-05,
214
+ "layer_types": [
215
+ "full_attention",
216
+ "sliding_attention",
217
+ "sliding_attention",
218
+ "sliding_attention",
219
+ "sliding_attention",
220
+ "full_attention",
221
+ "sliding_attention",
222
+ "sliding_attention",
223
+ "sliding_attention",
224
+ "sliding_attention",
225
+ "sliding_attention",
226
+ "full_attention",
227
+ "sliding_attention",
228
+ "sliding_attention",
229
+ "sliding_attention",
230
+ "sliding_attention",
231
+ "sliding_attention",
232
+ "full_attention",
233
+ "sliding_attention",
234
+ "sliding_attention",
235
+ "sliding_attention",
236
+ "sliding_attention",
237
+ "sliding_attention",
238
+ "full_attention",
239
+ "sliding_attention",
240
+ "sliding_attention",
241
+ "sliding_attention",
242
+ "sliding_attention",
243
+ "sliding_attention",
244
+ "full_attention",
245
+ "sliding_attention",
246
+ "sliding_attention",
247
+ "sliding_attention",
248
+ "sliding_attention",
249
+ "sliding_attention",
250
+ "full_attention",
251
+ "sliding_attention",
252
+ "sliding_attention",
253
+ "sliding_attention",
254
+ "sliding_attention",
255
+ "sliding_attention",
256
+ "full_attention",
257
+ "sliding_attention",
258
+ "sliding_attention",
259
+ "sliding_attention",
260
+ "sliding_attention",
261
+ "sliding_attention",
262
+ "full_attention"
263
+ ],
264
+ "mlp_layer_types": [
265
+ "dense",
266
+ "sparse",
267
+ "sparse",
268
+ "sparse",
269
+ "sparse",
270
+ "sparse",
271
+ "sparse",
272
+ "sparse",
273
+ "sparse",
274
+ "sparse",
275
+ "sparse",
276
+ "sparse",
277
+ "sparse",
278
+ "sparse",
279
+ "sparse",
280
+ "sparse",
281
+ "sparse",
282
+ "sparse",
283
+ "sparse",
284
+ "sparse",
285
+ "sparse",
286
+ "sparse",
287
+ "sparse",
288
+ "sparse",
289
+ "sparse",
290
+ "sparse",
291
+ "sparse",
292
+ "sparse",
293
+ "sparse",
294
+ "sparse",
295
+ "sparse",
296
+ "sparse",
297
+ "sparse",
298
+ "sparse",
299
+ "sparse",
300
+ "sparse",
301
+ "sparse",
302
+ "sparse",
303
+ "sparse",
304
+ "sparse",
305
+ "sparse",
306
+ "sparse",
307
+ "sparse",
308
+ "sparse",
309
+ "sparse",
310
+ "sparse",
311
+ "sparse",
312
+ "sparse"
313
+ ],
314
+ "rope_parameters": {
315
+ "full_attention": {
316
+ "rope_type": "default",
317
+ "rope_theta": 5000000,
318
+ "partial_rotary_factor": 0.334
319
+ },
320
+ "sliding_attention": {
321
+ "rope_type": "default",
322
+ "rope_theta": 10000,
323
+ "partial_rotary_factor": 0.334
324
+ }
325
+ }
326
  }