add missing files for legal scan

#2
Files changed (3) hide show
  1. README.md +103 -6
  2. tokenizer.json +2 -2
  3. tokenizer_config.json +240 -243
README.md CHANGED
@@ -1,11 +1,108 @@
1
  ---
 
 
 
 
 
2
  license: other
3
- license_name: other
4
- license_link: LICENSE
5
  ---
 
6
 
7
- **Disclaimer**
 
 
 
 
 
 
 
 
 
 
 
8
 
9
- This model is provided for research and evaluation purposes only.
10
- Quantization may introduce accuracy or behavioral differences compared to the original model.
11
- Users are responsible for validating the model in their own environments and complying with the original model license.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ base_model:
3
+ - MiniMaxAI/MiniMax-M2.7
4
+ language:
5
+ - en
6
+ library_name: transformers
7
  license: other
8
+ license_name: modified-mit
9
+ license_link: https://huggingface.co/MiniMaxAI/MiniMax-M2.7/blob/main/LICENSE
10
  ---
11
+ # Model Overview
12
 
13
+ - **Model Architecture:** MiniMaxM2ForCausalLM
14
+ - **Input:** Text
15
+ - **Output:** Text
16
+ - **Supported Hardware Microarchitecture:** AMD MI300/MI350/MI355 (emulation)
17
+ - **ROCm:** 7.2.2
18
+ - **Operating System(s):** Linux
19
+ - **Inference Engine:** [vLLM](https://docs.vllm.ai/en/latest/)
20
+ - **Model Optimizer:** [AMD-Quark](https://quark.docs.amd.com/latest/index.html) (v0.12)
21
+ - **Inference Engine:** [SGLang](https://docs.sglang.ai/)/[vLLM](https://docs.vllm.ai/en/latest/)
22
+ - **Model Optimizer:** [AMD-Quark](https://quark.docs.amd.com/latest/index.html)
23
+ - **Weight quantization:** NVFP4, Static
24
+ - **Activation quantization:** NVFP4, Dynamic
25
 
26
+
27
+ # Model Quantization
28
+
29
+ The model was quantized from [amd/MiniMax-M2.7-BF16](https://huggingface.co/amd/MiniMax-M2.7-BF16), originally from [MiniMax/MiniMax-M2.7](https://huggingface.co/MiniMax/MiniMax-M2.7), using [AMD-Quark](https://quark.docs.amd.com/latest/index.html). The weights are quantized to NVFP4 and activations are quantized to NVFP4.
30
+
31
+
32
+ **Quantization scripts:**
33
+ ```
34
+ cd /Quark/examples/torch/language_modeling/llm_ptq
35
+ exclude_layers="lm_head *block_sparse_moe.gate* *self_attn*"
36
+ python3 quantize_quark.py --model_dir amd/MiniMax-M2.7-BF16 \
37
+ --quant_scheme nvfp4 \
38
+ --exclude_layers $exclude_layers \
39
+ --num_calib_data 128 \
40
+ --model_export hf_format \
41
+ --multi_gpu balanced \
42
+ --trust_remote_code \
43
+ --output_dir amd/MiniMax-M2.7-NVFP4
44
+ ```
45
+
46
+
47
+ For further details or issues, please refer to the AMD-Quark documentation or contact the respective developers.
48
+
49
+ # Evaluation
50
+ The model was evaluated on gsm8k benchmarks using the vllm framework.
51
+
52
+ ### Accuracy
53
+
54
+ <table>
55
+ <tr>
56
+ <td><strong>Benchmark</strong>
57
+ </td>
58
+ <td><strong>MiniMaxAI/MiniMax-M2.7 </strong>
59
+ </td>
60
+ <td><strong>amd/MiniMax-M2.7-NVFP4(this model)</strong>
61
+ </td>
62
+ <td><strong>Recovery</strong>
63
+ </td>
64
+ </tr>
65
+ <tr>
66
+ <td>gsm8k (flexible-extract)
67
+ </td>
68
+ <td>91.81
69
+ </td>
70
+ <td>92.20
71
+ </td>
72
+ <td>100.04%
73
+ </td>
74
+ </tr>
75
+ </table>
76
+
77
+
78
+ ### Reproduction
79
+
80
+ The GSM8K result was obtained using the `lm-evaluation-harness` framework, based on the Docker image `rocm/vllm-dev:nightly_main_20260603`.
81
+
82
+ Install the lm-eval `(Version: 0.4.12)` in container first.
83
+ ```
84
+ pip install lm-eval
85
+ pip install lm-eval[api]
86
+ ```
87
+ #### Launching server
88
+ ```
89
+ vllm serve \
90
+ --model amd/MiniMax-M2.7-NVFP4 \
91
+ --trust-remote-code \
92
+ --host 0.0.0.0 \
93
+ --port 8011 \
94
+ --tensor-parallel-size 4 \
95
+ --enable-auto-tool-choice \
96
+ --tool-call-parser minimax_m2 \
97
+ --reasoning-parser minimax_m2_append_think
98
+ ```
99
+
100
+
101
+ #### Evaluating model in a new terminal
102
+ ```
103
+ python3 vllm/tests/evals/gsm8k/gsm8k_eval.py --host http://0.0.0.0 --port 8011
104
+ ```
105
+
106
+
107
+ # License
108
+ Modifications Copyright(c) 2026 Advanced Micro Devices, Inc. All rights reserved.
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ad01040a70bcef384b88cf9a00f75c1b73106acf4ab506fe04d75a89591111dc
3
- size 15523019
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:757622126525aeeb131756849d93298070ff3f0319c455ec8c5bb0f6b1cebbe8
3
+ size 9730160
tokenizer_config.json CHANGED
@@ -1,498 +1,495 @@
1
  {
2
- "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
- "200000": {
5
  "content": "]!p~[",
6
  "lstrip": false,
7
  "normalized": false,
8
  "rstrip": false,
9
  "single_word": false,
10
  "special": true
11
- },
12
- "200001": {
13
  "content": "<fim_prefix>",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
  "special": true
19
- },
20
- "200002": {
21
  "content": "<fim_middle>",
22
  "lstrip": false,
23
  "normalized": false,
24
  "rstrip": false,
25
  "single_word": false,
26
  "special": true
27
- },
28
- "200003": {
29
  "content": "<fim_suffix>",
30
  "lstrip": false,
31
  "normalized": false,
32
  "rstrip": false,
33
  "single_word": false,
34
  "special": true
35
- },
36
- "200004": {
37
  "content": "<fim_pad>",
38
  "lstrip": false,
39
  "normalized": false,
40
  "rstrip": false,
41
  "single_word": false,
42
  "special": true
43
- },
44
- "200005": {
45
  "content": "<reponame>",
46
  "lstrip": false,
47
  "normalized": false,
48
  "rstrip": false,
49
  "single_word": false,
50
  "special": true
51
- },
52
- "200006": {
53
  "content": "<filename>",
54
  "lstrip": false,
55
  "normalized": false,
56
  "rstrip": false,
57
  "single_word": false,
58
  "special": true
59
- },
60
- "200007": {
61
  "content": "<gh_stars>",
62
  "lstrip": false,
63
  "normalized": false,
64
  "rstrip": false,
65
  "single_word": false,
66
  "special": true
67
- },
68
- "200008": {
69
  "content": "<issue_start>",
70
  "lstrip": false,
71
  "normalized": false,
72
  "rstrip": false,
73
  "single_word": false,
74
  "special": true
75
- },
76
- "200009": {
77
  "content": "<issue_comment>",
78
  "lstrip": false,
79
  "normalized": false,
80
  "rstrip": false,
81
  "single_word": false,
82
  "special": true
83
- },
84
- "200010": {
85
  "content": "<issue_closed>",
86
  "lstrip": false,
87
  "normalized": false,
88
  "rstrip": false,
89
  "single_word": false,
90
  "special": true
91
- },
92
- "200011": {
93
- "content": "<jupyter_start>",
94
- "lstrip": false,
95
- "normalized": false,
96
- "rstrip": false,
97
- "single_word": false,
98
- "special": true
99
- },
100
- "200012": {
101
  "content": "<jupyter_text>",
102
  "lstrip": false,
103
  "normalized": false,
104
  "rstrip": false,
105
  "single_word": false,
106
  "special": true
107
- },
108
- "200013": {
109
  "content": "<jupyter_code>",
110
  "lstrip": false,
111
  "normalized": false,
112
  "rstrip": false,
113
  "single_word": false,
114
  "special": true
115
- },
116
- "200014": {
117
  "content": "<jupyter_output>",
118
  "lstrip": false,
119
  "normalized": false,
120
  "rstrip": false,
121
  "single_word": false,
122
  "special": true
123
- },
124
- "200015": {
125
- "content": "<empty_output>",
126
- "lstrip": false,
127
- "normalized": false,
128
- "rstrip": false,
129
- "single_word": false,
130
- "special": true
131
- },
132
- "200016": {
133
  "content": "<commit_before>",
134
  "lstrip": false,
135
  "normalized": false,
136
  "rstrip": false,
137
  "single_word": false,
138
  "special": true
139
- },
140
- "200017": {
141
  "content": "<commit_msg>",
142
  "lstrip": false,
143
  "normalized": false,
144
  "rstrip": false,
145
  "single_word": false,
146
  "special": true
147
- },
148
- "200018": {
149
  "content": "<commit_after>",
150
  "lstrip": false,
151
  "normalized": false,
152
  "rstrip": false,
153
  "single_word": false,
154
  "special": true
155
- },
156
- "200019": {
157
  "content": "]~b]",
158
  "lstrip": false,
159
  "normalized": false,
160
  "rstrip": false,
161
  "single_word": false,
162
  "special": true
163
- },
164
- "200020": {
165
  "content": "[e~[",
166
  "lstrip": false,
167
  "normalized": false,
168
  "rstrip": false,
169
  "single_word": false,
170
  "special": true
171
- },
172
- "200021": {
173
  "content": "]!d~[",
174
  "lstrip": false,
175
  "normalized": false,
176
  "rstrip": false,
177
  "single_word": false,
178
  "special": true
179
- },
180
- "200022": {
181
  "content": "<function_call>",
182
  "lstrip": false,
183
  "normalized": false,
184
  "rstrip": false,
185
  "single_word": false,
186
  "special": true
187
- },
188
- "200023": {
189
  "content": "<code_interpreter>",
190
  "lstrip": false,
191
  "normalized": false,
192
  "rstrip": false,
193
  "single_word": false,
194
  "special": true
195
- },
196
- "200024": {
197
  "content": "]<]speech[>[",
198
  "lstrip": false,
199
  "normalized": false,
200
  "rstrip": false,
201
  "single_word": false,
202
  "special": true
203
- },
204
- "200025": {
205
  "content": "]<]image[>[",
206
  "lstrip": false,
207
  "normalized": false,
208
  "rstrip": false,
209
  "single_word": false,
210
  "special": true
211
- },
212
- "200026": {
213
  "content": "]<]video[>[",
214
  "lstrip": false,
215
  "normalized": false,
216
  "rstrip": false,
217
  "single_word": false,
218
  "special": true
219
- },
220
- "200027": {
221
  "content": "]<]start of speech[>[",
222
  "lstrip": false,
223
  "normalized": false,
224
  "rstrip": false,
225
  "single_word": false,
226
  "special": true
227
- },
228
- "200028": {
229
  "content": "]<]end of speech[>[",
230
  "lstrip": false,
231
  "normalized": false,
232
  "rstrip": false,
233
  "single_word": false,
234
  "special": true
235
- },
236
- "200029": {
237
  "content": "]<]start of image[>[",
238
  "lstrip": false,
239
  "normalized": false,
240
  "rstrip": false,
241
  "single_word": false,
242
  "special": true
243
- },
244
- "200030": {
245
  "content": "]<]end of image[>[",
246
  "lstrip": false,
247
  "normalized": false,
248
  "rstrip": false,
249
  "single_word": false,
250
  "special": true
251
- },
252
- "200031": {
253
  "content": "]<]start of video[>[",
254
  "lstrip": false,
255
  "normalized": false,
256
  "rstrip": false,
257
  "single_word": false,
258
  "special": true
259
- },
260
- "200032": {
261
  "content": "]<]end of video[>[",
262
  "lstrip": false,
263
  "normalized": false,
264
  "rstrip": false,
265
  "single_word": false,
266
  "special": true
267
- },
268
- "200033": {
269
  "content": "]<]vision pad[>[",
270
  "lstrip": false,
271
  "normalized": false,
272
  "rstrip": false,
273
  "single_word": false,
274
  "special": true
275
- },
276
- "200034": {
277
  "content": "]~!b[",
278
  "lstrip": false,
279
  "normalized": false,
280
  "rstrip": false,
281
  "single_word": false,
282
  "special": true
283
- },
284
- "200035": {
285
  "content": "<jupyter_error>",
286
  "lstrip": false,
287
  "normalized": false,
288
  "rstrip": false,
289
  "single_word": false,
290
  "special": true
291
- },
292
- "200036": {
293
  "content": "<add_file>",
 
294
  "lstrip": false,
295
- "normalized": false,
296
  "rstrip": false,
297
- "single_word": false,
298
  "special": true
299
- },
300
- "200037": {
301
  "content": "<delete_file>",
302
  "lstrip": false,
303
  "normalized": false,
304
  "rstrip": false,
305
  "single_word": false,
306
  "special": true
307
- },
308
- "200038": {
309
  "content": "<rename_file>",
310
  "lstrip": false,
311
  "normalized": false,
312
  "rstrip": false,
313
  "single_word": false,
314
  "special": true
315
- },
316
- "200039": {
317
  "content": "<edit_file>",
318
  "lstrip": false,
319
  "normalized": false,
320
  "rstrip": false,
321
  "single_word": false,
322
  "special": true
323
- },
324
- "200040": {
325
  "content": "<commit_message>",
326
  "lstrip": false,
327
  "normalized": false,
328
  "rstrip": false,
329
  "single_word": false,
330
  "special": true
331
- },
332
- "200041": {
333
  "content": "<empty_source_file>",
334
  "lstrip": false,
335
  "normalized": false,
336
  "rstrip": false,
337
  "single_word": false,
338
  "special": true
339
- },
340
- "200042": {
341
  "content": "<repo_struct>",
342
  "lstrip": false,
343
  "normalized": false,
344
  "rstrip": false,
345
  "single_word": false,
346
  "special": true
347
- },
348
- "200043": {
349
- "content": "<code_context>",
350
- "lstrip": false,
351
- "normalized": false,
352
- "rstrip": false,
353
- "single_word": false,
354
- "special": true
355
- },
356
- "200044": {
357
- "content": "<file_content>",
358
- "lstrip": false,
359
- "normalized": false,
360
- "rstrip": false,
361
- "single_word": false,
362
- "special": true
363
- },
364
- "200045": {
365
- "content": "<source_files>",
366
- "lstrip": false,
367
- "normalized": false,
368
- "rstrip": false,
369
- "single_word": false,
370
- "special": true
371
- },
372
- "200046": {
373
- "content": "<pr_start>",
374
- "lstrip": false,
375
- "normalized": false,
376
- "rstrip": false,
377
- "single_word": false,
378
- "special": true
379
- },
380
- "200047": {
381
- "content": "<review_comment>",
382
- "lstrip": false,
383
- "normalized": false,
384
- "rstrip": false,
385
- "single_word": false,
386
- "special": true
387
- },
388
- "200048": {
389
- "content": "<filepath>",
390
- "lstrip": false,
391
- "normalized": false,
392
- "rstrip": false,
393
- "single_word": false,
394
- "special": true
395
- },
396
- "200049": {
397
- "content": "<file_sep>",
398
- "lstrip": false,
399
- "normalized": false,
400
- "rstrip": false,
401
- "single_word": false,
402
- "special": true
403
- },
404
- "200050": {
405
- "content": "<think>",
406
- "lstrip": false,
407
- "normalized": false,
408
- "rstrip": false,
409
- "single_word": false,
410
- "special": false
411
- },
412
- "200051": {
413
- "content": "</think>",
414
- "lstrip": false,
415
- "normalized": false,
416
- "rstrip": false,
417
- "single_word": false,
418
- "special": false
419
- },
420
- "200052": {
421
- "content": "<minimax:tool_call>",
422
- "lstrip": false,
423
- "normalized": false,
424
- "rstrip": false,
425
- "single_word": false,
426
- "special": false
427
- },
428
- "200053": {
429
- "content": "</minimax:tool_call>",
430
- "lstrip": false,
431
- "normalized": false,
432
- "rstrip": false,
433
- "single_word": false,
434
- "special": false
435
- }
436
  },
437
  "additional_special_tokens": [
438
- "<code_interpreter>",
439
- "<commit_after>",
440
- "<commit_before>",
441
- "<commit_msg>",
442
- "<empty_output>",
443
- "<filename>",
444
- "<fim_middle>",
445
- "<fim_pad>",
446
- "<fim_prefix>",
447
- "<fim_suffix>",
448
- "<function_call>",
449
- "<gh_stars>",
450
- "]<]speech[>[",
451
- "]<]image[>[",
452
- "]<]video[>[",
453
- "]<]start of speech[>[",
454
- "]<]end of speech[>[",
455
- "]<]start of image[>[",
456
- "]<]end of image[>[",
457
- "]<]start of video[>[",
458
- "]<]end of video[>[",
459
- "]<]vision pad[>[",
460
- "]~!b[",
461
- "<issue_closed>",
462
- "<issue_comment>",
463
- "<issue_start>",
464
- "<jupyter_code>",
465
- "<jupyter_output>",
466
- "<jupyter_start>",
467
- "<jupyter_text>",
468
- "<reponame>",
469
- "[e~[",
470
- "]!d~[",
471
- "]!p~[",
472
- "]~b]",
473
- "<jupyter_error>",
474
- "<add_file>",
475
- "<delete_file>",
476
- "<rename_file>",
477
- "<edit_file>",
478
- "<commit_message>",
479
- "<empty_source_file>",
480
- "<repo_struct>",
481
- "<code_context>",
482
- "<file_content>",
483
- "<source_files>",
484
- "<pr_start>",
485
- "<review_comment>",
486
- "<filepath>",
487
- "<file_sep>"
488
- ],
 
489
  "bos_token": "]~!b[",
490
  "clean_up_tokenization_spaces": false,
491
  "eos_token": "[e~[",
492
- "extra_special_tokens": {},
493
  "model_max_length": 40960000,
494
- "pad_token": "[e~[",
495
- "padding_side": "left",
496
  "tokenizer_class": "GPT2Tokenizer",
497
  "unk_token": "]!d~["
498
  }
 
1
  {
 
2
  "added_tokens_decoder": {
3
+ "200000": {
4
  "content": "]!p~[",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
  "special": true
10
+ },
11
+ "200001": {
12
  "content": "<fim_prefix>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
+ },
19
+ "200002": {
20
  "content": "<fim_middle>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
+ },
27
+ "200003": {
28
  "content": "<fim_suffix>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
+ },
35
+ "200004": {
36
  "content": "<fim_pad>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": true
42
+ },
43
+ "200005": {
44
  "content": "<reponame>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
  "special": true
50
+ },
51
+ "200006": {
52
  "content": "<filename>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
56
  "single_word": false,
57
  "special": true
58
+ },
59
+ "200007": {
60
  "content": "<gh_stars>",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
64
  "single_word": false,
65
  "special": true
66
+ },
67
+ "200008": {
68
  "content": "<issue_start>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
  "special": true
74
+ },
75
+ "200009": {
76
  "content": "<issue_comment>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
80
  "single_word": false,
81
  "special": true
82
+ },
83
+ "200010": {
84
  "content": "<issue_closed>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
  "special": true
90
+ },
91
+ "200011": {
92
+ "content": "<jupyter_start>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "200012": {
100
  "content": "<jupyter_text>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
  "special": true
106
+ },
107
+ "200013": {
108
  "content": "<jupyter_code>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
  "special": true
114
+ },
115
+ "200014": {
116
  "content": "<jupyter_output>",
117
  "lstrip": false,
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
  "special": true
122
+ },
123
+ "200015": {
124
+ "content": "<empty_output>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "200016": {
132
  "content": "<commit_before>",
133
  "lstrip": false,
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
  "special": true
138
+ },
139
+ "200017": {
140
  "content": "<commit_msg>",
141
  "lstrip": false,
142
  "normalized": false,
143
  "rstrip": false,
144
  "single_word": false,
145
  "special": true
146
+ },
147
+ "200018": {
148
  "content": "<commit_after>",
149
  "lstrip": false,
150
  "normalized": false,
151
  "rstrip": false,
152
  "single_word": false,
153
  "special": true
154
+ },
155
+ "200019": {
156
  "content": "]~b]",
157
  "lstrip": false,
158
  "normalized": false,
159
  "rstrip": false,
160
  "single_word": false,
161
  "special": true
162
+ },
163
+ "200020": {
164
  "content": "[e~[",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
168
  "single_word": false,
169
  "special": true
170
+ },
171
+ "200021": {
172
  "content": "]!d~[",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
176
  "single_word": false,
177
  "special": true
178
+ },
179
+ "200022": {
180
  "content": "<function_call>",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
184
  "single_word": false,
185
  "special": true
186
+ },
187
+ "200023": {
188
  "content": "<code_interpreter>",
189
  "lstrip": false,
190
  "normalized": false,
191
  "rstrip": false,
192
  "single_word": false,
193
  "special": true
194
+ },
195
+ "200024": {
196
  "content": "]<]speech[>[",
197
  "lstrip": false,
198
  "normalized": false,
199
  "rstrip": false,
200
  "single_word": false,
201
  "special": true
202
+ },
203
+ "200025": {
204
  "content": "]<]image[>[",
205
  "lstrip": false,
206
  "normalized": false,
207
  "rstrip": false,
208
  "single_word": false,
209
  "special": true
210
+ },
211
+ "200026": {
212
  "content": "]<]video[>[",
213
  "lstrip": false,
214
  "normalized": false,
215
  "rstrip": false,
216
  "single_word": false,
217
  "special": true
218
+ },
219
+ "200027": {
220
  "content": "]<]start of speech[>[",
221
  "lstrip": false,
222
  "normalized": false,
223
  "rstrip": false,
224
  "single_word": false,
225
  "special": true
226
+ },
227
+ "200028": {
228
  "content": "]<]end of speech[>[",
229
  "lstrip": false,
230
  "normalized": false,
231
  "rstrip": false,
232
  "single_word": false,
233
  "special": true
234
+ },
235
+ "200029": {
236
  "content": "]<]start of image[>[",
237
  "lstrip": false,
238
  "normalized": false,
239
  "rstrip": false,
240
  "single_word": false,
241
  "special": true
242
+ },
243
+ "200030": {
244
  "content": "]<]end of image[>[",
245
  "lstrip": false,
246
  "normalized": false,
247
  "rstrip": false,
248
  "single_word": false,
249
  "special": true
250
+ },
251
+ "200031": {
252
  "content": "]<]start of video[>[",
253
  "lstrip": false,
254
  "normalized": false,
255
  "rstrip": false,
256
  "single_word": false,
257
  "special": true
258
+ },
259
+ "200032": {
260
  "content": "]<]end of video[>[",
261
  "lstrip": false,
262
  "normalized": false,
263
  "rstrip": false,
264
  "single_word": false,
265
  "special": true
266
+ },
267
+ "200033": {
268
  "content": "]<]vision pad[>[",
269
  "lstrip": false,
270
  "normalized": false,
271
  "rstrip": false,
272
  "single_word": false,
273
  "special": true
274
+ },
275
+ "200034": {
276
  "content": "]~!b[",
277
  "lstrip": false,
278
  "normalized": false,
279
  "rstrip": false,
280
  "single_word": false,
281
  "special": true
282
+ },
283
+ "200035": {
284
  "content": "<jupyter_error>",
285
  "lstrip": false,
286
  "normalized": false,
287
  "rstrip": false,
288
  "single_word": false,
289
  "special": true
290
+ },
291
+ "200036": {
292
  "content": "<add_file>",
293
+ "single_word": false,
294
  "lstrip": false,
 
295
  "rstrip": false,
296
+ "normalized": false,
297
  "special": true
298
+ },
299
+ "200037": {
300
  "content": "<delete_file>",
301
  "lstrip": false,
302
  "normalized": false,
303
  "rstrip": false,
304
  "single_word": false,
305
  "special": true
306
+ },
307
+ "200038": {
308
  "content": "<rename_file>",
309
  "lstrip": false,
310
  "normalized": false,
311
  "rstrip": false,
312
  "single_word": false,
313
  "special": true
314
+ },
315
+ "200039": {
316
  "content": "<edit_file>",
317
  "lstrip": false,
318
  "normalized": false,
319
  "rstrip": false,
320
  "single_word": false,
321
  "special": true
322
+ },
323
+ "200040": {
324
  "content": "<commit_message>",
325
  "lstrip": false,
326
  "normalized": false,
327
  "rstrip": false,
328
  "single_word": false,
329
  "special": true
330
+ },
331
+ "200041": {
332
  "content": "<empty_source_file>",
333
  "lstrip": false,
334
  "normalized": false,
335
  "rstrip": false,
336
  "single_word": false,
337
  "special": true
338
+ },
339
+ "200042": {
340
  "content": "<repo_struct>",
341
  "lstrip": false,
342
  "normalized": false,
343
  "rstrip": false,
344
  "single_word": false,
345
  "special": true
346
+ },
347
+ "200043": {
348
+ "content": "<code_context>",
349
+ "single_word": false,
350
+ "lstrip": false,
351
+ "rstrip": false,
352
+ "normalized": false,
353
+ "special": true
354
+ },
355
+ "200044": {
356
+ "content": "<file_content>",
357
+ "single_word": false,
358
+ "lstrip": false,
359
+ "rstrip": false,
360
+ "normalized": false,
361
+ "special": true
362
+ },
363
+ "200045": {
364
+ "content": "<source_files>",
365
+ "single_word": false,
366
+ "lstrip": false,
367
+ "rstrip": false,
368
+ "normalized": false,
369
+ "special": true
370
+ },
371
+ "200046": {
372
+ "content": "<pr_start>",
373
+ "single_word": false,
374
+ "lstrip": false,
375
+ "rstrip": false,
376
+ "normalized": false,
377
+ "special": true
378
+ },
379
+ "200047": {
380
+ "content": "<review_comment>",
381
+ "single_word": false,
382
+ "lstrip": false,
383
+ "rstrip": false,
384
+ "normalized": false,
385
+ "special": true
386
+ },
387
+ "200048": {
388
+ "content": "<filepath>",
389
+ "single_word": false,
390
+ "lstrip": false,
391
+ "rstrip": false,
392
+ "normalized": false,
393
+ "special": true
394
+ },
395
+ "200049": {
396
+ "content": "<file_sep>",
397
+ "single_word": false,
398
+ "lstrip": false,
399
+ "rstrip": false,
400
+ "normalized": false,
401
+ "special": true
402
+ },
403
+ "200050": {
404
+ "content": "<think>",
405
+ "single_word": false,
406
+ "lstrip": false,
407
+ "rstrip": false,
408
+ "normalized": false,
409
+ "special": false
410
+ },
411
+ "200051": {
412
+ "content": "</think>",
413
+ "single_word": false,
414
+ "lstrip": false,
415
+ "rstrip": false,
416
+ "normalized": false,
417
+ "special": false
418
+ },
419
+ "200052": {
420
+ "content": "<minimax:tool_call>",
421
+ "single_word": false,
422
+ "lstrip": false,
423
+ "rstrip": false,
424
+ "normalized": false,
425
+ "special": false
426
+ },
427
+ "200053": {
428
+ "content": "</minimax:tool_call>",
429
+ "single_word": false,
430
+ "lstrip": false,
431
+ "rstrip": false,
432
+ "normalized": false,
433
+ "special": false
434
+ }
435
  },
436
  "additional_special_tokens": [
437
+ "<code_interpreter>",
438
+ "<commit_after>",
439
+ "<commit_before>",
440
+ "<commit_msg>",
441
+ "<empty_output>",
442
+ "<filename>",
443
+ "<fim_middle>",
444
+ "<fim_pad>",
445
+ "<fim_prefix>",
446
+ "<fim_suffix>",
447
+ "<function_call>",
448
+ "<gh_stars>",
449
+ "]<]speech[>[",
450
+ "]<]image[>[",
451
+ "]<]video[>[",
452
+ "]<]start of speech[>[",
453
+ "]<]end of speech[>[",
454
+ "]<]start of image[>[",
455
+ "]<]end of image[>[",
456
+ "]<]start of video[>[",
457
+ "]<]end of video[>[",
458
+ "]<]vision pad[>[",
459
+ "]~!b[",
460
+ "<issue_closed>",
461
+ "<issue_comment>",
462
+ "<issue_start>",
463
+ "<jupyter_code>",
464
+ "<jupyter_output>",
465
+ "<jupyter_start>",
466
+ "<jupyter_text>",
467
+ "<reponame>",
468
+ "[e~[",
469
+ "]!d~[",
470
+ "]!p~[",
471
+ "]~b]",
472
+ "<jupyter_error>",
473
+ "<add_file>",
474
+ "<delete_file>",
475
+ "<rename_file>",
476
+ "<edit_file>",
477
+ "<commit_message>",
478
+ "<empty_source_file>",
479
+ "<repo_struct>",
480
+ "<code_context>",
481
+ "<file_content>",
482
+ "<source_files>",
483
+ "<pr_start>",
484
+ "<review_comment>",
485
+ "<filepath>",
486
+ "<file_sep>"
487
+ ],
488
+ "add_prefix_space": false,
489
  "bos_token": "]~!b[",
490
  "clean_up_tokenization_spaces": false,
491
  "eos_token": "[e~[",
 
492
  "model_max_length": 40960000,
 
 
493
  "tokenizer_class": "GPT2Tokenizer",
494
  "unk_token": "]!d~["
495
  }