strata-engine commited on
Commit
005eebe
·
verified ·
1 Parent(s): 0dfe4c3

Add tokenizer_config.json for Inkling-2-Mini tokenizer/template support

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +548 -0
tokenizer_config.json ADDED
@@ -0,0 +1,548 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "model_max_length": 1048576,
4
+ "clean_up_tokenization_spaces": false,
5
+ "fix_mistral_regex": false,
6
+ "extra_special_tokens": {
7
+ "endoftext": "<|endoftext|>",
8
+ "message_user": "<|message_user|>",
9
+ "message_model": "<|message_model|>",
10
+ "message_system": "<|message_system|>",
11
+ "message_tool": "<|message_tool|>",
12
+ "content_text": "<|content_text|>",
13
+ "content_image": "<|content_image|>",
14
+ "content_model_end_sampling": "<|content_model_end_sampling|>",
15
+ "content_thinking": "<|content_thinking|>",
16
+ "end_message": "<|end_message|>",
17
+ "content_tool_error": "<|content_tool_error|>",
18
+ "content_xml": "<|content_xml|>",
19
+ "begin_of_text": "<|begin_of_text|>",
20
+ "content_invoke_tool_json": "<|content_invoke_tool_json|>",
21
+ "content_invoke_tool_text": "<|content_invoke_tool_text|>"
22
+ },
23
+ "added_tokens_decoder": {
24
+ "199998": {
25
+ "content": "<|unused|>",
26
+ "single_word": false,
27
+ "lstrip": false,
28
+ "rstrip": false,
29
+ "normalized": false,
30
+ "special": true
31
+ },
32
+ "199999": {
33
+ "content": "<|endoftext|>",
34
+ "single_word": false,
35
+ "lstrip": false,
36
+ "rstrip": false,
37
+ "normalized": false,
38
+ "special": true
39
+ },
40
+ "200000": {
41
+ "content": "<|message_user|>",
42
+ "single_word": false,
43
+ "lstrip": false,
44
+ "rstrip": false,
45
+ "normalized": false,
46
+ "special": true
47
+ },
48
+ "200001": {
49
+ "content": "<|message_model|>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ "200002": {
57
+ "content": "<|message_system|>",
58
+ "single_word": false,
59
+ "lstrip": false,
60
+ "rstrip": false,
61
+ "normalized": false,
62
+ "special": true
63
+ },
64
+ "200003": {
65
+ "content": "<|message_tool|>",
66
+ "single_word": false,
67
+ "lstrip": false,
68
+ "rstrip": false,
69
+ "normalized": false,
70
+ "special": true
71
+ },
72
+ "200004": {
73
+ "content": "<|content_text|>",
74
+ "single_word": false,
75
+ "lstrip": false,
76
+ "rstrip": false,
77
+ "normalized": false,
78
+ "special": true
79
+ },
80
+ "200005": {
81
+ "content": "<|content_image|>",
82
+ "single_word": false,
83
+ "lstrip": false,
84
+ "rstrip": false,
85
+ "normalized": false,
86
+ "special": true
87
+ },
88
+ "200006": {
89
+ "content": "<|content_model_end_sampling|>",
90
+ "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
+ },
96
+ "200007": {
97
+ "content": "<|unused_200007|>",
98
+ "single_word": false,
99
+ "lstrip": false,
100
+ "rstrip": false,
101
+ "normalized": false,
102
+ "special": true
103
+ },
104
+ "200008": {
105
+ "content": "<|content_thinking|>",
106
+ "single_word": false,
107
+ "lstrip": false,
108
+ "rstrip": false,
109
+ "normalized": false,
110
+ "special": true
111
+ },
112
+ "200009": {
113
+ "content": "<|unused_200009|>",
114
+ "single_word": false,
115
+ "lstrip": false,
116
+ "rstrip": false,
117
+ "normalized": false,
118
+ "special": true
119
+ },
120
+ "200010": {
121
+ "content": "<|end_message|>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": false,
126
+ "special": true
127
+ },
128
+ "200011": {
129
+ "content": "<|unused_200011|>",
130
+ "single_word": false,
131
+ "lstrip": false,
132
+ "rstrip": false,
133
+ "normalized": false,
134
+ "special": true
135
+ },
136
+ "200012": {
137
+ "content": "<|unused_200012|>",
138
+ "single_word": false,
139
+ "lstrip": false,
140
+ "rstrip": false,
141
+ "normalized": false,
142
+ "special": true
143
+ },
144
+ "200013": {
145
+ "content": "<|unused_200013|>",
146
+ "single_word": false,
147
+ "lstrip": false,
148
+ "rstrip": false,
149
+ "normalized": false,
150
+ "special": true
151
+ },
152
+ "200014": {
153
+ "content": "<|unused_200014|>",
154
+ "single_word": false,
155
+ "lstrip": false,
156
+ "rstrip": false,
157
+ "normalized": false,
158
+ "special": true
159
+ },
160
+ "200015": {
161
+ "content": "<|unused_200015|>",
162
+ "single_word": false,
163
+ "lstrip": false,
164
+ "rstrip": false,
165
+ "normalized": false,
166
+ "special": true
167
+ },
168
+ "200016": {
169
+ "content": "<|unused_200016|>",
170
+ "single_word": false,
171
+ "lstrip": false,
172
+ "rstrip": false,
173
+ "normalized": false,
174
+ "special": true
175
+ },
176
+ "200017": {
177
+ "content": "<|unused_200017|>",
178
+ "single_word": false,
179
+ "lstrip": false,
180
+ "rstrip": false,
181
+ "normalized": false,
182
+ "special": true
183
+ },
184
+ "200018": {
185
+ "content": "<|unused_200018|>",
186
+ "single_word": false,
187
+ "lstrip": false,
188
+ "rstrip": false,
189
+ "normalized": false,
190
+ "special": true
191
+ },
192
+ "200019": {
193
+ "content": "<|unused_200019|>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": false,
198
+ "special": true
199
+ },
200
+ "200020": {
201
+ "content": "<|unused_200020|>",
202
+ "single_word": false,
203
+ "lstrip": false,
204
+ "rstrip": false,
205
+ "normalized": false,
206
+ "special": true
207
+ },
208
+ "200021": {
209
+ "content": "<|unused_200021|>",
210
+ "single_word": false,
211
+ "lstrip": false,
212
+ "rstrip": false,
213
+ "normalized": false,
214
+ "special": true
215
+ },
216
+ "200022": {
217
+ "content": "<|content_tool_error|>",
218
+ "single_word": false,
219
+ "lstrip": false,
220
+ "rstrip": false,
221
+ "normalized": false,
222
+ "special": true
223
+ },
224
+ "200023": {
225
+ "content": "<|unused_200023|>",
226
+ "single_word": false,
227
+ "lstrip": false,
228
+ "rstrip": false,
229
+ "normalized": false,
230
+ "special": true
231
+ },
232
+ "200024": {
233
+ "content": "<|content_xml|>",
234
+ "single_word": false,
235
+ "lstrip": false,
236
+ "rstrip": false,
237
+ "normalized": false,
238
+ "special": true
239
+ },
240
+ "200025": {
241
+ "content": "<|unused_200025|>",
242
+ "single_word": false,
243
+ "lstrip": false,
244
+ "rstrip": false,
245
+ "normalized": false,
246
+ "special": true
247
+ },
248
+ "200026": {
249
+ "content": "<|unused_200026|>",
250
+ "single_word": false,
251
+ "lstrip": false,
252
+ "rstrip": false,
253
+ "normalized": false,
254
+ "special": true
255
+ },
256
+ "200027": {
257
+ "content": "<|unused_200027|>",
258
+ "single_word": false,
259
+ "lstrip": false,
260
+ "rstrip": false,
261
+ "normalized": false,
262
+ "special": true
263
+ },
264
+ "200028": {
265
+ "content": "<|begin_of_text|>",
266
+ "single_word": false,
267
+ "lstrip": false,
268
+ "rstrip": false,
269
+ "normalized": false,
270
+ "special": true
271
+ },
272
+ "200029": {
273
+ "content": "<|unused_200029|>",
274
+ "single_word": false,
275
+ "lstrip": false,
276
+ "rstrip": false,
277
+ "normalized": false,
278
+ "special": true
279
+ },
280
+ "200030": {
281
+ "content": "<|unused_200030|>",
282
+ "single_word": false,
283
+ "lstrip": false,
284
+ "rstrip": false,
285
+ "normalized": false,
286
+ "special": true
287
+ },
288
+ "200031": {
289
+ "content": "<|unused_200031|>",
290
+ "single_word": false,
291
+ "lstrip": false,
292
+ "rstrip": false,
293
+ "normalized": false,
294
+ "special": true
295
+ },
296
+ "200032": {
297
+ "content": "<|unused_200032|>",
298
+ "single_word": false,
299
+ "lstrip": false,
300
+ "rstrip": false,
301
+ "normalized": false,
302
+ "special": true
303
+ },
304
+ "200033": {
305
+ "content": "<|unused_200033|>",
306
+ "single_word": false,
307
+ "lstrip": false,
308
+ "rstrip": false,
309
+ "normalized": false,
310
+ "special": true
311
+ },
312
+ "200034": {
313
+ "content": "<|unused_200034|>",
314
+ "single_word": false,
315
+ "lstrip": false,
316
+ "rstrip": false,
317
+ "normalized": false,
318
+ "special": true
319
+ },
320
+ "200035": {
321
+ "content": "<|unused_200035|>",
322
+ "single_word": false,
323
+ "lstrip": false,
324
+ "rstrip": false,
325
+ "normalized": false,
326
+ "special": true
327
+ },
328
+ "200036": {
329
+ "content": "<|unused_200036|>",
330
+ "single_word": false,
331
+ "lstrip": false,
332
+ "rstrip": false,
333
+ "normalized": false,
334
+ "special": true
335
+ },
336
+ "200037": {
337
+ "content": "<|unused_200037|>",
338
+ "single_word": false,
339
+ "lstrip": false,
340
+ "rstrip": false,
341
+ "normalized": false,
342
+ "special": true
343
+ },
344
+ "200038": {
345
+ "content": "<|unused_200038|>",
346
+ "single_word": false,
347
+ "lstrip": false,
348
+ "rstrip": false,
349
+ "normalized": false,
350
+ "special": true
351
+ },
352
+ "200039": {
353
+ "content": "<|unused_200039|>",
354
+ "single_word": false,
355
+ "lstrip": false,
356
+ "rstrip": false,
357
+ "normalized": false,
358
+ "special": true
359
+ },
360
+ "200040": {
361
+ "content": "<|unused_200040|>",
362
+ "single_word": false,
363
+ "lstrip": false,
364
+ "rstrip": false,
365
+ "normalized": false,
366
+ "special": true
367
+ },
368
+ "200041": {
369
+ "content": "<|unused_200041|>",
370
+ "single_word": false,
371
+ "lstrip": false,
372
+ "rstrip": false,
373
+ "normalized": false,
374
+ "special": true
375
+ },
376
+ "200042": {
377
+ "content": "<|unused_200042|>",
378
+ "single_word": false,
379
+ "lstrip": false,
380
+ "rstrip": false,
381
+ "normalized": false,
382
+ "special": true
383
+ },
384
+ "200043": {
385
+ "content": "<|unused_200043|>",
386
+ "single_word": false,
387
+ "lstrip": false,
388
+ "rstrip": false,
389
+ "normalized": false,
390
+ "special": true
391
+ },
392
+ "200044": {
393
+ "content": "<|unused_200044|>",
394
+ "single_word": false,
395
+ "lstrip": false,
396
+ "rstrip": false,
397
+ "normalized": false,
398
+ "special": true
399
+ },
400
+ "200045": {
401
+ "content": "<|unused_200045|>",
402
+ "single_word": false,
403
+ "lstrip": false,
404
+ "rstrip": false,
405
+ "normalized": false,
406
+ "special": true
407
+ },
408
+ "200046": {
409
+ "content": "<|unused_200046|>",
410
+ "single_word": false,
411
+ "lstrip": false,
412
+ "rstrip": false,
413
+ "normalized": false,
414
+ "special": true
415
+ },
416
+ "200047": {
417
+ "content": "<|unused_200047|>",
418
+ "single_word": false,
419
+ "lstrip": false,
420
+ "rstrip": false,
421
+ "normalized": false,
422
+ "special": true
423
+ },
424
+ "200048": {
425
+ "content": "<|unused_200048|>",
426
+ "single_word": false,
427
+ "lstrip": false,
428
+ "rstrip": false,
429
+ "normalized": false,
430
+ "special": true
431
+ },
432
+ "200049": {
433
+ "content": "<|content_invoke_tool_json|>",
434
+ "single_word": false,
435
+ "lstrip": false,
436
+ "rstrip": false,
437
+ "normalized": false,
438
+ "special": true
439
+ },
440
+ "200050": {
441
+ "content": "<|unused_200050|>",
442
+ "single_word": false,
443
+ "lstrip": false,
444
+ "rstrip": false,
445
+ "normalized": false,
446
+ "special": true
447
+ },
448
+ "200051": {
449
+ "content": "<|unused_200051|>",
450
+ "single_word": false,
451
+ "lstrip": false,
452
+ "rstrip": false,
453
+ "normalized": false,
454
+ "special": true
455
+ },
456
+ "200052": {
457
+ "content": "<|unused_200052|>",
458
+ "single_word": false,
459
+ "lstrip": false,
460
+ "rstrip": false,
461
+ "normalized": false,
462
+ "special": true
463
+ },
464
+ "200053": {
465
+ "content": "<|unused_200053|>",
466
+ "single_word": false,
467
+ "lstrip": false,
468
+ "rstrip": false,
469
+ "normalized": false,
470
+ "special": true
471
+ },
472
+ "200054": {
473
+ "content": "<|unused_200054|>",
474
+ "single_word": false,
475
+ "lstrip": false,
476
+ "rstrip": false,
477
+ "normalized": false,
478
+ "special": true
479
+ },
480
+ "200055": {
481
+ "content": "<|unused_200055|>",
482
+ "single_word": false,
483
+ "lstrip": false,
484
+ "rstrip": false,
485
+ "normalized": false,
486
+ "special": true
487
+ },
488
+ "200056": {
489
+ "content": "<|unused_200056|>",
490
+ "single_word": false,
491
+ "lstrip": false,
492
+ "rstrip": false,
493
+ "normalized": false,
494
+ "special": true
495
+ },
496
+ "200057": {
497
+ "content": "<|content_invoke_tool_text|>",
498
+ "single_word": false,
499
+ "lstrip": false,
500
+ "rstrip": false,
501
+ "normalized": false,
502
+ "special": true
503
+ }
504
+ },
505
+ "response_template": {
506
+ "defaults": {
507
+ "role": "assistant"
508
+ },
509
+ "start_anchor_pattern": "<\\|end_message\\|>|<\\|content_model_end_sampling\\|>",
510
+ "fields": {
511
+ "reasoning_content": {
512
+ "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_thinking\\|>",
513
+ "close": "<|end_message|>",
514
+ "repeats": true,
515
+ "join": "",
516
+ "content": "text",
517
+ "content_args": {
518
+ "strip": false
519
+ }
520
+ },
521
+ "content": {
522
+ "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_(?:text|tool_error|invoke_tool_text)\\|>",
523
+ "close": "<|end_message|>",
524
+ "repeats": true,
525
+ "join": "",
526
+ "content": "text",
527
+ "content_args": {
528
+ "strip": false
529
+ }
530
+ },
531
+ "tool_calls": {
532
+ "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_invoke_tool_json\\|>",
533
+ "close": "<|end_message|>",
534
+ "repeats": true,
535
+ "content": "json",
536
+ "transform": {
537
+ "type": "function",
538
+ "function": {
539
+ "name": "{content.name}",
540
+ "arguments": "{content.args}"
541
+ }
542
+ }
543
+ }
544
+ }
545
+ },
546
+ "eos_token": "<|content_model_end_sampling|>",
547
+ "pad_token": "<|endoftext|>"
548
+ }