styx-100m / tokenizer /tokenizer.json
PrometheanStudio's picture
Add Styx 100M tokenizer
e3370ab verified
Raw History Blame Contribute Delete
26.1 kB
{
"version": 1,
"type": "ByteLevelBPE",
"config": {
"vocab_size": 1024,
"bos_token": "<|beginoftext|>",
"eos_token": "<|endoftext|>",
"pad_token": "<|pad|>",
"unk_token": "<|unk|>",
"extra_special_tokens": [],
"reserved_tokens": [],
"pre_tokenize": null,
"split_special": false
},
"merges": [
[
101,
32
],
[
115,
32
],
[
110,
32
],
[
97,
32
],
[
101,
114
],
[
111,
32
],
[
116,
104
],
[
105,
110
],
[
44,
32
],
[
101,
110
],
[
116,
32
],
[
97,
114
],
[
97,
110
],
[
111,
114
],
[
100,
32
],
[
32,
208
],
[
208,
190
],
[
101,
115
],
[
111,
110
],
[
116,
105
],
[
97,
108
],
[
121,
32
],
[
208,
181
],
[
208,
184
],
[
209,
130
],
[
208,
176
],
[
32,
32
],
[
101,
257
],
[
101,
108
],
[
111,
117
],
[
46,
32
],
[
262,
256
],
[
100,
256
],
[
272,
208
],
[
114,
101
],
[
105,
99
],
[
209,
129
],
[
116,
261
],
[
263,
103
],
[
111,
109
],
[
97,
257
],
[
97,
116
],
[
209,
128
],
[
208,
189
],
[
115,
116
],
[
113,
117
],
[
97,
99
],
[
268,
270
],
[
101,
258
],
[
111,
257
],
[
260,
32
],
[
111,
258
],
[
10,
10
],
[
102,
32
],
[
269,
32
],
[
294,
32
],
[
117,
110
],
[
114,
111
],
[
115,
105
],
[
45,
45
],
[
101,
99
],
[
265,
116
],
[
97,
109
],
[
100,
105
],
[
97,
258
],
[
97,
115
],
[
105,
257
],
[
112,
108
],
[
105,
116
],
[
97,
100
],
[
105,
108
],
[
101,
109
],
[
108,
259
],
[
121,
285
],
[
111,
309
],
[
208,
187
],
[
101,
270
],
[
99,
104
],
[
111,
108
],
[
117,
115
],
[
276,
32
],
[
97,
98
],
[
284,
32
],
[
97,
275
],
[
301,
256
],
[
105,
258
],
[
267,
32
],
[
101,
116
],
[
282,
282
],
[
209,
131
],
[
195,
179
],
[
117,
114
],
[
99,
295
],
[
208,
186
],
[
99,
105
],
[
99,
274
],
[
97,
266
],
[
208,
188
],
[
209,
143
],
[
116,
114
],
[
101,
120
],
[
108,
105
],
[
111,
115
],
[
209,
139
],
[
208,
178
],
[
46,
308
],
[
112,
313
],
[
195,
173
],
[
209,
140
],
[
195,
161
],
[
105,
100
],
[
105,
109
],
[
115,
117
],
[
114,
32
],
[
58,
32
],
[
97,
112
],
[
111,
119
],
[
112,
260
],
[
117,
108
],
[
329,
32
],
[
315,
315
],
[
195,
169
],
[
101,
100
],
[
84,
104
],
[
271,
191
],
[
116,
260
],
[
114,
105
],
[
97,
103
],
[
118,
256
],
[
112,
267
],
[
209,
135
],
[
114,
273
],
[
108,
32
],
[
273,
116
],
[
292,
280
],
[
111,
99
],
[
262,
352
],
[
107,
32
],
[
102,
310
],
[
118,
260
],
[
108,
101
],
[
46,
10
],
[
111,
100
],
[
115,
256
],
[
271,
184
],
[
208,
180
],
[
117,
109
],
[
119,
105
],
[
262,
32
],
[
108,
256
],
[
99,
320
],
[
111,
112
],
[
115,
101
],
[
102,
269
],
[
117,
258
],
[
105,
103
],
[
112,
117
],
[
104,
32
],
[
289,
178
],
[
32,
292
],
[
101,
264
],
[
119,
104
],
[
299,
279
],
[
346,
258
],
[
271,
178
],
[
208,
183
],
[
115,
264
],
[
265,
100
],
[
267,
256
],
[
109,
97
],
[
112,
114
],
[
108,
305
],
[
118,
105
],
[
111,
116
],
[
103,
114
],
[
111,
98
],
[
209,
133
],
[
312,
259
],
[
208,
191
],
[
275,
99
],
[
110,
101
],
[
100,
101
],
[
298,
281
],
[
102,
291
],
[
302,
105
],
[
105,
266
],
[
403,
404
],
[
116,
256
],
[
10,
344
],
[
339,
274
],
[
275,
274
],
[
98,
256
],
[
105,
115
],
[
116,
117
],
[
259,
288
],
[
271,
186
],
[
268,
100
],
[
45,
32
],
[
227,
129
],
[
111,
264
],
[
73,
32
],
[
271,
189
],
[
109,
101
],
[
104,
97
],
[
276,
108
],
[
265,
266
],
[
329,
369
],
[
280,
278
],
[
41,
32
],
[
117,
99
],
[
105,
114
],
[
97,
264
],
[
99,
108
],
[
339,
307
],
[
105,
122
],
[
271,
190
],
[
99,
117
],
[
112,
269
],
[
385,
259
],
[
115,
104
],
[
409,
109
],
[
112,
32
],
[
208,
179
],
[
108,
270
],
[
260,
256
],
[
97,
121
],
[
209,
142
],
[
379,
256
],
[
263,
116
],
[
100,
297
],
[
102,
114
],
[
280,
364
],
[
376,
376
],
[
285,
479
],
[
273,
264
],
[
297,
256
],
[
316,
116
],
[
39,
257
],
[
103,
117
],
[
107,
256
],
[
101,
266
],
[
289,
188
],
[
280,
272
],
[
99,
290
],
[
61,
32
],
[
58,
308
],
[
108,
296
],
[
333,
32
],
[
267,
105
],
[
208,
185
],
[
110,
111
],
[
330,
287
],
[
116,
97
],
[
298,
278
],
[
100,
273
],
[
119,
269
],
[
117,
116
],
[
99,
256
],
[
115,
112
],
[
208,
177
],
[
299,
272
],
[
289,
187
],
[
111,
111
],
[
102,
260
],
[
278,
280
],
[
372,
32
],
[
115,
99
],
[
102,
326
],
[
261,
288
],
[
96,
96
],
[
300,
114
],
[
295,
32
],
[
257,
303
],
[
101,
103
],
[
98,
108
],
[
101,
118
],
[
114,
256
],
[
99,
307
],
[
268,
116
],
[
271,
188
],
[
46,
271
],
[
97,
118
],
[
109,
256
],
[
100,
338
],
[
112,
111
],
[
331,
279
],
[
298,
279
],
[
97,
270
],
[
110,
261
],
[
104,
116
],
[
226,
128
],
[
121,
264
],
[
271,
180
],
[
34,
32
],
[
105,
32
],
[
119,
32
],
[
360,
281
],
[
115,
260
],
[
98,
117
],
[
115,
274
],
[
239,
188
],
[
289,
179
],
[
275,
307
],
[
98,
277
],
[
299,
359
],
[
268,
266
],
[
286,
69
],
[
341,
287
],
[
195,
177
],
[
105,
369
],
[
278,
418
],
[
208,
182
],
[
108,
277
],
[
290,
99
],
[
114,
97
],
[
100,
261
],
[
278,
298
],
[
365,
257
],
[
286,
73
],
[
209,
136
],
[
48,
48
],
[
116,
101
],
[
108,
261
],
[
281,
349
],
[
101,
102
],
[
332,
293
],
[
262,
101
],
[
118,
366
],
[
107,
101
],
[
317,
256
],
[
99,
273
],
[
115,
261
],
[
98,
105
],
[
111,
103
],
[
348,
261
],
[
286,
65
],
[
112,
290
],
[
363,
259
],
[
274,
283
],
[
105,
309
],
[
337,
405
],
[
112,
310
],
[
105,
300
],
[
486,
527
],
[
112,
101
],
[
110,
105
],
[
116,
347
],
[
98,
101
],
[
327,
32
],
[
291,
413
],
[
103,
265
],
[
278,
299
],
[
195,
186
],
[
506,
266
],
[
100,
260
],
[
262,
306
],
[
102,
263
],
[
109,
268
],
[
269,
256
],
[
473,
116
],
[
285,
266
],
[
264,
303
],
[
431,
106
],
[
117,
100
],
[
440,
419
],
[
289,
180
],
[
102,
116
],
[
351,
116
],
[
109,
117
],
[
109,
263
],
[
457,
281
],
[
102,
312
],
[
285,
110
],
[
262,
322
],
[
104,
284
],
[
346,
110
],
[
209,
134
],
[
99,
111
],
[
286,
83
],
[
119,
101
],
[
325,
261
],
[
109,
573
],
[
316,
105
],
[
335,
256
],
[
99,
101
],
[
259,
115
],
[
257,
293
],
[
459,
384
],
[
105,
265
],
[
108,
108
],
[
318,
323
],
[
286,
379
],
[
324,
277
],
[
109,
32
],
[
331,
281
],
[
295,
256
],
[
112,
358
],
[
112,
105
],
[
314,
109
],
[
311,
287
],
[
292,
349
],
[
98,
114
],
[
289,
182
],
[
116,
265
],
[
305,
288
],
[
263,
381
],
[
97,
277
],
[
289,
185
],
[
271,
177
],
[
301,
105
],
[
285,
114
],
[
275,
118
],
[
265,
275
],
[
292,
354
],
[
268,
103
],
[
108,
111
],
[
102,
101
],
[
331,
354
],
[
318,
256
],
[
259,
340
],
[
209,
137
],
[
103,
111
],
[
380,
298
],
[
286,
483
],
[
115,
334
],
[
105,
317
],
[
263,
256
],
[
115,
284
],
[
109,
614
],
[
65,
32
],
[
383,
256
],
[
259,
277
],
[
117,
32
],
[
87,
104
],
[
276,
388
],
[
104,
266
],
[
382,
112
],
[
377,
32
],
[
304,
328
],
[
299,
281
],
[
267,
103
],
[
115,
111
],
[
353,
279
],
[
275,
109
],
[
109,
335
],
[
102,
105
],
[
100,
465
],
[
452,
261
],
[
116,
269
],
[
331,
364
],
[
115,
119
],
[
109,
317
],
[
103,
32
],
[
291,
32
],
[
228,
184
],
[
120,
32
],
[
97,
106
],
[
227,
128
],
[
112,
97
],
[
485,
259
],
[
286,
80
],
[
118,
504
],
[
109,
111
],
[
556,
140
],
[
260,
257
],
[
97,
326
],
[
368,
32
],
[
326,
388
],
[
293,
287
],
[
286,
67
],
[
273,
266
],
[
414,
281
],
[
288,
328
],
[
112,
274
],
[
101,
46
],
[
265,
350
],
[
348,
109
],
[
323,
291
],
[
263,
468
],
[
257,
330
],
[
122,
32
],
[
108,
283
],
[
104,
101
],
[
444,
344
],
[
412,
378
],
[
319,
102
],
[
271,
183
],
[
209,
132
],
[
118,
306
],
[
99,
267
],
[
525,
96
],
[
356,
646
],
[
107,
311
],
[
289,
177
],
[
286,
72
],
[
280,
279
],
[
97,
117
],
[
367,
615
],
[
302,
104
],
[
457,
278
],
[
259,
328
],
[
119,
489
],
[
386,
279
],
[
274,
256
],
[
119,
723
],
[
115,
291
],
[
102,
108
],
[
99,
268
],
[
322,
259
],
[
209,
141
],
[
112,
387
],
[
58,
10
],
[
99,
334
],
[
386,
278
],
[
307,
287
],
[
289,
189
],
[
115,
121
],
[
32,
40
],
[
488,
488
],
[
115,
266
],
[
111,
611
],
[
102,
302
],
[
357,
103
],
[
297,
347
],
[
110,
429
],
[
99,
374
],
[
316,
266
],
[
260,
114
],
[
114,
284
],
[
632,
279
],
[
357,
495
],
[
268,
443
],
[
109,
267
],
[
373,
555
],
[
115,
307
],
[
117,
275
],
[
32,
345
],
[
317,
114
],
[
304,
338
],
[
112,
265
],
[
109,
398
],
[
283,
303
],
[
321,
256
],
[
116,
321
],
[
417,
605
],
[
267,
116
],
[
101,
288
],
[
514,
638
],
[
375,
406
],
[
302,
413
],
[
273,
256
],
[
117,
256
],
[
114,
324
],
[
112,
104
],
[
390,
360
],
[
118,
276
],
[
283,
288
],
[
115,
651
],
[
103,
105
],
[
41,
59
],
[
436,
551
],
[
316,
104
],
[
263,
100
],
[
517,
364
],
[
314,
300
],
[
99,
269
],
[
368,
503
],
[
99,
276
],
[
110,
402
],
[
50,
48
],
[
32,
280
],
[
273,
115
],
[
284,
108
],
[
265,
103
],
[
314,
98
],
[
276,
103
],
[
100,
306
],
[
97,
263
],
[
73,
110
],
[
791,
357
],
[
121,
262
],
[
119,
296
],
[
108,
263
],
[
99,
347
],
[
98,
111
],
[
362,
586
],
[
298,
359
],
[
285,
32
],
[
110,
318
],
[
99,
489
],
[
606,
260
],
[
349,
272
],
[
298,
345
],
[
296,
288
],
[
269,
105
],
[
104,
337
],
[
348,
323
],
[
263,
266
],
[
115,
46
],
[
630,
477
],
[
582,
565
],
[
358,
264
],
[
350,
419
],
[
412,
802
],
[
101,
106
],
[
51,
32
],
[
460,
277
],
[
285,
257
],
[
116,
306
],
[
554,
266
],
[
314,
274
],
[
260,
101
],
[
109,
260
],
[
107,
257
],
[
100,
114
],
[
566,
354
],
[
458,
319
],
[
323,
256
],
[
286,
76
],
[
268,
277
],
[
121,
112
],
[
109,
297
],
[
522,
692
],
[
285,
103
],
[
401,
281
],
[
381,
110
],
[
118,
284
],
[
101,
267
],
[
278,
353
],
[
116,
111
],
[
98,
321
],
[
50,
32
],
[
49,
32
],
[
112,
297
],
[
104,
296
],
[
99,
371
],
[
318,
32
],
[
286,
68
],
[
276,
587
],
[
116,
398
],
[
102,
334
],
[
391,
107
],
[
372,
258
],
[
318,
588
],
[
289,
183
],
[
115,
306
],
[
321,
264
],
[
115,
347
],
[
35,
32
],
[
425,
277
],
[
406,
447
],
[
261,
340
],
[
111,
102
],
[
822,
296
],
[
325,
259
],
[
115,
333
],
[
112,
398
],
[
45,
52
],
[
41,
10
],
[
561,
293
],
[
337,
616
],
[
334,
589
],
[
296,
277
],
[
280,
281
],
[
275,
265
],
[
256,
330
],
[
47,
47
],
[
420,
359
],
[
305,
277
],
[
278,
505
],
[
227,
130
],
[
107,
110
],
[
297,
311
],
[
257,
392
],
[
116,
327
],
[
312,
105
],
[
291,
336
],
[
265,
99
],
[
261,
277
],
[
835,
122
],
[
772,
300
],
[
271,
179
],
[
256,
287
],
[
67,
104
],
[
741,
519
],
[
300,
101
],
[
365,
32
],
[
111,
355
],
[
523,
256
],
[
480,
424
],
[
194,
191
],
[
105,
98
],
[
100,
259
],
[
48,
32
],
[
279,
353
],
[
272,
292
],
[
32,
765
],
[
647,
322
],
[
471,
515
],
[
396,
267
],
[
387,
729
],
[
97,
512
],
[
45,
50
],
[
286,
70
],
[
280,
414
],
[
263,
476
],
[
110,
291
],
[
104,
521
],
[
327,
323
],
[
279,
432
],
[
115,
370
],
[
99,
97
],
[
335,
266
],
[
267,
100
],
[
228,
190
],
[
101,
395
],
[
100,
274
],
[
713,
130
],
[
366,
543
],
[
301,
363
],
[
231,
154
],
[
976,
132
],
[
115,
361
],
[
10,
453
],
[
824,
98
],
[
439,
105
],
[
324,
257
],
[
283,
330
],
[
265,
264
],
[
108,
117
],
[
104,
322
],
[
627,
99
],
[
449,
32
],
[
377,
258
],
[
311,
259
],
[
276,
283
],
[
63,
32
],
[
706,
324
],
[
618,
492
],
[
323,
268
],
[
290,
103
],
[
106,
117
],
[
61,
61
],
[
386,
281
],
[
341,
259
],
[
319,
446
],
[
276,
470
],
[
118,
101
],
[
49,
57
],
[
34,
264
],
[
689,
352
],
[
633,
288
],
[
324,
121
],
[
278,
349
],
[
273,
355
],
[
116,
356
],
[
105,
260
],
[
89,
843
],
[
339,
384
],
[
319,
115
],
[
312,
270
],
[
267,
277
],
[
390,
281
],
[
380,
542
]
]
}