Hugging Face's logo

flexitok
/
mod-tokenizers-rtl_2digit

Undetermined
tokenizer
bpe
flexitok
fineweb2
Model card Files
xet
Community
gsaltintas commited on
Commit
14f88f7
·
verified ·
1 Parent(s): 7d273d1

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. README.md +1 -1
  2. merges.txt +90 -0
  3. tokenizer.json +464 -840
  4. tokenizer_config.json +0 -744
  5. vocab.json +103 -103
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `1234500119 mod 67` | `12, 34, 50, 0, 11, 9, , mod, , 67` | `60, 37, 92, 8, 12, 62, 6, 4, 6, 68` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `123500119 mod 67` | `1, 23, 50, 0, 1, 19, , mod, , 67` | `8, 30, 57, 7, 8, 26, 6, 4, 6, 74` |
merges.txt CHANGED
@@ -1 +1,91 @@
1
  #version: 0.2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  #version: 0.2
2
+ ['1', '0']
3
+ ['1', '1']
4
+ ['1', '2']
5
+ ['1', '3']
6
+ ['1', '4']
7
+ ['1', '5']
8
+ ['1', '6']
9
+ ['1', '7']
10
+ ['1', '8']
11
+ ['1', '9']
12
+ ['2', '0']
13
+ ['2', '1']
14
+ ['2', '2']
15
+ ['2', '3']
16
+ ['2', '4']
17
+ ['2', '5']
18
+ ['2', '6']
19
+ ['2', '7']
20
+ ['2', '8']
21
+ ['2', '9']
22
+ ['3', '0']
23
+ ['3', '1']
24
+ ['3', '2']
25
+ ['3', '3']
26
+ ['3', '4']
27
+ ['3', '5']
28
+ ['3', '6']
29
+ ['3', '7']
30
+ ['3', '8']
31
+ ['3', '9']
32
+ ['4', '0']
33
+ ['4', '1']
34
+ ['4', '2']
35
+ ['4', '3']
36
+ ['4', '4']
37
+ ['4', '5']
38
+ ['4', '6']
39
+ ['4', '7']
40
+ ['4', '8']
41
+ ['4', '9']
42
+ ['5', '0']
43
+ ['5', '1']
44
+ ['5', '2']
45
+ ['5', '3']
46
+ ['5', '4']
47
+ ['5', '5']
48
+ ['5', '6']
49
+ ['5', '7']
50
+ ['5', '8']
51
+ ['5', '9']
52
+ ['6', '0']
53
+ ['6', '1']
54
+ ['6', '2']
55
+ ['6', '3']
56
+ ['6', '4']
57
+ ['6', '5']
58
+ ['6', '6']
59
+ ['6', '7']
60
+ ['6', '8']
61
+ ['6', '9']
62
+ ['7', '0']
63
+ ['7', '1']
64
+ ['7', '2']
65
+ ['7', '3']
66
+ ['7', '4']
67
+ ['7', '5']
68
+ ['7', '6']
69
+ ['7', '7']
70
+ ['7', '8']
71
+ ['7', '9']
72
+ ['8', '0']
73
+ ['8', '1']
74
+ ['8', '2']
75
+ ['8', '3']
76
+ ['8', '4']
77
+ ['8', '5']
78
+ ['8', '6']
79
+ ['8', '7']
80
+ ['8', '8']
81
+ ['8', '9']
82
+ ['9', '0']
83
+ ['9', '1']
84
+ ['9', '2']
85
+ ['9', '3']
86
+ ['9', '4']
87
+ ['9', '5']
88
+ ['9', '6']
89
+ ['9', '7']
90
+ ['9', '8']
91
+ ['9', '9']
tokenizer.json CHANGED
@@ -65,843 +65,6 @@
65
  "rstrip": false,
66
  "normalized": false,
67
  "special": true
68
- },
69
- {
70
- "id": 7,
71
- "content": "10",
72
- "single_word": false,
73
- "lstrip": false,
74
- "rstrip": false,
75
- "normalized": true,
76
- "special": false
77
- },
78
- {
79
- "id": 8,
80
- "content": "0",
81
- "single_word": false,
82
- "lstrip": false,
83
- "rstrip": false,
84
- "normalized": true,
85
- "special": false
86
- },
87
- {
88
- "id": 9,
89
- "content": "2",
90
- "single_word": false,
91
- "lstrip": false,
92
- "rstrip": false,
93
- "normalized": true,
94
- "special": false
95
- },
96
- {
97
- "id": 10,
98
- "content": "78",
99
- "single_word": false,
100
- "lstrip": false,
101
- "rstrip": false,
102
- "normalized": true,
103
- "special": false
104
- },
105
- {
106
- "id": 11,
107
- "content": "6",
108
- "single_word": false,
109
- "lstrip": false,
110
- "rstrip": false,
111
- "normalized": true,
112
- "special": false
113
- },
114
- {
115
- "id": 12,
116
- "content": "11",
117
- "single_word": false,
118
- "lstrip": false,
119
- "rstrip": false,
120
- "normalized": true,
121
- "special": false
122
- },
123
- {
124
- "id": 13,
125
- "content": "7",
126
- "single_word": false,
127
- "lstrip": false,
128
- "rstrip": false,
129
- "normalized": true,
130
- "special": false
131
- },
132
- {
133
- "id": 14,
134
- "content": "91",
135
- "single_word": false,
136
- "lstrip": false,
137
- "rstrip": false,
138
- "normalized": true,
139
- "special": false
140
- },
141
- {
142
- "id": 15,
143
- "content": "48",
144
- "single_word": false,
145
- "lstrip": false,
146
- "rstrip": false,
147
- "normalized": true,
148
- "special": false
149
- },
150
- {
151
- "id": 16,
152
- "content": "37",
153
- "single_word": false,
154
- "lstrip": false,
155
- "rstrip": false,
156
- "normalized": true,
157
- "special": false
158
- },
159
- {
160
- "id": 17,
161
- "content": "60",
162
- "single_word": false,
163
- "lstrip": false,
164
- "rstrip": false,
165
- "normalized": true,
166
- "special": false
167
- },
168
- {
169
- "id": 18,
170
- "content": "57",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": false,
174
- "normalized": true,
175
- "special": false
176
- },
177
- {
178
- "id": 19,
179
- "content": "44",
180
- "single_word": false,
181
- "lstrip": false,
182
- "rstrip": false,
183
- "normalized": true,
184
- "special": false
185
- },
186
- {
187
- "id": 20,
188
- "content": "96",
189
- "single_word": false,
190
- "lstrip": false,
191
- "rstrip": false,
192
- "normalized": true,
193
- "special": false
194
- },
195
- {
196
- "id": 21,
197
- "content": "69",
198
- "single_word": false,
199
- "lstrip": false,
200
- "rstrip": false,
201
- "normalized": true,
202
- "special": false
203
- },
204
- {
205
- "id": 22,
206
- "content": "80",
207
- "single_word": false,
208
- "lstrip": false,
209
- "rstrip": false,
210
- "normalized": true,
211
- "special": false
212
- },
213
- {
214
- "id": 23,
215
- "content": "61",
216
- "single_word": false,
217
- "lstrip": false,
218
- "rstrip": false,
219
- "normalized": true,
220
- "special": false
221
- },
222
- {
223
- "id": 24,
224
- "content": "13",
225
- "single_word": false,
226
- "lstrip": false,
227
- "rstrip": false,
228
- "normalized": true,
229
- "special": false
230
- },
231
- {
232
- "id": 25,
233
- "content": "55",
234
- "single_word": false,
235
- "lstrip": false,
236
- "rstrip": false,
237
- "normalized": true,
238
- "special": false
239
- },
240
- {
241
- "id": 26,
242
- "content": "8",
243
- "single_word": false,
244
- "lstrip": false,
245
- "rstrip": false,
246
- "normalized": true,
247
- "special": false
248
- },
249
- {
250
- "id": 27,
251
- "content": "51",
252
- "single_word": false,
253
- "lstrip": false,
254
- "rstrip": false,
255
- "normalized": true,
256
- "special": false
257
- },
258
- {
259
- "id": 28,
260
- "content": "41",
261
- "single_word": false,
262
- "lstrip": false,
263
- "rstrip": false,
264
- "normalized": true,
265
- "special": false
266
- },
267
- {
268
- "id": 29,
269
- "content": "18",
270
- "single_word": false,
271
- "lstrip": false,
272
- "rstrip": false,
273
- "normalized": true,
274
- "special": false
275
- },
276
- {
277
- "id": 30,
278
- "content": "25",
279
- "single_word": false,
280
- "lstrip": false,
281
- "rstrip": false,
282
- "normalized": true,
283
- "special": false
284
- },
285
- {
286
- "id": 31,
287
- "content": "33",
288
- "single_word": false,
289
- "lstrip": false,
290
- "rstrip": false,
291
- "normalized": true,
292
- "special": false
293
- },
294
- {
295
- "id": 32,
296
- "content": "82",
297
- "single_word": false,
298
- "lstrip": false,
299
- "rstrip": false,
300
- "normalized": true,
301
- "special": false
302
- },
303
- {
304
- "id": 33,
305
- "content": "85",
306
- "single_word": false,
307
- "lstrip": false,
308
- "rstrip": false,
309
- "normalized": true,
310
- "special": false
311
- },
312
- {
313
- "id": 34,
314
- "content": "54",
315
- "single_word": false,
316
- "lstrip": false,
317
- "rstrip": false,
318
- "normalized": true,
319
- "special": false
320
- },
321
- {
322
- "id": 35,
323
- "content": "81",
324
- "single_word": false,
325
- "lstrip": false,
326
- "rstrip": false,
327
- "normalized": true,
328
- "special": false
329
- },
330
- {
331
- "id": 36,
332
- "content": "65",
333
- "single_word": false,
334
- "lstrip": false,
335
- "rstrip": false,
336
- "normalized": true,
337
- "special": false
338
- },
339
- {
340
- "id": 37,
341
- "content": "34",
342
- "single_word": false,
343
- "lstrip": false,
344
- "rstrip": false,
345
- "normalized": true,
346
- "special": false
347
- },
348
- {
349
- "id": 38,
350
- "content": "58",
351
- "single_word": false,
352
- "lstrip": false,
353
- "rstrip": false,
354
- "normalized": true,
355
- "special": false
356
- },
357
- {
358
- "id": 39,
359
- "content": "21",
360
- "single_word": false,
361
- "lstrip": false,
362
- "rstrip": false,
363
- "normalized": true,
364
- "special": false
365
- },
366
- {
367
- "id": 40,
368
- "content": "32",
369
- "single_word": false,
370
- "lstrip": false,
371
- "rstrip": false,
372
- "normalized": true,
373
- "special": false
374
- },
375
- {
376
- "id": 41,
377
- "content": "4",
378
- "single_word": false,
379
- "lstrip": false,
380
- "rstrip": false,
381
- "normalized": true,
382
- "special": false
383
- },
384
- {
385
- "id": 42,
386
- "content": "39",
387
- "single_word": false,
388
- "lstrip": false,
389
- "rstrip": false,
390
- "normalized": true,
391
- "special": false
392
- },
393
- {
394
- "id": 43,
395
- "content": "98",
396
- "single_word": false,
397
- "lstrip": false,
398
- "rstrip": false,
399
- "normalized": true,
400
- "special": false
401
- },
402
- {
403
- "id": 44,
404
- "content": "26",
405
- "single_word": false,
406
- "lstrip": false,
407
- "rstrip": false,
408
- "normalized": true,
409
- "special": false
410
- },
411
- {
412
- "id": 45,
413
- "content": "31",
414
- "single_word": false,
415
- "lstrip": false,
416
- "rstrip": false,
417
- "normalized": true,
418
- "special": false
419
- },
420
- {
421
- "id": 46,
422
- "content": "36",
423
- "single_word": false,
424
- "lstrip": false,
425
- "rstrip": false,
426
- "normalized": true,
427
- "special": false
428
- },
429
- {
430
- "id": 47,
431
- "content": "92",
432
- "single_word": false,
433
- "lstrip": false,
434
- "rstrip": false,
435
- "normalized": true,
436
- "special": false
437
- },
438
- {
439
- "id": 48,
440
- "content": "84",
441
- "single_word": false,
442
- "lstrip": false,
443
- "rstrip": false,
444
- "normalized": true,
445
- "special": false
446
- },
447
- {
448
- "id": 49,
449
- "content": "99",
450
- "single_word": false,
451
- "lstrip": false,
452
- "rstrip": false,
453
- "normalized": true,
454
- "special": false
455
- },
456
- {
457
- "id": 50,
458
- "content": "19",
459
- "single_word": false,
460
- "lstrip": false,
461
- "rstrip": false,
462
- "normalized": true,
463
- "special": false
464
- },
465
- {
466
- "id": 51,
467
- "content": "64",
468
- "single_word": false,
469
- "lstrip": false,
470
- "rstrip": false,
471
- "normalized": true,
472
- "special": false
473
- },
474
- {
475
- "id": 52,
476
- "content": "73",
477
- "single_word": false,
478
- "lstrip": false,
479
- "rstrip": false,
480
- "normalized": true,
481
- "special": false
482
- },
483
- {
484
- "id": 53,
485
- "content": "14",
486
- "single_word": false,
487
- "lstrip": false,
488
- "rstrip": false,
489
- "normalized": true,
490
- "special": false
491
- },
492
- {
493
- "id": 54,
494
- "content": "42",
495
- "single_word": false,
496
- "lstrip": false,
497
- "rstrip": false,
498
- "normalized": true,
499
- "special": false
500
- },
501
- {
502
- "id": 55,
503
- "content": "38",
504
- "single_word": false,
505
- "lstrip": false,
506
- "rstrip": false,
507
- "normalized": true,
508
- "special": false
509
- },
510
- {
511
- "id": 56,
512
- "content": "79",
513
- "single_word": false,
514
- "lstrip": false,
515
- "rstrip": false,
516
- "normalized": true,
517
- "special": false
518
- },
519
- {
520
- "id": 57,
521
- "content": "87",
522
- "single_word": false,
523
- "lstrip": false,
524
- "rstrip": false,
525
- "normalized": true,
526
- "special": false
527
- },
528
- {
529
- "id": 58,
530
- "content": "30",
531
- "single_word": false,
532
- "lstrip": false,
533
- "rstrip": false,
534
- "normalized": true,
535
- "special": false
536
- },
537
- {
538
- "id": 59,
539
- "content": "94",
540
- "single_word": false,
541
- "lstrip": false,
542
- "rstrip": false,
543
- "normalized": true,
544
- "special": false
545
- },
546
- {
547
- "id": 60,
548
- "content": "12",
549
- "single_word": false,
550
- "lstrip": false,
551
- "rstrip": false,
552
- "normalized": true,
553
- "special": false
554
- },
555
- {
556
- "id": 61,
557
- "content": "15",
558
- "single_word": false,
559
- "lstrip": false,
560
- "rstrip": false,
561
- "normalized": true,
562
- "special": false
563
- },
564
- {
565
- "id": 62,
566
- "content": "9",
567
- "single_word": false,
568
- "lstrip": false,
569
- "rstrip": false,
570
- "normalized": true,
571
- "special": false
572
- },
573
- {
574
- "id": 63,
575
- "content": "23",
576
- "single_word": false,
577
- "lstrip": false,
578
- "rstrip": false,
579
- "normalized": true,
580
- "special": false
581
- },
582
- {
583
- "id": 64,
584
- "content": "70",
585
- "single_word": false,
586
- "lstrip": false,
587
- "rstrip": false,
588
- "normalized": true,
589
- "special": false
590
- },
591
- {
592
- "id": 65,
593
- "content": "77",
594
- "single_word": false,
595
- "lstrip": false,
596
- "rstrip": false,
597
- "normalized": true,
598
- "special": false
599
- },
600
- {
601
- "id": 66,
602
- "content": "74",
603
- "single_word": false,
604
- "lstrip": false,
605
- "rstrip": false,
606
- "normalized": true,
607
- "special": false
608
- },
609
- {
610
- "id": 67,
611
- "content": "75",
612
- "single_word": false,
613
- "lstrip": false,
614
- "rstrip": false,
615
- "normalized": true,
616
- "special": false
617
- },
618
- {
619
- "id": 68,
620
- "content": "67",
621
- "single_word": false,
622
- "lstrip": false,
623
- "rstrip": false,
624
- "normalized": true,
625
- "special": false
626
- },
627
- {
628
- "id": 69,
629
- "content": "45",
630
- "single_word": false,
631
- "lstrip": false,
632
- "rstrip": false,
633
- "normalized": true,
634
- "special": false
635
- },
636
- {
637
- "id": 70,
638
- "content": "83",
639
- "single_word": false,
640
- "lstrip": false,
641
- "rstrip": false,
642
- "normalized": true,
643
- "special": false
644
- },
645
- {
646
- "id": 71,
647
- "content": "63",
648
- "single_word": false,
649
- "lstrip": false,
650
- "rstrip": false,
651
- "normalized": true,
652
- "special": false
653
- },
654
- {
655
- "id": 72,
656
- "content": "40",
657
- "single_word": false,
658
- "lstrip": false,
659
- "rstrip": false,
660
- "normalized": true,
661
- "special": false
662
- },
663
- {
664
- "id": 73,
665
- "content": "22",
666
- "single_word": false,
667
- "lstrip": false,
668
- "rstrip": false,
669
- "normalized": true,
670
- "special": false
671
- },
672
- {
673
- "id": 74,
674
- "content": "97",
675
- "single_word": false,
676
- "lstrip": false,
677
- "rstrip": false,
678
- "normalized": true,
679
- "special": false
680
- },
681
- {
682
- "id": 75,
683
- "content": "29",
684
- "single_word": false,
685
- "lstrip": false,
686
- "rstrip": false,
687
- "normalized": true,
688
- "special": false
689
- },
690
- {
691
- "id": 76,
692
- "content": "24",
693
- "single_word": false,
694
- "lstrip": false,
695
- "rstrip": false,
696
- "normalized": true,
697
- "special": false
698
- },
699
- {
700
- "id": 77,
701
- "content": "49",
702
- "single_word": false,
703
- "lstrip": false,
704
- "rstrip": false,
705
- "normalized": true,
706
- "special": false
707
- },
708
- {
709
- "id": 78,
710
- "content": "46",
711
- "single_word": false,
712
- "lstrip": false,
713
- "rstrip": false,
714
- "normalized": true,
715
- "special": false
716
- },
717
- {
718
- "id": 79,
719
- "content": "35",
720
- "single_word": false,
721
- "lstrip": false,
722
- "rstrip": false,
723
- "normalized": true,
724
- "special": false
725
- },
726
- {
727
- "id": 80,
728
- "content": "88",
729
- "single_word": false,
730
- "lstrip": false,
731
- "rstrip": false,
732
- "normalized": true,
733
- "special": false
734
- },
735
- {
736
- "id": 81,
737
- "content": "27",
738
- "single_word": false,
739
- "lstrip": false,
740
- "rstrip": false,
741
- "normalized": true,
742
- "special": false
743
- },
744
- {
745
- "id": 82,
746
- "content": "95",
747
- "single_word": false,
748
- "lstrip": false,
749
- "rstrip": false,
750
- "normalized": true,
751
- "special": false
752
- },
753
- {
754
- "id": 83,
755
- "content": "1",
756
- "single_word": false,
757
- "lstrip": false,
758
- "rstrip": false,
759
- "normalized": true,
760
- "special": false
761
- },
762
- {
763
- "id": 84,
764
- "content": "62",
765
- "single_word": false,
766
- "lstrip": false,
767
- "rstrip": false,
768
- "normalized": true,
769
- "special": false
770
- },
771
- {
772
- "id": 85,
773
- "content": "76",
774
- "single_word": false,
775
- "lstrip": false,
776
- "rstrip": false,
777
- "normalized": true,
778
- "special": false
779
- },
780
- {
781
- "id": 86,
782
- "content": "56",
783
- "single_word": false,
784
- "lstrip": false,
785
- "rstrip": false,
786
- "normalized": true,
787
- "special": false
788
- },
789
- {
790
- "id": 87,
791
- "content": "90",
792
- "single_word": false,
793
- "lstrip": false,
794
- "rstrip": false,
795
- "normalized": true,
796
- "special": false
797
- },
798
- {
799
- "id": 88,
800
- "content": "71",
801
- "single_word": false,
802
- "lstrip": false,
803
- "rstrip": false,
804
- "normalized": true,
805
- "special": false
806
- },
807
- {
808
- "id": 89,
809
- "content": "43",
810
- "single_word": false,
811
- "lstrip": false,
812
- "rstrip": false,
813
- "normalized": true,
814
- "special": false
815
- },
816
- {
817
- "id": 90,
818
- "content": "93",
819
- "single_word": false,
820
- "lstrip": false,
821
- "rstrip": false,
822
- "normalized": true,
823
- "special": false
824
- },
825
- {
826
- "id": 91,
827
- "content": "5",
828
- "single_word": false,
829
- "lstrip": false,
830
- "rstrip": false,
831
- "normalized": true,
832
- "special": false
833
- },
834
- {
835
- "id": 92,
836
- "content": "50",
837
- "single_word": false,
838
- "lstrip": false,
839
- "rstrip": false,
840
- "normalized": true,
841
- "special": false
842
- },
843
- {
844
- "id": 93,
845
- "content": "47",
846
- "single_word": false,
847
- "lstrip": false,
848
- "rstrip": false,
849
- "normalized": true,
850
- "special": false
851
- },
852
- {
853
- "id": 94,
854
- "content": "66",
855
- "single_word": false,
856
- "lstrip": false,
857
- "rstrip": false,
858
- "normalized": true,
859
- "special": false
860
- },
861
- {
862
- "id": 95,
863
- "content": "52",
864
- "single_word": false,
865
- "lstrip": false,
866
- "rstrip": false,
867
- "normalized": true,
868
- "special": false
869
- },
870
- {
871
- "id": 96,
872
- "content": "72",
873
- "single_word": false,
874
- "lstrip": false,
875
- "rstrip": false,
876
- "normalized": true,
877
- "special": false
878
- },
879
- {
880
- "id": 97,
881
- "content": "20",
882
- "single_word": false,
883
- "lstrip": false,
884
- "rstrip": false,
885
- "normalized": true,
886
- "special": false
887
- },
888
- {
889
- "id": 98,
890
- "content": "53",
891
- "single_word": false,
892
- "lstrip": false,
893
- "rstrip": false,
894
- "normalized": true,
895
- "special": false
896
- },
897
- {
898
- "id": 99,
899
- "content": "16",
900
- "single_word": false,
901
- "lstrip": false,
902
- "rstrip": false,
903
- "normalized": true,
904
- "special": false
905
  }
906
  ],
907
  "normalizer": null,
@@ -947,8 +110,469 @@
947
  "<pad>": 3,
948
  "mod": 4,
949
  "=": 5,
950
- " ": 6
951
- },
952
- "merges": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
953
  }
954
  }
 
65
  "rstrip": false,
66
  "normalized": false,
67
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  }
69
  ],
70
  "normalizer": null,
 
110
  "<pad>": 3,
111
  "mod": 4,
112
  "=": 5,
113
+ " ": 6,
114
+ "0": 7,
115
+ "1": 8,
116
+ "2": 9,
117
+ "3": 10,
118
+ "4": 11,
119
+ "5": 12,
120
+ "6": 13,
121
+ "7": 14,
122
+ "8": 15,
123
+ "9": 16,
124
+ "10": 17,
125
+ "11": 18,
126
+ "12": 19,
127
+ "13": 20,
128
+ "14": 21,
129
+ "15": 22,
130
+ "16": 23,
131
+ "17": 24,
132
+ "18": 25,
133
+ "19": 26,
134
+ "20": 27,
135
+ "21": 28,
136
+ "22": 29,
137
+ "23": 30,
138
+ "24": 31,
139
+ "25": 32,
140
+ "26": 33,
141
+ "27": 34,
142
+ "28": 35,
143
+ "29": 36,
144
+ "30": 37,
145
+ "31": 38,
146
+ "32": 39,
147
+ "33": 40,
148
+ "34": 41,
149
+ "35": 42,
150
+ "36": 43,
151
+ "37": 44,
152
+ "38": 45,
153
+ "39": 46,
154
+ "40": 47,
155
+ "41": 48,
156
+ "42": 49,
157
+ "43": 50,
158
+ "44": 51,
159
+ "45": 52,
160
+ "46": 53,
161
+ "47": 54,
162
+ "48": 55,
163
+ "49": 56,
164
+ "50": 57,
165
+ "51": 58,
166
+ "52": 59,
167
+ "53": 60,
168
+ "54": 61,
169
+ "55": 62,
170
+ "56": 63,
171
+ "57": 64,
172
+ "58": 65,
173
+ "59": 66,
174
+ "60": 67,
175
+ "61": 68,
176
+ "62": 69,
177
+ "63": 70,
178
+ "64": 71,
179
+ "65": 72,
180
+ "66": 73,
181
+ "67": 74,
182
+ "68": 75,
183
+ "69": 76,
184
+ "70": 77,
185
+ "71": 78,
186
+ "72": 79,
187
+ "73": 80,
188
+ "74": 81,
189
+ "75": 82,
190
+ "76": 83,
191
+ "77": 84,
192
+ "78": 85,
193
+ "79": 86,
194
+ "80": 87,
195
+ "81": 88,
196
+ "82": 89,
197
+ "83": 90,
198
+ "84": 91,
199
+ "85": 92,
200
+ "86": 93,
201
+ "87": 94,
202
+ "88": 95,
203
+ "89": 96,
204
+ "90": 97,
205
+ "91": 98,
206
+ "92": 99,
207
+ "93": 100,
208
+ "94": 101,
209
+ "95": 102,
210
+ "96": 103,
211
+ "97": 104,
212
+ "98": 105,
213
+ "99": 106
214
+ },
215
+ "merges": [
216
+ [
217
+ "1",
218
+ "0"
219
+ ],
220
+ [
221
+ "1",
222
+ "1"
223
+ ],
224
+ [
225
+ "1",
226
+ "2"
227
+ ],
228
+ [
229
+ "1",
230
+ "3"
231
+ ],
232
+ [
233
+ "1",
234
+ "4"
235
+ ],
236
+ [
237
+ "1",
238
+ "5"
239
+ ],
240
+ [
241
+ "1",
242
+ "6"
243
+ ],
244
+ [
245
+ "1",
246
+ "7"
247
+ ],
248
+ [
249
+ "1",
250
+ "8"
251
+ ],
252
+ [
253
+ "1",
254
+ "9"
255
+ ],
256
+ [
257
+ "2",
258
+ "0"
259
+ ],
260
+ [
261
+ "2",
262
+ "1"
263
+ ],
264
+ [
265
+ "2",
266
+ "2"
267
+ ],
268
+ [
269
+ "2",
270
+ "3"
271
+ ],
272
+ [
273
+ "2",
274
+ "4"
275
+ ],
276
+ [
277
+ "2",
278
+ "5"
279
+ ],
280
+ [
281
+ "2",
282
+ "6"
283
+ ],
284
+ [
285
+ "2",
286
+ "7"
287
+ ],
288
+ [
289
+ "2",
290
+ "8"
291
+ ],
292
+ [
293
+ "2",
294
+ "9"
295
+ ],
296
+ [
297
+ "3",
298
+ "0"
299
+ ],
300
+ [
301
+ "3",
302
+ "1"
303
+ ],
304
+ [
305
+ "3",
306
+ "2"
307
+ ],
308
+ [
309
+ "3",
310
+ "3"
311
+ ],
312
+ [
313
+ "3",
314
+ "4"
315
+ ],
316
+ [
317
+ "3",
318
+ "5"
319
+ ],
320
+ [
321
+ "3",
322
+ "6"
323
+ ],
324
+ [
325
+ "3",
326
+ "7"
327
+ ],
328
+ [
329
+ "3",
330
+ "8"
331
+ ],
332
+ [
333
+ "3",
334
+ "9"
335
+ ],
336
+ [
337
+ "4",
338
+ "0"
339
+ ],
340
+ [
341
+ "4",
342
+ "1"
343
+ ],
344
+ [
345
+ "4",
346
+ "2"
347
+ ],
348
+ [
349
+ "4",
350
+ "3"
351
+ ],
352
+ [
353
+ "4",
354
+ "4"
355
+ ],
356
+ [
357
+ "4",
358
+ "5"
359
+ ],
360
+ [
361
+ "4",
362
+ "6"
363
+ ],
364
+ [
365
+ "4",
366
+ "7"
367
+ ],
368
+ [
369
+ "4",
370
+ "8"
371
+ ],
372
+ [
373
+ "4",
374
+ "9"
375
+ ],
376
+ [
377
+ "5",
378
+ "0"
379
+ ],
380
+ [
381
+ "5",
382
+ "1"
383
+ ],
384
+ [
385
+ "5",
386
+ "2"
387
+ ],
388
+ [
389
+ "5",
390
+ "3"
391
+ ],
392
+ [
393
+ "5",
394
+ "4"
395
+ ],
396
+ [
397
+ "5",
398
+ "5"
399
+ ],
400
+ [
401
+ "5",
402
+ "6"
403
+ ],
404
+ [
405
+ "5",
406
+ "7"
407
+ ],
408
+ [
409
+ "5",
410
+ "8"
411
+ ],
412
+ [
413
+ "5",
414
+ "9"
415
+ ],
416
+ [
417
+ "6",
418
+ "0"
419
+ ],
420
+ [
421
+ "6",
422
+ "1"
423
+ ],
424
+ [
425
+ "6",
426
+ "2"
427
+ ],
428
+ [
429
+ "6",
430
+ "3"
431
+ ],
432
+ [
433
+ "6",
434
+ "4"
435
+ ],
436
+ [
437
+ "6",
438
+ "5"
439
+ ],
440
+ [
441
+ "6",
442
+ "6"
443
+ ],
444
+ [
445
+ "6",
446
+ "7"
447
+ ],
448
+ [
449
+ "6",
450
+ "8"
451
+ ],
452
+ [
453
+ "6",
454
+ "9"
455
+ ],
456
+ [
457
+ "7",
458
+ "0"
459
+ ],
460
+ [
461
+ "7",
462
+ "1"
463
+ ],
464
+ [
465
+ "7",
466
+ "2"
467
+ ],
468
+ [
469
+ "7",
470
+ "3"
471
+ ],
472
+ [
473
+ "7",
474
+ "4"
475
+ ],
476
+ [
477
+ "7",
478
+ "5"
479
+ ],
480
+ [
481
+ "7",
482
+ "6"
483
+ ],
484
+ [
485
+ "7",
486
+ "7"
487
+ ],
488
+ [
489
+ "7",
490
+ "8"
491
+ ],
492
+ [
493
+ "7",
494
+ "9"
495
+ ],
496
+ [
497
+ "8",
498
+ "0"
499
+ ],
500
+ [
501
+ "8",
502
+ "1"
503
+ ],
504
+ [
505
+ "8",
506
+ "2"
507
+ ],
508
+ [
509
+ "8",
510
+ "3"
511
+ ],
512
+ [
513
+ "8",
514
+ "4"
515
+ ],
516
+ [
517
+ "8",
518
+ "5"
519
+ ],
520
+ [
521
+ "8",
522
+ "6"
523
+ ],
524
+ [
525
+ "8",
526
+ "7"
527
+ ],
528
+ [
529
+ "8",
530
+ "8"
531
+ ],
532
+ [
533
+ "8",
534
+ "9"
535
+ ],
536
+ [
537
+ "9",
538
+ "0"
539
+ ],
540
+ [
541
+ "9",
542
+ "1"
543
+ ],
544
+ [
545
+ "9",
546
+ "2"
547
+ ],
548
+ [
549
+ "9",
550
+ "3"
551
+ ],
552
+ [
553
+ "9",
554
+ "4"
555
+ ],
556
+ [
557
+ "9",
558
+ "5"
559
+ ],
560
+ [
561
+ "9",
562
+ "6"
563
+ ],
564
+ [
565
+ "9",
566
+ "7"
567
+ ],
568
+ [
569
+ "9",
570
+ "8"
571
+ ],
572
+ [
573
+ "9",
574
+ "9"
575
+ ]
576
+ ]
577
  }
578
  }
tokenizer_config.json CHANGED
@@ -55,750 +55,6 @@
55
  "rstrip": false,
56
  "single_word": false,
57
  "special": true
58
- },
59
- "7": {
60
- "content": "10",
61
- "lstrip": false,
62
- "normalized": true,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": false
66
- },
67
- "8": {
68
- "content": "0",
69
- "lstrip": false,
70
- "normalized": true,
71
- "rstrip": false,
72
- "single_word": false,
73
- "special": false
74
- },
75
- "9": {
76
- "content": "2",
77
- "lstrip": false,
78
- "normalized": true,
79
- "rstrip": false,
80
- "single_word": false,
81
- "special": false
82
- },
83
- "10": {
84
- "content": "78",
85
- "lstrip": false,
86
- "normalized": true,
87
- "rstrip": false,
88
- "single_word": false,
89
- "special": false
90
- },
91
- "11": {
92
- "content": "6",
93
- "lstrip": false,
94
- "normalized": true,
95
- "rstrip": false,
96
- "single_word": false,
97
- "special": false
98
- },
99
- "12": {
100
- "content": "11",
101
- "lstrip": false,
102
- "normalized": true,
103
- "rstrip": false,
104
- "single_word": false,
105
- "special": false
106
- },
107
- "13": {
108
- "content": "7",
109
- "lstrip": false,
110
- "normalized": true,
111
- "rstrip": false,
112
- "single_word": false,
113
- "special": false
114
- },
115
- "14": {
116
- "content": "91",
117
- "lstrip": false,
118
- "normalized": true,
119
- "rstrip": false,
120
- "single_word": false,
121
- "special": false
122
- },
123
- "15": {
124
- "content": "48",
125
- "lstrip": false,
126
- "normalized": true,
127
- "rstrip": false,
128
- "single_word": false,
129
- "special": false
130
- },
131
- "16": {
132
- "content": "37",
133
- "lstrip": false,
134
- "normalized": true,
135
- "rstrip": false,
136
- "single_word": false,
137
- "special": false
138
- },
139
- "17": {
140
- "content": "60",
141
- "lstrip": false,
142
- "normalized": true,
143
- "rstrip": false,
144
- "single_word": false,
145
- "special": false
146
- },
147
- "18": {
148
- "content": "57",
149
- "lstrip": false,
150
- "normalized": true,
151
- "rstrip": false,
152
- "single_word": false,
153
- "special": false
154
- },
155
- "19": {
156
- "content": "44",
157
- "lstrip": false,
158
- "normalized": true,
159
- "rstrip": false,
160
- "single_word": false,
161
- "special": false
162
- },
163
- "20": {
164
- "content": "96",
165
- "lstrip": false,
166
- "normalized": true,
167
- "rstrip": false,
168
- "single_word": false,
169
- "special": false
170
- },
171
- "21": {
172
- "content": "69",
173
- "lstrip": false,
174
- "normalized": true,
175
- "rstrip": false,
176
- "single_word": false,
177
- "special": false
178
- },
179
- "22": {
180
- "content": "80",
181
- "lstrip": false,
182
- "normalized": true,
183
- "rstrip": false,
184
- "single_word": false,
185
- "special": false
186
- },
187
- "23": {
188
- "content": "61",
189
- "lstrip": false,
190
- "normalized": true,
191
- "rstrip": false,
192
- "single_word": false,
193
- "special": false
194
- },
195
- "24": {
196
- "content": "13",
197
- "lstrip": false,
198
- "normalized": true,
199
- "rstrip": false,
200
- "single_word": false,
201
- "special": false
202
- },
203
- "25": {
204
- "content": "55",
205
- "lstrip": false,
206
- "normalized": true,
207
- "rstrip": false,
208
- "single_word": false,
209
- "special": false
210
- },
211
- "26": {
212
- "content": "8",
213
- "lstrip": false,
214
- "normalized": true,
215
- "rstrip": false,
216
- "single_word": false,
217
- "special": false
218
- },
219
- "27": {
220
- "content": "51",
221
- "lstrip": false,
222
- "normalized": true,
223
- "rstrip": false,
224
- "single_word": false,
225
- "special": false
226
- },
227
- "28": {
228
- "content": "41",
229
- "lstrip": false,
230
- "normalized": true,
231
- "rstrip": false,
232
- "single_word": false,
233
- "special": false
234
- },
235
- "29": {
236
- "content": "18",
237
- "lstrip": false,
238
- "normalized": true,
239
- "rstrip": false,
240
- "single_word": false,
241
- "special": false
242
- },
243
- "30": {
244
- "content": "25",
245
- "lstrip": false,
246
- "normalized": true,
247
- "rstrip": false,
248
- "single_word": false,
249
- "special": false
250
- },
251
- "31": {
252
- "content": "33",
253
- "lstrip": false,
254
- "normalized": true,
255
- "rstrip": false,
256
- "single_word": false,
257
- "special": false
258
- },
259
- "32": {
260
- "content": "82",
261
- "lstrip": false,
262
- "normalized": true,
263
- "rstrip": false,
264
- "single_word": false,
265
- "special": false
266
- },
267
- "33": {
268
- "content": "85",
269
- "lstrip": false,
270
- "normalized": true,
271
- "rstrip": false,
272
- "single_word": false,
273
- "special": false
274
- },
275
- "34": {
276
- "content": "54",
277
- "lstrip": false,
278
- "normalized": true,
279
- "rstrip": false,
280
- "single_word": false,
281
- "special": false
282
- },
283
- "35": {
284
- "content": "81",
285
- "lstrip": false,
286
- "normalized": true,
287
- "rstrip": false,
288
- "single_word": false,
289
- "special": false
290
- },
291
- "36": {
292
- "content": "65",
293
- "lstrip": false,
294
- "normalized": true,
295
- "rstrip": false,
296
- "single_word": false,
297
- "special": false
298
- },
299
- "37": {
300
- "content": "34",
301
- "lstrip": false,
302
- "normalized": true,
303
- "rstrip": false,
304
- "single_word": false,
305
- "special": false
306
- },
307
- "38": {
308
- "content": "58",
309
- "lstrip": false,
310
- "normalized": true,
311
- "rstrip": false,
312
- "single_word": false,
313
- "special": false
314
- },
315
- "39": {
316
- "content": "21",
317
- "lstrip": false,
318
- "normalized": true,
319
- "rstrip": false,
320
- "single_word": false,
321
- "special": false
322
- },
323
- "40": {
324
- "content": "32",
325
- "lstrip": false,
326
- "normalized": true,
327
- "rstrip": false,
328
- "single_word": false,
329
- "special": false
330
- },
331
- "41": {
332
- "content": "4",
333
- "lstrip": false,
334
- "normalized": true,
335
- "rstrip": false,
336
- "single_word": false,
337
- "special": false
338
- },
339
- "42": {
340
- "content": "39",
341
- "lstrip": false,
342
- "normalized": true,
343
- "rstrip": false,
344
- "single_word": false,
345
- "special": false
346
- },
347
- "43": {
348
- "content": "98",
349
- "lstrip": false,
350
- "normalized": true,
351
- "rstrip": false,
352
- "single_word": false,
353
- "special": false
354
- },
355
- "44": {
356
- "content": "26",
357
- "lstrip": false,
358
- "normalized": true,
359
- "rstrip": false,
360
- "single_word": false,
361
- "special": false
362
- },
363
- "45": {
364
- "content": "31",
365
- "lstrip": false,
366
- "normalized": true,
367
- "rstrip": false,
368
- "single_word": false,
369
- "special": false
370
- },
371
- "46": {
372
- "content": "36",
373
- "lstrip": false,
374
- "normalized": true,
375
- "rstrip": false,
376
- "single_word": false,
377
- "special": false
378
- },
379
- "47": {
380
- "content": "92",
381
- "lstrip": false,
382
- "normalized": true,
383
- "rstrip": false,
384
- "single_word": false,
385
- "special": false
386
- },
387
- "48": {
388
- "content": "84",
389
- "lstrip": false,
390
- "normalized": true,
391
- "rstrip": false,
392
- "single_word": false,
393
- "special": false
394
- },
395
- "49": {
396
- "content": "99",
397
- "lstrip": false,
398
- "normalized": true,
399
- "rstrip": false,
400
- "single_word": false,
401
- "special": false
402
- },
403
- "50": {
404
- "content": "19",
405
- "lstrip": false,
406
- "normalized": true,
407
- "rstrip": false,
408
- "single_word": false,
409
- "special": false
410
- },
411
- "51": {
412
- "content": "64",
413
- "lstrip": false,
414
- "normalized": true,
415
- "rstrip": false,
416
- "single_word": false,
417
- "special": false
418
- },
419
- "52": {
420
- "content": "73",
421
- "lstrip": false,
422
- "normalized": true,
423
- "rstrip": false,
424
- "single_word": false,
425
- "special": false
426
- },
427
- "53": {
428
- "content": "14",
429
- "lstrip": false,
430
- "normalized": true,
431
- "rstrip": false,
432
- "single_word": false,
433
- "special": false
434
- },
435
- "54": {
436
- "content": "42",
437
- "lstrip": false,
438
- "normalized": true,
439
- "rstrip": false,
440
- "single_word": false,
441
- "special": false
442
- },
443
- "55": {
444
- "content": "38",
445
- "lstrip": false,
446
- "normalized": true,
447
- "rstrip": false,
448
- "single_word": false,
449
- "special": false
450
- },
451
- "56": {
452
- "content": "79",
453
- "lstrip": false,
454
- "normalized": true,
455
- "rstrip": false,
456
- "single_word": false,
457
- "special": false
458
- },
459
- "57": {
460
- "content": "87",
461
- "lstrip": false,
462
- "normalized": true,
463
- "rstrip": false,
464
- "single_word": false,
465
- "special": false
466
- },
467
- "58": {
468
- "content": "30",
469
- "lstrip": false,
470
- "normalized": true,
471
- "rstrip": false,
472
- "single_word": false,
473
- "special": false
474
- },
475
- "59": {
476
- "content": "94",
477
- "lstrip": false,
478
- "normalized": true,
479
- "rstrip": false,
480
- "single_word": false,
481
- "special": false
482
- },
483
- "60": {
484
- "content": "12",
485
- "lstrip": false,
486
- "normalized": true,
487
- "rstrip": false,
488
- "single_word": false,
489
- "special": false
490
- },
491
- "61": {
492
- "content": "15",
493
- "lstrip": false,
494
- "normalized": true,
495
- "rstrip": false,
496
- "single_word": false,
497
- "special": false
498
- },
499
- "62": {
500
- "content": "9",
501
- "lstrip": false,
502
- "normalized": true,
503
- "rstrip": false,
504
- "single_word": false,
505
- "special": false
506
- },
507
- "63": {
508
- "content": "23",
509
- "lstrip": false,
510
- "normalized": true,
511
- "rstrip": false,
512
- "single_word": false,
513
- "special": false
514
- },
515
- "64": {
516
- "content": "70",
517
- "lstrip": false,
518
- "normalized": true,
519
- "rstrip": false,
520
- "single_word": false,
521
- "special": false
522
- },
523
- "65": {
524
- "content": "77",
525
- "lstrip": false,
526
- "normalized": true,
527
- "rstrip": false,
528
- "single_word": false,
529
- "special": false
530
- },
531
- "66": {
532
- "content": "74",
533
- "lstrip": false,
534
- "normalized": true,
535
- "rstrip": false,
536
- "single_word": false,
537
- "special": false
538
- },
539
- "67": {
540
- "content": "75",
541
- "lstrip": false,
542
- "normalized": true,
543
- "rstrip": false,
544
- "single_word": false,
545
- "special": false
546
- },
547
- "68": {
548
- "content": "67",
549
- "lstrip": false,
550
- "normalized": true,
551
- "rstrip": false,
552
- "single_word": false,
553
- "special": false
554
- },
555
- "69": {
556
- "content": "45",
557
- "lstrip": false,
558
- "normalized": true,
559
- "rstrip": false,
560
- "single_word": false,
561
- "special": false
562
- },
563
- "70": {
564
- "content": "83",
565
- "lstrip": false,
566
- "normalized": true,
567
- "rstrip": false,
568
- "single_word": false,
569
- "special": false
570
- },
571
- "71": {
572
- "content": "63",
573
- "lstrip": false,
574
- "normalized": true,
575
- "rstrip": false,
576
- "single_word": false,
577
- "special": false
578
- },
579
- "72": {
580
- "content": "40",
581
- "lstrip": false,
582
- "normalized": true,
583
- "rstrip": false,
584
- "single_word": false,
585
- "special": false
586
- },
587
- "73": {
588
- "content": "22",
589
- "lstrip": false,
590
- "normalized": true,
591
- "rstrip": false,
592
- "single_word": false,
593
- "special": false
594
- },
595
- "74": {
596
- "content": "97",
597
- "lstrip": false,
598
- "normalized": true,
599
- "rstrip": false,
600
- "single_word": false,
601
- "special": false
602
- },
603
- "75": {
604
- "content": "29",
605
- "lstrip": false,
606
- "normalized": true,
607
- "rstrip": false,
608
- "single_word": false,
609
- "special": false
610
- },
611
- "76": {
612
- "content": "24",
613
- "lstrip": false,
614
- "normalized": true,
615
- "rstrip": false,
616
- "single_word": false,
617
- "special": false
618
- },
619
- "77": {
620
- "content": "49",
621
- "lstrip": false,
622
- "normalized": true,
623
- "rstrip": false,
624
- "single_word": false,
625
- "special": false
626
- },
627
- "78": {
628
- "content": "46",
629
- "lstrip": false,
630
- "normalized": true,
631
- "rstrip": false,
632
- "single_word": false,
633
- "special": false
634
- },
635
- "79": {
636
- "content": "35",
637
- "lstrip": false,
638
- "normalized": true,
639
- "rstrip": false,
640
- "single_word": false,
641
- "special": false
642
- },
643
- "80": {
644
- "content": "88",
645
- "lstrip": false,
646
- "normalized": true,
647
- "rstrip": false,
648
- "single_word": false,
649
- "special": false
650
- },
651
- "81": {
652
- "content": "27",
653
- "lstrip": false,
654
- "normalized": true,
655
- "rstrip": false,
656
- "single_word": false,
657
- "special": false
658
- },
659
- "82": {
660
- "content": "95",
661
- "lstrip": false,
662
- "normalized": true,
663
- "rstrip": false,
664
- "single_word": false,
665
- "special": false
666
- },
667
- "83": {
668
- "content": "1",
669
- "lstrip": false,
670
- "normalized": true,
671
- "rstrip": false,
672
- "single_word": false,
673
- "special": false
674
- },
675
- "84": {
676
- "content": "62",
677
- "lstrip": false,
678
- "normalized": true,
679
- "rstrip": false,
680
- "single_word": false,
681
- "special": false
682
- },
683
- "85": {
684
- "content": "76",
685
- "lstrip": false,
686
- "normalized": true,
687
- "rstrip": false,
688
- "single_word": false,
689
- "special": false
690
- },
691
- "86": {
692
- "content": "56",
693
- "lstrip": false,
694
- "normalized": true,
695
- "rstrip": false,
696
- "single_word": false,
697
- "special": false
698
- },
699
- "87": {
700
- "content": "90",
701
- "lstrip": false,
702
- "normalized": true,
703
- "rstrip": false,
704
- "single_word": false,
705
- "special": false
706
- },
707
- "88": {
708
- "content": "71",
709
- "lstrip": false,
710
- "normalized": true,
711
- "rstrip": false,
712
- "single_word": false,
713
- "special": false
714
- },
715
- "89": {
716
- "content": "43",
717
- "lstrip": false,
718
- "normalized": true,
719
- "rstrip": false,
720
- "single_word": false,
721
- "special": false
722
- },
723
- "90": {
724
- "content": "93",
725
- "lstrip": false,
726
- "normalized": true,
727
- "rstrip": false,
728
- "single_word": false,
729
- "special": false
730
- },
731
- "91": {
732
- "content": "5",
733
- "lstrip": false,
734
- "normalized": true,
735
- "rstrip": false,
736
- "single_word": false,
737
- "special": false
738
- },
739
- "92": {
740
- "content": "50",
741
- "lstrip": false,
742
- "normalized": true,
743
- "rstrip": false,
744
- "single_word": false,
745
- "special": false
746
- },
747
- "93": {
748
- "content": "47",
749
- "lstrip": false,
750
- "normalized": true,
751
- "rstrip": false,
752
- "single_word": false,
753
- "special": false
754
- },
755
- "94": {
756
- "content": "66",
757
- "lstrip": false,
758
- "normalized": true,
759
- "rstrip": false,
760
- "single_word": false,
761
- "special": false
762
- },
763
- "95": {
764
- "content": "52",
765
- "lstrip": false,
766
- "normalized": true,
767
- "rstrip": false,
768
- "single_word": false,
769
- "special": false
770
- },
771
- "96": {
772
- "content": "72",
773
- "lstrip": false,
774
- "normalized": true,
775
- "rstrip": false,
776
- "single_word": false,
777
- "special": false
778
- },
779
- "97": {
780
- "content": "20",
781
- "lstrip": false,
782
- "normalized": true,
783
- "rstrip": false,
784
- "single_word": false,
785
- "special": false
786
- },
787
- "98": {
788
- "content": "53",
789
- "lstrip": false,
790
- "normalized": true,
791
- "rstrip": false,
792
- "single_word": false,
793
- "special": false
794
- },
795
- "99": {
796
- "content": "16",
797
- "lstrip": false,
798
- "normalized": true,
799
- "rstrip": false,
800
- "single_word": false,
801
- "special": false
802
  }
803
  },
804
  "additional_special_tokens": [
 
55
  "rstrip": false,
56
  "single_word": false,
57
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  }
59
  },
60
  "additional_special_tokens": [
vocab.json CHANGED
@@ -1,109 +1,109 @@
1
  {
2
- "89": 2,
3
- "13": 24,
4
- "63": 71,
5
- "59": 5,
6
- "56": 86,
7
- "48": 15,
8
- "61": 23,
9
- "62": 84,
10
- "27": 81,
11
- "91": 14,
12
- "16": 99,
13
- "17": 0,
14
- "67": 68,
15
- "0": 8,
16
- "93": 90,
17
- "64": 51,
18
- "77": 65,
19
- "<pad>": 3,
20
- "28": 6,
21
- "5": 91,
22
- "55": 25,
23
- "25": 30,
24
- "21": 39,
25
- "95": 82,
26
- "84": 48,
27
- "65": 36,
28
- "73": 52,
29
- "35": 79,
30
- "31": 45,
31
- "39": 42,
32
- "37": 16,
33
- "10": 7,
34
- "24": 76,
35
- "12": 60,
36
- "2": 9,
37
- "14": 53,
38
- "92": 47,
39
- "44": 19,
40
- "43": 89,
41
- "47": 93,
42
- "79": 56,
43
- "9": 62,
44
- "45": 69,
45
- "38": 55,
46
- "23": 63,
47
- "=": 5,
48
- "22": 73,
49
- "66": 94,
50
- "72": 96,
51
- "7": 13,
52
- "75": 67,
53
- "3": 1,
54
- "34": 37,
55
- "82": 32,
56
- "69": 21,
57
- "30": 58,
58
  "mod": 4,
59
- "53": 98,
60
- "15": 61,
61
- "54": 34,
62
- "49": 77,
63
- "11": 12,
64
- "29": 75,
65
- "98": 43,
66
- "8": 26,
67
- "<unk>": 0,
68
- "46": 78,
69
- "58": 38,
70
- "68": 4,
71
- "18": 29,
72
- "26": 44,
73
- "97": 74,
74
- "6": 11,
75
- "36": 46,
76
- "33": 31,
77
- "81": 35,
78
- "32": 40,
79
- "85": 33,
80
- "76": 85,
81
- "52": 95,
82
  "<s>": 1,
83
- "40": 72,
84
- "94": 59,
85
- "20": 97,
86
- "1": 83,
87
- "87": 57,
88
- "71": 88,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  " ": 6,
90
- "78": 10,
91
- "60": 17,
92
- "88": 80,
93
- "99": 49,
94
- "80": 22,
95
- "41": 28,
96
- "42": 54,
97
- "83": 70,
98
- "50": 92,
99
- "96": 20,
100
- "74": 66,
101
- "86": 3,
102
- "57": 18,
103
- "51": 27,
104
- "70": 64,
105
- "90": 87,
106
  "</s>": 2,
107
- "4": 41,
108
- "19": 50
 
 
 
 
 
 
 
 
 
109
  }
 
1
  {
2
+ "5": 12,
3
+ "75": 82,
4
+ "60": 67,
5
+ "41": 48,
6
+ "49": 56,
7
+ "27": 34,
8
+ "6": 13,
9
+ "31": 38,
10
+ "65": 72,
11
+ "87": 94,
12
+ "0": 7,
13
+ "28": 35,
14
+ "22": 29,
15
+ "20": 27,
16
+ "3": 10,
17
+ "30": 37,
18
+ "63": 70,
19
+ "4": 11,
20
+ "12": 19,
21
+ "25": 32,
22
+ "80": 87,
23
+ "97": 104,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  "mod": 4,
25
+ "19": 26,
26
+ "21": 28,
27
+ "54": 61,
28
+ "92": 99,
29
+ "34": 41,
30
+ "81": 88,
31
+ "18": 25,
32
+ "76": 83,
33
+ "40": 47,
34
+ "9": 16,
35
+ "79": 86,
36
+ "85": 92,
37
+ "98": 105,
38
+ "74": 81,
39
+ "15": 22,
40
+ "58": 65,
41
+ "16": 23,
 
 
 
 
 
 
42
  "<s>": 1,
43
+ "83": 90,
44
+ "86": 93,
45
+ "24": 31,
46
+ "<unk>": 0,
47
+ "91": 98,
48
+ "82": 89,
49
+ "88": 95,
50
+ "7": 14,
51
+ "71": 78,
52
+ "32": 39,
53
+ "39": 46,
54
+ "72": 79,
55
+ "42": 49,
56
+ "8": 15,
57
+ "77": 84,
58
+ "90": 97,
59
+ "61": 68,
60
+ "<pad>": 3,
61
+ "48": 55,
62
+ "99": 106,
63
+ "33": 40,
64
+ "94": 101,
65
+ "=": 5,
66
+ "35": 42,
67
+ "67": 74,
68
+ "68": 75,
69
+ "59": 66,
70
+ "57": 64,
71
+ "64": 71,
72
+ "53": 60,
73
+ "52": 59,
74
+ "51": 58,
75
+ "1": 8,
76
+ "26": 33,
77
+ "44": 51,
78
+ "10": 17,
79
+ "14": 21,
80
+ "50": 57,
81
+ "89": 96,
82
+ "78": 85,
83
+ "46": 53,
84
+ "29": 36,
85
+ "43": 50,
86
+ "95": 102,
87
+ "11": 18,
88
+ "69": 76,
89
+ "36": 43,
90
+ "93": 100,
91
+ "73": 80,
92
+ "2": 9,
93
+ "70": 77,
94
  " ": 6,
95
+ "47": 54,
96
+ "38": 45,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  "</s>": 2,
98
+ "62": 69,
99
+ "13": 20,
100
+ "17": 24,
101
+ "66": 73,
102
+ "37": 44,
103
+ "56": 63,
104
+ "45": 52,
105
+ "84": 91,
106
+ "55": 62,
107
+ "23": 30,
108
+ "96": 103
109
  }