Upload tokenizer

tokenizer_config.json

@@ -264,7 +264,7 @@
       "single_word": false,
       "special": true
     },
-    "59219": {
+    "50285": {
       "content": "[unused0]",
       "lstrip": false,
       "normalized": true,
@@ -272,7 +272,7 @@
       "single_word": false,
       "special": false
     },
-    "59220": {
+    "50286": {
       "content": "[unused1]",
       "lstrip": false,
       "normalized": true,
@@ -280,7 +280,7 @@
       "single_word": false,
       "special": false
     },
-    "59221": {
+    "50287": {
       "content": "[unused2]",
       "lstrip": false,
       "normalized": true,
@@ -288,7 +288,7 @@
       "single_word": false,
       "special": false
     },
-    "59222": {
+    "50288": {
       "content": "[unused3]",
       "lstrip": false,
       "normalized": true,
@@ -296,7 +296,7 @@
       "single_word": false,
       "special": false
     },
-    "59223": {
+    "50289": {
       "content": "[unused4]",
       "lstrip": false,
       "normalized": true,
@@ -304,7 +304,7 @@
       "single_word": false,
       "special": false
     },
-    "59224": {
+    "50290": {
       "content": "[unused5]",
       "lstrip": false,
       "normalized": true,
@@ -312,7 +312,7 @@
       "single_word": false,
       "special": false
     },
-    "59225": {
+    "50291": {
       "content": "[unused6]",
       "lstrip": false,
       "normalized": true,
@@ -320,7 +320,7 @@
       "single_word": false,
       "special": false
     },
-    "59226": {
+    "50292": {
       "content": "[unused7]",
       "lstrip": false,
       "normalized": true,
@@ -328,7 +328,7 @@
       "single_word": false,
       "special": false
     },
-    "59227": {
+    "50293": {
       "content": "[unused8]",
       "lstrip": false,
       "normalized": true,
@@ -336,7 +336,7 @@
       "single_word": false,
       "special": false
     },
-    "59228": {
+    "50294": {
       "content": "[unused9]",
       "lstrip": false,
       "normalized": true,
@@ -344,7 +344,7 @@
       "single_word": false,
       "special": false
     },
-    "59229": {
+    "50295": {
       "content": "[unused10]",
       "lstrip": false,
       "normalized": true,
@@ -352,7 +352,7 @@
       "single_word": false,
       "special": false
     },
-    "59230": {
+    "50296": {
       "content": "[unused11]",
       "lstrip": false,
       "normalized": true,
@@ -360,7 +360,7 @@
       "single_word": false,
       "special": false
     },
-    "59231": {
+    "50297": {
       "content": "[unused12]",
       "lstrip": false,
       "normalized": true,
@@ -368,7 +368,7 @@
       "single_word": false,
       "special": false
     },
-    "59232": {
+    "50298": {
       "content": "[unused13]",
       "lstrip": false,
       "normalized": true,
@@ -376,7 +376,7 @@
       "single_word": false,
       "special": false
     },
-    "59233": {
+    "50299": {
       "content": "[unused14]",
       "lstrip": false,
       "normalized": true,
@@ -384,7 +384,7 @@
       "single_word": false,
       "special": false
     },
-    "59234": {
+    "50300": {
       "content": "[unused15]",
       "lstrip": false,
       "normalized": true,
@@ -392,7 +392,7 @@
       "single_word": false,
       "special": false
     },
-    "59235": {
+    "50301": {
       "content": "[unused16]",
       "lstrip": false,
       "normalized": true,
@@ -400,7 +400,7 @@
       "single_word": false,
       "special": false
     },
-    "59236": {
+    "50302": {
       "content": "[unused17]",
       "lstrip": false,
       "normalized": true,
@@ -408,7 +408,7 @@
       "single_word": false,
       "special": false
     },
-    "59237": {
+    "50303": {
       "content": "[unused18]",
       "lstrip": false,
       "normalized": true,
@@ -416,7 +416,7 @@
       "single_word": false,
       "special": false
     },
-    "59238": {
+    "50304": {
       "content": "[unused19]",
       "lstrip": false,
       "normalized": true,
@@ -424,7 +424,7 @@
       "single_word": false,
       "special": false
     },
-    "59239": {
+    "50305": {
       "content": "[unused20]",
       "lstrip": false,
       "normalized": true,
@@ -432,7 +432,7 @@
       "single_word": false,
       "special": false
     },
-    "59240": {
+    "50306": {
       "content": "[unused21]",
       "lstrip": false,
       "normalized": true,
@@ -440,7 +440,7 @@
       "single_word": false,
       "special": false
     },
-    "59241": {
+    "50307": {
       "content": "[unused22]",
       "lstrip": false,
       "normalized": true,
@@ -448,7 +448,7 @@
       "single_word": false,
       "special": false
     },
-    "59242": {
+    "50308": {
       "content": "[unused23]",
       "lstrip": false,
       "normalized": true,
@@ -456,7 +456,7 @@
       "single_word": false,
       "special": false
     },
-    "59243": {
+    "50309": {
       "content": "[unused24]",
       "lstrip": false,
       "normalized": true,
@@ -464,7 +464,7 @@
       "single_word": false,
       "special": false
     },
-    "59244": {
+    "50310": {
       "content": "[unused25]",
       "lstrip": false,
       "normalized": true,
@@ -472,7 +472,7 @@
       "single_word": false,
       "special": false
     },
-    "59245": {
+    "50311": {
       "content": "[unused26]",
       "lstrip": false,
       "normalized": true,
@@ -480,7 +480,7 @@
       "single_word": false,
       "special": false
     },
-    "59246": {
+    "50312": {
       "content": "[unused27]",
       "lstrip": false,
       "normalized": true,
@@ -488,7 +488,7 @@
       "single_word": false,
       "special": false
     },
-    "59247": {
+    "50313": {
       "content": "[unused28]",
       "lstrip": false,
       "normalized": true,
@@ -496,7 +496,7 @@
       "single_word": false,
       "special": false
     },
-    "59248": {
+    "50314": {
       "content": "[unused29]",
       "lstrip": false,
       "normalized": true,
@@ -504,7 +504,7 @@
       "single_word": false,
       "special": false
     },
-    "59249": {
+    "50315": {
       "content": "[unused30]",
       "lstrip": false,
       "normalized": true,
@@ -512,7 +512,7 @@
       "single_word": false,
       "special": false
     },
-    "59250": {
+    "50316": {
       "content": "[unused31]",
       "lstrip": false,
       "normalized": true,
@@ -520,7 +520,7 @@
       "single_word": false,
       "special": false
     },
-    "59251": {
+    "50317": {
       "content": "[unused32]",
       "lstrip": false,
       "normalized": true,
@@ -528,7 +528,7 @@
       "single_word": false,
       "special": false
     },
-    "59252": {
+    "50318": {
       "content": "[unused33]",
       "lstrip": false,
       "normalized": true,
@@ -536,7 +536,7 @@
       "single_word": false,
       "special": false
     },
-    "59253": {
+    "50319": {
       "content": "[unused34]",
       "lstrip": false,
       "normalized": true,
@@ -544,7 +544,7 @@
       "single_word": false,
       "special": false
     },
-    "59254": {
+    "50320": {
       "content": "[unused35]",
       "lstrip": false,
       "normalized": true,
@@ -552,7 +552,7 @@
       "single_word": false,
       "special": false
     },
-    "59255": {
+    "50321": {
       "content": "[unused36]",
       "lstrip": false,
       "normalized": true,
@@ -560,7 +560,7 @@
       "single_word": false,
       "special": false
     },
-    "59256": {
+    "50322": {
       "content": "[unused37]",
       "lstrip": false,
       "normalized": true,
@@ -568,7 +568,7 @@
       "single_word": false,
       "special": false
     },
-    "59257": {
+    "50323": {
       "content": "[unused38]",
       "lstrip": false,
       "normalized": true,
@@ -576,7 +576,7 @@
       "single_word": false,
       "special": false
     },
-    "59258": {
+    "50324": {
       "content": "[unused39]",
       "lstrip": false,
       "normalized": true,
@@ -584,7 +584,7 @@
       "single_word": false,
       "special": false
     },
-    "59259": {
+    "50325": {
       "content": "[unused40]",
       "lstrip": false,
       "normalized": true,
@@ -592,7 +592,7 @@
       "single_word": false,
       "special": false
     },
-    "59260": {
+    "50326": {
       "content": "[unused41]",
       "lstrip": false,
       "normalized": true,
@@ -600,7 +600,7 @@
       "single_word": false,
       "special": false
     },
-    "59261": {
+    "50327": {
       "content": "[unused42]",
       "lstrip": false,
       "normalized": true,
@@ -608,7 +608,7 @@
       "single_word": false,
       "special": false
     },
-    "59262": {
+    "50328": {
       "content": "[unused43]",
       "lstrip": false,
       "normalized": true,
@@ -616,7 +616,7 @@
       "single_word": false,
       "special": false
     },
-    "59263": {
+    "50329": {
       "content": "[unused44]",
       "lstrip": false,
       "normalized": true,
@@ -624,7 +624,7 @@
       "single_word": false,
       "special": false
     },
-    "59264": {
+    "50330": {
       "content": "[unused45]",
       "lstrip": false,
       "normalized": true,
@@ -632,7 +632,7 @@
       "single_word": false,
       "special": false
     },
-    "59265": {
+    "50331": {
       "content": "[unused46]",
       "lstrip": false,
       "normalized": true,
@@ -640,7 +640,7 @@
       "single_word": false,
       "special": false
     },
-    "59266": {
+    "50332": {
       "content": "[unused47]",
       "lstrip": false,
       "normalized": true,
@@ -648,7 +648,7 @@
       "single_word": false,
       "special": false
     },
-    "59267": {
+    "50333": {
       "content": "[unused48]",
       "lstrip": false,
       "normalized": true,
@@ -656,7 +656,7 @@
       "single_word": false,
       "special": false
     },
-    "59268": {
+    "50334": {
       "content": "[unused49]",
       "lstrip": false,
       "normalized": true,
@@ -664,7 +664,7 @@
       "single_word": false,
       "special": false
     },
-    "59269": {
+    "50335": {
       "content": "[unused50]",
       "lstrip": false,
       "normalized": true,
@@ -672,7 +672,7 @@
       "single_word": false,
       "special": false
     },
-    "59270": {
+    "50336": {
       "content": "[unused51]",
       "lstrip": false,
       "normalized": true,
@@ -680,7 +680,7 @@
       "single_word": false,
       "special": false
     },
-    "59271": {
+    "50337": {
       "content": "[unused52]",
       "lstrip": false,
       "normalized": true,
@@ -688,7 +688,7 @@
       "single_word": false,
       "special": false
     },
-    "59272": {
+    "50338": {
       "content": "[unused53]",
       "lstrip": false,
       "normalized": true,
@@ -696,7 +696,7 @@
       "single_word": false,
       "special": false
     },
-    "59273": {
+    "50339": {
       "content": "[unused54]",
       "lstrip": false,
       "normalized": true,
@@ -704,7 +704,7 @@
       "single_word": false,
       "special": false
     },
-    "59274": {
+    "50340": {
       "content": "[unused55]",
       "lstrip": false,
       "normalized": true,
@@ -712,7 +712,7 @@
       "single_word": false,
       "special": false
     },
-    "59275": {
+    "50341": {
       "content": "[unused56]",
       "lstrip": false,
       "normalized": true,
@@ -720,7 +720,7 @@
       "single_word": false,
       "special": false
     },
-    "59276": {
+    "50342": {
       "content": "[unused57]",
       "lstrip": false,
       "normalized": true,
@@ -728,7 +728,7 @@
       "single_word": false,
       "special": false
     },
-    "59277": {
+    "50343": {
       "content": "[unused58]",
       "lstrip": false,
       "normalized": true,
@@ -736,7 +736,7 @@
       "single_word": false,
       "special": false
     },
-    "59278": {
+    "50344": {
       "content": "[unused59]",
       "lstrip": false,
       "normalized": true,
@@ -744,7 +744,7 @@
       "single_word": false,
       "special": false
     },
-    "59279": {
+    "50345": {
       "content": "[unused60]",
       "lstrip": false,
       "normalized": true,
@@ -752,7 +752,7 @@
       "single_word": false,
       "special": false
     },
-    "59280": {
+    "50346": {
       "content": "[unused61]",
       "lstrip": false,
       "normalized": true,
@@ -760,7 +760,7 @@
       "single_word": false,
       "special": false
     },
-    "59281": {
+    "50347": {
       "content": "[unused62]",
       "lstrip": false,
       "normalized": true,
@@ -768,7 +768,7 @@
       "single_word": false,
       "special": false
     },
-    "59282": {
+    "50348": {
       "content": "[unused63]",
       "lstrip": false,
       "normalized": true,
@@ -776,7 +776,7 @@
       "single_word": false,
       "special": false
     },
-    "59283": {
+    "50349": {
       "content": "[unused64]",
       "lstrip": false,
       "normalized": true,
@@ -784,7 +784,7 @@
       "single_word": false,
       "special": false
     },
-    "59284": {
+    "50350": {
       "content": "[unused65]",
       "lstrip": false,
       "normalized": true,
@@ -792,7 +792,7 @@
       "single_word": false,
       "special": false
     },
-    "59285": {
+    "50351": {
       "content": "[unused66]",
       "lstrip": false,
       "normalized": true,
@@ -800,7 +800,7 @@
       "single_word": false,
       "special": false
     },
-    "59286": {
+    "50352": {
       "content": "[unused67]",
       "lstrip": false,
       "normalized": true,
@@ -808,7 +808,7 @@
       "single_word": false,
       "special": false
     },
-    "59287": {
+    "50353": {
       "content": "[unused68]",
       "lstrip": false,
       "normalized": true,
@@ -816,7 +816,7 @@
       "single_word": false,
       "special": false
     },
-    "59288": {
+    "50354": {
       "content": "[unused69]",
       "lstrip": false,
       "normalized": true,
@@ -824,7 +824,7 @@
       "single_word": false,
       "special": false
     },
-    "59289": {
+    "50355": {
       "content": "[unused70]",
       "lstrip": false,
       "normalized": true,
@@ -832,7 +832,7 @@
       "single_word": false,
       "special": false
     },
-    "59290": {
+    "50356": {
       "content": "[unused71]",
       "lstrip": false,
       "normalized": true,
@@ -840,7 +840,7 @@
       "single_word": false,
       "special": false
     },
-    "59291": {
+    "50357": {
       "content": "[unused72]",
       "lstrip": false,
       "normalized": true,
@@ -848,7 +848,7 @@
       "single_word": false,
       "special": false
     },
-    "59292": {
+    "50358": {
       "content": "[unused73]",
       "lstrip": false,
       "normalized": true,
@@ -856,7 +856,7 @@
       "single_word": false,
       "special": false
     },
-    "59293": {
+    "50359": {
       "content": "[unused74]",
       "lstrip": false,
       "normalized": true,
@@ -864,7 +864,7 @@
       "single_word": false,
       "special": false
     },
-    "59294": {
+    "50360": {
       "content": "[unused75]",
       "lstrip": false,
       "normalized": true,
@@ -872,7 +872,7 @@
       "single_word": false,
       "special": false
     },
-    "59295": {
+    "50361": {
       "content": "[unused76]",
       "lstrip": false,
       "normalized": true,
@@ -880,7 +880,7 @@
       "single_word": false,
       "special": false
     },
-    "59296": {
+    "50362": {
       "content": "[unused77]",
       "lstrip": false,
       "normalized": true,
@@ -888,7 +888,7 @@
       "single_word": false,
       "special": false
     },
-    "59297": {
+    "50363": {
       "content": "[unused78]",
       "lstrip": false,
       "normalized": true,
@@ -896,7 +896,7 @@
       "single_word": false,
       "special": false
     },
-    "59298": {
+    "50364": {
       "content": "[unused79]",
       "lstrip": false,
       "normalized": true,
@@ -904,7 +904,7 @@
       "single_word": false,
       "special": false
     },
-    "59299": {
+    "50365": {
       "content": "[unused80]",
       "lstrip": false,
       "normalized": true,
@@ -912,7 +912,7 @@
       "single_word": false,
       "special": false
     },
-    "59300": {
+    "50366": {
       "content": "[unused81]",
       "lstrip": false,
       "normalized": true,
@@ -920,7 +920,7 @@
       "single_word": false,
       "special": false
     },
-    "59301": {
+    "50367": {
       "content": "[unused82]",
       "lstrip": false,
       "normalized": true,