Upload tokenizer

Browse files

Files changed (2) hide show

added_tokens.json +976 -0
tokenizer.json +0 -0

added_tokens.json CHANGED Viewed

@@ -29,112 +29,1088 @@
   "v_tok_1021": 51286,
   "v_tok_1022": 51287,
   "v_tok_1023": 51288,
   "v_tok_103": 50368,
   "v_tok_104": 50369,
   "v_tok_105": 50370,
   "v_tok_106": 50371,
   "v_tok_107": 50372,
   "v_tok_108": 50373,
   "v_tok_109": 50374,
   "v_tok_11": 50276,
   "v_tok_110": 50375,
   "v_tok_111": 50376,
   "v_tok_112": 50377,
   "v_tok_113": 50378,
   "v_tok_114": 50379,
   "v_tok_115": 50380,
   "v_tok_116": 50381,
   "v_tok_117": 50382,
   "v_tok_118": 50383,
   "v_tok_119": 50384,
   "v_tok_12": 50277,
   "v_tok_120": 50385,
   "v_tok_121": 50386,
   "v_tok_122": 50387,
   "v_tok_123": 50388,
   "v_tok_124": 50389,
   "v_tok_125": 50390,
   "v_tok_126": 50391,
   "v_tok_127": 50392,
   "v_tok_128": 50393,
   "v_tok_129": 50394,
   "v_tok_13": 50278,
   "v_tok_130": 50395,
   "v_tok_131": 50396,
   "v_tok_132": 50397,
   "v_tok_133": 50398,
   "v_tok_134": 50399,
   "v_tok_135": 50400,
   "v_tok_136": 50401,
   "v_tok_137": 50402,
   "v_tok_138": 50403,
   "v_tok_139": 50404,
   "v_tok_14": 50279,
   "v_tok_140": 50405,
   "v_tok_141": 50406,
   "v_tok_142": 50407,
   "v_tok_143": 50408,
   "v_tok_144": 50409,
   "v_tok_145": 50410,
   "v_tok_146": 50411,
   "v_tok_147": 50412,
   "v_tok_148": 50413,
   "v_tok_149": 50414,
   "v_tok_15": 50280,
   "v_tok_150": 50415,
   "v_tok_151": 50416,
   "v_tok_152": 50417,
   "v_tok_153": 50418,
   "v_tok_154": 50419,
   "v_tok_155": 50420,
   "v_tok_156": 50421,
   "v_tok_157": 50422,
   "v_tok_158": 50423,
   "v_tok_159": 50424,
   "v_tok_16": 50281,
   "v_tok_160": 50425,
   "v_tok_161": 50426,
   "v_tok_162": 50427,
   "v_tok_163": 50428,
   "v_tok_164": 50429,
   "v_tok_165": 50430,
   "v_tok_166": 50431,
   "v_tok_167": 50432,
   "v_tok_168": 50433,
   "v_tok_169": 50434,
   "v_tok_17": 50282,
   "v_tok_170": 50435,
   "v_tok_171": 50436,
   "v_tok_172": 50437,
   "v_tok_173": 50438,
   "v_tok_174": 50439,
   "v_tok_175": 50440,
   "v_tok_176": 50441,
   "v_tok_177": 50442,
   "v_tok_178": 50443,
   "v_tok_179": 50444,
   "v_tok_18": 50283,
   "v_tok_180": 50445,
   "v_tok_181": 50446,
   "v_tok_182": 50447,
   "v_tok_183": 50448,
   "v_tok_184": 50449,
   "v_tok_185": 50450,
   "v_tok_186": 50451,
   "v_tok_187": 50452,
   "v_tok_188": 50453,
   "v_tok_189": 50454,
   "v_tok_19": 50284,
   "v_tok_190": 50455,
   "v_tok_191": 50456,
   "v_tok_192": 50457,
   "v_tok_193": 50458,
   "v_tok_194": 50459,
   "v_tok_195": 50460,
   "v_tok_196": 50461,
   "v_tok_197": 50462,
   "v_tok_198": 50463,
   "v_tok_199": 50464,
   "v_tok_2": 50267,
   "v_tok_20": 50285,
   "v_tok_200": 50465,

   "v_tok_1021": 51286,
   "v_tok_1022": 51287,
   "v_tok_1023": 51288,
+  "v_tok_1024": 51289,
+  "v_tok_1025": 51290,
+  "v_tok_1026": 51291,
+  "v_tok_1027": 51292,
+  "v_tok_1028": 51293,
+  "v_tok_1029": 51294,
   "v_tok_103": 50368,
+  "v_tok_1030": 51295,
+  "v_tok_1031": 51296,
+  "v_tok_1032": 51297,
+  "v_tok_1033": 51298,
+  "v_tok_1034": 51299,
+  "v_tok_1035": 51300,
+  "v_tok_1036": 51301,
+  "v_tok_1037": 51302,
+  "v_tok_1038": 51303,
+  "v_tok_1039": 51304,
   "v_tok_104": 50369,
+  "v_tok_1040": 51305,
+  "v_tok_1041": 51306,
+  "v_tok_1042": 51307,
+  "v_tok_1043": 51308,
+  "v_tok_1044": 51309,
+  "v_tok_1045": 51310,
+  "v_tok_1046": 51311,
+  "v_tok_1047": 51312,
+  "v_tok_1048": 51313,
+  "v_tok_1049": 51314,
   "v_tok_105": 50370,
+  "v_tok_1050": 51315,
+  "v_tok_1051": 51316,
+  "v_tok_1052": 51317,
+  "v_tok_1053": 51318,
+  "v_tok_1054": 51319,
+  "v_tok_1055": 51320,
+  "v_tok_1056": 51321,
+  "v_tok_1057": 51322,
+  "v_tok_1058": 51323,
+  "v_tok_1059": 51324,
   "v_tok_106": 50371,
+  "v_tok_1060": 51325,
+  "v_tok_1061": 51326,
+  "v_tok_1062": 51327,
+  "v_tok_1063": 51328,
+  "v_tok_1064": 51329,
+  "v_tok_1065": 51330,
+  "v_tok_1066": 51331,
+  "v_tok_1067": 51332,
+  "v_tok_1068": 51333,
+  "v_tok_1069": 51334,
   "v_tok_107": 50372,
+  "v_tok_1070": 51335,
+  "v_tok_1071": 51336,
+  "v_tok_1072": 51337,
+  "v_tok_1073": 51338,
+  "v_tok_1074": 51339,
+  "v_tok_1075": 51340,
+  "v_tok_1076": 51341,
+  "v_tok_1077": 51342,
+  "v_tok_1078": 51343,
+  "v_tok_1079": 51344,
   "v_tok_108": 50373,
+  "v_tok_1080": 51345,
+  "v_tok_1081": 51346,
+  "v_tok_1082": 51347,
+  "v_tok_1083": 51348,
+  "v_tok_1084": 51349,
+  "v_tok_1085": 51350,
+  "v_tok_1086": 51351,
+  "v_tok_1087": 51352,
+  "v_tok_1088": 51353,
+  "v_tok_1089": 51354,
   "v_tok_109": 50374,
+  "v_tok_1090": 51355,
+  "v_tok_1091": 51356,
+  "v_tok_1092": 51357,
+  "v_tok_1093": 51358,
+  "v_tok_1094": 51359,
+  "v_tok_1095": 51360,
+  "v_tok_1096": 51361,
+  "v_tok_1097": 51362,
+  "v_tok_1098": 51363,
+  "v_tok_1099": 51364,
   "v_tok_11": 50276,
   "v_tok_110": 50375,
+  "v_tok_1100": 51365,
+  "v_tok_1101": 51366,
+  "v_tok_1102": 51367,
+  "v_tok_1103": 51368,
+  "v_tok_1104": 51369,
+  "v_tok_1105": 51370,
+  "v_tok_1106": 51371,
+  "v_tok_1107": 51372,
+  "v_tok_1108": 51373,
+  "v_tok_1109": 51374,
   "v_tok_111": 50376,
+  "v_tok_1110": 51375,
+  "v_tok_1111": 51376,
+  "v_tok_1112": 51377,
+  "v_tok_1113": 51378,
+  "v_tok_1114": 51379,
+  "v_tok_1115": 51380,
+  "v_tok_1116": 51381,
+  "v_tok_1117": 51382,
+  "v_tok_1118": 51383,
+  "v_tok_1119": 51384,
   "v_tok_112": 50377,
+  "v_tok_1120": 51385,
+  "v_tok_1121": 51386,
+  "v_tok_1122": 51387,
+  "v_tok_1123": 51388,
+  "v_tok_1124": 51389,
+  "v_tok_1125": 51390,
+  "v_tok_1126": 51391,
+  "v_tok_1127": 51392,
+  "v_tok_1128": 51393,
+  "v_tok_1129": 51394,
   "v_tok_113": 50378,
+  "v_tok_1130": 51395,
+  "v_tok_1131": 51396,
+  "v_tok_1132": 51397,
+  "v_tok_1133": 51398,
+  "v_tok_1134": 51399,
+  "v_tok_1135": 51400,
+  "v_tok_1136": 51401,
+  "v_tok_1137": 51402,
+  "v_tok_1138": 51403,
+  "v_tok_1139": 51404,
   "v_tok_114": 50379,
+  "v_tok_1140": 51405,
+  "v_tok_1141": 51406,
+  "v_tok_1142": 51407,
+  "v_tok_1143": 51408,
+  "v_tok_1144": 51409,
+  "v_tok_1145": 51410,
+  "v_tok_1146": 51411,
+  "v_tok_1147": 51412,
+  "v_tok_1148": 51413,
+  "v_tok_1149": 51414,
   "v_tok_115": 50380,
+  "v_tok_1150": 51415,
+  "v_tok_1151": 51416,
+  "v_tok_1152": 51417,
+  "v_tok_1153": 51418,
+  "v_tok_1154": 51419,
+  "v_tok_1155": 51420,
+  "v_tok_1156": 51421,
+  "v_tok_1157": 51422,
+  "v_tok_1158": 51423,
+  "v_tok_1159": 51424,
   "v_tok_116": 50381,
+  "v_tok_1160": 51425,
+  "v_tok_1161": 51426,
+  "v_tok_1162": 51427,
+  "v_tok_1163": 51428,
+  "v_tok_1164": 51429,
+  "v_tok_1165": 51430,
+  "v_tok_1166": 51431,
+  "v_tok_1167": 51432,
+  "v_tok_1168": 51433,
+  "v_tok_1169": 51434,
   "v_tok_117": 50382,
+  "v_tok_1170": 51435,
+  "v_tok_1171": 51436,
+  "v_tok_1172": 51437,
+  "v_tok_1173": 51438,
+  "v_tok_1174": 51439,
+  "v_tok_1175": 51440,
+  "v_tok_1176": 51441,
+  "v_tok_1177": 51442,
+  "v_tok_1178": 51443,
+  "v_tok_1179": 51444,
   "v_tok_118": 50383,
+  "v_tok_1180": 51445,
+  "v_tok_1181": 51446,
+  "v_tok_1182": 51447,
+  "v_tok_1183": 51448,
+  "v_tok_1184": 51449,
+  "v_tok_1185": 51450,
+  "v_tok_1186": 51451,
+  "v_tok_1187": 51452,
+  "v_tok_1188": 51453,
+  "v_tok_1189": 51454,
   "v_tok_119": 50384,
+  "v_tok_1190": 51455,
+  "v_tok_1191": 51456,
+  "v_tok_1192": 51457,
+  "v_tok_1193": 51458,
+  "v_tok_1194": 51459,
+  "v_tok_1195": 51460,
+  "v_tok_1196": 51461,
+  "v_tok_1197": 51462,
+  "v_tok_1198": 51463,
+  "v_tok_1199": 51464,
   "v_tok_12": 50277,
   "v_tok_120": 50385,
+  "v_tok_1200": 51465,
+  "v_tok_1201": 51466,
+  "v_tok_1202": 51467,
+  "v_tok_1203": 51468,
+  "v_tok_1204": 51469,
+  "v_tok_1205": 51470,
+  "v_tok_1206": 51471,
+  "v_tok_1207": 51472,
+  "v_tok_1208": 51473,
+  "v_tok_1209": 51474,
   "v_tok_121": 50386,
+  "v_tok_1210": 51475,
+  "v_tok_1211": 51476,
+  "v_tok_1212": 51477,
+  "v_tok_1213": 51478,
+  "v_tok_1214": 51479,
+  "v_tok_1215": 51480,
+  "v_tok_1216": 51481,
+  "v_tok_1217": 51482,
+  "v_tok_1218": 51483,
+  "v_tok_1219": 51484,
   "v_tok_122": 50387,
+  "v_tok_1220": 51485,
+  "v_tok_1221": 51486,
+  "v_tok_1222": 51487,
+  "v_tok_1223": 51488,
+  "v_tok_1224": 51489,
+  "v_tok_1225": 51490,
+  "v_tok_1226": 51491,
+  "v_tok_1227": 51492,
+  "v_tok_1228": 51493,
+  "v_tok_1229": 51494,
   "v_tok_123": 50388,
+  "v_tok_1230": 51495,
+  "v_tok_1231": 51496,
+  "v_tok_1232": 51497,
+  "v_tok_1233": 51498,
+  "v_tok_1234": 51499,
+  "v_tok_1235": 51500,
+  "v_tok_1236": 51501,
+  "v_tok_1237": 51502,
+  "v_tok_1238": 51503,
+  "v_tok_1239": 51504,
   "v_tok_124": 50389,
+  "v_tok_1240": 51505,
+  "v_tok_1241": 51506,
+  "v_tok_1242": 51507,
+  "v_tok_1243": 51508,
+  "v_tok_1244": 51509,
+  "v_tok_1245": 51510,
+  "v_tok_1246": 51511,
+  "v_tok_1247": 51512,
+  "v_tok_1248": 51513,
+  "v_tok_1249": 51514,
   "v_tok_125": 50390,
+  "v_tok_1250": 51515,
+  "v_tok_1251": 51516,
+  "v_tok_1252": 51517,
+  "v_tok_1253": 51518,
+  "v_tok_1254": 51519,
+  "v_tok_1255": 51520,
+  "v_tok_1256": 51521,
+  "v_tok_1257": 51522,
+  "v_tok_1258": 51523,
+  "v_tok_1259": 51524,
   "v_tok_126": 50391,
+  "v_tok_1260": 51525,
+  "v_tok_1261": 51526,
+  "v_tok_1262": 51527,
+  "v_tok_1263": 51528,
+  "v_tok_1264": 51529,
+  "v_tok_1265": 51530,
+  "v_tok_1266": 51531,
+  "v_tok_1267": 51532,
+  "v_tok_1268": 51533,
+  "v_tok_1269": 51534,
   "v_tok_127": 50392,
+  "v_tok_1270": 51535,
+  "v_tok_1271": 51536,
+  "v_tok_1272": 51537,
+  "v_tok_1273": 51538,
+  "v_tok_1274": 51539,
+  "v_tok_1275": 51540,
+  "v_tok_1276": 51541,
+  "v_tok_1277": 51542,
+  "v_tok_1278": 51543,
+  "v_tok_1279": 51544,
   "v_tok_128": 50393,
+  "v_tok_1280": 51545,
+  "v_tok_1281": 51546,
+  "v_tok_1282": 51547,
+  "v_tok_1283": 51548,
+  "v_tok_1284": 51549,
+  "v_tok_1285": 51550,
+  "v_tok_1286": 51551,
+  "v_tok_1287": 51552,
+  "v_tok_1288": 51553,
+  "v_tok_1289": 51554,
   "v_tok_129": 50394,
+  "v_tok_1290": 51555,
+  "v_tok_1291": 51556,
+  "v_tok_1292": 51557,
+  "v_tok_1293": 51558,
+  "v_tok_1294": 51559,
+  "v_tok_1295": 51560,
+  "v_tok_1296": 51561,
+  "v_tok_1297": 51562,
+  "v_tok_1298": 51563,
+  "v_tok_1299": 51564,
   "v_tok_13": 50278,
   "v_tok_130": 50395,
+  "v_tok_1300": 51565,
+  "v_tok_1301": 51566,
+  "v_tok_1302": 51567,
+  "v_tok_1303": 51568,
+  "v_tok_1304": 51569,
+  "v_tok_1305": 51570,
+  "v_tok_1306": 51571,
+  "v_tok_1307": 51572,
+  "v_tok_1308": 51573,
+  "v_tok_1309": 51574,
   "v_tok_131": 50396,
+  "v_tok_1310": 51575,
+  "v_tok_1311": 51576,
+  "v_tok_1312": 51577,
+  "v_tok_1313": 51578,
+  "v_tok_1314": 51579,
+  "v_tok_1315": 51580,
+  "v_tok_1316": 51581,
+  "v_tok_1317": 51582,
+  "v_tok_1318": 51583,
+  "v_tok_1319": 51584,
   "v_tok_132": 50397,
+  "v_tok_1320": 51585,
+  "v_tok_1321": 51586,
+  "v_tok_1322": 51587,
+  "v_tok_1323": 51588,
+  "v_tok_1324": 51589,
+  "v_tok_1325": 51590,
+  "v_tok_1326": 51591,
+  "v_tok_1327": 51592,
+  "v_tok_1328": 51593,
+  "v_tok_1329": 51594,
   "v_tok_133": 50398,
+  "v_tok_1330": 51595,
+  "v_tok_1331": 51596,
+  "v_tok_1332": 51597,
+  "v_tok_1333": 51598,
+  "v_tok_1334": 51599,
+  "v_tok_1335": 51600,
+  "v_tok_1336": 51601,
+  "v_tok_1337": 51602,
+  "v_tok_1338": 51603,
+  "v_tok_1339": 51604,
   "v_tok_134": 50399,
+  "v_tok_1340": 51605,
+  "v_tok_1341": 51606,
+  "v_tok_1342": 51607,
+  "v_tok_1343": 51608,
+  "v_tok_1344": 51609,
+  "v_tok_1345": 51610,
+  "v_tok_1346": 51611,
+  "v_tok_1347": 51612,
+  "v_tok_1348": 51613,
+  "v_tok_1349": 51614,
   "v_tok_135": 50400,
+  "v_tok_1350": 51615,
+  "v_tok_1351": 51616,
+  "v_tok_1352": 51617,
+  "v_tok_1353": 51618,
+  "v_tok_1354": 51619,
+  "v_tok_1355": 51620,
+  "v_tok_1356": 51621,
+  "v_tok_1357": 51622,
+  "v_tok_1358": 51623,
+  "v_tok_1359": 51624,
   "v_tok_136": 50401,
+  "v_tok_1360": 51625,
+  "v_tok_1361": 51626,
+  "v_tok_1362": 51627,
+  "v_tok_1363": 51628,
+  "v_tok_1364": 51629,
+  "v_tok_1365": 51630,
+  "v_tok_1366": 51631,
+  "v_tok_1367": 51632,
+  "v_tok_1368": 51633,
+  "v_tok_1369": 51634,
   "v_tok_137": 50402,
+  "v_tok_1370": 51635,
+  "v_tok_1371": 51636,
+  "v_tok_1372": 51637,
+  "v_tok_1373": 51638,
+  "v_tok_1374": 51639,
+  "v_tok_1375": 51640,
+  "v_tok_1376": 51641,
+  "v_tok_1377": 51642,
+  "v_tok_1378": 51643,
+  "v_tok_1379": 51644,
   "v_tok_138": 50403,
+  "v_tok_1380": 51645,
+  "v_tok_1381": 51646,
+  "v_tok_1382": 51647,
+  "v_tok_1383": 51648,
+  "v_tok_1384": 51649,
+  "v_tok_1385": 51650,
+  "v_tok_1386": 51651,
+  "v_tok_1387": 51652,
+  "v_tok_1388": 51653,
+  "v_tok_1389": 51654,
   "v_tok_139": 50404,
+  "v_tok_1390": 51655,
+  "v_tok_1391": 51656,
+  "v_tok_1392": 51657,
+  "v_tok_1393": 51658,
+  "v_tok_1394": 51659,
+  "v_tok_1395": 51660,
+  "v_tok_1396": 51661,
+  "v_tok_1397": 51662,
+  "v_tok_1398": 51663,
+  "v_tok_1399": 51664,
   "v_tok_14": 50279,
   "v_tok_140": 50405,
+  "v_tok_1400": 51665,
+  "v_tok_1401": 51666,
+  "v_tok_1402": 51667,
+  "v_tok_1403": 51668,
+  "v_tok_1404": 51669,
+  "v_tok_1405": 51670,
+  "v_tok_1406": 51671,
+  "v_tok_1407": 51672,
+  "v_tok_1408": 51673,
+  "v_tok_1409": 51674,
   "v_tok_141": 50406,
+  "v_tok_1410": 51675,
+  "v_tok_1411": 51676,
+  "v_tok_1412": 51677,
+  "v_tok_1413": 51678,
+  "v_tok_1414": 51679,
+  "v_tok_1415": 51680,
+  "v_tok_1416": 51681,
+  "v_tok_1417": 51682,
+  "v_tok_1418": 51683,
+  "v_tok_1419": 51684,
   "v_tok_142": 50407,
+  "v_tok_1420": 51685,
+  "v_tok_1421": 51686,
+  "v_tok_1422": 51687,
+  "v_tok_1423": 51688,
+  "v_tok_1424": 51689,
+  "v_tok_1425": 51690,
+  "v_tok_1426": 51691,
+  "v_tok_1427": 51692,
+  "v_tok_1428": 51693,
+  "v_tok_1429": 51694,
   "v_tok_143": 50408,
+  "v_tok_1430": 51695,
+  "v_tok_1431": 51696,
+  "v_tok_1432": 51697,
+  "v_tok_1433": 51698,
+  "v_tok_1434": 51699,
+  "v_tok_1435": 51700,
+  "v_tok_1436": 51701,
+  "v_tok_1437": 51702,
+  "v_tok_1438": 51703,
+  "v_tok_1439": 51704,
   "v_tok_144": 50409,
+  "v_tok_1440": 51705,
+  "v_tok_1441": 51706,
+  "v_tok_1442": 51707,
+  "v_tok_1443": 51708,
+  "v_tok_1444": 51709,
+  "v_tok_1445": 51710,
+  "v_tok_1446": 51711,
+  "v_tok_1447": 51712,
+  "v_tok_1448": 51713,
+  "v_tok_1449": 51714,
   "v_tok_145": 50410,
+  "v_tok_1450": 51715,
+  "v_tok_1451": 51716,
+  "v_tok_1452": 51717,
+  "v_tok_1453": 51718,
+  "v_tok_1454": 51719,
+  "v_tok_1455": 51720,
+  "v_tok_1456": 51721,
+  "v_tok_1457": 51722,
+  "v_tok_1458": 51723,
+  "v_tok_1459": 51724,
   "v_tok_146": 50411,
+  "v_tok_1460": 51725,
+  "v_tok_1461": 51726,
+  "v_tok_1462": 51727,
+  "v_tok_1463": 51728,
+  "v_tok_1464": 51729,
+  "v_tok_1465": 51730,
+  "v_tok_1466": 51731,
+  "v_tok_1467": 51732,
+  "v_tok_1468": 51733,
+  "v_tok_1469": 51734,
   "v_tok_147": 50412,
+  "v_tok_1470": 51735,
+  "v_tok_1471": 51736,
+  "v_tok_1472": 51737,
+  "v_tok_1473": 51738,
+  "v_tok_1474": 51739,
+  "v_tok_1475": 51740,
+  "v_tok_1476": 51741,
+  "v_tok_1477": 51742,
+  "v_tok_1478": 51743,
+  "v_tok_1479": 51744,
   "v_tok_148": 50413,
+  "v_tok_1480": 51745,
+  "v_tok_1481": 51746,
+  "v_tok_1482": 51747,
+  "v_tok_1483": 51748,
+  "v_tok_1484": 51749,
+  "v_tok_1485": 51750,
+  "v_tok_1486": 51751,
+  "v_tok_1487": 51752,
+  "v_tok_1488": 51753,
+  "v_tok_1489": 51754,
   "v_tok_149": 50414,
+  "v_tok_1490": 51755,
+  "v_tok_1491": 51756,
+  "v_tok_1492": 51757,
+  "v_tok_1493": 51758,
+  "v_tok_1494": 51759,
+  "v_tok_1495": 51760,
+  "v_tok_1496": 51761,
+  "v_tok_1497": 51762,
+  "v_tok_1498": 51763,
+  "v_tok_1499": 51764,
   "v_tok_15": 50280,
   "v_tok_150": 50415,
+  "v_tok_1500": 51765,
+  "v_tok_1501": 51766,
+  "v_tok_1502": 51767,
+  "v_tok_1503": 51768,
+  "v_tok_1504": 51769,
+  "v_tok_1505": 51770,
+  "v_tok_1506": 51771,
+  "v_tok_1507": 51772,
+  "v_tok_1508": 51773,
+  "v_tok_1509": 51774,
   "v_tok_151": 50416,
+  "v_tok_1510": 51775,
+  "v_tok_1511": 51776,
+  "v_tok_1512": 51777,
+  "v_tok_1513": 51778,
+  "v_tok_1514": 51779,
+  "v_tok_1515": 51780,
+  "v_tok_1516": 51781,
+  "v_tok_1517": 51782,
+  "v_tok_1518": 51783,
+  "v_tok_1519": 51784,
   "v_tok_152": 50417,
+  "v_tok_1520": 51785,
+  "v_tok_1521": 51786,
+  "v_tok_1522": 51787,
+  "v_tok_1523": 51788,
+  "v_tok_1524": 51789,
+  "v_tok_1525": 51790,
+  "v_tok_1526": 51791,
+  "v_tok_1527": 51792,
+  "v_tok_1528": 51793,
+  "v_tok_1529": 51794,
   "v_tok_153": 50418,
+  "v_tok_1530": 51795,
+  "v_tok_1531": 51796,
+  "v_tok_1532": 51797,
+  "v_tok_1533": 51798,
+  "v_tok_1534": 51799,
+  "v_tok_1535": 51800,
+  "v_tok_1536": 51801,
+  "v_tok_1537": 51802,
+  "v_tok_1538": 51803,
+  "v_tok_1539": 51804,
   "v_tok_154": 50419,
+  "v_tok_1540": 51805,
+  "v_tok_1541": 51806,
+  "v_tok_1542": 51807,
+  "v_tok_1543": 51808,
+  "v_tok_1544": 51809,
+  "v_tok_1545": 51810,
+  "v_tok_1546": 51811,
+  "v_tok_1547": 51812,
+  "v_tok_1548": 51813,
+  "v_tok_1549": 51814,
   "v_tok_155": 50420,
+  "v_tok_1550": 51815,
+  "v_tok_1551": 51816,
+  "v_tok_1552": 51817,
+  "v_tok_1553": 51818,
+  "v_tok_1554": 51819,
+  "v_tok_1555": 51820,
+  "v_tok_1556": 51821,
+  "v_tok_1557": 51822,
+  "v_tok_1558": 51823,
+  "v_tok_1559": 51824,
   "v_tok_156": 50421,
+  "v_tok_1560": 51825,
+  "v_tok_1561": 51826,
+  "v_tok_1562": 51827,
+  "v_tok_1563": 51828,
+  "v_tok_1564": 51829,
+  "v_tok_1565": 51830,
+  "v_tok_1566": 51831,
+  "v_tok_1567": 51832,
+  "v_tok_1568": 51833,
+  "v_tok_1569": 51834,
   "v_tok_157": 50422,
+  "v_tok_1570": 51835,
+  "v_tok_1571": 51836,
+  "v_tok_1572": 51837,
+  "v_tok_1573": 51838,
+  "v_tok_1574": 51839,
+  "v_tok_1575": 51840,
+  "v_tok_1576": 51841,
+  "v_tok_1577": 51842,
+  "v_tok_1578": 51843,
+  "v_tok_1579": 51844,
   "v_tok_158": 50423,
+  "v_tok_1580": 51845,
+  "v_tok_1581": 51846,
+  "v_tok_1582": 51847,
+  "v_tok_1583": 51848,
+  "v_tok_1584": 51849,
+  "v_tok_1585": 51850,
+  "v_tok_1586": 51851,
+  "v_tok_1587": 51852,
+  "v_tok_1588": 51853,
+  "v_tok_1589": 51854,
   "v_tok_159": 50424,
+  "v_tok_1590": 51855,
+  "v_tok_1591": 51856,
+  "v_tok_1592": 51857,
+  "v_tok_1593": 51858,
+  "v_tok_1594": 51859,
+  "v_tok_1595": 51860,
+  "v_tok_1596": 51861,
+  "v_tok_1597": 51862,
+  "v_tok_1598": 51863,
+  "v_tok_1599": 51864,
   "v_tok_16": 50281,
   "v_tok_160": 50425,
+  "v_tok_1600": 51865,
+  "v_tok_1601": 51866,
+  "v_tok_1602": 51867,
+  "v_tok_1603": 51868,
+  "v_tok_1604": 51869,
+  "v_tok_1605": 51870,
+  "v_tok_1606": 51871,
+  "v_tok_1607": 51872,
+  "v_tok_1608": 51873,
+  "v_tok_1609": 51874,
   "v_tok_161": 50426,
+  "v_tok_1610": 51875,
+  "v_tok_1611": 51876,
+  "v_tok_1612": 51877,
+  "v_tok_1613": 51878,
+  "v_tok_1614": 51879,
+  "v_tok_1615": 51880,
+  "v_tok_1616": 51881,
+  "v_tok_1617": 51882,
+  "v_tok_1618": 51883,
+  "v_tok_1619": 51884,
   "v_tok_162": 50427,
+  "v_tok_1620": 51885,
+  "v_tok_1621": 51886,
+  "v_tok_1622": 51887,
+  "v_tok_1623": 51888,
+  "v_tok_1624": 51889,
+  "v_tok_1625": 51890,
+  "v_tok_1626": 51891,
+  "v_tok_1627": 51892,
+  "v_tok_1628": 51893,
+  "v_tok_1629": 51894,
   "v_tok_163": 50428,
+  "v_tok_1630": 51895,
+  "v_tok_1631": 51896,
+  "v_tok_1632": 51897,
+  "v_tok_1633": 51898,
+  "v_tok_1634": 51899,
+  "v_tok_1635": 51900,
+  "v_tok_1636": 51901,
+  "v_tok_1637": 51902,
+  "v_tok_1638": 51903,
+  "v_tok_1639": 51904,
   "v_tok_164": 50429,
+  "v_tok_1640": 51905,
+  "v_tok_1641": 51906,
+  "v_tok_1642": 51907,
+  "v_tok_1643": 51908,
+  "v_tok_1644": 51909,
+  "v_tok_1645": 51910,
+  "v_tok_1646": 51911,
+  "v_tok_1647": 51912,
+  "v_tok_1648": 51913,
+  "v_tok_1649": 51914,
   "v_tok_165": 50430,
+  "v_tok_1650": 51915,
+  "v_tok_1651": 51916,
+  "v_tok_1652": 51917,
+  "v_tok_1653": 51918,
+  "v_tok_1654": 51919,
+  "v_tok_1655": 51920,
+  "v_tok_1656": 51921,
+  "v_tok_1657": 51922,
+  "v_tok_1658": 51923,
+  "v_tok_1659": 51924,
   "v_tok_166": 50431,
+  "v_tok_1660": 51925,
+  "v_tok_1661": 51926,
+  "v_tok_1662": 51927,
+  "v_tok_1663": 51928,
+  "v_tok_1664": 51929,
+  "v_tok_1665": 51930,
+  "v_tok_1666": 51931,
+  "v_tok_1667": 51932,
+  "v_tok_1668": 51933,
+  "v_tok_1669": 51934,
   "v_tok_167": 50432,
+  "v_tok_1670": 51935,
+  "v_tok_1671": 51936,
+  "v_tok_1672": 51937,
+  "v_tok_1673": 51938,
+  "v_tok_1674": 51939,
+  "v_tok_1675": 51940,
+  "v_tok_1676": 51941,
+  "v_tok_1677": 51942,
+  "v_tok_1678": 51943,
+  "v_tok_1679": 51944,
   "v_tok_168": 50433,
+  "v_tok_1680": 51945,
+  "v_tok_1681": 51946,
+  "v_tok_1682": 51947,
+  "v_tok_1683": 51948,
+  "v_tok_1684": 51949,
+  "v_tok_1685": 51950,
+  "v_tok_1686": 51951,
+  "v_tok_1687": 51952,
+  "v_tok_1688": 51953,
+  "v_tok_1689": 51954,
   "v_tok_169": 50434,
+  "v_tok_1690": 51955,
+  "v_tok_1691": 51956,
+  "v_tok_1692": 51957,
+  "v_tok_1693": 51958,
+  "v_tok_1694": 51959,
+  "v_tok_1695": 51960,
+  "v_tok_1696": 51961,
+  "v_tok_1697": 51962,
+  "v_tok_1698": 51963,
+  "v_tok_1699": 51964,
   "v_tok_17": 50282,
   "v_tok_170": 50435,
+  "v_tok_1700": 51965,
+  "v_tok_1701": 51966,
+  "v_tok_1702": 51967,
+  "v_tok_1703": 51968,
+  "v_tok_1704": 51969,
+  "v_tok_1705": 51970,
+  "v_tok_1706": 51971,
+  "v_tok_1707": 51972,
+  "v_tok_1708": 51973,
+  "v_tok_1709": 51974,
   "v_tok_171": 50436,
+  "v_tok_1710": 51975,
+  "v_tok_1711": 51976,
+  "v_tok_1712": 51977,
+  "v_tok_1713": 51978,
+  "v_tok_1714": 51979,
+  "v_tok_1715": 51980,
+  "v_tok_1716": 51981,
+  "v_tok_1717": 51982,
+  "v_tok_1718": 51983,
+  "v_tok_1719": 51984,
   "v_tok_172": 50437,
+  "v_tok_1720": 51985,
+  "v_tok_1721": 51986,
+  "v_tok_1722": 51987,
+  "v_tok_1723": 51988,
+  "v_tok_1724": 51989,
+  "v_tok_1725": 51990,
+  "v_tok_1726": 51991,
+  "v_tok_1727": 51992,
+  "v_tok_1728": 51993,
+  "v_tok_1729": 51994,
   "v_tok_173": 50438,
+  "v_tok_1730": 51995,
+  "v_tok_1731": 51996,
+  "v_tok_1732": 51997,
+  "v_tok_1733": 51998,
+  "v_tok_1734": 51999,
+  "v_tok_1735": 52000,
+  "v_tok_1736": 52001,
+  "v_tok_1737": 52002,
+  "v_tok_1738": 52003,
+  "v_tok_1739": 52004,
   "v_tok_174": 50439,
+  "v_tok_1740": 52005,
+  "v_tok_1741": 52006,
+  "v_tok_1742": 52007,
+  "v_tok_1743": 52008,
+  "v_tok_1744": 52009,
+  "v_tok_1745": 52010,
+  "v_tok_1746": 52011,
+  "v_tok_1747": 52012,
+  "v_tok_1748": 52013,
+  "v_tok_1749": 52014,
   "v_tok_175": 50440,
+  "v_tok_1750": 52015,
+  "v_tok_1751": 52016,
+  "v_tok_1752": 52017,
+  "v_tok_1753": 52018,
+  "v_tok_1754": 52019,
+  "v_tok_1755": 52020,
+  "v_tok_1756": 52021,
+  "v_tok_1757": 52022,
+  "v_tok_1758": 52023,
+  "v_tok_1759": 52024,
   "v_tok_176": 50441,
+  "v_tok_1760": 52025,
+  "v_tok_1761": 52026,
+  "v_tok_1762": 52027,
+  "v_tok_1763": 52028,
+  "v_tok_1764": 52029,
+  "v_tok_1765": 52030,
+  "v_tok_1766": 52031,
+  "v_tok_1767": 52032,
+  "v_tok_1768": 52033,
+  "v_tok_1769": 52034,
   "v_tok_177": 50442,
+  "v_tok_1770": 52035,
+  "v_tok_1771": 52036,
+  "v_tok_1772": 52037,
+  "v_tok_1773": 52038,
+  "v_tok_1774": 52039,
+  "v_tok_1775": 52040,
+  "v_tok_1776": 52041,
+  "v_tok_1777": 52042,
+  "v_tok_1778": 52043,
+  "v_tok_1779": 52044,
   "v_tok_178": 50443,
+  "v_tok_1780": 52045,
+  "v_tok_1781": 52046,
+  "v_tok_1782": 52047,
+  "v_tok_1783": 52048,
+  "v_tok_1784": 52049,
+  "v_tok_1785": 52050,
+  "v_tok_1786": 52051,
+  "v_tok_1787": 52052,
+  "v_tok_1788": 52053,
+  "v_tok_1789": 52054,
   "v_tok_179": 50444,
+  "v_tok_1790": 52055,
+  "v_tok_1791": 52056,
+  "v_tok_1792": 52057,
+  "v_tok_1793": 52058,
+  "v_tok_1794": 52059,
+  "v_tok_1795": 52060,
+  "v_tok_1796": 52061,
+  "v_tok_1797": 52062,
+  "v_tok_1798": 52063,
+  "v_tok_1799": 52064,
   "v_tok_18": 50283,
   "v_tok_180": 50445,
+  "v_tok_1800": 52065,
+  "v_tok_1801": 52066,
+  "v_tok_1802": 52067,
+  "v_tok_1803": 52068,
+  "v_tok_1804": 52069,
+  "v_tok_1805": 52070,
+  "v_tok_1806": 52071,
+  "v_tok_1807": 52072,
+  "v_tok_1808": 52073,
+  "v_tok_1809": 52074,
   "v_tok_181": 50446,
+  "v_tok_1810": 52075,
+  "v_tok_1811": 52076,
+  "v_tok_1812": 52077,
+  "v_tok_1813": 52078,
+  "v_tok_1814": 52079,
+  "v_tok_1815": 52080,
+  "v_tok_1816": 52081,
+  "v_tok_1817": 52082,
+  "v_tok_1818": 52083,
+  "v_tok_1819": 52084,
   "v_tok_182": 50447,
+  "v_tok_1820": 52085,
+  "v_tok_1821": 52086,
+  "v_tok_1822": 52087,
+  "v_tok_1823": 52088,
+  "v_tok_1824": 52089,
+  "v_tok_1825": 52090,
+  "v_tok_1826": 52091,
+  "v_tok_1827": 52092,
+  "v_tok_1828": 52093,
+  "v_tok_1829": 52094,
   "v_tok_183": 50448,
+  "v_tok_1830": 52095,
+  "v_tok_1831": 52096,
+  "v_tok_1832": 52097,
+  "v_tok_1833": 52098,
+  "v_tok_1834": 52099,
+  "v_tok_1835": 52100,
+  "v_tok_1836": 52101,
+  "v_tok_1837": 52102,
+  "v_tok_1838": 52103,
+  "v_tok_1839": 52104,
   "v_tok_184": 50449,
+  "v_tok_1840": 52105,
+  "v_tok_1841": 52106,
+  "v_tok_1842": 52107,
+  "v_tok_1843": 52108,
+  "v_tok_1844": 52109,
+  "v_tok_1845": 52110,
+  "v_tok_1846": 52111,
+  "v_tok_1847": 52112,
+  "v_tok_1848": 52113,
+  "v_tok_1849": 52114,
   "v_tok_185": 50450,
+  "v_tok_1850": 52115,
+  "v_tok_1851": 52116,
+  "v_tok_1852": 52117,
+  "v_tok_1853": 52118,
+  "v_tok_1854": 52119,
+  "v_tok_1855": 52120,
+  "v_tok_1856": 52121,
+  "v_tok_1857": 52122,
+  "v_tok_1858": 52123,
+  "v_tok_1859": 52124,
   "v_tok_186": 50451,
+  "v_tok_1860": 52125,
+  "v_tok_1861": 52126,
+  "v_tok_1862": 52127,
+  "v_tok_1863": 52128,
+  "v_tok_1864": 52129,
+  "v_tok_1865": 52130,
+  "v_tok_1866": 52131,
+  "v_tok_1867": 52132,
+  "v_tok_1868": 52133,
+  "v_tok_1869": 52134,
   "v_tok_187": 50452,
+  "v_tok_1870": 52135,
+  "v_tok_1871": 52136,
+  "v_tok_1872": 52137,
+  "v_tok_1873": 52138,
+  "v_tok_1874": 52139,
+  "v_tok_1875": 52140,
+  "v_tok_1876": 52141,
+  "v_tok_1877": 52142,
+  "v_tok_1878": 52143,
+  "v_tok_1879": 52144,
   "v_tok_188": 50453,
+  "v_tok_1880": 52145,
+  "v_tok_1881": 52146,
+  "v_tok_1882": 52147,
+  "v_tok_1883": 52148,
+  "v_tok_1884": 52149,
+  "v_tok_1885": 52150,
+  "v_tok_1886": 52151,
+  "v_tok_1887": 52152,
+  "v_tok_1888": 52153,
+  "v_tok_1889": 52154,
   "v_tok_189": 50454,
+  "v_tok_1890": 52155,
+  "v_tok_1891": 52156,
+  "v_tok_1892": 52157,
+  "v_tok_1893": 52158,
+  "v_tok_1894": 52159,
+  "v_tok_1895": 52160,
+  "v_tok_1896": 52161,
+  "v_tok_1897": 52162,
+  "v_tok_1898": 52163,
+  "v_tok_1899": 52164,
   "v_tok_19": 50284,
   "v_tok_190": 50455,
+  "v_tok_1900": 52165,
+  "v_tok_1901": 52166,
+  "v_tok_1902": 52167,
+  "v_tok_1903": 52168,
+  "v_tok_1904": 52169,
+  "v_tok_1905": 52170,
+  "v_tok_1906": 52171,
+  "v_tok_1907": 52172,
+  "v_tok_1908": 52173,
+  "v_tok_1909": 52174,
   "v_tok_191": 50456,
+  "v_tok_1910": 52175,
+  "v_tok_1911": 52176,
+  "v_tok_1912": 52177,
+  "v_tok_1913": 52178,
+  "v_tok_1914": 52179,
+  "v_tok_1915": 52180,
+  "v_tok_1916": 52181,
+  "v_tok_1917": 52182,
+  "v_tok_1918": 52183,
+  "v_tok_1919": 52184,
   "v_tok_192": 50457,
+  "v_tok_1920": 52185,
+  "v_tok_1921": 52186,
+  "v_tok_1922": 52187,
+  "v_tok_1923": 52188,
+  "v_tok_1924": 52189,
+  "v_tok_1925": 52190,
+  "v_tok_1926": 52191,
+  "v_tok_1927": 52192,
+  "v_tok_1928": 52193,
+  "v_tok_1929": 52194,
   "v_tok_193": 50458,
+  "v_tok_1930": 52195,
+  "v_tok_1931": 52196,
+  "v_tok_1932": 52197,
+  "v_tok_1933": 52198,
+  "v_tok_1934": 52199,
+  "v_tok_1935": 52200,
+  "v_tok_1936": 52201,
+  "v_tok_1937": 52202,
+  "v_tok_1938": 52203,
+  "v_tok_1939": 52204,
   "v_tok_194": 50459,
+  "v_tok_1940": 52205,
+  "v_tok_1941": 52206,
+  "v_tok_1942": 52207,
+  "v_tok_1943": 52208,
+  "v_tok_1944": 52209,
+  "v_tok_1945": 52210,
+  "v_tok_1946": 52211,
+  "v_tok_1947": 52212,
+  "v_tok_1948": 52213,
+  "v_tok_1949": 52214,
   "v_tok_195": 50460,
+  "v_tok_1950": 52215,
+  "v_tok_1951": 52216,
+  "v_tok_1952": 52217,
+  "v_tok_1953": 52218,
+  "v_tok_1954": 52219,
+  "v_tok_1955": 52220,
+  "v_tok_1956": 52221,
+  "v_tok_1957": 52222,
+  "v_tok_1958": 52223,
+  "v_tok_1959": 52224,
   "v_tok_196": 50461,
+  "v_tok_1960": 52225,
+  "v_tok_1961": 52226,
+  "v_tok_1962": 52227,
+  "v_tok_1963": 52228,
+  "v_tok_1964": 52229,
+  "v_tok_1965": 52230,
+  "v_tok_1966": 52231,
+  "v_tok_1967": 52232,
+  "v_tok_1968": 52233,
+  "v_tok_1969": 52234,
   "v_tok_197": 50462,
+  "v_tok_1970": 52235,
+  "v_tok_1971": 52236,
+  "v_tok_1972": 52237,
+  "v_tok_1973": 52238,
+  "v_tok_1974": 52239,
+  "v_tok_1975": 52240,
+  "v_tok_1976": 52241,
+  "v_tok_1977": 52242,
+  "v_tok_1978": 52243,
+  "v_tok_1979": 52244,
   "v_tok_198": 50463,
+  "v_tok_1980": 52245,
+  "v_tok_1981": 52246,
+  "v_tok_1982": 52247,
+  "v_tok_1983": 52248,
+  "v_tok_1984": 52249,
+  "v_tok_1985": 52250,
+  "v_tok_1986": 52251,
+  "v_tok_1987": 52252,
+  "v_tok_1988": 52253,
+  "v_tok_1989": 52254,
   "v_tok_199": 50464,
+  "v_tok_1990": 52255,
+  "v_tok_1991": 52256,
+  "v_tok_1992": 52257,
+  "v_tok_1993": 52258,
+  "v_tok_1994": 52259,
+  "v_tok_1995": 52260,
+  "v_tok_1996": 52261,
+  "v_tok_1997": 52262,
+  "v_tok_1998": 52263,
+  "v_tok_1999": 52264,
   "v_tok_2": 50267,
   "v_tok_20": 50285,
   "v_tok_200": 50465,

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff