Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -137,10 +137,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "vocab": {
-      "<pad>": 0,
-      "<unk>": 1,
-      "<s>": 2,
-      "</s>": 3,
       "<mask>": 4,
       "er": 5,
       "in": 6,
@@ -363,8 +359,8 @@
       "ef": 223,
       "▁pr": 224,
       "ating": 225,
-      "ick": 226,
-      "ment": 227,
+      "ment": 226,
+      "ick": 227,
       "li": 228,
       "ys": 229,
       "aw": 230,
@@ -422,8 +418,8 @@
       "ew": 282,
       "▁le": 283,
       "▁en": 284,
-      "▁ro": 285,
-      "▁I": 286,
+      "▁I": 285,
+      "▁ro": 286,
       "ear": 287,
       "ille": 288,
       "▁par": 289,
@@ -456,17 +452,17 @@
       "ox": 316,
       "ge": 317,
       "row": 318,
-      "▁V": 319,
-      "pr": 320,
+      "pr": 319,
+      "▁V": 320,
       "ash": 321,
       "let": 322,
-      "led": 323,
-      "int": 324,
+      "int": 323,
+      "led": 324,
       "eth": 325,
       "ative": 326,
       "▁tw": 327,
-      "att": 328,
-      "ont": 329,
+      "ont": 328,
+      "att": 329,
       "ore": 330,
       "▁ad": 331,
       "ling": 332,
@@ -479,13 +475,13 @@
       "▁wh": 339,
       "ail": 340,
       "▁pol": 341,
-      "fl": 342,
-      "▁Ch": 343,
-      "▁y": 344,
+      "▁y": 342,
+      "fl": 343,
+      "▁Ch": 344,
       "ax": 345,
       "az": 346,
-      "ater": 347,
-      "ice": 348,
+      "ice": 347,
+      "ater": 348,
       "▁tr": 349,
       "old": 350,
       "ence": 351,
@@ -508,12 +504,12 @@
       "▁under": 368,
       "yt": 369,
       "rig": 370,
-      "ates": 371,
-      "ik": 372,
+      "ik": 371,
+      "ates": 372,
       "▁sl": 373,
       "ann": 374,
-      "yp": 375,
-      "ept": 376,
+      "ept": 375,
+      "yp": 376,
       "ile": 377,
       "▁for": 378,
       "vers": 379,
@@ -539,9 +535,9 @@
       "ey": 399,
       "▁quasi": 400,
       "ors": 401,
-      "ily": 402,
-      "rans": 403,
-      "ull": 404,
+      "ull": 402,
+      "ily": 403,
+      "rans": 404,
       "ship": 405,
       "▁ne": 406,
       "ea": 407,
@@ -554,8 +550,8 @@
       "uck": 414,
       "oot": 415,
       "ark": 416,
-      "ush": 417,
-      "erv": 418,
+      "erv": 417,
+      "ush": 418,
       "ton": 419,
       "ou": 420,
       "ork": 421,
@@ -655,11 +651,11 @@
       "itis": 515,
       "ari": 516,
       "▁Al": 517,
-      "orr": 518,
-      "ual": 519,
-      "▁int": 520,
-      "yg": 521,
-      "eli": 522,
+      "eli": 518,
+      "orr": 519,
+      "ual": 520,
+      "▁int": 521,
+      "yg": 522,
       "ress": 523,
       "iveness": 524,
       "au": 525,
@@ -670,8 +666,8 @@
       "agg": 530,
       "ank": 531,
       "les": 532,
-      "ify": 533,
-      "ern": 534,
+      "ern": 533,
+      "ify": 534,
       "ible": 535,
       "rat": 536,
       "▁hyper": 537,
@@ -751,9 +747,9 @@
       "▁Z": 611,
       "yll": 612,
       "▁cal": 613,
-      "ines": 614,
-      "▁nond": 615,
-      "ren": 616,
+      "ren": 614,
+      "ines": 615,
+      "▁nond": 616,
       "ring": 617,
       "oon": 618,
       "▁col": 619,
@@ -768,8 +764,8 @@
       "eless": 628,
       "arn": 629,
       "▁Ar": 630,
-      "▁syn": 631,
-      "opl": 632,
+      "opl": 631,
+      "▁syn": 632,
       "▁squ": 633,
       "▁inf": 634,
       "▁se": 635,
@@ -787,9 +783,9 @@
       "work": 647,
       "be": 648,
       "col": 649,
-      "▁Sh": 650,
-      "ics": 651,
-      "ven": 652,
+      "ven": 650,
+      "▁Sh": 651,
+      "ics": 652,
       "▁unre": 653,
       "ipp": 654,
       "▁nons": 655,
@@ -805,8 +801,8 @@
       "ru": 665,
       "▁pred": 666,
       "form": 667,
-      "amm": 668,
-      "▁end": 669,
+      "▁end": 668,
+      "amm": 669,
       "ild": 670,
       "ower": 671,
       "erg": 672,
@@ -816,12 +812,12 @@
       "off": 676,
       "▁Ph": 677,
       "▁ung": 678,
-      "▁mult": 679,
-      "aster": 680,
-      "iously": 681,
-      "agn": 682,
-      "iph": 683,
-      "▁ver": 684,
+      "▁ver": 679,
+      "▁mult": 680,
+      "aster": 681,
+      "iously": 682,
+      "agn": 683,
+      "iph": 684,
       "ological": 685,
       "rous": 686,
       "aur": 687,
@@ -854,8 +850,8 @@
       "orth": 714,
       "▁ref": 715,
       "eral": 716,
-      "▁op": 717,
-      "ency": 718,
+      "ency": 717,
+      "▁op": 718,
       "que": 719,
       "▁unh": 720,
       "▁uncon": 721,
@@ -877,9 +873,9 @@
       "urs": 737,
       "▁rh": 738,
       "▁sul": 739,
-      "leg": 740,
-      "iferous": 741,
-      "yc": 742,
+      "yc": 740,
+      "leg": 741,
+      "iferous": 742,
       "▁def": 743,
       "press": 744,
       "eal": 745,
@@ -891,10 +887,10 @@
       "ump": 751,
       "▁wor": 752,
       "iate": 753,
-      "ured": 754,
-      "▁dep": 755,
-      "rec": 756,
-      "ean": 757,
+      "▁dep": 754,
+      "ured": 755,
+      "ean": 756,
+      "rec": 757,
       "ech": 758,
       "sych": 759,
       "ently": 760,
@@ -911,9 +907,9 @@
       "ering": 771,
       "vent": 772,
       "mon": 773,
-      "ially": 774,
-      "arr": 775,
-      "ject": 776,
+      "ject": 774,
+      "ially": 775,
+      "arr": 776,
       "▁py": 777,
       "▁the": 778,
       "alk": 779,
@@ -934,11 +930,11 @@
       "org": 794,
       "unc": 795,
       "▁cap": 796,
-      "▁at": 797,
-      "outh": 798,
-      "▁back": 799,
-      "▁prot": 800,
-      "ili": 801,
+      "ili": 797,
+      "▁at": 798,
+      "outh": 799,
+      "▁back": 800,
+      "▁prot": 801,
       "rown": 802,
       "▁x": 803,
       "ranch": 804,
@@ -962,8 +958,8 @@
       "right": 822,
       "omat": 823,
       "rist": 824,
-      "omb": 825,
-      "▁ser": 826,
+      "▁ser": 825,
+      "omb": 826,
       "ites": 827,
       "▁reg": 828,
       "nesses": 829,
@@ -983,13 +979,13 @@
       "unk": 843,
       "▁ass": 844,
       "aped": 845,
-      "ened": 846,
-      "▁emb": 847,
-      "imb": 848,
-      "vol": 849,
-      "osph": 850,
-      "ries": 851,
-      "ras": 852,
+      "ras": 846,
+      "ened": 847,
+      "▁emb": 848,
+      "imb": 849,
+      "vol": 850,
+      "osph": 851,
+      "ries": 852,
       "aced": 853,
       "hear": 854,
       "rem": 855,
@@ -1037,12 +1033,12 @@
       "eng": 897,
       "iac": 898,
       "udd": 899,
-      "over": 900,
-      "bor": 901,
-      "elt": 902,
-      "▁hand": 903,
-      "ingness": 904,
-      "ek": 905,
+      "ek": 900,
+      "over": 901,
+      "▁hand": 902,
+      "bor": 903,
+      "elt": 904,
+      "ingness": 905,
       "umin": 906,
       "ooth": 907,
       "▁world": 908,
@@ -1055,11 +1051,11 @@
       "▁rel": 915,
       "enz": 916,
       "ara": 917,
-      "ops": 918,
-      "ella": 919,
-      "▁comm": 920,
-      "zz": 921,
-      "ext": 922,
+      "ext": 918,
+      "ops": 919,
+      "ella": 920,
+      "▁comm": 921,
+      "zz": 922,
       "lic": 923,
       "arth": 924,
       "unct": 925,
@@ -1087,8 +1083,8 @@
       "headed": 947,
       "usc": 948,
       "iter": 949,
-      "ibility": 950,
-      "po": 951,
+      "po": 950,
+      "ibility": 951,
       "occ": 952,
       "▁gre": 953,
       "ellow": 954,
@@ -1133,9 +1129,9 @@
       "ishness": 993,
       "osp": 994,
       "▁ly": 995,
-      "aint": 996,
-      "eled": 997,
-      "rate": 998,
+      "rate": 996,
+      "aint": 997,
+      "eled": 998,
       "▁ang": 999,
       "elect": 1000,
       "ages": 1001,
@@ -1182,26 +1178,26 @@
       "rew": 1042,
       "oral": 1043,
       "▁med": 1044,
-      "obl": 1045,
-      "osm": 1046,
+      "me": 1045,
+      "obl": 1046,
       "▁unst": 1047,
-      "atin": 1048,
-      "▁can": 1049,
-      "oic": 1050,
-      "elic": 1051,
-      "▁mar": 1052,
-      "▁exc": 1053,
-      "utter": 1054,
-      "atively": 1055,
-      "ophil": 1056,
-      "elling": 1057,
-      "abb": 1058,
-      "opt": 1059,
-      "uth": 1060,
-      "ators": 1061,
-      "▁psych": 1062,
-      "ze": 1063,
-      "me": 1064,
+      "osm": 1048,
+      "atin": 1049,
+      "▁can": 1050,
+      "oic": 1051,
+      "elic": 1052,
+      "▁mar": 1053,
+      "▁exc": 1054,
+      "utter": 1055,
+      "atively": 1056,
+      "ophil": 1057,
+      "elling": 1058,
+      "abb": 1059,
+      "opt": 1060,
+      "uth": 1061,
+      "ators": 1062,
+      "▁psych": 1063,
+      "ze": 1064,
       "ants": 1065,
       "itor": 1066,
       "vert": 1067,
@@ -1236,10 +1232,10 @@
       "ete": 1096,
       "▁ev": 1097,
       "uls": 1098,
-      "▁del": 1099,
-      "ophag": 1100,
-      "▁dist": 1101,
-      "plic": 1102,
+      "plic": 1099,
+      "▁del": 1100,
+      "ophag": 1101,
+      "▁dist": 1102,
       "▁ped": 1103,
       "ution": 1104,
       "alt": 1105,
@@ -1255,9 +1251,9 @@
       "elike": 1115,
       "ilt": 1116,
       "uble": 1117,
-      "aries": 1118,
-      "bed": 1119,
-      "▁pat": 1120,
+      "▁pat": 1118,
+      "aries": 1119,
+      "bed": 1120,
       "▁sulph": 1121,
       "ises": 1122,
       "ately": 1123,
@@ -1289,35 +1285,35 @@
       "ipl": 1149,
       "irt": 1150,
       "yte": 1151,
-      "▁Mar": 1152,
-      "▁gal": 1153,
-      "▁overd": 1154,
-      "▁In": 1155,
+      "▁In": 1152,
+      "▁Mar": 1153,
+      "▁gal": 1154,
+      "▁overd": 1155,
       "▁fr": 1156,
       "ider": 1157,
       "olic": 1158,
       "tail": 1159,
       "▁ost": 1160,
-      "pect": 1161,
-      "▁sym": 1162,
-      "opath": 1163,
-      "ogg": 1164,
+      "ogg": 1161,
+      "pect": 1162,
+      "▁sym": 1163,
+      "opath": 1164,
       "yth": 1165,
       "▁cle": 1166,
       "opter": 1167,
       "▁water": 1168,
-      "lex": 1169,
-      "etal": 1170,
-      "ison": 1171,
-      "▁wind": 1172,
-      "action": 1173,
-      "otyp": 1174,
-      "▁reb": 1175,
-      "ident": 1176,
-      "inate": 1177,
-      "▁pur": 1178,
-      "kn": 1179,
-      "ma": 1180,
+      "ma": 1169,
+      "lex": 1170,
+      "etal": 1171,
+      "ison": 1172,
+      "▁wind": 1173,
+      "action": 1174,
+      "otyp": 1175,
+      "▁reb": 1176,
+      "ident": 1177,
+      "inate": 1178,
+      "▁pur": 1179,
+      "kn": 1180,
       "ino": 1181,
       "▁go": 1182,
       "cons": 1183,
@@ -1446,11 +1442,11 @@
       "▁sac": 1306,
       "iation": 1307,
       "ih": 1308,
-      "orial": 1309,
-      "▁micro": 1310,
-      "alg": 1311,
-      "oil": 1312,
-      "▁Un": 1313,
+      "▁Un": 1309,
+      "orial": 1310,
+      "▁micro": 1311,
+      "alg": 1312,
+      "oil": 1313,
       "▁gen": 1314,
       "▁two": 1315,
       "▁disp": 1316,
@@ -1531,13 +1527,13 @@
       "go": 1391,
       "to": 1392,
       "omen": 1393,
-      "▁mat": 1394,
-      "▁chrom": 1395,
-      "yb": 1396,
-      "awn": 1397,
-      "urr": 1398,
-      "ired": 1399,
-      "▁act": 1400,
+      "▁act": 1394,
+      "▁mat": 1395,
+      "▁chrom": 1396,
+      "yb": 1397,
+      "awn": 1398,
+      "urr": 1399,
+      "ired": 1400,
       "▁cam": 1401,
       "▁meg": 1402,
       "▁ter": 1403,
@@ -1555,36 +1551,36 @@
       "ka": 1415,
       "osc": 1416,
       "▁Or": 1417,
-      "emic": 1418,
-      "ences": 1419,
-      "▁hard": 1420,
-      "oscope": 1421,
-      "pe": 1422,
-      "den": 1423,
-      "aven": 1424,
+      "aven": 1418,
+      "emic": 1419,
+      "ences": 1420,
+      "▁hard": 1421,
+      "oscope": 1422,
+      "pe": 1423,
+      "den": 1424,
       "ytic": 1425,
       "▁amb": 1426,
       "▁unv": 1427,
       "▁mill": 1428,
       "auc": 1429,
-      "ynam": 1430,
-      "▁ren": 1431,
-      "oidal": 1432,
-      "▁fire": 1433,
-      "ape": 1434,
-      "key": 1435,
-      "▁En": 1436,
-      "rong": 1437,
-      "▁bel": 1438,
-      "▁inv": 1439,
-      "▁pet": 1440,
-      "▁pers": 1441,
-      "▁pros": 1442,
-      "ration": 1443,
-      "ini": 1444,
-      "osh": 1445,
-      "ove": 1446,
-      "▁ol": 1447,
+      "▁ol": 1430,
+      "ynam": 1431,
+      "▁ren": 1432,
+      "oidal": 1433,
+      "▁fire": 1434,
+      "ape": 1435,
+      "key": 1436,
+      "▁En": 1437,
+      "rong": 1438,
+      "▁bel": 1439,
+      "▁inv": 1440,
+      "▁pet": 1441,
+      "▁pers": 1442,
+      "▁pros": 1443,
+      "ration": 1444,
+      "ini": 1445,
+      "osh": 1446,
+      "ove": 1447,
       "▁gro": 1448,
       "▁leg": 1449,
       "ection": 1450,
@@ -1837,18 +1833,18 @@
       "rah": 1697,
       "▁du": 1698,
       "asis": 1699,
-      "esia": 1700,
-      "▁bur": 1701,
-      "▁nit": 1702,
-      "▁not": 1703,
-      "ionist": 1704,
-      "▁heart": 1705,
-      "idi": 1706,
-      "oms": 1707,
-      "ulc": 1708,
-      "ared": 1709,
-      "ases": 1710,
-      "eler": 1711,
+      "eler": 1700,
+      "esia": 1701,
+      "▁bur": 1702,
+      "▁nit": 1703,
+      "▁not": 1704,
+      "ionist": 1705,
+      "▁heart": 1706,
+      "idi": 1707,
+      "oms": 1708,
+      "ulc": 1709,
+      "ared": 1710,
+      "ases": 1711,
       "worm": 1712,
       "▁log": 1713,
       "ining": 1714,
@@ -1968,44 +1964,44 @@
       "itting": 1828,
       "ky": 1829,
       "aer": 1830,
-      "oney": 1831,
-      "▁unch": 1832,
-      "▁recon": 1833,
-      "ada": 1834,
-      "orh": 1835,
-      "uzz": 1836,
-      "van": 1837,
-      "▁Ac": 1838,
-      "aric": 1839,
-      "idal": 1840,
-      "▁cow": 1841,
-      "▁tro": 1842,
-      "onger": 1843,
-      "usion": 1844,
-      "▁prim": 1845,
-      "lessly": 1846,
-      "escence": 1847,
-      "yz": 1848,
-      "irl": 1849,
-      "aled": 1850,
-      "eted": 1851,
-      "iner": 1852,
-      "▁cyt": 1853,
-      "ethyl": 1854,
-      "woman": 1855,
-      "▁nonv": 1856,
-      "▁oste": 1857,
-      "▁subd": 1858,
-      "▁overf": 1859,
-      "▁semip": 1860,
-      "lem": 1861,
-      "obb": 1862,
-      "ona": 1863,
-      "ulf": 1864,
-      "yan": 1865,
-      "▁benz": 1866,
-      "ochrom": 1867,
-      "ama": 1868,
+      "lem": 1831,
+      "oney": 1832,
+      "▁unch": 1833,
+      "▁recon": 1834,
+      "ada": 1835,
+      "ama": 1836,
+      "orh": 1837,
+      "uzz": 1838,
+      "van": 1839,
+      "▁Ac": 1840,
+      "aric": 1841,
+      "idal": 1842,
+      "▁cow": 1843,
+      "▁tro": 1844,
+      "onger": 1845,
+      "usion": 1846,
+      "▁prim": 1847,
+      "lessly": 1848,
+      "escence": 1849,
+      "yz": 1850,
+      "irl": 1851,
+      "aled": 1852,
+      "eted": 1853,
+      "iner": 1854,
+      "▁cyt": 1855,
+      "ethyl": 1856,
+      "woman": 1857,
+      "▁nonv": 1858,
+      "▁oste": 1859,
+      "▁subd": 1860,
+      "▁overf": 1861,
+      "▁semip": 1862,
+      "obb": 1863,
+      "ona": 1864,
+      "ulf": 1865,
+      "yan": 1866,
+      "▁benz": 1867,
+      "ochrom": 1868,
       "vel": 1869,
       "agon": 1870,
       "unch": 1871,
@@ -2038,105 +2034,109 @@
       "▁dig": 1898,
       "rogen": 1899,
       "orship": 1900,
-      "rained": 1901,
-      "▁trich": 1902,
-      "erb": 1903,
-      "ken": 1904,
-      "pan": 1905,
-      "▁ax": 1906,
-      "▁ri": 1907,
-      "ested": 1908,
-      "osoph": 1909,
-      "▁hist": 1910,
-      "▁sept": 1911,
-      "cephal": 1912,
-      "e": 1913,
-      "▁": 1914,
-      "i": 1915,
-      "a": 1916,
-      "n": 1917,
-      "o": 1918,
-      "r": 1919,
-      "s": 1920,
-      "t": 1921,
-      "l": 1922,
-      "c": 1923,
-      "u": 1924,
-      "d": 1925,
-      "p": 1926,
-      "m": 1927,
-      "h": 1928,
-      "g": 1929,
-      "y": 1930,
-      "b": 1931,
-      "f": 1932,
-      "-": 1933,
-      "v": 1934,
-      "k": 1935,
-      "w": 1936,
-      "z": 1937,
-      "x": 1938,
-      "S": 1939,
-      "C": 1940,
-      "A": 1941,
-      "q": 1942,
-      "M": 1943,
-      "P": 1944,
-      "j": 1945,
-      "B": 1946,
-      "T": 1947,
-      "L": 1948,
-      "'": 1949,
-      "D": 1950,
-      "H": 1951,
-      "G": 1952,
-      "E": 1953,
-      "R": 1954,
-      "N": 1955,
-      "F": 1956,
-      "K": 1957,
-      "O": 1958,
-      "I": 1959,
-      "W": 1960,
-      ".": 1961,
-      "J": 1962,
-      "V": 1963,
-      "U": 1964,
-      "Z": 1965,
-      "Y": 1966,
-      "/": 1967,
-      "Q": 1968,
-      "X": 1969,
-      ":": 1970,
-      "[": 1971,
-      "]": 1972,
-      "(": 1973,
-      ")": 1974,
-      "2": 1975,
-      "1": 1976,
-      "0": 1977,
-      "=": 1978,
-      ",": 1979,
-      "#": 1980,
-      "3": 1981,
-      "4": 1982,
-      "?": 1983,
-      "\"": 1984,
-      "!": 1985,
-      "&": 1986,
-      "5": 1987,
-      "6": 1988,
-      "8": 1989,
-      "`": 1990,
-      "9": 1991,
-      "%": 1992,
-      "@": 1993,
-      "7": 1994,
-      "<": 1995,
-      ">": 1996,
-      "$": 1997,
-      "+": 1998,
-      ";": 1999
+      "e": 1901,
+      "▁": 1902,
+      "i": 1903,
+      "a": 1904,
+      "n": 1905,
+      "o": 1906,
+      "r": 1907,
+      "s": 1908,
+      "t": 1909,
+      "l": 1910,
+      "c": 1911,
+      "u": 1912,
+      "d": 1913,
+      "p": 1914,
+      "m": 1915,
+      "h": 1916,
+      "g": 1917,
+      "y": 1918,
+      "b": 1919,
+      "f": 1920,
+      "-": 1921,
+      "v": 1922,
+      "k": 1923,
+      "w": 1924,
+      "z": 1925,
+      "x": 1926,
+      "S": 1927,
+      "C": 1928,
+      "A": 1929,
+      "q": 1930,
+      "M": 1931,
+      "P": 1932,
+      "j": 1933,
+      "B": 1934,
+      "T": 1935,
+      "L": 1936,
+      "'": 1937,
+      "D": 1938,
+      "H": 1939,
+      "G": 1940,
+      "E": 1941,
+      "R": 1942,
+      "N": 1943,
+      "F": 1944,
+      "K": 1945,
+      "O": 1946,
+      "I": 1947,
+      "W": 1948,
+      ".": 1949,
+      "J": 1950,
+      "V": 1951,
+      "U": 1952,
+      "Z": 1953,
+      "Y": 1954,
+      "/": 1955,
+      "Q": 1956,
+      "X": 1957,
+      ":": 1958,
+      "[": 1959,
+      "]": 1960,
+      "(": 1961,
+      ")": 1962,
+      "2": 1963,
+      "1": 1964,
+      "0": 1965,
+      "=": 1966,
+      ",": 1967,
+      "ı": 1968,
+      "#": 1969,
+      "3": 1970,
+      "4": 1971,
+      "?": 1972,
+      "\"": 1973,
+      "ü": 1974,
+      "!": 1975,
+      "&": 1976,
+      "5": 1977,
+      "ş": 1978,
+      "6": 1979,
+      "8": 1980,
+      "`": 1981,
+      "ö": 1982,
+      "9": 1983,
+      "ğ": 1984,
+      "%": 1985,
+      "@": 1986,
+      "7": 1987,
+      "<": 1988,
+      ">": 1989,
+      "Ö": 1990,
+      "ç": 1991,
+      "$": 1992,
+      "+": 1993,
+      ";": 1994,
+      "Ç": 1995,
+      "Ü": 1996,
+      "Ğ": 1997,
+      "İ": 1998,
+      "Ş": 1999,
+      "<pad>": 0,
+      "<unk>": 1,
+      "<s>": 2,
+      "</s>": 3
     },
     "merges": []
   }