{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "_", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFD" }, { "type": "StripAccents" }, { "type": "Lowercase" } ] }, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": null, "decoder": null, "model": { "type": "WordLevel", "vocab": { "_": 0, "[PAD]": 1, "[UNK]": 2, "[BOS]": 3, "[EOS]": 4, "a": 5, "b": 6, "c": 7, "d": 8, "e": 9, "f": 10, "g": 11, "h": 12, "i": 13, "j": 14, "k": 15, "l": 16, "m": 17, "n": 18, "o": 19, "p": 20, "q": 21, "r": 22, "s": 23, "t": 24, "u": 25, "v": 26, "w": 27, "x": 28, "y": 29, "z": 30, "0": 31, "1": 32, "2": 33, "3": 34, "4": 35, "5": 36, "6": 37, "7": 38, "8": 39, "9": 40, "10": 41, "11": 42, "12": 43, "13": 44, "14": 45, "15": 46, "16": 47, "17": 48, "18": 49, "19": 50, "20": 51, "21": 52, "22": 53, "23": 54, "24": 55, "25": 56, "26": 57, "27": 58, "28": 59, "29": 60, "30": 61, "31": 62, "32": 63, "33": 64, "34": 65, "35": 66, "36": 67, "37": 68, "38": 69, "39": 70, "40": 71, "41": 72, "42": 73, "43": 74, "44": 75, "45": 76, "46": 77, "47": 78, "48": 79, "49": 80, "50": 81, "51": 82, "52": 83, "53": 84, "54": 85, "55": 86, "56": 87, "57": 88, "58": 89, "59": 90, "60": 91, "61": 92, "62": 93, "63": 94, "64": 95, "65": 96, "66": 97, "67": 98, "68": 99, "69": 100, "70": 101, "71": 102, "72": 103, "73": 104, "74": 105, "75": 106, "76": 107, "77": 108, "78": 109, "79": 110, "80": 111, "81": 112, "82": 113, "83": 114, "84": 115, "85": 116, "86": 117, "87": 118, "88": 119, "89": 120, "90": 121, "91": 122, "92": 123, "93": 124, "94": 125, "95": 126, "96": 127, "97": 128, "98": 129, "99": 130, "100": 131, "101": 132, "102": 133, "103": 134, "104": 135, "105": 136, "106": 137, "107": 138, "108": 139, "109": 140, "110": 141, "111": 142, "112": 143, "113": 144, "114": 145, "115": 146, "116": 147, "117": 148, "118": 149, "119": 150, "120": 151, "121": 152, "122": 153, "123": 154, "124": 155, "125": 156, "126": 157, "127": 158, "128": 159, "129": 160, "130": 161, "131": 162, "132": 163, "133": 164, "134": 165, "135": 166, "136": 167, "137": 168, "138": 169, "139": 170, "140": 171, "141": 172, "142": 173, "143": 174, "144": 175, "145": 176, "146": 177, "147": 178, "148": 179, "149": 180, "150": 181, "151": 182, "152": 183, "153": 184, "154": 185, "155": 186, "156": 187, "157": 188, "158": 189, "159": 190, "160": 191, "161": 192, "162": 193, "163": 194, "164": 195, "165": 196, "166": 197, "167": 198, "168": 199, "169": 200, "170": 201, "171": 202, "172": 203, "173": 204, "174": 205, "175": 206, "176": 207, "177": 208, "178": 209, "179": 210, "180": 211, "181": 212, "182": 213, "183": 214, "184": 215, "185": 216, "186": 217, "187": 218, "188": 219, "189": 220, "190": 221, "191": 222, "192": 223, "193": 224, "194": 225, "195": 226, "196": 227, "197": 228, "198": 229, "199": 230, "200": 231, "201": 232, "202": 233, "203": 234, "204": 235, "205": 236, "206": 237, "207": 238, "208": 239, "209": 240, "210": 241, "211": 242, "212": 243, "213": 244, "214": 245, "215": 246, "216": 247, "217": 248, "218": 249, "219": 250, "220": 251, "221": 252, "222": 253, "223": 254, "224": 255, "225": 256, "226": 257, "227": 258, "228": 259, "229": 260, "230": 261, "231": 262, "232": 263, "233": 264, "234": 265, "235": 266, "236": 267, "237": 268, "238": 269, "239": 270, "240": 271, "241": 272, "242": 273, "243": 274, "244": 275, "245": 276, "246": 277, "247": 278, "248": 279, "249": 280, "250": 281, "251": 282, "252": 283, "253": 284, "254": 285, "255": 286, "256": 287, "257": 288, "258": 289, "259": 290, "260": 291, "261": 292, "262": 293, "263": 294, "264": 295, "265": 296, "266": 297, "267": 298, "268": 299, "269": 300, "270": 301, "271": 302, "272": 303, "273": 304, "274": 305, "275": 306, "276": 307, "277": 308, "278": 309, "279": 310, "280": 311, "281": 312, "282": 313, "283": 314, "284": 315, "285": 316, "286": 317, "287": 318, "288": 319, "289": 320, "290": 321, "291": 322, "292": 323, "293": 324, "294": 325, "295": 326, "296": 327, "297": 328, "298": 329, "299": 330, "300": 331, "301": 332, "302": 333, "303": 334, "304": 335, "305": 336, "306": 337, "307": 338, "308": 339, "309": 340, "310": 341, "311": 342, "312": 343, "313": 344, "314": 345, "315": 346, "316": 347, "317": 348, "318": 349, "319": 350, "320": 351, "321": 352, "322": 353, "323": 354, "324": 355, "325": 356, "326": 357, "327": 358, "328": 359, "329": 360, "330": 361, "331": 362, "332": 363, "333": 364, "334": 365, "335": 366, "336": 367, "337": 368, "338": 369, "339": 370, "340": 371, "341": 372, "342": 373, "343": 374, "344": 375, "345": 376, "346": 377, "347": 378, "348": 379, "349": 380, "350": 381, "351": 382, "352": 383, "353": 384, "354": 385, "355": 386, "356": 387, "357": 388, "358": 389, "359": 390, "360": 391, "361": 392, "362": 393, "363": 394, "364": 395, "365": 396, "366": 397, "367": 398, "368": 399, "369": 400, "370": 401, "371": 402, "372": 403, "373": 404, "374": 405, "375": 406, "376": 407, "377": 408, "378": 409, "379": 410, "380": 411, "381": 412, "382": 413, "383": 414, "384": 415, "385": 416, "386": 417, "387": 418, "388": 419, "389": 420, "390": 421, "391": 422, "392": 423, "393": 424, "394": 425, "395": 426, "396": 427, "397": 428, "398": 429, "399": 430, "400": 431, "401": 432, "402": 433, "403": 434, "404": 435, "405": 436, "406": 437, "407": 438, "408": 439, "409": 440, "410": 441, "411": 442, "412": 443, "413": 444, "414": 445, "415": 446, "416": 447, "417": 448, "418": 449, "419": 450, "420": 451, "421": 452, "422": 453, "423": 454, "424": 455, "425": 456, "426": 457, "427": 458, "428": 459, "429": 460, "430": 461, "431": 462, "432": 463, "433": 464, "434": 465, "435": 466, "436": 467, "437": 468, "438": 469, "439": 470, "440": 471, "441": 472, "442": 473, "443": 474, "444": 475, "445": 476, "446": 477, "447": 478, "448": 479, "449": 480, "450": 481, "451": 482, "452": 483, "453": 484, "454": 485, "455": 486, "456": 487, "457": 488, "458": 489, "459": 490, "460": 491, "461": 492, "462": 493, "463": 494, "464": 495, "465": 496, "466": 497, "467": 498, "468": 499, "469": 500, "470": 501, "471": 502, "472": 503, "473": 504, "474": 505, "475": 506, "476": 507, "477": 508, "478": 509, "479": 510, "480": 511, "481": 512, "482": 513, "483": 514, "484": 515, "485": 516, "486": 517, "487": 518, "488": 519, "489": 520, "490": 521, "491": 522, "492": 523, "493": 524, "494": 525, "495": 526, "496": 527, "497": 528, "498": 529, "499": 530, "500": 531, "501": 532, "502": 533, "503": 534, "504": 535, "505": 536, "506": 537, "507": 538, "508": 539, "509": 540, "510": 541, "511": 542, "512": 543, "513": 544, "514": 545, "515": 546, "516": 547, "517": 548, "518": 549, "519": 550, "520": 551, "521": 552, "522": 553, "523": 554, "524": 555, "525": 556, "526": 557, "527": 558, "528": 559, "529": 560, "530": 561, "531": 562, "532": 563, "533": 564, "534": 565, "535": 566, "536": 567, "537": 568, "538": 569, "539": 570, "540": 571, "541": 572, "542": 573, "543": 574, "544": 575, "545": 576, "546": 577, "547": 578, "548": 579, "549": 580, "550": 581, "551": 582, "552": 583, "553": 584, "554": 585, "555": 586, "556": 587, "557": 588, "558": 589, "559": 590, "560": 591, "561": 592, "562": 593, "563": 594, "564": 595, "565": 596, "566": 597, "567": 598, "568": 599, "569": 600, "570": 601, "571": 602, "572": 603, "573": 604, "574": 605, "575": 606, "576": 607, "577": 608, "578": 609, "579": 610, "580": 611, "581": 612, "582": 613, "583": 614, "584": 615, "585": 616, "586": 617, "587": 618, "588": 619, "589": 620, "590": 621, "591": 622, "592": 623, "593": 624, "594": 625, "595": 626, "596": 627, "597": 628, "598": 629, "599": 630, "600": 631, "601": 632, "602": 633, "603": 634, "604": 635, "605": 636, "606": 637, "607": 638, "608": 639, "609": 640, "610": 641, "611": 642, "612": 643, "613": 644, "614": 645, "615": 646, "616": 647, "617": 648, "618": 649, "619": 650, "620": 651, "621": 652, "622": 653, "623": 654, "624": 655, "625": 656, "626": 657, "627": 658, "628": 659, "629": 660, "630": 661, "631": 662, "632": 663, "633": 664, "634": 665, "635": 666, "636": 667, "637": 668, "638": 669, "639": 670, "640": 671, "641": 672, "642": 673, "643": 674, "644": 675, "645": 676, "646": 677, "647": 678, "648": 679, "649": 680, "650": 681, "651": 682, "652": 683, "653": 684, "654": 685, "655": 686, "656": 687, "657": 688, "658": 689, "659": 690, "660": 691, "661": 692, "662": 693, "663": 694, "664": 695, "665": 696, "666": 697, "667": 698, "668": 699, "669": 700, "670": 701, "671": 702, "672": 703, "673": 704, "674": 705, "675": 706, "676": 707, "677": 708, "678": 709, "679": 710, "680": 711, "681": 712, "682": 713, "683": 714, "684": 715, "685": 716, "686": 717, "687": 718, "688": 719, "689": 720, "690": 721, "691": 722, "692": 723, "693": 724, "694": 725, "695": 726, "696": 727, "697": 728, "698": 729, "699": 730, "700": 731, "701": 732, "702": 733, "703": 734, "704": 735, "705": 736, "706": 737, "707": 738, "708": 739, "709": 740, "710": 741, "711": 742, "712": 743, "713": 744, "714": 745, "715": 746, "716": 747, "717": 748, "718": 749, "719": 750, "720": 751, "721": 752, "722": 753, "723": 754, "724": 755, "725": 756, "726": 757, "727": 758, "728": 759, "729": 760, "730": 761, "731": 762, "732": 763, "733": 764, "734": 765, "735": 766, "736": 767, "737": 768, "738": 769, "739": 770, "740": 771, "741": 772, "742": 773, "743": 774, "744": 775, "745": 776, "746": 777, "747": 778, "748": 779, "749": 780, "750": 781, "751": 782, "752": 783, "753": 784, "754": 785, "755": 786, "756": 787, "757": 788, "758": 789, "759": 790, "760": 791, "761": 792, "762": 793, "763": 794, "764": 795, "765": 796, "766": 797, "767": 798, "768": 799, "769": 800, "770": 801, "771": 802, "772": 803, "773": 804, "774": 805, "775": 806, "776": 807, "777": 808, "778": 809, "779": 810, "780": 811, "781": 812, "782": 813, "783": 814, "784": 815, "785": 816, "786": 817, "787": 818, "788": 819, "789": 820, "790": 821, "791": 822, "792": 823, "793": 824, "794": 825, "795": 826, "796": 827, "797": 828, "798": 829, "799": 830, "800": 831, "801": 832, "802": 833, "803": 834, "804": 835, "805": 836, "806": 837, "807": 838, "808": 839, "809": 840, "810": 841, "811": 842, "812": 843, "813": 844, "814": 845, "815": 846, "816": 847, "817": 848, "818": 849, "819": 850, "820": 851, "821": 852, "822": 853, "823": 854, "824": 855, "825": 856, "826": 857, "827": 858, "828": 859, "829": 860, "830": 861, "831": 862, "832": 863, "833": 864, "834": 865, "835": 866, "836": 867, "837": 868, "838": 869, "839": 870, "840": 871, "841": 872, "842": 873, "843": 874, "844": 875, "845": 876, "846": 877, "847": 878, "848": 879, "849": 880, "850": 881, "851": 882, "852": 883, "853": 884, "854": 885, "855": 886, "856": 887, "857": 888, "858": 889, "859": 890, "860": 891, "861": 892, "862": 893, "863": 894, "864": 895, "865": 896, "866": 897, "867": 898, "868": 899, "869": 900, "870": 901, "871": 902, "872": 903, "873": 904, "874": 905, "875": 906, "876": 907, "877": 908, "878": 909, "879": 910, "880": 911, "881": 912, "882": 913, "883": 914, "884": 915, "885": 916, "886": 917, "887": 918, "888": 919, "889": 920, "890": 921, "891": 922, "892": 923, "893": 924, "894": 925, "895": 926, "896": 927, "897": 928, "898": 929, "899": 930, "900": 931, "901": 932, "902": 933, "903": 934, "904": 935, "905": 936, "906": 937, "907": 938, "908": 939, "909": 940, "910": 941, "911": 942, "912": 943, "913": 944, "914": 945, "915": 946, "916": 947, "917": 948, "918": 949, "919": 950, "920": 951, "921": 952, "922": 953, "923": 954, "924": 955, "925": 956, "926": 957, "927": 958, "928": 959, "929": 960, "930": 961, "931": 962, "932": 963, "933": 964, "934": 965, "935": 966, "936": 967, "937": 968, "938": 969, "939": 970, "940": 971, "941": 972, "942": 973, "943": 974, "944": 975, "945": 976, "946": 977, "947": 978, "948": 979, "949": 980, "950": 981, "951": 982, "952": 983, "953": 984, "954": 985, "955": 986, "956": 987, "957": 988, "958": 989, "959": 990, "960": 991, "961": 992, "962": 993, "963": 994, "964": 995, "965": 996, "966": 997, "967": 998, "968": 999, "969": 1000, "970": 1001, "971": 1002, "972": 1003, "973": 1004, "974": 1005, "975": 1006, "976": 1007, "977": 1008, "978": 1009, "979": 1010, "980": 1011, "981": 1012, "982": 1013, "983": 1014, "984": 1015, "985": 1016, "986": 1017, "987": 1018, "988": 1019, "989": 1020, "990": 1021, "991": 1022, "992": 1023, "993": 1024, "994": 1025, "995": 1026, "996": 1027, "997": 1028, "998": 1029, "999": 1030, "1000": 1031, "1001": 1032, "1002": 1033, "1003": 1034, "1004": 1035, "1005": 1036, "1006": 1037, "1007": 1038, "1008": 1039, "1009": 1040, "1010": 1041, "1011": 1042, "1012": 1043, "1013": 1044, "1014": 1045, "1015": 1046, "1016": 1047, "1017": 1048, "1018": 1049, "1019": 1050, "1020": 1051, "1021": 1052, "1022": 1053, "1023": 1054, "1024": 1055, "|": 1056, "?": 1057, "(": 1058, ")": 1059 }, "unk_token": "[UNK]" } }