{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "_", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFD" }, { "type": "StripAccents" }, { "type": "Lowercase" } ] }, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": null, "decoder": null, "model": { "type": "WordLevel", "vocab": { "_": 0, "[PAD]": 1, "[UNK]": 2, "[BOS]": 3, "[EOS]": 4, "a": 5, "b": 6, "c": 7, "d": 8, "e": 9, "f": 10, "g": 11, "h": 12, "i": 13, "j": 14, "k": 15, "l": 16, "m": 17, "n": 18, "o": 19, "p": 20, "q": 21, "r": 22, "s": 23, "t": 24, "u": 25, "v": 26, "w": 27, "x": 28, "y": 29, "z": 30, "0": 31, "1": 32, "2": 33, "3": 34, "4": 35, "5": 36, "6": 37, "7": 38, "8": 39, "9": 40, "10": 41, "11": 42, "12": 43, "13": 44, "14": 45, "15": 46, "16": 47, "17": 48, "18": 49, "19": 50, "20": 51, "21": 52, "22": 53, "23": 54, "24": 55, "25": 56, "26": 57, "27": 58, "28": 59, "29": 60, "30": 61, "31": 62, "32": 63, "33": 64, "34": 65, "35": 66, "36": 67, "37": 68, "38": 69, "39": 70, "40": 71, "41": 72, "42": 73, "43": 74, "44": 75, "45": 76, "46": 77, "47": 78, "48": 79, "49": 80, "50": 81, "51": 82, "52": 83, "53": 84, "54": 85, "55": 86, "56": 87, "57": 88, "58": 89, "59": 90, "60": 91, "61": 92, "62": 93, "63": 94, "64": 95, "65": 96, "66": 97, "67": 98, "68": 99, "69": 100, "70": 101, "71": 102, "72": 103, "73": 104, "74": 105, "75": 106, "76": 107, "77": 108, "78": 109, "79": 110, "80": 111, "81": 112, "82": 113, "83": 114, "84": 115, "85": 116, "86": 117, "87": 118, "88": 119, "89": 120, "90": 121, "91": 122, "92": 123, "93": 124, "94": 125, "95": 126, "96": 127, "97": 128, "98": 129, "99": 130, "100": 131, "101": 132, "102": 133, "103": 134, "104": 135, "105": 136, "106": 137, "107": 138, "108": 139, "109": 140, "110": 141, "111": 142, "112": 143, "113": 144, "114": 145, "115": 146, "116": 147, "117": 148, "118": 149, "119": 150, "120": 151, "121": 152, "122": 153, "123": 154, "124": 155, "125": 156, "126": 157, "127": 158, "128": 159, "129": 160, "130": 161, "131": 162, "132": 163, "133": 164, "134": 165, "135": 166, "136": 167, "137": 168, "138": 169, "139": 170, "140": 171, "141": 172, "142": 173, "143": 174, "144": 175, "145": 176, "146": 177, "147": 178, "148": 179, "149": 180, "150": 181, "151": 182, "152": 183, "153": 184, "154": 185, "155": 186, "156": 187, "157": 188, "158": 189, "159": 190, "160": 191, "161": 192, "162": 193, "163": 194, "164": 195, "165": 196, "166": 197, "167": 198, "168": 199, "169": 200, "170": 201, "171": 202, "172": 203, "173": 204, "174": 205, "175": 206, "176": 207, "177": 208, "178": 209, "179": 210, "180": 211, "181": 212, "182": 213, "183": 214, "184": 215, "185": 216, "186": 217, "187": 218, "188": 219, "189": 220, "190": 221, "191": 222, "192": 223, "193": 224, "194": 225, "195": 226, "196": 227, "197": 228, "198": 229, "199": 230, "200": 231, "201": 232, "202": 233, "203": 234, "204": 235, "205": 236, "206": 237, "207": 238, "208": 239, "209": 240, "210": 241, "211": 242, "212": 243, "213": 244, "214": 245, "215": 246, "216": 247, "217": 248, "218": 249, "219": 250, "220": 251, "221": 252, "222": 253, "223": 254, "224": 255, "225": 256, "226": 257, "227": 258, "228": 259, "229": 260, "230": 261, "231": 262, "232": 263, "233": 264, "234": 265, "235": 266, "236": 267, "237": 268, "238": 269, "239": 270, "240": 271, "241": 272, "242": 273, "243": 274, "244": 275, "245": 276, "246": 277, "247": 278, "248": 279, "249": 280, "250": 281, "251": 282, "252": 283, "253": 284, "254": 285, "255": 286, "256": 287, "257": 288, "258": 289, "259": 290, "260": 291, "261": 292, "262": 293, "263": 294, "264": 295, "265": 296, "266": 297, "267": 298, "268": 299, "269": 300, "270": 301, "271": 302, "272": 303, "273": 304, "274": 305, "275": 306, "276": 307, "277": 308, "278": 309, "279": 310, "280": 311, "281": 312, "282": 313, "283": 314, "284": 315, "285": 316, "286": 317, "287": 318, "288": 319, "289": 320, "290": 321, "291": 322, "292": 323, "293": 324, "294": 325, "295": 326, "296": 327, "297": 328, "298": 329, "299": 330, "300": 331, "301": 332, "302": 333, "303": 334, "304": 335, "305": 336, "306": 337, "307": 338, "308": 339, "309": 340, "310": 341, "311": 342, "312": 343, "313": 344, "314": 345, "315": 346, "316": 347, "317": 348, "318": 349, "319": 350, "320": 351, "321": 352, "322": 353, "323": 354, "324": 355, "325": 356, "326": 357, "327": 358, "328": 359, "329": 360, "330": 361, "331": 362, "332": 363, "333": 364, "334": 365, "335": 366, "336": 367, "337": 368, "338": 369, "339": 370, "340": 371, "341": 372, "342": 373, "343": 374, "344": 375, "345": 376, "346": 377, "347": 378, "348": 379, "349": 380, "350": 381, "351": 382, "352": 383, "353": 384, "354": 385, "355": 386, "356": 387, "357": 388, "358": 389, "359": 390, "360": 391, "361": 392, "362": 393, "363": 394, "364": 395, "365": 396, "366": 397, "367": 398, "368": 399, "369": 400, "370": 401, "371": 402, "372": 403, "373": 404, "374": 405, "375": 406, "376": 407, "377": 408, "378": 409, "379": 410, "380": 411, "381": 412, "382": 413, "383": 414, "384": 415, "385": 416, "386": 417, "387": 418, "388": 419, "389": 420, "390": 421, "391": 422, "392": 423, "393": 424, "394": 425, "395": 426, "396": 427, "397": 428, "398": 429, "399": 430, "400": 431, "401": 432, "402": 433, "403": 434, "404": 435, "405": 436, "406": 437, "407": 438, "408": 439, "409": 440, "410": 441, "411": 442, "412": 443, "413": 444, "414": 445, "415": 446, "416": 447, "417": 448, "418": 449, "419": 450, "420": 451, "421": 452, "422": 453, "423": 454, "424": 455, "425": 456, "426": 457, "427": 458, "428": 459, "429": 460, "430": 461, "431": 462, "432": 463, "433": 464, "434": 465, "435": 466, "436": 467, "437": 468, "438": 469, "439": 470, "440": 471, "441": 472, "442": 473, "443": 474, "444": 475, "445": 476, "446": 477, "447": 478, "448": 479, "449": 480, "450": 481, "451": 482, "452": 483, "453": 484, "454": 485, "455": 486, "456": 487, "457": 488, "458": 489, "459": 490, "460": 491, "461": 492, "462": 493, "463": 494, "464": 495, "465": 496, "466": 497, "467": 498, "468": 499, "469": 500, "470": 501, "471": 502, "472": 503, "473": 504, "474": 505, "475": 506, "476": 507, "477": 508, "478": 509, "479": 510, "480": 511, "481": 512, "482": 513, "483": 514, "484": 515, "485": 516, "486": 517, "487": 518, "488": 519, "489": 520, "490": 521, "491": 522, "492": 523, "493": 524, "494": 525, "495": 526, "496": 527, "497": 528, "498": 529, "499": 530, "500": 531, "501": 532, "502": 533, "503": 534, "504": 535, "505": 536, "506": 537, "507": 538, "508": 539, "509": 540, "510": 541, "511": 542, "512": 543, "513": 544, "514": 545, "515": 546, "516": 547, "517": 548, "518": 549, "519": 550, "520": 551, "521": 552, "522": 553, "523": 554, "524": 555, "525": 556, "526": 557, "527": 558, "528": 559, "529": 560, "530": 561, "531": 562, "532": 563, "533": 564, "534": 565, "535": 566, "536": 567, "537": 568, "538": 569, "539": 570, "540": 571, "541": 572, "542": 573, "543": 574, "544": 575, "545": 576, "546": 577, "547": 578, "548": 579, "549": 580, "550": 581, "551": 582, "552": 583, "553": 584, "554": 585, "555": 586, "556": 587, "557": 588, "558": 589, "559": 590, "560": 591, "561": 592, "562": 593, "563": 594, "564": 595, "565": 596, "566": 597, "567": 598, "568": 599, "569": 600, "570": 601, "571": 602, "572": 603, "573": 604, "574": 605, "575": 606, "576": 607, "577": 608, "578": 609, "579": 610, "580": 611, "581": 612, "582": 613, "583": 614, "584": 615, "585": 616, "586": 617, "587": 618, "588": 619, "589": 620, "590": 621, "591": 622, "592": 623, "593": 624, "594": 625, "595": 626, "596": 627, "597": 628, "598": 629, "599": 630, "600": 631, "|": 632, "?": 633 }, "unk_token": "[UNK]" } }