{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "": 3, "!": 4, "\"": 5, "#": 6, "$": 7, "%": 8, "&": 9, "'": 10, "(": 11, ")": 12, "*": 13, "+": 14, ",": 15, "-": 16, ".": 17, "/": 18, "0": 19, "1": 20, "2": 21, "3": 22, "4": 23, "5": 24, "6": 25, "7": 26, "8": 27, "9": 28, ":": 29, ";": 30, "<": 31, "=": 32, ">": 33, "?": 34, "@": 35, "A": 36, "B": 37, "C": 38, "D": 39, "E": 40, "F": 41, "G": 42, "H": 43, "I": 44, "J": 45, "K": 46, "L": 47, "M": 48, "N": 49, "O": 50, "P": 51, "Q": 52, "R": 53, "S": 54, "T": 55, "U": 56, "V": 57, "W": 58, "X": 59, "Y": 60, "Z": 61, "[": 62, "\\": 63, "]": 64, "^": 65, "_": 66, "`": 67, "a": 68, "b": 69, "c": 70, "d": 71, "e": 72, "f": 73, "g": 74, "h": 75, "i": 76, "j": 77, "k": 78, "l": 79, "m": 80, "n": 81, "o": 82, "p": 83, "q": 84, "r": 85, "s": 86, "t": 87, "u": 88, "v": 89, "w": 90, "x": 91, "y": 92, "z": 93, "{": 94, "|": 95, "}": 96, "~": 97, "¡": 98, "¢": 99, "£": 100, "¤": 101, "¥": 102, "¦": 103, "§": 104, "¨": 105, "©": 106, "ª": 107, "«": 108, "¬": 109, "®": 110, "¯": 111, "°": 112, "±": 113, "²": 114, "³": 115, "´": 116, "µ": 117, "¶": 118, "·": 119, "¸": 120, "¹": 121, "º": 122, "»": 123, "¼": 124, "½": 125, "¾": 126, "¿": 127, "À": 128, "Á": 129, "Â": 130, "Ã": 131, "Ä": 132, "Å": 133, "Æ": 134, "Ç": 135, "È": 136, "É": 137, "Ê": 138, "Ë": 139, "Ì": 140, "Í": 141, "Î": 142, "Ï": 143, "Ð": 144, "Ñ": 145, "Ò": 146, "Ó": 147, "Ô": 148, "Õ": 149, "Ö": 150, "×": 151, "Ø": 152, "Ù": 153, "Ú": 154, "Û": 155, "Ü": 156, "Ý": 157, "Þ": 158, "ß": 159, "à": 160, "á": 161, "â": 162, "ã": 163, "ä": 164, "å": 165, "æ": 166, "ç": 167, "è": 168, "é": 169, "ê": 170, "ë": 171, "ì": 172, "í": 173, "î": 174, "ï": 175, "ð": 176, "ñ": 177, "ò": 178, "ó": 179, "ô": 180, "õ": 181, "ö": 182, "÷": 183, "ø": 184, "ù": 185, "ú": 186, "û": 187, "ü": 188, "ý": 189, "þ": 190, "ÿ": 191, "Ā": 192, "ā": 193, "Ă": 194, "ă": 195, "Ą": 196, "ą": 197, "Ć": 198, "ć": 199, "Ĉ": 200, "ĉ": 201, "Ċ": 202, "ċ": 203, "Č": 204, "č": 205, "Ď": 206, "ď": 207, "Đ": 208, "đ": 209, "Ē": 210, "ē": 211, "Ĕ": 212, "ĕ": 213, "Ė": 214, "ė": 215, "Ę": 216, "ę": 217, "Ě": 218, "ě": 219, "Ĝ": 220, "ĝ": 221, "Ğ": 222, "ğ": 223, "Ġ": 224, "ġ": 225, "Ģ": 226, "ģ": 227, "Ĥ": 228, "ĥ": 229, "Ħ": 230, "ħ": 231, "Ĩ": 232, "ĩ": 233, "Ī": 234, "ī": 235, "Ĭ": 236, "ĭ": 237, "Į": 238, "į": 239, "İ": 240, "ı": 241, "IJ": 242, "ij": 243, "Ĵ": 244, "ĵ": 245, "Ķ": 246, "ķ": 247, "ĸ": 248, "Ĺ": 249, "ĺ": 250, "Ļ": 251, "ļ": 252, "Ľ": 253, "ľ": 254, "Ŀ": 255, "ŀ": 256, "Ł": 257, "ł": 258, "Ń": 259, "en": 260, "ken": 261, "er": 262, "oken": 263, "token": 264, "iz": 265, "izer": 266, "in": 267, "ĠĠ": 268, "tokenizer": 269, "ra": 270, "rain": 271, "ad": 272, "Ġ=": 273, "Ġa": 274, "Ġi": 275, "lo": 276, "ĠĠĠ": 277, "\",": 278, "ac": 279, "al": 280, "ed": 281, "ĠB": 282, "Ġtokenizer": 283, "\"<": 284, "Token": 285, "el": 286, "mp": 287, "or": 288, "te": 289, "co": 290, "nd": 291, "pr": 292, "train": 293, ">\",": 294, "ev": 295, "fr": 296, "om": 297, "to": 298, "yte": 299, "ers": 300, "load": 301, "Tokenizer": 302, "cod": 303, "evel": 304, "from": 305, "Level": 306, "PE": 307, "ecod": 308, "os": 309, "un": 310, "Ġand": 311, "Ġimp": 312, "Ġtokenizers": 313, "ort": 314, "yteLevel": 315, "Ġimport": 316, "=\"<": 317, "ID": 318, "Train": 319, "bac": 320, "ds": 321, "ids": 322, "kend": 323, "Ġit": 324, "all": 325, "ĠByteLevel": 326, "pre": 327, "unk": 328, "backend": 329, "\"\"": 330, "(\"": 331, "EP": 332, "REP": 333, "ex": 334, "eTrain": 335, "gin": 336, "hu": 337, "ns": 338, "th": 339, "Ġs": 340, "Ġu": 341, "Ġload": 342, "ace": 343, "REPO": 344, "))": 345, ":\",": 346, "Au": 347, "Decod": 348, "EX": 349, "FI": 350, "LE": 351, "TEX": 352, "ab": 353, "amp": 354, "bos": 355, "decod": 356, "eos": 357, "for": 358, "ggin": 359, "le": 360, "mers": 361, "pad": 362, "st": 363, "Ġ\"": 364, "Ġ\"<": 365, "Ġtrain": 366, "ĠAu": 367, "int": 368, "rans": 369, "ĠBPE": 370, "print": 371, "toTokenizer": 372, "examp": 373, "Ġloaded": 374, "FILE": 375, "TEXT": 376, "formers": 377, "gging": 378, "ĠAutoTokenizer": 379, "ransformers": 380, "example": 381, "()": 382, "00": 383, ">\"": 384, "AB": 385, "AL": 386, "CI": 387, "CAB": 388, "EN": 389, "Fa": 390, "Hu": 391, "IZ": 392, "KEN": 393, "OCAB": 394, "OKEN": 395, "Pr": 396, "SPE": 397, "SIZ": 398, "Th": 399, "TOKEN": 400, "VOCAB": 401, "av": 402, "ec": 403, "et": 404, "face": 405, "hf": 406, "hab": 407, "it": 408, "ial": 409, "my": 410, "mall": 411, "on": 412, "py": 413, "pload": 414, "peTrain": 415, "phab": 416, "transformers": 417, "ver": 418, "xt": 419, "Ġ1": 420, "Ġb": 421, "Ġin": 422, "Ġlo": 423, "Ġto": 424, "ĠTokenizer": 425, "ĠHu": 426, "ĠPr": 427, "Ġtransformers": 428, "tokens": 429, "Ġas": 430, "alphab": 431, "edTokenizer": 432, "ĠBpeTrain": 433, "trained": 434, "ĠByteLevelDecod": 435, "pretrained": 436, "\"\"\"": 437, "eTrainedTokenizer": 438, "hub": 439, "hugging": 440, "Ġsmall": 441, "Ġupload": 442, "CIAL": 443, "Fast": 444, "SPECIAL": 445, "SIZE": 446, "TOKENS": 447, "ave": 448, "ĠPreTrainedTokenizer": 449, "alphabet": 450, "ĠBpeTrainer": 451, "ĠByteLevelDecoder": 452, "huggingface": 453, "ĠPreTrainedTokenizerFast": 454, "\")": 455, "([": 456, ".\"": 457, "BPE": 458, "ByteLevel": 459, "Cr": 460, "Fal": 461, "Face": 462, "Hel": 463, "Ins": 464, "Lo": 465, "Save": 466, "Transformers": 467, "Wra": 468, "[\"<": 469, "],": 470, "am": 471, "ate": 472, "and": 473, "bj": 474, "ce": 475, "call": 476, "cab": 477, "del": 478, "elo": 479, "eate": 480, "fi": 481, "il": 482, "ip": 483, "is": 484, "ly": 485, "level": 486, "mo": 487, "nver": 488, "nam": 489, "ou": 490, "oth": 491, "obj": 492, "ocab": 493, "pu": 494, "pace": 495, "pec": 496, "pip": 497, "run": 498, "se": 499, "sh": 500, "siz": 501, "sers": 502, "save": 503, "space": 504, "tall": 505, "txt": 506, "tand": 507, "tly": 508, "uth": 509, "vocab": 510, "wil": 511 }, "merges": [ [ "e", "n" ], [ "k", "en" ], [ "e", "r" ], [ "o", "ken" ], [ "t", "oken" ], [ "i", "z" ], [ "iz", "er" ], [ "i", "n" ], [ "Ġ", "Ġ" ], [ "token", "izer" ], [ "r", "a" ], [ "ra", "in" ], [ "a", "d" ], [ "Ġ", "=" ], [ "Ġ", "a" ], [ "Ġ", "i" ], [ "l", "o" ], [ "ĠĠ", "Ġ" ], [ "\"", "," ], [ "a", "c" ], [ "a", "l" ], [ "e", "d" ], [ "Ġ", "B" ], [ "Ġ", "tokenizer" ], [ "\"", "<" ], [ "T", "oken" ], [ "e", "l" ], [ "m", "p" ], [ "o", "r" ], [ "t", "e" ], [ "c", "o" ], [ "n", "d" ], [ "p", "r" ], [ "t", "rain" ], [ ">", "\"," ], [ "e", "v" ], [ "f", "r" ], [ "o", "m" ], [ "t", "o" ], [ "y", "te" ], [ "er", "s" ], [ "lo", "ad" ], [ "Token", "izer" ], [ "co", "d" ], [ "ev", "el" ], [ "fr", "om" ], [ "L", "evel" ], [ "P", "E" ], [ "e", "cod" ], [ "o", "s" ], [ "u", "n" ], [ "Ġa", "nd" ], [ "Ġi", "mp" ], [ "Ġtokenizer", "s" ], [ "or", "t" ], [ "yte", "Level" ], [ "Ġimp", "ort" ], [ "=", "\"<" ], [ "I", "D" ], [ "T", "rain" ], [ "b", "ac" ], [ "d", "s" ], [ "i", "ds" ], [ "ken", "d" ], [ "Ġi", "t" ], [ "al", "l" ], [ "ĠB", "yteLevel" ], [ "pr", "e" ], [ "un", "k" ], [ "bac", "kend" ], [ "\"", "\"" ], [ "(", "\"" ], [ "E", "P" ], [ "R", "EP" ], [ "e", "x" ], [ "e", "Train" ], [ "g", "in" ], [ "h", "u" ], [ "n", "s" ], [ "t", "h" ], [ "Ġ", "s" ], [ "Ġ", "u" ], [ "Ġ", "load" ], [ "ac", "e" ], [ "REP", "O" ], [ ")", ")" ], [ ":", "\"," ], [ "A", "u" ], [ "D", "ecod" ], [ "E", "X" ], [ "F", "I" ], [ "L", "E" ], [ "T", "EX" ], [ "a", "b" ], [ "a", "mp" ], [ "b", "os" ], [ "d", "ecod" ], [ "e", "os" ], [ "f", "or" ], [ "g", "gin" ], [ "l", "e" ], [ "m", "ers" ], [ "p", "ad" ], [ "s", "t" ], [ "Ġ", "\"" ], [ "Ġ", "\"<" ], [ "Ġ", "train" ], [ "Ġ", "Au" ], [ "in", "t" ], [ "ra", "ns" ], [ "ĠB", "PE" ], [ "pr", "int" ], [ "to", "Tokenizer" ], [ "ex", "amp" ], [ "Ġload", "ed" ], [ "FI", "LE" ], [ "TEX", "T" ], [ "for", "mers" ], [ "ggin", "g" ], [ "ĠAu", "toTokenizer" ], [ "rans", "formers" ], [ "examp", "le" ], [ "(", ")" ], [ "0", "0" ], [ ">", "\"" ], [ "A", "B" ], [ "A", "L" ], [ "C", "I" ], [ "C", "AB" ], [ "E", "N" ], [ "F", "a" ], [ "H", "u" ], [ "I", "Z" ], [ "K", "EN" ], [ "O", "CAB" ], [ "O", "KEN" ], [ "P", "r" ], [ "S", "PE" ], [ "S", "IZ" ], [ "T", "h" ], [ "T", "OKEN" ], [ "V", "OCAB" ], [ "a", "v" ], [ "e", "c" ], [ "e", "t" ], [ "f", "ace" ], [ "h", "f" ], [ "h", "ab" ], [ "i", "t" ], [ "i", "al" ], [ "m", "y" ], [ "m", "all" ], [ "o", "n" ], [ "p", "y" ], [ "p", "load" ], [ "p", "eTrain" ], [ "p", "hab" ], [ "t", "ransformers" ], [ "v", "er" ], [ "x", "t" ], [ "Ġ", "1" ], [ "Ġ", "b" ], [ "Ġ", "in" ], [ "Ġ", "lo" ], [ "Ġ", "to" ], [ "Ġ", "Tokenizer" ], [ "Ġ", "Hu" ], [ "Ġ", "Pr" ], [ "Ġ", "transformers" ], [ "token", "s" ], [ "Ġa", "s" ], [ "al", "phab" ], [ "ed", "Tokenizer" ], [ "ĠB", "peTrain" ], [ "train", "ed" ], [ "ĠByteLevel", "Decod" ], [ "pre", "trained" ], [ "\"\"", "\"" ], [ "eTrain", "edTokenizer" ], [ "hu", "b" ], [ "hu", "gging" ], [ "Ġs", "mall" ], [ "Ġu", "pload" ], [ "CI", "AL" ], [ "Fa", "st" ], [ "SPE", "CIAL" ], [ "SIZ", "E" ], [ "TOKEN", "S" ], [ "av", "e" ], [ "ĠPr", "eTrainedTokenizer" ], [ "alphab", "et" ], [ "ĠBpeTrain", "er" ], [ "ĠByteLevelDecod", "er" ], [ "hugging", "face" ], [ "ĠPreTrainedTokenizer", "Fast" ], [ "\"", ")" ], [ "(", "[" ], [ ".", "\"" ], [ "B", "PE" ], [ "B", "yteLevel" ], [ "C", "r" ], [ "F", "al" ], [ "F", "ace" ], [ "H", "el" ], [ "I", "ns" ], [ "L", "o" ], [ "S", "ave" ], [ "T", "ransformers" ], [ "W", "ra" ], [ "[", "\"<" ], [ "]", "," ], [ "a", "m" ], [ "a", "te" ], [ "a", "nd" ], [ "b", "j" ], [ "c", "e" ], [ "c", "all" ], [ "c", "ab" ], [ "d", "el" ], [ "e", "lo" ], [ "e", "ate" ], [ "f", "i" ], [ "i", "l" ], [ "i", "p" ], [ "i", "s" ], [ "l", "y" ], [ "l", "evel" ], [ "m", "o" ], [ "n", "ver" ], [ "n", "am" ], [ "o", "u" ], [ "o", "th" ], [ "o", "bj" ], [ "o", "cab" ], [ "p", "u" ], [ "p", "ace" ], [ "p", "ec" ], [ "p", "ip" ], [ "r", "un" ], [ "s", "e" ], [ "s", "h" ], [ "s", "iz" ], [ "s", "ers" ], [ "s", "ave" ], [ "s", "pace" ], [ "t", "all" ], [ "t", "xt" ], [ "t", "and" ], [ "t", "ly" ], [ "u", "th" ], [ "v", "ocab" ], [ "w", "il" ] ] } }