{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": { "": { "id": "", "ids": [ 1 ], "tokens": [ "" ] } } }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "Ā": 3, "ā": 4, "Ă": 5, "ă": 6, "Ą": 7, "ą": 8, "Ć": 9, "ć": 10, "Ĉ": 11, "ĉ": 12, "Ċ": 13, "ċ": 14, "Č": 15, "č": 16, "Ď": 17, "ď": 18, "Đ": 19, "đ": 20, "Ē": 21, "ē": 22, "Ĕ": 23, "ĕ": 24, "Ė": 25, "ė": 26, "Ę": 27, "ę": 28, "Ě": 29, "ě": 30, "Ĝ": 31, "ĝ": 32, "Ğ": 33, "ğ": 34, "Ġ": 35, "!": 36, "\"": 37, "#": 38, "$": 39, "%": 40, "&": 41, "'": 42, "(": 43, ")": 44, "*": 45, "+": 46, ",": 47, "-": 48, ".": 49, "/": 50, "0": 51, "1": 52, "2": 53, "3": 54, "4": 55, "5": 56, "6": 57, "7": 58, "8": 59, "9": 60, ":": 61, ";": 62, "<": 63, "=": 64, ">": 65, "?": 66, "@": 67, "A": 68, "B": 69, "C": 70, "D": 71, "E": 72, "F": 73, "G": 74, "H": 75, "I": 76, "J": 77, "K": 78, "L": 79, "M": 80, "N": 81, "O": 82, "P": 83, "Q": 84, "R": 85, "S": 86, "T": 87, "U": 88, "V": 89, "W": 90, "X": 91, "Y": 92, "Z": 93, "[": 94, "\\": 95, "]": 96, "^": 97, "_": 98, "`": 99, "a": 100, "b": 101, "c": 102, "d": 103, "e": 104, "f": 105, "g": 106, "h": 107, "i": 108, "j": 109, "k": 110, "l": 111, "m": 112, "n": 113, "o": 114, "p": 115, "q": 116, "r": 117, "s": 118, "t": 119, "u": 120, "v": 121, "w": 122, "x": 123, "y": 124, "z": 125, "{": 126, "|": 127, "}": 128, "~": 129, "ġ": 130, "Ģ": 131, "ģ": 132, "Ĥ": 133, "ĥ": 134, "Ħ": 135, "ħ": 136, "Ĩ": 137, "ĩ": 138, "Ī": 139, "ī": 140, "Ĭ": 141, "ĭ": 142, "Į": 143, "į": 144, "İ": 145, "ı": 146, "IJ": 147, "ij": 148, "Ĵ": 149, "ĵ": 150, "Ķ": 151, "ķ": 152, "ĸ": 153, "Ĺ": 154, "ĺ": 155, "Ļ": 156, "ļ": 157, "Ľ": 158, "ľ": 159, "Ŀ": 160, "ŀ": 161, "Ł": 162, "ł": 163, "¡": 164, "¢": 165, "£": 166, "¤": 167, "¥": 168, "¦": 169, "§": 170, "¨": 171, "©": 172, "ª": 173, "«": 174, "¬": 175, "Ń": 176, "®": 177, "¯": 178, "°": 179, "±": 180, "²": 181, "³": 182, "´": 183, "µ": 184, "¶": 185, "·": 186, "¸": 187, "¹": 188, "º": 189, "»": 190, "¼": 191, "½": 192, "¾": 193, "¿": 194, "À": 195, "Á": 196, "Â": 197, "Ã": 198, "Ä": 199, "Å": 200, "Æ": 201, "Ç": 202, "È": 203, "É": 204, "Ê": 205, "Ë": 206, "Ì": 207, "Í": 208, "Î": 209, "Ï": 210, "Ð": 211, "Ñ": 212, "Ò": 213, "Ó": 214, "Ô": 215, "Õ": 216, "Ö": 217, "×": 218, "Ø": 219, "Ù": 220, "Ú": 221, "Û": 222, "Ü": 223, "Ý": 224, "Þ": 225, "ß": 226, "à": 227, "á": 228, "â": 229, "ã": 230, "ä": 231, "å": 232, "æ": 233, "ç": 234, "è": 235, "é": 236, "ê": 237, "ë": 238, "ì": 239, "í": 240, "î": 241, "ï": 242, "ð": 243, "ñ": 244, "ò": 245, "ó": 246, "ô": 247, "õ": 248, "ö": 249, "÷": 250, "ø": 251, "ù": 252, "ú": 253, "û": 254, "ü": 255, "ý": 256, "þ": 257, "ÿ": 258 }, "merges": [] } }