{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<\\s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 329, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 330, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "vocab": { "": 0, "<\\s>": 1, "": 2, "": 3, "!": 4, "%": 5, "&": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "A": 31, "B": 32, "C": 33, "D": 34, "E": 35, "F": 36, "G": 37, "H": 38, "I": 39, "J": 40, "K": 41, "L": 42, "M": 43, "N": 44, "O": 45, "P": 46, "Q": 47, "R": 48, "S": 49, "T": 50, "U": 51, "V": 52, "W": 53, "X": 54, "Y": 55, "Z": 56, "[": 57, "\\": 58, "]": 59, "_": 60, "`": 61, "a": 62, "b": 63, "c": 64, "d": 65, "e": 66, "f": 67, "g": 68, "h": 69, "i": 70, "j": 71, "k": 72, "l": 73, "m": 74, "n": 75, "o": 76, "p": 77, "q": 78, "r": 79, "s": 80, "t": 81, "u": 82, "v": 83, "w": 84, "x": 85, "y": 86, "z": 87, "|": 88, "~": 89, "¡": 90, "¢": 91, "£": 92, "¤": 93, "¥": 94, "¦": 95, "§": 96, "¨": 97, "©": 98, "ª": 99, "«": 100, "¬": 101, "®": 102, "¯": 103, "°": 104, "±": 105, "²": 106, "³": 107, "µ": 108, "¶": 109, "·": 110, "¸": 111, "¹": 112, "¼": 113, "½": 114, "¾": 115, "¿": 116, "Â": 117, "Ã": 118, "Ë": 119, "à": 120, "â": 121, "Ġ": 122, "Ģ": 123, "ģ": 124, "Ĥ": 125, "ĥ": 126, "Ħ": 127, "ħ": 128, "Ĩ": 129, "ĩ": 130, "Ī": 131, "ī": 132, "Ĭ": 133, "ĭ": 134, "Į": 135, "į": 136, "ı": 137, "IJ": 138, "ij": 139, "Ĵ": 140, "ĵ": 141, "Ķ": 142, "ķ": 143, "ĸ": 144, "Ĺ": 145, "ĺ": 146, "Ļ": 147, "ļ": 148, "Ľ": 149, "ľ": 150, "Ŀ": 151, "ŀ": 152, "Ł": 153, "ł": 154, "Ń": 155, "à¤": 156, "à¥": 157, "à¦": 158, "à¨": 159, "âĢ": 160, "à§": 161, "à©": 162, "s>": 163, "<\\": 164, "", "< \\", "< p", "< s>", "a d", " °", " ²", "à ¡", "à ¨", "à ©", "à ·", "à Ĺ", "Ë ļ", "â Ī", "ठ¡", "ठ¢", "ठ£", "ठ¤", "ठ¥", "ठ¦", "ठ§", "ठ¨", "ठª", "ठ«", "ठ¬", "ठ®", "ठ¯", "ठ°", "ठ²", "ठ³", "ठµ", "ठ¶", "ठ·", "ठ¸", "ठ¹", "ठ¼", "ठ½", "ठ¾", "ठ¿", "ठģ", "ठĤ", "ठĥ", "ठħ", "ठĨ", "ठĩ", "ठĪ", "ठī", "ठĬ", "ठĭ", "ठį", "ठı", "ठIJ", "ठij", "ठĵ", "ठĶ", "ठķ", "ठĸ", "ठĹ", "ठĺ", "ठĻ", "ठļ", "ठĽ", "ठľ", "ठĿ", "ठŀ", "ठŁ", "ठł", "ठŃ", "ॠ¤", "ॠ¥", "ॠ¦", "ॠ§", "ॠ¨", "ॠ©", "ॠª", "ॠ«", "ॠ¬", "ॠ®", "ॠ¯", "ॠĢ", "ॠģ", "ॠĤ", "ॠĥ", "ॠĦ", "ॠħ", "ॠĩ", "ॠĪ", "ॠī", "ॠĬ", "ॠĭ", "ॠĮ", "ॠį", "ॠIJ", "ॠĺ", "ॠĻ", "ॠļ", "ॠĽ", "ॠľ", "ॠĿ", "ॠŀ", "ॠŁ", "ॠł", "ॠŃ", "ঠ¤", "ঠ¥", "ঠ¨", "ঠª", "ঠ¬", "ঠ®", "ঠ¯", "ঠ²", "ঠ¸", "ঠ¹", "ঠ¼", "ঠ¾", "ঠ¿", "ঠĨ", "ঠĩ", "ঠı", "ঠķ", "ঠĹ", "ঠļ", "ঠĽ", "ঠľ", "ঠł", "ਠ¤", "ਠ¦", "ਠ¨", "ਠª", "ਠ¬", "ਠ®", "ਠ²", "ਠµ", "ਠ¶", "ਠ¸", "ਠ¹", "ਠ¼", "ਠ¾", "ਠ¿", "ਠħ", "ਠĨ", "ਠī", "ਠķ", "ਠĸ", "ਠļ", "âĢ ¦", "âĢ Į", "âĢ į", "âĢ ĵ", "âĢ ĺ", "âĢ Ļ", "âĢ ľ", "âĢ Ŀ", "ৠ°", "ৠ±", "ৠĪ", "ৠĭ", "ৠį", "ৠŁ", "à© °", "à© ±", "à© ģ", "à© Ĥ", "à© ĩ", "à© Ī", "<\\ s>", "

" ] } }