{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "[\\s\\S]" }, "behavior": "Isolated", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": { "": { "id": "", "ids": [ 2 ], "tokens": [ "" ] }, "": { "id": "", "ids": [ 1 ], "tokens": [ "" ] } } }, "decoder": { "type": "Fuse" }, "model": { "type": "WordLevel", "vocab": { "": 0, "": 1, "": 2, "": 3, "a": 4, "b": 5, "c": 6, "d": 7, "e": 8, "f": 9, "g": 10, "h": 11, "i": 12, "j": 13, "k": 14, "l": 15, "m": 16, "n": 17, "o": 18, "p": 19, "q": 20, "r": 21, "s": 22, "t": 23, "u": 24, "v": 25, "w": 26, "x": 27, "y": 28, "z": 29, "A": 30, "B": 31, "C": 32, "D": 33, "E": 34, "F": 35, "G": 36, "H": 37, "I": 38, "J": 39, "K": 40, "L": 41, "M": 42, "N": 43, "O": 44, "P": 45, "Q": 46, "R": 47, "S": 48, "T": 49, "U": 50, "V": 51, "W": 52, "X": 53, "Y": 54, "Z": 55, "ɣ": 56, "Ɣ": 57, "ɛ": 58, "Ɛ": 59, "č": 60, "Č": 61, "ǧ": 62, "Ǧ": 63, "ţ": 64, "Ţ": 65, "ḍ": 66, "Ḍ": 67, "ḥ": 68, "Ḥ": 69, "ṛ": 70, "Ṛ": 71, "ṣ": 72, "Ṣ": 73, "ṭ": 74, "Ṭ": 75, "ẓ": 76, "Ẓ": 77, "é": 78, "è": 79, "ê": 80, "ë": 81, "à": 82, "â": 83, "î": 84, "ï": 85, "ô": 86, "ù": 87, "û": 88, "ç": 89, "É": 90, "È": 91, "Ê": 92, "Ë": 93, "À": 94, "Â": 95, "Î": 96, "Ï": 97, "Ô": 98, "Ù": 99, "Û": 100, "Ç": 101, "0": 102, "1": 103, "2": 104, "3": 105, "4": 106, "5": 107, "6": 108, "7": 109, "8": 110, "9": 111, " ": 112, ".": 113, ",": 114, "?": 115, "!": 116, "«": 117, "»": 118, "-": 119, "—": 120, ":": 121, ";": 122, "'": 123, "\"": 124, "/": 125, "(": 126, ")": 127, "[": 128, "]": 129, "{": 130, "}": 131, "%": 132, "*": 133, "+": 134, "=": 135, "_": 136, "~": 137, "–": 138, "…": 139, "\n": 140, "\t": 141, "ⴰ": 142, "ⴱ": 143, "ⴳ": 144, "ⴷ": 145, "ⴹ": 146, "ⴻ": 147, "ⴼ": 148, "ⴽ": 149, "ⵀ": 150, "ⵃ": 151, "ⵄ": 152, "ⵅ": 153, "ⵇ": 154, "ⵉ": 155, "ⵊ": 156, "ⵍ": 157, "ⵎ": 158, "ⵏ": 159, "ⵓ": 160, "ⵔ": 161, "ⵕ": 162, "ⵙ": 163, "ⵚ": 164, "ⵛ": 165, "ⵜ": 166, "ⵟ": 167, "ⵡ": 168, "ⵢ": 169, "ⵣ": 170, "ⵥ": 171, "ⵖ": 172 }, "unk_token": "" } }