{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "Replace", "pattern": { "String": "⸗\n" }, "content": "" }, { "type": "Replace", "pattern": { "String": "-\n" }, "content": "" }, { "type": "NFC" }, { "type": "Replace", "pattern": { "Regex": "(?<=\\p{L}\\p{M}*)Σ(?!\\p{M}*\\p{L})" }, "content": "ς" }, { "type": "Lowercase" }, { "type": "Replace", "pattern": { "String": "aͤ" }, "content": "ä" }, { "type": "Replace", "pattern": { "String": "oͤ" }, "content": "ö" }, { "type": "Replace", "pattern": { "String": "uͤ" }, "content": "ü" }, { "type": "Replace", "pattern": { "String": "ͤ" }, "content": "" }, { "type": "Replace", "pattern": { "String": "ſ" }, "content": "s" }, { "type": "Replace", "pattern": { "String": "ꝛ" }, "content": "r" }, { "type": "Replace", "pattern": { "String": "⸗" }, "content": "-" }, { "type": "Replace", "pattern": { "String": "„" }, "content": "\"" }, { "type": "Replace", "pattern": { "String": "“" }, "content": "\"" }, { "type": "Replace", "pattern": { "String": "”" }, "content": "\"" }, { "type": "Replace", "pattern": { "String": "»" }, "content": "\"" }, { "type": "Replace", "pattern": { "String": "«" }, "content": "\"" }, { "type": "Replace", "pattern": { "String": "‘" }, "content": "'" }, { "type": "Replace", "pattern": { "String": "’" }, "content": "'" }, { "type": "Replace", "pattern": { "String": "‚" }, "content": "'" }, { "type": "Replace", "pattern": { "String": "–" }, "content": "-" }, { "type": "Replace", "pattern": { "String": "—" }, "content": "-" }, { "type": "Replace", "pattern": { "Regex": "\\d" }, "content": "0" }, { "type": "Replace", "pattern": { "Regex": "\\s+" }, "content": " " }, { "type": "Strip", "strip_left": true, "strip_right": true } ] }, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "[\\s\\S]" }, "behavior": "Isolated", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": { "type": "Fuse" }, "model": { "type": "WordLevel", "vocab": { "": 0, "": 1, " ": 2, "e": 3, "n": 4, "i": 5, "r": 6, "s": 7, "t": 8, "a": 9, "d": 10, "h": 11, "l": 12, "u": 13, "c": 14, "g": 15, "m": 16, "o": 17, "b": 18, "w": 19, "f": 20, ",": 21, ".": 22, "0": 23, "z": 24, "k": 25, "v": 26, "p": 27, "ü": 28, "ä": 29, "/": 30, "ß": 31, "ö": 32, "j": 33, "y": 34, "-": 35, ")": 36, ";": 37, "(": 38, ":": 39, "\"": 40, "x": 41, "q": 42, "'": 43, "!": 44, "_": 45, "?": 46, "¬": 47, "*": 48, "§": 49, "=": 50, "é": 51, "æ": 52, "‒": 53, "&": 54, "]": 55, "α": 56, "ñ": 57, "[": 58, "ν": 59, "ο": 60, "ẽ": 61, "τ": 62, "ι": 63, "ε": 64, "+": 65, "ς": 66, "̃": 67, "â": 68, "à": 69, "è": 70, "σ": 71, "ô": 72, "π": 73, "μ": 74, "λ": 75, "ê": 76, "ρ": 77, "κ": 78, "ë": 79, "°": 80, "½": 81, "·": 82, "î": 83, "δ": 84, "á": 85, "†": 86, "υ": 87, "γ": 88, "‟": 89, "…": 90, "ͦ": 91, "ί": 92, "⁾": 93, "ω": 94, "η": 95, "œ": 96, "φ": 97, "β": 98, "ϑ": 99, "û": 100, "%": 101, "í": 102, "έ": 103, "ἀ": 104, "ά": 105, "ό": 106, "ἐ": 107, "ů": 108, "ϱ": 109, "χ": 110, "′": 111, "č": 112, "ò": 113, "ç": 114, "́": 115, "ʒ": 116, "¼": 117, "|": 118, "ὸ": 119, "ὶ": 120, "": 121, "─": 122, "ó": 123, "̄": 124, "ύ": 125, "ù": 126, "ï": 127, "ṅ": 128, "þ": 129, "ĕ": 130, "ῖ": 131, "″": 132, "ā": 133, "⋹": 134, "¾": 135, "¹": 136, "ũ": 137, "ú": 138, "ή": 139, "ῦ": 140, "θ": 141, "ῶ": 142, "š": 143, "×": 144, "ē": 145, "ξ": 146, "ὰ": 147, "ý": 148, "ɟ": 149, "õ": 150, "̅": 151, "ἰ": 152, "ø": 153, "ζ": 154, "ž": 155, "²": 156, "〃": 157, "ῆ": 158, "ꝟ": 159, "ã": 160, "ὴ": 161, "℔": 162, "ꝙ": 163, "ὐ": 164, "̆": 165, "⅓": 166, "¶": 167, "ἔ": 168, "£": 169, "å": 170, "│": 171, "⁽": 172, "³": 173, "□": 174, "℥": 175, "ϰ": 176, "ώ": 177, "ψ": 178, "⁴": 179, "∞": 180, "ř": 181, "ἄ": 182, "ę": 183, "ŭ": 184, "⅔": 185, "ὲ": 186, "ĭ": 187, "⁵": 188, "℞": 189, "ÿ": 190, "ū": 191, "⁶": 192, "⏑": 193, "#": 194, "≠": 195, "̶": 196, "ὁ": 197, "√": 198, "⁷": 199, "ὑ": 200, "ī": 201, "ì": 202, "⁸": 203, "ἱ": 204, "⁹": 205, "ἡ": 206, "ð": 207, "ł": 208, "̕": 209, "ʼ": 210, "⃛": 211, "}": 212, "☉": 213, "{": 214, "ą": 215, "ὀ": 216, "〟": 217, "ś": 218, "ῳ": 219, "₤": 220, "ὺ": 221, "⁰": 222, "ῷ": 223, "‥": 224, "⅛": 225, "ᾶ": 226, "ō": 227, "ă": 228, "ὅ": 229, "∫": 230, "ἑ": 231, "‰": 232, "ŷ": 233, "ż": 234, ">": 235, "ἴ": 236, "℈": 237, "<": 238, "ě": 239, "‧": 240, "♂": 241, "♀": 242, "ď": 243, "ḱ": 244, "⅕": 245, "᾽": 246, "ὄ": 247, "⅜": 248, "ˈ": 249, "ć": 250, "˙": 251, "ῥ": 252, "ἁ": 253, "‴": 254, "•": 255 }, "unk_token": "" } }