| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<eos>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "<unk>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": { | |
| "type": "Sequence", | |
| "normalizers": [ | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "⸗\n" | |
| }, | |
| "content": "" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "-\n" | |
| }, | |
| "content": "" | |
| }, | |
| { | |
| "type": "NFC" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "Regex": "(?<=\\p{L}\\p{M}*)Σ(?!\\p{M}*\\p{L})" | |
| }, | |
| "content": "ς" | |
| }, | |
| { | |
| "type": "Lowercase" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "aͤ" | |
| }, | |
| "content": "ä" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "oͤ" | |
| }, | |
| "content": "ö" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "uͤ" | |
| }, | |
| "content": "ü" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "ͤ" | |
| }, | |
| "content": "" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "ſ" | |
| }, | |
| "content": "s" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "ꝛ" | |
| }, | |
| "content": "r" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "⸗" | |
| }, | |
| "content": "-" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "„" | |
| }, | |
| "content": "\"" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "“" | |
| }, | |
| "content": "\"" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "”" | |
| }, | |
| "content": "\"" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "»" | |
| }, | |
| "content": "\"" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "«" | |
| }, | |
| "content": "\"" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "‘" | |
| }, | |
| "content": "'" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "’" | |
| }, | |
| "content": "'" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "‚" | |
| }, | |
| "content": "'" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "–" | |
| }, | |
| "content": "-" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "String": "—" | |
| }, | |
| "content": "-" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "Regex": "\\d" | |
| }, | |
| "content": "0" | |
| }, | |
| { | |
| "type": "Replace", | |
| "pattern": { | |
| "Regex": "\\s+" | |
| }, | |
| "content": " " | |
| }, | |
| { | |
| "type": "Strip", | |
| "strip_left": true, | |
| "strip_right": true | |
| } | |
| ] | |
| }, | |
| "pre_tokenizer": { | |
| "type": "Split", | |
| "pattern": { | |
| "Regex": "[\\s\\S]" | |
| }, | |
| "behavior": "Isolated", | |
| "invert": false | |
| }, | |
| "post_processor": { | |
| "type": "TemplateProcessing", | |
| "single": [ | |
| { | |
| "Sequence": { | |
| "id": "A", | |
| "type_id": 0 | |
| } | |
| } | |
| ], | |
| "pair": [ | |
| { | |
| "Sequence": { | |
| "id": "A", | |
| "type_id": 0 | |
| } | |
| }, | |
| { | |
| "Sequence": { | |
| "id": "B", | |
| "type_id": 1 | |
| } | |
| } | |
| ], | |
| "special_tokens": {} | |
| }, | |
| "decoder": { | |
| "type": "Fuse" | |
| }, | |
| "model": { | |
| "type": "WordLevel", | |
| "vocab": { | |
| "<eos>": 0, | |
| "<unk>": 1, | |
| " ": 2, | |
| "e": 3, | |
| "n": 4, | |
| "i": 5, | |
| "r": 6, | |
| "s": 7, | |
| "t": 8, | |
| "a": 9, | |
| "d": 10, | |
| "h": 11, | |
| "l": 12, | |
| "u": 13, | |
| "c": 14, | |
| "g": 15, | |
| "m": 16, | |
| "o": 17, | |
| "b": 18, | |
| "w": 19, | |
| "f": 20, | |
| ",": 21, | |
| ".": 22, | |
| "0": 23, | |
| "z": 24, | |
| "k": 25, | |
| "v": 26, | |
| "p": 27, | |
| "ü": 28, | |
| "ä": 29, | |
| "/": 30, | |
| "ß": 31, | |
| "ö": 32, | |
| "j": 33, | |
| "y": 34, | |
| "-": 35, | |
| ")": 36, | |
| ";": 37, | |
| "(": 38, | |
| ":": 39, | |
| "\"": 40, | |
| "x": 41, | |
| "q": 42, | |
| "'": 43, | |
| "!": 44, | |
| "_": 45, | |
| "?": 46, | |
| "¬": 47, | |
| "*": 48, | |
| "§": 49, | |
| "=": 50, | |
| "é": 51, | |
| "æ": 52, | |
| "‒": 53, | |
| "&": 54, | |
| "]": 55, | |
| "α": 56, | |
| "ñ": 57, | |
| "[": 58, | |
| "ν": 59, | |
| "ο": 60, | |
| "ẽ": 61, | |
| "τ": 62, | |
| "ι": 63, | |
| "ε": 64, | |
| "+": 65, | |
| "ς": 66, | |
| "̃": 67, | |
| "â": 68, | |
| "à": 69, | |
| "è": 70, | |
| "σ": 71, | |
| "ô": 72, | |
| "π": 73, | |
| "μ": 74, | |
| "λ": 75, | |
| "ê": 76, | |
| "ρ": 77, | |
| "κ": 78, | |
| "ë": 79, | |
| "°": 80, | |
| "½": 81, | |
| "·": 82, | |
| "î": 83, | |
| "δ": 84, | |
| "á": 85, | |
| "†": 86, | |
| "υ": 87, | |
| "γ": 88, | |
| "‟": 89, | |
| "…": 90, | |
| "ͦ": 91, | |
| "ί": 92, | |
| "⁾": 93, | |
| "ω": 94, | |
| "η": 95, | |
| "œ": 96, | |
| "φ": 97, | |
| "β": 98, | |
| "ϑ": 99, | |
| "û": 100, | |
| "%": 101, | |
| "í": 102, | |
| "έ": 103, | |
| "ἀ": 104, | |
| "ά": 105, | |
| "ό": 106, | |
| "ἐ": 107, | |
| "ů": 108, | |
| "ϱ": 109, | |
| "χ": 110, | |
| "′": 111, | |
| "č": 112, | |
| "ò": 113, | |
| "ç": 114, | |
| "́": 115, | |
| "ʒ": 116, | |
| "¼": 117, | |
| "|": 118, | |
| "ὸ": 119, | |
| "ὶ": 120, | |
| "": 121, | |
| "─": 122, | |
| "ó": 123, | |
| "̄": 124, | |
| "ύ": 125, | |
| "ù": 126, | |
| "ï": 127, | |
| "ṅ": 128, | |
| "þ": 129, | |
| "ĕ": 130, | |
| "ῖ": 131, | |
| "″": 132, | |
| "ā": 133, | |
| "⋹": 134, | |
| "¾": 135, | |
| "¹": 136, | |
| "ũ": 137, | |
| "ú": 138, | |
| "ή": 139, | |
| "ῦ": 140, | |
| "θ": 141, | |
| "ῶ": 142, | |
| "š": 143, | |
| "×": 144, | |
| "ē": 145, | |
| "ξ": 146, | |
| "ὰ": 147, | |
| "ý": 148, | |
| "ɟ": 149, | |
| "õ": 150, | |
| "̅": 151, | |
| "ἰ": 152, | |
| "ø": 153, | |
| "ζ": 154, | |
| "ž": 155, | |
| "²": 156, | |
| "〃": 157, | |
| "ῆ": 158, | |
| "ꝟ": 159, | |
| "ã": 160, | |
| "ὴ": 161, | |
| "℔": 162, | |
| "ꝙ": 163, | |
| "ὐ": 164, | |
| "̆": 165, | |
| "⅓": 166, | |
| "¶": 167, | |
| "ἔ": 168, | |
| "£": 169, | |
| "å": 170, | |
| "│": 171, | |
| "⁽": 172, | |
| "³": 173, | |
| "□": 174, | |
| "℥": 175, | |
| "ϰ": 176, | |
| "ώ": 177, | |
| "ψ": 178, | |
| "⁴": 179, | |
| "∞": 180, | |
| "ř": 181, | |
| "ἄ": 182, | |
| "ę": 183, | |
| "ŭ": 184, | |
| "⅔": 185, | |
| "ὲ": 186, | |
| "ĭ": 187, | |
| "⁵": 188, | |
| "℞": 189, | |
| "ÿ": 190, | |
| "ū": 191, | |
| "⁶": 192, | |
| "⏑": 193, | |
| "#": 194, | |
| "≠": 195, | |
| "̶": 196, | |
| "ὁ": 197, | |
| "√": 198, | |
| "⁷": 199, | |
| "ὑ": 200, | |
| "ī": 201, | |
| "ì": 202, | |
| "⁸": 203, | |
| "ἱ": 204, | |
| "⁹": 205, | |
| "ἡ": 206, | |
| "ð": 207, | |
| "ł": 208, | |
| "̕": 209, | |
| "ʼ": 210, | |
| "⃛": 211, | |
| "}": 212, | |
| "☉": 213, | |
| "{": 214, | |
| "ą": 215, | |
| "ὀ": 216, | |
| "〟": 217, | |
| "ś": 218, | |
| "ῳ": 219, | |
| "₤": 220, | |
| "ὺ": 221, | |
| "⁰": 222, | |
| "ῷ": 223, | |
| "‥": 224, | |
| "⅛": 225, | |
| "ᾶ": 226, | |
| "ō": 227, | |
| "ă": 228, | |
| "ὅ": 229, | |
| "∫": 230, | |
| "ἑ": 231, | |
| "‰": 232, | |
| "ŷ": 233, | |
| "ż": 234, | |
| ">": 235, | |
| "ἴ": 236, | |
| "℈": 237, | |
| "<": 238, | |
| "ě": 239, | |
| "‧": 240, | |
| "♂": 241, | |
| "♀": 242, | |
| "ď": 243, | |
| "ḱ": 244, | |
| "⅕": 245, | |
| "᾽": 246, | |
| "ὄ": 247, | |
| "⅜": 248, | |
| "ˈ": 249, | |
| "ć": 250, | |
| "˙": 251, | |
| "ῥ": 252, | |
| "ἁ": 253, | |
| "‴": 254, | |
| "•": 255 | |
| }, | |
| "unk_token": "<unk>" | |
| } | |
| } |