Upload tokenizer
4edd597 | { |
| " ": 16, |
| "!": 5, |
| "\"": 11, |
| "'": 176, |
| ",": 3, |
| ".": 4, |
| ":": 2, |
| ";": 1, |
| "<pad>": 177, |
| "<unk>": 178, |
| "?": 6, |
| "A": 17, |
| "B": 18, |
| "C": 19, |
| "D": 20, |
| "E": 21, |
| "F": 22, |
| "G": 23, |
| "H": 24, |
| "I": 25, |
| "J": 26, |
| "K": 27, |
| "L": 28, |
| "M": 29, |
| "N": 30, |
| "O": 31, |
| "P": 32, |
| "Q": 33, |
| "R": 34, |
| "S": 35, |
| "T": 36, |
| "U": 37, |
| "V": 38, |
| "W": 39, |
| "X": 40, |
| "Y": 41, |
| "Z": 42, |
| "_": 0, |
| "a": 43, |
| "b": 44, |
| "c": 45, |
| "d": 46, |
| "e": 47, |
| "f": 48, |
| "g": 49, |
| "h": 50, |
| "i": 51, |
| "j": 52, |
| "k": 53, |
| "l": 54, |
| "m": 55, |
| "n": 56, |
| "o": 57, |
| "p": 58, |
| "q": 59, |
| "r": 60, |
| "s": 61, |
| "t": 62, |
| "u": 63, |
| "v": 64, |
| "w": 65, |
| "x": 66, |
| "y": 67, |
| "z": 68, |
| "¡": 7, |
| "«": 12, |
| "»": 13, |
| "¿": 8, |
| "æ": 72, |
| "ç": 78, |
| "ð": 81, |
| "ø": 116, |
| "ħ": 98, |
| "ŋ": 112, |
| "œ": 120, |
| "ǀ": 152, |
| "ǁ": 153, |
| "ǂ": 154, |
| "ǃ": 155, |
| "ɐ": 70, |
| "ɑ": 69, |
| "ɒ": 71, |
| "ɓ": 73, |
| "ɔ": 76, |
| "ɕ": 77, |
| "ɖ": 80, |
| "ɗ": 79, |
| "ɘ": 84, |
| "ə": 83, |
| "ɚ": 85, |
| "ɛ": 86, |
| "ɜ": 87, |
| "ɝ": 88, |
| "ɞ": 89, |
| "ɟ": 90, |
| "ɠ": 93, |
| "ɡ": 92, |
| "ɢ": 94, |
| "ɣ": 139, |
| "ɤ": 140, |
| "ɥ": 99, |
| "ɦ": 96, |
| "ɧ": 97, |
| "ɨ": 101, |
| "ɪ": 102, |
| "ɫ": 106, |
| "ɬ": 105, |
| "ɭ": 104, |
| "ɮ": 107, |
| "ɯ": 110, |
| "ɰ": 111, |
| "ɱ": 109, |
| "ɲ": 114, |
| "ɳ": 113, |
| "ɴ": 115, |
| "ɵ": 117, |
| "ɶ": 121, |
| "ɸ": 118, |
| "ɹ": 123, |
| "ɺ": 124, |
| "ɻ": 126, |
| "ɽ": 129, |
| "ɾ": 125, |
| "ʀ": 127, |
| "ʁ": 128, |
| "ʂ": 130, |
| "ʃ": 131, |
| "ʄ": 91, |
| "ʈ": 132, |
| "ʉ": 134, |
| "ʊ": 135, |
| "ʋ": 136, |
| "ʌ": 138, |
| "ʍ": 141, |
| "ʎ": 143, |
| "ʏ": 144, |
| "ʐ": 146, |
| "ʑ": 145, |
| "ʒ": 147, |
| "ʔ": 148, |
| "ʕ": 150, |
| "ʘ": 122, |
| "ʙ": 74, |
| "ʛ": 95, |
| "ʜ": 100, |
| "ʝ": 103, |
| "ʟ": 108, |
| "ʡ": 149, |
| "ʢ": 151, |
| "ʤ": 82, |
| "ʧ": 133, |
| "ʰ": 162, |
| "ʱ": 163, |
| "ʲ": 164, |
| "ʴ": 161, |
| "ʷ": 165, |
| "ʼ": 160, |
| "ˈ": 156, |
| "ˌ": 157, |
| "ː": 158, |
| "ˑ": 159, |
| "˞": 168, |
| "ˠ": 166, |
| "ˤ": 167, |
| "̩": 175, |
| "β": 75, |
| "θ": 119, |
| "χ": 142, |
| "ᵻ": 177, |
| "—": 9, |
| "“": 14, |
| "”": 15, |
| "…": 10, |
| "↑": 170, |
| "→": 171, |
| "↓": 169, |
| "↗": 172, |
| "↘": 173, |
| "ⱱ": 137 |
| } |
|
|
