Upload tokenizer
f4d9da7 verified | { |
| "$": 1, |
| "&": 2, |
| "+": 3, |
| "/": 4, |
| "0": 5, |
| "1": 6, |
| "2": 7, |
| "3": 8, |
| "4": 9, |
| "5": 10, |
| "6": 11, |
| "7": 12, |
| "8": 13, |
| "9": 14, |
| "[": 15, |
| "[PAD]": 288, |
| "[UNK]": 287, |
| "]": 16, |
| "a": 17, |
| "b": 18, |
| "c": 19, |
| "d": 20, |
| "e": 21, |
| "f": 22, |
| "g": 23, |
| "h": 24, |
| "i": 25, |
| "j": 26, |
| "k": 27, |
| "l": 28, |
| "m": 29, |
| "n": 30, |
| "o": 31, |
| "p": 32, |
| "q": 33, |
| "r": 34, |
| "s": 35, |
| "t": 36, |
| "u": 37, |
| "v": 38, |
| "w": 39, |
| "x": 40, |
| "y": 41, |
| "z": 42, |
| "|": 0, |
| "~": 43, |
| "£": 44, |
| "¥": 45, |
| "°": 46, |
| "²": 47, |
| "½": 48, |
| "¾": 49, |
| "ã": 50, |
| "õ": 51, |
| "ሀ": 52, |
| "ሁ": 53, |
| "ሂ": 54, |
| "ሃ": 55, |
| "ሄ": 56, |
| "ህ": 57, |
| "ሆ": 58, |
| "ለ": 59, |
| "ሉ": 60, |
| "ሊ": 61, |
| "ላ": 62, |
| "ሌ": 63, |
| "ል": 64, |
| "ሎ": 65, |
| "ሏ": 66, |
| "ሐ": 67, |
| "ሓ": 68, |
| "ሔ": 69, |
| "ሕ": 70, |
| "መ": 71, |
| "ሙ": 72, |
| "ሚ": 73, |
| "ማ": 74, |
| "ሜ": 75, |
| "ም": 76, |
| "ሞ": 77, |
| "ሟ": 78, |
| "ሠ": 79, |
| "ሣ": 80, |
| "ሥ": 81, |
| "ሦ": 82, |
| "ረ": 83, |
| "ሩ": 84, |
| "ሪ": 85, |
| "ራ": 86, |
| "ሬ": 87, |
| "ር": 88, |
| "ሮ": 89, |
| "ሯ": 90, |
| "ሰ": 91, |
| "ሱ": 92, |
| "ሲ": 93, |
| "ሳ": 94, |
| "ሴ": 95, |
| "ስ": 96, |
| "ሶ": 97, |
| "ሷ": 98, |
| "ሸ": 99, |
| "ሹ": 100, |
| "ሺ": 101, |
| "ሻ": 102, |
| "ሼ": 103, |
| "ሽ": 104, |
| "ሾ": 105, |
| "ቀ": 106, |
| "ቁ": 107, |
| "ቂ": 108, |
| "ቃ": 109, |
| "ቄ": 110, |
| "ቅ": 111, |
| "ቆ": 112, |
| "ቋ": 113, |
| "በ": 114, |
| "ቡ": 115, |
| "ቢ": 116, |
| "ባ": 117, |
| "ቤ": 118, |
| "ብ": 119, |
| "ቦ": 120, |
| "ቧ": 121, |
| "ቨ": 122, |
| "ቪ": 123, |
| "ቫ": 124, |
| "ቭ": 125, |
| "ቮ": 126, |
| "ተ": 127, |
| "ቱ": 128, |
| "ቲ": 129, |
| "ታ": 130, |
| "ቴ": 131, |
| "ት": 132, |
| "ቶ": 133, |
| "ቷ": 134, |
| "ቸ": 135, |
| "ቹ": 136, |
| "ቺ": 137, |
| "ቻ": 138, |
| "ቼ": 139, |
| "ች": 140, |
| "ቾ": 141, |
| "ቿ": 142, |
| "ኀ": 143, |
| "ኃ": 144, |
| "ኅ": 145, |
| "ኋ": 146, |
| "ነ": 147, |
| "ኑ": 148, |
| "ኒ": 149, |
| "ና": 150, |
| "ኔ": 151, |
| "ን": 152, |
| "ኖ": 153, |
| "ኗ": 154, |
| "ኘ": 155, |
| "ኙ": 156, |
| "ኚ": 157, |
| "ኛ": 158, |
| "ኝ": 159, |
| "ኞ": 160, |
| "አ": 161, |
| "ኡ": 162, |
| "ኢ": 163, |
| "ኣ": 164, |
| "ኤ": 165, |
| "እ": 166, |
| "ኦ": 167, |
| "ከ": 168, |
| "ኩ": 169, |
| "ኪ": 170, |
| "ካ": 171, |
| "ኬ": 172, |
| "ክ": 173, |
| "ኮ": 174, |
| "ኳ": 175, |
| "ኸ": 176, |
| "ወ": 177, |
| "ዉ": 178, |
| "ዊ": 179, |
| "ዋ": 180, |
| "ዌ": 181, |
| "ው": 182, |
| "ዎ": 183, |
| "ዐ": 184, |
| "ዑ": 185, |
| "ዓ": 186, |
| "ዕ": 187, |
| "ዖ": 188, |
| "ዘ": 189, |
| "ዙ": 190, |
| "ዚ": 191, |
| "ዛ": 192, |
| "ዜ": 193, |
| "ዝ": 194, |
| "ዞ": 195, |
| "ዟ": 196, |
| "ዠ": 197, |
| "ዢ": 198, |
| "ዣ": 199, |
| "ዥ": 200, |
| "ዦ": 201, |
| "የ": 202, |
| "ዩ": 203, |
| "ዪ": 204, |
| "ያ": 205, |
| "ዬ": 206, |
| "ይ": 207, |
| "ዮ": 208, |
| "ደ": 209, |
| "ዱ": 210, |
| "ዲ": 211, |
| "ዳ": 212, |
| "ዴ": 213, |
| "ድ": 214, |
| "ዶ": 215, |
| "ዷ": 216, |
| "ጀ": 217, |
| "ጁ": 218, |
| "ጂ": 219, |
| "ጃ": 220, |
| "ጄ": 221, |
| "ጅ": 222, |
| "ጆ": 223, |
| "ጇ": 224, |
| "ገ": 225, |
| "ጉ": 226, |
| "ጊ": 227, |
| "ጋ": 228, |
| "ጌ": 229, |
| "ግ": 230, |
| "ጎ": 231, |
| "ጓ": 232, |
| "ጠ": 233, |
| "ጡ": 234, |
| "ጢ": 235, |
| "ጣ": 236, |
| "ጤ": 237, |
| "ጥ": 238, |
| "ጦ": 239, |
| "ጧ": 240, |
| "ጨ": 241, |
| "ጩ": 242, |
| "ጪ": 243, |
| "ጫ": 244, |
| "ጭ": 245, |
| "ጮ": 246, |
| "ጳ": 247, |
| "ጴ": 248, |
| "ጵ": 249, |
| "ጸ": 250, |
| "ጹ": 251, |
| "ጻ": 252, |
| "ጽ": 253, |
| "ጾ": 254, |
| "ጿ": 255, |
| "ፀ": 256, |
| "ፁ": 257, |
| "ፃ": 258, |
| "ፅ": 259, |
| "ፈ": 260, |
| "ፉ": 261, |
| "ፊ": 262, |
| "ፋ": 263, |
| "ፌ": 264, |
| "ፍ": 265, |
| "ፎ": 266, |
| "ፏ": 267, |
| "ፐ": 268, |
| "ፑ": 269, |
| "ፒ": 270, |
| "ፓ": 271, |
| "ፔ": 272, |
| "ፕ": 273, |
| "ፖ": 274, |
| "ፗ": 275, |
| "፡": 276, |
| "።": 277, |
| "፣": 278, |
| "፤": 279, |
| "፥": 280, |
| "፦": 281, |
| "–": 282, |
| "—": 283, |
| "’": 284, |
| "‹": 285, |
| "›": 286 |
| } |
|
|