Upload tokenizer
ff1e6f5 | { |
| " ": 1, |
| "!": 272, |
| "#": 302, |
| "%": 278, |
| "(": 275, |
| ")": 280, |
| "*": 301, |
| "+": 283, |
| "-": 285, |
| ".": 282, |
| "/": 287, |
| "0": 288, |
| "1": 289, |
| "2": 290, |
| "3": 291, |
| "4": 292, |
| "5": 293, |
| "6": 294, |
| "7": 295, |
| "8": 296, |
| "9": 297, |
| "<BOS>": 2, |
| "<EOS>": 3, |
| "<pad>": 0, |
| "?": 303, |
| "«": 276, |
| "»": 279, |
| "ሀ": 4, |
| "ሁ": 5, |
| "ሂ": 6, |
| "ሃ": 7, |
| "ሄ": 8, |
| "ህ": 9, |
| "ሆ": 10, |
| "ለ": 11, |
| "ሉ": 12, |
| "ሊ": 13, |
| "ላ": 14, |
| "ሌ": 15, |
| "ል": 16, |
| "ሎ": 17, |
| "ሏ": 18, |
| "ሐ": 19, |
| "ሑ": 20, |
| "ሒ": 21, |
| "ሓ": 22, |
| "ሔ": 23, |
| "ሕ": 24, |
| "ሖ": 25, |
| "መ": 26, |
| "ሙ": 27, |
| "ሚ": 28, |
| "ማ": 29, |
| "ሜ": 30, |
| "ም": 31, |
| "ሞ": 32, |
| "ሟ": 33, |
| "ሠ": 34, |
| "ሡ": 35, |
| "ሢ": 36, |
| "ሣ": 37, |
| "ሤ": 38, |
| "ሥ": 39, |
| "ሦ": 40, |
| "ሧ": 41, |
| "ረ": 42, |
| "ሩ": 43, |
| "ሪ": 44, |
| "ራ": 45, |
| "ሬ": 46, |
| "ር": 47, |
| "ሮ": 48, |
| "ሯ": 49, |
| "ሰ": 50, |
| "ሱ": 51, |
| "ሲ": 52, |
| "ሳ": 53, |
| "ሴ": 54, |
| "ስ": 55, |
| "ሶ": 56, |
| "ሷ": 57, |
| "ሸ": 58, |
| "ሹ": 59, |
| "ሺ": 60, |
| "ሻ": 61, |
| "ሼ": 62, |
| "ሽ": 63, |
| "ሾ": 64, |
| "ሿ": 65, |
| "ቀ": 66, |
| "ቁ": 67, |
| "ቂ": 68, |
| "ቃ": 69, |
| "ቄ": 70, |
| "ቅ": 71, |
| "ቆ": 72, |
| "ቋ": 73, |
| "በ": 74, |
| "ቡ": 75, |
| "ቢ": 76, |
| "ባ": 77, |
| "ቤ": 78, |
| "ብ": 79, |
| "ቦ": 80, |
| "ቧ": 81, |
| "ቨ": 82, |
| "ቩ": 83, |
| "ቪ": 84, |
| "ቫ": 85, |
| "ቬ": 86, |
| "ቭ": 87, |
| "ቮ": 88, |
| "ቯ": 89, |
| "ተ": 90, |
| "ቱ": 91, |
| "ቲ": 92, |
| "ታ": 93, |
| "ቴ": 94, |
| "ት": 95, |
| "ቶ": 96, |
| "ቷ": 97, |
| "ቸ": 98, |
| "ቹ": 99, |
| "ቺ": 100, |
| "ቻ": 101, |
| "ቼ": 102, |
| "ች": 103, |
| "ቾ": 104, |
| "ቿ": 105, |
| "ኀ": 106, |
| "ኁ": 107, |
| "ኂ": 108, |
| "ኃ": 109, |
| "ኄ": 110, |
| "ኅ": 111, |
| "ኋ": 112, |
| "ነ": 113, |
| "ኑ": 114, |
| "ኒ": 115, |
| "ና": 116, |
| "ኔ": 117, |
| "ን": 118, |
| "ኖ": 119, |
| "ኗ": 120, |
| "ኘ": 121, |
| "ኙ": 122, |
| "ኚ": 123, |
| "ኛ": 124, |
| "ኜ": 125, |
| "ኝ": 126, |
| "ኞ": 127, |
| "ኟ": 128, |
| "አ": 129, |
| "ኡ": 130, |
| "ኢ": 131, |
| "ኣ": 132, |
| "ኤ": 133, |
| "እ": 134, |
| "ኦ": 135, |
| "ኧ": 136, |
| "ከ": 137, |
| "ኩ": 138, |
| "ኪ": 139, |
| "ካ": 140, |
| "ኬ": 141, |
| "ክ": 142, |
| "ኮ": 143, |
| "ኳ": 144, |
| "ኸ": 145, |
| "ኹ": 146, |
| "ኺ": 147, |
| "ኻ": 148, |
| "ኼ": 149, |
| "ኽ": 150, |
| "ኾ": 151, |
| "ዀ": 152, |
| "ዃ": 153, |
| "ወ": 154, |
| "ዉ": 155, |
| "ዊ": 156, |
| "ዋ": 157, |
| "ዌ": 158, |
| "ው": 159, |
| "ዎ": 160, |
| "ዏ": 161, |
| "ዐ": 162, |
| "ዑ": 163, |
| "ዒ": 164, |
| "ዓ": 165, |
| "ዔ": 166, |
| "ዕ": 167, |
| "ዖ": 168, |
| "ዘ": 169, |
| "ዙ": 170, |
| "ዚ": 171, |
| "ዛ": 172, |
| "ዜ": 173, |
| "ዝ": 174, |
| "ዞ": 175, |
| "ዟ": 176, |
| "ዠ": 177, |
| "ዡ": 178, |
| "ዢ": 179, |
| "ዣ": 180, |
| "ዤ": 181, |
| "ዥ": 182, |
| "ዦ": 183, |
| "ዧ": 184, |
| "የ": 185, |
| "ዩ": 186, |
| "ዪ": 187, |
| "ያ": 188, |
| "ዬ": 189, |
| "ይ": 190, |
| "ዮ": 191, |
| "ደ": 192, |
| "ዱ": 193, |
| "ዲ": 194, |
| "ዳ": 195, |
| "ዴ": 196, |
| "ድ": 197, |
| "ዶ": 198, |
| "ዷ": 199, |
| "ጀ": 200, |
| "ጁ": 201, |
| "ጂ": 202, |
| "ጃ": 203, |
| "ጄ": 204, |
| "ጅ": 205, |
| "ጆ": 206, |
| "ጇ": 207, |
| "ገ": 208, |
| "ጉ": 209, |
| "ጊ": 210, |
| "ጋ": 211, |
| "ጌ": 212, |
| "ግ": 213, |
| "ጎ": 214, |
| "ጓ": 215, |
| "ጠ": 216, |
| "ጡ": 217, |
| "ጢ": 218, |
| "ጣ": 219, |
| "ጤ": 220, |
| "ጥ": 221, |
| "ጦ": 222, |
| "ጧ": 223, |
| "ጨ": 224, |
| "ጩ": 225, |
| "ጪ": 226, |
| "ጫ": 227, |
| "ጬ": 228, |
| "ጭ": 229, |
| "ጮ": 230, |
| "ጯ": 231, |
| "ጰ": 232, |
| "ጱ": 233, |
| "ጲ": 234, |
| "ጳ": 235, |
| "ጴ": 236, |
| "ጵ": 237, |
| "ጶ": 238, |
| "ጷ": 239, |
| "ጸ": 240, |
| "ጹ": 241, |
| "ጺ": 242, |
| "ጻ": 243, |
| "ጼ": 244, |
| "ጽ": 245, |
| "ጾ": 246, |
| "ጿ": 247, |
| "ፀ": 248, |
| "ፁ": 249, |
| "ፂ": 250, |
| "ፃ": 251, |
| "ፄ": 252, |
| "ፅ": 253, |
| "ፆ": 254, |
| "ፇ": 255, |
| "ፈ": 256, |
| "ፉ": 257, |
| "ፊ": 258, |
| "ፋ": 259, |
| "ፌ": 260, |
| "ፍ": 261, |
| "ፎ": 262, |
| "ፏ": 263, |
| "ፐ": 264, |
| "ፑ": 265, |
| "ፒ": 266, |
| "ፓ": 267, |
| "ፔ": 268, |
| "ፕ": 269, |
| "ፖ": 270, |
| "ፗ": 271, |
| "፝": 300, |
| "፡": 298, |
| "።": 286, |
| "፣": 284, |
| "፤": 299, |
| "፥": 277, |
| "፦": 273, |
| "‹": 274, |
| "›": 281 |
| } |
|
|