Upload tokenizer

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "</s>": 305,
+  "<PAD>": 307,
+  "<mask>": 308,
+  "<s>": 304,
+  "<unk>": 306
+}

merges.txt

@@ -0,0 +1,305 @@
+#version: 0.2
+"<pad>": 0,
+" ": 1,
+"<BOS>": 2,
+"<EOS>": 3,
+"\u1200": 4,
+"\u1201": 5,
+"\u1202": 6,
+"\u1203": 7,
+"\u1204": 8,
+"\u1205": 9,
+"\u1206": 10,
+"\u1208": 11,
+"\u1209": 12,
+"\u120a": 13,
+"\u120b": 14,
+"\u120c": 15,
+"\u120d": 16,
+"\u120e": 17,
+"\u120f": 18,
+"\u1210": 19,
+"\u1211": 20,
+"\u1212": 21,
+"\u1213": 22,
+"\u1214": 23,
+"\u1215": 24,
+"\u1216": 25,
+"\u1218": 26,
+"\u1219": 27,
+"\u121a": 28,
+"\u121b": 29,
+"\u121c": 30,
+"\u121d": 31,
+"\u121e": 32,
+"\u121f": 33,
+"\u1220": 34,
+"\u1221": 35,
+"\u1222": 36,
+"\u1223": 37,
+"\u1224": 38,
+"\u1225": 39,
+"\u1226": 40,
+"\u1227": 41,
+"\u1228": 42,
+"\u1229": 43,
+"\u122a": 44,
+"\u122b": 45,
+"\u122c": 46,
+"\u122d": 47,
+"\u122e": 48,
+"\u122f": 49,
+"\u1230": 50,
+"\u1231": 51,
+"\u1232": 52,
+"\u1233": 53,
+"\u1234": 54,
+"\u1235": 55,
+"\u1236": 56,
+"\u1237": 57,
+"\u1238": 58,
+"\u1239": 59,
+"\u123a": 60,
+"\u123b": 61,
+"\u123c": 62,
+"\u123d": 63,
+"\u123e": 64,
+"\u123f": 65,
+"\u1240": 66,
+"\u1241": 67,
+"\u1242": 68,
+"\u1243": 69,
+"\u1244": 70,
+"\u1245": 71,
+"\u1246": 72,
+"\u124b": 73,
+"\u1260": 74,
+"\u1261": 75,
+"\u1262": 76,
+"\u1263": 77,
+"\u1264": 78,
+"\u1265": 79,
+"\u1266": 80,
+"\u1267": 81,
+"\u1268": 82,
+"\u1269": 83,
+"\u126a": 84,
+"\u126b": 85,
+"\u126c": 86,
+"\u126d": 87,
+"\u126e": 88,
+"\u126f": 89,
+"\u1270": 90,
+"\u1271": 91,
+"\u1272": 92,
+"\u1273": 93,
+"\u1274": 94,
+"\u1275": 95,
+"\u1276": 96,
+"\u1277": 97,
+"\u1278": 98,
+"\u1279": 99,
+"\u127a": 100,
+"\u127b": 101,
+"\u127c": 102,
+"\u127d": 103,
+"\u127e": 104,
+"\u127f": 105,
+"\u1280": 106,
+"\u1281": 107,
+"\u1282": 108,
+"\u1283": 109,
+"\u1284": 110,
+"\u1285": 111,
+"\u128b": 112,
+"\u1290": 113,
+"\u1291": 114,
+"\u1292": 115,
+"\u1293": 116,
+"\u1294": 117,
+"\u1295": 118,
+"\u1296": 119,
+"\u1297": 120,
+"\u1298": 121,
+"\u1299": 122,
+"\u129a": 123,
+"\u129b": 124,
+"\u129c": 125,
+"\u129d": 126,
+"\u129e": 127,
+"\u129f": 128,
+"\u12a0": 129,
+"\u12a1": 130,
+"\u12a2": 131,
+"\u12a3": 132,
+"\u12a4": 133,
+"\u12a5": 134,
+"\u12a6": 135,
+"\u12a7": 136,
+"\u12a8": 137,
+"\u12a9": 138,
+"\u12aa": 139,
+"\u12ab": 140,
+"\u12ac": 141,
+"\u12ad": 142,
+"\u12ae": 143,
+"\u12b3": 144,
+"\u12b8": 145,
+"\u12b9": 146,
+"\u12ba": 147,
+"\u12bb": 148,
+"\u12bc": 149,
+"\u12bd": 150,
+"\u12be": 151,
+"\u12c0": 152,
+"\u12c3": 153,
+"\u12c8": 154,
+"\u12c9": 155,
+"\u12ca": 156,
+"\u12cb": 157,
+"\u12cc": 158,
+"\u12cd": 159,
+"\u12ce": 160,
+"\u12cf": 161,
+"\u12d0": 162,
+"\u12d1": 163,
+"\u12d2": 164,
+"\u12d3": 165,
+"\u12d4": 166,
+"\u12d5": 167,
+"\u12d6": 168,
+"\u12d8": 169,
+"\u12d9": 170,
+"\u12da": 171,
+"\u12db": 172,
+"\u12dc": 173,
+"\u12dd": 174,
+"\u12de": 175,
+"\u12df": 176,
+"\u12e0": 177,
+"\u12e1": 178,
+"\u12e2": 179,
+"\u12e3": 180,
+"\u12e4": 181,
+"\u12e5": 182,
+"\u12e6": 183,
+"\u12e7": 184,
+"\u12e8": 185,
+"\u12e9": 186,
+"\u12ea": 187,
+"\u12eb": 188,
+"\u12ec": 189,
+"\u12ed": 190,
+"\u12ee": 191,
+"\u12f0": 192,
+"\u12f1": 193,
+"\u12f2": 194,
+"\u12f3": 195,
+"\u12f4": 196,
+"\u12f5": 197,
+"\u12f6": 198,
+"\u12f7": 199,
+"\u1300": 200,
+"\u1301": 201,
+"\u1302": 202,
+"\u1303": 203,
+"\u1304": 204,
+"\u1305": 205,
+"\u1306": 206,
+"\u1307": 207,
+"\u1308": 208,
+"\u1309": 209,
+"\u130a": 210,
+"\u130b": 211,
+"\u130c": 212,
+"\u130d": 213,
+"\u130e": 214,
+"\u1313": 215,
+"\u1320": 216,
+"\u1321": 217,
+"\u1322": 218,
+"\u1323": 219,
+"\u1324": 220,
+"\u1325": 221,
+"\u1326": 222,
+"\u1327": 223,
+"\u1328": 224,
+"\u1329": 225,
+"\u132a": 226,
+"\u132b": 227,
+"\u132c": 228,
+"\u132d": 229,
+"\u132e": 230,
+"\u132f": 231,
+"\u1330": 232,
+"\u1331": 233,
+"\u1332": 234,
+"\u1333": 235,
+"\u1334": 236,
+"\u1335": 237,
+"\u1336": 238,
+"\u1337": 239,
+"\u1338": 240,
+"\u1339": 241,
+"\u133a": 242,
+"\u133b": 243,
+"\u133c": 244,
+"\u133d": 245,
+"\u133e": 246,
+"\u133f": 247,
+"\u1340": 248,
+"\u1341": 249,
+"\u1342": 250,
+"\u1343": 251,
+"\u1344": 252,
+"\u1345": 253,
+"\u1346": 254,
+"\u1347": 255,
+"\u1348": 256,
+"\u1349": 257,
+"\u134a": 258,
+"\u134b": 259,
+"\u134c": 260,
+"\u134d": 261,
+"\u134e": 262,
+"\u134f": 263,
+"\u1350": 264,
+"\u1351": 265,
+"\u1352": 266,
+"\u1353": 267,
+"\u1354": 268,
+"\u1355": 269,
+"\u1356": 270,
+"\u1357": 271,
+"!": 272,
+"\u1366": 273,
+"\u2039": 274,
+"(": 275,
+"\u00ab": 276,
+"\u1365": 277,
+"%": 278,
+"\u00bb": 279,
+")": 280,
+"\u203a": 281,
+".": 282,
+"+": 283,
+"\u1363": 284,
+"-": 285,
+"\u1362": 286,
+"/": 287,
+"0": 288,
+"1": 289,
+"2": 290,
+"3": 291,
+"4": 292,
+"5": 293,
+"6": 294,
+"7": 295,
+"8": 296,
+"9": 297,
+"\u1361": 298,
+"\u1364": 299,
+"\u135d": 300,
+"*": 301,
+"#": 302,
+"?": 303

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<PAD>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "304": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "305": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "306": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "307": {
+      "content": "<PAD>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "308": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD>",
+  "sep_token": "</s>",
+  "tokenizer_class": "BartTokenizer",
+  "unk_token": "<unk>"
+}

vocab.json

@@ -0,0 +1,306 @@
+{
+  " ": 1,
+  "!": 272,
+  "#": 302,
+  "%": 278,
+  "(": 275,
+  ")": 280,
+  "*": 301,
+  "+": 283,
+  "-": 285,
+  ".": 282,
+  "/": 287,
+  "0": 288,
+  "1": 289,
+  "2": 290,
+  "3": 291,
+  "4": 292,
+  "5": 293,
+  "6": 294,
+  "7": 295,
+  "8": 296,
+  "9": 297,
+  "<BOS>": 2,
+  "<EOS>": 3,
+  "<pad>": 0,
+  "?": 303,
+  "«": 276,
+  "»": 279,
+  "ሀ": 4,
+  "ሁ": 5,
+  "ሂ": 6,
+  "ሃ": 7,
+  "ሄ": 8,
+  "ህ": 9,
+  "ሆ": 10,
+  "ለ": 11,
+  "ሉ": 12,
+  "ሊ": 13,
+  "ላ": 14,
+  "ሌ": 15,
+  "ል": 16,
+  "ሎ": 17,
+  "ሏ": 18,
+  "ሐ": 19,
+  "ሑ": 20,
+  "ሒ": 21,
+  "ሓ": 22,
+  "ሔ": 23,
+  "ሕ": 24,
+  "ሖ": 25,
+  "መ": 26,
+  "ሙ": 27,
+  "ሚ": 28,
+  "ማ": 29,
+  "ሜ": 30,
+  "ም": 31,
+  "ሞ": 32,
+  "ሟ": 33,
+  "ሠ": 34,
+  "ሡ": 35,
+  "ሢ": 36,
+  "ሣ": 37,
+  "ሤ": 38,
+  "ሥ": 39,
+  "ሦ": 40,
+  "ሧ": 41,
+  "ረ": 42,
+  "ሩ": 43,
+  "ሪ": 44,
+  "ራ": 45,
+  "ሬ": 46,
+  "ር": 47,
+  "ሮ": 48,
+  "ሯ": 49,
+  "ሰ": 50,
+  "ሱ": 51,
+  "ሲ": 52,
+  "ሳ": 53,
+  "ሴ": 54,
+  "ስ": 55,
+  "ሶ": 56,
+  "ሷ": 57,
+  "ሸ": 58,
+  "ሹ": 59,
+  "ሺ": 60,
+  "ሻ": 61,
+  "ሼ": 62,
+  "ሽ": 63,
+  "ሾ": 64,
+  "ሿ": 65,
+  "ቀ": 66,
+  "ቁ": 67,
+  "ቂ": 68,
+  "ቃ": 69,
+  "ቄ": 70,
+  "ቅ": 71,
+  "ቆ": 72,
+  "ቋ": 73,
+  "በ": 74,
+  "ቡ": 75,
+  "ቢ": 76,
+  "ባ": 77,
+  "ቤ": 78,
+  "ብ": 79,
+  "ቦ": 80,
+  "ቧ": 81,
+  "ቨ": 82,
+  "ቩ": 83,
+  "ቪ": 84,
+  "ቫ": 85,
+  "ቬ": 86,
+  "ቭ": 87,
+  "ቮ": 88,
+  "ቯ": 89,
+  "ተ": 90,
+  "ቱ": 91,
+  "ቲ": 92,
+  "ታ": 93,
+  "ቴ": 94,
+  "ት": 95,
+  "ቶ": 96,
+  "ቷ": 97,
+  "ቸ": 98,
+  "ቹ": 99,
+  "ቺ": 100,
+  "ቻ": 101,
+  "ቼ": 102,
+  "ች": 103,
+  "ቾ": 104,
+  "ቿ": 105,
+  "ኀ": 106,
+  "ኁ": 107,
+  "ኂ": 108,
+  "ኃ": 109,
+  "ኄ": 110,
+  "ኅ": 111,
+  "ኋ": 112,
+  "ነ": 113,
+  "ኑ": 114,
+  "ኒ": 115,
+  "ና": 116,
+  "ኔ": 117,
+  "ን": 118,
+  "ኖ": 119,
+  "ኗ": 120,
+  "ኘ": 121,
+  "ኙ": 122,
+  "ኚ": 123,
+  "ኛ": 124,
+  "ኜ": 125,
+  "ኝ": 126,
+  "ኞ": 127,
+  "ኟ": 128,
+  "አ": 129,
+  "ኡ": 130,
+  "ኢ": 131,
+  "ኣ": 132,
+  "ኤ": 133,
+  "እ": 134,
+  "ኦ": 135,
+  "ኧ": 136,
+  "ከ": 137,
+  "ኩ": 138,
+  "ኪ": 139,
+  "ካ": 140,
+  "ኬ": 141,
+  "ክ": 142,
+  "ኮ": 143,
+  "ኳ": 144,
+  "ኸ": 145,
+  "ኹ": 146,
+  "ኺ": 147,
+  "ኻ": 148,
+  "ኼ": 149,
+  "ኽ": 150,
+  "ኾ": 151,
+  "ዀ": 152,
+  "ዃ": 153,
+  "ወ": 154,
+  "ዉ": 155,
+  "ዊ": 156,
+  "ዋ": 157,
+  "ዌ": 158,
+  "ው": 159,
+  "ዎ": 160,
+  "ዏ": 161,
+  "ዐ": 162,
+  "ዑ": 163,
+  "ዒ": 164,
+  "ዓ": 165,
+  "ዔ": 166,
+  "ዕ": 167,
+  "ዖ": 168,
+  "ዘ": 169,
+  "ዙ": 170,
+  "ዚ": 171,
+  "ዛ": 172,
+  "ዜ": 173,
+  "ዝ": 174,
+  "ዞ": 175,
+  "ዟ": 176,
+  "ዠ": 177,
+  "ዡ": 178,
+  "ዢ": 179,
+  "ዣ": 180,
+  "ዤ": 181,
+  "ዥ": 182,
+  "ዦ": 183,
+  "ዧ": 184,
+  "የ": 185,
+  "ዩ": 186,
+  "ዪ": 187,
+  "ያ": 188,
+  "ዬ": 189,
+  "ይ": 190,
+  "ዮ": 191,
+  "ደ": 192,
+  "ዱ": 193,
+  "ዲ": 194,
+  "ዳ": 195,
+  "ዴ": 196,
+  "ድ": 197,
+  "ዶ": 198,
+  "ዷ": 199,
+  "ጀ": 200,
+  "ጁ": 201,
+  "ጂ": 202,
+  "ጃ": 203,
+  "ጄ": 204,
+  "ጅ": 205,
+  "ጆ": 206,
+  "ጇ": 207,
+  "ገ": 208,
+  "ጉ": 209,
+  "ጊ": 210,
+  "ጋ": 211,
+  "ጌ": 212,
+  "ግ": 213,
+  "ጎ": 214,
+  "ጓ": 215,
+  "ጠ": 216,
+  "ጡ": 217,
+  "ጢ": 218,
+  "ጣ": 219,
+  "ጤ": 220,
+  "ጥ": 221,
+  "ጦ": 222,
+  "ጧ": 223,
+  "ጨ": 224,
+  "ጩ": 225,
+  "ጪ": 226,
+  "ጫ": 227,
+  "ጬ": 228,
+  "ጭ": 229,
+  "ጮ": 230,
+  "ጯ": 231,
+  "ጰ": 232,
+  "ጱ": 233,
+  "ጲ": 234,
+  "ጳ": 235,
+  "ጴ": 236,
+  "ጵ": 237,
+  "ጶ": 238,
+  "ጷ": 239,
+  "ጸ": 240,
+  "ጹ": 241,
+  "ጺ": 242,
+  "ጻ": 243,
+  "ጼ": 244,
+  "ጽ": 245,
+  "ጾ": 246,
+  "ጿ": 247,
+  "ፀ": 248,
+  "ፁ": 249,
+  "ፂ": 250,
+  "ፃ": 251,
+  "ፄ": 252,
+  "ፅ": 253,
+  "ፆ": 254,
+  "ፇ": 255,
+  "ፈ": 256,
+  "ፉ": 257,
+  "ፊ": 258,
+  "ፋ": 259,
+  "ፌ": 260,
+  "ፍ": 261,
+  "ፎ": 262,
+  "ፏ": 263,
+  "ፐ": 264,
+  "ፑ": 265,
+  "ፒ": 266,
+  "ፓ": 267,
+  "ፔ": 268,
+  "ፕ": 269,
+  "ፖ": 270,
+  "ፗ": 271,
+  "፝": 300,
+  "፡": 298,
+  "።": 286,
+  "፣": 284,
+  "፤": 299,
+  "፥": 277,
+  "፦": 273,
+  "‹": 274,
+  "›": 281
+}