Upload tokenizer

tokenizer.json

@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "<|vsep|>",
+      "content": "</|psep|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "<|bsep|>",
+      "content": "<|vsep|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 4,
-      "content": "<|pad|>",
+      "content": "<|bsep|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -50,7 +50,7 @@
     },
     {
       "id": 5,
-      "content": "<|meter_0|>",
+      "content": "</|bsep|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
     },
     {
       "id": 6,
-      "content": "<|meter_1|>",
+      "content": "<|pad|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,7 +68,7 @@
     },
     {
       "id": 7,
-      "content": "<|meter_2|>",
+      "content": "<|meter_0|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -77,7 +77,7 @@
     },
     {
       "id": 8,
-      "content": "<|meter_3|>",
+      "content": "<|meter_1|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -86,7 +86,7 @@
     },
     {
       "id": 9,
-      "content": "<|meter_4|>",
+      "content": "<|meter_2|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "<|meter_5|>",
+      "content": "<|meter_3|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "<|meter_6|>",
+      "content": "<|meter_4|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 12,
-      "content": "<|meter_7|>",
+      "content": "<|meter_5|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 13,
-      "content": "<|meter_8|>",
+      "content": "<|meter_6|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,7 +131,7 @@
     },
     {
       "id": 14,
-      "content": "<|meter_9|>",
+      "content": "<|meter_7|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -140,7 +140,7 @@
     },
     {
       "id": 15,
-      "content": "<|meter_10|>",
+      "content": "<|meter_8|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -149,7 +149,7 @@
     },
     {
       "id": 16,
-      "content": "<|meter_11|>",
+      "content": "<|meter_9|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -158,7 +158,7 @@
     },
     {
       "id": 17,
-      "content": "<|meter_12|>",
+      "content": "<|meter_10|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -167,7 +167,7 @@
     },
     {
       "id": 18,
-      "content": "<|meter_13|>",
+      "content": "<|meter_11|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -176,7 +176,7 @@
     },
     {
       "id": 19,
-      "content": "<|meter_14|>",
+      "content": "<|meter_12|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -185,7 +185,7 @@
     },
     {
       "id": 20,
-      "content": "<|meter_15|>",
+      "content": "<|meter_13|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -194,7 +194,7 @@
     },
     {
       "id": 21,
-      "content": "<|res_0|>",
+      "content": "<|meter_14|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -203,7 +203,7 @@
     },
     {
       "id": 22,
-      "content": "<|res_1|>",
+      "content": "<|meter_15|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -212,7 +212,7 @@
     },
     {
       "id": 23,
-      "content": "<|res_2|>",
+      "content": "<|res_0|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -221,7 +221,7 @@
     },
     {
       "id": 24,
-      "content": "<|res_3|>",
+      "content": "<|res_1|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -230,7 +230,7 @@
     },
     {
       "id": 25,
-      "content": "<|res_4|>",
+      "content": "<|res_2|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -239,7 +239,7 @@
     },
     {
       "id": 26,
-      "content": "<|res_5|>",
+      "content": "<|res_3|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -248,7 +248,7 @@
     },
     {
       "id": 27,
-      "content": "<|res_6|>",
+      "content": "<|res_4|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -257,7 +257,7 @@
     },
     {
       "id": 28,
-      "content": "<|res_7|>",
+      "content": "<|res_5|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -266,7 +266,7 @@
     },
     {
       "id": 29,
-      "content": "<|res_8|>",
+      "content": "<|res_6|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -275,7 +275,7 @@
     },
     {
       "id": 30,
-      "content": "<|res_9|>",
+      "content": "<|res_7|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -284,7 +284,7 @@
     },
     {
       "id": 31,
-      "content": "<|res_10|>",
+      "content": "<|res_8|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -293,16 +293,7 @@
     },
     {
       "id": 32,
-      "content": "<|res_11|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 33,
-      "content": "<|res_12|>",
+      "content": "<|res_9|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -327,39 +318,39 @@
     "vocab": {
       "<|endoftext|>": 0,
       "<|psep|>": 1,
-      "<|vsep|>": 2,
-      "<|bsep|>": 3,
-      "<|pad|>": 4,
-      "<|meter_0|>": 5,
-      "<|meter_1|>": 6,
-      "<|meter_2|>": 7,
-      "<|meter_3|>": 8,
-      "<|meter_4|>": 9,
-      "<|meter_5|>": 10,
-      "<|meter_6|>": 11,
-      "<|meter_7|>": 12,
-      "<|meter_8|>": 13,
-      "<|meter_9|>": 14,
-      "<|meter_10|>": 15,
-      "<|meter_11|>": 16,
-      "<|meter_12|>": 17,
-      "<|meter_13|>": 18,
-      "<|meter_14|>": 19,
-      "<|meter_15|>": 20,
-      "<|res_0|>": 21,
-      "<|res_1|>": 22,
-      "<|res_2|>": 23,
-      "<|res_3|>": 24,
-      "<|res_4|>": 25,
-      "<|res_5|>": 26,
-      "<|res_6|>": 27,
-      "<|res_7|>": 28,
-      "<|res_8|>": 29,
-      "<|res_9|>": 30,
-      "<|res_10|>": 31,
-      "<|res_11|>": 32,
-      "<|res_12|>": 33,
-      " ": 34,
+      "</|psep|>": 2,
+      "<|vsep|>": 3,
+      "<|bsep|>": 4,
+      "</|bsep|>": 5,
+      "<|pad|>": 6,
+      "<|meter_0|>": 7,
+      "<|meter_1|>": 8,
+      "<|meter_2|>": 9,
+      "<|meter_3|>": 10,
+      "<|meter_4|>": 11,
+      "<|meter_5|>": 12,
+      "<|meter_6|>": 13,
+      "<|meter_7|>": 14,
+      "<|meter_8|>": 15,
+      "<|meter_9|>": 16,
+      "<|meter_10|>": 17,
+      "<|meter_11|>": 18,
+      "<|meter_12|>": 19,
+      "<|meter_13|>": 20,
+      "<|meter_14|>": 21,
+      "<|meter_15|>": 22,
+      "<|res_0|>": 23,
+      "<|res_1|>": 24,
+      "<|res_2|>": 25,
+      "<|res_3|>": 26,
+      "<|res_4|>": 27,
+      "<|res_5|>": 28,
+      "<|res_6|>": 29,
+      "<|res_7|>": 30,
+      "<|res_8|>": 31,
+      "<|res_9|>": 32,
+      " ": 33,
+      "/": 34,
       "0": 35,
       "1": 36,
       "2": 37,

vocab.json

@@ -1 +1 @@
-{"<|endoftext|>":0,"<|psep|>":1,"<|vsep|>":2,"<|bsep|>":3,"<|pad|>":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"<|meter_14|>":19,"<|meter_15|>":20,"<|res_0|>":21,"<|res_1|>":22,"<|res_2|>":23,"<|res_3|>":24,"<|res_4|>":25,"<|res_5|>":26,"<|res_6|>":27,"<|res_7|>":28,"<|res_8|>":29,"<|res_9|>":30,"<|res_10|>":31,"<|res_11|>":32,"<|res_12|>":33," ":34,"0":35,"1":36,"2":37,"3":38,"4":39,"5":40,"6":41,"7":42,"8":43,"9":44,"<":45,">":46,"_":47,"b":48,"e":49,"m":50,"p":51,"r":52,"s":53,"t":54,"v":55,"|":56,"~":57,"ء":58,"أ":59,"ؤ":60,"ئ":61,"ا":62,"ب":63,"ة":64,"ت":65,"ث":66,"ج":67,"ح":68,"خ":69,"د":70,"ذ":71,"ر":72,"ز":73,"س":74,"ش":75,"ص":76,"ض":77,"ط":78,"ظ":79,"ع":80,"غ":81,"ف":82,"ق":83,"ك":84,"ل":85,"م":86,"ن":87,"ه":88,"و":89,"ى":90,"ي":91,"ً":92,"ٌ":93,"ٍ":94,"َ":95,"ُ":96,"ِ":97,"ّ":98,"ْ":99}
+{"<|endoftext|>":0,"<|psep|>":1,"</|psep|>":2,"<|vsep|>":3,"<|bsep|>":4,"</|bsep|>":5,"<|pad|>":6,"<|meter_0|>":7,"<|meter_1|>":8,"<|meter_2|>":9,"<|meter_3|>":10,"<|meter_4|>":11,"<|meter_5|>":12,"<|meter_6|>":13,"<|meter_7|>":14,"<|meter_8|>":15,"<|meter_9|>":16,"<|meter_10|>":17,"<|meter_11|>":18,"<|meter_12|>":19,"<|meter_13|>":20,"<|meter_14|>":21,"<|meter_15|>":22,"<|res_0|>":23,"<|res_1|>":24,"<|res_2|>":25,"<|res_3|>":26,"<|res_4|>":27,"<|res_5|>":28,"<|res_6|>":29,"<|res_7|>":30,"<|res_8|>":31,"<|res_9|>":32," ":33,"/":34,"0":35,"1":36,"2":37,"3":38,"4":39,"5":40,"6":41,"7":42,"8":43,"9":44,"<":45,">":46,"_":47,"b":48,"e":49,"m":50,"p":51,"r":52,"s":53,"t":54,"v":55,"|":56,"~":57,"ء":58,"أ":59,"ؤ":60,"ئ":61,"ا":62,"ب":63,"ة":64,"ت":65,"ث":66,"ج":67,"ح":68,"خ":69,"د":70,"ذ":71,"ر":72,"ز":73,"س":74,"ش":75,"ص":76,"ض":77,"ط":78,"ظ":79,"ع":80,"غ":81,"ف":82,"ق":83,"ك":84,"ل":85,"م":86,"ن":87,"ه":88,"و":89,"ى":90,"ي":91,"ً":92,"ٌ":93,"ٍ":94,"َ":95,"ُ":96,"ِ":97,"ّ":98,"ْ":99}