Upload tokenizer

merges.txt

@@ -0,0 +1 @@
+#version: 0.2

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer.json

@@ -0,0 +1,283 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<|vsep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<|bsep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "|psep|",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "<|meter_0|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": "<|meter_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 7,
+      "content": "<|meter_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 8,
+      "content": "<|meter_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 9,
+      "content": "<|meter_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 10,
+      "content": "<|meter_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 11,
+      "content": "<|meter_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 12,
+      "content": "<|meter_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 13,
+      "content": "<|meter_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 14,
+      "content": "<|meter_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 15,
+      "content": "<|meter_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 16,
+      "content": "<|meter_11|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 17,
+      "content": "<|meter_12|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 18,
+      "content": "<|meter_13|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "UnicodeScripts"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "<|endoftext|>": 0,
+      "<|vsep|>": 1,
+      "<|bsep|>": 2,
+      "<|pad|>": 3,
+      "|psep|": 4,
+      "<|meter_0|>": 5,
+      "<|meter_1|>": 6,
+      "<|meter_2|>": 7,
+      "<|meter_3|>": 8,
+      "<|meter_4|>": 9,
+      "<|meter_5|>": 10,
+      "<|meter_6|>": 11,
+      "<|meter_7|>": 12,
+      "<|meter_8|>": 13,
+      "<|meter_9|>": 14,
+      "<|meter_10|>": 15,
+      "<|meter_11|>": 16,
+      "<|meter_12|>": 17,
+      "<|meter_13|>": 18,
+      " ": 19,
+      "0": 20,
+      "1": 21,
+      "2": 22,
+      "3": 23,
+      "4": 24,
+      "5": 25,
+      "6": 26,
+      "7": 27,
+      "8": 28,
+      "9": 29,
+      ":": 30,
+      "<": 31,
+      ">": 32,
+      "_": 33,
+      "b": 34,
+      "e": 35,
+      "m": 36,
+      "p": 37,
+      "r": 38,
+      "s": 39,
+      "t": 40,
+      "v": 41,
+      "|": 42,
+      "ء": 43,
+      "آ": 44,
+      "أ": 45,
+      "ؤ": 46,
+      "إ": 47,
+      "ئ": 48,
+      "ا": 49,
+      "ب": 50,
+      "ة": 51,
+      "ت": 52,
+      "ث": 53,
+      "ج": 54,
+      "ح": 55,
+      "خ": 56,
+      "د": 57,
+      "ذ": 58,
+      "ر": 59,
+      "ز": 60,
+      "س": 61,
+      "ش": 62,
+      "ص": 63,
+      "ض": 64,
+      "ط": 65,
+      "ظ": 66,
+      "ع": 67,
+      "غ": 68,
+      "ف": 69,
+      "ق": 70,
+      "ك": 71,
+      "ل": 72,
+      "م": 73,
+      "ن": 74,
+      "ه": 75,
+      "و": 76,
+      "ى": 77,
+      "ي": 78,
+      "ً": 79,
+      "ٌ": 80,
+      "ٍ": 81,
+      "َ": 82,
+      "ُ": 83,
+      "ِ": 84,
+      "ّ": 85,
+      "ْ": 86
+    },
+    "merges": []
+  }
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

vocab.json

@@ -0,0 +1 @@
+{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"|psep|":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18," ":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,"<":31,">":32,"_":33,"b":34,"e":35,"m":36,"p":37,"r":38,"s":39,"t":40,"v":41,"|":42,"ء":43,"آ":44,"أ":45,"ؤ":46,"إ":47,"ئ":48,"ا":49,"ب":50,"ة":51,"ت":52,"ث":53,"ج":54,"ح":55,"خ":56,"د":57,"ذ":58,"ر":59,"ز":60,"س":61,"ش":62,"ص":63,"ض":64,"ط":65,"ظ":66,"ع":67,"غ":68,"ف":69,"ق":70,"ك":71,"ل":72,"م":73,"ن":74,"ه":75,"و":76,"ى":77,"ي":78,"ً":79,"ٌ":80,"ٍ":81,"َ":82,"ُ":83,"ِ":84,"ّ":85,"ْ":86}