Upload tokenizer

special_tokens_map.json

@@ -1,5 +1,9 @@
 {
   "additional_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>",
     "▁<PRE>",
     "▁<MID>",
     "▁<SUF>",

tokenizer.json

@@ -170,6 +170,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ccb9c8b6b561889acea59191d66986d314e7cbd6a78abc6e49b139ca91c1e6
+size 500058

tokenizer_config.json

@@ -60,6 +60,10 @@
     }
   },
   "additional_special_tokens": [
+    "▁<PRE>",
+    "▁<MID>",
+    "▁<SUF>",
+    "▁<EOT>",
     "▁<PRE>",
     "▁<MID>",
     "▁<SUF>",
@@ -67,6 +71,7 @@
   ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
+  "device_map": "auto",
   "eos_token": "</s>",
   "eot_token": "▁<EOT>",
   "fill_token": "<FILL_ME>",