Upload tokenizer

added_tokens.json

@@ -22,6 +22,7 @@
   "<|fim_prefix|>": 128026,
   "<|fim_suffix|>": 128028,
   "<|im_end|>": 128003,
+  "<|im_mask|>": 128255,
   "<|im_start|>": 128002,
   "<|image_pad|>": 128024,
   "<|object_ref_end|>": 128016,
@@ -162,7 +163,6 @@
   "<|reserved_special_token_220|>": 128252,
   "<|reserved_special_token_221|>": 128253,
   "<|reserved_special_token_222|>": 128254,
-  "<|reserved_special_token_223|>": 128255,
   "<|reserved_special_token_22|>": 128054,
   "<|reserved_special_token_23|>": 128055,
   "<|reserved_special_token_24|>": 128056,

special_tokens_map.json

@@ -14,7 +14,7 @@
     "single_word": false
   },
   "mask_token": {
-    "content": "<|end_of_text|>",
+    "content": "<|im_mask|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf32d7719f1732eef0b7d9752ec708a65efe14c1d8901b9243fa0dc8af325ece
-size 17209390
+oid sha256:6994d9adc5f6c1cec4a446e3a3f98ffa7efeefd4a76c057accf09971fbf68420
+size 17209371

tokenizer_config.json

@@ -2043,7 +2043,7 @@
       "special": true
     },
     "128255": {
-      "content": "<|reserved_special_token_223|>",
+      "content": "<|im_mask|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2056,10 +2056,11 @@
   "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
-  "mask_token": "<|end_of_text|>",
+  "mask_token": "<|im_mask|>",
   "model_input_names": [
     "input_ids",
-    "attention_mask"
+    "attention_mask",
+    "position_ids"
   ],
   "model_max_length": 131072,
   "pad_token": "<|end_of_text|>",