Upload tokenizer

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<image_soft_token>": 262144,
+  "<|im_end|>": 262145
+}

special_tokens_map.json

@@ -1,8 +1,14 @@
 {
   "additional_special_tokens": [
-    "<start_of_turn>",
-    "<end_of_turn>"
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
+  "boi_token": "<start_of_image>",
   "bos_token": {
     "content": "<bos>",
     "lstrip": false,
@@ -10,6 +16,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "eoi_token": "<end_of_image>",
   "eos_token": {
     "content": "<eos>",
     "lstrip": false,
@@ -17,6 +24,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "image_token": "<image_soft_token>",
   "pad_token": {
     "content": "<pad>",
     "lstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
-size 34362873
+oid sha256:3f8d1880ce5dece8db6d91f5fa876d4915eae3adf7044a0e6ed59e7053242463
+size 33384755

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
-size 4241003
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074