Create tokenizer/tokenizer_config.json

split_files/tokenizer/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "im_start": "<|im_start|>",
+    "im_end": "<|im_end|>",
+    "object_ref_start": "<|object_ref_start|>",
+    "object_ref_end": "<|object_ref_end|>",
+    "box_start": "<|box_start|>",
+    "box_end": "<|box_end|>",
+    "quad_start": "<|quad_start|>",
+    "quad_end": "<|quad_end|>",
+    "vision_start": "<|vision_start|>",
+    "vision_end": "<|vision_end|>",
+    "vision_pad": "<|vision_pad|>",
+    "image_pad": "<|image_pad|>",
+    "video_pad": "<|video_pad|>"
+  },
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}