Update tokenizer_config.json

tokenizer_config.json

@@ -1,3 +1,67 @@
+Hugging Face's logo
+Hugging Face
+Models
+Datasets
+Spaces
+Posts
+Docs
+Enterprise
+Pricing
+
+
+
+
+Qwen
+/
+Qwen2-VL-2B-Instruct 
+
+like
+378
+
+Follow
+
+Qwen
+10.6k
+Image-Text-to-Text
+Transformers
+Safetensors
+English
+qwen2_vl
+multimodal
+conversational
+text-generation-inference
+Inference Endpoints
+
+arxiv:
+2409.12191
+
+arxiv:
+2308.12966
+
+License:
+apache-2.0
+Model card
+Files and versions
+Community
+23
+Qwen2-VL-2B-Instruct
+/
+tokenizer_config.json
+
+yangapku's picture
+yangapku
+Initial commit
+e28f5d3
+5 months ago
+raw
+
+Copy download link
+history
+blame
+contribute
+delete
+
+4.19 kB
 {
   "add_prefix_space": false,
   "added_tokens_decoder": {
@@ -114,31 +178,16 @@
       "special": true
     }
   },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
+  "additional_special_tokens": ["<|im_start|>", "<|im_end|>", "<|object_ref_start|>","<|object_ref_end|>","<|box_start|>","<|box_end|>","<|quad_start|>","<|quad_end|>","<|vision_start|>","<|vision_end|>","<|vision_pad|>","<|image_pad|>","<|video_pad|>"],
   "bos_token": null,
   "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
+  "padding_side": "left",
   "errors": "replace",
-  "model_max_length": 4096,
-  "pad_token": null,
-  "padding_side": "right",
-  "processor_class": "Qwen2VLProcessor",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null
-}
+}