Image-Text-to-Text
Transformers
Safetensors
qwen3_5_moe
Merge
evolutionary-merge
darwin
darwin-v5
model-mri
reasoning
advanced-reasoning
chain-of-thought
thinking
qwen3.5
qwen
Mixture of Experts
mixture-of-experts
claude-opus
distillation
multimodal
vision-language
gpqa
benchmark
open-source
apache-2.0
layer-wise-merge
moe-merge
dead-expert-revival
coding-agent
tool-calling
long-context
262k-context
conversational
Eval Results (legacy)
Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
|
@@ -304,9 +304,9 @@
|
|
| 304 |
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 305 |
"processor_class": "Qwen3VLProcessor",
|
| 306 |
"split_special_tokens": false,
|
| 307 |
-
"tokenizer_class": "
|
| 308 |
"unk_token": null,
|
| 309 |
"video_token": "<|video_pad|>",
|
| 310 |
"vision_bos_token": "<|vision_start|>",
|
| 311 |
"vision_eos_token": "<|vision_end|>"
|
| 312 |
-
}
|
|
|
|
| 304 |
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 305 |
"processor_class": "Qwen3VLProcessor",
|
| 306 |
"split_special_tokens": false,
|
| 307 |
+
"tokenizer_class": "Qwen2TokenizerFast",
|
| 308 |
"unk_token": null,
|
| 309 |
"video_token": "<|video_pad|>",
|
| 310 |
"vision_bos_token": "<|vision_start|>",
|
| 311 |
"vision_eos_token": "<|vision_end|>"
|
| 312 |
+
}
|