fnruha0921
/

temp_backup

Model card Files Files and versions

fnruha0921 commited on 12 days ago

Commit

1487d29

·

verified ·

1 Parent(s): e69d47e

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -25,7 +25,7 @@
   "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
   "processor_class": "Qwen3VLProcessor",
   "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null,
   "video_token": "<|video_pad|>",
   "vision_bos_token": "<|vision_start|>",

   "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
   "processor_class": "Qwen3VLProcessor",
   "split_special_tokens": false,
+  "tokenizer_class": "Qwen3_5TokenizerFast",
   "unk_token": null,
   "video_token": "<|video_pad|>",
   "vision_bos_token": "<|vision_start|>",