redSBA_ai / tokenizer_config.json
redSBA's picture
Upload folder using huggingface_hub
e1523d7 verified
Raw History Blame Contribute Delete
3.52 kB
{"add_prefix_space": false, "backend": "tokenizers", "bos_token": null, "clean_up_tokenization_spaces": false, "eos_token": "<|im_end|>", "errors": "replace", "extra_special_tokens": {}, "is_local": false, "local_files_only": false, "model_max_length": 32768, "pad_token": "<|PAD_TOKEN|>", "padding_side": "left", "split_special_tokens": false, "tokenizer_class": "Qwen2Tokenizer", "unk_token": null, "added_tokens_decoder": {"151643": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151644": {"content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151645": {"content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151646": {"content": "<|object_ref_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151647": {"content": "<|object_ref_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151648": {"content": "<|box_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151649": {"content": "<|box_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151650": {"content": "<|quad_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151651": {"content": "<|quad_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151652": {"content": "<|vision_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151653": {"content": "<|vision_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151654": {"content": "<|vision_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151655": {"content": "<|image_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151656": {"content": "<|video_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "151657": {"content": "<tool_call>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151658": {"content": "</tool_call>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151659": {"content": "<|fim_prefix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151660": {"content": "<|fim_middle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151661": {"content": "<|fim_suffix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151662": {"content": "<|fim_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151663": {"content": "<|repo_name|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151664": {"content": "<|file_sep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "151665": {"content": "<|PAD_TOKEN|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}}}