kappa-tokenizer / enhancement_metadata.json
masatof's picture
Upload folder using huggingface_hub
ceb81be verified
{
"base_model": "Qwen/Qwen3-4B",
"source_tokens_from": "Kwaipilot/KAT-V1-40B",
"original_vocab_size": 151643,
"final_vocab_size": 151643,
"tokens_added": 6,
"existing_tokens": {
"</think>": 151668,
"</tool_call>": 151658,
"<think>": 151667,
"<tool_call>": 151657,
"<|box_end|>": 151649,
"<|box_start|>": 151648,
"<|endoftext|>": 151643,
"<|file_sep|>": 151664,
"<|fim_middle|>": 151660,
"<|fim_pad|>": 151662,
"<|fim_prefix|>": 151659,
"<|fim_suffix|>": 151661,
"<|im_end|>": 151645,
"<|im_start|>": 151644,
"<|image_pad|>": 151655,
"<|object_ref_end|>": 151647,
"<|object_ref_start|>": 151646,
"<|quad_end|>": 151651,
"<|quad_start|>": 151650,
"<|repo_name|>": 151663,
"<|video_pad|>": 151656,
"<|vision_end|>": 151653,
"<|vision_pad|>": 151654,
"<|vision_start|>": 151652
},
"added_tokens": {
"</answer>": 151669,
"</judge>": 151670,
"<answer>": 151671,
"<judge>": 151672,
"<think_off>": 151673,
"<think_on>": 151674
},
"all_kat_tokens": {
"</answer>": 151669,
"</judge>": 151670,
"</think>": 151668,
"</tool_call>": 151658,
"<answer>": 151671,
"<judge>": 151672,
"<think>": 151667,
"<think_off>": 151673,
"<think_on>": 151674,
"<tool_call>": 151657,
"<|box_end|>": 151649,
"<|box_start|>": 151648,
"<|endoftext|>": 151643,
"<|file_sep|>": 151664,
"<|fim_middle|>": 151660,
"<|fim_pad|>": 151662,
"<|fim_prefix|>": 151659,
"<|fim_suffix|>": 151661,
"<|im_end|>": 151645,
"<|im_start|>": 151644,
"<|image_pad|>": 151655,
"<|object_ref_end|>": 151647,
"<|object_ref_start|>": 151646,
"<|quad_end|>": 151651,
"<|quad_start|>": 151650,
"<|repo_name|>": 151663,
"<|video_pad|>": 151656,
"<|vision_end|>": 151653,
"<|vision_pad|>": 151654,
"<|vision_start|>": 151652
},
"kat_token_categories": {
"reasoning": [
"<judge>",
"</judge>",
"<think>",
"</think>",
"<think_on>",
"<think_off>"
],
"output": [
"<answer>",
"</answer>"
],
"tools": [
"<tool_call>",
"</tool_call>"
],
"chat": [
"<|im_start|>",
"<|im_end|>"
],
"multimodal": [
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"code": [
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|fim_pad|>",
"<|repo_name|>",
"<|file_sep|>"
],
"structure": [
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>"
],
"special": [
"<|endoftext|>"
]
}
}