Hugging Face's logo

masatof
/
kappa-tokenizer

Model card Files
xet
 Community
kappa-tokenizer
File size: 2,804 Bytes
ceb81be
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
{
  "base_model": "Qwen/Qwen3-4B",
  "source_tokens_from": "Kwaipilot/KAT-V1-40B",
  "original_vocab_size": 151643,
  "final_vocab_size": 151643,
  "tokens_added": 6,
  "existing_tokens": {
    "</think>": 151668,
    "</tool_call>": 151658,
    "<think>": 151667,
    "<tool_call>": 151657,
    "<|box_end|>": 151649,
    "<|box_start|>": 151648,
    "<|endoftext|>": 151643,
    "<|file_sep|>": 151664,
    "<|fim_middle|>": 151660,
    "<|fim_pad|>": 151662,
    "<|fim_prefix|>": 151659,
    "<|fim_suffix|>": 151661,
    "<|im_end|>": 151645,
    "<|im_start|>": 151644,
    "<|image_pad|>": 151655,
    "<|object_ref_end|>": 151647,
    "<|object_ref_start|>": 151646,
    "<|quad_end|>": 151651,
    "<|quad_start|>": 151650,
    "<|repo_name|>": 151663,
    "<|video_pad|>": 151656,
    "<|vision_end|>": 151653,
    "<|vision_pad|>": 151654,
    "<|vision_start|>": 151652
  },
  "added_tokens": {
    "</answer>": 151669,
    "</judge>": 151670,
    "<answer>": 151671,
    "<judge>": 151672,
    "<think_off>": 151673,
    "<think_on>": 151674
  },
  "all_kat_tokens": {
    "</answer>": 151669,
    "</judge>": 151670,
    "</think>": 151668,
    "</tool_call>": 151658,
    "<answer>": 151671,
    "<judge>": 151672,
    "<think>": 151667,
    "<think_off>": 151673,
    "<think_on>": 151674,
    "<tool_call>": 151657,
    "<|box_end|>": 151649,
    "<|box_start|>": 151648,
    "<|endoftext|>": 151643,
    "<|file_sep|>": 151664,
    "<|fim_middle|>": 151660,
    "<|fim_pad|>": 151662,
    "<|fim_prefix|>": 151659,
    "<|fim_suffix|>": 151661,
    "<|im_end|>": 151645,
    "<|im_start|>": 151644,
    "<|image_pad|>": 151655,
    "<|object_ref_end|>": 151647,
    "<|object_ref_start|>": 151646,
    "<|quad_end|>": 151651,
    "<|quad_start|>": 151650,
    "<|repo_name|>": 151663,
    "<|video_pad|>": 151656,
    "<|vision_end|>": 151653,
    "<|vision_pad|>": 151654,
    "<|vision_start|>": 151652
  },
  "kat_token_categories": {
    "reasoning": [
      "<judge>",
      "</judge>",
      "<think>",
      "</think>",
      "<think_on>",
      "<think_off>"
    ],
    "output": [
      "<answer>",
      "</answer>"
    ],
    "tools": [
      "<tool_call>",
      "</tool_call>"
    ],
    "chat": [
      "<|im_start|>",
      "<|im_end|>"
    ],
    "multimodal": [
      "<|vision_start|>",
      "<|vision_end|>",
      "<|vision_pad|>",
      "<|image_pad|>",
      "<|video_pad|>"
    ],
    "code": [
      "<|fim_prefix|>",
      "<|fim_middle|>",
      "<|fim_suffix|>",
      "<|fim_pad|>",
      "<|repo_name|>",
      "<|file_sep|>"
    ],
    "structure": [
      "<|box_start|>",
      "<|box_end|>",
      "<|quad_start|>",
      "<|quad_end|>",
      "<|object_ref_start|>",
      "<|object_ref_end|>"
    ],
    "special": [
      "<|endoftext|>"
    ]
  }
}