strata-engine commited on
Commit
9d8bc00
·
verified ·
1 Parent(s): 005eebe

Add special_tokens_map.json for Inkling-2-Mini tokenizer/template support

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +21 -0
special_tokens_map.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|message_user|>",
5
+ "<|message_model|>",
6
+ "<|message_system|>",
7
+ "<|message_tool|>",
8
+ "<|content_text|>",
9
+ "<|content_image|>",
10
+ "<|content_model_end_sampling|>",
11
+ "<|content_thinking|>",
12
+ "<|end_message|>",
13
+ "<|content_tool_error|>",
14
+ "<|content_xml|>",
15
+ "<|begin_of_text|>",
16
+ "<|content_invoke_tool_json|>",
17
+ "<|content_invoke_tool_text|>"
18
+ ],
19
+ "eos_token": "<|content_model_end_sampling|>",
20
+ "pad_token": "<|endoftext|>"
21
+ }