almaghrabima commited on
Commit
6e193ce
·
verified ·
1 Parent(s): 08e8fc2

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +22 -0
special_tokens_map.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|bos|>",
3
+ "eos_token": "<|end_of_text|>",
4
+ "unk_token": "<|unk|>",
5
+ "pad_token": "<|pad|>",
6
+ "mask_token": "<|mask|>",
7
+ "additional_special_tokens": [
8
+ "<|assistant_end|>",
9
+ "<|assistant_start|>",
10
+ "<|bos|>",
11
+ "<|end_of_text|>",
12
+ "<|mask|>",
13
+ "<|output_end|>",
14
+ "<|output_start|>",
15
+ "<|pad|>",
16
+ "<|python_end|>",
17
+ "<|python_start|>",
18
+ "<|unk|>",
19
+ "<|user_end|>",
20
+ "<|user_start|>"
21
+ ]
22
+ }