dineth554 commited on
Commit
46ab720
·
verified ·
1 Parent(s): 6bf8769

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +17 -0
tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 16000,
3
+ "special_tokens": {
4
+ "<|pad|>": 0,
5
+ "<|eos|>": 1,
6
+ "<|unk|>": 2,
7
+ "<|system|>": 3,
8
+ "<|user|>": 4,
9
+ "<|assistant|>": 5,
10
+ "<|code|>": 6,
11
+ "<|comment|>": 7,
12
+ "<|indent|>": 8,
13
+ "<|newline|>": 9,
14
+ "<|tab|>": 10,
15
+ "<|space|>": 11
16
+ }
17
+ }