meutsabdahal commited on
Commit
bf1ccbb
·
verified ·
1 Parent(s): 8be09c8

Upload model, tokenizer, and model card

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +25 -0
tokenizer_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<s>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "cls_token": "<cls>",
6
+ "eos_token": "</s>",
7
+ "from_flax": true,
8
+ "is_local": false,
9
+ "local_files_only": false,
10
+ "mask_token": "<mask>",
11
+ "model_max_length": 512,
12
+ "pad_token": "<pad>",
13
+ "sep_token": "<sep>",
14
+ "special_tokens": [
15
+ "<s>",
16
+ "<pad>",
17
+ "</s>",
18
+ "<unk>",
19
+ "<cls>",
20
+ "<sep>",
21
+ "<mask>"
22
+ ],
23
+ "tokenizer_class": "TokenizersBackend",
24
+ "unk_token": "<unk>"
25
+ }