saracandu commited on
Commit
07d0126
·
verified ·
1 Parent(s): f99d658

added tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +11 -0
tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name_or_path": "temporal_logic_tokenizer",
3
+ "special_tokens_map_file": "special_tokens_map.json",
4
+ "tokenizer_file": "tokenizer.json",
5
+ "do_lower_case": false,
6
+ "model_max_length": 500,
7
+ "bos_token": "/s",
8
+ "eos_token": "s",
9
+ "unk_token": "unk",
10
+ "pad_token": "pad"
11
+ }