McClain commited on
Commit
23f2016
·
verified ·
1 Parent(s): c8bd3a2

Add tokenizer_config.json for AutoTokenizer compatibility

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +16 -0
tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoTokenizer": [
4
+ "tokenization_plasmid_lm.PlasmidLMTokenizer",
5
+ null
6
+ ]
7
+ },
8
+ "tokenizer_class": "PlasmidLMTokenizer",
9
+ "bos_token": "<BOS>",
10
+ "eos_token": "<EOS>",
11
+ "unk_token": "<UNK>",
12
+ "pad_token": "<PAD>",
13
+ "sep_token": "<SEP>",
14
+ "model_max_length": 16384,
15
+ "use_fast": false
16
+ }