robinfaro commited on
Commit
7b00130
·
verified ·
1 Parent(s): be43987

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -1
tokenizer_config.json CHANGED
@@ -16,5 +16,8 @@
16
  "extra_special_tokens": {},
17
  "model_max_length": 1024,
18
  "tokenizer_class": "GPT2Tokenizer",
19
- "unk_token": "<|endoftext|>"
 
 
 
20
  }
 
16
  "extra_special_tokens": {},
17
  "model_max_length": 1024,
18
  "tokenizer_class": "GPT2Tokenizer",
19
+ "unk_token": "<|endoftext|>",
20
+ "auto_map": {
21
+ "AutoTokenizer": ["transformers.GPT2TokenizerFast", null]
22
+ },
23
  }