robinfaro commited on
Commit
2749609
·
verified ·
1 Parent(s): 90e0b0a

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -4
tokenizer_config.json CHANGED
@@ -16,8 +16,5 @@
16
  "extra_special_tokens": {},
17
  "model_max_length": 1024,
18
  "tokenizer_class": "GPT2Tokenizer",
19
- "unk_token": "<|endoftext|>",
20
- "auto_map": {
21
- "AutoTokenizer": ["GPT2TokenizerFast", null]
22
- }
23
  }
 
16
  "extra_special_tokens": {},
17
  "model_max_length": 1024,
18
  "tokenizer_class": "GPT2Tokenizer",
19
+ "unk_token": "<|endoftext|>"
 
 
 
20
  }