ArthurZ HF Staff commited on
Commit
d913439
·
verified ·
1 Parent(s): 7427a19

Update tokenizer_config.json

Browse files

This should make it compatible with v5 + corrects it because it does not follow gpt2.
GPT2 does not have a normalizer, and has a different pre-tokenizer https://huggingface.co/openai-community/gpt2/raw/main/tokenizer.json

Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -491,7 +491,7 @@
491
  "eos_token": "[e~[",
492
  "extra_special_tokens": {},
493
  "model_max_length": 40960000,
494
- "tokenizer_class": "GPT2Tokenizer",
495
  "unk_token": "]!d~[",
496
  "tool_parser_type": "minimax_m2"
497
  }
 
491
  "eos_token": "[e~[",
492
  "extra_special_tokens": {},
493
  "model_max_length": 40960000,
494
+ "tokenizer_class": "TokenizersBackend",
495
  "unk_token": "]!d~[",
496
  "tool_parser_type": "minimax_m2"
497
  }