Added AutoTokenizer support

README.md

@@ -26,12 +26,11 @@ Tokenizer requirements:
 
 #### Simple FillMaskPipeline
 ```python
-from transformers import AutoModelForSeq2SeqLM, pipeline
-from tokenization_bart_japanese import BartJapaneseTokenizer
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
 
 model_name = "Formzu/bart-base-japanese"
 model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
-tokenizer = BartJapaneseTokenizer.from_pretrained(model_name)
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
 
 masked_text = "天気が<mask>から散歩しましょう。"
 
@@ -46,15 +45,14 @@ print(out)
 ```
 #### Text Generation
 ```python
-from transformers import AutoModelForSeq2SeqLM
-from tokenization_bart_japanese import BartJapaneseTokenizer
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 import torch
 
 device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
 
 model_name = "Formzu/bart-base-japanese"
 model = AutoModelForSeq2SeqLM.from_pretrained(model_name).to(device)
-tokenizer = BartJapaneseTokenizer.from_pretrained(model_name)
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
 
 masked_text = "天気が<mask>から散歩しましょう。"
 

tokenizer_config.json

@@ -18,5 +18,6 @@
   "tgt_lang": null,
   "tokenizer_class": "BartJapaneseTokenizer",
   "tokenizer_file": null,
-  "unk_token": "<unk>"
+  "unk_token": "<unk>",
+  "auto_map": {"AutoTokenizer": ["tokenization_bart_japanese.BartJapaneseTokenizer", null]}
 }