Fix transformers v5 auto_map + HF init

tokenizer/hf_tokenizer.py

@@ -16,6 +16,8 @@ class MGPT2Tokenizer(PreTrainedTokenizer):
     """
 
     model_input_names = ["input_ids", "attention_mask"]
+    # Let `PreTrainedTokenizer.from_pretrained()` know which file it should pass to `__init__`.
+    vocab_files_names = {"model_file": "tokenizer.model"}
 
     def __init__(self, model_file: str, **kwargs: Any):
         if not model_file.endswith(".model"):

tokenizer_config.json

@@ -17,6 +17,9 @@
   "tokenizer_class": "MGPT2Tokenizer",
   "unk_token": null,
   "auto_map": {
-    "AutoTokenizer": "tokenization_mgpt2.MGPT2Tokenizer"
+    "AutoTokenizer": [
+      "tokenization_mgpt2.MGPT2Tokenizer",
+      null
+    ]
   }
 }