Release v1.1: PMI Phrase Merging & Smart Morphology

tokenization_df_arc.py

@@ -179,6 +179,12 @@ class DFArcTokenizer(PreTrainedTokenizerFast):
     morphological segmentation, and phrase merging before tokenization.
     """
     
+    vocab_files_names = {
+        "vocab_file": "tokenizer.json",
+        "tokenizer_file": "tokenizer.json",
+        "phrases_file": "phrase_vocab.json"
+    }
+
     def __init__(
         self,
         vocab_file: Optional[str] = None,