Upload folder using huggingface_hub

tokenization_df_arc.py

@@ -213,11 +213,6 @@ class DFArcTokenizer(PreTrainedTokenizerFast):
             text = self.phrase_helper.merge_phrases(text)
         return super().encode(text, *args, **kwargs)
 
-    def _encode_plus(self, text, *args, **kwargs):
-        if isinstance(text, str):
-            text = self.normalizer_helper.normalize(text)
-            text = self.morph_helper.segment_text(text)
-            text = self.phrase_helper.merge_phrases(text)
     def convert_tokens_to_string(self, tokens: List[str]) -> str:
         """
         Converts a sequence of tokens (string) in a single string.