Adapt tokenization_interns1.py to transformers>=5.0.0

tokenization_interns1.py

@@ -25,10 +25,12 @@ import regex as re
 import sentencepiece as spm
 
 from transformers.tokenization_utils_base import AddedToken, TextInput
-from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.utils import logging
-# from transformers.utils.import_utils import requires
-
+from packaging import version
+if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+    from transformers.tokenization_python import PreTrainedTokenizer
+else:
+    from transformers.tokenization_utils import PreTrainedTokenizer
 
 logger = logging.get_logger(__name__)
 
@@ -566,6 +568,7 @@ class InternS1Tokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             split_special_tokens=split_special_tokens,
+            special_tokens_pattern="bos_eos",
             **kwargs,
         )
 
@@ -715,9 +718,6 @@ class InternS1Tokenizer(PreTrainedTokenizer):
 
         text, kwargs = self.prepare_for_tokenization(text, **kwargs)
 
-        if kwargs:
-            logger.warning(f"Keyword arguments {kwargs} not recognized.")
-
         if hasattr(self, "do_lower_case") and self.do_lower_case:
             # convert non-special tokens to lowercase. Might be super slow as well?
             escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]