Add tokenizer compatibility shim

sitecustomize.py

@@ -5,6 +5,13 @@ try:
 except Exception:  # pragma: no cover
     huggingface_hub = None
 
+try:
+    from transformers import PreTrainedTokenizer, PreTrainedTokenizerBase, PreTrainedTokenizerFast
+except Exception:  # pragma: no cover
+    PreTrainedTokenizer = None
+    PreTrainedTokenizerBase = None
+    PreTrainedTokenizerFast = None
+
 
 if huggingface_hub is not None and not hasattr(huggingface_hub, "HfFolder"):
     class HfFolder:
@@ -27,3 +34,12 @@ if huggingface_hub is not None and not hasattr(huggingface_hub, "HfFolder"):
                 cls.path_token.unlink()
 
     huggingface_hub.HfFolder = HfFolder
+
+
+def _compat_batch_encode_plus(self, *args, **kwargs):
+    return self(*args, **kwargs)
+
+
+for tokenizer_cls in (PreTrainedTokenizerBase, PreTrainedTokenizer, PreTrainedTokenizerFast):
+    if tokenizer_cls is not None and not hasattr(tokenizer_cls, "batch_encode_plus"):
+        tokenizer_cls.batch_encode_plus = _compat_batch_encode_plus