Upload base_tokenizer.py with huggingface_hub

base_tokenizer.py

@@ -0,0 +1,9 @@
+from transformers import PreTrainedTokenizerBase
+
+
+class BaseSequenceTokenizer:
+    def __init__(self, tokenizer: PreTrainedTokenizerBase):
+        self.tokenizer = tokenizer
+
+    def __call__(self, sequences, **kwargs):
+        raise NotImplementedError