Fixing tokenizer return type

tokenizer.py

@@ -26,14 +26,14 @@ class KmerTokenizer(PreTrainedTokenizer):
         self.unk_token = "[UNK]"
         # self.pad_token = "[PAD]"
 
-    def _tokenize(self, text, **kwargs):
+    def tokenize(self, text, **kwargs):
         splits = [text[i:i + self.k] for i in range(0, len(text) - self.k + 1, self.stride)]
         if kwargs.get('return_tensors') == 'pt':
             return torch.tensor(splits)
         return splits
 
     def _encode(self, text, **kwargs):
-        tokens = self._tokenize(text, **kwargs)
+        tokens = self.tokenize(text, **kwargs)
         token_ids = self.convert_tokens_to_ids(tokens)
         if kwargs.get('return_tensors') == 'pt':
             return torch.tensor(token_ids)