sd

tokenizeConfig.py

@@ -90,7 +90,7 @@ class OBITokenizer(PreTrainedTokenizer):
     def _tokenize(self, text):
         """Returns a tokenized string."""
         encoding = self.tokenizer.encode(text)
-        return encoding.tokens
+        return encoding.ids
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
@@ -102,7 +102,7 @@ class OBITokenizer(PreTrainedTokenizer):
 
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) into a single string."""
-        return self.tokenizer.decode(tokens).strip()
+        return self.tokenizer.decode(tokens)
 
     def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
         """