Update tokenizer_script.py

tokenizer_script.py

@@ -76,7 +76,7 @@ class CharacterTokenizer(PreTrainedTokenizer):
 
         return (vocab_file,)
 
-    def batch_encode(self, texts, add_special_tokens=False, padding=False, max_length=None):
+    def batch_encode(self, texts, add_special_tokens=False, padding=False, truncation=True, max_length=None):
         encoded_texts = [self.encode(text, add_special_tokens) for text in texts]
         # Handle max_length (truncation)
         if max_length is not None:
@@ -123,7 +123,6 @@ class CharacterTokenizer(PreTrainedTokenizer):
     def convert_tokens_to_string(self, tokens):
         return "".join(tokens)
     
-
     @classmethod
     def from_json(cls, vocab_file, **kwargs):
         with open(vocab_file, 'r', encoding='utf-8') as f: