Update tokenizer_script.py

tokenizer_script.py

@@ -82,13 +82,13 @@ class CharacterTokenizer(PreTrainedTokenizer):
         if max_length is not None:
             encoded_texts = [ids[:max_length] for ids in encoded_texts]
         if add_special_tokens:
-            bos_token_id = self.convert_tokens_to_ids(tokenizer.bos_token)
-            eos_token_id = self.convert_tokens_to_ids(tokenizer.eos_token)
+            bos_token_id = self.convert_tokens_to_ids(self.bos_token)
+            eos_token_id = self.convert_tokens_to_ids(self.eos_token)
             encoded_texts = [[bos_token_id] + ids + [eos_token_id] for ids in encoded_texts]
         # Handle padding
         if padding:
             # properly handle padding side
-            pad_id = self.vocab.get(tokenizer.pad_token, 0)
+            pad_id = self.vocab.get(self.pad_token, 0)
             max_len = max(len(ids) for ids in encoded_texts) if max_length is None else max_length
             if tokenizer.padding_side == "right":
               encoded_texts = [ids + [pad_id] * (max_len - len(ids)) for ids in encoded_texts]