Update tokenizer.py

tokenizer.py

@@ -53,12 +53,11 @@ class DNAKmerTokenizer(PreTrainedTokenizer):
             "|".join(re.escape(token) for token in self.special_tokens)
         )
         self.dna_pattern = re.compile(f"[A-Z]{{{self.k}}}|[A-Z]+")
-        self.bos_token = "<s>"
-        self.eos_token = "</s>"
-        self.mask_token = "<mask>"
-        self.bos_token_id = self._convert_token_to_id(self.bos_token)
-        self.eos_token_id = self._convert_token_to_id(self.eos_token)
-        self.mask_token_id = self._convert_token_to_id(self.mask_token)
+        kwargs.setdefault("bos_token", "<s>")
+        kwargs.setdefault("eos_token", "</s>")
+        kwargs.setdefault("mask_token", "<mask>")
+        kwargs.setdefault("unk_token", "<oov>")
+        kwargs.setdefault("pad_token", "<pad>")
         super().__init__(**kwargs)
 
     @property