Update char_tokenizer.py

char_tokenizer.py

@@ -39,7 +39,20 @@ class CharTokenizer(PreTrainedTokenizer):
         do_lower_case=False,
         *args,
         **kwargs
-    ):  
+    ):
+        self.do_lower_case = do_lower_case
+        self.space_token = space_token
+        
+        if not vocab_file or not os.path.isfile(vocab_file):
+            self.vocab = OrderedDict()
+            special_tokens = [pad_token, unk_token, bos_token, eos_token, cls_token, sep_token, mask_token]
+            for i, token in enumerate(special_tokens):
+                self.vocab[token] = i
+        else:
+            self.vocab = load_vocab(vocab_file)
+        
+        self.ids_to_tokens = OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])
+
         super().__init__(
             pad_token=pad_token,
             unk_token=unk_token,
@@ -51,15 +64,6 @@ class CharTokenizer(PreTrainedTokenizer):
             do_lower_case=do_lower_case,
             **kwargs
         )
-        self.do_lower_case = do_lower_case
-        self.space_token = space_token
-
-        if not vocab_file or not os.path.isfile(vocab_file):
-            self.vocab = OrderedDict()
-            self.ids_to_tokens = OrderedDict()
-        else:
-            self.vocab = load_vocab(vocab_file)
-            self.ids_to_tokens = OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])
 
     def train(self, file_path):
         vocab = set()
@@ -74,9 +78,10 @@ class CharTokenizer(PreTrainedTokenizer):
         special_tokens = [self.pad_token, self.unk_token, self.bos_token, self.eos_token]
         vocab = special_tokens + vocab
 
+        self.vocab = OrderedDict()
         for i, ch in enumerate(vocab):
             self.vocab[ch] = i
-        self.ids_to_tokens = vocab
+        self.ids_to_tokens = OrderedDict([(i, ch) for i, ch in enumerate(vocab)])
 
     @property
     def vocab_size(self):
@@ -91,7 +96,7 @@ class CharTokenizer(PreTrainedTokenizer):
         return self.vocab.get(token, self.vocab[self.unk_token])
 
     def _convert_id_to_token(self, index):
-        return self.ids_to_tokens[index]
+        return self.ids_to_tokens.get(index, self.unk_token)
     
     def prepare_for_tokenization(
         self, text, is_split_into_words: bool = False, spaces=0, **kwargs