Upload tokenizer

tokenizer_script.py

@@ -12,6 +12,7 @@ class CharacterTokenizer(PreTrainedTokenizer):
 
     def __init__(
         self,
+        vocab=None,
         unk_token="[UNK]",
         pad_token="[PAD]",
         bos_token="[BOS]",
@@ -19,19 +20,19 @@ class CharacterTokenizer(PreTrainedTokenizer):
         sep_token="[SEP]",
         **kwargs
     ):
-
-        vocab = {}
-        # Add special tokens
-        special_tokens = [
-            unk_token,
-            pad_token,
-            bos_token,
-            eos_token,
-            sep_token,
-          ]
-        for token in special_tokens:
-            if token not in vocab:
-                vocab[token] = len(vocab)
+        if vocab is None:
+          vocab = {}
+          # Add special tokens
+          special_tokens = [
+              unk_token,
+              pad_token,
+              bos_token,
+              eos_token,
+              sep_token,
+            ]
+          for token in special_tokens:
+              if token not in vocab:
+                  vocab[token] = len(vocab)
         self.vocab = vocab
         self.inv_vocab = {v: k for k, v in self.vocab.items()}
 
@@ -113,16 +114,13 @@ class CharacterTokenizer(PreTrainedTokenizer):
 
     @classmethod
     def from_json(cls, vocab_file, **kwargs):
-      print('vocab file is:',vocab_file)
-      with open(vocab_file, 'r', encoding='utf-8') as f:
-          vocab = json.load(f)
-      
-      return cls(vocab=vocab, **kwargs)
+        with open(vocab_file, 'r', encoding='utf-8') as f:
+            vocab = json.load(f)
+        return cls(vocab=vocab, **kwargs)
     
     @classmethod
     def from_vocab(cls, vocab, **kwargs):
-      print('vocab are:',vocab)
-      return cls(vocab=vocab, **kwargs)
+        return cls(vocab=vocab, **kwargs)
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):