Upload tokenizer

tokenizer_script.py

@@ -113,14 +113,16 @@ class CharacterTokenizer(PreTrainedTokenizer):
 
     @classmethod
     def from_json(cls, vocab_file, **kwargs):
-        with open(vocab_file, 'r', encoding='utf-8') as f:
-            vocab = json.load(f)
-        
-        return cls(vocab=vocab, **kwargs)
+      print('vocab file is:',vocab_file)
+      with open(vocab_file, 'r', encoding='utf-8') as f:
+          vocab = json.load(f)
+      
+      return cls(vocab=vocab, **kwargs)
     
     @classmethod
     def from_vocab(cls, vocab, **kwargs):
-        return cls(vocab=vocab, **kwargs)
+      print('vocab are:',vocab)
+      return cls(vocab=vocab, **kwargs)
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):