Update tokenizer.py

tokenizer.py

@@ -5,7 +5,6 @@ from transformers import PreTrainedTokenizer
 
 
 class AlphabetTokenizer(PreTrainedTokenizer):
-    vocab_files_names = {"vocab_file": "vocab.json"}
     special_tokens_dict = {
         'unk_token': '[UNK]',
         'sep_token': '[SEP]',
@@ -15,7 +14,7 @@ class AlphabetTokenizer(PreTrainedTokenizer):
     }
 
     def __init__(self, **kwargs):
-        self.alphabet = [chr(i) for i in range(65, 65+19)] + [chr(i).lower() for i in range(65, 65+19)] + [str(i) for i in range(0, 10)] + ['.', '+', '-', ' ', 'W']
+        self.alphabet = [chr(i) for i in range(65, 65+19)] + [chr(i).lower() for i in range(65, 65+19)] + [str(i) for i in range(0, 10)] + ['.', '+', '-', ' ', 'W', '>', 'X']
         self.vocab = {char: i for i, char in enumerate(self.alphabet)}
         self.inv_vocab = {i: char for char, i in self.vocab.items()}
 
@@ -78,19 +77,3 @@ class AlphabetTokenizer(PreTrainedTokenizer):
             f.write(json.dumps(self.vocab, ensure_ascii=False))
 
         return (vocab_file,)
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs):
-        """Load the tokenizer from a pretrained model vocabulary."""
-        tokenizer = cls(*init_inputs, **kwargs)
-        vocab_file = os.path.join(pretrained_model_name_or_path, tokenizer.vocab_files_names["vocab_file"])
-        if os.path.isfile(vocab_file):
-            with open(vocab_file, "r", encoding="utf-8") as f:
-                vocab = json.load(f)
-            tokenizer.vocab = vocab
-            tokenizer.inv_vocab = {v: k for k, v in vocab.items()}
-
-        # override default _add_tokens of special tokens, and we added manually
-        tokenizer._added_tokens_decoder = {}
-        tokenizer.add_special_tokens(cls.special_tokens_dict)
-        return tokenizer