Update tokenizer.py

tokenizer.py

@@ -19,7 +19,9 @@ class MedicalcodeTokenizer(PreTrainedTokenizer):
         super().__init__(**kwargs)
 
     def _tokenize(self, text):
-        return self.code2tok.get(text, [])
+        if text in self.code2tok:
+            return self.code2tok[text]
+        return self._infer_and_register(text)
 
     def _convert_token_to_id(self, token):
         return self.vocab.get(token, self.vocab.get(self.unk_token, 0))
@@ -27,6 +29,12 @@ class MedicalcodeTokenizer(PreTrainedTokenizer):
     def _convert_id_to_token(self, idx):
         return self.ids_to_tokens.get(idx, self.unk_token)
 
+    def embed(self, text):
+        tokens = self._tokenize(text)         
+        if text in self.code2emb:
+            return self.code2emb[text]
+        return tokens
+
     def get_vocab(self):
         return self.vocab