Update tokenizer.py

tokenizer.py

@@ -4,7 +4,7 @@ import torch
 from transformers import PreTrainedTokenizer
 
 
-class MedTok(PreTrainedTokenizer):
+class MedicalcodeTokenizer(PreTrainedTokenizer):
     def __init__(self, vocab_file, code2tokens_file, embedding_file, **kwargs):
         print(f"Loading vocab from: {vocab_file}")
         print(f"Loading token map from: {code2tokens_file}")
@@ -104,26 +104,4 @@ class MedTok(PreTrainedTokenizer):
         vocab_file = os.path.join(pretrained_model_name_or_path, "vocab.json")
         code2tokens_file = os.path.join(pretrained_model_name_or_path, "code2tokens.json")
         embedding_file = os.path.join(pretrained_model_name_or_path, "code2embeddings.json")
-        return cls(vocab_file=vocab_file, code2tokens_file=code2tokens_file, embedding_file=embedding_file, **kwargs)
-
-
-
-'''vocab_path = "vocab.json"
-token_path = "code2tokens.json"
-embedding_path = "code2embeddings.json"
-
-tokenizer = MedTok(
-    vocab_file=vocab_path,
-    code2tokens_file=token_path,
-    embedding_file=embedding_path,
-    unk_token='[UNK]',
-    pad_token='[PAD]',
-)
-
-tokens = tokenizer.tokenize("E11.9")
-ids = tokenizer.encode("E11.9")
-embed = tokenizer.embed("E11.9")
-print("Tokens:", tokens)
-print("Token IDs:", ids)
-print("Decoded:", tokenizer.decode(tokens))
-print("Embedding:", embed)'''
+        return cls(vocab_file=vocab_file, code2tokens_file=code2tokens_file, embedding_file=embedding_file, **kwargs)