Hugging Face's logo

mims-harvard
/
MedTok

Model card Files
xet
Community
Blair1213 commited on
Commit
22c914e
·
verified ·
1 Parent(s): 05ea635

Update tokenizer.py

Browse files
Files changed (1) hide show
  1. tokenizer.py +2 -2
tokenizer.py CHANGED
@@ -87,12 +87,12 @@ class MedicalcodeTokenizer(PreTrainedTokenizer):
87
  with open(os.path.join(save_directory, "code2embeddings.json"), "w") as f:
88
  json.dump(self.code2embed, f, indent=2)
89
  tokenizer_config = {
90
- "tokenizer_class": "MedTok",
91
  "vocab_file": "vocab.json",
92
  "code2tokens_file": "code2tokens.json",
93
  "code2embedding_file": "code2embeddings.json",
94
  "auto_map": {
95
- "AutoTokenizer": "tokenizer.MedTok"
96
  }
97
  }
98
  with open(os.path.join(save_directory, "tokenizer_config.json"), "w") as f:
 
87
  with open(os.path.join(save_directory, "code2embeddings.json"), "w") as f:
88
  json.dump(self.code2embed, f, indent=2)
89
  tokenizer_config = {
90
+ "tokenizer_class": "tokenizer.MedicalcodeTokenizer",
91
  "vocab_file": "vocab.json",
92
  "code2tokens_file": "code2tokens.json",
93
  "code2embedding_file": "code2embeddings.json",
94
  "auto_map": {
95
+ "AutoTokenizer": "MedTok"
96
  }
97
  }
98
  with open(os.path.join(save_directory, "tokenizer_config.json"), "w") as f: