Hugging Face's logo

mims-harvard
/
MedTok

Model card Files
xet
 Community
Blair1213 commited on
Commit
b272d5a
·
verified ·
1 Parent(s): 3896e14

Update tokenizer.py

Browse files
Files changed (1) hide show
  1. tokenizer.py +3 -3
tokenizer.py CHANGED
@@ -60,7 +60,7 @@ class MedicalcodeTokenizer(PreTrainedTokenizer):
60
 
61
  @classmethod
62
  def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
63
- vocab_file = os.path.join(pretrained_model_name_or_path, "vocab.json")
64
- code2tokens_file = os.path.join(pretrained_model_name_or_path, "code2tokens.json")
65
- embedding_file = os.path.join(pretrained_model_name_or_path, "code2embeddings.json")
66
  return cls(vocab_file=vocab_file, code2tokens_file=code2tokens_file, embedding_file=embedding_file, **kwargs)
 
60
 
61
  @classmethod
62
  def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
63
+ vocab_file = hf_hub_download(pretrained_model_name_or_path, "vocab.json")
64
+ code2tokens_file = hf_hub_download(pretrained_model_name_or_path, "code2tokens.json")
65
+ embedding_file = hf_hub_download(pretrained_model_name_or_path, "code2embeddings.json")
66
  return cls(vocab_file=vocab_file, code2tokens_file=code2tokens_file, embedding_file=embedding_file, **kwargs)