Update README.md

README.md

@@ -40,7 +40,6 @@ import itertools
 import torch
 
 from transformers import AutoModelForMaskedLM, AutoTokenizer
-from transformers.utils import cached_path,hf_bucket_url
 
 
 # get sparse vector from dense vectors with shape batch_size * seq_len * vocab_size
@@ -67,8 +66,8 @@ def transform_sparse_vector_to_dict(sparse_vector):
     
 # download the idf file from model hub. idf is used to give weights for query tokens
 def get_tokenizer_idf(tokenizer):
-    url = hf_bucket_url("opensearch-project/opensearch-neural-sparse-encoding-doc-v1","idf.json")
-    local_cached_path = cached_path(url)
+    from huggingface_hub import hf_hub_download
+    local_cached_path = hf_hub_download(repo_id="opensearch-project/opensearch-neural-sparse-encoding-doc-v1", filename="idf.json")
     with open(local_cached_path) as f:
         idf = json.load(f)
     idf_vector = [0]*tokenizer.vocab_size