Tom Aarsen commited on
Commit ·
e8e8b51
1
Parent(s): 65e48be
Set repo_type="dataset" to specify that we're downloading the indices from a dataset repo
Browse files
app.py
CHANGED
|
@@ -14,8 +14,8 @@ title_text_int8_dataset = load_dataset("sentence-transformers/quantized-retrieva
|
|
| 14 |
# title_text_int8_dataset = load_from_disk("wikipedia-mxbai-embed-int8-index").select_columns(["url", "title", "text", "embedding"])
|
| 15 |
|
| 16 |
# Load the binary indices
|
| 17 |
-
binary_index_path = hf_hub_download(repo_id="sentence-transformers/quantized-retrieval-data", filename="wikipedia_ubinary_faiss_50m.index", local_dir=".")
|
| 18 |
-
binary_ivf_index_path = hf_hub_download(repo_id="sentence-transformers/quantized-retrieval-data", filename="wikipedia_ubinary_ivf_faiss_50m.index", local_dir=".")
|
| 19 |
|
| 20 |
binary_index: faiss.IndexBinaryFlat = faiss.read_index_binary(binary_index_path)
|
| 21 |
binary_ivf_index: faiss.IndexBinaryIVF = faiss.read_index_binary(binary_ivf_index_path)
|
|
|
|
| 14 |
# title_text_int8_dataset = load_from_disk("wikipedia-mxbai-embed-int8-index").select_columns(["url", "title", "text", "embedding"])
|
| 15 |
|
| 16 |
# Load the binary indices
|
| 17 |
+
binary_index_path = hf_hub_download(repo_id="sentence-transformers/quantized-retrieval-data", filename="wikipedia_ubinary_faiss_50m.index", local_dir=".", repo_type="dataset")
|
| 18 |
+
binary_ivf_index_path = hf_hub_download(repo_id="sentence-transformers/quantized-retrieval-data", filename="wikipedia_ubinary_ivf_faiss_50m.index", local_dir=".", repo_type="dataset")
|
| 19 |
|
| 20 |
binary_index: faiss.IndexBinaryFlat = faiss.read_index_binary(binary_index_path)
|
| 21 |
binary_ivf_index: faiss.IndexBinaryIVF = faiss.read_index_binary(binary_ivf_index_path)
|