update snapshot download, fix error

app/app.py

@@ -8,7 +8,7 @@ import markdown
 ROOT_FOLDER = Path(__file__).resolve().parent.parent
 
 sys.path.append(str(ROOT_FOLDER))
-import sys
+import pickle
 import os
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
 from src.retrieval_helpers import enrich_search_results,enrich_bm25_search_results
@@ -62,22 +62,34 @@ VECTOR_STORE_DIR = ROOT / "data" / "processed"
 
 @st.cache_resource
 def load_vector_store_cached():
-    login(token=HF_TOKEN, add_to_git_credential=False)
+
+    hf_token = st.secrets["HF_TOKEN"]
+    login(token=hf_token, add_to_git_credential=False)
     VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True)
 
-    snapshot_path = snapshot_download(
-        repo_id="rishadaz/amazon_retriever-storage",
-        repo_type="dataset",
-        local_dir=str(VECTOR_STORE_DIR),
-        split='full',
-        token=HF_TOKEN,
-    )
+    # Skip download if already cached on disk
+    if not any(VECTOR_STORE_DIR.iterdir()):
+        snapshot_path = Path(snapshot_download(
+            repo_id="rishadaz/amazon_retriever-storage",
+            repo_type="dataset",
+            local_dir=str(VECTOR_STORE_DIR),
+            token=hf_token,
+        ))
+    else:
+        snapshot_path = VECTOR_STORE_DIR
+
+    mini_index_path = snapshot_path / "tokenisation" / "bm25_index_mini.pkl"
+    embeddings_dir  = snapshot_path / "embeddings"
+
+    if not mini_index_path.exists():
+        raise FileNotFoundError(f"BM25 index not found at {mini_index_path}")
+    if not embeddings_dir.exists():
+        raise FileNotFoundError(f"Embeddings dir not found at {embeddings_dir}")
 
-    mini_index_path = Path(snapshot_path) / "tokenisation" / "bm25_index_mini.pkl"
-    embeddings_dir  = Path(snapshot_path) / "embeddings"
+    with open(mini_index_path, "rb") as f:
+        bm25_retriever = pickle.load(f)
 
-    vector_store    = load_vector_store(embeddings_dir)
-    bm25_retriever  = load(mini_index_path)
+    vector_store = load_vector_store(embeddings_dir)
 
     return vector_store, bm25_retriever