Mozzicstar commited on
Commit ·
a5eb084
1
Parent(s): 283c442
Add vectorstore debug logging
Browse files- scripts/query_qa.py +16 -2
scripts/query_qa.py
CHANGED
|
@@ -50,10 +50,24 @@ def embed_text_hf(texts, model_id="sentence-transformers/all-mpnet-base-v2", api
|
|
| 50 |
|
| 51 |
|
| 52 |
def load_vectorstore(persist_dir="vectorstore"):
|
|
|
|
|
|
|
| 53 |
persist_dir = Path(persist_dir)
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
docs = pickle.load(f)
|
|
|
|
|
|
|
| 57 |
return index, docs
|
| 58 |
|
| 59 |
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
def load_vectorstore(persist_dir="vectorstore"):
|
| 53 |
+
import logging
|
| 54 |
+
logger = logging.getLogger(__name__)
|
| 55 |
persist_dir = Path(persist_dir)
|
| 56 |
+
|
| 57 |
+
index_path = persist_dir / "faiss_index.bin"
|
| 58 |
+
meta_path = persist_dir / "metadata.pkl"
|
| 59 |
+
|
| 60 |
+
logger.info(f"Loading vectorstore from {persist_dir}")
|
| 61 |
+
logger.info(f"Index exists: {index_path.exists()}, size: {index_path.stat().st_size if index_path.exists() else 0}")
|
| 62 |
+
logger.info(f"Metadata exists: {meta_path.exists()}, size: {meta_path.stat().st_size if meta_path.exists() else 0}")
|
| 63 |
+
|
| 64 |
+
index = faiss.read_index(str(index_path))
|
| 65 |
+
logger.info(f"FAISS index loaded, ntotal: {index.ntotal}")
|
| 66 |
+
|
| 67 |
+
with open(meta_path, "rb") as f:
|
| 68 |
docs = pickle.load(f)
|
| 69 |
+
logger.info(f"Metadata loaded, {len(docs)} documents")
|
| 70 |
+
|
| 71 |
return index, docs
|
| 72 |
|
| 73 |
|