Spaces:
Runtime error
Runtime error
File size: 1,577 Bytes
c893230 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | """Compare SQLite document status vs FAISS chunk counts."""
from __future__ import annotations
import sqlite3
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.config import settings
from app.vectorstore.factory import get_vectorstore
def main() -> None:
db_path = Path("dev.db")
if not db_path.is_file():
print("No dev.db in project root")
return
conn = sqlite3.connect(db_path)
status_counts = dict(
conn.execute("SELECT status, COUNT(*) FROM documents GROUP BY status").fetchall()
)
print("DB document status:", status_counts)
print("FAISS_INDEX_PATH:", settings.faiss_index_path)
print("VECTORSTORE_BACKEND:", settings.vectorstore_backend)
vs = get_vectorstore()
ntotal = getattr(vs, "total_vectors", lambda: 0)()
print("FAISS ntotal:", ntotal, "index_loaded:", getattr(vs, "index_loaded", False))
complete_ids = [r[0] for r in conn.execute(
"SELECT id FROM documents WHERE status='complete' LIMIT 300"
).fetchall()]
missing = sum(1 for doc_id in complete_ids if vs.count_for_doc(doc_id) == 0)
print(f"complete docs sampled: {len(complete_ids)}, missing from FAISS: {missing}")
failed = conn.execute(
"SELECT error_message FROM documents WHERE status='failed' "
"AND error_message IS NOT NULL LIMIT 3"
).fetchall()
if failed:
print("sample errors:")
for (msg,) in failed:
print(" ", (msg or "")[:160])
conn.close()
if __name__ == "__main__":
main()
|