"""Compare SQLite document status vs FAISS chunk counts.""" from __future__ import annotations import sqlite3 import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from app.config import settings from app.vectorstore.factory import get_vectorstore def main() -> None: db_path = Path("dev.db") if not db_path.is_file(): print("No dev.db in project root") return conn = sqlite3.connect(db_path) status_counts = dict( conn.execute("SELECT status, COUNT(*) FROM documents GROUP BY status").fetchall() ) print("DB document status:", status_counts) print("FAISS_INDEX_PATH:", settings.faiss_index_path) print("VECTORSTORE_BACKEND:", settings.vectorstore_backend) vs = get_vectorstore() ntotal = getattr(vs, "total_vectors", lambda: 0)() print("FAISS ntotal:", ntotal, "index_loaded:", getattr(vs, "index_loaded", False)) complete_ids = [r[0] for r in conn.execute( "SELECT id FROM documents WHERE status='complete' LIMIT 300" ).fetchall()] missing = sum(1 for doc_id in complete_ids if vs.count_for_doc(doc_id) == 0) print(f"complete docs sampled: {len(complete_ids)}, missing from FAISS: {missing}") failed = conn.execute( "SELECT error_message FROM documents WHERE status='failed' " "AND error_message IS NOT NULL LIMIT 3" ).fetchall() if failed: print("sample errors:") for (msg,) in failed: print(" ", (msg or "")[:160]) conn.close() if __name__ == "__main__": main()