scripts/check_vectorstore.py

"""Compare SQLite document status vs FAISS chunk counts."""
from __future__ import annotations

import sqlite3
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from app.config import settings
from app.vectorstore.factory import get_vectorstore


def main() -> None:
    db_path = Path("dev.db")
    if not db_path.is_file():
        print("No dev.db in project root")
        return

    conn = sqlite3.connect(db_path)
    status_counts = dict(
        conn.execute("SELECT status, COUNT(*) FROM documents GROUP BY status").fetchall()
    )
    print("DB document status:", status_counts)
    print("FAISS_INDEX_PATH:", settings.faiss_index_path)
    print("VECTORSTORE_BACKEND:", settings.vectorstore_backend)

    vs = get_vectorstore()
    ntotal = getattr(vs, "total_vectors", lambda: 0)()
    print("FAISS ntotal:", ntotal, "index_loaded:", getattr(vs, "index_loaded", False))

    complete_ids = [r[0] for r in conn.execute(
        "SELECT id FROM documents WHERE status='complete' LIMIT 300"
    ).fetchall()]
    missing = sum(1 for doc_id in complete_ids if vs.count_for_doc(doc_id) == 0)
    print(f"complete docs sampled: {len(complete_ids)}, missing from FAISS: {missing}")

    failed = conn.execute(
        "SELECT error_message FROM documents WHERE status='failed' "
        "AND error_message IS NOT NULL LIMIT 3"
    ).fetchall()
    if failed:
        print("sample errors:")
        for (msg,) in failed:
            print(" ", (msg or "")[:160])

    conn.close()


if __name__ == "__main__":
    main()