scripts/rebuild_qdrant.py

"""
Rebuild the Qdrant ``interview_questions`` collection from the local dataset.

This recreates, byte-for-byte, the vector database the original app used:
  * collection name : interview_questions
  * vector size     : 384  (all-MiniLM-L6-v2)
  * distance        : COSINE
  * payload         : {"job_role": <lower>, "question": ..., "answer": ...}

The original cluster was deleted after inactivity, but every question is
preserved in ``data/shuffled_questions.json`` (4233 Q&A pairs, 26 roles),
which is exactly what populated Qdrant in the first place.

Usage:
    export QDRANT_API_URL="https://<your-cluster>.qdrant.io:6333"
    export QDRANT_API_KEY="<your-key>"
    python scripts/rebuild_qdrant.py

It is safe to re-run: the collection is recreated from scratch each time.
"""

import json
import logging
import os
import sys

logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

COLLECTION_NAME = "interview_questions"
VECTOR_SIZE = 384
DATA_FILE = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
    "data",
    "shuffled_questions.json",
)


def main() -> int:
    url = os.getenv("QDRANT_API_URL")
    key = os.getenv("QDRANT_API_KEY")
    if not url or not key:
        logging.error(
            "Set QDRANT_API_URL and QDRANT_API_KEY environment variables first."
        )
        return 1

    # Qdrant cloud URLs need the :6333 REST port; add it if the user pasted
    # the bare hostname from the dashboard.
    if url.endswith("/"):
        url = url[:-1]
    if ".qdrant.io" in url and not url.rsplit(":", 1)[-1].isdigit():
        url = url + ":6333"

    from qdrant_client import QdrantClient
    from qdrant_client.http.models import Distance, PointStruct, VectorParams
    from sentence_transformers import SentenceTransformer

    logging.info("Loading dataset from %s", DATA_FILE)
    with open(DATA_FILE, "r", encoding="utf-8") as f:
        rows = json.load(f)
    logging.info("Loaded %d Q&A rows", len(rows))

    logging.info("Loading embedding model all-MiniLM-L6-v2 (first run downloads it)")
    model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")

    client = QdrantClient(url=url, api_key=key, check_compatibility=False, timeout=120)

    logging.info("Recreating collection '%s' (size=%d, COSINE)", COLLECTION_NAME, VECTOR_SIZE)
    client.recreate_collection(
        collection_name=COLLECTION_NAME,
        vectors_config=VectorParams(size=VECTOR_SIZE, distance=Distance.COSINE),
    )

    # The app filters questions by job_role, which requires a keyword payload
    # index — without it Qdrant returns HTTP 400 and the interview silently
    # falls back to generic default questions.
    client.create_payload_index(
        collection_name=COLLECTION_NAME,
        field_name="job_role",
        field_schema="keyword",
    )
    logging.info("Created keyword payload index on 'job_role'")

    # Build the points. We embed the QUESTION text, exactly like the original
    # notebook, and store role/question/answer in the payload.
    questions, payloads = [], []
    for item in rows:
        try:
            role = item["Job Role"].lower().strip()
            question = item["Questions"].strip()
            answer = item["Answers"].strip()
        except (KeyError, AttributeError):
            continue
        if not question:
            continue
        questions.append(question)
        payloads.append({"job_role": role, "question": question, "answer": answer})

    logging.info("Embedding %d questions...", len(questions))
    vectors = model.encode(questions, batch_size=128, show_progress_bar=True)

    batch_size = 64
    total = len(questions)
    for start in range(0, total, batch_size):
        end = min(start + batch_size, total)
        points = [
            PointStruct(id=i, vector=vectors[i].tolist(), payload=payloads[i])
            for i in range(start, end)
        ]
        for attempt in range(1, 4):
            try:
                client.upsert(collection_name=COLLECTION_NAME, points=points, wait=True)
                break
            except Exception as exc:
                logging.warning("Batch %d-%d attempt %d failed: %s", start, end, attempt, exc)
                if attempt == 3:
                    raise
        logging.info("Uploaded %d/%d", end, total)

    info = client.get_collection(COLLECTION_NAME)
    logging.info(
        "Done. Collection '%s' now has %s points (distance=%s).",
        COLLECTION_NAME,
        info.points_count,
        info.config.params.vectors.distance,
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())