scripts/ingest_topics.py

#!/usr/bin/env python
"""One-time ingest of the topic vocabulary into the vantage_topics Qdrant collection.

Run once (or whenever seeds/topics.csv changes):
    python scripts/ingest_topics.py

Each point stores:
    vector  — 384-dim MiniLM dense embedding of the keyword string
    payload — {keyword: str, source: str}

The TopicIndex adapter then does a single search against this collection at runtime
instead of re-embedding 708 strings on every API cold start.
"""
from __future__ import annotations

import csv
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent / "packages" / "vantage_core"))

from qdrant_client import QdrantClient
from qdrant_client import models as qm

from vantage_core.adapters.embedder_st import STEmbedder
from vantage_core.config import get_settings
from vantage_core.constants import EMBEDDING_DIM, TOPICS_COLLECTION_NAME
from vantage_core.log import get_logger

log = get_logger(__name__)

_SEEDS = Path(__file__).parent.parent / "packages" / "vantage_core" / "seeds" / "topics.csv"


def main() -> None:
    s      = get_settings()
    client = QdrantClient(url=s.qdrant_url, api_key=s.qdrant_api_key, timeout=60)

    # Read topics
    with _SEEDS.open() as fh:
        rows = list(csv.DictReader(fh))
    keywords = [r["keyword"] for r in rows]
    sources  = [r.get("source", "") for r in rows]
    log.info("ingest_topics  topics=%d  file=%s", len(keywords), _SEEDS)

    # Create collection (dense-only — BM25 overkill for short keyword strings)
    if not client.collection_exists(TOPICS_COLLECTION_NAME):
        client.create_collection(
            collection_name = TOPICS_COLLECTION_NAME,
            vectors_config  = qm.VectorParams(size=EMBEDDING_DIM, distance=qm.Distance.COSINE),
        )
        log.info("ingest_topics  created collection %r", TOPICS_COLLECTION_NAME)
    else:
        log.info("ingest_topics  collection %r already exists — upserting", TOPICS_COLLECTION_NAME)

    # Embed all keywords in one batch
    embedder = STEmbedder()
    log.info("ingest_topics  embedding %d keywords ...", len(keywords))
    vectors = embedder.embed(keywords)

    # Upsert — point id = row index (stable, idempotent)
    points = [
        qm.PointStruct(
            id      = i,
            vector  = vec,
            payload = {"keyword": kw, "source": src},
        )
        for i, (kw, src, vec) in enumerate(zip(keywords, sources, vectors))
    ]
    client.upsert(collection_name=TOPICS_COLLECTION_NAME, points=points)
    count = client.count(TOPICS_COLLECTION_NAME).count
    log.info("ingest_topics  done  collection=%r  points=%d", TOPICS_COLLECTION_NAME, count)
    print(f"✓ {count} topics ingested into '{TOPICS_COLLECTION_NAME}'")


if __name__ == "__main__":
    main()