ingest.py

"""
One-time script to build ChromaDB vector store from knowledge base.
Run once before starting the app: python ingest.py
Embeds all 37 knowledge base documents using multilingual MiniLM.
"""
from knowledge_base import KNOWLEDGE_BASE
from sentence_transformers import SentenceTransformer
import chromadb

print("Loading embedding model...")
embedder = SentenceTransformer(
    'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2'
)

print("Setting up ChromaDB...")
chroma_client = chromadb.PersistentClient(path="./chroma_db")

# Delete existing collection to rebuild fresh
try:
    chroma_client.delete_collection("banking_hindi")
    print("Deleted existing collection.")
except:
    pass

collection = chroma_client.get_or_create_collection("banking_hindi")

documents = []
metadatas = []
ids = []

for doc in KNOWLEDGE_BASE:
    full_text = f"{doc['title']}\n{doc['content']}"
    documents.append(full_text)
    metadatas.append({
        "id": doc["id"],
        "title": doc["title"],
        "category": doc["category"]
    })
    ids.append(doc["id"])

print(f"Embedding {len(documents)} documents...")
embeddings = embedder.encode(
    documents, 
    show_progress_bar=True,
    batch_size=8
).tolist()

collection.add(
    documents=documents,
    embeddings=embeddings,
    metadatas=metadatas,
    ids=ids
)
print(f"✅ Successfully ingested {len(documents)} documents into ChromaDB")
print(f"Collection count: {collection.count()}")