scripts/build_vector_store.py

from pathlib import Path
from langchain_community.document_loaders import DirectoryLoader, TextLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_community.embeddings import HuggingFaceEmbeddings

DOCS_PATH = Path("data/documents")
STORE_PATH = "data/vectorstore"
EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"

def build():
    """Build vector store from policy documents"""
    print("Building vector store...")
    
    if not DOCS_PATH.exists():
        print(f"❌ Documents directory not found: {DOCS_PATH}")
        print("Please create policy documents in data/documents/")
        return
    
    # Load docs
    loader = DirectoryLoader(
        DOCS_PATH,
        glob="**/*.md",
        loader_cls=TextLoader,
        show_progress=True
    )
    docs = loader.load()
    
    if not docs:
        print(f"❌ No documents found in {DOCS_PATH}")
        return
    
    print(f"Loaded {len(docs)} documents")

    # Chunk — 512 tokens, 50 overlap preserves sentence boundaries
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=512,
        chunk_overlap=50,
        separators=["\n\n", "\n", ".", " "]
    )
    chunks = splitter.split_documents(docs)
    print(f"Created {len(chunks)} chunks")

    # Embed and persist
    embeddings = HuggingFaceEmbeddings(model_name=EMBED_MODEL)
    vectorstore = Chroma.from_documents(
        chunks,
        embeddings,
        persist_directory=STORE_PATH,
        collection_name="olist_policies"
    )
    print(f"✅ Vector store built at {STORE_PATH} ({len(chunks)} vectors)")

if __name__ == "__main__":
    build()