""" Build the Chroma vector database for Week 8 (Day 2 notebook — no Jupyter needed). Usage: cd week8 uv run python build_vectorstore.py Requires HF_TOKEN in .env (repo root or week8). Uses the lite dataset by default (~faster than full 400k products). """ import os import sys from pathlib import Path from huggingface_hub import login from sentence_transformers import SentenceTransformer from tqdm import tqdm import chromadb from agents.items import Item from env_utils import REPO_ROOT, load_project_env load_project_env() DB = "products_vectorstore" COLLECTION_NAME = "products" BATCH_SIZE = 1000 LITE_MODE = os.getenv("LITE_MODE", "true").lower() in ("true", "1", "yes") HF_USER = "ed-donner" def main(): if not os.getenv("HF_TOKEN"): print(f"ERROR: HF_TOKEN is missing.") print(f" Add HF_TOKEN=hf_... to: {REPO_ROOT / '.env'}") sys.exit(1) login(token=os.environ["HF_TOKEN"], add_to_git_credential=False) dataset = f"{HF_USER}/items_lite" if LITE_MODE else f"{HF_USER}/items_full" print(f"Loading dataset: {dataset} (LITE_MODE={LITE_MODE})") train, val, test = Item.from_hub(dataset) print(f"Loaded {len(train):,} training items, {len(val):,} val, {len(test):,} test") client = chromadb.PersistentClient(path=DB) existing = [c.name for c in client.list_collections()] if COLLECTION_NAME in existing: print(f"Collection '{COLLECTION_NAME}' already exists in {DB}/") count = client.get_collection(COLLECTION_NAME).count() print(f" → {count:,} vectors. Delete products_vectorstore/ to rebuild.") return print("Loading embedding model (first run downloads ~90MB)...") encoder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") collection = client.create_collection(COLLECTION_NAME) print(f"Encoding {len(train):,} products in batches of {BATCH_SIZE}...") for i in tqdm(range(0, len(train), BATCH_SIZE), desc="Building vectorstore"): batch = train[i : i + BATCH_SIZE] documents = [item.summary for item in batch] vectors = encoder.encode(documents).astype(float).tolist() metadatas = [{"category": item.category, "price": item.price} for item in batch] ids = [f"doc_{j}" for j in range(i, i + len(batch))] collection.add( ids=ids, documents=documents, embeddings=vectors, metadatas=metadatas, ) count = collection.count() print(f"\nDone! {count:,} vectors saved to {Path(DB).resolve()}") if __name__ == "__main__": main()