Updated

requirements.txt

@@ -2,14 +2,12 @@ fastapi
 uvicorn[standard]
 pydantic
 spitch
-langchain
 langchain-community
-langdetect
-httpx
+langchain-core
+langchain-huggingface>=0.0.8
+langchain-chroma
 huggingface_hub
+httpx
+langdetect
 python-multipart
-langchain-huggingface>=0.0.8
 pandas
-langchain_chroma
-langchain_core
-sentence-transformers

smebuilder_vector.py

@@ -1,8 +1,8 @@
 import os
 import pandas as pd
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import Chroma
-from langchain.schema import Document
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+from langchain_core.documents import Document
 
 # ----------------- CONFIG -----------------
 DATASET_PATH = "sme_builder_dataset.csv"
@@ -11,13 +11,11 @@ COLLECTION_NAME = "landing_page_generation_examples"
 EMBEDDING_MODEL = os.getenv("HF_EMBEDDING_MODEL", "intfloat/e5-large-v2")
 HF_CACHE_DIR = os.getenv("HF_CACHE_DIR", "/app/huggingface_cache")
 
-# ensure directories exist
 os.makedirs(HF_CACHE_DIR, exist_ok=True)
 os.makedirs(DB_LOCATION, exist_ok=True)
 
 # ----------------- LOAD DATASET -----------------
 if not os.path.exists(DATASET_PATH):
-    # If dataset is optional, consider returning an empty retriever. For now raise so developer notices.
     raise FileNotFoundError(f"Dataset file not found: {DATASET_PATH}")
 
 df = pd.read_csv(DATASET_PATH)
@@ -26,8 +24,8 @@ df = pd.read_csv(DATASET_PATH)
 embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
 
 # ----------------- VECTOR STORE -----------------
-# if directory is empty then we should add documents; otherwise assume already persisted
-add_documents = not bool(os.listdir(DB_LOCATION))
+# Only add documents if DB is empty
+add_documents = not os.listdir(DB_LOCATION)
 
 vector_store = Chroma(
     collection_name=COLLECTION_NAME,
@@ -38,30 +36,19 @@ vector_store = Chroma(
 if add_documents:
     documents = []
     for i, row in df.iterrows():
-        # build a single text blob per row combining prompt + code + sector
-        content_pieces = [
-            str(row.get("prompt", "")).strip(),
-            str(row.get("html_code", "")).strip(),
-            str(row.get("css_code", "")).strip(),
-            str(row.get("js_code", "")).strip(),
-            str(row.get("sector", "")).strip(),
-        ]
-        content = " \n".join([p for p in content_pieces if p])
-        if not content:
-            continue
+        content = " ".join([
+            str(row.get("prompt", "")),
+            str(row.get("html_code", "")),
+            str(row.get("css_code", "")),
+            str(row.get("js_code", "")),
+            str(row.get("sector", ""))
+        ]).strip()
         documents.append(Document(page_content=content, metadata={"id": str(i)}))
-
+    
     if documents:
         vector_store.add_documents(documents=documents)
 
 # ----------------- RETRIEVER -----------------
-retriever = vector_store.as_retriever(search_kwargs={"k": 8})
-
-# Helpful info (no heavy introspection)
-try:
-    # avoid private attributes; just confirm connectivity
-    count = len(vector_store._collection.get()["ids"]) if hasattr(vector_store, "_collection") else "unknown"
-except Exception:
-    count = "unknown"
+retriever = vector_store.as_retriever(search_kwargs={"k": 20})
 
-print(f"SME vector store initialized. collection={COLLECTION_NAME}, documents={count}")
+print(f"SME vector store initialized. collection={COLLECTION_NAME}, documents={vector_store._collection.count()}")