Adding chromadb build

app.py

@@ -9,17 +9,52 @@ from langchain.document_loaders import DataFrameLoader
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores import Chroma
 
+import kagglehub
+from kagglehub import KaggleDatasetAdapter
+import pandas as pd 
+
+# Download dataset
+# Load the latest version
+df = kagglehub.load_dataset(
+  KaggleDatasetAdapter.PANDAS,
+  "tobiasbueck/multilingual-customer-support-tickets",
+  file_path,
+)
+
+df = df[df['language'] == 'en']
+# Check for non-string items in body
+non_string_body = df[~df['body'].apply(lambda x: isinstance(x, str))].index
+non_string_answers = df[~df['answer'].apply(lambda x: isinstance(x, str))].index
+non_string_ids = non_string_body.union(non_string_answers)
+# Drop those rows
+df = df.drop(index=non_string_ids)
+df['q_and_a'] = 'Question: ' + df['body'] + ' Answer: ' + df['answer']
+df_train, df_holdout = train_test_split(df, test_size=0.2, random_state=42)
+df_val, df_test = train_test_split(df_holdout, test_size=0.5, random_state=42)
+
+persist_directory = './chroma_db'
+!rm -rf ./chroma_db  # remove old database files if any
+loader = DataFrameLoader(
+    df_train,
+    page_content_column="q_and_a")
+documents = loader.load()
+
+vectordb = Chroma.from_documents(
+    documents=documents,
+    embedding=embedding,
+    persist_directory=persist_directory
+)
 
 # Get OpenAI setup
 openai_api_key = os.getenv("openai_token")
 embedding = OpenAIEmbeddings(openai_api_key=openai_api_key)
 
-@st.cache_resource
-def get_vectordb():
-    embedding = OpenAIEmbeddings(openai_api_key=os.getenv("openai_token"))
-    return Chroma(persist_directory="./chroma_db", embedding_function=embedding)
+# @st.cache_resource
+# def get_vectordb():
+#     embedding = OpenAIEmbeddings(openai_api_key=os.getenv("openai_token"))
+#     return Chroma(persist_directory="./chroma_db", embedding_function=embedding)
 
-vectordb = get_vectordb()
+# vectordb = get_vectordb()
 
 # # Setup vector database
 # persist_directory = './chroma_db'