UPDATE: trainChatbot

app.py

@@ -271,7 +271,7 @@ async def loadPDF(vectorstore: str, pdf: UploadFile = File(...)):
         .insert({"username": username, 
                  "chatbotName": chatbotName, 
                  "dataSourceName": fileName, 
-                 "sourceEndpoint": "\loadPDF", 
+                 "sourceEndpoint": "/loadPDF", 
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()
     )
@@ -299,7 +299,7 @@ async def loadImagePDF(vectorstore: str, pdf: UploadFile = File(...)):
         .insert({"username": username, 
                  "chatbotName": chatbotName, 
                  "dataSourceName": fileName, 
-                 "sourceEndpoint": "\loadImagePDF", 
+                 "sourceEndpoint": "/loadImagePDF", 
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()
     )
@@ -330,7 +330,7 @@ async def loadText(addTextConfig: AddText):
         .insert({"username": username, 
                  "chatbotName": chatbotName, 
                  "dataSourceName": fileName, 
-                 "sourceEndpoint": "\loadText", 
+                 "sourceEndpoint": "/loadText", 
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()
     )
@@ -339,28 +339,6 @@ async def loadText(addTextConfig: AddText):
     }
 
 
-
-@app.post("/addText")
-async def addText(addTextConfig: AddText):
-    vectorstore, text, source = addTextConfig.vectorstore, addTextConfig.text, addTextConfig.source
-    text = base64.b64decode(text.encode("utf-8")).decode("utf-8")
-    username, chatbotname = vectorstore.split("$")[1], vectorstore.split("$")[2]
-    df = pd.DataFrame(supabase.table("ConversAI_ChatbotInfo").select("*").execute().data)
-    currentCount = df[(df["user_id"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
-    newCount = currentCount + len(text)
-    limit = supabase.table("ConversAI_UserConfig").select("tokenLimit").eq("user_id", username).execute().data[0][
-        "tokenLimit"]
-    if newCount < int(limit):
-        supabase.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("user_id", username).eq(
-            "chatbotname", chatbotname).execute()
-        output = addDocuments(text=text, source=source, vectorstore=vectorstore)
-        return output
-    else:
-        return {
-            "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
-        }
-
-
 class AddQAPair(BaseModel):
     vectorstore: str
     question: str
@@ -410,7 +388,7 @@ async def loadWebURLs(loadWebsite: LoadWebsite):
         .insert({"username": username, 
                  "chatbotName": chatbotName, 
                  "dataSourceName": fileName, 
-                 "sourceEndpoint": "\loadWebURLs", 
+                 "sourceEndpoint": "/loadWebURLs", 
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()
     )
@@ -467,8 +445,8 @@ class YtTranscript(BaseModel):
     urls: list[str]
 
 
-@app.post("/getYoutubeTranscript")
-async def getYoutubeTranscript(ytTranscript: YtTranscript):
+@app.post("/loadYoutubeTranscript")
+async def loadYoutubeTranscript(ytTranscript: YtTranscript):
     vectorstore, urls = ytTranscript.vectorstore, ytTranscript.urls
     username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
     text = getTranscript(urls=urls)
@@ -484,7 +462,7 @@ async def getYoutubeTranscript(ytTranscript: YtTranscript):
         .insert({"username": username, 
                  "chatbotName": chatbotName, 
                  "dataSourceName": fileName, 
-                 "sourceEndpoint": "\getYoutubeTranscript", 
+                 "sourceEndpoint": "/getYoutubeTranscript", 
                  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
         .execute()
     )
@@ -523,7 +501,42 @@ async def chatHistory(vectorstore: str):
     return response
 
 
-# @app.post("/trainChatbot")
-# async def chatHistory(vectorstore: str):
-#     username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
-#     return response
+@app.post("/listChatbotSources")
+async def listChatbotSources(vectorstore: str):
+    username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
+    result = supabase.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq("chatbotName", chatbotName).execute().data
+    return result
+
+
+
+@app.post("/trainChatbot")
+async def trainChatbot(vectorstore: str):
+    texts = []
+    sources = []
+    fileTypes = [supabase.table("ConversAI_ChatbotDataSources").select("sourceEndpoint").eq("sourceContentURL", x).execute().data[0]["sourceEndpoint"] for x in sources]
+    for source, fileType in zip(sources, fileTypes):
+        if ((fileType == "/loadPDF") | (fileType == "/loadImagePDF")):
+            r = requests.get(source)
+            file = eval(r.content.decode("utf-8"))
+            content = file["output"]
+            fileSource = file["source"]
+            texts.append(".".join([base64.b64decode(content[key].encode("utf-8")).decode("utf-8") for key in content.keys()]).replace("\n", " "))
+            sources.append(fileSource)
+        elif fileType == "/loadText":
+            r = requests.get(source)
+            file = eval(r.content.decode("utf-8"))
+            content = file["output"]
+            fileSource = file["source"]
+            texts.append(content.replace("\n", " "))
+            sources.append(fileSource)
+        elif ((fileType == "/loadWebURLs") | (fileType == "/loadYoutubeTranscript")):
+            r = requests.get(source)
+            file = eval(r.content.decode("utf-8"))
+            content = file["output"]
+            fileSource = file["source"]
+            texts.append(".".join([base64.b64decode(content[key].encode("utf-8")).decode("utf-8") for key in content.keys()]).replace("\n", " "))
+            sources.append(fileSource)
+        else:
+            pass
+    texts = [(text, source) for text, source in zip(texts, sources)]
+    return addDocuments(texts = texts, vectorstore = vectorstore)        

functions.py

@@ -113,6 +113,7 @@ def createTable(tablename: str):
         prefer_grpc=True,
         api_key=os.environ["QDRANT_API_KEY"],
         collection_name=tablename,
+        force_recreate=True,
         retrieval_mode=RetrievalMode.HYBRID
     )
     return {
@@ -120,7 +121,7 @@ def createTable(tablename: str):
     }
 
 
-def addDocuments(text: str, source: str, vectorstore: str):
+def addDocuments(texts: list[tuple[str]], vectorstore: str):
     global vectorEmbeddings
     global sparseEmbeddings
     splitter = RecursiveCharacterTextSplitter(
@@ -128,20 +129,22 @@ def addDocuments(text: str, source: str, vectorstore: str):
         chunk_overlap=250,
         add_start_index=True
     )
-    text = text.replace("\n", " ")
-    text = text.translate(str.maketrans('', '', string.punctuation.replace(".", "")))
-    texts = [Document(page_content=text, metadata={"source": source})]
-    texts = splitter.split_documents(texts)
-    ids = [str(uuid4()) for _ in range(len(texts))]
-    vectorstore = QdrantVectorStore.from_existing_collection(
+    sources = [textTuple[1] for textTuple in texts]
+    texts = [textTuple[0].replace("\n", " ") for textTuple in texts]
+    texts = [text.translate(str.maketrans('', '', string.punctuation.replace(".", ""))) for text in texts]
+    texts = [Document(page_content=text, metadata={"source": source}) for text, source in zip(texts, sources)]
+    documents = splitter.split_documents(texts)
+    vectorstore = QdrantVectorStore.from_documents(
+        documents=documents,
         embedding=vectorEmbeddings,
         sparse_embedding=sparseEmbeddings,
-        collection_name=vectorstore,
         url=os.environ["QDRANT_URL"],
+        prefer_grpc=True,
         api_key=os.environ["QDRANT_API_KEY"],
+        collection_name=vectorstore,
+        force_recreate=True,
         retrieval_mode=RetrievalMode.HYBRID
     )
-    vectorstore.add_documents(documents=texts, ids=ids)
     return {
         "output": "SUCCESS"
     }