UPDATE: urls

app.py

@@ -33,30 +33,20 @@ async def newChatbot(chatbotName: str, username: str):
     return createTable(tablename = chatbotName)
 
 
-@app.post("/addPDF")
+@app.post("/getRawPDFText")
 async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
     pdf = await pdf.read()
     reader = PdfReader(io.BytesIO(pdf))
     text = ""
     for page in reader.pages:
         text += page.extract_text()
-    return addDocuments(text = text, vectorstore = vectorstore)
+    return text
 
 
-@app.post("/addText")
+@app.post("/addData")
 async def addText(vectorstore: str, text: str):
     return addDocuments(text = text, vectorstore = vectorstore)
 
-
-@app.post("/addWebsite")
-async def addWebsite(vectorstore: str, websiteUrl: str):
-    urls = getLinks(websiteUrl)
-    loader = UnstructuredURLLoader(urls=urls)
-    docs = loader.load()
-    text = "\n\n\n\n".join([f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
-    return addDocuments(text = text, vectorstore = vectorstore)
-
-
 @app.post("/answerQuery")
 async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-70b-8192"):
     return answerQuery(query=query, vectorstore=vectorstore, llmModel=llmModel)
@@ -70,8 +60,6 @@ async def delete(chatbotName: str):
 async def delete(username: str):
     return listTables(username=username)
 
-@app.post("/getLinks")
+@app.post("/getWebsiteData")
 async def crawlUrl(baseUrl: str):
-    return {
-        "urls": getLinks(url=baseUrl, timeout=30)
-        }
+    return getRawWebText(url=baseUrl, timeout=30)

functions.py

@@ -258,8 +258,7 @@ def listTables(username: str):
         }
     
 
-
-def getLinks(url: str, timeout = 30):
+def getRawWebText(url: str, timeout = 30):
     start = time.time()
     def getLinksFromPage(url: str) -> list:
         response = requests.get(url)
@@ -290,9 +289,5 @@ def getLinks(url: str, timeout = 30):
     allLinks = {}
     foundLinks = list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))
     for link in foundLinks:
-        allLinks[link] = len(BeautifulSoup(requests.get(link).text, "lxml").body.get_text(" ", strip = True))
-    return allLinks
-
-
-def getTextLength(text: str):
-    return len(text)
+        allLinks[link] = BeautifulSoup(requests.get(link).text, "lxml").body.get_text(" ", strip = True)
+    return allLinks