UPDATE: New Endpoints

app.py

@@ -1,11 +1,13 @@
 import io
 from functions import *
 from PyPDF2 import PdfReader
+import pandas as pd
 from fastapi import FastAPI, File, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from langchain_community.document_loaders import UnstructuredURLLoader
 
 
+
 app = FastAPI(title = "ConversAI", root_path = "/api/v1")
 app.add_middleware(
     CORSMiddleware,
@@ -29,23 +31,63 @@ async def login(username: str, password: str):
 
 @app.post("/newChatbot")
 async def newChatbot(chatbotName: str, username: str):
+    client.table("ConversAI_ChatbotInfo").insert({"username": username, "chatbotname": chatbotName}).execute()
     chatbotName = f"convai-{username}-{chatbotName}"
     return createTable(tablename = chatbotName)
 
 
-@app.post("/getRawPDFText")
+@app.post("/addPDF")
 async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
     pdf = await pdf.read()
     reader = PdfReader(io.BytesIO(pdf))
     text = ""
     for page in reader.pages:
         text += page.extract_text()
-    return text
-
-
-@app.post("/addData")
+    username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
+    df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
+    currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"][0]
+    newCount = currentCount + len(text)
+    if newCount < 400000:
+        client.table("ConversAI_ChatbotInfo").update({"charactercount", newCount}).eq("username", username).eq("chatbotname", chatbotname).execute()
+        return addDocuments(text = text, vectorstore = vectorstore)
+    else:
+        return {
+            "output": "DOCUMENT EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
+        }
+
+
+@app.post("/addText")
 async def addText(vectorstore: str, text: str):
-    return addDocuments(text = text, vectorstore = vectorstore)
+    username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
+    df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
+    currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"][0]
+    newCount = currentCount + len(text)
+    if newCount < 400000:
+        client.table("ConversAI_ChatbotInfo").update({"charactercount", newCount}).eq("username", username).eq("chatbotname", chatbotname).execute()
+        return addDocuments(text = text, vectorstore = vectorstore)
+    else:
+        return {
+            "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
+        }
+
+
+@app.post("/addWebsite")
+async def addWebsite(vectorstore: str, websiteUrl: str):
+    urls = getLinks(websiteUrl)
+    loader = UnstructuredURLLoader(urls=urls)
+    docs = loader.load()
+    text = "\n\n\n\n".join([f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
+    username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
+    df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
+    currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"][0]
+    newCount = currentCount + len(text)
+    if newCount < 400000:
+        client.table("ConversAI_ChatbotInfo").update({"charactercount", newCount}).eq("username", username).eq("chatbotname", chatbotname).execute()
+        return addDocuments(text = text, vectorstore = vectorstore)
+    else:
+        return {
+            "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
+        }
 
 @app.post("/answerQuery")
 async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-70b-8192"):
@@ -54,12 +96,24 @@ async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-7
 
 @app.post("/deleteChatbot")
 async def delete(chatbotName: str):
+    username, chatbotName = chatbotName.split("-")[1], chatbotName.split("-")[2]
+    client.table('ConversAI_ChatbotInfo').delete().eq('username', username).eq('chatbotname', chatbotName).execute()
     return deleteTable(tableName=chatbotName)
 
 @app.post("/listChatbots")
 async def delete(username: str):
     return listTables(username=username)
 
-@app.post("/getWebsiteData")
+@app.post("/getLinks")
 async def crawlUrl(baseUrl: str):
-    return getRawWebText(url=baseUrl, timeout=30)
+    return {
+        "urls": getLinks(url=baseUrl, timeout=30)
+        }
+
+@app.post("/getCurrentCount")
+async def getCount(vectorstore: str):
+    username, chatbotName = chatbotName.split("-")[1], chatbotName.split("-")[2]
+    df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
+    return {
+        "currentCount": df[(df['username'] == username) & (df['chatbotname'] == chatbotName)]['charactercount'][0]
+        }

functions.py

@@ -258,7 +258,7 @@ def listTables(username: str):
         }
     
 
-def getRawWebText(url: str, timeout = 30):
+def getLinks(url: str, timeout = 30):
     start = time.time()
     def getLinksFromPage(url: str) -> list:
         response = requests.get(url)
@@ -286,8 +286,4 @@ def getRawWebText(url: str, timeout = 30):
             break
         else:
             uniqueLinks = uniqueLinks.union(set(getLinksFromPage(link)))
-    allLinks = {}
-    foundLinks = list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))
-    for link in foundLinks:
-        allLinks[link] = BeautifulSoup(requests.get(link).text, "lxml").body.get_text(" ", strip = True)
-    return allLinks
+    return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))

requirements.txt

@@ -13,6 +13,7 @@ langchain-groq
 lxml
 PyPDF2
 python-dotenv
+pandas
 sentence-transformers
 supabase
 unstructured