DEBUG: updating getLinks

app.py

@@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from langchain_community.document_loaders import UnstructuredURLLoader
 from src.api.speech_api import speech_translator_router
 from functions import client as supabase
+from urllib.parse import urlparse
 
 app = FastAPI(title="ConversAI", root_path="/api/v1")
 
@@ -224,11 +225,11 @@ async def addText(addQaPair: AddQAPair):
 
 @app.post("/addWebsite")
 async def addWebsite(vectorstore: str, websiteUrls: list[str]):
-    urls = websiteUrls
-    loader = UnstructuredURLLoader(urls=urls)
+    loader = UnstructuredURLLoader(urls=websiteUrls)
     docs = loader.load()
     text = "\n\n".join(
-        [f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
+        [f"{docs[doc].page_content}" for doc in range(len(docs))]
+    )
     username, chatbotname = vectorstore.split("$")[1], vectorstore.split("$")[2]
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
     currentCount = df[(df["user_id"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
@@ -238,7 +239,7 @@ async def addWebsite(vectorstore: str, websiteUrls: list[str]):
     if newCount < int(limit):
         client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("user_id", username).eq(
             "chatbotname", chatbotname).execute()
-        return addDocuments(text=text, source="website", vectorstore=vectorstore)
+        return addDocuments(text=text, source=urlparse(websiteUrls[0]).netloc, vectorstore=vectorstore)
     else:
         return {
             "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."

functions.py

@@ -154,6 +154,7 @@ def addDocuments(text: str, source: str, vectorstore: str):
 def format_docs(docs: str):
     context = ""
     for doc in docs:
+        print("METADATA ::: ", type(doc.metadata))
         context += f"CONTENT: {doc.page_content}\nSOURCE: {doc.metadata} \n\n\n"
     if context == "":
         context = "No context found"
@@ -255,7 +256,7 @@ def listTables(username: str):
 
 def getLinks(url: str, timeout=30):
     start = time.time()
-    
+
     def getLinksFromPage(url: str) -> list:
         response = requests.get(url)
         soup = BeautifulSoup(response.content, "lxml")