UPDATE: web crawler

app.py

@@ -1,11 +1,9 @@
 import io
-import re
 from functions import *
 from PyPDF2 import PdfReader
-from bs4 import BeautifulSoup
 from fastapi import FastAPI, File, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
-from langchain_community.document_loaders import RecursiveUrlLoader
+from langchain_community.document_loaders import UnstructuredURLLoader
 
 
 app = FastAPI(title = "ConversAI", root_path = "/api/v1")
@@ -52,12 +50,10 @@ async def addText(vectorstore: str, text: str):
 
 @app.post("/addWebsite")
 async def addWebsite(vectorstore: str, websiteUrl: str):
-    def bs4_extractor(html: str) -> str:
-        soup = BeautifulSoup(html, "lxml")
-        return re.sub(r"\n\n+", "\n\n", soup.text).strip()
-    loader = RecursiveUrlLoader(websiteUrl, max_depth=2, timeout = 60, extractor=bs4_extractor)
+    urls = getLinks("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
+    loader = UnstructuredURLLoader(urls=urls)
     docs = loader.load()
-    text = "\n\n".join([docs[doc].page_content for doc in range(len(docs))])
+    text = "\n\n\n\n".join([f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
     return addDocuments(text = text, vectorstore = vectorstore)
 
 

functions.py

@@ -18,9 +18,14 @@ from langchain.retrievers.document_compressors import FlashrankRerank
 from supabase.client import create_client
 from qdrant_client import QdrantClient
 from langchain_groq import ChatGroq
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse
 from supabase import create_client
 from dotenv import load_dotenv
 import os
+import time
+import requests
+
 
 load_dotenv("secrets.env")
 client = create_client(os.environ["SUPABASE_URL"], os.environ["SUPABASE_KEY"])
@@ -243,4 +248,32 @@ def listTables(username: str):
     except Exception as e:
         return {
             "error": e
-        }
+        }
+    
+
+def getLinks(url: str, timeout = 30):
+  start = time.time()
+  def getLinksFromPage(url: str):
+    response = requests.get(url)
+    htmlContent = response.content
+    soup = BeautifulSoup(htmlContent, "lxml")
+    anchorTags = soup.find_all("a")
+    allLinks = []
+    for tag in anchorTags:
+      if "href" in tag.attrs:
+        if urlparse(tag.attrs["href"]).netloc == urlparse(url).netloc:
+          allLinks.append(tag.attrs["href"])
+        else:
+          continue
+      else:
+        continue
+    return allLinks
+  links = getLinksFromPage(url)
+  uniqueLinks = set()
+  for link in links:
+    now = time.time()
+    if now - start > timeout:
+      break
+    else:
+      uniqueLinks = uniqueLinks.union(set(getLinksFromPage(link)))
+  return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))

requirements.txt

@@ -14,4 +14,6 @@ lxml
 PyPDF2
 python-dotenv
 sentence-transformers
-supabase
+supabase
+unstructured
+urllib3