UPDATE: Web Crawler

app.py

@@ -57,7 +57,7 @@ async def addWebsite(vectorstore: str, websiteUrl: str):
         return re.sub(r"\n\n+", "\n\n", soup.text).strip()
     loader = RecursiveUrlLoader(websiteUrl, max_depth=2, timeout = 60, extractor=bs4_extractor)
     docs = loader.load()
-    text = "\n\n".join([docs[doc].page_content for doc in docs])
+    text = "\n\n".join([docs[doc].page_content for doc in range(len(docs))])
     return addDocuments(text = text, vectorstore = vectorstore)