UPDATE: adding urls

app.py

@@ -68,4 +68,10 @@ async def delete(chatbotName: str):
 
 @app.post("/listChatbots")
 async def delete(username: str):
-    return listTables(username=username)
+    return listTables(username=username)
+
+@app.post("/getLinks")
+async def crawlUrl(baseUrl: str):
+    return {
+        "urls": getLinks(url=baseUrl, timeout=30)
+        }

functions.py

@@ -290,4 +290,8 @@ def getLinks(url: str, timeout = 30):
             break
         else:
             uniqueLinks = uniqueLinks.union(set(getLinksFromPage(link)))
-    return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))
+    return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))
+
+
+def getTextLength(text: str):
+    return len(text)