Update app.py

app.py

@@ -59,7 +59,7 @@ def extract_text(url: str, language: str = "ja", length: int = 150000,use_jina:b
         return {"message": error_msg}
 
 @app.post("/httpx_bs", tags=["Text Extraction and beautiful soup"], dependencies=[Depends(validate_token)])
-def extract_text(url: str, length: int = 150000):
+def httpx_bs(url: str, length: int = 150000):
     try:
         response = httpx.get(url)
         text_content = str(convert_to_markdown(response,url))
@@ -75,6 +75,37 @@ def extract_text(url: str, length: int = 150000):
         error_msg = str(e)
         return {"message": error_msg}
 
+@app.post("/extract_from_url", tags=["Text Extraction from URL"], dependencies=[Depends(validate_token)])
+def extract_from_url(url: str, length: int = 150000, tool: str = "httpx"):
+    try:
+        if tool == "jina":
+            response = requests.get("https://r.jina.ai/" + url)
+            text_content = response.text
+        elif tool == "httpx":
+            response = httpx.get(url)
+            text_content = str(convert_to_markdown(response.text, url))
+        elif tool == "requests":
+            response = requests.get(url, timeout=10)
+            text_content = str(convert_to_markdown(response.text, url))
+        elif tool == "webbaseloader":
+            loader = WebBaseLoader(url)
+            docs = loader.load()
+            text_content = docs[0].page_content
+        else:
+            raise ValueError("Invalid tool specified. Choose from 'jina', 'httpx', 'requests', or 'webbaseloader'.")
+
+        if len(text_content) < length:
+            return {"text_content": text_content}
+        else:
+            return {
+                "text_content": text_content[: int(length / 2)]
+                + text_content[len(text_content) - int(length / 2) :]
+            }
+    except Exception as e:
+        error_msg = str(e)
+        return {"message": error_msg}
+
+
 def convert_to_markdown(response_text,url):
     # if response.status_code != 200:
     #     return f"エラー: ステータスコード {response.status_code}"