Update app.py

app.py

@@ -44,7 +44,8 @@ def extract_text(url: str, language: str = "ja", length: int = 150000,use_jina:b
                 response = requests.get("https://r.jina.ai/"+ url)
                 text_content = response.text
             else:
-                text_content = str(fetch_and_convert_to_markdown(requests.get(url,timeout = 10)))
+                response = requests.get(url,timeout = 10)
+                text_content = str(convert_to_markdown(response.text,url))
 
         if len(text_content) < length:
             return {"text_content": text_content}
@@ -58,9 +59,10 @@ def extract_text(url: str, language: str = "ja", length: int = 150000,use_jina:b
         return {"message": error_msg}
 
 @app.post("/httpx_bs", tags=["Text Extraction and beautiful soup"], dependencies=[Depends(validate_token)])
-def extract_text(url: str):
+def extract_text(url: str, length: int = 150000):
     try:
-        text_content = str(fetch_and_convert_to_markdown(httpx.get(url)))
+        response = httpx.get(url)
+        text_content = str(convert_to_markdown(response,url))
 
         if len(text_content) < length:
             return {"text_content": text_content}
@@ -73,11 +75,11 @@ def extract_text(url: str):
         error_msg = str(e)
         return {"message": error_msg}
 
-def fetch_and_convert_to_markdown(response):
-    if response.status_code != 200:
-        return f"エラー: ステータスコード {response.status_code}"
+def convert_to_markdown(response_text,url):
+    # if response.status_code != 200:
+    #     return f"エラー: ステータスコード {response.status_code}"
 
-    soup = BeautifulSoup(response.text, 'html.parser')
+    soup = BeautifulSoup(response_text, 'html.parser')
     markdown = ""
 
     # タイトル