Update app.py

app.py

@@ -130,6 +130,21 @@ def extract_background_from_url(url: str):
     background_section = extract_background(content)
     return {"background": background_section}
 
+
+@app.get("/extract-page-text/")
+def extract_page_content(url: str, max_char: int = 5000)    
+    try:
+        response = requests.get(url, timeout=20, verify=False)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, 'html.parser')
+        full_text = soup.get_text()
+        
+        text = full_text[:max_char]
+        return {"text_content": text}
+    except requests.RequestException as e:
+        return {"error": f"Error fetching the page: {str(e)}"}
+
 def search_web(topic, max_references):
     """Search the web using DuckDuckGo and return results."""
     doc_list = []