Update app.py

app.py

@@ -171,6 +171,28 @@ def visit_webpage(url: str) -> str:
     return (text[:500], text[500:1000])
 
 
+def visit_webpage(url: str) -> str:
+    headers = {
+        "User-Agent": "Mozilla/5.0"
+    }
+
+    response = requests.get(url, headers=headers, timeout=10)
+    response.raise_for_status()
+
+    soup = BeautifulSoup(response.text, "html.parser")
+
+    # Remove scripts/styles
+    for tag in soup(["script", "style"]):
+        tag.extract()
+
+    # Extract more elements (not just <p>)
+    elements = soup.find_all(["p", "dd"])
+
+    text = " \n ".join(el.get_text(strip=False) for el in elements)
+
+    return (text[:1000], )
+
+
 def web_search(query: str, num_results: int = 10):
     """
     Search the internet for the query provided
@@ -705,7 +727,6 @@ class BasicAgent:
             response = self.safe_app.invoke(state)
             
             agent_answer = response["output"]
-            agent_answer = "FunkMonk"
         else:
             agent_answer = fixed_answer
             # agent_answer = self.agent.run(question)