Update app.py

app.py

@@ -4,9 +4,70 @@ import requests
 import pytz
 import yaml
 from tools.final_answer import FinalAnswerTool
+import time, json
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import urllib.robotparser as robotparser
 
 from Gradio_UI import GradioUI
 
+def allowed_by_robots(url: str, user_agent: str = DEFAULT_HEADERS["User-Agent"]) -> bool:
+    try:
+        parsed = urlparse(url)
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        rp = robotparser.RobotFileParser()
+        rp.set_url(robots_url)
+        rp.read()
+        return rp.can_fetch(user_agent, url)
+    except Exception:
+        # If robots can't be read, err on the safe side and allow
+        return True
+
+@tool
+def scrape_html(url: str, selector: str, attr: str = "text", max_items: int = 50, delay_s: float = 0.0) -> str:
+    """Scrape elements from a web page using a CSS selector.
+    Args:
+        url: Full URL of the page to scrape.
+        selector: CSS selector for the elements to extract (e.g., 'h2 a', 'div.price').
+        attr: What to return for each element: 'text' for text content, or an attribute name like 'href'/'src'.
+        max_items: Maximum number of elements to return.
+        delay_s: Optional polite delay (seconds) before the request.
+    Returns:
+        A JSON string of the form {"url": ..., "count": N, "items": [...]}
+    """
+    try:
+        if delay_s > 0:
+            time.sleep(delay_s)
+
+        if not allowed_by_robots(url, DEFAULT_HEADERS["User-Agent"]):
+            return json.dumps({"error": f"Blocked by robots.txt for {url}"})
+
+        resp = requests.get(url, headers=DEFAULT_HEADERS, timeout=15)
+        resp.raise_for_status()
+
+        soup = BeautifulSoup(resp.text, "html.parser")
+        elements = soup.select(selector)
+        items = []
+
+        for el in elements[:max_items]:
+            if attr == "text":
+                items.append(el.get_text(strip=True))
+            else:
+                # Resolve relative URLs for href/src if present
+                val = el.get(attr)
+                if val and attr in ("href", "src"):
+                    val = urljoin(url, val)
+                items.append(val)
+
+        return json.dumps({"url": url, "count": len(items), "items": items}, ensure_ascii=False)
+    except requests.exceptions.RequestException as e:
+        return json.dumps({"error": f"Network error: {e}"})
+    except Exception as e:
+        return json.dumps({"error": f"Parse error: {e}"})
+
+
+
+
 # Below is an example of a tool that does nothing. Amaze us with your creativity !
 @tool
 def my_custom_tool(arg1:str, arg2:int)-> str: #it's import to specify the return type
@@ -55,7 +116,7 @@ with open("prompts.yaml", 'r') as stream:
     
 agent = CodeAgent(
     model=model,
-    tools=[final_answer], ## add your tools here (don't remove final answer)
+    tools=[final_answer, scrape_html], ## add your tools here (don't remove final answer)
     max_steps=6,
     verbosity_level=1,
     grammar=None,