"""Web tools: search the internet and read the content of a web page.""" from __future__ import annotations import re import requests _HEADERS = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" ) } def web_search(context: dict, query: str, max_results: int = 5) -> str: """Run a DuckDuckGo web search and return the top results as text.""" # Package was renamed duckduckgo_search -> ddgs; support both. try: try: from ddgs import DDGS # type: ignore except ImportError: from duckduckgo_search import DDGS # type: ignore except Exception as exc: # noqa: BLE001 return f"web_search unavailable: {exc}" hits = [] for attempt in range(3): try: with DDGS() as ddgs: hits = list(ddgs.text(query, max_results=max_results)) if hits: break except Exception: # noqa: BLE001 - shared-IP rate limits, retry pass if not hits: return ( "No web results (search may be rate-limited from this host). " "Try wikipedia_search / read_wikipedia for factual lookups instead." ) lines = [] for i, h in enumerate(hits, 1): title = h.get("title", "") href = h.get("href") or h.get("url", "") body = h.get("body", "") lines.append(f"{i}. {title}\n {href}\n {body}") return "\n".join(lines) # Wikimedia's robot policy requires a descriptive User-Agent (browser UAs get 403). _WIKI_HEADERS = { "User-Agent": "gaia-agents-course/1.0 (HF Agents course student project)", "Accept": "application/json", } def _wiki_api(params: dict) -> dict: params = {"format": "json", **params} resp = requests.get( "https://en.wikipedia.org/w/api.php", params=params, headers=_WIKI_HEADERS, timeout=25 ) resp.raise_for_status() return resp.json() def wikipedia_search(context: dict, query: str, max_results: int = 5) -> str: """Search English Wikipedia and return matching page titles with snippets.""" try: data = _wiki_api( {"action": "query", "list": "search", "srsearch": query, "srlimit": max_results} ) except Exception as exc: # noqa: BLE001 return f"wikipedia_search error: {exc}" results = data.get("query", {}).get("search", []) if not results: return "No Wikipedia pages found." lines = [] for i, r in enumerate(results, 1): snippet = re.sub(r"<[^>]+>", "", r.get("snippet", "")) lines.append(f"{i}. {r.get('title')} — {snippet}") return "\n".join(lines) def read_wikipedia(context: dict, title: str, max_chars: int = 8000) -> str: """Return the plain-text content of an English Wikipedia article by title.""" try: data = _wiki_api( { "action": "query", "prop": "extracts", "explaintext": 1, "redirects": 1, "titles": title, } ) except Exception as exc: # noqa: BLE001 return f"read_wikipedia error: {exc}" pages = data.get("query", {}).get("pages", {}) page = next(iter(pages.values()), {}) text = page.get("extract") if not text: return f"No Wikipedia article titled '{title}'. Try wikipedia_search first." if len(text) > max_chars: text = text[:max_chars] + "\n\n...[truncated]" return f"[Wikipedia: {page.get('title', title)}]\n{text}" def visit_webpage(context: dict, url: str, max_chars: int = 8000) -> str: """Download a web page and return its main text content as Markdown.""" try: resp = requests.get(url, headers=_HEADERS, timeout=25) resp.raise_for_status() except Exception as exc: # noqa: BLE001 return f"visit_webpage error: {exc}" try: from markdownify import markdownify text = markdownify(resp.text) except Exception: # noqa: BLE001 from bs4 import BeautifulSoup text = BeautifulSoup(resp.text, "html.parser").get_text("\n") text = re.sub(r"\n{3,}", "\n\n", text).strip() if len(text) > max_chars: text = text[:max_chars] + "\n\n...[truncated]" return text SCHEMAS = [ { "type": "function", "function": { "name": "web_search", "description": "Search the web with DuckDuckGo. Returns a list of titles, URLs and snippets.", "parameters": { "type": "object", "properties": { "query": {"type": "string", "description": "The search query."}, "max_results": {"type": "integer", "description": "How many results (default 5)."}, }, "required": ["query"], }, }, }, { "type": "function", "function": { "name": "visit_webpage", "description": "Fetch a URL and return its readable text content as Markdown.", "parameters": { "type": "object", "properties": { "url": {"type": "string", "description": "The full URL to visit."}, }, "required": ["url"], }, }, }, { "type": "function", "function": { "name": "wikipedia_search", "description": ( "Search English Wikipedia for page titles. Prefer this over web_search " "for encyclopedic facts (people, places, events, works, species) — it is " "reliable and not rate-limited." ), "parameters": { "type": "object", "properties": { "query": {"type": "string", "description": "What to search for."}, "max_results": {"type": "integer", "description": "How many titles (default 5)."}, }, "required": ["query"], }, }, }, { "type": "function", "function": { "name": "read_wikipedia", "description": "Read the full plain-text of an English Wikipedia article by its exact title.", "parameters": { "type": "object", "properties": { "title": {"type": "string", "description": "Exact article title."}, }, "required": ["title"], }, }, }, ] FUNCTIONS = { "web_search": web_search, "visit_webpage": visit_webpage, "wikipedia_search": wikipedia_search, "read_wikipedia": read_wikipedia, }