"""Web search via DuckDuckGo HTML endpoint (no API key, no new deps). Used as optional grounding for chat ("use web") and the local-deals feature. Degrades gracefully: any network/parse failure returns []. """ import html import logging import re from urllib.parse import parse_qs, unquote, urlparse logger = logging.getLogger(__name__) _SEARCH_URL = "https://html.duckduckgo.com/html/" _UA = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ) # Title _RESULT_RE = re.compile( r']*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>(.*?)', re.IGNORECASE | re.DOTALL, ) # snippet (sometimes a /
) _SNIPPET_RE = re.compile( r'<(?:a|td|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)', re.IGNORECASE | re.DOTALL, ) _TAG_RE = re.compile(r"<[^>]+>") def _strip_html(s: str) -> str: return html.unescape(_TAG_RE.sub("", s)).strip() def _clean_url(href: str) -> str: """Resolve DDG redirect links //duckduckgo.com/l/?uddg=.""" href = html.unescape(href) if href.startswith("//"): href = "https:" + href parsed = urlparse(href) if "duckduckgo.com" in parsed.netloc and parsed.path.startswith("/l"): uddg = parse_qs(parsed.query).get("uddg", [""])[0] if uddg: return unquote(uddg) return href def search_web(query: str, max_results: int = 5) -> list[dict]: """Search DuckDuckGo. Returns [{"title", "snippet", "url"}]; [] on failure.""" try: import httpx resp = httpx.get( _SEARCH_URL, params={"q": query}, headers={"User-Agent": _UA}, timeout=10.0, follow_redirects=True, ) resp.raise_for_status() page = resp.text links = _RESULT_RE.findall(page) snippets = [_strip_html(s) for s in _SNIPPET_RE.findall(page)] results = [] for i, (href, title_html) in enumerate(links[:max_results]): results.append( { "title": _strip_html(title_html), "snippet": snippets[i] if i < len(snippets) else "", "url": _clean_url(href), } ) return results except Exception as e: logger.warning("web search failed for %r: %s", query, e) return [] def format_results(results: list[dict]) -> str: """Readable text block for prompt injection.""" if not results: return "" lines = ["WEB SEARCH RESULTS:"] for i, r in enumerate(results, 1): lines.append(f"{i}. {r.get('title', '')}") if r.get("snippet"): lines.append(f" {r['snippet']}") if r.get("url"): lines.append(f" Source: {r['url']}") return "\n".join(lines)