src/tools/web_browser.py

"""
Frox AI — Web Browser Tool

Distinct from web_search: this fetches a specific URL and extracts its
readable text, rather than searching for pages. Plain HTTP client +
HTML parsing — no AI model or third-party service involved beyond the
page's own server.
"""
from __future__ import annotations

import re
from typing import Optional
from urllib.parse import urlparse

try:
    import httpx
except ImportError:
    httpx = None

try:
    from bs4 import BeautifulSoup
    HAS_BS4 = True
except ImportError:
    HAS_BS4 = False

from tools.registry import tool, ToolContext


MAX_CONTENT_CHARS = 15000   # keep tool results from blowing the model's context budget
ALLOWED_SCHEMES = {"http", "https"}
BLOCKED_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"}   # basic SSRF guard


def _validate_url(url: str):
    parsed = urlparse(url)
    if parsed.scheme not in ALLOWED_SCHEMES:
        raise ValueError(f"Unsupported URL scheme: {parsed.scheme!r} (only http/https allowed)")
    host = (parsed.hostname or "").lower()
    if host in BLOCKED_HOSTS or host.startswith("169.254.") or host.startswith("10.") \
       or host.startswith("192.168.") or re.match(r"^172\.(1[6-9]|2\d|3[01])\.", host):
        raise ValueError(f"Fetching internal/private addresses is not allowed: {host}")


def _extract_text(html: str) -> str:
    if HAS_BS4:
        soup = BeautifulSoup(html, "html.parser")
        for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
            tag.decompose()
        text = soup.get_text(separator="\n")
    else:
        # Minimal fallback if beautifulsoup4 isn't installed: strip tags with regex.
        text = re.sub(r"<script.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r"<style.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
        text = re.sub(r"<[^>]+>", "\n", text)

    lines = [line.strip() for line in text.splitlines()]
    lines = [line for line in lines if line]
    return "\n".join(lines)


def _extract_title(html: str) -> str:
    match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
    return match.group(1).strip() if match else ""


@tool(
    name="web_browser",
    description="Fetch a specific URL and return its readable text content",
    timeout=15.0,
)
async def web_browser(ctx: ToolContext, url: str) -> dict:
    """
    Args:
        url: The full URL to fetch (http/https only).
    """
    if httpx is None:
        raise RuntimeError("httpx is required for web_browser — pip install httpx")

    _validate_url(url)

    async with httpx.AsyncClient(
        timeout=10.0, follow_redirects=True,
        headers={"User-Agent": "FroxAI-Browser/1.1"},
    ) as client:
        resp = await client.get(url)
        resp.raise_for_status()
        content_type = resp.headers.get("content-type", "")

        if "text/html" not in content_type and "application/xhtml" not in content_type:
            return {
                "url": url, "title": "", "content_type": content_type,
                "text": resp.text[:MAX_CONTENT_CHARS],
                "truncated": len(resp.text) > MAX_CONTENT_CHARS,
            }

        html = resp.text

    title = _extract_title(html)
    text = _extract_text(html)
    truncated = len(text) > MAX_CONTENT_CHARS

    return {
        "url": url,
        "title": title,
        "text": text[:MAX_CONTENT_CHARS],
        "truncated": truncated,
    }