File size: 3,454 Bytes
296a506 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """
Frox AI — Web Browser Tool
Distinct from web_search: this fetches a specific URL and extracts its
readable text, rather than searching for pages. Plain HTTP client +
HTML parsing — no AI model or third-party service involved beyond the
page's own server.
"""
from __future__ import annotations
import re
from typing import Optional
from urllib.parse import urlparse
try:
import httpx
except ImportError:
httpx = None
try:
from bs4 import BeautifulSoup
HAS_BS4 = True
except ImportError:
HAS_BS4 = False
from tools.registry import tool, ToolContext
MAX_CONTENT_CHARS = 15000 # keep tool results from blowing the model's context budget
ALLOWED_SCHEMES = {"http", "https"}
BLOCKED_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} # basic SSRF guard
def _validate_url(url: str):
parsed = urlparse(url)
if parsed.scheme not in ALLOWED_SCHEMES:
raise ValueError(f"Unsupported URL scheme: {parsed.scheme!r} (only http/https allowed)")
host = (parsed.hostname or "").lower()
if host in BLOCKED_HOSTS or host.startswith("169.254.") or host.startswith("10.") \
or host.startswith("192.168.") or re.match(r"^172\.(1[6-9]|2\d|3[01])\.", host):
raise ValueError(f"Fetching internal/private addresses is not allowed: {host}")
def _extract_text(html: str) -> str:
if HAS_BS4:
soup = BeautifulSoup(html, "html.parser")
for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
tag.decompose()
text = soup.get_text(separator="\n")
else:
# Minimal fallback if beautifulsoup4 isn't installed: strip tags with regex.
text = re.sub(r"<script.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r"<style.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
text = re.sub(r"<[^>]+>", "\n", text)
lines = [line.strip() for line in text.splitlines()]
lines = [line for line in lines if line]
return "\n".join(lines)
def _extract_title(html: str) -> str:
match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
return match.group(1).strip() if match else ""
@tool(
name="web_browser",
description="Fetch a specific URL and return its readable text content",
timeout=15.0,
)
async def web_browser(ctx: ToolContext, url: str) -> dict:
"""
Args:
url: The full URL to fetch (http/https only).
"""
if httpx is None:
raise RuntimeError("httpx is required for web_browser — pip install httpx")
_validate_url(url)
async with httpx.AsyncClient(
timeout=10.0, follow_redirects=True,
headers={"User-Agent": "FroxAI-Browser/1.1"},
) as client:
resp = await client.get(url)
resp.raise_for_status()
content_type = resp.headers.get("content-type", "")
if "text/html" not in content_type and "application/xhtml" not in content_type:
return {
"url": url, "title": "", "content_type": content_type,
"text": resp.text[:MAX_CONTENT_CHARS],
"truncated": len(resp.text) > MAX_CONTENT_CHARS,
}
html = resp.text
title = _extract_title(html)
text = _extract_text(html)
truncated = len(text) > MAX_CONTENT_CHARS
return {
"url": url,
"title": title,
"text": text[:MAX_CONTENT_CHARS],
"truncated": truncated,
}
|