tools/visit_webpage.py

"""
tools/visit_webpage.py —— 工具③：打开并阅读一个网页

增强版：支持表格保留为纯文本（便于 LLM 解析），增加重试和超时控制，User-Agent 轮换。
"""

import re
import time
import requests
from bs4 import BeautifulSoup

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
]

def visit_webpage(url: str) -> str:
    """
    Fetch a web page, extract its main text content, and return as plain text.
    Tables are converted to CSV-like format for easier parsing by LLM.
    Includes retries and timeout handling.
    """
    # 重试配置
    max_retries = 2
    timeout = 15  # 秒
    
    for attempt in range(max_retries + 1):
        try:
            headers = {"User-Agent": USER_AGENTS[attempt % len(USER_AGENTS)]}
            resp = requests.get(url, headers=headers, timeout=timeout)
            resp.raise_for_status()
            soup = BeautifulSoup(resp.text, "html.parser")
            
            # 移除脚本和样式标签
            for tag in soup(["script", "style", "nav", "footer", "header"]):
                tag.decompose()
            
            # 处理表格：转换为 CSV 文本
            for table in soup.find_all("table"):
                rows = []
                for tr in table.find_all("tr"):
                    cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                    if cells:
                        rows.append(",".join(cells))
                if rows:
                    table_text = "\n".join(rows)
                    # 用注释标记表格，帮助 LLM 识别
                    table.replace_with(soup.new_string(f"\n[TABLE START]\n{table_text}\n[TABLE END]\n"))
            
            # 获取纯文本
            text = soup.get_text(separator="\n", strip=True)
            # 压缩多余空行
            text = re.sub(r"\n{3,}", "\n\n", text)
            # 限制长度
            max_len = 8000
            if len(text) > max_len:
                text = text[:max_len] + "\n...[truncated]"
            return text
        
        except requests.exceptions.RequestException as e:
            if attempt == max_retries:
                return f"Failed to fetch webpage after {max_retries+1} attempts: {str(e)}"
            time.sleep(1)  # 重试前等待
        except Exception as e:
            return f"Error parsing webpage: {str(e)}"
    
    return "Unexpected error in visit_webpage"