tools/url_reader.py

import requests
from langchain_core.documents import Document
from langchain_core.tools import tool

from src.config import TAVILY_API_KEY

_EXTRACT_URL = "https://api.tavily.com/extract"


@tool
def url_reader(url: str) -> list:
    """Read and extract content from a URL.

    Use this when the user's query contains a URL and you need to access the webpage content.

    Args:
        url: The URL to extract content from.
    """
    try:
        resp = requests.post(
            _EXTRACT_URL,
            json={"urls": [url], "api_key": TAVILY_API_KEY},
            timeout=30,
        )
        resp.raise_for_status()
        data = resp.json()
    except Exception as e:
        return [Document(page_content=f"Không thể đọc URL: {e}", metadata={"source": url})]

    results = data.get("results", [])
    if not results:
        failed = data.get("failed_results", [])
        reason = failed[0].get("error", "không có nội dung") if failed else "không có nội dung"
        return [Document(page_content=f"Không thể trích xuất nội dung từ URL: {reason}", metadata={"source": url})]

    return [
        Document(
            page_content=r.get("raw_content", ""),
            metadata={"source": r.get("url", url), "title": r.get("url", url)},
        )
        for r in results
        if isinstance(r, dict) and r.get("raw_content")
    ]