import requests from langchain_core.documents import Document from langchain_core.tools import tool from src.config import TAVILY_API_KEY _EXTRACT_URL = "https://api.tavily.com/extract" @tool def url_reader(url: str) -> list: """Read and extract content from a URL. Use this when the user's query contains a URL and you need to access the webpage content. Args: url: The URL to extract content from. """ try: resp = requests.post( _EXTRACT_URL, json={"urls": [url], "api_key": TAVILY_API_KEY}, timeout=30, ) resp.raise_for_status() data = resp.json() except Exception as e: return [Document(page_content=f"Không thể đọc URL: {e}", metadata={"source": url})] results = data.get("results", []) if not results: failed = data.get("failed_results", []) reason = failed[0].get("error", "không có nội dung") if failed else "không có nội dung" return [Document(page_content=f"Không thể trích xuất nội dung từ URL: {reason}", metadata={"source": url})] return [ Document( page_content=r.get("raw_content", ""), metadata={"source": r.get("url", url), "title": r.get("url", url)}, ) for r in results if isinstance(r, dict) and r.get("raw_content") ]