|
|
|
|
|
|
|
|
|
|
|
from typing import Annotated |
|
|
|
|
|
from langchain_core.tools import tool |
|
|
|
|
|
from src.config.crawler.crawler import Crawler |
|
|
from loguru import logger |
|
|
|
|
|
|
|
|
@tool |
|
|
def crawl_tool( |
|
|
url: Annotated[str, "The url to crawl."], |
|
|
) -> str: |
|
|
"""Use this to crawl a url and get a readable content in markdown format.""" |
|
|
try: |
|
|
crawler = Crawler() |
|
|
article = crawler.crawl(url) |
|
|
return {"url": url, "crawled_content": article.to_markdown()[:1000]} |
|
|
except BaseException as e: |
|
|
error_msg = f"Failed to crawl. Error: {repr(e)}" |
|
|
logger.error(error_msg) |
|
|
return error_msg |
|
|
|
