Add visit webpage tool. Update imports

app.py

@@ -3,9 +3,14 @@ import gradio as gr
 import requests
 import inspect
 import pandas as pd
-from llama_index.core.agent.workflow import AgentWorkflow, ToolCallResult, AgentStream
+from llama_index.core.agent.workflow import AgentWorkflow
 from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
-from tools import APIProcessor, parse_youtube_video, transcribe_image_from_link
+from tools import (
+    APIProcessor,
+    parse_youtube_video,
+    transcribe_image_from_link,
+    transcribe_webpage,
+)
 from search import GoogleSearch
 from dotenv import load_dotenv
 
@@ -40,6 +45,7 @@ class BasicAgent:
                 get_and_process_question_attachment,
                 parse_youtube_video,
                 transcribe_image_from_link,
+                transcribe_webpage,
             ],
             llm=self.llm,
             system_prompt=SYSTEM_PROMPT,

requirements.txt

@@ -6,4 +6,7 @@ openai
 pandas
 aiohttp
 Pillow
-yt-dlp
+yt-dlp
+markdownify
+llama-index-utils-workflow
+llama-index-llms-huggingface-api

tools.py

@@ -7,6 +7,9 @@ import tempfile
 import os
 import io
 import yt_dlp
+import re
+from markdownify import markdownify
+from requests.exceptions import RequestException
 
 
 def transcribe_image_from_link(image_link: str) -> str:
@@ -46,6 +49,37 @@ def transcribe_image_from_link(image_link: str) -> str:
     return transcribed_text
 
 
+def transcribe_webpage(website_url: str) -> str:
+    """Visits website url and returns markdown of contents"""
+    try:
+        # Send a GET request to the URL with a 20-second timeout
+        response = requests.get(website_url, timeout=20)
+        response.raise_for_status()  # Raise an exception for bad status codes
+
+        # Convert the HTML content to Markdown
+        markdown_content = markdownify(response.text).strip()
+
+        with open("webpage_content.md", "a") as f:
+            f.write("""Response from the webpage:\n\n""")
+            f.write(response.text)
+
+        # Remove multiple line breaks
+        markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
+
+        with open("webpage_content.md", "a") as f:
+            f.write("\n\nMarkdown content:\n\n")
+            f.write(markdown_content)
+
+        return markdown_content
+
+    except requests.exceptions.Timeout:
+        return "The request timed out. Please try again later or check the URL."
+    except RequestException as e:
+        return f"Error fetching the webpage: {str(e)}"
+    except Exception as e:
+        return f"An unexpected error occurred: {str(e)}"
+
+
 def parse_youtube_video(youtube_url: str) -> str:
     """Returns text transcript of a youtube video
     Args: