Spaces:
Sleeping
Sleeping
Update fetch_and_read_URL
Browse files
app.py
CHANGED
|
@@ -10,17 +10,18 @@ from tools.final_answer import FinalAnswerTool
|
|
| 10 |
from Gradio_UI import GradioUI
|
| 11 |
|
| 12 |
@tool
|
| 13 |
-
def fetch_and_summarize_url(url:str, char_limit:int = 1500)-> dict:
|
| 14 |
-
|
| 15 |
-
|
|
|
|
| 16 |
Args:
|
| 17 |
-
url: the URL
|
| 18 |
char_limit: Maximum number of characters (default 1500).
|
| 19 |
"""
|
| 20 |
# STEP A - Fetch URL
|
| 21 |
try:
|
| 22 |
response = requests.get(url, timeout=10)
|
| 23 |
-
response.raise_for_status()
|
| 24 |
html_content = response.text
|
| 25 |
except Exception as e:
|
| 26 |
return {
|
|
@@ -28,45 +29,32 @@ def fetch_and_summarize_url(url:str, char_limit:int = 1500)-> dict: #it's import
|
|
| 28 |
"reading_time_minutes": 0.0,
|
| 29 |
"error": f"Failed to fetch URL: {str(e)}"
|
| 30 |
}
|
| 31 |
-
|
| 32 |
-
# STEP B — Extract text from HTML
|
| 33 |
-
soup = BeautifulSoup(html_content, "html.parser")
|
| 34 |
|
| 35 |
-
#
|
|
|
|
| 36 |
for tag in soup(["script", "style", "header", "footer", "nav", "noscript"]):
|
| 37 |
tag.extract()
|
|
|
|
| 38 |
|
| 39 |
-
#
|
| 40 |
-
raw_text = soup.get_text(separator=" ")
|
| 41 |
-
|
| 42 |
-
# Clean whitespace / newlines
|
| 43 |
-
raw_text = re.sub(r"\s+", " ", raw_text).strip()
|
| 44 |
-
|
| 45 |
-
# STEP C — Summarize using the LLM
|
| 46 |
try:
|
| 47 |
-
# Truncate text to avoid huge prompts
|
| 48 |
truncated_text = raw_text[:char_limit]
|
| 49 |
-
|
| 50 |
-
# Ask the LLM to summarize
|
| 51 |
-
summary_prompt = f"Summarize the following text into 4–6 sentences:\n\n{truncated_text}"
|
| 52 |
-
|
| 53 |
-
# Use the agent model to generate the summary
|
| 54 |
summary = final_answer.run(summary_prompt).strip()
|
| 55 |
-
|
| 56 |
except Exception as e:
|
| 57 |
summary = f"Error summarizing text: {str(e)}"
|
| 58 |
|
| 59 |
-
# STEP D
|
| 60 |
try:
|
| 61 |
word_count = len(raw_text.split())
|
| 62 |
reading_time_minutes = round(word_count / 200, 2)
|
| 63 |
-
except
|
| 64 |
reading_time_minutes = 0.0
|
| 65 |
-
|
| 66 |
return {
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
}
|
| 70 |
|
| 71 |
@tool
|
| 72 |
def get_current_time_in_timezone(timezone: str) -> str:
|
|
|
|
| 10 |
from Gradio_UI import GradioUI
|
| 11 |
|
| 12 |
@tool
|
| 13 |
+
def fetch_and_summarize_url(url: str, char_limit: int = 1500) -> dict:
|
| 14 |
+
"""A tool that fetches the content from a given URL, extracts the meaningful text,
|
| 15 |
+
and returns a structured summary with the estimated reading time.
|
| 16 |
+
|
| 17 |
Args:
|
| 18 |
+
url: the URL where data will be fetched from.
|
| 19 |
char_limit: Maximum number of characters (default 1500).
|
| 20 |
"""
|
| 21 |
# STEP A - Fetch URL
|
| 22 |
try:
|
| 23 |
response = requests.get(url, timeout=10)
|
| 24 |
+
response.raise_for_status()
|
| 25 |
html_content = response.text
|
| 26 |
except Exception as e:
|
| 27 |
return {
|
|
|
|
| 29 |
"reading_time_minutes": 0.0,
|
| 30 |
"error": f"Failed to fetch URL: {str(e)}"
|
| 31 |
}
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
+
# STEP B - Extract text
|
| 34 |
+
soup = BeautifulSoup(html_content, "html.parser")
|
| 35 |
for tag in soup(["script", "style", "header", "footer", "nav", "noscript"]):
|
| 36 |
tag.extract()
|
| 37 |
+
raw_text = re.sub(r"\s+", " ", soup.get_text(separator=" ")).strip()
|
| 38 |
|
| 39 |
+
# STEP C - Summarize
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
try:
|
|
|
|
| 41 |
truncated_text = raw_text[:char_limit]
|
| 42 |
+
summary_prompt = f"Summarize the following text into 4-6 sentences:\n\n{truncated_text}"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
summary = final_answer.run(summary_prompt).strip()
|
|
|
|
| 44 |
except Exception as e:
|
| 45 |
summary = f"Error summarizing text: {str(e)}"
|
| 46 |
|
| 47 |
+
# STEP D - Reading time
|
| 48 |
try:
|
| 49 |
word_count = len(raw_text.split())
|
| 50 |
reading_time_minutes = round(word_count / 200, 2)
|
| 51 |
+
except:
|
| 52 |
reading_time_minutes = 0.0
|
| 53 |
+
|
| 54 |
return {
|
| 55 |
+
"summary": summary,
|
| 56 |
+
"reading_time_minutes": reading_time_minutes
|
| 57 |
+
}
|
| 58 |
|
| 59 |
@tool
|
| 60 |
def get_current_time_in_timezone(timezone: str) -> str:
|