Update app.py

app.py

@@ -1197,23 +1197,56 @@ def cache_response(query, response, ttl=3600):
 
 
 tavily_tool = TavilySearchResults(
-    max_results=12,
+    max_results=13,
     search_depth="advanced",
-    topic="news",
     days=1,
     include_answer=True,
     include_raw_content=True,
-    # include_domains=[...],
     exclude_domains=['example.com'],
-    # name="...",            # overwrite default tool name
-    # description="...",     # overwrite default tool description
-    # args_schema=...,       # overwrite default args_schema: BaseModel
 )
 
+def validate_tavily_results(tavily_tool):
+    """
+    Fetch and validate results from TavilySearchResults.
+    Ensures that results do not lead to 'Page Not Found' pages.
+    """
+    results = tavily_tool.search()
+    valid_results = []
+
+    for result in results:
+        url = result.get('url')
+        if not url:
+            continue
+
+        try:
+            # Fetch page content
+            response = requests.get(url, timeout=10)
+            if response.status_code != 200:
+                continue
+
+            # Parse page content to check for "Page Not Found"
+            soup = BeautifulSoup(response.content, 'html.parser')
+            page_text = soup.get_text()
+
+            # Common markers of "Page Not Found"
+            if any(marker in page_text for marker in ["Page Not Found", "404", "Error", "Not Available"]):
+                continue
+
+            # Add valid result
+            valid_results.append(result)
+
+        except requests.RequestException:
+            # Skip URLs causing errors
+            continue
+
+    return valid_results
+
+
+
 #Compile all tool functions into a list
 tools = [
     knowledge_base_tool,  # Tool for querying the knowledge base and retrieving responses
-    tavily_tool,
+    {"tool": tavily_tool, "validate": validate_tavily_results(tavily_tool)},
     #google_search_tool,  # Tool for performing a Google search and retrieving search result snippets
 ]