Update app.py

app.py

@@ -1191,11 +1191,6 @@ def cache_response(query, response, ttl=3600):
 
 
 
-
-
-
-
-
 tavily_tool = TavilySearchResults(
     max_results=13,
     search_depth="advanced",
@@ -1210,11 +1205,24 @@ def validate_tavily_results(query):
     Fetch and validate results from TavilySearchResults.
     Ensures that results do not lead to 'Page Not Found' pages.
     """
-    results = tavily_tool.run(query=query)
+    # Use the Tavily tool to fetch results
+    results = tavily_tool.run(query)  # Pass the query as tool_input to the Tavily tool
+
+    # Check if results are in expected format (list of dicts). Adjust if necessary.
+    if isinstance(results, str):
+        # If results are returned as a string, parse them into a structured format
+        # For example, if results are JSON-formatted strings
+        import json
+        try:
+            results = json.loads(results)
+        except json.JSONDecodeError:
+            # If parsing fails, split by newlines assuming URLs are separated by lines
+            results = [{"title": f"Result {i+1}", "url": url.strip()} for i, url in enumerate(results.splitlines())]
+
     valid_results = []
 
     for result in results:
-        url = result.get('url')
+        url = result.get("url")
         if not url:
             continue
 
@@ -1225,18 +1233,17 @@ def validate_tavily_results(query):
                 continue
 
             # Parse page content to check for "Page Not Found"
-            soup = BeautifulSoup(response.content, 'html.parser')
-            page_text = soup.get_text()
+            soup = BeautifulSoup(response.content, "html.parser")
+            page_text = soup.get_text(separator=' ', strip=True)
 
             # Common markers of "Page Not Found"
-            if any(marker in page_text for marker in ["Page Not Found", "404", "Error", "Not Available"]):
+            if any(marker.lower() in page_text.lower() for marker in ["Page Not Found", "404", "Error", "Not Available"]):
                 continue
 
             # Add valid result
             valid_results.append(result)
 
         except requests.RequestException:
-            # Skip URLs causing errors
             continue
 
     return valid_results
@@ -1247,7 +1254,6 @@ def tavily_tool_with_validation(query):
     return "\n".join([f"{res.get('title')}: {res.get('url')}" for res in valid_results])
 
 
-
 #Compile all tool functions into a list
 tools = [
     knowledge_base_tool,  # Tool for querying the knowledge base and retrieving responses