improving web searching - added web scrapping when the search returns links - fixed content_type identification

manage_agents.py

@@ -292,6 +292,9 @@ class MainAgent:
         # Create LLM with tools bound for tool-using capabilities
         self.general_tools = [self.wikipedia_tool, self.web_search_tool]
         self.llm_with_tools = self.llm.bind_tools(self.general_tools)
+        
+        # Tool usage tracking
+        self.last_used_tool = None
     
     def _format_question(self, question: str) -> str:
         """Format the question to be more specific and clear"""
@@ -349,15 +352,20 @@ class MainAgent:
     
     def process_question(self, task_id: str, question: str, file_name: str = "") -> str:
         try:
+            # Reset tool tracking
+            self.last_used_tool = None
+            
             # First check if we can answer this directly without tools
             direct_answer = self.content_translate.answer_or_flag(question)
             if direct_answer != "TOOLS_REQUIRED":
+                self.last_used_tool = "direct"
                 return direct_answer
             
             # If we have a file to process, use specialized tools
             if file_name:
                 # Identify content type based on file extension
                 content_type, parameter, task_id = self.content_type_agent.identify_content_type(question, file_name, task_id)
+                self.last_used_tool = content_type
                 
                 if content_type in self.tools:
                     tool = self.tools[content_type]
@@ -451,6 +459,30 @@ class MainAgent:
 
                             response = self.llm.invoke(audio_analysis_prompt)
                             return response.content if hasattr(response, 'content') else str(response)
+                        elif content_type == "youtube":
+                            result = tool._run(task_id, question=question)
+
+                            # Use specialized prompt for YouTube analysis
+                            youtube_analysis_prompt = f"""
+                            Analyze this YouTube video and provide an extremely concise answer:
+
+                            Question: {question}
+
+                            YouTube Video:
+                            {result}
+
+                            Instructions:
+                            1. Pay careful attention to the specific format requested in the question
+                            2. Extract only the information needed to answer the question
+                            
+                            When answering, provide ONLY the precise answer requested. 
+                            Do not include explanations, steps, reasoning, or additional text.
+                            Be direct and specific. GAIA benchmark requires exact matching answers.
+                            For example, if asked "What is the color of the sky?", respond simply with "blue".
+                            """
+
+                            response = self.llm.invoke(youtube_analysis_prompt)
+                            return response.content if hasattr(response, 'content') else str(response)
                         else:
                             # Even for other tools, pass the question if the method accepts it
                             try:
@@ -489,12 +521,45 @@ class MainAgent:
                 else:
                     return f"Unsupported file type: {content_type}"
             
+            # For general questions (no files), check for special content types first
+            # This is important for things like YouTube URLs that don't have a file
+            content_type, parameter, _ = self.content_type_agent.identify_content_type(question, "", task_id)
+            
+            # Handle YouTube URLs in general questions
+            if content_type == "youtube":
+                self.last_used_tool = "youtube"
+                youtube_url = parameter if parameter.startswith("http") else question
+                result = self.youtube_tool._run(youtube_url, question=question)
+                
+                # Use specialized prompt for YouTube analysis
+                youtube_analysis_prompt = f"""
+                Analyze this YouTube video and provide an extremely concise answer:
+
+                Question: {question}
+
+                YouTube Video:
+                {result}
+
+                Instructions:
+                1. Pay careful attention to the specific format requested in the question
+                2. Extract only the information needed to answer the question
+                
+                When answering, provide ONLY the precise answer requested. 
+                Do not include explanations, steps, reasoning, or additional text.
+                Be direct and specific. GAIA benchmark requires exact matching answers.
+                For example, if asked "What is the color of the sky?", respond simply with "blue".
+                """
+
+                response = self.llm.invoke(youtube_analysis_prompt)
+                return response.content if hasattr(response, 'content') else str(response)
+            
             # For general questions (no files), use improved search strategy
             question_lower = question.lower()
             answer = None
             
             # Check for Wikipedia specific questions first
             if "wikipedia" in question_lower:
+                self.last_used_tool = "wiki"
                 question = self._format_question(question)
                 wiki_result = self.wikipedia_tool._run(question)
                 answer = self._generate_answer_from_context(question, wiki_result)
@@ -502,6 +567,7 @@ class MainAgent:
                     return answer
             
             # Use general web search
+            self.last_used_tool = "web"
             query = self._format_question(question)
             web_result = self.web_search_tool._run(query)
             answer = self._generate_answer_from_context(question, web_result)
@@ -511,6 +577,7 @@ class MainAgent:
             
             # If no good answer from web search, try with Wikipedia as a last resource
             if "wikipedia" not in question_lower:  # Only if not already tried
+                self.last_used_tool = "wiki"
                 question = self._format_question(question)
                 wiki_result = self.wikipedia_tool._run(question)
                 answer = self._generate_answer_from_context(question, wiki_result)
@@ -518,6 +585,7 @@ class MainAgent:
                     return answer
             
             # If we still don't have a good answer, use the general tools approach
+            self.last_used_tool = "general"
             answer = self._get_answer_using_tools(question)
             
             return answer

tools.py

@@ -5,9 +5,9 @@ import requests
 from langchain.tools import BaseTool
 from langchain.schema import Document
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.tools import WikipediaQueryRun, DuckDuckGoSearchRun
+from langchain_community.tools import WikipediaQueryRun, DuckDuckGoSearchResults
 from langchain_community.document_loaders import PythonLoader
-from langchain_community.utilities import WikipediaAPIWrapper
+from langchain_community.utilities import WikipediaAPIWrapper, DuckDuckGoSearchAPIWrapper
 import pytube
 from PIL import Image
 import pandas as pd
@@ -154,6 +154,8 @@ class YouTubeVideoTool(BaseContentTool):
                 for entry in transcript_list
             ])
             
+            print(f"Transcript text: {transcript_text}")
+            
             return transcript_text
             
         except Exception as e:
@@ -696,14 +698,120 @@ class WebSearchTool(BaseTool):
     """Tool for web search using DuckDuckGo"""
     name: str = "web_search"
     description: str = "Search the web for information. Useful for questions about current events, specific facts, or topics not covered in Wikipedia."
-    search_tool: DuckDuckGoSearchRun = Field(default_factory=DuckDuckGoSearchRun)
+    search_tool: DuckDuckGoSearchResults = Field(default_factory=DuckDuckGoSearchResults)
+    
+    def _extract_links_from_results(self, search_result: str) -> list:
+        """Extract links from search results using string splitting"""
+        links = []
+        try:
+            # Split by 'link:' and process each part except the first one
+            parts = search_result.split('link:')
+            
+            # Skip the first part (before the first 'link:')
+            for part in parts[1:]:
+                # Get the URL by splitting at the first comma
+                url = part.split(',')[0].strip()
+                if url.startswith('http'):
+                    links.append(url)
+            
+            # Add debug output
+        except Exception as e:
+            print(f"Error extracting links: {str(e)}")
+            
+        return links
+    
+    def _is_promising_link(self, link: str, query: str) -> bool:
+        """Determine if a link is promising based on the query"""
+        query_terms = set(query.lower().split())
+        
+        # Exclude common non-content sites
+        excluded_domains = [
+            'youtube.com', 'facebook.com', 'twitter.com', 'instagram.com',
+            'pinterest.com', 'reddit.com', 'tiktok.com', 'linkedin.com'
+        ]
+        
+        for domain in excluded_domains:
+            if domain in link:
+                return False
+                
+        # Prefer certain credible domains
+        preferred_domains = [
+            'wikipedia.org', 'britannica.com', 'scholarpedia.org', 
+            '.edu', '.gov', '.org'
+        ]
+        
+        for domain in preferred_domains:
+            if domain in link:
+                return True
+                
+        return True  # Default to True to allow scraping
+    
+    def _scrape_page_content(self, url: str) -> str:
+        """Scrape the content of a webpage"""
+        try:
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+            }
+            response = requests.get(url, headers=headers, timeout=10)
+            response.raise_for_status()
+            
+            # Check if we got HTML content
+            content_type = response.headers.get('Content-Type', '')
+            if 'text/html' not in content_type:
+                return ""
+                
+            # Use BeautifulSoup to parse the HTML
+            from bs4 import BeautifulSoup
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            # Remove script and style elements
+            for script in soup(["script", "style", "nav", "footer", "header"]):
+                script.decompose()
+                
+            # Extract text content
+            text = soup.get_text(separator=' ', strip=True)
+            
+            # Clean up the text
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            text = '\n'.join(chunk for chunk in chunks if chunk)
+            
+            # Limit the length
+            return text[:5000]  # Limit to 5000 chars
+            
+        except Exception as e:
+            return f"Error scraping page content {str(e)}"
 
     def _run(self, query: str) -> str:
         """Search the web and return results as a string"""
         try:
-            search_result = self.search_tool.invoke(query)
+            # First perform the DuckDuckGo search
+            search_result = self.search_tool.run(query, max_results=5)
+            
+            # Extract links from the search results
+            links = self._extract_links_from_results(search_result)
+            
+            # Process up to 3 promising links
+            additional_content = []
+            processed_count = 0
+            
+            for link in links:
+                if processed_count >= 3:
+                    break
+                    
+                if self._is_promising_link(link, query):
+                    content = self._scrape_page_content(link)
+                    if content:
+                        additional_content.append(f"Additional content from {link}:\n{content}\n")
+                        processed_count += 1
+            
+            # Combine the search results with the additional content
+            combined_result = search_result
+            
+            if additional_content:
+                combined_result += "\n\n" + "\n\n".join(additional_content)
             
-            return search_result
+            return combined_result
             
         except Exception as e:
             return f"Error searching the web: {str(e)}"