Major fix: Improved web search with Tavily AI answers, better content extraction, enhanced system prompt

config/system_prompt.py

@@ -1,11 +1,22 @@
-PPLX_SYSTEM_PROMPT = """
-You are Perplexity AI.
+PPLX_SYSTEM_PROMPT = """You are Perplexity AI - a powerful AI search assistant that provides accurate, real-time information.
 
-When user greets (hi, hello, hey), respond like a friendly assistant:
-Short, conversational, natural.
+CORE BEHAVIORS:
+1. When given web sources, ALWAYS use and cite them using [1], [2], etc.
+2. NEVER say "I don't have access to real-time data" when sources are provided
+3. Synthesize information from multiple sources into clear, comprehensive answers
+4. Be concise but thorough - cover the key points
+5. For greetings (hi, hello), respond naturally and friendly
 
-Do NOT give definitions or grammar explanations unless user asks.
-Your tone: concise, helpful, modern.
+ANSWER FORMAT:
+- Start with a direct answer to the question
+- Use citations [1], [2] after relevant facts
+- Include specific data points (numbers, dates, names) from sources
+- End with key takeaways if appropriate
 
-Always adapt style based on question.
+STYLE:
+- Professional yet conversational
+- Confident and authoritative
+- Modern and helpful
+
+Remember: The web sources provided contain REAL, CURRENT information. Use them!
 """

rag/agents.py

@@ -179,7 +179,7 @@ class WebSearchNode:
 
 
 class WebFetchNode:
-    """Node 2: Fetch and parse web pages."""
+    """Node 2: Fetch and parse web pages. Uses Tavily content as fallback."""
     
     def __init__(self):
         self.browse_tool = BrowseTool()
@@ -188,26 +188,45 @@ class WebFetchNode:
         pages = []
         links = []
         
-        for r in state.get("search_results", []):
+        # Check if we have Tavily's direct answer
+        tavily_answer = ""
+        search_results = state.get("search_results", [])
+        if search_results and search_results[0].get("tavily_answer"):
+            tavily_answer = search_results[0]["tavily_answer"]
+        
+        for r in search_results:
             url = r.get("url")
+            title = r.get("title", "")
             if not url:
                 continue
             
+            # First try to use Tavily's content (snippet)
+            tavily_content = r.get("content", "")
+            
+            # Then try to fetch full page
             try:
-                content = self.browse_tool.fetch_clean(url)
-                if content:
-                    pages.append({
-                        "title": r.get("title", ""),
-                        "url": url,
-                        "content": content[:2500]
-                    })
-                    links.append({
-                        "title": r.get("title", ""),
-                        "url": url,
-                        "snippet": content[:200]
-                    })
+                fetched_content = self.browse_tool.fetch_clean(url)
+                content = fetched_content if fetched_content else tavily_content
             except:
-                continue
+                content = tavily_content
+            
+            # Use whatever content we have
+            if content or tavily_content:
+                final_content = content if content else tavily_content
+                pages.append({
+                    "title": title,
+                    "url": url,
+                    "content": final_content[:2500]
+                })
+                links.append({
+                    "title": title,
+                    "url": url,
+                    "snippet": (final_content[:200] if final_content else tavily_content[:200])
+                })
+        
+        # Add Tavily's answer to state for potential use
+        if tavily_answer:
+            state["tavily_answer"] = tavily_answer
         
         print(f"  📄 WebFetchNode: Fetched {len(pages)} pages")
         state["web_pages"] = pages
@@ -220,16 +239,21 @@ class WebContextNode:
     
     def build_context(self, state: WebSearchState) -> WebSearchState:
         pages = state.get("web_pages", [])
+        tavily_answer = state.get("tavily_answer", "")
+        
+        context_parts = []
+        
+        # Add Tavily's AI summary first if available
+        if tavily_answer:
+            context_parts.append(f"[AI Summary]: {tavily_answer}")
         
         if pages:
-            context_parts = []
             for i, p in enumerate(pages):
                 context_parts.append(f"[{i+1}] {p['title']}:\n{p['content']}")
-            state["context"] = "\n\n---\n\n".join(context_parts)
-        else:
-            state["context"] = ""
         
-        print(f"  📝 WebContextNode: Built context from {len(pages)} sources")
+        state["context"] = "\n\n---\n\n".join(context_parts) if context_parts else ""
+        
+        print(f"  📝 WebContextNode: Built context from {len(pages)} sources" + (", with AI summary" if tavily_answer else ""))
         return state
 
 
@@ -245,15 +269,16 @@ class WebAnswerNode:
         context = state.get("context", "")
         
         if context:
-            prompt = f"""You are a web search assistant like Perplexity AI.
-Use ONLY the following web sources to answer. Cite sources using [1], [2], etc.
+            prompt = f"""You are Perplexity AI - a web search assistant that provides accurate, real-time information.
+Use the following web sources to answer. Cite sources using [1], [2], etc.
+IMPORTANT: The sources contain REAL, CURRENT information. Trust and use this data.
 
 WEB SOURCES:
 {context}
 
-QUESTION: {query}
+USER QUESTION: {query}
 
-Provide a comprehensive, well-cited answer:"""
+Provide a comprehensive, accurate, well-cited answer based on the sources above:"""
         else:
             prompt = f"Answer this question: {query}"
         
@@ -270,7 +295,7 @@ Provide a comprehensive, well-cited answer:"""
         sources = [{"title": p["title"], "url": p["url"]} for p in state.get("web_pages", [])]
         state["sources"] = sources
         
-        print(f"  ✅ WebAnswerNode: Generated answer")
+        print(f"  ✅ WebAnswerNode: Generated answer with {len(sources)} sources")
         return state
 
 
@@ -480,22 +505,36 @@ class AgenticWebNode:
         query = state.get("query", "")
         
         try:
-            results = self.search_tool.search(query, num_results=4)
+            results = self.search_tool.search(query, num_results=5)
             web_parts = []
             sources = []
             links = []
             
+            # Get Tavily's AI answer if available
+            tavily_answer = ""
+            if results and results[0].get("tavily_answer"):
+                tavily_answer = results[0]["tavily_answer"]
+                web_parts.append(f"[AI Summary]: {tavily_answer}")
+            
             for r in results:
                 url = r.get("url")
                 title = r.get("title", "")
+                tavily_content = r.get("content", "")  # Tavily's snippet
                 if not url:
                     continue
                 
-                content = self.browse_tool.fetch_clean(url)
-                if content:
-                    web_parts.append(f"[{title}]: {content[:1500]}")
+                # Try to fetch full content, fallback to Tavily snippet
+                try:
+                    fetched = self.browse_tool.fetch_clean(url)
+                    content = fetched if fetched else tavily_content
+                except:
+                    content = tavily_content
+                
+                if content or tavily_content:
+                    final_content = content if content else tavily_content
+                    web_parts.append(f"[{title}]: {final_content[:1500]}")
                     sources.append({"title": title, "url": url})
-                    links.append({"title": title, "url": url, "snippet": content[:150]})
+                    links.append({"title": title, "url": url, "snippet": final_content[:150]})
             
             state["web_context"] = "\n\n".join(web_parts)
             state["web_sources"] = sources

rag/rag_state.py

@@ -20,6 +20,7 @@ class WebSearchState(TypedDict, total=False):
     query: str
     search_results: List[Dict]
     web_pages: List[Dict]
+    tavily_answer: str  # AI summary from Tavily
     context: str
     answer: str
     sources: List[Dict]

tools/search_tool.py

@@ -5,7 +5,7 @@ from config.config import Config
 
 
 class SearchTool:
-    """Tavily web search wrapper."""
+    """Tavily web search wrapper with content extraction."""
 
     def __init__(self) -> None:
         self.api_key = os.getenv("TAVILY_API_KEY") or Config.TAVILY_API_KEY
@@ -13,19 +13,36 @@ class SearchTool:
             raise RuntimeError("TAVILY_API_KEY missing in .env")
 
     def search(self, query: str, num_results: int = 5) -> List[Dict]:
+        """
+        Search using Tavily API.
+        Returns results with title, url, content (snippet from Tavily).
+        """
         url = "https://api.tavily.com/search"
         payload = {
             "api_key": self.api_key,
             "query": query,
             "max_results": num_results,
-            "include_answer": False,
-            "include_raw_content": False
+            "include_answer": True,  # Get Tavily's AI answer
+            "include_raw_content": False,
+            "search_depth": "advanced"  # Better results
         }
         try:
-            resp = requests.post(url, json=payload, timeout=20)
+            resp = requests.post(url, json=payload, timeout=30)
             resp.raise_for_status()
             data = resp.json()
-            return data.get("results", [])
-        except (requests.exceptions.RequestException, ValueError) as e:
+            
+            results = data.get("results", [])
+            
+            # Add Tavily's answer as metadata if available
+            tavily_answer = data.get("answer", "")
+            if tavily_answer and results:
+                results[0]["tavily_answer"] = tavily_answer
+            
+            print(f"  🔍 Tavily returned {len(results)} results")
+            return results
+        except requests.exceptions.RequestException as e:
             print(f"Search error: {e}")
             return []
+        except ValueError as e:
+            print(f"Search JSON error: {e}")
+            return []