Update app.py

app.py

@@ -1,16 +1,19 @@
+python
+
 import gradio as gr
 from openai import OpenAI
 import requests
 import json
 import os
 import logging
-from typing import Dict, List, Tuple, Optional
-from newspaper import Article
-import markdown
+from typing import Dict, List, Optional, Tuple
 from datetime import datetime
 from bs4 import BeautifulSoup
 from googlesearch import search
 from newsapi import NewsApiClient
+import markdown
+import re
+import time
 
 # Set up logging
 logging.basicConfig(
@@ -23,10 +26,9 @@ class RaindropSearchBot:
     def __init__(self):
         self.openai_api_key = os.getenv('openaikey')
         self.raindrop_api_token = os.getenv('raindroptoken')
-        self.serpapi_key = os.getenv('serpapikey')
         self.newsapi_key = os.getenv('newsapikey')
         
-        if not all([self.openai_api_key, self.raindrop_api_token, self.serpapi_key, self.newsapi_key]):
+        if not all([self.openai_api_key, self.raindrop_api_token, self.newsapi_key]):
             raise EnvironmentError(
                 "Missing required environment variables. Please ensure all API keys are set."
             )
@@ -34,6 +36,38 @@ class RaindropSearchBot:
         self.client = OpenAI(api_key=self.openai_api_key)
         self.newsapi = NewsApiClient(api_key=self.newsapi_key)
 
+    def get_google_results(self, query: str, num_results: int = 5) -> List[Dict]:
+        """Get Google search results using googlesearch-python."""
+        try:
+            search_results = []
+            for result in search(query, num_results=num_results, advanced=True):
+                search_results.append({
+                    'title': result.title,
+                    'link': result.url,
+                    'snippet': result.description
+                })
+            return search_results
+                
+        except Exception as e:
+            logger.error(f"Google search error: {e}")
+            return []
+
+    def get_news_results(self, query: str, num_results: int = 5) -> List[Dict]:
+        """Get news articles using NewsAPI."""
+        try:
+            news_results = self.newsapi.get_everything(
+                q=query,
+                language='en',
+                sort_by='relevancy',
+                page_size=num_results
+            )
+            
+            return news_results.get('articles', [])
+            
+        except Exception as e:
+            logger.error(f"News API error: {e}")
+            return []
+
     def extract_content_from_url(self, url: str) -> Optional[str]:
         """Extract main content from a URL using BeautifulSoup."""
         try:
@@ -84,18 +118,18 @@ class RaindropSearchBot:
             url = item.get('link') or item.get('url')
             if not url:
                 return item
-    
+
             # For Raindrop items, use existing excerpt if available
             if source_type == 'raindrop' and item.get('excerpt'):
                 content = item['excerpt']
             else:
                 content = self.extract_content_from_url(url)
-    
+
             if not content:
                 logger.warning(f"No content extracted from {url}")
                 item['detailed_summary'] = "Content extraction failed."
                 return item
-    
+
             # Generate summary focused on the query topic
             try:
                 prompt = f"""
@@ -110,7 +144,7 @@ class RaindropSearchBot:
                 4. Keep the summary to 2-3 paragraphs
                 5. Highlight any unique insights from this source
                 """
-    
+
                 response = self.client.chat.completions.create(
                     model="gpt-4o-mini",
                     messages=[{"role": "user", "content": prompt}],
@@ -124,77 +158,12 @@ class RaindropSearchBot:
             except Exception as e:
                 logger.error(f"Error generating summary: {e}")
                 item['detailed_summary'] = "Summary generation failed."
-    
+
             return item
             
         except Exception as e:
             logger.error(f"Error processing item: {e}")
             return item
-    
-    def get_google_results(self, query: str, num_results: int = 5) -> List[Dict]:
-        """Get Google search results using googlesearch-python."""
-        try:
-            search_results = []
-            for result in search(query, num_results=num_results, advanced=True):
-                search_results.append({
-                    'title': result.title,
-                    'link': result.url,
-                    'snippet': result.description
-                })
-            return search_results
-                
-        except Exception as e:
-            logger.error(f"Google search error: {e}")
-            return []
-
-    def get_news_results(self, query: str, num_results: int = 5) -> List[Dict]:
-        """Get news articles using NewsAPI."""
-        try:
-            news_results = self.newsapi.get_everything(
-                q=query,
-                language='en',
-                sort_by='relevancy',
-                page_size=num_results
-            )
-            
-            return news_results.get('articles', [])
-            
-        except Exception as e:
-            logger.error(f"News API error: {e}")
-            return []
-
-    def generate_search_query(self, user_request: str) -> str:
-        """Convert user request to optimized search terms."""
-        logger.info(f"Generating search query for: {user_request}")
-        
-        prompt = f"""
-        You are a search expert. Create a search query to find relevant documents about:
-        {user_request}
-        
-        Guidelines:
-        - Focus on key concepts and synonyms
-        - Use simple keywords that would appear in titles or descriptions
-        - Avoid complex operators or special characters
-        - Return only the search terms, no explanation
-        - Include alternative phrasings
-        - Keep it concise (max 3-4 key terms/phrases)
-        
-        Return only the search query terms.
-        """
-
-        try:
-            response = self.client.chat.completions.create(
-                model="gpt-4o-mini",
-                messages=[{"role": "user", "content": prompt}],
-                temperature=0.3,
-                max_tokens=50
-            )
-            search_query = response.choices[0].message.content.strip()
-            logger.info(f"Generated search query: {search_query}")
-            return search_query
-        except Exception as e:
-            logger.error(f"Error generating search query: {e}")
-            return user_request
 
     def search_raindrop(self, search_query: str) -> List[Dict]:
         """Search Raindrop.io with enhanced error handling and logging."""
@@ -244,122 +213,6 @@ class RaindropSearchBot:
             logger.error(f"Search error: {e}")
             return []
 
-    def analyze_results(self, results: List[Dict], google_results: List[Dict], 
-                       news_results: List[Dict], user_query: str) -> str:
-        """Generate an analysis of all search results."""
-        if not any([results, google_results, news_results]):
-            return "No relevant results found. Try modifying your search terms."
-
-        # Create context for analysis
-        context = f"Based on the search query: '{user_query}'\n\n"
-        context += "Analyze these sources:\n\n"
-        
-        # Add Raindrop results
-        if results:
-            context += "Bookmarked Sources:\n"
-            for item in results:
-                context += f"Title: {item.get('title', 'No title')}\n"
-                if item.get('excerpt'):
-                    context += f"Content: {item['excerpt'][:500]}...\n"
-                context += f"Created: {item.get('created', 'No date')}\n\n"
-
-        # Add Google results
-        if google_results:
-            context += "Web Sources:\n"
-            for item in google_results:
-                context += f"Title: {item.get('title', 'No title')}\n"
-                if item.get('snippet'):
-                    context += f"Content: {item['snippet']}\n\n"
-
-        # Add News results
-        if news_results:
-            context += "News Sources:\n"
-            for item in news_results:
-                context += f"Title: {item.get('title', 'No title')}\n"
-                if item.get('description'):
-                    context += f"Content: {item['description']}\n"
-                context += f"Published: {item.get('publishedAt', 'No date')}\n\n"
-
-        try:
-            prompt = f"""
-            Based on all available sources, provide a comprehensive analysis of {user_query}.
-            
-            Requirements:
-            1. Focus on the most relevant and recent information
-            2. Organize the response by key themes
-            3. Include only factual information from the sources
-            4. Highlight any significant developments or changes
-            5. Note any conflicts or differences between sources
-            6. Prioritize official sources and recent developments
-            
-            Context:
-            {context}
-            """
-
-            response = self.client.chat.completions.create(
-                model="gpt-4o-mini",
-                messages=[{"role": "user", "content": prompt}],
-                temperature=0.5,
-                max_tokens=1000
-            )
-            
-            analysis = response.choices[0].message.content
-            return analysis
-        except Exception as e:
-            logger.error(f"Analysis generation error: {e}")
-            return "Error generating analysis."
-
-    def format_results(self, results: Tuple[List[Dict], List[Dict], List[Dict]], 
-                      essay: str) -> str:
-        """Format the essay and results with detailed summaries."""
-        raindrop_results, google_results, news_results = results
-        
-        output = f"{essay}\n\n"
-        output += "---\n\n"
-        output += "# References and Detailed Summaries\n\n"
-        
-        ref_counter = 1
-
-        # Format Raindrop results
-        if raindrop_results:
-            output += "## 🔍 Bookmarked Sources\n\n"
-            for item in raindrop_results:
-                output += f"### [{ref_counter}] {item.get('title', 'No Title')}\n"
-                output += f"**Link**: {item.get('link')}\n"
-                if item.get('tags'):
-                    output += f"**Tags**: {', '.join(item['tags'])}\n"
-                if item.get('created'):
-                    output += f"**Created**: {item['created'][:10]}\n"
-                output += "\n**Summary**:\n"
-                output += f"{item.get('detailed_summary', 'No summary available.')}\n\n"
-                ref_counter += 1
-
-        # Format Google results
-        if google_results:
-            output += "## 🌐 Web Sources\n\n"
-            for item in google_results:
-                output += f"### [{ref_counter}] {item.get('title', 'No Title')}\n"
-                output += f"**Link**: {item.get('link')}\n"
-                output += "\n**Summary**:\n"
-                output += f"{item.get('detailed_summary', 'No summary available.')}\n\n"
-                ref_counter += 1
-
-        # Format News results
-        if news_results:
-            output += "## 📰 Recent News\n\n"
-            for item in news_results:
-                output += f"### [{ref_counter}] {item.get('title', 'No Title')}\n"
-                output += f"**Link**: {item.get('url')}\n"
-                if item.get('source', {}).get('name'):
-                    output += f"**Source**: {item['source']['name']}\n"
-                if item.get('publishedAt'):
-                    output += f"**Published**: {item['publishedAt'][:10]}\n"
-                output += "\n**Summary**:\n"
-                output += f"{item.get('detailed_summary', 'No summary available.')}\n\n"
-                ref_counter += 1
-
-        return output
-
     def process_all_results(self, raindrop_results: List[Dict], 
                           google_results: List[Dict], 
                           news_results: List[Dict]) -> Tuple[List[Dict], List[Dict], List[Dict]]:
@@ -439,7 +292,58 @@ class RaindropSearchBot:
         except Exception as e:
             logger.error(f"Error generating essay: {e}")
             return "Error generating analysis."
-    
+
+    def format_results(self, results: Tuple[List[Dict], List[Dict], List[Dict]], 
+                      essay: str) -> str:
+        """Format the essay and results with detailed summaries."""
+        raindrop_results, google_results, news_results = results
+        
+        output = f"{essay}\n\n"
+        output += "---\n\n"
+        output += "# References and Detailed Summaries\n\n"
+        
+        ref_counter = 1
+
+        # Format Raindrop results
+        if raindrop_results:
+            output += "## 🔍 Bookmarked Sources\n\n"
+            for item in raindrop_results:
+                output += f"### [{ref_counter}] {item.get('title', 'No Title')}\n"
+                output += f"**Link**: {item.get('link')}\n"
+                if item.get('tags'):
+                    output += f"**Tags**: {', '.join(item['tags'])}\n"
+                if item.get('created'):
+                    output += f"**Created**: {item['created'][:10]}\n"
+                output += "\n**Summary**:\n"
+                output += f"{item.get('detailed_summary', 'No summary available.')}\n\n"
+                ref_counter += 1
+
+        # Format Google results
+        if google_results:
+            output += "## 🌐 Web Sources\n\n"
+            for item in google_results:
+                output += f"### [{ref_counter}] {item.get('title', 'No Title')}\n"
+                output += f"**Link**: {item.get('link')}\n"
+                output += "\n**Summary**:\n"
+                output += f"{item.get('detailed_summary', 'No summary available.')}\n\n"
+                ref_counter += 1
+
+        # Format News results
+        if news_results:
+            output += "## 📰 Recent News\n\n"
+            for item in news_results:
+                output += f"### [{ref_counter}] {item.get('title', 'No Title')}\n"
+                output += f"**Link**: {item.get('url')}\n"
+                if item.get('source', {}).get('name'):
+                    output += f"**Source**: {item['source']['name']}\n"
+                if item.get('publishedAt'):
+                    output += f"**Published**: {item['publishedAt'][:10]}\n"
+                output += "\n**Summary**:\n"
+                output += f"{item.get('detailed_summary', 'No summary available.')}\n\n"
+                ref_counter += 1
+
+        return output
+
     def process_request(self, user_request: str) -> str:
         """Process the user request with enhanced content collection and analysis."""
         try:
@@ -469,6 +373,39 @@ class RaindropSearchBot:
             logger.error(f"Error processing request: {e}", exc_info=True)
             return f"An error occurred while processing your request. Please try again."
 
+    def generate_search_query(self, user_request: str) -> str:
+        """Convert user request to optimized search terms."""
+        logger.info(f"Generating search query for: {user_request}")
+        
+        prompt = f"""
+        You are a search expert. Create a search query to find relevant documents about:
+        {user_request}
+        
+        Guidelines:
+        - Focus on key concepts and synonyms
+        - Use simple keywords that would appear in titles or descriptions
+        - Avoid complex operators or special characters
+        - Return only the search terms, no explanation
+        - Include alternative phrasings
+        - Keep it concise (max 3-4 key terms/phrases)
+        
+        Return only the search query terms.
+        """
+
+        try:
+            response = self.client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0.3,
+                max_tokens=50
+            )
+            search_query = response.choices[0].message.content.strip()
+            logger.info(f"Generated search query: {search_query}")
+            return search_query
+        except Exception as e:
+            logger.error(f"Error generating search query: {e}")
+            return user_request
+
 # Initialize bot
 bot = RaindropSearchBot()
 
@@ -511,4 +448,4 @@ with gr.Blocks(title="Enhanced Search Assistant", theme=gr.themes.Soft()) as dem
 
 # Launch the interface
 if __name__ == "__main__":
-    demo.launch(share=True)
+    demo.launch(share=True)