Update app.py

app.py

@@ -5,7 +5,8 @@ import json
 import httpx
 import os
 import logging
-from typing import Dict, List, Optional, Tuple
+from typing import Optional, List, Dict, Tuple
+from itertools import cycle
 from datetime import datetime
 from bs4 import BeautifulSoup
 from googlesearch import search
@@ -25,8 +26,6 @@ logger = logging.getLogger(__name__)
 
 class RaindropSearchBot:
     def __init__(self):
-        self.min_delay = 2  # Minimum delay in seconds
-        self.max_delay = 5  # Maximum delay in seconds
         self.openai_api_key = os.getenv('openaikey')
         self.raindrop_api_token = os.getenv('raindroptoken')
         self.newsapi_key = os.getenv('newsapikey')
@@ -45,35 +44,102 @@ class RaindropSearchBot:
             )
         )
         self.newsapi = NewsApiClient(api_key=self.newsapi_key)
-
+        self.min_delay = 5  # Increased minimum delay
+        self.max_delay = 15  # Increased maximum delay
+        self.ua = UserAgent()
+        self.setup_proxies()
+
+    def get_alternative_search_results(self, query: str) -> List[Dict]:
+        """Implement alternative search engine if Google fails"""
+        # Example using DuckDuckGo (you'll need to install duckduckgo-search)
+        try:
+            from duckduckgo_search import ddg
+            
+            self.random_delay()
+            results = ddg(query, max_results=5)
+            
+            return [{
+                'title': result['title'],
+                'link': result['link'],
+                'snippet': result['snippet']
+            } for result in results]
+            
+        except Exception as e:
+            logger.error(f"Alternative search failed: {e}")
+            return []
+    
+    def search_with_fallback(self, query: str) -> List[Dict]:
+        """Search with fallback to alternative search engines"""
+        try:
+            return self.get_google_results(query)
+        except Exception as e:
+            logger.warning(f"Google search failed: {e}")
+            try:
+                # Implement alternative search engine here
+                # For example: DuckDuckGo, Bing, etc.
+                return self.get_alternative_search_results(query)
+            except Exception as e:
+                logger.error(f"All search attempts failed: {e}")
+                return []
+    
+    def setup_proxies(self):
+        """Setup proxy rotation"""
+        # Free proxy list - replace with your paid proxy service for better reliability
+        self.proxies = [
+            'http://proxy1.example.com:8080',
+            'http://proxy2.example.com:8080',
+            # Add more proxies here
+        ]
+        self.proxy_cycle = cycle(self.proxies)
+    
     def random_delay(self):
-        """Add a random delay between requests"""
-        delay = random.uniform(self.min_delay, self.max_delay)
+        """Enhanced random delay with jitter"""
+        base_delay = random.uniform(self.min_delay, self.max_delay)
+        jitter = random.uniform(-1, 1)  # Add/subtract up to 1 second
+        delay = max(0, base_delay + jitter)
         time.sleep(delay)
 
-    @retry(wait=wait_exponential(multiplier=1, min=4, max=10),
-           stop=stop_after_attempt(3))
-
     def get_google_results(self, query: str, num_results: int = 5) -> List[Dict]:
-        """Get Google search results using googlesearch-python with retry and delay."""
+        """Get Google search results with improved handling"""
         try:
             search_results = []
-            for result in search(query, num_results=num_results, advanced=True):
-                search_results.append({
-                    'title': result.title,
-                    'link': result.url,
-                    'snippet': result.description
-                })
-                # Add random delay between each search result
+            session = self.create_session()
+            
+            # Break the search into smaller chunks
+            chunk_size = 3
+            for i in range(0, num_results, chunk_size):
+                # Add substantial random delay between chunks
                 self.random_delay()
+                
+                try:
+                    chunk_results = list(search(
+                        query,
+                        num_results=min(chunk_size, num_results - i),
+                        advanced=True,
+                        lang="en",
+                        sleep_interval=random.uniform(5, 10),  # Random delay between requests
+                        timeout=30
+                    ))
+                    
+                    for result in chunk_results:
+                        search_results.append({
+                            'title': result.title,
+                            'link': result.url,
+                            'snippet': result.description
+                        })
+                        
+                    # Add random delay between chunks
+                    time.sleep(random.uniform(8, 15))
+                    
+                except Exception as e:
+                    logger.warning(f"Error in search chunk {i}: {e}")
+                    continue
+                    
             return search_results
                 
         except Exception as e:
             logger.error(f"Google search error: {e}")
-            return []
-
-    @retry(wait=wait_exponential(multiplier=1, min=4, max=10),
-           stop=stop_after_attempt(3))
+            raise
 
     def get_news_results(self, query: str, num_results: int = 5) -> List[Dict]:
         """Get news articles using NewsAPI with retry and delay."""
@@ -150,16 +216,34 @@ class RaindropSearchBot:
             return None
 
     def get_random_user_agent(self) -> str:
-        """Return a random user agent string to avoid detection."""
-        user_agents = [
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-        ]
-        return random.choice(user_agents)
+        """Get random user agent using fake-useragent"""
+        return self.ua.random
+
+    def create_session(self) -> requests.Session:
+        """Create a session with random user agent and proxy"""
+        session = requests.Session()
+        session.headers.update({
+            'User-Agent': self.get_random_user_agent(),
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Cache-Control': 'max-age=0'
+        })
+        session.proxies = self.get_next_proxy()
+        return session
+
+    @retry(
+        wait=wait_exponential(multiplier=1, min=4, max=20),
+        stop=stop_after_attempt(3),
+        reraise=True
+    )
     
     def get_content_and_summary(self, request: str, item: Dict, source_type: str) -> Dict:
         """Get content and generate summary for a single item."""
@@ -266,14 +350,14 @@ class RaindropSearchBot:
             logger.error(f"Search error: {e}")
             return []
 
-    def process_all_results(self, raindrop_results: List[Dict], 
+    def process_all_results(self, request, raindrop_results: List[Dict], 
                           google_results: List[Dict], 
                           news_results: List[Dict]) -> Tuple[List[Dict], List[Dict], List[Dict]]:
         """Process and enrich all results with content and summaries."""
         
         processed_raindrop = []
         for item in raindrop_results:
-            processed_item = self.get_content_and_summary(item, 'raindrop')
+            processed_item = self.get_content_and_summary(request, item, 'raindrop')
             if processed_item.get('detailed_summary'):
                 processed_raindrop.append(processed_item)
             # Add delay between processing items
@@ -281,7 +365,7 @@ class RaindropSearchBot:
 
         processed_google = []
         for item in google_results:
-            processed_item = self.get_content_and_summary(item, 'google')
+            processed_item = self.get_content_and_summary(request, item, 'google')
             if processed_item.get('detailed_summary'):
                 processed_google.append(processed_item)
             # Add delay between processing items
@@ -289,7 +373,7 @@ class RaindropSearchBot:
 
         processed_news = []
         for item in news_results:
-            processed_item = self.get_content_and_summary(item, 'news')
+            processed_item = self.get_content_and_summary(request, item, 'news')
             if processed_item.get('detailed_summary'):
                 processed_news.append(processed_item)
             # Add delay between processing items
@@ -446,7 +530,7 @@ class RaindropSearchBot:
             
             # Process all results to get content and summaries
             processed_results = self.process_all_results(
-                raindrop_results, google_results, news_results
+                user_request, raindrop_results, google_results, news_results
             )
             
             # Generate essay-style analysis