Update app.py

app.py

@@ -62,22 +62,36 @@ class SearchEngineInterface:
     def __init__(self):
         self.session = None
         self.headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5',
-            'Accept-Encoding': 'gzip, deflate',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Encoding': 'gzip, deflate, br',
             'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Cache-Control': 'max-age=0',
         }
     
     async def get_session(self):
-        """Get or create aiohttp session"""
-        if self.session is None:
-            connector = aiohttp.TCPConnector(limit=10)
-            timeout = aiohttp.ClientTimeout(total=30)
+        """Get or create aiohttp session with better configuration"""
+        if self.session is None or self.session.closed:
+            connector = aiohttp.TCPConnector(
+                limit=20,
+                limit_per_host=5,
+                ttl_dns_cache=300,
+                use_dns_cache=True,
+                keepalive_timeout=30,
+                enable_cleanup_closed=True
+            )
+            timeout = aiohttp.ClientTimeout(total=45, connect=15, sock_read=30)
             self.session = aiohttp.ClientSession(
                 headers=self.headers,
                 connector=connector,
-                timeout=timeout
+                timeout=timeout,
+                trust_env=True
             )
         return self.session
     
@@ -209,57 +223,224 @@ class SearchEngineInterface:
             return []
     
     async def close(self):
-        """Close the session"""
-        if self.session:
+        """Close the session safely"""
+        if self.session and not self.session.closed:
             await self.session.close()
+            # Wait a bit for the underlying connections to close
+            await asyncio.sleep(0.1)
 
 class ContentScraper:
-    """Scrape and parse article content using newspaper3k"""
+    """Scrape and parse article content using newspaper3k with robust error handling"""
     
     def __init__(self):
         self.session = None
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'cross-site',
+            'Sec-Fetch-User': '?1',
+            'Cache-Control': 'no-cache',
+            'Pragma': 'no-cache'
+        }
+        # Domains known to block scrapers - we'll handle these differently
+        self.blocked_domains = {
+            'bloomberg.com', 'wsj.com', 'ft.com', 'nytimes.com', 
+            'washingtonpost.com', 'economist.com', 'reuters.com'
+        }
     
     async def get_session(self):
-        """Get or create aiohttp session"""
-        if self.session is None:
-            connector = aiohttp.TCPConnector(limit=20)
-            timeout = aiohttp.ClientTimeout(total=30)
+        """Get or create aiohttp session with robust configuration"""
+        if self.session is None or self.session.closed:
+            connector = aiohttp.TCPConnector(
+                limit=30,
+                limit_per_host=10,
+                ttl_dns_cache=300,
+                use_dns_cache=True,
+                keepalive_timeout=60,
+                enable_cleanup_closed=True,
+                ssl=False  # Disable SSL verification for problematic sites
+            )
+            timeout = aiohttp.ClientTimeout(total=60, connect=20, sock_read=40)
             self.session = aiohttp.ClientSession(
+                headers=self.headers,
                 connector=connector,
-                timeout=timeout
+                timeout=timeout,
+                trust_env=True
             )
         return self.session
     
+    def is_blocked_domain(self, url: str) -> bool:
+        """Check if domain is known to block scrapers"""
+        from urllib.parse import urlparse
+        try:
+            domain = urlparse(url).netloc.lower()
+            return any(blocked in domain for blocked in self.blocked_domains)
+        except:
+            return False
+    
+    async def scrape_article_fallback(self, url: str) -> Tuple[str, Optional[str]]:
+        """Fallback scraping method using direct HTTP request"""
+        try:
+            session = await self.get_session()
+            
+            # Add random delay to avoid rate limiting
+            await asyncio.sleep(0.5)
+            
+            async with session.get(url, allow_redirects=True) as response:
+                if response.status == 200:
+                    html = await response.text()
+                    soup = BeautifulSoup(html, 'html.parser')
+                    
+                    # Remove script and style elements
+                    for script in soup(["script", "style", "nav", "header", "footer", "aside"]):
+                        script.decompose()
+                    
+                    # Try to find main content
+                    content_selectors = [
+                        'article', '.article-body', '.entry-content', '.post-content',
+                        '.content', '.main-content', '[data-module="ArticleBody"]',
+                        '.story-body', '.article-content', 'main'
+                    ]
+                    
+                    content = ""
+                    for selector in content_selectors:
+                        elements = soup.select(selector)
+                        if elements:
+                            content = ' '.join(elem.get_text(strip=True) for elem in elements)
+                            if len(content) > 200:  # Minimum content length
+                                break
+                    
+                    # If no content found, get all paragraph text
+                    if not content or len(content) < 100:
+                        paragraphs = soup.find_all('p')
+                        content = ' '.join(p.get_text(strip=True) for p in paragraphs if len(p.get_text(strip=True)) > 20)
+                    
+                    # Try to extract publication date
+                    pub_date = None
+                    date_selectors = [
+                        'time[datetime]', '.published-date', '.post-date', 
+                        '.article-date', '[data-testid="timestamp"]'
+                    ]
+                    
+                    for selector in date_selectors:
+                        date_elem = soup.select_one(selector)
+                        if date_elem:
+                            pub_date = date_elem.get('datetime') or date_elem.get_text(strip=True)
+                            break
+                    
+                    return content[:3000], pub_date  # Limit content length
+                else:
+                    return "", None
+        except Exception as e:
+            print(f"Fallback scraping failed for {url}: {e}")
+            return "", None
+    
     async def scrape_article(self, url: str) -> Tuple[str, Optional[str]]:
-        """Scrape article content and publication date"""
+        """Scrape article content with multiple fallback strategies"""
         try:
-            # Use newspaper3k for article extraction
+            # First, try newspaper3k with custom configuration
             article = Article(url)
-            article.download()
-            article.parse()
-            
-            content = article.text
-            pub_date = article.publish_date.isoformat() if article.publish_date else None
+            article.set_config({
+                'browser_user_agent': self.headers['User-Agent'],
+                'request_timeout': 30,
+                'number_threads': 1,
+                'verbose': False,
+                'fetch_images': False,
+                'memoize_articles': False,
+                'use_cached_categories': False
+            })
+            
+            # Try newspaper3k first
+            try:
+                article.download()
+                article.parse()
+                
+                if article.text and len(article.text.strip()) > 100:
+                    content = article.text.strip()
+                    pub_date = article.publish_date.isoformat() if article.publish_date else None
+                    return content[:3000], pub_date
+            except Exception as e:
+                print(f"Newspaper3k failed for {url}: {e}")
+            
+            # If newspaper3k fails or domain is blocked, try fallback
+            content, pub_date = await self.scrape_article_fallback(url)
+            if content and len(content.strip()) > 50:
+                return content, pub_date
+                
+            return "", None
             
-            return content, pub_date
         except Exception as e:
-            print(f"Error scraping {url}: {e}")
+            print(f"All scraping methods failed for {url}: {e}")
             return "", None
     
-    async def scrape_multiple(self, search_results: List[SearchResult]) -> List[SearchResult]:
-        """Scrape multiple articles in parallel"""
+    async def scrape_multiple(self, search_results: List[SearchResult], max_successful: int = None) -> List[SearchResult]:
+        """Scrape multiple articles with robust error handling and retry logic"""
+        if not search_results:
+            return search_results
+        
+        max_successful = max_successful or len(search_results)
+        successful_scraped = 0
+        semaphore = asyncio.Semaphore(5)  # Limit concurrent requests
+        
+        async def scrape_with_semaphore(result: SearchResult) -> SearchResult:
+            nonlocal successful_scraped
+            
+            if successful_scraped >= max_successful:
+                return result
+                
+            async with semaphore:
+                try:
+                    # Skip if already have enough successful results
+                    if successful_scraped >= max_successful:
+                        return result
+                    
+                    content, pub_date = await self.scrape_article(result.url)
+                    
+                    if content and len(content.strip()) > 50:
+                        result.content = content
+                        result.publication_date = pub_date
+                        successful_scraped += 1
+                        print(f"✅ Successfully scraped: {result.url[:60]}...")
+                    else:
+                        print(f"⚠️ No content extracted from: {result.url[:60]}...")
+                        
+                except Exception as e:
+                    print(f"❌ Failed to scrape {result.url[:60]}...: {e}")
+                
+                return result
+        
+        # Process all URLs but stop when we have enough successful results
         tasks = []
         for result in search_results:
-            tasks.append(self.scrape_article(result.url))
+            if successful_scraped < max_successful:
+                tasks.append(scrape_with_semaphore(result))
+            else:
+                break
+        
+        if tasks:
+            scraped_results = await asyncio.gather(*tasks, return_exceptions=True)
+            
+            # Filter out exceptions and return successful results
+            valid_results = []
+            for result in scraped_results:
+                if not isinstance(result, Exception):
+                    valid_results.append(result)
+        else:
+            valid_results = search_results
         
-        scraped_data = await asyncio.gather(*tasks, return_exceptions=True)
+        # Return results with content first, then others
+        results_with_content = [r for r in valid_results if r.content.strip()]
+        results_without_content = [r for r in valid_results if not r.content.strip()]
         
-        for i, (content, pub_date) in enumerate(scraped_data):
-            if not isinstance(content, Exception):
-                search_results[i].content = content
-                search_results[i].publication_date = pub_date
+        print(f"📊 Scraping summary: {len(results_with_content)} successful, {len(results_without_content)} failed")
         
-        return search_results
+        return results_with_content + results_without_content
     
     async def close(self):
         """Close the session"""
@@ -475,7 +656,7 @@ class AISearchEngine:
                                  temperature: float,
                                  max_results: int,
                                  max_tokens: int) -> Tuple[str, str]:
-        """Main search and summarization pipeline"""
+        """Main search and summarization pipeline with robust error handling"""
         
         start_time = time.time()
         status_updates = []
@@ -500,53 +681,90 @@ class AISearchEngine:
             if not search_tasks:
                 return "No search engines selected", "\n".join(status_updates)
             
-            search_results_lists = await asyncio.gather(*search_tasks)
+            search_results_lists = await asyncio.gather(*search_tasks, return_exceptions=True)
             
-            # Combine and deduplicate results
+            # Combine and deduplicate results, handling exceptions
             all_results = []
             seen_urls = set()
             
             for results_list in search_results_lists:
-                for result in results_list:
-                    if result.url not in seen_urls:
-                        all_results.append(result)
-                        seen_urls.add(result.url)
+                if not isinstance(results_list, Exception) and results_list:
+                    for result in results_list:
+                        if result.url not in seen_urls and result.url.startswith('http'):
+                            all_results.append(result)
+                            seen_urls.add(result.url)
             
             status_updates.append(f"Found {len(all_results)} unique results")
             
             if not all_results:
-                return "No search results found", "\n".join(status_updates)
+                return "No search results found. This might be due to rate limiting or network issues. Please try again.", "\n".join(status_updates)
             
-            # Step 3: Content Scraping
+            # Step 3: Content Scraping with intelligent retry and fallback
             status_updates.append("📄 Scraping article content...")
-            scraped_results = await self.content_scraper.scrape_multiple(all_results[:max_results])
             
-            # Filter results with content
-            results_with_content = [r for r in scraped_results if r.content.strip()]
-            status_updates.append(f"Successfully scraped {len(results_with_content)} articles")
+            # Prioritize results and scrape intelligently
+            target_successful = min(max_results, len(all_results))
+            scraped_results = await self.content_scraper.scrape_multiple(
+                all_results[:max_results * 2],  # Try more URLs to ensure we get enough content
+                max_successful=target_successful
+            )
+            
+            # Filter results with meaningful content
+            results_with_content = [r for r in scraped_results if r.content.strip() and len(r.content.strip()) > 100]
+            status_updates.append(f"Successfully scraped {len(results_with_content)} articles with meaningful content")
+            
+            # If we don't have enough content, try to get some from snippets
+            if len(results_with_content) < 3:
+                status_updates.append("Using search snippets as fallback content...")
+                for result in scraped_results:
+                    if not result.content.strip() and result.snippet.strip():
+                        result.content = result.snippet
+                        results_with_content.append(result)
+                        if len(results_with_content) >= 5:  # Reasonable minimum
+                            break
+            
+            if not results_with_content:
+                return "No article content could be extracted. This might be due to anti-bot protections. Please try a different query or try again later.", "\n".join(status_updates)
             
             # Step 4: Optional Embedding-based Filtering
             if use_embeddings and results_with_content:
                 status_updates.append("🧠 Filtering results using embeddings...")
-                filtered_results = self.embedding_filter.filter_by_relevance(query, results_with_content)
-                status_updates.append(f"Filtered to {len(filtered_results)} most relevant results")
-            else:
-                filtered_results = results_with_content
+                try:
+                    filtered_results = self.embedding_filter.filter_by_relevance(query, results_with_content)
+                    if filtered_results:
+                        results_with_content = filtered_results
+                        status_updates.append(f"Filtered to {len(filtered_results)} most relevant results")
+                    else:
+                        status_updates.append("Embedding filter returned no results, using all scraped content")
+                except Exception as e:
+                    status_updates.append(f"Embedding filtering failed, using all results: {str(e)}")
             
-            if not filtered_results:
+            if not results_with_content:
                 return "No relevant results found after filtering", "\n".join(status_updates)
             
             # Step 5: LLM Summarization
             status_updates.append(f"🤖 Generating summary using {model}...")
             
-            if model.startswith("Groq"):
-                summary = await self.llm_summarizer.summarize_with_groq(
-                    query, filtered_results, temperature, max_tokens
-                )
-            else:  # OpenRouter
-                summary = await self.llm_summarizer.summarize_with_openrouter(
-                    query, filtered_results, temperature, max_tokens
-                )
+            try:
+                if model.startswith("Groq"):
+                    summary = await self.llm_summarizer.summarize_with_groq(
+                        query, results_with_content, temperature, max_tokens
+                    )
+                else:  # OpenRouter
+                    summary = await self.llm_summarizer.summarize_with_openrouter(
+                        query, results_with_content, temperature, max_tokens
+                    )
+                
+                # Check if summarization failed
+                if summary.startswith("Error") or summary.startswith("Groq API error") or summary.startswith("OpenRouter API error"):
+                    # Provide a basic summary from the content
+                    basic_summary = self.create_basic_summary(query, results_with_content)
+                    summary = f"AI summarization failed, but here's what I found:\n\n{basic_summary}\n\n---\n⚠️ Original error: {summary}"
+                
+            except Exception as e:
+                # Fallback to basic summary
+                basic_summary = self.create_basic_summary(query, results_with_content)
+                summary = f"AI summarization encountered an error, but here's what I found:\n\n{basic_summary}\n\n---\n⚠️ Error: {str(e)}"
             
             # Add metadata
             end_time = time.time()
@@ -556,7 +774,6 @@ class AISearchEngine:
             metadata += f"- Processing time: {processing_time:.2f} seconds\n"
             metadata += f"- Results found: {len(all_results)}\n"
             metadata += f"- Articles scraped: {len(results_with_content)}\n"
-            metadata += f"- Results used for summary: {len(filtered_results)}\n"
             metadata += f"- Search engines: {', '.join(search_engines)}\n"
             metadata += f"- Model: {model}\n"
             metadata += f"- Embeddings used: {use_embeddings}\n"
@@ -572,9 +789,27 @@ class AISearchEngine:
             return error_msg, "\n".join(status_updates)
         
         finally:
-            # Cleanup
-            await self.search_interface.close()
-            await self.content_scraper.close()
+            # Cleanup - but don't close sessions immediately to allow reuse
+            try:
+                # Don't close sessions here as they might be reused
+                pass
+            except Exception as e:
+                print(f"Cleanup error: {e}")
+    
+    def create_basic_summary(self, query: str, results: List[SearchResult]) -> str:
+        """Create a basic summary when AI summarization fails"""
+        summary_parts = [f"Based on search results for: **{query}**\n"]
+        
+        for i, result in enumerate(results[:5], 1):
+            content_preview = result.content[:300] + "..." if len(result.content) > 300 else result.content
+            summary_parts.append(f"**{i}. {result.title}**")
+            summary_parts.append(f"Source: {result.url}")
+            if result.publication_date:
+                summary_parts.append(f"Date: {result.publication_date}")
+            summary_parts.append(f"Content: {content_preview}")
+            summary_parts.append("")
+        
+        return "\n".join(summary_parts)
 
 # Global search engine instance
 search_engine = None