Update services/research_engine.py

services/research_engine.py

@@ -16,7 +16,6 @@ from googleapiclient.errors import HttpError
 class ResearchEngine:
     """Enhanced research engine with improved scraping and error handling"""
 
-    # Global number of simultaneous scrapes (must match adapter pool_maxsize)
     _MAX_SCRAPERS: int = 5
 
     def __init__(self):
@@ -33,7 +32,6 @@ class ResearchEngine:
             logging.error("Error initializing Google Custom Search: %s", e)
             raise
 
-        # Prepare a single requests.Session with connection pool and retry/backoff:
         self.session = requests.Session()
         retries = Retry(
             total=3,
@@ -47,17 +45,13 @@ class ResearchEngine:
         self.session.mount("http://", adapter)
         self.session.mount("https://", adapter)
 
-        # Preload a small list of realistic user‑agents
         self.user_agents = [
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) "
-            "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",
             "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0"
         ]
 
     def search_multiple_sources(self, query: str, context: str) -> Dict[str, Any]:
-        """Search multiple sources and aggregate results"""
         now = time.time()
         results = {
             'google_results': [],
@@ -67,7 +61,6 @@ class ResearchEngine:
             'metadata': {}
         }
 
-        # Google Search
         try:
             data = self._search_google(query, context)
             results['google_results'] = data['items']
@@ -77,15 +70,13 @@ class ResearchEngine:
             logging.error("Google Search API error: %s", e)
             raise RuntimeError("Google Search API failure")
 
-        # News API (if available)
         news = self._search_news_api(query)
         if news:
             results['news_results'] = news.get('articles', [])
             results['sources'].extend([a.get('url') for a in news['articles'] if a.get('url')])
             logging.info("Added %d news sources", len(results['news_results']))
 
-        # Scrape top N sources concurrently
-        unique_urls = list(dict.fromkeys(results['sources']))  # drop duplicates
+        unique_urls = list(dict.fromkeys(results['sources']))
         urls = unique_urls[:self._MAX_SCRAPERS]
         results['scraped_content'] = list(self._parallel_scrape(urls))
 
@@ -97,11 +88,8 @@ class ResearchEngine:
         return results
 
     def _search_google(self, query: str, context: str) -> Dict[str, Any]:
-        """Call Google Custom Search API for given query+context"""
         professional_query = f"{query} {context}"
-        resp = self.google_service.cse() \
-            .list(q=professional_query, cx=self.google_cx, num=10) \
-            .execute()
+        resp = self.google_service.cse().list(q=professional_query, cx=self.google_cx, num=10).execute()
 
         items = []
         sources = []
@@ -118,7 +106,6 @@ class ResearchEngine:
         return {'items': items, 'sources': sources}
 
     def _search_news_api(self, query: str) -> Optional[Dict[str, Any]]:
-        """Optionally call NewsAPI for current events (last 30 days relevancy)"""
         api_key = os.getenv("NEWS_API_KEY")
         if not api_key:
             logging.warning("No NEWS_API_KEY – skipping News search")
@@ -143,7 +130,6 @@ class ResearchEngine:
             return None
 
     def _parallel_scrape(self, urls: List[str]) -> List[Dict[str, Any]]:
-        """Fetch page content in parallel with rotating UA, timeouts, and quick failure on 403 or errors."""
         out = []
         future_to_url = {}
 
@@ -152,7 +138,7 @@ class ResearchEngine:
                 "User-Agent": random.choice(self.user_agents),
                 "Accept-Language": "en-US,en;q=0.9",
                 "Accept": "text/html,application/xhtml+xml;q=0.9,image/webp,*/*;q=0.8",
-                "Connection": "keep‑alive"
+                "Connection": "keep-alive"
             }
             try:
                 resp = self.session.get(u, headers=headers, timeout=(3, 10))
@@ -160,12 +146,16 @@ class ResearchEngine:
                     logging.warning("Scraping blocked (403) for %s", u)
                     return None
                 resp.raise_for_status()
+                if resp.encoding.lower() in ("iso-8859-1", "latin-1"):
+                    resp.encoding = resp.apparent_encoding
                 text = resp.text or ""
                 if len(text) < 100:
                     return None
                 return {'url': u, 'content': text[:2000], 'timestamp': time.time()}
             except requests.exceptions.ReadTimeout:
                 logging.warning("Timeout scraping %s", u)
+            except UnicodeEncodeError:
+                return {'url': u, 'content': resp.content.decode('utf-8', errors='replace')[:2000], 'timestamp': time.time()}
             except Exception as e:
                 logging.warning("Scraping failure for %s: %s", u, e)
             return None
@@ -173,17 +163,15 @@ class ResearchEngine:
         with ThreadPoolExecutor(max_workers=self._MAX_SCRAPERS) as ex:
             for u in urls:
                 future_to_url[ex.submit(crawl, u)] = u
-
             for f in as_completed(future_to_url):
                 res = f.result()
                 if res:
                     out.append(res)
-                time.sleep(random.uniform(0.5, 1.0))  # polite pacing
+                time.sleep(random.uniform(0.5, 1.0))
 
         return out
 
     def extract_key_data_points(self, research_results: Dict[str, Any]) -> List[Dict[str, Any]]:
-        """Extract top 10 value‑type stats from snippets and content (monetary, % etc.)"""
         data_points = []
         for itm in research_results.get('google_results', []):
             val = self._extract_numbers_and_stats(itm.get('snippet', ""))
@@ -194,7 +182,6 @@ class ResearchEngine:
                     'context': itm.get('snippet', ""),
                     'type': 'statistic'
                 })
-
         for cnt in research_results.get('scraped_content', []):
             val = self._extract_numbers_and_stats(cnt.get('content', ""))
             if val:
@@ -204,7 +191,6 @@ class ResearchEngine:
                     'context': cnt.get('content', "")[:200],
                     'type': 'detailed_analysis'
                 })
-
         return data_points[:10]
 
     def _extract_numbers_and_stats(self, text: str) -> Optional[str]: