fix news dashboard

app/pages/05_Dashboard.py

@@ -450,88 +450,79 @@ def fetch_economic_calendar():
 status_placeholder = st.empty()
 
 # Execute all news fetching operations in parallel using ThreadPoolExecutor
-with st.spinner("Loading news from 8 sources..."):
-    with ThreadPoolExecutor(max_workers=8) as executor:
-        # Submit all tasks with source name attached
-        futures_map = {
-            executor.submit(fetch_twitter_news): 'twitter',
-            executor.submit(fetch_reddit_news): 'reddit',
-            executor.submit(fetch_rss_news): 'rss',
-            executor.submit(fetch_ai_tech_news): 'ai_tech',
-            executor.submit(fetch_prediction_markets): 'predictions',
-            executor.submit(fetch_sectoral_news): 'sectoral_news',
-            executor.submit(fetch_market_events): 'market_events',
-            executor.submit(fetch_economic_calendar): 'economic_calendar'
-        }
+_fetch_tasks = [
+    (fetch_twitter_news, 'twitter'),
+    (fetch_reddit_news, 'reddit'),
+    (fetch_rss_news, 'rss'),
+    (fetch_ai_tech_news, 'ai_tech'),
+    (fetch_prediction_markets, 'predictions'),
+    (fetch_sectoral_news, 'sectoral_news'),
+    (fetch_market_events, 'market_events'),
+    (fetch_economic_calendar, 'economic_calendar'),
+]
 
-        # Track errors and completion
-        fetch_errors = []
-        completed_sources = []
-
-        # Process results as they complete (progressive loading)
-        try:
-            for future in as_completed(futures_map, timeout=90):
-                source_name = futures_map[future]
-
-                try:
-                    result_df, error = future.result()
-
-                    # Update status
-                    completed_sources.append(source_name)
-                    status_placeholder.info(f"🔍 Loaded {len(completed_sources)}/8 sources ({', '.join(completed_sources)})")
-
-                    if source_name == 'twitter':
-                        twitter_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                    elif source_name == 'reddit':
-                        reddit_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                    elif source_name == 'rss':
-                        rss_all_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                        # Get main page news subset for RSS
-                        if not rss_all_df.empty and 'from_web' in rss_all_df.columns:
-                            rss_main_df = rss_all_df[rss_all_df['from_web'] == True].copy()
-                    elif source_name == 'ai_tech':
-                        ai_tech_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                    elif source_name == 'predictions':
-                        predictions_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                    elif source_name == 'sectoral_news':
-                        sectoral_news_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                    elif source_name == 'market_events':
-                        market_events_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-                    elif source_name == 'economic_calendar':
-                        economic_calendar_df = result_df
-                        if error:
-                            fetch_errors.append(error)
-
-                except Exception as e:
-                    fetch_errors.append(f"Error fetching {source_name} news: {e}")
-                    completed_sources.append(f"{source_name} (error)")
-                    status_placeholder.warning(f"⚠️ {source_name} failed, continuing with other sources...")
-
-        except TimeoutError:
-            # Handle timeout gracefully - continue with whatever results we have
-            fetch_errors.append("⏱️ Some sources timed out after 90 seconds - displaying available results")
-            status_placeholder.warning(f"⚠️ {len(completed_sources)}/8 sources loaded (some timed out)")
-
-            # Mark incomplete sources
-            all_sources = set(futures_map.values())
-            incomplete_sources = all_sources - set(completed_sources)
-            for source in incomplete_sources:
-                fetch_errors.append(f"{source} timed out - skipped")
-                completed_sources.append(f"{source} (timeout)")
+def _apply_result(source_name, result_df, error):
+    global twitter_df, reddit_df, rss_all_df, rss_main_df, ai_tech_df
+    global predictions_df, sectoral_news_df, market_events_df, economic_calendar_df
+    if source_name == 'twitter':
+        twitter_df = result_df
+    elif source_name == 'reddit':
+        reddit_df = result_df
+    elif source_name == 'rss':
+        rss_all_df = result_df
+        if not rss_all_df.empty and 'from_web' in rss_all_df.columns:
+            rss_main_df = rss_all_df[rss_all_df['from_web'] == True].copy()
+    elif source_name == 'ai_tech':
+        ai_tech_df = result_df
+    elif source_name == 'predictions':
+        predictions_df = result_df
+    elif source_name == 'sectoral_news':
+        sectoral_news_df = result_df
+    elif source_name == 'market_events':
+        market_events_df = result_df
+    elif source_name == 'economic_calendar':
+        economic_calendar_df = result_df
+    if error:
+        fetch_errors.append(error)
+
+fetch_errors = []
+completed_sources = []
+
+with st.spinner("Loading news from 8 sources..."):
+    try:
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            futures_map = {executor.submit(fn): name for fn, name in _fetch_tasks}
+            try:
+                for future in as_completed(futures_map, timeout=90):
+                    source_name = futures_map[future]
+                    try:
+                        result_df, error = future.result()
+                        completed_sources.append(source_name)
+                        status_placeholder.info(f"🔍 Loaded {len(completed_sources)}/8 sources ({', '.join(completed_sources)})")
+                        _apply_result(source_name, result_df, error)
+                    except Exception as e:
+                        fetch_errors.append(f"Error fetching {source_name} news: {e}")
+                        completed_sources.append(f"{source_name} (error)")
+                        status_placeholder.warning(f"⚠️ {source_name} failed, continuing with other sources...")
+            except TimeoutError:
+                fetch_errors.append("⏱️ Some sources timed out after 90 seconds - displaying available results")
+                status_placeholder.warning(f"⚠️ {len(completed_sources)}/8 sources loaded (some timed out)")
+                all_sources = set(futures_map.values())
+                for source in all_sources - set(completed_sources):
+                    fetch_errors.append(f"{source} timed out - skipped")
+                    completed_sources.append(f"{source} (timeout)")
+    except RuntimeError:
+        # OS thread limit reached — fall back to sequential fetching
+        fetch_errors.append("⚠️ Thread limit reached, falling back to sequential fetch")
+        for fn, name in _fetch_tasks:
+            try:
+                result_df, error = fn()
+                completed_sources.append(name)
+                status_placeholder.info(f"🔍 Loaded {len(completed_sources)}/8 sources (sequential mode)")
+                _apply_result(name, result_df, error)
+            except Exception as e:
+                fetch_errors.append(f"Error fetching {name} news: {e}")
+                completed_sources.append(f"{name} (error)")
 
     # Clear the status message after all sources complete
     status_placeholder.success(f"✅ Loaded {len(completed_sources)}/8 sources successfully")