add progressive loading (show results as they arrive)

app/pages/05_Dashboard.py

@@ -204,7 +204,7 @@ force_refresh = st.session_state.get('force_refresh', False)
 
 # Fetch news from all sources IN PARALLEL for maximum performance
 import pandas as pd
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 twitter_df = pd.DataFrame()
 reddit_df = pd.DataFrame()
@@ -294,52 +294,63 @@ def fetch_ai_tech_news():
         return pd.DataFrame(), f"AI/Tech news unavailable: {e}"
     return pd.DataFrame(), None
 
-with st.spinner("🔍 Fetching latest financial & tech news in parallel..."):
-    # Execute all news fetching operations in parallel using ThreadPoolExecutor
-    with ThreadPoolExecutor(max_workers=4) as executor:
-        # Submit all tasks
-        future_twitter = executor.submit(fetch_twitter_news)
-        future_reddit = executor.submit(fetch_reddit_news)
-        future_rss = executor.submit(fetch_rss_news)
-        future_ai_tech = executor.submit(fetch_ai_tech_news)
-
-        # Collect results as they complete
-        futures = {
-            'twitter': future_twitter,
-            'reddit': future_reddit,
-            'rss': future_rss,
-            'ai_tech': future_ai_tech
-        }
-
-        # Track errors to display later (avoid cluttering top of page)
-        fetch_errors = []
-
-        for source_name, future in futures.items():
-            try:
-                result_df, error = future.result(timeout=90)  # 90 second timeout per source
-
-                if source_name == 'twitter':
-                    twitter_df = result_df
-                    if error:
-                        fetch_errors.append(error)
-                elif source_name == 'reddit':
-                    reddit_df = result_df
-                    if error:
-                        fetch_errors.append(error)
-                elif source_name == 'rss':
-                    rss_all_df = result_df
-                    if error:
-                        fetch_errors.append(error)
-                    # Get main page news subset for RSS
-                    if not rss_all_df.empty and 'from_web' in rss_all_df.columns:
-                        rss_main_df = rss_all_df[rss_all_df['from_web'] == True].copy()
-                elif source_name == 'ai_tech':
-                    ai_tech_df = result_df
-                    if error:
-                        fetch_errors.append(error)
-
-            except Exception as e:
-                fetch_errors.append(f"Error fetching {source_name} news: {e}")
+# Progressive loading: Display results as they arrive
+# Create a status placeholder to show progress
+status_placeholder = st.empty()
+status_placeholder.info("🔍 Starting news fetch from 4 sources...")
+
+# Execute all news fetching operations in parallel using ThreadPoolExecutor
+with ThreadPoolExecutor(max_workers=4) as executor:
+    # Submit all tasks with source name attached
+    futures_map = {
+        executor.submit(fetch_twitter_news): 'twitter',
+        executor.submit(fetch_reddit_news): 'reddit',
+        executor.submit(fetch_rss_news): 'rss',
+        executor.submit(fetch_ai_tech_news): 'ai_tech'
+    }
+
+    # Track errors and completion
+    fetch_errors = []
+    completed_sources = []
+
+    # Process results as they complete (progressive loading)
+    for future in as_completed(futures_map, timeout=90):
+        source_name = futures_map[future]
+
+        try:
+            result_df, error = future.result()
+
+            # Update status
+            completed_sources.append(source_name)
+            status_placeholder.info(f"🔍 Loaded {len(completed_sources)}/4 sources ({', '.join(completed_sources)})")
+
+            if source_name == 'twitter':
+                twitter_df = result_df
+                if error:
+                    fetch_errors.append(error)
+            elif source_name == 'reddit':
+                reddit_df = result_df
+                if error:
+                    fetch_errors.append(error)
+            elif source_name == 'rss':
+                rss_all_df = result_df
+                if error:
+                    fetch_errors.append(error)
+                # Get main page news subset for RSS
+                if not rss_all_df.empty and 'from_web' in rss_all_df.columns:
+                    rss_main_df = rss_all_df[rss_all_df['from_web'] == True].copy()
+            elif source_name == 'ai_tech':
+                ai_tech_df = result_df
+                if error:
+                    fetch_errors.append(error)
+
+        except Exception as e:
+            fetch_errors.append(f"Error fetching {source_name} news: {e}")
+            completed_sources.append(f"{source_name} (error)")
+            status_placeholder.warning(f"⚠️ {source_name} failed, continuing with other sources...")
+
+# Clear the status message after all sources complete
+status_placeholder.success(f"✅ Loaded {len(completed_sources)}/4 sources successfully")
 
 # Debug output (remove in production)
 if st.session_state.get('debug_mode', False):