optimize news module performance, improve news UI, etc

app/components/news.py

@@ -6,8 +6,8 @@ from datetime import datetime
 import html as html_module
 
 
-def display_news_card(news_item: dict):
-    """Display a single news card with professional styling using Streamlit components."""
+def display_tradingview_news_card(news_item: dict):
+    """Display a single news card with TradingView-inspired styling."""
 
     # Calculate time ago
     time_diff = datetime.now() - news_item['timestamp']
@@ -16,76 +16,242 @@ def display_news_card(news_item: dict):
     elif time_diff.seconds < 3600:
         time_ago = f"{time_diff.seconds // 60}m ago"
     else:
-        time_ago = f"{time_diff.seconds // 3600}h ago"
+        hours = time_diff.seconds // 3600
+        time_ago = f"{hours}h ago" if hours < 24 else f"{time_diff.days}d ago"
+
+    # Impact badge colors (TradingView style)
+    impact_colors = {
+        'high': '#F23645',     # Red
+        'medium': '#FF9800',   # Orange
+        'low': '#089981'       # Green
+    }
 
-    # Create container with custom styling
-    with st.container():
-        # Add custom CSS for this card
-        st.markdown("""
-        <style>
-        .news-card {
-            background: linear-gradient(135deg, #1f2937 0%, #111827 100%);
-            border: 1px solid #374151;
-            border-radius: 12px;
+    # Sentiment colors
+    sentiment_colors = {
+        'positive': '#089981',   # Green
+        'negative': '#F23645',   # Red
+        'neutral': '#787B86'     # Gray
+    }
+
+    impact_color = impact_colors.get(news_item['impact'], '#787B86')
+    sentiment_color = sentiment_colors.get(news_item['sentiment'], '#787B86')
+
+    # Escape HTML in text
+    summary = html_module.escape(news_item.get('summary', '').strip())
+    source = html_module.escape(news_item['source'])
+    category = html_module.escape(news_item['category'])
+    url = html_module.escape(news_item['url'])
+
+    # TradingView-style card HTML
+    card_html = f"""
+    <div style="
+        background: linear-gradient(135deg, #1E222D 0%, #131722 100%);
+        border: 1px solid #2A2E39;
+        border-radius: 8px;
+        padding: 16px;
+        margin-bottom: 12px;
+        transition: all 0.2s ease;
+        cursor: pointer;
+        position: relative;
+        overflow: hidden;
+    " onmouseover="this.style.borderColor='#3861FB'; this.style.transform='translateY(-2px)'; this.style.boxShadow='0 4px 12px rgba(56, 97, 251, 0.15)';"
+       onmouseout="this.style.borderColor='#2A2E39'; this.style.transform='translateY(0)'; this.style.boxShadow='none';">
+
+        <!-- Left colored indicator bar -->
+        <div style="
+            position: absolute;
+            left: 0;
+            top: 0;
+            bottom: 0;
+            width: 3px;
+            background: {impact_color};
+        "></div>
+
+        <!-- Header row -->
+        <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; margin-left: 8px;">
+            <div style="display: flex; align-items: center; gap: 8px; flex-wrap: wrap;">
+                <span style="
+                    color: #3861FB;
+                    font-weight: 600;
+                    font-size: 13px;
+                    font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+                ">{source}</span>
+
+                <span style="
+                    background: {impact_color};
+                    color: white;
+                    padding: 2px 8px;
+                    border-radius: 4px;
+                    font-size: 10px;
+                    font-weight: 700;
+                    letter-spacing: 0.5px;
+                ">{news_item['impact'].upper()}</span>
+
+                <span style="
+                    color: {sentiment_color};
+                    font-size: 11px;
+                    font-weight: 600;
+                    padding: 2px 6px;
+                    border: 1px solid {sentiment_color};
+                    border-radius: 4px;
+                ">{'▲' if news_item['sentiment'] == 'positive' else '▼' if news_item['sentiment'] == 'negative' else '●'} {news_item['sentiment'].upper()}</span>
+
+                <span style="
+                    color: #787B86;
+                    font-size: 11px;
+                    background: rgba(120, 123, 134, 0.1);
+                    padding: 2px 6px;
+                    border-radius: 4px;
+                ">#{category}</span>
+            </div>
+
+            <span style="color: #787B86; font-size: 11px; white-space: nowrap;">{time_ago}</span>
+        </div>
+
+        <!-- News summary -->
+        <div style="
+            color: #D1D4DC;
+            font-size: 14px;
+            line-height: 1.5;
+            margin-bottom: 8px;
+            margin-left: 8px;
+            font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+        ">{summary}</div>
+
+        <!-- Read more link -->
+        <a href="{url}" target="_blank" style="
+            color: #3861FB;
+            font-size: 12px;
+            text-decoration: none;
+            margin-left: 8px;
+            display: inline-flex;
+            align-items: center;
+            gap: 4px;
+            font-weight: 500;
+        " onmouseover="this.style.color='#5880FF';" onmouseout="this.style.color='#3861FB';">
+            Read Full Story →
+        </a>
+    </div>
+    """
+
+    st.markdown(card_html, unsafe_allow_html=True)
+
+
+def display_news_card(news_item: dict):
+    """Wrapper to maintain compatibility - calls TradingView-style card."""
+    display_tradingview_news_card(news_item)
+
+
+def display_scrollable_news_section(df: pd.DataFrame, section_title: str, section_icon: str,
+                                    section_subtitle: str, max_items: int = 20, height: str = "600px"):
+    """Display a scrollable news section with TradingView styling."""
+
+    if df.empty:
+        st.markdown(f"""
+        <div style="
+            background: linear-gradient(135deg, #1E222D 0%, #131722 100%);
+            border: 1px solid #2A2E39;
+            border-radius: 8px;
             padding: 20px;
-            margin-bottom: 16px;
-        }
-        </style>
+            text-align: center;
+            color: #787B86;
+        ">
+            <p style="font-size: 16px; margin: 0;">📭 No news available for this section</p>
+        </div>
         """, unsafe_allow_html=True)
+        return
 
-        # Header row with source and badges
-        col1, col2 = st.columns([3, 1])
-
-        with col1:
-            # Source and badges in one line
-            badge_cols = st.columns([2, 1, 1, 1])
-            with badge_cols[0]:
-                st.markdown(f"**:blue[{news_item['source']}]**")
-            with badge_cols[1]:
-                if news_item['impact'] == 'high':
-                    st.markdown("🔴 **HIGH**")
-                elif news_item['impact'] == 'medium':
-                    st.markdown("🟡 **MED**")
-                else:
-                    st.markdown("🟢 **LOW**")
-            with badge_cols[2]:
-                sentiment_emoji = {'positive': '📈', 'negative': '📉', 'neutral': '➡️'}
-                st.markdown(f"{sentiment_emoji.get(news_item['sentiment'], '➡️')} {news_item['sentiment'].title()}")
-            with badge_cols[3]:
-                st.markdown(f"**#{news_item['category']}**")
+    # Section header
+    st.markdown(f"""
+    <div style="
+        background: linear-gradient(135deg, #2A2E39 0%, #1E222D 100%);
+        border: 1px solid #363A45;
+        border-radius: 8px 8px 0 0;
+        padding: 16px 20px;
+        margin-bottom: 0;
+    ">
+        <div style="display: flex; justify-content: space-between; align-items: center;">
+            <div>
+                <h3 style="
+                    color: #D1D4DC;
+                    margin: 0;
+                    font-size: 18px;
+                    font-weight: 600;
+                    font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+                ">{section_icon} {section_title}</h3>
+                <p style="
+                    color: #787B86;
+                    margin: 4px 0 0 0;
+                    font-size: 12px;
+                ">{section_subtitle}</p>
+            </div>
+            <div style="
+                background: rgba(56, 97, 251, 0.15);
+                color: #3861FB;
+                padding: 6px 12px;
+                border-radius: 6px;
+                font-size: 13px;
+                font-weight: 600;
+            ">{len(df.head(max_items))} stories</div>
+        </div>
+    </div>
+    """, unsafe_allow_html=True)
 
-        with col2:
-            st.markdown(f"[**Read More →**]({news_item['url']})")
+    # Scrollable container with TradingView-style scrollbar
+    st.markdown(f"""
+    <style>
+    .news-scroll-container {{
+        height: {height};
+        overflow-y: auto;
+        background: #0D0E13;
+        border: 1px solid #2A2E39;
+        border-top: none;
+        border-radius: 0 0 8px 8px;
+        padding: 16px;
+    }}
+
+    /* TradingView-style scrollbar */
+    .news-scroll-container::-webkit-scrollbar {{
+        width: 8px;
+    }}
+
+    .news-scroll-container::-webkit-scrollbar-track {{
+        background: #1E222D;
+        border-radius: 4px;
+    }}
+
+    .news-scroll-container::-webkit-scrollbar-thumb {{
+        background: #363A45;
+        border-radius: 4px;
+    }}
+
+    .news-scroll-container::-webkit-scrollbar-thumb:hover {{
+        background: #434651;
+    }}
+    </style>
+    """, unsafe_allow_html=True)
 
-        # Title/Summary
-        st.markdown(f"### {news_item.get('summary', '').strip()}")
+    # Start scrollable container
+    st.markdown('<div class="news-scroll-container">', unsafe_allow_html=True)
 
-        # Meta information
-        st.caption(f"🕐 {time_ago}")
+    # Display news items
+    for idx, row in df.head(max_items).iterrows():
+        display_tradingview_news_card(row.to_dict())
 
-        st.markdown("---")
+    # End scrollable container
+    st.markdown('</div>', unsafe_allow_html=True)
 
 
 def display_news_feed(df: pd.DataFrame, max_items: int = 20):
-    """Display a feed of news items."""
+    """Display a feed of news items (legacy compatibility)."""
 
     if df.empty:
         st.info("📭 No news available. Adjust your filters or refresh the feed.")
         return
 
-    # Add custom CSS for animations
-    st.markdown("""
-    <style>
-    @keyframes pulse {
-        0%, 100% { opacity: 1; }
-        50% { opacity: 0.6; }
-    }
-    </style>
-    """, unsafe_allow_html=True)
-
     # Display news items
     for idx, row in df.head(max_items).iterrows():
-        display_news_card(row.to_dict())
+        display_tradingview_news_card(row.to_dict())
 
 
 def display_news_statistics(stats: dict):
@@ -167,36 +333,142 @@ def display_category_breakdown(stats: dict):
 
 
 def display_breaking_news_banner(df: pd.DataFrame):
-    """Display breaking news banner at the top using Streamlit components."""
+    """Display breaking news banner at the top with TradingView styling."""
 
     breaking = df[df['is_breaking'] == True] if not df.empty and 'is_breaking' in df.columns else pd.DataFrame()
 
     if not breaking.empty:
         latest = breaking.iloc[0]
 
-        # Create container with red background styling
-        with st.container():
-            st.markdown("""
-            <style>
-            .breaking-news-container {
-                background: linear-gradient(135deg, #dc2626 0%, #991b1b 100%);
-                border-radius: 12px;
-                padding: 20px;
-                margin-bottom: 24px;
-                border: 2px solid #fca5a5;
-            }
-            </style>
-            """, unsafe_allow_html=True)
-
-            # Layout with emoji and content
-            col1, col2, col3 = st.columns([1, 8, 2])
-
-            with col1:
-                st.markdown("# 🚨")
-
-            with col2:
-                st.markdown(f"**:red[BREAKING NEWS • {latest['source'].upper()}]**")
-                st.markdown(f"## {latest.get('summary', '').strip()}")
-
-            with col3:
-                st.markdown(f"[**READ NOW →**]({latest['url']})")
+        # Escape HTML
+        summary = html_module.escape(latest.get('summary', '').strip())
+        source = html_module.escape(latest['source'])
+        url = html_module.escape(latest['url'])
+
+        # Calculate time ago
+        time_diff = datetime.now() - latest['timestamp']
+        if time_diff.seconds < 60:
+            time_ago = f"{time_diff.seconds}s ago"
+        elif time_diff.seconds < 3600:
+            time_ago = f"{time_diff.seconds // 60}m ago"
+        else:
+            hours = time_diff.seconds // 3600
+            time_ago = f"{hours}h ago" if hours < 24 else f"{time_diff.days}d ago"
+
+        # TradingView-style breaking news banner
+        banner_html = f"""
+        <style>
+        @keyframes pulse-glow {{
+            0%, 100% {{ box-shadow: 0 0 20px rgba(242, 54, 69, 0.6); }}
+            50% {{ box-shadow: 0 0 30px rgba(242, 54, 69, 0.9); }}
+        }}
+
+        @keyframes slide-in {{
+            from {{ transform: translateX(-10px); opacity: 0; }}
+            to {{ transform: translateX(0); opacity: 1; }}
+        }}
+        </style>
+
+        <div style="
+            background: linear-gradient(135deg, #F23645 0%, #C91B28 100%);
+            border: 2px solid #FF6B78;
+            border-radius: 12px;
+            padding: 20px 24px;
+            margin-bottom: 24px;
+            animation: pulse-glow 2s ease-in-out infinite;
+            position: relative;
+            overflow: hidden;
+        ">
+            <!-- Animated background pattern -->
+            <div style="
+                position: absolute;
+                top: 0;
+                left: 0;
+                right: 0;
+                bottom: 0;
+                background: repeating-linear-gradient(
+                    45deg,
+                    transparent,
+                    transparent 10px,
+                    rgba(255, 255, 255, 0.03) 10px,
+                    rgba(255, 255, 255, 0.03) 20px
+                );
+                pointer-events: none;
+            "></div>
+
+            <!-- Content -->
+            <div style="position: relative; z-index: 1;">
+                <div style="display: flex; align-items: center; gap: 16px; margin-bottom: 12px;">
+                    <!-- Animated icon -->
+                    <div style="
+                        font-size: 32px;
+                        animation: pulse-glow 1s ease-in-out infinite;
+                        filter: drop-shadow(0 2px 8px rgba(0, 0, 0, 0.3));
+                    ">🚨</div>
+
+                    <!-- Header -->
+                    <div style="flex: 1;">
+                        <div style="
+                            color: white;
+                            font-size: 14px;
+                            font-weight: 700;
+                            letter-spacing: 1.5px;
+                            text-transform: uppercase;
+                            margin-bottom: 4px;
+                            font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+                            text-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+                        ">⚡ Breaking News</div>
+                        <div style="
+                            color: rgba(255, 255, 255, 0.9);
+                            font-size: 11px;
+                            display: flex;
+                            align-items: center;
+                            gap: 8px;
+                        ">
+                            <span style="
+                                background: rgba(255, 255, 255, 0.2);
+                                padding: 2px 8px;
+                                border-radius: 4px;
+                                font-weight: 600;
+                            ">{source}</span>
+                            <span style="opacity: 0.8;">•</span>
+                            <span style="opacity: 0.8;">{time_ago}</span>
+                        </div>
+                    </div>
+
+                    <!-- Read button -->
+                    <a href="{url}" target="_blank" style="
+                        background: white;
+                        color: #F23645;
+                        padding: 10px 20px;
+                        border-radius: 6px;
+                        font-size: 13px;
+                        font-weight: 700;
+                        text-decoration: none;
+                        display: inline-flex;
+                        align-items: center;
+                        gap: 6px;
+                        transition: all 0.2s ease;
+                        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2);
+                    " onmouseover="this.style.transform='translateY(-2px)'; this.style.boxShadow='0 4px 12px rgba(0, 0, 0, 0.3)';"
+                       onmouseout="this.style.transform='translateY(0)'; this.style.boxShadow='0 2px 8px rgba(0, 0, 0, 0.2)';">
+                        READ NOW →
+                    </a>
+                </div>
+
+                <!-- News summary -->
+                <div style="
+                    color: white;
+                    font-size: 16px;
+                    font-weight: 500;
+                    line-height: 1.5;
+                    margin-left: 48px;
+                    font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+                    text-shadow: 0 1px 2px rgba(0, 0, 0, 0.2);
+                    animation: slide-in 0.5s ease-out;
+                ">{summary}</div>
+            </div>
+        </div>
+        """
+
+        st.markdown(banner_html, unsafe_allow_html=True)

app/pages/05_Dashboard.py

@@ -12,10 +12,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from components.styles import DARK_THEME_CSS
 from components.news import (
-    display_news_feed,
     display_news_statistics,
     display_category_breakdown,
-    display_breaking_news_banner
+    display_breaking_news_banner,
+    display_scrollable_news_section
 )
 
 # Import news scrapers
@@ -220,50 +220,57 @@ if not all_news_df.empty:
 
 st.markdown("---")
 
-# ---- SECTION 1: Twitter/X Breaking News (Highest Priority) ----
-st.markdown("## 🐦 Twitter/X Financial News Feed")
-st.caption("Real-time breaking news from premium Twitter/X accounts (WalterBloomberg, Reuters, Bloomberg, FT, etc.)")
+# ---- THREE-COLUMN SCROLLABLE NEWS LAYOUT (TradingView Style) ----
 
-if not twitter_filtered.empty:
-    display_news_feed(twitter_filtered, max_items=15)
-elif not twitter_df.empty:
-    st.info("📭 No Twitter news matches your current filters.")
-else:
-    st.info("⏳ Twitter news scraping in progress... This may take 30-60 seconds on first load.")
+col1, col2, col3 = st.columns(3)
 
-st.markdown("---")
-
-# ---- SECTION 2: Main Page News (Web-Scraped) ----
-st.markdown("## 🔥 Top Stories from Main Pages")
-st.caption("Latest headlines directly scraped from news source homepages")
-
-if not rss_main_filtered.empty:
-    display_news_feed(rss_main_filtered, max_items=10)
-elif not rss_main_df.empty:
-    st.info("📭 No main page news matches your filters.")
-else:
-    st.info("⏳ Main page news will appear here...")
-
-st.markdown("---")
-
-# ---- SECTION 3: RSS Feed News (Lowest Priority) ----
-col1, col2, col3 = st.columns([2, 1, 1])
 with col1:
-    st.markdown("## 📰 RSS Feed News")
+    # SECTION 1: Twitter/X Breaking News
+    if not twitter_filtered.empty:
+        display_scrollable_news_section(
+            twitter_filtered,
+            section_title="Twitter/X News",
+            section_icon="🐦",
+            section_subtitle="Real-time breaking news from premium accounts",
+            max_items=50,
+            height="700px"
+        )
+    elif not twitter_df.empty:
+        st.info("📭 No Twitter news matches your current filters.")
+    else:
+        st.info("⏳ Twitter news scraping in progress... This may take 30-60 seconds on first load.")
+
 with col2:
-    show_count = st.selectbox("Show", [10, 20, 50, 100], index=1, label_visibility="collapsed")
+    # SECTION 2: Main Page News (Web-Scraped)
+    if not rss_main_filtered.empty:
+        display_scrollable_news_section(
+            rss_main_filtered,
+            section_title="Top Headlines",
+            section_icon="🔥",
+            section_subtitle="Latest from main pages",
+            max_items=50,
+            height="700px"
+        )
+    elif not rss_main_df.empty:
+        st.info("📭 No main page news matches your filters.")
+    else:
+        st.info("⏳ Main page news will appear here...")
+
 with col3:
+    # SECTION 3: RSS Feed News
     if not rss_all_filtered.empty:
-        st.caption(f"Displaying {min(show_count, len(rss_all_filtered))} of {len(rss_all_filtered)} stories")
-
-st.caption("Aggregated news from RSS feeds across all sources")
-
-if not rss_all_filtered.empty:
-    display_news_feed(rss_all_filtered, max_items=show_count)
-elif not rss_all_df.empty:
-    st.info("📭 No RSS news matches your current filters.")
-else:
-    st.info("⏳ RSS feed news will appear here...")
+        display_scrollable_news_section(
+            rss_all_filtered,
+            section_title="RSS Feed",
+            section_icon="📰",
+            section_subtitle="Aggregated from all sources",
+            max_items=100,
+            height="700px"
+        )
+    elif not rss_all_df.empty:
+        st.info("📭 No RSS news matches your current filters.")
+    else:
+        st.info("⏳ RSS feed news will appear here...")
 
 # Auto-refresh logic
 if auto_refresh:

app/services/twitter_news_playwright.py

@@ -208,51 +208,58 @@ class TwitterFinanceMonitor:
         logger.warning("Chromium not found in standard paths")
         return '/usr/bin/chromium'  # Fallback
 
-    def _scrape_twitter_profile(self, source_name: str, source_info: Dict, timeout: int = 15) -> List[Dict]:
-        """Scrape tweets from a single Twitter profile using Playwright"""
+    def _scrape_twitter_profile(self, source_name: str, source_info: Dict, timeout: int = 12) -> List[Dict]:
+        """Scrape tweets from a single Twitter profile using Playwright (optimized for speed)"""
         if not PLAYWRIGHT_AVAILABLE:
             logger.warning("Playwright not available")
             return []
 
         try:
             with sync_playwright() as p:
-                # Launch lightweight browser
+                # Launch lightweight browser with aggressive performance flags
                 browser = p.chromium.launch(
                     executable_path=self.chromium_path,
                     headless=True,
-                    args=['--disable-blink-features=AutomationControlled']
+                    args=[
+                        '--disable-blink-features=AutomationControlled',
+                        '--disable-dev-shm-usage',  # Overcome limited resource problems
+                        '--no-sandbox',  # Required for some environments
+                        '--disable-setuid-sandbox',
+                        '--disable-gpu',  # Not needed in headless
+                        '--disable-software-rasterizer'
+                    ]
                 )
                 context = browser.new_context(
                     user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
                 )
                 page = context.new_page()
 
-                # Block images, fonts, and css for speed
+                # Block images, fonts, css, and videos for maximum speed
                 def route_intercept(route):
-                    if route.request.resource_type in ["image", "media", "font", "stylesheet"]:
+                    if route.request.resource_type in ["image", "media", "font", "stylesheet", "video"]:
                         route.abort()
                     else:
                         route.continue_()
 
                 page.route("**/*", route_intercept)
 
-                # Navigate to profile
-                logger.info(f"Scraping {source_name} from {source_info['url']}")
-                page.goto(source_info['url'], timeout=timeout * 1000)
+                # Navigate to profile with reduced timeout
+                logger.info(f"Scraping {source_name}...")
+                page.goto(source_info['url'], timeout=timeout * 1000, wait_until="domcontentloaded")  # Don't wait for full load
 
-                # Wait for tweets to load
+                # Wait for tweets to load with reduced timeout
                 try:
-                    page.wait_for_selector("article", timeout=timeout * 1000)
+                    page.wait_for_selector("article", timeout=8000)  # Fixed 8 second wait
                 except PlaywrightTimeoutError:
                     logger.warning(f"Timeout waiting for tweets from {source_name}")
                     browser.close()
                     return []
 
-                # Extract tweet texts
+                # Extract tweet texts (limit to 15 for speed)
                 tweet_elements = page.locator("article div[data-testid='tweetText']").all()
 
                 news_items = []
-                for idx, element in enumerate(tweet_elements[:20]):  # Limit to 20 most recent
+                for idx, element in enumerate(tweet_elements[:15]):  # Reduced from 20 to 15 for speed
                     try:
                         text = element.text_content()
                         if not text or len(text) < 10:
@@ -308,7 +315,7 @@ class TwitterFinanceMonitor:
     def scrape_twitter_news(_self, max_tweets: int = 100) -> List[Dict]:
         """
         Scrape latest financial news from Twitter using Playwright
-        Runs in parallel for better performance
+        Runs in parallel for better performance - 19 sources in ~30-45 seconds
         """
         if not PLAYWRIGHT_AVAILABLE:
             logger.info("Playwright not available - using mock data")
@@ -317,17 +324,26 @@ class TwitterFinanceMonitor:
         all_news = []
         seen_texts = set()
 
-        # Scrape sources in parallel with timeout
-        with ThreadPoolExecutor(max_workers=3) as executor:
+        # Sort sources by weight (priority) - scrape high-value sources first
+        sorted_sources = sorted(
+            _self.SOURCES.items(),
+            key=lambda x: x[1]['weight'],
+            reverse=True
+        )
+
+        # Scrape sources in parallel with higher concurrency
+        # 8 workers = 19 sources in 3 batches (~30-45 seconds total)
+        with ThreadPoolExecutor(max_workers=8) as executor:
             futures = []
-            for name, info in _self.SOURCES.items():
-                future = executor.submit(_self._scrape_twitter_profile, name, info, timeout=15)
+            for name, info in sorted_sources:
+                # Reduced timeout for faster failures
+                future = executor.submit(_self._scrape_twitter_profile, name, info, timeout=12)
                 futures.append((future, name))
 
             for future, source_name in futures:
                 try:
-                    # Wait max 20 seconds per source
-                    news_items = future.result(timeout=20)
+                    # Wait max 15 seconds per source (down from 20)
+                    news_items = future.result(timeout=15)
 
                     # Deduplicate based on text similarity
                     unique_items = []
@@ -342,7 +358,7 @@ class TwitterFinanceMonitor:
                         logger.info(f"Fetched {len(unique_items)} unique tweets from {source_name}")
 
                 except FuturesTimeoutError:
-                    logger.warning(f"Timeout scraping {source_name}")
+                    logger.warning(f"Timeout scraping {source_name} - skipping")
                 except Exception as e:
                     logger.error(f"Error processing {source_name}: {e}")