init project

.env.example

@@ -0,0 +1,16 @@
+# Financial Platform Environment Variables
+
+# DeepSeek API Key (for AI-powered insights)
+DEEPSEEK_API_KEY=your-deepseek-api-key-here
+
+# News Service URL (for news aggregation with sentiment analysis)
+NEWS_SERVICE_URL=http://localhost:5000
+
+# Alpha Vantage API Key (optional, for forex data)
+ALPHA_VANTAGE_KEY=your-alpha-vantage-key-here
+
+# Twitter/X Credentials (for real-time news monitoring via Twikit)
+# Create a Twitter account or use existing credentials
+TWITTER_USERNAME=your-twitter-username
+TWITTER_EMAIL=your-twitter-email@example.com
+TWITTER_PASSWORD=your-twitter-password

.gitignore

@@ -0,0 +1,42 @@
+# Ignore virtual environment directories
+.venv/
+# Ignore environment variable files
+.env
+# Ignore lock files
+uv.lock
+# Ignore Python bytecode files
+*.pyc
+*.pyo
+__pycache__/
+*/__pycache__/
+**/__pycache__/
+# Ignore Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+# Ignore IDE specific files
+.idea/
+# Ignore logs
+logs/
+# ML model files
+ml_models/
+# Ignore experimental result files
+exp_results/
+# Ignore png and jpg files
+*.png
+*.jpg
+# Ignore .ruff
+.ruff_cache
+# Test files
+test_*
+test_*.py
+*_test.py
+tests/__pycache__/
+# Ignore md files
+*.md
+#
+docs/
+#
+*_example.py
+#
+tests/
+#
+README_old.md

Dockerfile

@@ -0,0 +1,72 @@
+FROM python:3.13-slim
+
+WORKDIR /app
+
+# Install system dependencies for Playwright and Chromium
+RUN apt-get update && apt-get install -y \
+    # Build tools
+    build-essential \
+    git \
+    # Chromium browser and driver
+    chromium \
+    chromium-driver \
+    # Playwright dependencies
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libdbus-1-3 \
+    libxkbcommon0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libasound2 \
+    libatspi2.0-0 \
+    libxshmfence1 \
+    # Utilities
+    curl \
+    wget \
+    ca-certificates \
+    fonts-liberation \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Find Chromium installation and create symlink if needed
+RUN if [ -f /usr/bin/chromium-browser ]; then \
+        ln -sf /usr/bin/chromium-browser /usr/bin/chromium; \
+    elif [ -f /usr/lib/chromium/chromium ]; then \
+        ln -sf /usr/lib/chromium/chromium /usr/bin/chromium; \
+    fi
+
+# Verify Chromium is accessible
+RUN which chromium || (echo "ERROR: Chromium not found!" && exit 1)
+
+# Set Playwright to use system Chromium
+ENV PLAYWRIGHT_BROWSERS_PATH=0
+ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
+
+# Copy application code
+COPY . .
+
+# Set Streamlit configuration for HuggingFace Spaces
+ENV STREAMLIT_SERVER_PORT=7860
+ENV STREAMLIT_SERVER_ADDRESS=0.0.0.0
+ENV STREAMLIT_SERVER_HEADLESS=true
+ENV STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
+
+# Expose Streamlit port
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK CMD curl --fail http://localhost:7860/_stcore/health || exit 1
+
+# Run Streamlit (corrected app file path from main.py to app.py)
+CMD ["streamlit", "run", "app/app.py", "--server.port=7860", "--server.address=0.0.0.0"]

README.md

@@ -1,12 +1,193 @@
 ---
 title: UnifiedFinancialPlatform
-emoji: 🏢
-colorFrom: yellow
-colorTo: pink
+emoji: 📈
+colorFrom: blue
+colorTo: green
 sdk: docker
+app_port: 7860
 pinned: false
 license: apache-2.0
-short_description: Unified Financial Platform
+short_description: Unified Financial Platform. Multi-asset analysis with OpenBB and AI insights
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Financial Analysis Platform
+
+A comprehensive multi-asset financial analysis platform built with Streamlit, providing real-time data, technical indicators, and AI-powered insights.
+
+## Features
+
+### 📈 Stock Analysis
+- Real-time stock price data from OpenBB
+- Technical indicators (SMA, EMA, RSI)
+- Company profile and financial statements
+- Revenue and net income trends
+- TradingView chart integration
+- Profitability metrics analysis
+
+### ₿ Cryptocurrency (Coming Soon)
+- Real-time cryptocurrency prices
+- Market cap and 24h volume
+- Technical indicators for crypto assets
+- TradingView crypto charts
+
+### 💱 Forex Trading (Coming Soon)
+- Foreign exchange rate analysis
+- Major, minor, and exotic pairs
+- Pip calculator
+- Economic calendar integration
+
+### 🔍 Market Screener (Coming Soon)
+- Multi-criteria filtering
+- Technical pattern recognition
+- Sort by volume, price change, RSI
+- Export results to CSV
+
+### 📰 News & AI Dashboard ✅ LIVE
+- **23 Premium Sources** across 4 tiers for comprehensive coverage
+  - **Tier 1**: Bloomberg (×2), Reuters, FT, WSJ, The Economist, CNBC, MarketWatch (8 sources)
+  - **Tier 2**: BBC World, AFP, Al Jazeera, Politico, DW News (5 sources)
+  - **Tier 3**: Federal Reserve (2.0x), ECB (2.0x), Lagarde, BoE, IMF, World Bank, US Treasury (7 sources)
+  - **Tier 4**: Zero Hedge, First Squawk, Live Squawk (3 sources)
+- **Low-latency monitoring** with 3-minute cache for trading decisions
+- **Intelligent categorization**: Macro, Markets, Geopolitical
+- **Professional sentiment analysis** (Positive/Negative/Neutral)
+- **Weighted impact scoring**: Source credibility × engagement × recency
+- **Breaking news detection** with instant alerts and priority display
+- **Smart filtering** by category, sentiment, and impact level
+- **Auto-refresh mode** for continuous monitoring during trading hours
+- Powered by **Twikit** for real-time Twitter/X intelligence (free, no API costs)
+
+## Installation
+
+1. Clone the repository:
+```bash
+git clone <repository-url>
+cd FinancialPlatform
+```
+
+2. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+3. Create a `.env` file based on `.env.example`:
+```bash
+cp .env.example .env
+```
+
+4. Configure your API keys and Twitter credentials in `.env`:
+```
+DEEPSEEK_API_KEY=your-key-here
+NEWS_SERVICE_URL=http://localhost:5000
+ALPHA_VANTAGE_KEY=your-key-here
+
+# Twitter/X Credentials (required for real-time news monitoring)
+TWITTER_USERNAME=your-twitter-username
+TWITTER_EMAIL=your-email@example.com
+TWITTER_PASSWORD=your-password
+```
+
+**Note**: Twitter credentials are required for real-time news monitoring. Without credentials, the system will use demo/mock data.
+
+## Usage
+
+Run the application:
+```bash
+streamlit run app/app.py
+```
+
+The application will open in your default web browser at `http://localhost:8501`.
+
+## Project Structure
+
+```
+FinancialPlatform/
+├── app/
+│   ├── app.py                      # Main landing page
+│   ├── pages/
+│   │   ├── 01_Stocks.py           # Stock analysis page
+│   │   ├── 02_Crypto.py           # Cryptocurrency analysis
+│   │   ├── 03_Forex.py            # Forex analysis
+│   │   ├── 04_Screener.py         # Market screener
+│   │   └── 05_Dashboard.py        # News & AI dashboard
+│   ├── components/
+│   │   ├── chart.py               # Chart creation utilities
+│   │   ├── data_sources.py        # Data fetching functions
+│   │   ├── ui.py                  # UI component functions
+│   │   └── styles.py              # Dark theme CSS
+│   └── utils/
+│       ├── config.py              # Configuration management
+│       └── formatters.py          # Data formatting utilities
+├── requirements.txt
+├── .env.example
+└── README.md
+```
+
+## Technology Stack
+
+- **Frontend**: Streamlit
+- **Data Sources**: OpenBB SDK, yfinance
+- **Charting**: Plotly, TradingView widgets
+- **AI**: DeepSeek API (planned)
+- **Styling**: Custom CSS with dark theme
+
+## Features in Development
+
+- [ ] Cryptocurrency data integration (Binance API)
+- [ ] Forex data integration (Alpha Vantage)
+- [ ] Market screener with advanced filters
+- [ ] News aggregation service
+- [ ] AI-powered trading insights
+- [ ] Sentiment analysis
+- [ ] Additional technical indicators (MACD, Bollinger Bands, ATR)
+
+## Configuration
+
+### Environment Variables
+
+- `DEEPSEEK_API_KEY`: API key for AI-powered insights
+- `NEWS_SERVICE_URL`: URL for news aggregation service
+- `ALPHA_VANTAGE_KEY`: API key for forex data (optional)
+
+### Cache Settings
+
+Data caching is configured in `utils/config.py`:
+- Price data: 1 hour TTL
+- Fundamental data: 24 hours TTL
+- News data: 15 minutes TTL
+
+## Deployment
+
+### HuggingFace Spaces
+
+This application is optimized for deployment on HuggingFace Spaces:
+
+1. Create a new Space on HuggingFace
+2. Set the Space type to "Streamlit"
+3. Add your environment variables in the Space settings:
+   - `DEEPSEEK_API_KEY`
+   - `NEWS_SERVICE_URL`
+   - `ALPHA_VANTAGE_KEY`
+4. Push your code to the Space repository
+
+### Local Development
+
+For local development with hot-reload:
+```bash
+streamlit run app/app.py --server.runOnSave=true
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+Apache 2.0 License
+
+## Acknowledgments
+
+- OpenBB for financial data API
+- TradingView for chart widgets
+- Streamlit for the amazing web framework
+

app/app.py

@@ -0,0 +1,163 @@
+"""Financial Analysis Dashboard - Main Application Landing Page."""
+
+import streamlit as st
+from components.styles import DARK_THEME_CSS
+
+# ---- Configuration ----
+st.set_page_config(
+    page_title="Financial Dashboard",
+    page_icon="📈",
+    layout="wide",
+    initial_sidebar_state="expanded",
+    menu_items={
+        "About": "A professional financial analysis platform with multi-asset support"
+    }
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# ---- Header ----
+st.markdown("# 📈 Financial Analysis Platform")
+st.markdown("### Professional multi-asset analysis with technical indicators, AI insights, and real-time data")
+
+st.markdown("---")
+
+# ---- Feature Overview ----
+col1, col2, col3 = st.columns(3)
+
+with col1:
+    st.markdown("""
+    <div style="padding: 1.5rem; background: linear-gradient(135deg, #1f2937 0%, #111827 100%); border-radius: 10px; border: 1px solid #30363d;">
+        <h3>📈 Stock Analysis</h3>
+        <p>Comprehensive stock analysis with technical indicators, financial metrics, and TradingView charts.</p>
+        <ul>
+            <li>Real-time price data</li>
+            <li>Technical indicators (SMA, EMA, RSI)</li>
+            <li>Financial statements</li>
+            <li>Company profiles</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+with col2:
+    st.markdown("""
+    <div style="padding: 1.5rem; background: linear-gradient(135deg, #1f2937 0%, #111827 100%); border-radius: 10px; border: 1px solid #30363d;">
+        <h3>₿ Cryptocurrency</h3>
+        <p>Track and analyze major cryptocurrencies with real-time market data.</p>
+        <ul>
+            <li>BTC, ETH, and major altcoins</li>
+            <li>24h volume & market cap</li>
+            <li>Price charts & indicators</li>
+            <li>Market sentiment</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+with col3:
+    st.markdown("""
+    <div style="padding: 1.5rem; background: linear-gradient(135deg, #1f2937 0%, #111827 100%); border-radius: 10px; border: 1px solid #30363d;">
+        <h3>💱 Forex Trading</h3>
+        <p>Foreign exchange analysis for major, minor, and exotic currency pairs.</p>
+        <ul>
+            <li>Major pairs (EUR/USD, GBP/USD)</li>
+            <li>Real-time exchange rates</li>
+            <li>Technical analysis</li>
+            <li>Pip calculator</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+st.markdown("<br>", unsafe_allow_html=True)
+
+col4, col5 = st.columns(2)
+
+with col4:
+    st.markdown("""
+    <div style="padding: 1.5rem; background: linear-gradient(135deg, #1f2937 0%, #111827 100%); border-radius: 10px; border: 1px solid #30363d;">
+        <h3>🔍 Market Screener</h3>
+        <p>Advanced screening tools to find investment opportunities across markets.</p>
+        <ul>
+            <li>Multi-criteria filtering</li>
+            <li>Technical pattern recognition</li>
+            <li>Sort by volume, price change, RSI</li>
+            <li>Export results to CSV</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+with col5:
+    st.markdown("""
+    <div style="padding: 1.5rem; background: linear-gradient(135deg, #1f2937 0%, #111827 100%); border-radius: 10px; border: 1px solid #30363d;">
+        <h3>🤖 News & AI Dashboard</h3>
+        <p>AI-powered market insights with sentiment analysis and trading recommendations.</p>
+        <ul>
+            <li>Real-time news aggregation</li>
+            <li>Sentiment analysis</li>
+            <li>AI trading insights</li>
+            <li>Market trend detection</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+st.markdown("---")
+
+# ---- Quick Start ----
+st.markdown("## 🚀 Quick Start")
+st.markdown("Use the sidebar to navigate to different sections:")
+
+quick_col1, quick_col2, quick_col3 = st.columns(3)
+
+with quick_col1:
+    if st.button("📈 Stock Analysis", use_container_width=True):
+        st.switch_page("pages/01_Stocks.py")
+
+with quick_col2:
+    if st.button("₿ Cryptocurrency", use_container_width=True):
+        st.info("Coming soon!")
+
+with quick_col3:
+    if st.button("💱 Forex Trading", use_container_width=True):
+        st.info("Coming soon!")
+
+st.markdown("<br>", unsafe_allow_html=True)
+
+quick_col4, quick_col5 = st.columns(2)
+
+with quick_col4:
+    if st.button("🔍 Market Screener", use_container_width=True):
+        st.info("Coming soon!")
+
+with quick_col5:
+    if st.button("🤖 News & AI Dashboard", use_container_width=True):
+        st.info("Coming soon!")
+
+st.markdown("---")
+
+# ---- Sidebar ----
+with st.sidebar:
+    st.markdown("## 📋 Navigation")
+    st.info("Select a page from the sidebar to get started.")
+
+    st.markdown("---")
+    st.markdown("## ℹ️ About")
+    st.markdown("""
+    This platform provides comprehensive financial analysis across multiple asset classes:
+
+    - **Stocks**: Technical & fundamental analysis
+    - **Crypto**: Real-time cryptocurrency tracking
+    - **Forex**: Currency pair analysis
+    - **Screener**: Find investment opportunities
+    - **Dashboard**: AI-powered insights
+    """)
+
+    st.markdown("---")
+    st.markdown("### 🔧 Features")
+    st.markdown("""
+    - ✅ Real-time data
+    - ✅ Technical indicators
+    - ✅ TradingView integration
+    - ✅ Dark theme UI
+    - ✅ AI-powered insights
+    - ✅ News sentiment analysis
+    """)

app/charts.py

@@ -0,0 +1,142 @@
+"""Chart creation utilities for the financial dashboard."""
+
+import plotly.graph_objects as go
+import pandas as pd
+
+
+def get_dark_theme_layout():
+    """Get common dark theme layout settings for all charts."""
+    return dict(
+        plot_bgcolor="#0d1117",
+        paper_bgcolor="#0e1117",
+        font=dict(color="#e6edf3", size=12, family="Arial, sans-serif"),
+        xaxis=dict(
+            gridcolor="#30363d",
+            showgrid=True,
+            zeroline=False,
+            color="#8b949e"
+        ),
+        yaxis=dict(
+            gridcolor="#30363d",
+            showgrid=True,
+            zeroline=False,
+            color="#8b949e"
+        ),
+        legend=dict(
+            bgcolor="rgba(13, 17, 23, 0.8)",
+            bordercolor="#30363d",
+            borderwidth=1,
+            font=dict(color="#e6edf3")
+        ),
+        hoverlabel=dict(
+            bgcolor="#0d1117",
+            bordercolor="#30363d",
+            font=dict(color="#e6edf3")
+        )
+    )
+
+
+def create_price_chart(df: pd.DataFrame, symbol: str, period: int) -> go.Figure:
+    """Create price chart with SMA and EMA indicators."""
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["close"],
+        name="Close Price",
+        line=dict(color="#0066ff", width=2.5)
+    ))
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["SMA"],
+        name=f"SMA {period}",
+        line=dict(color="#00d084", width=2, dash="dash")
+    ))
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["EMA"],
+        name=f"EMA {period}",
+        line=dict(color="#ffa500", width=2, dash="dot")
+    ))
+
+    layout = get_dark_theme_layout()
+    fig.update_layout(
+        title=f"{symbol} - Price with Moving Averages",
+        xaxis_title="Date",
+        yaxis_title="Price ($)",
+        hovermode="x unified",
+        template="plotly_dark",
+        height=500,
+        margin=dict(l=0, r=0, t=40, b=0),
+        **layout
+    )
+
+    return fig
+
+
+def create_rsi_chart(df: pd.DataFrame, symbol: str) -> go.Figure:
+    """Create RSI (Relative Strength Index) chart."""
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["RSI"],
+        name="RSI",
+        line=dict(color="#ff3838", width=2.5),
+        fill="tozeroy",
+        fillcolor="rgba(255, 56, 56, 0.15)"
+    ))
+
+    fig.add_hline(y=70, line_dash="dash", line_color="rgba(255, 165, 0, 0.6)",
+                  annotation_text="Overbought (70)")
+    fig.add_hline(y=30, line_dash="dash", line_color="rgba(0, 208, 132, 0.6)",
+                  annotation_text="Oversold (30)")
+    fig.add_hline(y=50, line_dash="dot", line_color="rgba(139, 148, 158, 0.3)")
+
+    layout = get_dark_theme_layout()
+    layout["yaxis"]["range"] = [0, 100]
+
+    fig.update_layout(
+        title=f"{symbol} - Relative Strength Index (RSI)",
+        xaxis_title="Date",
+        yaxis_title="RSI",
+        hovermode="x unified",
+        template="plotly_dark",
+        height=500,
+        margin=dict(l=0, r=0, t=40, b=0),
+        **layout
+    )
+
+    return fig
+
+
+def create_financial_chart(income_data: pd.DataFrame) -> go.Figure:
+    """Create financial revenue and net income chart."""
+    fig = go.Figure()
+
+    fig.add_trace(go.Bar(
+        x=income_data['period_ending'],
+        y=income_data['total_revenue'],
+        name="Total Revenue",
+        marker=dict(color='#0066ff', opacity=0.9),
+        yaxis='y1'
+    ))
+
+    fig.add_trace(go.Bar(
+        x=income_data['period_ending'],
+        y=income_data['net_income'],
+        name="Net Income",
+        marker=dict(color='#00d084', opacity=0.9),
+        yaxis='y1'
+    ))
+
+    layout = get_dark_theme_layout()
+    fig.update_layout(
+        title="Revenue & Net Income (Annual)",
+        xaxis_title="Period",
+        yaxis_title="Amount ($)",
+        hovermode="x unified",
+        template="plotly_dark",
+        height=400,
+        barmode='group',
+        margin=dict(l=0, r=0, t=40, b=0),
+        **layout
+    )
+
+    return fig

app/components/__init__.py

@@ -0,0 +1 @@
+"""Components package for financial platform UI."""

app/components/chart.py

@@ -0,0 +1,142 @@
+"""Chart creation utilities for the financial dashboard."""
+
+import plotly.graph_objects as go
+import pandas as pd
+
+
+def get_dark_theme_layout():
+    """Get common dark theme layout settings for all charts."""
+    return dict(
+        plot_bgcolor="#0d1117",
+        paper_bgcolor="#0e1117",
+        font=dict(color="#e6edf3", size=12, family="Arial, sans-serif"),
+        xaxis=dict(
+            gridcolor="#30363d",
+            showgrid=True,
+            zeroline=False,
+            color="#8b949e"
+        ),
+        yaxis=dict(
+            gridcolor="#30363d",
+            showgrid=True,
+            zeroline=False,
+            color="#8b949e"
+        ),
+        legend=dict(
+            bgcolor="rgba(13, 17, 23, 0.8)",
+            bordercolor="#30363d",
+            borderwidth=1,
+            font=dict(color="#e6edf3")
+        ),
+        hoverlabel=dict(
+            bgcolor="#0d1117",
+            bordercolor="#30363d",
+            font=dict(color="#e6edf3")
+        )
+    )
+
+
+def create_price_chart(df: pd.DataFrame, symbol: str, period: int) -> go.Figure:
+    """Create price chart with SMA and EMA indicators."""
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["close"],
+        name="Close Price",
+        line=dict(color="#0066ff", width=2.5)
+    ))
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["SMA"],
+        name=f"SMA {period}",
+        line=dict(color="#00d084", width=2, dash="dash")
+    ))
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["EMA"],
+        name=f"EMA {period}",
+        line=dict(color="#ffa500", width=2, dash="dot")
+    ))
+
+    layout = get_dark_theme_layout()
+    fig.update_layout(
+        title=f"{symbol} - Price with Moving Averages",
+        xaxis_title="Date",
+        yaxis_title="Price ($)",
+        hovermode="x unified",
+        template="plotly_dark",
+        height=500,
+        margin=dict(l=0, r=0, t=40, b=0),
+        **layout
+    )
+
+    return fig
+
+
+def create_rsi_chart(df: pd.DataFrame, symbol: str) -> go.Figure:
+    """Create RSI (Relative Strength Index) chart."""
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatter(
+        x=df.index, y=df["RSI"],
+        name="RSI",
+        line=dict(color="#ff3838", width=2.5),
+        fill="tozeroy",
+        fillcolor="rgba(255, 56, 56, 0.15)"
+    ))
+
+    fig.add_hline(y=70, line_dash="dash", line_color="rgba(255, 165, 0, 0.6)",
+                  annotation_text="Overbought (70)")
+    fig.add_hline(y=30, line_dash="dash", line_color="rgba(0, 208, 132, 0.6)",
+                  annotation_text="Oversold (30)")
+    fig.add_hline(y=50, line_dash="dot", line_color="rgba(139, 148, 158, 0.3)")
+
+    layout = get_dark_theme_layout()
+    layout["yaxis"]["range"] = [0, 100]
+
+    fig.update_layout(
+        title=f"{symbol} - Relative Strength Index (RSI)",
+        xaxis_title="Date",
+        yaxis_title="RSI",
+        hovermode="x unified",
+        template="plotly_dark",
+        height=500,
+        margin=dict(l=0, r=0, t=40, b=0),
+        **layout
+    )
+
+    return fig
+
+
+def create_financial_chart(income_data: pd.DataFrame) -> go.Figure:
+    """Create financial revenue and net income chart."""
+    fig = go.Figure()
+
+    fig.add_trace(go.Bar(
+        x=income_data['period_ending'],
+        y=income_data['total_revenue'],
+        name="Total Revenue",
+        marker=dict(color='#0066ff', opacity=0.9),
+        yaxis='y1'
+    ))
+
+    fig.add_trace(go.Bar(
+        x=income_data['period_ending'],
+        y=income_data['net_income'],
+        name="Net Income",
+        marker=dict(color='#00d084', opacity=0.9),
+        yaxis='y1'
+    ))
+
+    layout = get_dark_theme_layout()
+    fig.update_layout(
+        title="Revenue & Net Income (Annual)",
+        xaxis_title="Period",
+        yaxis_title="Amount ($)",
+        hovermode="x unified",
+        template="plotly_dark",
+        height=400,
+        barmode='group',
+        margin=dict(l=0, r=0, t=40, b=0),
+        **layout
+    )
+
+    return fig

app/components/data_sources.py

@@ -0,0 +1,81 @@
+"""Data fetching and processing utilities for the financial dashboard."""
+
+import pandas as pd
+from openbb import sdk
+import streamlit as st
+
+
+@st.cache_data(ttl=3600)
+def load_stock_data(symbol: str) -> pd.DataFrame:
+    """Load historical stock price data with caching."""
+    df = sdk.equity.price.historical(symbol=symbol).to_dataframe()
+    return df
+
+
+@st.cache_data(ttl=86400)
+def load_company_profile(symbol: str):
+    """Load company profile information with caching."""
+    profile_response = sdk.equity.profile(symbol=symbol)
+    profile_info = profile_response.results[0] if hasattr(profile_response, 'results') and profile_response.results else None
+    return profile_info
+
+
+@st.cache_data(ttl=86400)
+def load_income_statement(symbol: str) -> pd.DataFrame:
+    """Load company income statement data with caching."""
+    income_stmt = sdk.equity.fundamental.income(symbol=symbol).to_dataframe()
+    return income_stmt
+
+
+def calculate_technical_indicators(df: pd.DataFrame, period: int) -> pd.DataFrame:
+    """Calculate SMA, EMA, and RSI indicators."""
+    df = df.copy()
+    df["SMA"] = df["close"].rolling(period).mean()
+    df["EMA"] = df["close"].ewm(span=period, adjust=False).mean()
+
+    # Calculate RSI
+    delta = df["close"].diff()
+    gain = delta.clip(lower=0)
+    loss = -1 * delta.clip(upper=0)
+    avg_gain = gain.rolling(period).mean()
+    avg_loss = loss.rolling(period).mean()
+    rs = avg_gain / avg_loss
+    df["RSI"] = 100 - (100 / (1 + rs))
+
+    return df
+
+
+def get_price_metrics(df: pd.DataFrame) -> dict:
+    """Calculate key price metrics."""
+    current_price = df["close"].iloc[-1]
+    prev_close = df["close"].iloc[-2] if len(df) > 1 else df["close"].iloc[0]
+    price_change = current_price - prev_close
+    price_change_pct = (price_change / prev_close) * 100 if prev_close != 0 else 0
+
+    return {
+        "current_price": current_price,
+        "price_change": price_change,
+        "price_change_pct": price_change_pct,
+        "high_52w": df['high'].max(),
+        "low_52w": df['low'].min(),
+    }
+
+
+def get_profitability_metrics(income_data: pd.Series) -> dict:
+    """Calculate profitability metrics from income statement."""
+    total_rev = income_data.get('total_revenue', 0)
+    gross_prof = income_data.get('gross_profit', 0)
+    net_inc = income_data.get('net_income', 0)
+    operating_inc = income_data.get('operating_income', 0)
+
+    metrics = {}
+
+    if total_rev and total_rev > 0:
+        metrics["gross_margin"] = (gross_prof / total_rev) * 100 if pd.notna(gross_prof) else 0
+        metrics["net_margin"] = (net_inc / total_rev) * 100 if pd.notna(net_inc) else 0
+        if operating_inc:
+            metrics["operating_margin"] = (operating_inc / total_rev) * 100
+    else:
+        metrics = {"gross_margin": 0, "net_margin": 0}
+
+    return metrics

app/components/news.py

@@ -0,0 +1,723 @@
+"""News display components for the financial dashboard."""
+
+import streamlit as st
+import pandas as pd
+from datetime import datetime
+import html as html_module
+
+
+def display_tradingview_news_card(news_item: dict):
+    """Display a single news card with TradingView-inspired styling."""
+
+    # Calculate time ago
+    time_diff = datetime.now() - news_item['timestamp']
+    if time_diff.seconds < 60:
+        time_ago = f"{time_diff.seconds}s ago"
+    elif time_diff.seconds < 3600:
+        time_ago = f"{time_diff.seconds // 60}m ago"
+    else:
+        hours = time_diff.seconds // 3600
+        time_ago = f"{hours}h ago" if hours < 24 else f"{time_diff.days}d ago"
+
+    # Impact badge colors (TradingView style)
+    impact_colors = {
+        'high': '#F23645',     # Red
+        'medium': '#FF9800',   # Orange
+        'low': '#089981'       # Green
+    }
+
+    # Sentiment colors
+    sentiment_colors = {
+        'positive': '#089981',   # Green
+        'negative': '#F23645',   # Red
+        'neutral': '#787B86'     # Gray
+    }
+
+    impact_color = impact_colors.get(news_item['impact'], '#787B86')
+    sentiment_color = sentiment_colors.get(news_item['sentiment'], '#787B86')
+
+    # Escape HTML in text
+    summary = html_module.escape(news_item.get('summary', '').strip())
+    source = html_module.escape(news_item['source'])
+    category = html_module.escape(news_item['category'])
+    url = html_module.escape(news_item['url'])
+
+    # TradingView-style card HTML
+    card_html = f"""
+    <div style="
+        background: linear-gradient(135deg, #1E222D 0%, #131722 100%);
+        border: 1px solid #2A2E39;
+        border-radius: 8px;
+        padding: 16px;
+        margin-bottom: 12px;
+        transition: all 0.2s ease;
+        cursor: pointer;
+        position: relative;
+        overflow: hidden;
+    " onmouseover="this.style.borderColor='#3861FB'; this.style.transform='translateY(-2px)'; this.style.boxShadow='0 4px 12px rgba(56, 97, 251, 0.15)';"
+       onmouseout="this.style.borderColor='#2A2E39'; this.style.transform='translateY(0)'; this.style.boxShadow='none';">
+
+        <!-- Left colored indicator bar -->
+        <div style="
+            position: absolute;
+            left: 0;
+            top: 0;
+            bottom: 0;
+            width: 3px;
+            background: {impact_color};
+        "></div>
+
+        <!-- Header row -->
+        <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; margin-left: 8px;">
+            <div style="display: flex; align-items: center; gap: 8px; flex-wrap: wrap;">
+                <span style="
+                    color: #3861FB;
+                    font-weight: 600;
+                    font-size: 13px;
+                    font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+                ">{source}</span>
+
+                <span style="
+                    background: {impact_color};
+                    color: white;
+                    padding: 2px 8px;
+                    border-radius: 4px;
+                    font-size: 10px;
+                    font-weight: 700;
+                    letter-spacing: 0.5px;
+                ">{news_item['impact'].upper()}</span>
+
+                <span style="
+                    color: {sentiment_color};
+                    font-size: 11px;
+                    font-weight: 600;
+                    padding: 2px 6px;
+                    border: 1px solid {sentiment_color};
+                    border-radius: 4px;
+                ">{'▲' if news_item['sentiment'] == 'positive' else '▼' if news_item['sentiment'] == 'negative' else '●'} {news_item['sentiment'].upper()}</span>
+
+                <span style="
+                    color: #787B86;
+                    font-size: 11px;
+                    background: rgba(120, 123, 134, 0.1);
+                    padding: 2px 6px;
+                    border-radius: 4px;
+                ">#{category}</span>
+            </div>
+
+            <span style="color: #787B86; font-size: 11px; white-space: nowrap;">{time_ago}</span>
+        </div>
+
+        <!-- News summary -->
+        <div style="
+            color: #D1D4DC;
+            font-size: 14px;
+            line-height: 1.5;
+            margin-bottom: 8px;
+            margin-left: 8px;
+            font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;
+        ">{summary}</div>
+
+        <!-- Read more link -->
+        <a href="{url}" target="_blank" style="
+            color: #3861FB;
+            font-size: 12px;
+            text-decoration: none;
+            margin-left: 8px;
+            display: inline-flex;
+            align-items: center;
+            gap: 4px;
+            font-weight: 500;
+        " onmouseover="this.style.color='#5880FF';" onmouseout="this.style.color='#3861FB';">
+            Read Full Story →
+        </a>
+    </div>
+    """
+
+    st.markdown(card_html, unsafe_allow_html=True)
+
+
+def display_news_card(news_item: dict):
+    """Wrapper to maintain compatibility - calls TradingView-style card."""
+    display_tradingview_news_card(news_item)
+
+
+def display_scrollable_news_section(df: pd.DataFrame, section_title: str, section_icon: str,
+                                    section_subtitle: str, max_items: int = 20, height: str = "600px"):
+    """Display a scrollable news section with TradingView styling."""
+
+    if df.empty:
+        st.markdown(f"""
+        <div style="
+            background: linear-gradient(135deg, #1E222D 0%, #131722 100%);
+            border: 1px solid #2A2E39;
+            border-radius: 8px;
+            padding: 20px;
+            text-align: center;
+            color: #787B86;
+        ">
+            <p style="font-size: 16px; margin: 0;">📭 No news available for this section</p>
+        </div>
+        """, unsafe_allow_html=True)
+        return
+
+    # Build header HTML (no leading whitespace)
+    header_html = f"""<div style="background: linear-gradient(135deg, #2A2E39 0%, #1E222D 100%); border: 1px solid #363A45; border-radius: 8px 8px 0 0; padding: 16px 20px; margin-bottom: 0;">
+<div style="display: flex; justify-content: space-between; align-items: center;">
+<div>
+<h3 style="color: #D1D4DC; margin: 0; font-size: 18px; font-weight: 600;">{section_icon} {section_title}</h3>
+<p style="color: #787B86; margin: 4px 0 0 0; font-size: 12px;">{section_subtitle}</p>
+</div>
+<div style="background: rgba(56, 97, 251, 0.15); color: #3861FB; padding: 6px 12px; border-radius: 6px; font-size: 13px; font-weight: 600;">{len(df.head(max_items))} stories</div>
+</div>
+</div>"""
+
+    # Render header
+    st.markdown(header_html, unsafe_allow_html=True)
+
+    # Build all news cards HTML
+    news_cards_html = ""
+    for idx, row in df.head(max_items).iterrows():
+        news_item = row.to_dict()
+
+        # Calculate time ago
+        time_diff = datetime.now() - news_item['timestamp']
+        if time_diff.seconds < 60:
+            time_ago = f"{time_diff.seconds}s ago"
+        elif time_diff.seconds < 3600:
+            time_ago = f"{time_diff.seconds // 60}m ago"
+        else:
+            hours = time_diff.seconds // 3600
+            time_ago = f"{hours}h ago" if hours < 24 else f"{time_diff.days}d ago"
+
+        # Impact and sentiment colors
+        impact_colors = {'high': '#F23645', 'medium': '#FF9800', 'low': '#089981'}
+        sentiment_colors = {'positive': '#089981', 'negative': '#F23645', 'neutral': '#787B86'}
+
+        impact_color = impact_colors.get(news_item['impact'], '#787B86')
+        sentiment_color = sentiment_colors.get(news_item['sentiment'], '#787B86')
+
+        # Escape HTML
+        title = html_module.escape(str(news_item.get('title', '')).strip())
+        summary = html_module.escape(str(news_item.get('summary', '')).strip())
+        source = html_module.escape(news_item['source'])
+        category = html_module.escape(news_item['category'])
+        url = html_module.escape(news_item['url'])
+
+        sentiment_symbol = '▲' if news_item['sentiment'] == 'positive' else '▼' if news_item['sentiment'] == 'negative' else '●'
+
+        # Build card HTML (no leading whitespace)
+        news_cards_html += f"""<div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 16px; margin-bottom: 12px; transition: all 0.2s ease; cursor: pointer; position: relative; overflow: hidden;" onmouseover="this.style.borderColor='#3861FB'; this.style.transform='translateY(-2px)'; this.style.boxShadow='0 4px 12px rgba(56, 97, 251, 0.15)';" onmouseout="this.style.borderColor='#2A2E39'; this.style.transform='translateY(0)'; this.style.boxShadow='none';">
+<div style="position: absolute; left: 0; top: 0; bottom: 0; width: 3px; background: {impact_color};"></div>
+<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; margin-left: 8px;">
+<div style="display: flex; align-items: center; gap: 8px; flex-wrap: wrap;">
+<span style="color: #3861FB; font-weight: 600; font-size: 13px; font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;">{source}</span>
+<span style="background: {impact_color}; color: white; padding: 2px 8px; border-radius: 4px; font-size: 10px; font-weight: 700; letter-spacing: 0.5px;">{news_item['impact'].upper()}</span>
+<span style="color: {sentiment_color}; font-size: 11px; font-weight: 600; padding: 2px 6px; border: 1px solid {sentiment_color}; border-radius: 4px;">{sentiment_symbol} {news_item['sentiment'].upper()}</span>
+<span style="color: #787B86; font-size: 11px; background: rgba(120, 123, 134, 0.1); padding: 2px 6px; border-radius: 4px;">#{category}</span>
+</div>
+<span style="color: #787B86; font-size: 11px; white-space: nowrap;">{time_ago}</span>
+</div>
+<div style="color: #E0E3EB; font-size: 14px; font-weight: 600; margin-bottom: 6px; margin-left: 8px; font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;">{title if title else summary}</div>
+<div style="color: #D1D4DC; font-size: 13px; line-height: 1.5; margin-bottom: 8px; margin-left: 8px; font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif;">{summary if title else ''}</div>
+<a href="{url}" target="_blank" style="color: #3861FB; font-size: 12px; text-decoration: none; margin-left: 8px; display: inline-flex; align-items: center; gap: 4px; font-weight: 500;" onmouseover="this.style.color='#5880FF';" onmouseout="this.style.color='#3861FB';">Read Full Story →</a>
+</div>
+"""
+
+    # Generate unique class name to avoid conflicts
+    import random
+    unique_id = f"news-scroll-{random.randint(10000, 99999)}"
+
+    # Render scrollable container with all news cards using st.markdown (no leading whitespace)
+    scrollable_html = f"""<style>
+.{unique_id} {{
+height: {height};
+overflow-y: auto;
+background: #0D0E13;
+border: 1px solid #2A2E39;
+border-top: none;
+border-radius: 0 0 8px 8px;
+padding: 16px;
+}}
+.{unique_id}::-webkit-scrollbar {{
+width: 8px;
+}}
+.{unique_id}::-webkit-scrollbar-track {{
+background: #1E222D;
+border-radius: 4px;
+}}
+.{unique_id}::-webkit-scrollbar-thumb {{
+background: #363A45;
+border-radius: 4px;
+}}
+.{unique_id}::-webkit-scrollbar-thumb:hover {{
+background: #434651;
+}}
+</style>
+<div class="{unique_id}">
+{news_cards_html}
+</div>
+"""
+
+    st.markdown(scrollable_html, unsafe_allow_html=True)
+
+
+def display_news_feed(df: pd.DataFrame, max_items: int = 20):
+    """Display a feed of news items (legacy compatibility)."""
+
+    if df.empty:
+        st.info("📭 No news available. Adjust your filters or refresh the feed.")
+        return
+
+    # Display news items
+    for idx, row in df.head(max_items).iterrows():
+        display_tradingview_news_card(row.to_dict())
+
+
+def display_news_statistics(stats: dict):
+    """Display news feed statistics in metric cards."""
+
+    col1, col2, col3, col4 = st.columns(4)
+
+    with col1:
+        st.metric(
+            "Total Stories",
+            f"{stats['total']}",
+            help="Total news items in feed"
+        )
+
+    with col2:
+        st.metric(
+            "High Impact",
+            f"{stats['high_impact']}",
+            delta=f"{(stats['high_impact']/max(stats['total'], 1)*100):.0f}%",
+            help="High-impact market-moving news"
+        )
+
+    with col3:
+        st.metric(
+            "Breaking News",
+            f"{stats['breaking']}",
+            delta="LIVE" if stats['breaking'] > 0 else None,
+            help="Breaking news alerts"
+        )
+
+    with col4:
+        st.metric(
+            "Last Update",
+            stats['last_update'],
+            help="Time of last news fetch"
+        )
+
+
+def display_category_breakdown(stats: dict):
+    """Display news breakdown by category using Streamlit components."""
+
+    if 'by_category' not in stats:
+        return
+
+    st.markdown("### 📊 News by Category")
+
+    categories = stats['by_category']
+    total = sum(categories.values())
+
+    if total == 0:
+        st.info("No categorized news available")
+        return
+
+    col1, col2, col3 = st.columns(3)
+
+    with col1:
+        macro_count = categories.get('macro', 0)
+        macro_pct = (macro_count / total) * 100
+        with st.container():
+            st.markdown("**:blue[📈 MACRO]**")
+            st.markdown(f"# {macro_count}")
+            st.caption(f"{macro_pct:.1f}% of total")
+
+    with col2:
+        geo_count = categories.get('geopolitical', 0)
+        geo_pct = (geo_count / total) * 100
+        with st.container():
+            st.markdown("**:orange[🌍 GEOPOLITICAL]**")
+            st.markdown(f"# {geo_count}")
+            st.caption(f"{geo_pct:.1f}% of total")
+
+    with col3:
+        markets_count = categories.get('markets', 0)
+        markets_pct = (markets_count / total) * 100
+        with st.container():
+            st.markdown("**:green[💹 MARKETS]**")
+            st.markdown(f"# {markets_count}")
+            st.caption(f"{markets_pct:.1f}% of total")
+
+
+def display_breaking_news_banner(df: pd.DataFrame):
+    """Display breaking news banner at the top with TradingView styling and ML-based impact score."""
+
+    # With ML-based scoring, we trust that the passed DataFrame already contains
+    # the highest-impact news, so no need to filter by is_breaking
+    # (The scorer already selected the most impactful news)
+    if not df.empty:
+        latest = df.iloc[0]
+
+        # Escape HTML
+        summary = html_module.escape(latest.get('summary', '').strip())
+        source = html_module.escape(latest['source'])
+        url = html_module.escape(latest['url'])
+
+        # Get impact score if available
+        impact_score = latest.get('breaking_score', 0)
+        score_display = f"{impact_score:.1f}" if impact_score > 0 else "N/A"
+
+        # Determine score color and label
+        if impact_score >= 80:
+            score_color = "#FF3B30"  # Critical red
+            score_label = "CRITICAL"
+        elif impact_score >= 60:
+            score_color = "#FF9500"  # High orange
+            score_label = "HIGH"
+        elif impact_score >= 40:
+            score_color = "#FFCC00"  # Medium yellow
+            score_label = "MEDIUM"
+        else:
+            score_color = "#34C759"  # Low green
+            score_label = "LOW"
+
+        # Calculate time ago
+        time_diff = datetime.now() - latest['timestamp']
+        if time_diff.seconds < 60:
+            time_ago = f"{time_diff.seconds}s ago"
+        elif time_diff.seconds < 3600:
+            time_ago = f"{time_diff.seconds // 60}m ago"
+        else:
+            hours = time_diff.seconds // 3600
+            time_ago = f"{hours}h ago" if hours < 24 else f"{time_diff.days}d ago"
+
+        # TradingView-style breaking news banner with impact score (no leading whitespace)
+        banner_html = f"""<style>
+@keyframes pulse-glow {{
+0%, 100% {{ box-shadow: 0 0 20px rgba(242, 54, 69, 0.6); }}
+50% {{ box-shadow: 0 0 30px rgba(242, 54, 69, 0.9); }}
+}}
+@keyframes slide-in {{
+from {{ transform: translateX(-10px); opacity: 0; }}
+to {{ transform: translateX(0); opacity: 1; }}
+}}
+</style>
+<div style="background: linear-gradient(135deg, #F23645 0%, #C91B28 100%); border: 2px solid #FF6B78; border-radius: 12px; padding: 20px 24px; margin-bottom: 24px; animation: pulse-glow 2s ease-in-out infinite; position: relative; overflow: hidden;">
+<div style="position: absolute; top: 0; left: 0; right: 0; bottom: 0; background: repeating-linear-gradient(45deg, transparent, transparent 10px, rgba(255, 255, 255, 0.03) 10px, rgba(255, 255, 255, 0.03) 20px); pointer-events: none;"></div>
+<div style="position: relative; z-index: 1;">
+<div style="display: flex; align-items: center; gap: 16px; margin-bottom: 12px;">
+<div style="font-size: 32px; animation: pulse-glow 1s ease-in-out infinite; filter: drop-shadow(0 2px 8px rgba(0, 0, 0, 0.3));">🚨</div>
+<div style="flex: 1;">
+<div style="color: white; font-size: 14px; font-weight: 700; letter-spacing: 1.5px; text-transform: uppercase; margin-bottom: 4px; font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif; text-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);">⚡ Breaking News</div>
+<div style="color: rgba(255, 255, 255, 0.9); font-size: 11px; display: flex; align-items: center; gap: 8px; flex-wrap: wrap;">
+<span style="background: rgba(255, 255, 255, 0.2); padding: 2px 8px; border-radius: 4px; font-weight: 600;">{source}</span>
+<span style="opacity: 0.8;">•</span>
+<span style="opacity: 0.8;">{time_ago}</span>
+<span style="opacity: 0.8;">•</span>
+<span style="background: {score_color}; color: white; padding: 2px 8px; border-radius: 4px; font-weight: 700; font-size: 10px; letter-spacing: 0.5px;">📊 IMPACT: {score_display}/100 ({score_label})</span>
+</div>
+</div>
+<a href="{url}" target="_blank" style="background: white; color: #F23645; padding: 10px 20px; border-radius: 6px; font-size: 13px; font-weight: 700; text-decoration: none; display: inline-flex; align-items: center; gap: 6px; transition: all 0.2s ease; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2);" onmouseover="this.style.transform='translateY(-2px)'; this.style.boxShadow='0 4px 12px rgba(0, 0, 0, 0.3)';" onmouseout="this.style.transform='translateY(0)'; this.style.boxShadow='0 2px 8px rgba(0, 0, 0, 0.2)';">READ NOW →</a>
+</div>
+<div style="color: white; font-size: 16px; font-weight: 500; line-height: 1.5; margin-left: 48px; font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.2); animation: slide-in 0.5s ease-out;">{summary}</div>
+</div>
+</div>"""
+
+        st.markdown(banner_html, unsafe_allow_html=True)
+
+
+def display_prediction_card(prediction_item: dict):
+    """Display a single prediction market card with probability visualization."""
+
+    # Escape HTML in text
+    title = html_module.escape(prediction_item.get('title', '').strip())
+    source = html_module.escape(prediction_item['source'])
+    url = html_module.escape(prediction_item['url'])
+
+    # Get probabilities
+    yes_prob = prediction_item.get('yes_probability', 50.0)
+    no_prob = prediction_item.get('no_probability', 50.0)
+
+    # Determine bar color based on probabilities
+    if yes_prob > 60:
+        bar_color = '#089981'  # Green - likely YES
+        sentiment_text = 'YES LIKELY'
+    elif no_prob > 60:
+        bar_color = '#F23645'  # Red - likely NO
+        sentiment_text = 'NO LIKELY'
+    else:
+        bar_color = '#FF9800'  # Orange - balanced
+        sentiment_text = 'BALANCED'
+
+    # Format end date if available
+    end_date = prediction_item.get('end_date')
+    if end_date:
+        if isinstance(end_date, str):
+            end_date_display = end_date
+        else:
+            days_until = (end_date - datetime.now()).days
+            end_date_display = f"Closes in {days_until}d" if days_until > 0 else "Closed"
+    else:
+        end_date_display = ""
+
+    # Volume display
+    volume = prediction_item.get('volume', 0)
+    if volume > 1000000:
+        volume_display = f"${volume/1000000:.1f}M volume"
+    elif volume > 1000:
+        volume_display = f"${volume/1000:.1f}K volume"
+    elif volume > 0:
+        volume_display = f"${volume:.0f} volume"
+    else:
+        volume_display = ""
+
+    # Prediction card HTML
+    card_html = f"""
+    <div style="
+        background: linear-gradient(135deg, #1E222D 0%, #131722 100%);
+        border: 1px solid #2A2E39;
+        border-radius: 8px;
+        padding: 16px;
+        margin-bottom: 12px;
+        transition: all 0.2s ease;
+        cursor: pointer;
+    " onmouseover="this.style.borderColor='#3861FB'; this.style.transform='translateY(-2px)';"
+       onmouseout="this.style.borderColor='#2A2E39'; this.style.transform='translateY(0)';">
+
+        <!-- Header -->
+        <div style="margin-bottom: 12px;">
+            <div style="display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 8px;">
+                <span style="color: #3861FB; font-weight: 600; font-size: 13px;">{source}</span>
+                <span style="
+                    background: {bar_color};
+                    color: white;
+                    padding: 2px 8px;
+                    border-radius: 4px;
+                    font-size: 10px;
+                    font-weight: 700;
+                ">{sentiment_text}</span>
+            </div>
+            <div style="color: #D1D4DC; font-size: 14px; font-weight: 500; line-height: 1.4; margin-bottom: 8px;">
+                {title}
+            </div>
+        </div>
+
+        <!-- Probability Visualization -->
+        <div style="margin-bottom: 10px;">
+            <div style="display: flex; justify-content: space-between; margin-bottom: 4px;">
+                <span style="color: #089981; font-size: 12px; font-weight: 600;">YES {yes_prob:.1f}%</span>
+                <span style="color: #F23645; font-size: 12px; font-weight: 600;">NO {no_prob:.1f}%</span>
+            </div>
+            <!-- Horizontal probability bar -->
+            <div style="
+                display: flex;
+                height: 8px;
+                border-radius: 4px;
+                overflow: hidden;
+                background: #2A2E39;
+            ">
+                <div style="
+                    width: {yes_prob}%;
+                    background: #089981;
+                    transition: width 0.3s ease;
+                "></div>
+                <div style="
+                    width: {no_prob}%;
+                    background: #F23645;
+                    transition: width 0.3s ease;
+                "></div>
+            </div>
+        </div>
+
+        <!-- Footer info -->
+        <div style="display: flex; justify-content: space-between; align-items: center;">
+            <div style="color: #787B86; font-size: 11px;">
+                {end_date_display}{" • " + volume_display if volume_display and end_date_display else volume_display}
+            </div>
+            <a href="{url}" target="_blank" style="
+                color: #3861FB;
+                font-size: 11px;
+                font-weight: 600;
+                text-decoration: none;
+            ">View Market →</a>
+        </div>
+    </div>
+    """
+
+    st.markdown(card_html, unsafe_allow_html=True)
+
+
+def display_economic_event_card(event_item: dict):
+    """Display a single economic event card with forecast/actual comparison."""
+
+    # Escape HTML
+    title = html_module.escape(event_item.get('event_name', event_item.get('title', '')).strip())
+    country = html_module.escape(event_item.get('country', 'US'))
+    url = html_module.escape(event_item.get('url', ''))
+
+    # Get values
+    forecast = event_item.get('forecast')
+    previous = event_item.get('previous')
+    actual = event_item.get('actual')
+    importance = event_item.get('importance', 'medium')
+
+    # Importance badge color
+    importance_colors = {
+        'high': '#F23645',
+        'medium': '#FF9800',
+        'low': '#787B86'
+    }
+    importance_color = importance_colors.get(importance, '#787B86')
+
+    # Time to event
+    time_to_event = event_item.get('time_to_event', '')
+
+    # Format values with unit detection
+    def format_value(val):
+        if val is None:
+            return '-'
+        if isinstance(val, (int, float)):
+            # Check if it looks like a percentage
+            if abs(val) < 100:
+                return f"{val:.1f}%"
+            else:
+                return f"{val:.1f}"
+        return str(val)
+
+    forecast_display = format_value(forecast)
+    previous_display = format_value(previous)
+    actual_display = format_value(actual)
+
+    # Determine if beat/miss
+    beat_miss_html = ""
+    if actual is not None and forecast is not None:
+        if actual > forecast:
+            beat_miss_html = '<span style="color: #089981; font-weight: 700;">[BEAT]</span>'
+        elif actual < forecast:
+            beat_miss_html = '<span style="color: #F23645; font-weight: 700;">[MISS]</span>'
+
+    # Country flag emojis
+    country_flags = {
+        'US': '🇺🇸',
+        'EU': '🇪🇺',
+        'UK': '🇬🇧',
+        'JP': '🇯🇵',
+        'CN': '🇨🇳',
+        'CA': '🇨🇦',
+        'AU': '🇦🇺'
+    }
+    flag = country_flags.get(country, '🌍')
+
+    # Event card HTML
+    card_html = f"""
+    <div style="
+        background: linear-gradient(135deg, #1E222D 0%, #131722 100%);
+        border: 1px solid #2A2E39;
+        border-radius: 8px;
+        padding: 16px;
+        margin-bottom: 12px;
+        transition: all 0.2s ease;
+    " onmouseover="this.style.borderColor='#3861FB'; this.style.transform='translateY(-2px)';"
+       onmouseout="this.style.borderColor='#2A2E39'; this.style.transform='translateY(0)';">
+
+        <!-- Header -->
+        <div style="display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 12px;">
+            <div style="flex: 1;">
+                <div style="display: flex; align-items: center; gap: 8px; margin-bottom: 6px;">
+                    <span style="font-size: 20px;">{flag}</span>
+                    <span style="
+                        background: {importance_color};
+                        color: white;
+                        padding: 2px 8px;
+                        border-radius: 4px;
+                        font-size: 10px;
+                        font-weight: 700;
+                    ">{importance.upper()}</span>
+                </div>
+                <div style="color: #D1D4DC; font-size: 14px; font-weight: 500; line-height: 1.4;">
+                    {title}
+                </div>
+            </div>
+            {f'<div style="color: #3861FB; font-size: 12px; font-weight: 600; white-space: nowrap; margin-left: 12px;">{time_to_event}</div>' if time_to_event else ''}
+        </div>
+
+        <!-- Values comparison -->
+        <div style="background: #0D0E13; border-radius: 6px; padding: 10px; margin-bottom: 8px;">
+            <div style="display: flex; justify-content: space-between; margin-bottom: 6px;">
+                <span style="color: #787B86; font-size: 11px;">Forecast:</span>
+                <span style="color: #D1D4DC; font-size: 12px; font-weight: 600;">{forecast_display}</span>
+            </div>
+            <div style="display: flex; justify-content: space-between; margin-bottom: 6px;">
+                <span style="color: #787B86; font-size: 11px;">Previous:</span>
+                <span style="color: #D1D4DC; font-size: 12px; font-weight: 600;">{previous_display}</span>
+            </div>
+            {f'<div style="display: flex; justify-content: space-between;"><span style="color: #787B86; font-size: 11px;">Actual:</span><span style="color: #D1D4DC; font-size: 12px; font-weight: 600;">{actual_display} {beat_miss_html}</span></div>' if actual is not None else ''}
+        </div>
+    </div>
+    """
+
+    st.markdown(card_html, unsafe_allow_html=True)
+
+
+def display_economic_calendar_widget(events_df: pd.DataFrame):
+    """Display economic calendar widget showing upcoming events."""
+
+    if events_df.empty:
+        st.info("📅 No upcoming economic events in the next 7 days")
+        return
+
+    # Build widget HTML with single-line styles (no leading whitespace)
+    widget_html = """<div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 12px; padding: 20px; margin-bottom: 20px;">
+<div style="margin-bottom: 16px;">
+<h3 style="color: #D1D4DC; font-size: 18px; font-weight: 600; margin: 0;">📅 Economic Calendar</h3>
+<p style="color: #787B86; font-size: 13px; margin: 4px 0 0 0;">Upcoming high-impact events</p>
+</div>"""
+
+    # Show top 10 events
+    for idx, event in events_df.head(10).iterrows():
+        # Get event details
+        event_name = html_module.escape(event.get('event_name', event.get('title', '')))
+        country = html_module.escape(event.get('country', 'US'))
+        importance = event.get('importance', 'medium')
+        time_to_event = event.get('time_to_event', '')
+        forecast = event.get('forecast')
+
+        # Country flags
+        country_flags = {
+            'US': '🇺🇸',
+            'EU': '🇪🇺',
+            'UK': '🇬🇧',
+            'JP': '🇯🇵',
+            'CN': '🇨🇳'
+        }
+        flag = country_flags.get(country, '🌍')
+
+        # Importance stars
+        stars = '⭐' * ({'high': 3, 'medium': 2, 'low': 1}.get(importance, 1))
+
+        # Format forecast
+        forecast_display = f"{forecast:.1f}" if forecast is not None else "N/A"
+
+        # Importance color
+        importance_color = '#F23645' if importance == 'high' else '#FF9800' if importance == 'medium' else '#787B86'
+
+        # Build event HTML (no leading whitespace, single-line styles)
+        event_html = f"""<div style="background: #0D0E13; border-left: 3px solid {importance_color}; border-radius: 6px; padding: 12px; margin-bottom: 10px;">
+<div style="display: flex; justify-content: space-between; align-items: center;">
+<div style="flex: 1;">
+<div style="color: #D1D4DC; font-size: 13px; font-weight: 500; margin-bottom: 4px;">{flag} {event_name}</div>
+<div style="color: #787B86; font-size: 11px;">{stars} Forecast: {forecast_display}</div>
+</div>
+<div style="color: #3861FB; font-size: 12px; font-weight: 600; white-space: nowrap; margin-left: 12px;">{time_to_event}</div>
+</div>
+</div>
+"""
+
+        widget_html += event_html
+
+    widget_html += "</div>"
+
+    st.markdown(widget_html, unsafe_allow_html=True)

app/components/styles.py

@@ -0,0 +1,331 @@
+"""Dark theme CSS styles for the financial dashboard."""
+
+DARK_THEME_CSS = """
+    <style>
+    :root {
+        --primary-color: #0066ff;
+        --secondary-color: #1f77e2;
+        --success-color: #00d084;
+        --danger-color: #ff3838;
+        --warning-color: #ffa500;
+        --bg-dark: #0e1117;
+        --bg-darker: #010409;
+        --text-primary: #e6edf3;
+        --text-secondary: #8b949e;
+        --border-color: #30363d;
+    }
+
+    /* Main background */
+    html, body {
+        background-color: var(--bg-darker) !important;
+        color: var(--text-primary) !important;
+        margin: 0 !important;
+        padding: 0 !important;
+    }
+
+    /* Streamlit containers */
+    .main, [data-testid="stAppViewContainer"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    /* Hide header and footer */
+    [data-testid="stHeader"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    [data-testid="stToolbar"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    .stApp {
+        background-color: var(--bg-dark) !important;
+    }
+
+    [data-testid="stDecoration"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    [data-testid="stSidebar"] {
+        background-color: #0d1117 !important;
+        border-right: 1px solid var(--border-color);
+    }
+
+    /* Text colors */
+    p, span, div, h1, h2, h3, h4, h5, h6, label, li, a {
+        color: var(--text-primary) !important;
+    }
+
+    /* Headings */
+    h1, h2, h3 {
+        color: var(--text-primary) !important;
+        font-weight: 700 !important;
+    }
+
+    /* Links */
+    a {
+        color: var(--primary-color) !important;
+        text-decoration: none !important;
+    }
+
+    a:hover {
+        color: var(--secondary-color) !important;
+        text-decoration: underline !important;
+    }
+
+    /* Labels and text inputs */
+    label {
+        color: var(--text-primary) !important;
+        font-weight: 500 !important;
+    }
+
+    /* Paragraph text */
+    p {
+        color: var(--text-primary) !important;
+        line-height: 1.6 !important;
+    }
+
+    /* Metric card styling */
+    [data-testid="metric-container"] {
+        background: linear-gradient(135deg, #1f2937 0%, #111827 100%) !important;
+        border: 1px solid var(--border-color) !important;
+        border-radius: 10px !important;
+        padding: 1.5rem !important;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3) !important;
+    }
+
+    .metric-card {
+        background: linear-gradient(135deg, #1f2937 0%, #111827 100%);
+        padding: 1.5rem;
+        border-radius: 10px;
+        border: 1px solid var(--border-color);
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3);
+    }
+
+    .metric-value {
+        font-size: 2.5rem;
+        font-weight: 700;
+        color: var(--primary-color);
+        margin: 0.5rem 0;
+    }
+
+    .metric-label {
+        font-size: 0.875rem;
+        color: var(--text-secondary);
+        text-transform: uppercase;
+        letter-spacing: 0.05em;
+    }
+
+    .section-title {
+        color: var(--text-primary);
+        border-bottom: 2px solid var(--primary-color);
+        padding-bottom: 1rem;
+        margin-top: 2rem;
+        margin-bottom: 1.5rem;
+    }
+
+    /* Button styling */
+    .stButton > button {
+        background: linear-gradient(135deg, var(--primary-color) 0%, var(--secondary-color) 100%) !important;
+        color: #ffffff !important;
+        border: none !important;
+        border-radius: 8px !important;
+        padding: 0.75rem 2rem !important;
+        font-weight: 700 !important;
+        transition: all 0.3s ease !important;
+        box-shadow: 0 4px 6px rgba(0, 102, 255, 0.2) !important;
+    }
+
+    .stButton > button:hover {
+        box-shadow: 0 8px 16px rgba(0, 102, 255, 0.4) !important;
+        transform: translateY(-2px) !important;
+    }
+
+    .stButton > button:active {
+        transform: translateY(0) !important;
+    }
+
+    /* Input fields */
+    [data-testid="stTextInput"] input,
+    [data-testid="stSlider"] input {
+        background-color: #161b22 !important;
+        border: 1px solid var(--border-color) !important;
+        color: var(--text-primary) !important;
+        border-radius: 6px !important;
+    }
+
+    [data-testid="stTextInput"] input::placeholder {
+        color: var(--text-secondary) !important;
+    }
+
+    /* Slider */
+    [data-testid="stSlider"] {
+        color: var(--primary-color) !important;
+    }
+
+    /* Tabs */
+    [data-testid="stTabs"] [role="tablist"] {
+        background-color: transparent !important;
+        border-bottom: 2px solid var(--border-color) !important;
+    }
+
+    [data-testid="stTabs"] [role="tab"] {
+        color: var(--text-secondary) !important;
+        background-color: transparent !important;
+        border: none !important;
+        padding: 1rem 1.5rem !important;
+    }
+
+    [data-testid="stTabs"] [role="tab"][aria-selected="true"] {
+        color: var(--primary-color) !important;
+        border-bottom: 3px solid var(--primary-color) !important;
+    }
+
+    /* Dataframe */
+    [data-testid="dataframe"] {
+        background-color: #0d1117 !important;
+    }
+
+    .dataframe {
+        background-color: #0d1117 !important;
+        color: var(--text-primary) !important;
+    }
+
+    /* Info/Error boxes */
+    [data-testid="stInfo"],
+    [data-testid="stSuccess"],
+    [data-testid="stWarning"],
+    [data-testid="stError"] {
+        background-color: rgba(0, 102, 255, 0.1) !important;
+        border-left: 4px solid var(--primary-color) !important;
+        border-radius: 6px !important;
+    }
+
+    [data-testid="stError"] {
+        background-color: rgba(255, 56, 56, 0.1) !important;
+        border-left-color: var(--danger-color) !important;
+    }
+
+    /* Markdown */
+    [data-testid="stMarkdown"] {
+        color: var(--text-primary) !important;
+    }
+
+    /* Expander */
+    [data-testid="stExpander"] {
+        background-color: #161b22 !important;
+        border: 1px solid var(--border-color) !important;
+        border-radius: 6px !important;
+    }
+
+    /* Metric text styling */
+    [data-testid="metric-container"] p {
+        color: var(--text-primary) !important;
+    }
+
+    [data-testid="metric-container"] [data-testid="stMetricValue"] {
+        color: var(--primary-color) !important;
+        font-weight: 700 !important;
+    }
+
+    /* Slider label color */
+    [data-testid="stSlider"] label {
+        color: var(--text-primary) !important;
+    }
+
+    /* Text input label */
+    [data-testid="stTextInput"] label {
+        color: var(--text-primary) !important;
+    }
+
+    /* Write and markdown text */
+    [data-testid="stMarkdownContainer"] p {
+        color: var(--text-primary) !important;
+    }
+
+    [data-testid="stMarkdownContainer"] strong {
+        color: var(--primary-color) !important;
+        font-weight: 600 !important;
+    }
+
+    /* Spinner text */
+    [data-testid="stSpinner"] {
+        color: var(--primary-color) !important;
+    }
+
+    /* Column separators */
+    hr {
+        border-color: var(--border-color) !important;
+    }
+
+    /* Scrollbar */
+    ::-webkit-scrollbar {
+        width: 8px;
+        height: 8px;
+    }
+
+    ::-webkit-scrollbar-track {
+        background: #0d1117;
+    }
+
+    ::-webkit-scrollbar-thumb {
+        background: var(--border-color);
+        border-radius: 4px;
+    }
+
+    ::-webkit-scrollbar-thumb:hover {
+        background: var(--primary-color);
+    }
+
+    /* Selection highlighting */
+    ::selection {
+        background-color: var(--primary-color);
+        color: #fff;
+    }
+
+    /* Fix all white backgrounds */
+    .stApp > header {
+        background-color: var(--bg-dark) !important;
+    }
+
+    .stApp > header::before {
+        background: none !important;
+    }
+
+    .stApp > header::after {
+        background: none !important;
+    }
+
+    /* Streamlit elements background */
+    [data-testid="stVerticalBlock"] {
+        background-color: transparent !important;
+    }
+
+    [data-testid="stVerticalBlockBorderWrapper"] {
+        background-color: transparent !important;
+    }
+
+    /* Remove white decorative elements */
+    .st-emotion-cache-1gvbgyg {
+        background-color: var(--bg-dark) !important;
+    }
+
+    .st-emotion-cache-1jicfl2 {
+        background-color: var(--bg-dark) !important;
+    }
+
+    /* Ensure all root divs are dark */
+    div[class*="st-"] {
+        background-color: transparent !important;
+    }
+
+    /* Modal and overlay backgrounds */
+    .stModal {
+        background-color: var(--bg-dark) !important;
+    }
+
+    /* Alert boxes background */
+    .stAlert {
+        background-color: rgba(0, 102, 255, 0.1) !important;
+    }
+    </style>
+"""

app/components/ui.py

@@ -0,0 +1,174 @@
+"""UI component functions for the financial dashboard."""
+
+import streamlit as st
+import pandas as pd
+import sys
+import os
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from utils.formatters import format_financial_value
+from components.data_sources import get_profitability_metrics
+
+
+def display_price_metrics(metrics: dict):
+    """Display key price metrics in columns."""
+    st.markdown('<div class="section-title">📊 Price Metrics</div>', unsafe_allow_html=True)
+
+    col1, col2, col3, col4 = st.columns(4)
+
+    with col1:
+        st.metric("Current Price", f"${metrics['current_price']:.2f}",
+                 f"{metrics['price_change']:+.2f}", delta_color="normal")
+
+    with col2:
+        st.metric("Day Change %", f"{metrics['price_change_pct']:+.2f}%",
+                 None, delta_color="normal")
+
+    with col3:
+        st.metric("52W High", f"${metrics['high_52w']:.2f}")
+
+    with col4:
+        st.metric("52W Low", f"${metrics['low_52w']:.2f}")
+
+
+def display_company_info(profile_info):
+    """Display company information."""
+    st.markdown('<div class="section-title">📋 Company Information</div>', unsafe_allow_html=True)
+
+    if profile_info:
+        info_col1, info_col2 = st.columns(2)
+        with info_col1:
+            st.write(f"**Company Name:** {getattr(profile_info, 'name', 'N/A')}")
+            st.write(f"**Sector:** {getattr(profile_info, 'sector', 'N/A')}")
+            st.write(f"**Industry:** {getattr(profile_info, 'industry', 'N/A')}")
+
+        with info_col2:
+            st.write(f"**Country:** {getattr(profile_info, 'country', 'N/A')}")
+            st.write(f"**Exchange:** {getattr(profile_info, 'exchange', 'N/A')}")
+            st.write(f"**Website:** {getattr(profile_info, 'website', 'N/A')}")
+
+
+def display_financial_metrics(income_stmt: pd.DataFrame):
+    """Display financial metrics from income statement."""
+    st.markdown('<div class="section-title">💰 Financial Metrics</div>', unsafe_allow_html=True)
+
+    latest_income = income_stmt.iloc[0] if len(income_stmt) > 0 else None
+
+    if latest_income is not None:
+        # First row of metrics
+        fin_col1, fin_col2, fin_col3, fin_col4 = st.columns(4)
+
+        with fin_col1:
+            revenue = latest_income.get('total_revenue', 0)
+            if pd.notna(revenue) and revenue > 0:
+                st.metric("Total Revenue", format_financial_value(revenue))
+            else:
+                st.metric("Total Revenue", "N/A")
+
+        with fin_col2:
+            net_income = latest_income.get('net_income', 0)
+            if pd.notna(net_income) and net_income > 0:
+                st.metric("Net Income", format_financial_value(net_income))
+            else:
+                st.metric("Net Income", "N/A")
+
+        with fin_col3:
+            gross_profit = latest_income.get('gross_profit', 0)
+            if pd.notna(gross_profit) and gross_profit > 0:
+                st.metric("Gross Profit", format_financial_value(gross_profit))
+            else:
+                st.metric("Gross Profit", "N/A")
+
+        with fin_col4:
+            operating_income = latest_income.get('operating_income', 0)
+            if pd.notna(operating_income) and operating_income > 0:
+                st.metric("Operating Income", format_financial_value(operating_income))
+            else:
+                st.metric("Operating Income", "N/A")
+
+        # Second row of metrics
+        fin_col5, fin_col6, fin_col7, fin_col8 = st.columns(4)
+
+        with fin_col5:
+            eps = latest_income.get('diluted_earnings_per_share', 0)
+            if pd.notna(eps):
+                st.metric("EPS (Diluted)", f"${eps:.2f}")
+            else:
+                st.metric("EPS (Diluted)", "N/A")
+
+        with fin_col6:
+            ebitda = latest_income.get('ebitda', 0)
+            if pd.notna(ebitda) and ebitda > 0:
+                st.metric("EBITDA", format_financial_value(ebitda))
+            else:
+                st.metric("EBITDA", "N/A")
+
+        with fin_col7:
+            cogs = latest_income.get('cost_of_revenue', 0)
+            if pd.notna(cogs) and cogs > 0:
+                st.metric("Cost of Revenue", format_financial_value(cogs))
+            else:
+                st.metric("Cost of Revenue", "N/A")
+
+        with fin_col8:
+            rd_expense = latest_income.get('research_and_development_expense', 0)
+            if pd.notna(rd_expense) and rd_expense > 0:
+                st.metric("R&D Expense", format_financial_value(rd_expense))
+            else:
+                st.metric("R&D Expense", "N/A")
+
+
+def display_income_statement(income_stmt: pd.DataFrame):
+    """Display formatted income statement table."""
+    st.markdown("### Income Statement")
+
+    if not income_stmt.empty:
+        display_columns = [
+            'period_ending',
+            'total_revenue',
+            'cost_of_revenue',
+            'gross_profit',
+            'operating_income',
+            'net_income',
+            'diluted_earnings_per_share',
+            'ebitda'
+        ]
+
+        available_cols = [col for col in display_columns if col in income_stmt.columns]
+        financial_display = income_stmt[available_cols].copy()
+
+        for col in financial_display.columns:
+            if col != 'period_ending':
+                financial_display[col] = financial_display[col].apply(
+                    lambda x: format_financial_value(x)
+                )
+
+        st.dataframe(financial_display, use_container_width=True, hide_index=True)
+
+
+def display_profitability_metrics(income_stmt: pd.DataFrame):
+    """Display profitability metrics."""
+    st.markdown("### Profitability Metrics")
+
+    prof_col1, prof_col2 = st.columns(2)
+    latest_data = income_stmt.iloc[0]
+    metrics = get_profitability_metrics(latest_data)
+
+    with prof_col1:
+        if "gross_margin" in metrics:
+            st.metric("Gross Margin", f"{metrics['gross_margin']:.2f}%")
+        if "net_margin" in metrics:
+            st.metric("Net Profit Margin", f"{metrics['net_margin']:.2f}%")
+
+    with prof_col2:
+        if "operating_margin" in metrics:
+            st.metric("Operating Margin", f"{metrics['operating_margin']:.2f}%")
+
+        if len(income_stmt) > 1:
+            prev_revenue = income_stmt.iloc[1].get('total_revenue', 0)
+            total_rev = latest_data.get('total_revenue', 0)
+            if prev_revenue and prev_revenue > 0:
+                revenue_growth = ((total_rev - prev_revenue) / prev_revenue) * 100
+                st.metric("Revenue Growth (YoY)", f"{revenue_growth:+.2f}%")

app/data.py

@@ -0,0 +1,88 @@
+"""Data fetching and processing utilities for the financial dashboard."""
+
+import pandas as pd
+from openbb import sdk
+
+
+def load_stock_data(symbol: str) -> pd.DataFrame:
+    """Load historical stock price data."""
+    df = sdk.equity.price.historical(symbol=symbol).to_dataframe()
+    return df
+
+
+def load_company_profile(symbol: str):
+    """Load company profile information."""
+    profile_response = sdk.equity.profile(symbol=symbol)
+    profile_info = profile_response.results[0] if hasattr(profile_response, 'results') and profile_response.results else None
+    return profile_info
+
+
+def load_income_statement(symbol: str) -> pd.DataFrame:
+    """Load company income statement data."""
+    income_stmt = sdk.equity.fundamental.income(symbol=symbol).to_dataframe()
+    return income_stmt
+
+
+def calculate_technical_indicators(df: pd.DataFrame, period: int) -> pd.DataFrame:
+    """Calculate SMA, EMA, and RSI indicators."""
+    df["SMA"] = df["close"].rolling(period).mean()
+    df["EMA"] = df["close"].ewm(span=period, adjust=False).mean()
+
+    # Calculate RSI
+    delta = df["close"].diff()
+    gain = delta.clip(lower=0)
+    loss = -1 * delta.clip(upper=0)
+    avg_gain = gain.rolling(period).mean()
+    avg_loss = loss.rolling(period).mean()
+    rs = avg_gain / avg_loss
+    df["RSI"] = 100 - (100 / (1 + rs))
+
+    return df
+
+
+def format_financial_value(value) -> str:
+    """Format financial values with appropriate units."""
+    if pd.isna(value):
+        return "N/A"
+    if abs(value) >= 1e9:
+        return f"${value/1e9:.2f}B"
+    elif abs(value) >= 1e6:
+        return f"${value/1e6:.2f}M"
+    else:
+        return f"${value:.2f}"
+
+
+def get_price_metrics(df: pd.DataFrame) -> dict:
+    """Calculate key price metrics."""
+    current_price = df["close"].iloc[-1]
+    prev_close = df["close"].iloc[-2] if len(df) > 1 else df["close"].iloc[0]
+    price_change = current_price - prev_close
+    price_change_pct = (price_change / prev_close) * 100 if prev_close != 0 else 0
+
+    return {
+        "current_price": current_price,
+        "price_change": price_change,
+        "price_change_pct": price_change_pct,
+        "high_52w": df['high'].max(),
+        "low_52w": df['low'].min(),
+    }
+
+
+def get_profitability_metrics(income_data: pd.Series) -> dict:
+    """Calculate profitability metrics from income statement."""
+    total_rev = income_data.get('total_revenue', 0)
+    gross_prof = income_data.get('gross_profit', 0)
+    net_inc = income_data.get('net_income', 0)
+    operating_inc = income_data.get('operating_income', 0)
+
+    metrics = {}
+
+    if total_rev and total_rev > 0:
+        metrics["gross_margin"] = (gross_prof / total_rev) * 100 if pd.notna(gross_prof) else 0
+        metrics["net_margin"] = (net_inc / total_rev) * 100 if pd.notna(net_inc) else 0
+        if operating_inc:
+            metrics["operating_margin"] = (operating_inc / total_rev) * 100
+    else:
+        metrics = {"gross_margin": 0, "net_margin": 0}
+
+    return metrics

app/main.py

@@ -0,0 +1,148 @@
+"""Financial Analysis Dashboard - Main Application."""
+
+import streamlit as st
+from dotenv import load_dotenv
+import os
+
+from styles import DARK_THEME_CSS
+from data import (
+    load_stock_data,
+    load_company_profile,
+    load_income_statement,
+    calculate_technical_indicators,
+    get_price_metrics,
+)
+from charts import (
+    create_price_chart,
+    create_rsi_chart,
+    create_financial_chart,
+)
+from ui import (
+    display_price_metrics,
+    display_company_info,
+    display_financial_metrics,
+    display_income_statement,
+    display_profitability_metrics,
+)
+
+
+# ---- Configuration ----
+load_dotenv()
+token = os.getenv("TOKEN")
+
+st.set_page_config(
+    page_title="Financial Dashboard",
+    page_icon="📈",
+    layout="wide",
+    initial_sidebar_state="expanded",
+    menu_items={
+        "About": "A professional financial analysis dashboard with technical indicators"
+    }
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# ---- Header ----
+st.markdown("# 📈 Financial Analysis Dashboard")
+st.markdown("Real-time technical analysis with multiple indicators")
+
+# ---- Sidebar Configuration ----
+with st.sidebar:
+    st.markdown("## ⚙️ Settings")
+    symbol = st.text_input("Stock Ticker", "AAPL", help="Enter a valid stock ticker symbol").upper()
+    period = st.slider("Indicator Period", 5, 50, 20, help="Period for SMA, EMA, and RSI calculations")
+
+    st.markdown("---")
+    st.markdown("### About")
+    st.info("This dashboard provides real-time technical analysis with comprehensive financial metrics.")
+
+
+def main():
+    """Main application logic."""
+    if st.button("�� Load Dashboard", key="load_btn", use_container_width=True):
+        try:
+            # Load data
+            with st.spinner("Loading data..."):
+                df = load_stock_data(symbol)
+                profile_info = load_company_profile(symbol)
+                income_stmt = load_income_statement(symbol)
+
+            # Calculate technical indicators
+            df = calculate_technical_indicators(df, period)
+
+            # Display price metrics
+            metrics = get_price_metrics(df)
+            display_price_metrics(metrics)
+
+            # Display company information
+            display_company_info(profile_info)
+
+            # Display financial metrics
+            if not income_stmt.empty:
+                display_financial_metrics(income_stmt)
+
+                # Financial history chart
+                st.markdown('<div class="section-title">📊 Revenue & Net Income Trend</div>', unsafe_allow_html=True)
+                income_chart_data = income_stmt[['period_ending', 'total_revenue', 'net_income']].dropna()
+
+                if len(income_chart_data) > 0:
+                    fig_financial = create_financial_chart(income_chart_data)
+                    st.plotly_chart(fig_financial, use_container_width=True)
+
+            # ---- Tabs ----
+            tab1, tab2, tab3, tab4 = st.tabs([
+                "📈 Price & Moving Averages",
+                "📊 RSI Indicator",
+                "📉 TradingView",
+                "📋 Financials"
+            ])
+
+            # Tab 1: Price & Moving Averages
+            with tab1:
+                fig_price = create_price_chart(df, symbol, period)
+                st.plotly_chart(fig_price, use_container_width=True)
+
+            # Tab 2: RSI Indicator
+            with tab2:
+                fig_rsi = create_rsi_chart(df, symbol)
+                st.plotly_chart(fig_rsi, use_container_width=True)
+
+            # Tab 3: TradingView
+            with tab3:
+                tradingview_html = f"""
+                <div class="tradingview-widget-container">
+                  <div id="tradingview_{symbol}"></div>
+                  <script type="text/javascript" src="https://s3.tradingview.com/tv.js"></script>
+                  <script type="text/javascript">
+                    new TradingView.widget({{
+                      "width": "100%",
+                      "height": 600,
+                      "symbol": "{symbol}",
+                      "interval": "D",
+                      "timezone": "Etc/UTC",
+                      "theme": "dark",
+                      "style": "1",
+                      "locale": "en",
+                      "enable_publishing": false,
+                      "allow_symbol_change": true,
+                      "container_id": "tradingview_{symbol}"
+                    }});
+                  </script>
+                </div>
+                """
+                st.components.v1.html(tradingview_html, height=650)
+
+            # Tab 4: Detailed Financials
+            with tab4:
+                if not income_stmt.empty:
+                    display_income_statement(income_stmt)
+                    display_profitability_metrics(income_stmt)
+
+        except Exception as e:
+            st.error(f"Error loading data for {symbol}: {str(e)}")
+            st.info("Please check the ticker symbol and try again.")
+
+
+if __name__ == "__main__":
+    main()

app/pages/01_Stocks.py

@@ -0,0 +1,145 @@
+"""Stock Analysis Page - Comprehensive stock analysis with technical indicators."""
+
+import streamlit as st
+import sys
+import os
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from components.styles import DARK_THEME_CSS
+from components.data_sources import (
+    load_stock_data,
+    load_company_profile,
+    load_income_statement,
+    calculate_technical_indicators,
+    get_price_metrics,
+)
+from components.chart import (
+    create_price_chart,
+    create_rsi_chart,
+    create_financial_chart,
+)
+from components.ui import (
+    display_price_metrics,
+    display_company_info,
+    display_financial_metrics,
+    display_income_statement,
+    display_profitability_metrics,
+)
+
+
+# ---- Page Configuration ----
+st.set_page_config(
+    page_title="Stocks - Financial Dashboard",
+    page_icon="📈",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# ---- Header ----
+st.markdown("# 📈 Stock Analysis")
+st.markdown("Real-time technical analysis with comprehensive financial metrics")
+
+# ---- Sidebar Configuration ----
+with st.sidebar:
+    st.markdown("## ⚙️ Settings")
+    symbol = st.text_input("Stock Ticker", "AAPL", help="Enter a valid stock ticker symbol").upper()
+    period = st.slider("Indicator Period", 5, 50, 20, help="Period for SMA, EMA, and RSI calculations")
+
+    st.markdown("---")
+    st.markdown("### About")
+    st.info("Analyze stocks with technical indicators, financials, and TradingView charts.")
+
+
+def main():
+    """Main stock analysis logic."""
+    if st.button("📊 Load Stock Data", key="load_btn", use_container_width=True):
+        try:
+            # Load data
+            with st.spinner("Loading data..."):
+                df = load_stock_data(symbol)
+                profile_info = load_company_profile(symbol)
+                income_stmt = load_income_statement(symbol)
+
+            # Calculate technical indicators
+            df = calculate_technical_indicators(df, period)
+
+            # Display price metrics
+            metrics = get_price_metrics(df)
+            display_price_metrics(metrics)
+
+            # Display company information
+            display_company_info(profile_info)
+
+            # Display financial metrics
+            if not income_stmt.empty:
+                display_financial_metrics(income_stmt)
+
+                # Financial history chart
+                st.markdown('<div class="section-title">📊 Revenue & Net Income Trend</div>', unsafe_allow_html=True)
+                income_chart_data = income_stmt[['period_ending', 'total_revenue', 'net_income']].dropna()
+
+                if len(income_chart_data) > 0:
+                    fig_financial = create_financial_chart(income_chart_data)
+                    st.plotly_chart(fig_financial, use_container_width=True)
+
+            # ---- Tabs ----
+            tab1, tab2, tab3, tab4 = st.tabs([
+                "📈 Price & Moving Averages",
+                "📊 RSI Indicator",
+                "📉 TradingView",
+                "📋 Financials"
+            ])
+
+            # Tab 1: Price & Moving Averages
+            with tab1:
+                fig_price = create_price_chart(df, symbol, period)
+                st.plotly_chart(fig_price, use_container_width=True)
+
+            # Tab 2: RSI Indicator
+            with tab2:
+                fig_rsi = create_rsi_chart(df, symbol)
+                st.plotly_chart(fig_rsi, use_container_width=True)
+
+            # Tab 3: TradingView
+            with tab3:
+                tradingview_html = f"""
+                <div class="tradingview-widget-container">
+                  <div id="tradingview_{symbol}"></div>
+                  <script type="text/javascript" src="https://s3.tradingview.com/tv.js"></script>
+                  <script type="text/javascript">
+                    new TradingView.widget({{
+                      "width": "100%",
+                      "height": 600,
+                      "symbol": "{symbol}",
+                      "interval": "D",
+                      "timezone": "Etc/UTC",
+                      "theme": "dark",
+                      "style": "1",
+                      "locale": "en",
+                      "enable_publishing": false,
+                      "allow_symbol_change": true,
+                      "container_id": "tradingview_{symbol}"
+                    }});
+                  </script>
+                </div>
+                """
+                st.components.v1.html(tradingview_html, height=650)
+
+            # Tab 4: Detailed Financials
+            with tab4:
+                if not income_stmt.empty:
+                    display_income_statement(income_stmt)
+                    display_profitability_metrics(income_stmt)
+
+        except Exception as e:
+            st.error(f"Error loading data for {symbol}: {str(e)}")
+            st.info("Please check the ticker symbol and try again.")
+
+
+if __name__ == "__main__":
+    main()

app/pages/02_Crypto.py

@@ -0,0 +1,74 @@
+"""Cryptocurrency Analysis Page - Track and analyze cryptocurrencies."""
+
+import streamlit as st
+import sys
+import os
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from components.styles import DARK_THEME_CSS
+
+
+# ---- Page Configuration ----
+st.set_page_config(
+    page_title="Crypto - Financial Dashboard",
+    page_icon="₿",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# ---- Header ----
+st.markdown("# ₿ Cryptocurrency Analysis")
+st.markdown("Track and analyze major cryptocurrencies with real-time market data")
+
+st.markdown("---")
+
+# ---- Sidebar Configuration ----
+with st.sidebar:
+    st.markdown("## ⚙️ Settings")
+    crypto_symbol = st.selectbox(
+        "Cryptocurrency",
+        ["BTC/USD", "ETH/USD", "BNB/USD", "ADA/USD", "SOL/USD"],
+        help="Select a cryptocurrency pair"
+    )
+    period = st.slider("Indicator Period", 5, 50, 20, help="Period for technical indicators")
+
+    st.markdown("---")
+    st.markdown("### About")
+    st.info("Analyze cryptocurrencies with technical indicators and real-time market data.")
+
+
+# ---- Main Content ----
+st.info("🚧 This page is under development. Cryptocurrency analysis features coming soon!")
+
+st.markdown("""
+### Planned Features:
+
+- **Real-time Price Data**: Live cryptocurrency prices from Binance
+- **Market Metrics**: 24h volume, market cap, price changes
+- **Technical Indicators**: SMA, EMA, RSI, MACD for crypto assets
+- **TradingView Charts**: Interactive crypto charts
+- **Market Sentiment**: Community sentiment analysis
+- **Top Movers**: Biggest gainers and losers in 24h
+
+Stay tuned for updates!
+""")
+
+# Placeholder metrics
+col1, col2, col3, col4 = st.columns(4)
+
+with col1:
+    st.metric("Current Price", "N/A", "N/A")
+
+with col2:
+    st.metric("24h Change", "N/A", "N/A")
+
+with col3:
+    st.metric("24h Volume", "N/A")
+
+with col4:
+    st.metric("Market Cap", "N/A")

app/pages/03_Forex.py

@@ -0,0 +1,74 @@
+"""Forex Trading Analysis Page - Analyze foreign exchange pairs."""
+
+import streamlit as st
+import sys
+import os
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from components.styles import DARK_THEME_CSS
+
+
+# ---- Page Configuration ----
+st.set_page_config(
+    page_title="Forex - Financial Dashboard",
+    page_icon="💱",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# ---- Header ----
+st.markdown("# 💱 Forex Trading Analysis")
+st.markdown("Foreign exchange analysis for major, minor, and exotic currency pairs")
+
+st.markdown("---")
+
+# ---- Sidebar Configuration ----
+with st.sidebar:
+    st.markdown("## ⚙️ Settings")
+    forex_pair = st.selectbox(
+        "Currency Pair",
+        ["EUR/USD", "GBP/USD", "USD/JPY", "USD/CHF", "AUD/USD", "USD/CAD"],
+        help="Select a forex pair"
+    )
+    period = st.slider("Indicator Period", 5, 50, 20, help="Period for technical indicators")
+
+    st.markdown("---")
+    st.markdown("### About")
+    st.info("Analyze forex pairs with technical indicators and real-time exchange rates.")
+
+
+# ---- Main Content ----
+st.info("🚧 This page is under development. Forex analysis features coming soon!")
+
+st.markdown("""
+### Planned Features:
+
+- **Real-time Exchange Rates**: Live forex rates from multiple sources
+- **Major, Minor & Exotic Pairs**: Comprehensive coverage
+- **Technical Analysis**: Full suite of technical indicators
+- **Pip Calculator**: Calculate pip values for position sizing
+- **Economic Calendar**: Important economic events
+- **TradingView Charts**: Interactive forex charts
+
+Stay tuned for updates!
+""")
+
+# Placeholder metrics
+col1, col2, col3, col4 = st.columns(4)
+
+with col1:
+    st.metric("Current Rate", "N/A", "N/A")
+
+with col2:
+    st.metric("24h Change", "N/A", "N/A")
+
+with col3:
+    st.metric("Bid Price", "N/A")
+
+with col4:
+    st.metric("Ask Price", "N/A")

app/pages/04_Screener.py

@@ -0,0 +1,74 @@
+"""Market Screener Page - Find investment opportunities across markets."""
+
+import streamlit as st
+import sys
+import os
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from components.styles import DARK_THEME_CSS
+
+
+# ---- Page Configuration ----
+st.set_page_config(
+    page_title="Screener - Financial Dashboard",
+    page_icon="🔍",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# ---- Header ----
+st.markdown("# 🔍 Market Screener")
+st.markdown("Advanced screening tools to find investment opportunities across markets")
+
+st.markdown("---")
+
+# ---- Sidebar Configuration ----
+with st.sidebar:
+    st.markdown("## ⚙️ Screening Filters")
+
+    asset_type = st.selectbox(
+        "Asset Type",
+        ["Stocks", "Crypto", "Forex"],
+        help="Select asset type to screen"
+    )
+
+    st.markdown("### Price Filters")
+    min_price = st.number_input("Min Price ($)", value=0.0, step=1.0)
+    max_price = st.number_input("Max Price ($)", value=1000.0, step=10.0)
+
+    st.markdown("### Technical Filters")
+    rsi_min = st.slider("Min RSI", 0, 100, 30)
+    rsi_max = st.slider("Max RSI", 0, 100, 70)
+
+    volume_min = st.number_input("Min Volume", value=1000000, step=100000)
+
+    st.markdown("---")
+    if st.button("🔍 Run Screener", use_container_width=True):
+        st.info("Screening in progress...")
+
+
+# ---- Main Content ----
+st.info("🚧 This page is under development. Market screener features coming soon!")
+
+st.markdown("""
+### Planned Features:
+
+- **Multi-Asset Screening**: Stocks, crypto, and forex
+- **Technical Filters**: RSI, MACD, moving averages, volume
+- **Fundamental Filters**: P/E ratio, market cap, revenue growth
+- **Pattern Recognition**: Chart patterns and technical setups
+- **Custom Criteria**: Build your own screening rules
+- **Export Results**: Download screening results as CSV
+- **Saved Screens**: Save your favorite screening criteria
+
+Stay tuned for updates!
+""")
+
+# Placeholder table
+st.markdown("### Screening Results")
+st.info("No screening results yet. Configure filters and run the screener.")

app/pages/05_Dashboard.py

@@ -0,0 +1,951 @@
+"""
+News & AI Dashboard Page - Real-time Financial Intelligence
+Powered by professional-grade news monitoring with low-latency delivery
+"""
+
+import streamlit as st
+import sys
+import os
+import logging
+
+# Suppress noisy Playwright asyncio errors
+logging.getLogger('asyncio').setLevel(logging.CRITICAL)
+logging.getLogger('playwright').setLevel(logging.WARNING)
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from components.styles import DARK_THEME_CSS
+from components.news import (
+    display_news_statistics,
+    display_category_breakdown,
+    display_breaking_news_banner,
+    display_scrollable_news_section,
+    display_prediction_card,
+    display_economic_event_card,
+    display_economic_calendar_widget
+)
+from utils.breaking_news_scorer import get_breaking_news_scorer
+from utils.ai_summary_store import init_storage, enqueue_items, fetch_summaries, get_status
+from utils.ai_summary_worker import start_worker_if_needed
+
+# Import news scrapers
+try:
+    from services.news_scraper import FinanceNewsScraper
+    RSS_AVAILABLE = True
+except ImportError:
+    RSS_AVAILABLE = False
+
+try:
+    from services.twitter_news_playwright import TwitterFinanceMonitor
+    TWITTER_AVAILABLE = True
+except ImportError:
+    TWITTER_AVAILABLE = False
+
+try:
+    from services.reddit_news import RedditFinanceMonitor
+    REDDIT_AVAILABLE = True
+except ImportError:
+    REDDIT_AVAILABLE = False
+
+try:
+    from services.ai_tech_news import AITechNewsScraper
+    AI_TECH_AVAILABLE = True
+except ImportError:
+    AI_TECH_AVAILABLE = False
+
+try:
+    from services.prediction_markets import PredictionMarketsScraper
+    PREDICTIONS_AVAILABLE = True
+except ImportError:
+    PREDICTIONS_AVAILABLE = False
+
+try:
+    from services.sectoral_news import SectoralNewsScraper
+    SECTORAL_AVAILABLE = True
+except ImportError:
+    SECTORAL_AVAILABLE = False
+
+try:
+    from services.market_events import MarketEventsScraper
+    EVENTS_AVAILABLE = True
+except ImportError:
+    EVENTS_AVAILABLE = False
+
+try:
+    from services.economic_calendar import EconomicCalendarService
+    CALENDAR_AVAILABLE = True
+except ImportError:
+    CALENDAR_AVAILABLE = False
+
+
+# ---- Page Configuration ----
+st.set_page_config(
+    page_title="News Dashboard - Financial Platform",
+    page_icon="📰",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# ---- Apply Dark Theme ----
+st.markdown(DARK_THEME_CSS, unsafe_allow_html=True)
+
+# Initialize news monitors (with caching)
+if 'rss_monitor' not in st.session_state and RSS_AVAILABLE:
+    st.session_state.rss_monitor = FinanceNewsScraper()
+
+if 'twitter_monitor' not in st.session_state and TWITTER_AVAILABLE:
+    st.session_state.twitter_monitor = TwitterFinanceMonitor()
+
+if 'reddit_monitor' not in st.session_state and REDDIT_AVAILABLE:
+    st.session_state.reddit_monitor = RedditFinanceMonitor()
+
+if 'ai_tech_monitor' not in st.session_state and AI_TECH_AVAILABLE:
+    st.session_state.ai_tech_monitor = AITechNewsScraper()
+
+if 'prediction_markets_monitor' not in st.session_state and PREDICTIONS_AVAILABLE:
+    st.session_state.prediction_markets_monitor = PredictionMarketsScraper()
+
+if 'sectoral_news_monitor' not in st.session_state and SECTORAL_AVAILABLE:
+    st.session_state.sectoral_news_monitor = SectoralNewsScraper()
+
+if 'market_events_monitor' not in st.session_state and EVENTS_AVAILABLE:
+    st.session_state.market_events_monitor = MarketEventsScraper()
+
+if 'economic_calendar_service' not in st.session_state and CALENDAR_AVAILABLE:
+    st.session_state.economic_calendar_service = EconomicCalendarService()
+
+rss_monitor = st.session_state.get('rss_monitor')
+twitter_monitor = st.session_state.get('twitter_monitor')
+reddit_monitor = st.session_state.get('reddit_monitor')
+ai_tech_monitor = st.session_state.get('ai_tech_monitor')
+prediction_markets_monitor = st.session_state.get('prediction_markets_monitor')
+sectoral_news_monitor = st.session_state.get('sectoral_news_monitor')
+market_events_monitor = st.session_state.get('market_events_monitor')
+economic_calendar_service = st.session_state.get('economic_calendar_service')
+
+# Initialize unified cache manager
+if 'news_cache_manager' not in st.session_state:
+    from utils.news_cache import NewsCacheManager
+    st.session_state.news_cache_manager = NewsCacheManager(default_ttl=180)
+
+cache_manager = st.session_state.news_cache_manager
+
+# ---- Header ----
+st.markdown("# 🤖 Live Financial News & AI Dashboard")
+st.markdown("AI-powered market insights with sentiment analysis and trading recommendations. Real-time macro, markets & geopolitical intelligence")
+
+st.markdown("---")
+
+# ---- Sidebar Filters ----
+with st.sidebar:
+    st.markdown("## ⚙️ News Filters")
+
+    # Category filter
+    category_filter = st.selectbox(
+        "Category",
+        ["all", "macro", "markets", "geopolitical"],
+        format_func=lambda x: x.upper() if x != "all" else "ALL CATEGORIES",
+        help="Filter by news category"
+    )
+
+    # Sentiment filter
+    sentiment_filter = st.selectbox(
+        "Sentiment",
+        ["all", "positive", "negative", "neutral"],
+        format_func=lambda x: x.upper() if x != "all" else "ALL SENTIMENTS",
+        help="Filter by market sentiment"
+    )
+
+    # Impact filter
+    impact_filter = st.selectbox(
+        "Impact Level",
+        ["all", "high", "medium", "low"],
+        format_func=lambda x: x.upper() if x != "all" else "ALL IMPACT LEVELS",
+        help="Filter by market impact"
+    )
+
+    st.markdown("---")
+
+    # Refresh controls
+    st.markdown("### 🔄 Refresh Settings")
+
+    col1, col2 = st.columns(2)
+    with col1:
+        if st.button("🔄 Refresh Now", use_container_width=True, type="primary"):
+            st.session_state.force_refresh = True
+            st.rerun()
+
+    with col2:
+        auto_refresh = st.checkbox("Auto-refresh", value=True, help="Auto-refresh every 3 minutes")
+
+    if auto_refresh:
+        st.info("⏱️ Auto-refresh enabled (3 min)")
+
+    st.markdown("---")
+    st.markdown("### 📊 Feed Statistics")
+
+    # Get cache statistics from cache manager
+    cache_stats = cache_manager.get_statistics()
+
+    # Calculate totals from cache
+    total_stories = (
+        cache_stats['twitter']['items'] +
+        cache_stats['reddit']['items'] +
+        cache_stats['rss']['items'] +
+        cache_stats.get('ai_tech', {}).get('items', 0)
+    )
+
+    # Display metrics
+    st.metric("Total Stories", total_stories)
+    st.metric("Cache Status", "✅ Active" if total_stories > 0 else "⏳ Loading")
+
+    # Show cache age for transparency
+    if cache_stats['twitter']['is_valid']:
+        age = int(cache_stats['twitter']['age_seconds'])
+        st.caption(f"🕐 Cache age: {age}s / 180s")
+    else:
+        st.caption("🔄 Fetching fresh data...")
+
+    st.markdown("---")
+    st.markdown("### ℹ️ Sources")
+
+    # Count total sources
+    twitter_sources = len(twitter_monitor.SOURCES) if twitter_monitor else 0
+    reddit_sources = len(reddit_monitor.SUBREDDITS) if reddit_monitor else 0
+    rss_sources = len(rss_monitor.SOURCES) if rss_monitor else 0
+    ai_tech_sources = len(ai_tech_monitor.SOURCES) if ai_tech_monitor else 0
+    prediction_sources = 3  # Polymarket, Metaculus, CME FedWatch
+    sectoral_sources = 7  # 7 sectors
+    events_sources = 3  # Earnings, indicators, central banks
+    total_sources = twitter_sources + reddit_sources + rss_sources + ai_tech_sources + prediction_sources + sectoral_sources + events_sources
+
+    st.markdown(f"""
+    <div style='font-size: 11px; line-height: 1.6;'>
+
+    **Twitter/X Accounts ({twitter_sources})**
+    • WalterBloomberg • FXHedge • DeItaone
+    • Reuters • Bloomberg • FT • WSJ
+    • CNBC • BBC • MarketWatch
+    • The Economist • AP • AFP
+
+    **Reddit Communities ({reddit_sources})**
+    • r/wallstreetbets • r/stocks • r/investing
+    • r/algotrading • r/economics • r/geopolitics
+    • r/options • r/SecurityAnalysis
+
+    **RSS + Web Scraping ({rss_sources})**
+    • CNBC • Bloomberg • FT • WSJ
+    • BBC • Yahoo Finance • Google News
+    • The Economist • Fed (2.0x) • ECB (2.0x) • IMF
+
+    **AI & Tech Sources ({ai_tech_sources})**
+    • OpenAI • Google AI • Microsoft AI • Meta AI
+    • DeepMind • Anthropic • AWS AI • NVIDIA
+    • TechCrunch • The Verge • VentureBeat
+    • MIT Tech Review • Wired • Ars Technica
+
+    **Prediction Markets ({prediction_sources})**
+    • Polymarket • Metaculus • CME FedWatch
+
+    **Sectoral Coverage ({sectoral_sources})**
+    • Finance • Tech • Energy • Healthcare
+    • Consumer • Industrials • Real Estate
+
+    **Market Events ({events_sources})**
+    • Earnings Calendar • Economic Indicators
+    • Central Bank Events (Fed, ECB, BoE, BoJ)
+
+    **Total: {total_sources} Premium Sources**
+    </div>
+    """, unsafe_allow_html=True)
+
+
+# ---- Main Content Area ----
+
+# Check for forced refresh (don't clear yet - wait until after fetching)
+force_refresh = st.session_state.get('force_refresh', False)
+
+# Initialize AI summary store/worker (shared across sessions/processes)
+init_storage()
+start_worker_if_needed()
+
+# Fetch news from all sources IN PARALLEL for maximum performance
+import pandas as pd
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+twitter_df = pd.DataFrame()
+reddit_df = pd.DataFrame()
+rss_all_df = pd.DataFrame()
+rss_main_df = pd.DataFrame()
+ai_tech_df = pd.DataFrame()
+predictions_df = pd.DataFrame()
+sectoral_news_df = pd.DataFrame()
+market_events_df = pd.DataFrame()
+economic_calendar_df = pd.DataFrame()
+
+def fetch_twitter_news():
+    """Fetch Twitter/X news via cache manager"""
+    try:
+        if twitter_monitor:
+            # Use cache manager for smart caching
+            twitter_news = cache_manager.get_news(
+                source='twitter',
+                fetcher_func=twitter_monitor.scrape_twitter_news,
+                force_refresh=force_refresh,
+                max_tweets=50
+            )
+            if twitter_news:
+                df = pd.DataFrame(twitter_news)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"Twitter scraping unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_reddit_news():
+    """Fetch Reddit news via cache manager"""
+    try:
+        if reddit_monitor:
+            # Use cache manager for smart caching
+            reddit_news = cache_manager.get_news(
+                source='reddit',
+                fetcher_func=reddit_monitor.scrape_reddit_news,
+                force_refresh=force_refresh,
+                max_posts=50,
+                hours=12
+            )
+            if reddit_news:
+                df = pd.DataFrame(reddit_news)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"Reddit scraping unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_rss_news():
+    """Fetch RSS + Web scraped news via cache manager"""
+    try:
+        if rss_monitor:
+            # Use cache manager for smart caching
+            rss_news = cache_manager.get_news(
+                source='rss',
+                fetcher_func=rss_monitor.scrape_news,
+                force_refresh=force_refresh,
+                max_items=100
+            )
+            if rss_news:
+                df = pd.DataFrame(rss_news)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                    return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"RSS scraping unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_ai_tech_news():
+    """Fetch AI/Tech news via cache manager"""
+    try:
+        if ai_tech_monitor:
+            # Use cache manager for smart caching
+            ai_tech_news = cache_manager.get_news(
+                source='ai_tech',
+                fetcher_func=ai_tech_monitor.scrape_ai_tech_news,
+                force_refresh=force_refresh,
+                max_items=100,
+                hours=48
+            )
+            if ai_tech_news:
+                df = pd.DataFrame(ai_tech_news)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                    return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"AI/Tech news unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_prediction_markets():
+    """Fetch prediction market data via cache manager"""
+    try:
+        if prediction_markets_monitor:
+            predictions = cache_manager.get_news(
+                source='predictions',
+                fetcher_func=prediction_markets_monitor.scrape_predictions,
+                force_refresh=force_refresh,
+                max_items=50
+            )
+            if predictions:
+                df = pd.DataFrame(predictions)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                    return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"Prediction markets unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_sectoral_news():
+    """Fetch sectoral news via cache manager"""
+    try:
+        if sectoral_news_monitor:
+            sectoral_news = cache_manager.get_news(
+                source='sectoral_news',
+                fetcher_func=sectoral_news_monitor.scrape_sectoral_news,
+                force_refresh=force_refresh,
+                max_items=50,
+                hours=24
+            )
+            if sectoral_news:
+                df = pd.DataFrame(sectoral_news)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                    return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"Sectoral news unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_market_events():
+    """Fetch market events via cache manager"""
+    try:
+        if market_events_monitor:
+            events = cache_manager.get_news(
+                source='market_events',
+                fetcher_func=market_events_monitor.scrape_market_events,
+                force_refresh=force_refresh,
+                max_items=50,
+                days_ahead=14
+            )
+            if events:
+                df = pd.DataFrame(events)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                    return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"Market events unavailable: {e}"
+    return pd.DataFrame(), None
+
+def fetch_economic_calendar():
+    """Fetch economic calendar via cache manager"""
+    try:
+        if economic_calendar_service:
+            calendar_events = cache_manager.get_news(
+                source='economic_calendar',
+                fetcher_func=economic_calendar_service.get_upcoming_events,
+                force_refresh=force_refresh,
+                days_ahead=7,
+                min_importance='medium'
+            )
+            if calendar_events:
+                df = pd.DataFrame(calendar_events)
+                if not df.empty:
+                    df['timestamp'] = pd.to_datetime(df['timestamp'])
+                    return df, None
+    except Exception as e:
+        return pd.DataFrame(), f"Economic calendar unavailable: {e}"
+    return pd.DataFrame(), None
+
+# Progressive loading: Display results as they arrive
+# Create a status placeholder to show progress
+status_placeholder = st.empty()
+
+# Execute all news fetching operations in parallel using ThreadPoolExecutor
+with st.spinner("Loading news from 8 sources..."):
+    with ThreadPoolExecutor(max_workers=8) as executor:
+        # Submit all tasks with source name attached
+        futures_map = {
+            executor.submit(fetch_twitter_news): 'twitter',
+            executor.submit(fetch_reddit_news): 'reddit',
+            executor.submit(fetch_rss_news): 'rss',
+            executor.submit(fetch_ai_tech_news): 'ai_tech',
+            executor.submit(fetch_prediction_markets): 'predictions',
+            executor.submit(fetch_sectoral_news): 'sectoral_news',
+            executor.submit(fetch_market_events): 'market_events',
+            executor.submit(fetch_economic_calendar): 'economic_calendar'
+        }
+
+        # Track errors and completion
+        fetch_errors = []
+        completed_sources = []
+
+        # Process results as they complete (progressive loading)
+        try:
+            for future in as_completed(futures_map, timeout=90):
+                source_name = futures_map[future]
+
+                try:
+                    result_df, error = future.result()
+
+                    # Update status
+                    completed_sources.append(source_name)
+                    status_placeholder.info(f"🔍 Loaded {len(completed_sources)}/8 sources ({', '.join(completed_sources)})")
+
+                    if source_name == 'twitter':
+                        twitter_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                    elif source_name == 'reddit':
+                        reddit_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                    elif source_name == 'rss':
+                        rss_all_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                        # Get main page news subset for RSS
+                        if not rss_all_df.empty and 'from_web' in rss_all_df.columns:
+                            rss_main_df = rss_all_df[rss_all_df['from_web'] == True].copy()
+                    elif source_name == 'ai_tech':
+                        ai_tech_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                    elif source_name == 'predictions':
+                        predictions_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                    elif source_name == 'sectoral_news':
+                        sectoral_news_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                    elif source_name == 'market_events':
+                        market_events_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+                    elif source_name == 'economic_calendar':
+                        economic_calendar_df = result_df
+                        if error:
+                            fetch_errors.append(error)
+
+                except Exception as e:
+                    fetch_errors.append(f"Error fetching {source_name} news: {e}")
+                    completed_sources.append(f"{source_name} (error)")
+                    status_placeholder.warning(f"⚠️ {source_name} failed, continuing with other sources...")
+
+        except TimeoutError:
+            # Handle timeout gracefully - continue with whatever results we have
+            fetch_errors.append("⏱️ Some sources timed out after 90 seconds - displaying available results")
+            status_placeholder.warning(f"⚠️ {len(completed_sources)}/8 sources loaded (some timed out)")
+
+            # Mark incomplete sources
+            all_sources = set(futures_map.values())
+            incomplete_sources = all_sources - set(completed_sources)
+            for source in incomplete_sources:
+                fetch_errors.append(f"{source} timed out - skipped")
+                completed_sources.append(f"{source} (timeout)")
+
+    # Clear the status message after all sources complete
+    status_placeholder.success(f"✅ Loaded {len(completed_sources)}/8 sources successfully")
+
+# Debug logging (console only, not displayed on page)
+import logging
+logger = logging.getLogger(__name__)
+logger.info(f"News Fetch Results: Twitter={len(twitter_df)}, Reddit={len(reddit_df)}, RSS={len(rss_all_df)}, AI/Tech={len(ai_tech_df)}, Predictions={len(predictions_df)}, Sectoral={len(sectoral_news_df)}, Events={len(market_events_df)}, Calendar={len(economic_calendar_df)}")
+logger.info(f"Availability: Predictions={PREDICTIONS_AVAILABLE}, Sectoral={SECTORAL_AVAILABLE}, Events={EVENTS_AVAILABLE}, Calendar={CALENDAR_AVAILABLE}")
+if fetch_errors:
+    for err in fetch_errors:
+        logger.warning(f"Fetch error: {err}")
+
+# Batch AI summarization after all sources are collected
+ai_summary_dfs = [
+    twitter_df,
+    reddit_df,
+    rss_all_df,
+    ai_tech_df,
+    sectoral_news_df,
+    market_events_df,
+    economic_calendar_df,
+    predictions_df,
+]
+
+all_items = []
+for df in ai_summary_dfs:
+    if df.empty:
+        continue
+    all_items.extend(df.to_dict("records"))
+
+if all_items:
+    enqueue_items(all_items)
+
+# Clear force refresh flag after fetching is complete
+if force_refresh:
+    st.session_state.force_refresh = False
+
+# Apply filters using cache manager (with filter result caching)
+filters = {
+    'category': category_filter,
+    'sentiment': sentiment_filter,
+    'impact': impact_filter
+}
+
+twitter_filtered = cache_manager.get_filtered_news(twitter_df, filters, 'twitter') if not twitter_df.empty else twitter_df
+reddit_filtered = cache_manager.get_filtered_news(reddit_df, filters, 'reddit') if not reddit_df.empty else reddit_df
+rss_main_filtered = cache_manager.get_filtered_news(rss_main_df, filters, 'rss_main') if not rss_main_df.empty else rss_main_df
+rss_all_filtered = cache_manager.get_filtered_news(rss_all_df, filters, 'rss_all') if not rss_all_df.empty else rss_all_df
+
+# Combine Twitter and Reddit for first column
+twitter_reddit_df = pd.concat([twitter_filtered, reddit_filtered], ignore_index=True) if not twitter_filtered.empty or not reddit_filtered.empty else pd.DataFrame()
+if not twitter_reddit_df.empty:
+    twitter_reddit_df = twitter_reddit_df.sort_values('timestamp', ascending=False)
+
+# Combine all for breaking news banner
+all_news_df = pd.concat([twitter_filtered, reddit_filtered, rss_all_filtered], ignore_index=True) if not twitter_filtered.empty or not reddit_filtered.empty or not rss_all_filtered.empty else pd.DataFrame()
+
+# Display breaking news banner with ML-based scoring
+if not all_news_df.empty:
+    # Initialize the breaking news scorer
+    scorer = get_breaking_news_scorer()
+
+    # Convert DataFrame to list of dicts for scoring
+    all_news_list = all_news_df.to_dict('records')
+
+    # Get top breaking news using multi-factor impact scoring
+    # Only show news with impact score >= 40 (medium-high impact threshold)
+    breaking_news_items = scorer.get_breaking_news(all_news_list, top_n=1)
+
+    if breaking_news_items and breaking_news_items[0]['breaking_score'] >= 40.0:
+        # Display the highest-impact news in the banner
+        breaking_df = pd.DataFrame([breaking_news_items[0]])
+        display_breaking_news_banner(breaking_df)
+    else:
+        # If no high-impact news found, show informational message with score
+        if breaking_news_items:
+            top_score = breaking_news_items[0]['breaking_score']
+            st.info(f"📊 Monitoring financial markets - highest impact score: {top_score:.1f}/100 (threshold: 40)")
+        else:
+            st.info("📊 Monitoring financial markets - no news items available for scoring")
+else:
+    # No news data available at all
+    st.info("📊 Loading financial news - breaking news banner will appear when data is available")
+
+st.markdown("---")
+
+# ---- ECONOMIC CALENDAR WIDGET ----
+if not economic_calendar_df.empty:
+    display_economic_calendar_widget(economic_calendar_df)
+    st.markdown("---")
+
+# ---- FOUR-COLUMN SCROLLABLE NEWS LAYOUT (TradingView Style) ----
+
+col1, col2, col3, col4 = st.columns(4)
+
+with col1:
+    # SECTION 1: Twitter/X & Reddit Breaking News
+    if not twitter_reddit_df.empty:
+        display_scrollable_news_section(
+            twitter_reddit_df,
+            section_title="Twitter/X & Reddit News",
+            section_icon="🌐",
+            section_subtitle="Real-time news from premium accounts & communities (last 12h)",
+            max_items=100,
+            height="700px"
+        )
+    elif not twitter_df.empty or not reddit_df.empty:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px;">📭</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">No matches found</div>
+            <div style="color: #787B86; font-size: 13px;">Try adjusting your filters to see Twitter/X & Reddit news</div>
+        </div>
+        """, unsafe_allow_html=True)
+    else:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading Twitter/X & Reddit News</div>
+            <div style="color: #787B86; font-size: 13px;">Fetching real-time news from premium sources...</div>
+            <div style="color: #787B86; font-size: 12px; margin-top: 8px; opacity: 0.7;">This may take 30-60 seconds on first load</div>
+        </div>
+        <style>
+        @keyframes pulse {
+            0%, 100% { opacity: 1; transform: scale(1); }
+            50% { opacity: 0.6; transform: scale(1.1); }
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+with col2:
+    # SECTION 2: Main Page News (Web-Scraped)
+    if not rss_main_filtered.empty:
+        display_scrollable_news_section(
+            rss_main_filtered,
+            section_title="Top Headlines",
+            section_icon="🔥",
+            section_subtitle="Latest from main pages",
+            max_items=50,
+            height="700px"
+        )
+    elif not rss_main_df.empty:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px;">📭</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">No matches found</div>
+            <div style="color: #787B86; font-size: 13px;">Try adjusting your filters to see top headlines</div>
+        </div>
+        """, unsafe_allow_html=True)
+    else:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading Top Headlines</div>
+            <div style="color: #787B86; font-size: 13px;">Fetching latest news from major outlets...</div>
+            <div style="color: #787B86; font-size: 12px; margin-top: 8px; opacity: 0.7;">Web scraping main pages</div>
+        </div>
+        <style>
+        @keyframes pulse {
+            0%, 100% { opacity: 1; transform: scale(1); }
+            50% { opacity: 0.6; transform: scale(1.1); }
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+with col3:
+    # SECTION 3: RSS Feed News
+    if not rss_all_filtered.empty:
+        display_scrollable_news_section(
+            rss_all_filtered,
+            section_title="RSS Feed",
+            section_icon="📰",
+            section_subtitle="Aggregated from all sources",
+            max_items=100,
+            height="700px"
+        )
+    elif not rss_all_df.empty:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px;">📭</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">No matches found</div>
+            <div style="color: #787B86; font-size: 13px;">Try adjusting your filters to see RSS feed news</div>
+        </div>
+        """, unsafe_allow_html=True)
+    else:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading RSS Feed</div>
+            <div style="color: #787B86; font-size: 13px;">Aggregating news from all RSS sources...</div>
+            <div style="color: #787B86; font-size: 12px; margin-top: 8px; opacity: 0.7;">Bloomberg, Reuters, FT, WSJ & more</div>
+        </div>
+        <style>
+        @keyframes pulse {
+            0%, 100% { opacity: 1; transform: scale(1); }
+            50% { opacity: 0.6; transform: scale(1.1); }
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+with col4:
+    # SECTION 4: AI & Tech News
+    if not ai_tech_df.empty:
+        display_scrollable_news_section(
+            ai_tech_df,
+            section_title="AI & Tech News",
+            section_icon="🤖",
+            section_subtitle="Latest from tech giants & AI research",
+            max_items=100,
+            height="700px"
+        )
+    else:
+        # Debug: Check if there's an AI/Tech specific error
+        ai_tech_error = next((err for err in fetch_errors if 'ai_tech' in err.lower() or 'AI/Tech' in err), None) if 'fetch_errors' in locals() else None
+
+        if ai_tech_error:
+            # Show error message
+            st.markdown(f"""
+            <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+                <div style="font-size: 48px; margin-bottom: 16px;">⚠️</div>
+                <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">AI & Tech News Unavailable</div>
+                <div style="color: #787B86; font-size: 13px;">{ai_tech_error}</div>
+            </div>
+            """, unsafe_allow_html=True)
+        else:
+            # Show loading message
+            st.markdown("""
+            <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+                <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+                <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading AI & Tech News</div>
+                <div style="color: #787B86; font-size: 13px;">Aggregating from tech blogs & research...</div>
+                <div style="color: #787B86; font-size: 12px; margin-top: 8px; opacity: 0.7;">OpenAI, Google AI, Microsoft, Meta & more</div>
+                <div style="color: #FF9500; font-size: 12px; margin-top: 12px;">If this persists, check the "Source Fetch Warnings" section below</div>
+            </div>
+            <style>
+            @keyframes pulse {
+                0%, 100% { opacity: 1; transform: scale(1); }
+                50% { opacity: 0.6; transform: scale(1.1); }
+            }
+            </style>
+            """, unsafe_allow_html=True)
+
+# ---- SECOND ROW: MARKET INTELLIGENCE (3 COLUMNS) ----
+st.markdown("---")
+st.markdown("## 📊 Market Intelligence - Predictions, Sectors & Events")
+
+col5, col6, col7 = st.columns(3)
+
+with col5:
+    # Prediction Markets Column
+    if not predictions_df.empty:
+        display_scrollable_news_section(
+            predictions_df,
+            section_title="Prediction Markets",
+            section_icon="🎲",
+            section_subtitle="Polymarket, Metaculus & CME FedWatch",
+            max_items=50,
+            height="600px"
+        )
+    else:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading Prediction Markets</div>
+            <div style="color: #787B86; font-size: 13px;">Fetching market forecasts...</div>
+        </div>
+        <style>
+        @keyframes pulse {
+            0%, 100% { opacity: 1; transform: scale(1); }
+            50% { opacity: 0.6; transform: scale(1.1); }
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+with col6:
+    # Sectoral News Column
+    if not sectoral_news_df.empty:
+        display_scrollable_news_section(
+            sectoral_news_df,
+            section_title="Sectoral News",
+            section_icon="🏭",
+            section_subtitle="7 sectors: Finance, Tech, Energy & more",
+            max_items=50,
+            height="600px"
+        )
+    else:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading Sectoral News</div>
+            <div style="color: #787B86; font-size: 13px;">Aggregating sector-specific news...</div>
+        </div>
+        <style>
+        @keyframes pulse {
+            0%, 100% { opacity: 1; transform: scale(1); }
+            50% { opacity: 0.6; transform: scale(1.1); }
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+with col7:
+    # Market Events Column
+    if not market_events_df.empty:
+        display_scrollable_news_section(
+            market_events_df,
+            section_title="Market Events",
+            section_icon="📈",
+            section_subtitle="Earnings, indicators & central banks",
+            max_items=50,
+            height="600px"
+        )
+    else:
+        st.markdown("""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 30px; text-align: center;">
+            <div style="font-size: 48px; margin-bottom: 16px; animation: pulse 2s ease-in-out infinite;">⏳</div>
+            <div style="color: #D1D4DC; font-size: 16px; font-weight: 600; margin-bottom: 8px;">Loading Market Events</div>
+            <div style="color: #787B86; font-size: 13px;">Fetching earnings & economic indicators...</div>
+        </div>
+        <style>
+        @keyframes pulse {
+            0%, 100% { opacity: 1; transform: scale(1); }
+            50% { opacity: 0.6; transform: scale(1.1); }
+        }
+        </style>
+        """, unsafe_allow_html=True)
+
+# Display fetch errors in expander (less intrusive)
+if 'fetch_errors' in locals() and fetch_errors:
+    with st.expander("⚠️ Source Fetch Warnings", expanded=False):
+        for error in fetch_errors:
+            st.caption(f"• {error}")
+
+# ---- AI SUMMARY METRICS ----
+total_items = sum(len(df) for df in ai_summary_dfs if not df.empty)
+ai_summarized = 0
+for df in ai_summary_dfs:
+    if df.empty or "summary_ai" not in df.columns:
+        continue
+    ai_summarized += df["summary_ai"].fillna("").astype(str).str.strip().ne("").sum()
+
+ai_summary_pct = (ai_summarized / total_items * 100) if total_items else 0.0
+
+st.markdown("---")
+@st.fragment(run_every=60)
+def render_ai_summary_section():
+    summaries = fetch_summaries(limit=50)
+    status = get_status()
+    last_update_text = status.get("last_update") or "N/A"
+    buffer_remaining = status.get("buffer_remaining_seconds")
+    buffer_text = "N/A"
+    if buffer_remaining is not None:
+        buffer_text = f"{int(buffer_remaining)}s"
+
+    st.markdown("## 🤖 AI Summary")
+    st.markdown(
+        f"""
+        <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 20px; margin-bottom: 12px;">
+            <div style="color: #E0E3EB; font-size: 16px; font-weight: 600; margin-bottom: 6px;">Current AI Summarizations</div>
+            <div style="color: #D1D4DC; font-size: 14px; line-height: 1.6;">
+                {ai_summarized} / {total_items} items summarized
+                <span style="color: #787B86; font-size: 12px; margin-left: 8px;">({ai_summary_pct:.1f}% coverage)</span>
+            </div>
+            <div style="color: #787B86; font-size: 12px; margin-top: 6px;">Last update: {last_update_text}</div>
+            <div style="color: #787B86; font-size: 12px;">Buffer: {status.get("buffer_size", 0)} items, next flush in {buffer_text}</div>
+            <div style="color: #787B86; font-size: 12px;">Cache: {status.get("total_summaries", 0)} summaries, batch max ~{status.get("batch_max_chars", 0)} chars</div>
+        </div>
+        """,
+        unsafe_allow_html=True,
+    )
+
+    if summaries:
+        for item in summaries:
+            source = item.get("source", "")
+            summary = item.get("summary", "")
+            title = item.get("title", "")
+            st.markdown(
+                f"""
+                <div style="background: #131722; border: 1px solid #2A2E39; border-radius: 6px; padding: 10px; margin-bottom: 8px;">
+                    <div style="color: #E0E3EB; font-size: 13px; font-weight: 600;">{source} — {title}</div>
+                    <div style="color: #D1D4DC; font-size: 13px; margin-top: 4px;">{summary}</div>
+                </div>
+                """,
+                unsafe_allow_html=True,
+            )
+    else:
+        st.info("AI summaries will appear after the 2-minute buffering window completes.")
+
+render_ai_summary_section()
+
+# Auto-refresh logic
+if auto_refresh:
+    import time
+    time.sleep(180)  # 3 minutes
+    st.rerun()
+
+# ---- Footer with Instructions ----
+st.markdown("---")
+st.markdown("""
+### 💡 How to Use This Dashboard
+
+**For Traders:**
+- Monitor breaking news in real-time for market-moving events
+- Filter by category to focus on macro, markets, or geopolitical news
+- Use sentiment analysis to gauge market mood
+- High-impact news items require immediate attention
+
+**Tips:**
+- Enable auto-refresh for continuous monitoring during trading hours
+- Focus on "HIGH IMPACT" news for potential volatility
+- Breaking news (🔴) indicates urgent market-moving information
+- Check engagement metrics (likes + retweets) for news importance
+
+**Data Source:** Dual-mode scraping - RSS feeds + direct web page parsing from Reuters, Bloomberg, FT, WSJ, CNBC, Google News, Yahoo Finance, Fed, ECB and more
+**Update Frequency:** 3-minute cache for low-latency delivery
+**No Authentication Required:** Public sources - works out of the box
+""")

app/services/__init__.py

@@ -0,0 +1 @@
+"""Services package for financial platform."""

app/services/ai_tech_news.py

@@ -0,0 +1,293 @@
+"""
+AI & Tech News Scraper
+Fetches news from popular tech resources and big tech company blogs
+"""
+
+import feedparser
+import requests
+from bs4 import BeautifulSoup
+from datetime import datetime, timedelta
+from typing import List, Dict
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class AITechNewsScraper:
+    """Scraper for AI and tech news from major sources and company blogs"""
+
+    # AI/Tech News Sources (RSS + Web)
+    SOURCES = {
+        # Major Tech News
+        'TechCrunch AI': {
+            'url': 'https://techcrunch.com/category/artificial-intelligence/feed/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'The Verge AI': {
+            'url': 'https://www.theverge.com/ai-artificial-intelligence/rss/index.xml',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'VentureBeat AI': {
+            'url': 'https://venturebeat.com/category/ai/feed/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'MIT Technology Review AI': {
+            'url': 'https://www.technologyreview.com/topic/artificial-intelligence/feed',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'Ars Technica AI': {
+            'url': 'https://feeds.arstechnica.com/arstechnica/technology-lab',
+            'type': 'rss',
+            'category': 'tech'
+        },
+        'Wired AI': {
+            'url': 'https://www.wired.com/feed/tag/ai/latest/rss',
+            'type': 'rss',
+            'category': 'ai'
+        },
+
+        # Big Tech Company Blogs
+        'OpenAI Blog': {
+            'url': 'https://openai.com/blog/rss.xml',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'Google AI Blog': {
+            'url': 'https://blog.google/technology/ai/rss/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'Microsoft AI Blog': {
+            'url': 'https://blogs.microsoft.com/ai/feed/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'Meta AI Blog': {
+            'url': 'https://ai.meta.com/blog/rss/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'DeepMind Blog': {
+            'url': 'https://deepmind.google/blog/rss.xml',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'Anthropic News': {
+            'url': 'https://www.anthropic.com/news/rss.xml',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'AWS AI Blog': {
+            'url': 'https://aws.amazon.com/blogs/machine-learning/feed/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+        'NVIDIA AI Blog': {
+            'url': 'https://blogs.nvidia.com/feed/',
+            'type': 'rss',
+            'category': 'ai'
+        },
+
+        # Research & Academia
+        'Stanford HAI': {
+            'url': 'https://hai.stanford.edu/news/rss.xml',
+            'type': 'rss',
+            'category': 'research'
+        },
+        'Berkeley AI Research': {
+            'url': 'https://bair.berkeley.edu/blog/feed.xml',
+            'type': 'rss',
+            'category': 'research'
+        },
+    }
+
+    def __init__(self):
+        """Initialize the AI/Tech news scraper"""
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+        })
+
+    def scrape_ai_tech_news(self, max_items: int = 100, hours: int = 48) -> List[Dict]:
+        """
+        Scrape AI and tech news from all sources
+
+        Args:
+            max_items: Maximum number of news items to return
+            hours: Only include news from the last N hours
+
+        Returns:
+            List of news items with standardized format
+        """
+        all_news = []
+        cutoff_time = datetime.now() - timedelta(hours=hours)
+
+        for source_name, source_config in self.SOURCES.items():
+            try:
+                if source_config['type'] == 'rss':
+                    news_items = self._scrape_rss_feed(
+                        source_name,
+                        source_config['url'],
+                        source_config['category'],
+                        cutoff_time
+                    )
+                    all_news.extend(news_items)
+                    logger.info(f"Scraped {len(news_items)} items from {source_name}")
+
+            except Exception as e:
+                logger.error(f"Error scraping {source_name}: {e}")
+                continue
+
+        # Sort by timestamp (newest first)
+        all_news.sort(key=lambda x: x['timestamp'], reverse=True)
+
+        # Limit to max_items
+        return all_news[:max_items]
+
+    def _scrape_rss_feed(self, source_name: str, feed_url: str,
+                        category: str, cutoff_time: datetime) -> List[Dict]:
+        """Scrape a single RSS feed"""
+        news_items = []
+
+        try:
+            feed = feedparser.parse(feed_url)
+
+            for entry in feed.entries:
+                try:
+                    # Parse timestamp
+                    if hasattr(entry, 'published_parsed') and entry.published_parsed:
+                        timestamp = datetime(*entry.published_parsed[:6])
+                    elif hasattr(entry, 'updated_parsed') and entry.updated_parsed:
+                        timestamp = datetime(*entry.updated_parsed[:6])
+                    else:
+                        timestamp = datetime.now()
+
+                    # Skip old news
+                    if timestamp < cutoff_time:
+                        continue
+
+                    # Extract title and summary
+                    title = entry.get('title', 'No title')
+                    summary = entry.get('summary', entry.get('description', ''))
+
+                    # Clean HTML from summary
+                    if summary:
+                        soup = BeautifulSoup(summary, 'html.parser')
+                        summary = soup.get_text().strip()
+                        # Limit summary length
+                        if len(summary) > 300:
+                            summary = summary[:297] + '...'
+
+                    # Determine impact and sentiment based on keywords
+                    impact = self._determine_impact(title, summary)
+                    sentiment = self._determine_sentiment(title, summary)
+
+                    news_item = {
+                        'title': title,
+                        'summary': summary or title,
+                        'source': source_name,
+                        'url': entry.get('link', ''),
+                        'timestamp': timestamp,
+                        'category': category,
+                        'impact': impact,
+                        'sentiment': sentiment,
+                        'is_breaking': self._is_breaking_news(title, summary),
+                        'likes': 0,  # No engagement data for RSS
+                        'retweets': 0,
+                        'reddit_score': 0,
+                        'reddit_comments': 0
+                    }
+
+                    news_items.append(news_item)
+
+                except Exception as e:
+                    logger.error(f"Error parsing entry from {source_name}: {e}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"Error fetching RSS feed {feed_url}: {e}")
+
+        return news_items
+
+    def _determine_impact(self, title: str, summary: str) -> str:
+        """Determine impact level based on keywords"""
+        text = f"{title} {summary}".lower()
+
+        high_impact_keywords = [
+            'breakthrough', 'announce', 'launch', 'release', 'new model',
+            'gpt', 'claude', 'gemini', 'llama', 'chatgpt',
+            'billion', 'trillion', 'acquisition', 'merger',
+            'regulation', 'ban', 'lawsuit', 'security breach',
+            'major', 'significant', 'revolutionary', 'first-ever'
+        ]
+
+        medium_impact_keywords = [
+            'update', 'improve', 'enhance', 'study', 'research',
+            'partnership', 'collaboration', 'funding', 'investment',
+            'expands', 'grows', 'adopts', 'implements'
+        ]
+
+        for keyword in high_impact_keywords:
+            if keyword in text:
+                return 'high'
+
+        for keyword in medium_impact_keywords:
+            if keyword in text:
+                return 'medium'
+
+        return 'low'
+
+    def _determine_sentiment(self, title: str, summary: str) -> str:
+        """Determine sentiment based on keywords"""
+        text = f"{title} {summary}".lower()
+
+        positive_keywords = [
+            'breakthrough', 'success', 'achieve', 'improve', 'advance',
+            'innovative', 'revolutionary', 'launch', 'release', 'win',
+            'growth', 'expand', 'partnership', 'collaboration'
+        ]
+
+        negative_keywords = [
+            'fail', 'issue', 'problem', 'concern', 'worry', 'risk',
+            'ban', 'lawsuit', 'breach', 'hack', 'leak', 'crisis',
+            'decline', 'loss', 'shutdown', 'controversy'
+        ]
+
+        positive_count = sum(1 for kw in positive_keywords if kw in text)
+        negative_count = sum(1 for kw in negative_keywords if kw in text)
+
+        if positive_count > negative_count:
+            return 'positive'
+        elif negative_count > positive_count:
+            return 'negative'
+        else:
+            return 'neutral'
+
+    def _is_breaking_news(self, title: str, summary: str) -> bool:
+        """Determine if news is breaking"""
+        text = f"{title} {summary}".lower()
+
+        breaking_indicators = [
+            'breaking', 'just announced', 'just released', 'just launched',
+            'alert', 'urgent', 'developing', 'live', 'now:'
+        ]
+
+        return any(indicator in text for indicator in breaking_indicators)
+
+    def get_statistics(self) -> Dict:
+        """Get statistics - returns empty for backward compatibility"""
+        return {
+            'total': 0,
+            'high_impact': 0,
+            'breaking': 0,
+            'last_update': 'Managed by cache',
+            'by_category': {
+                'ai': 0,
+                'tech': 0,
+                'research': 0
+            }
+        }

app/services/economic_calendar.py

@@ -0,0 +1,385 @@
+"""
+Economic Calendar Scraper - Investing.com
+Scrapes upcoming economic events, indicators, and releases
+No API key required - web scraping approach
+"""
+
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import logging
+import re
+
+import requests
+from bs4 import BeautifulSoup
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class EconomicCalendarService:
+    """
+    Scrapes economic calendar data from Investing.com
+    Focus: High and medium importance events
+    """
+
+    def __init__(self):
+        """Initialize scraper with session"""
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Referer': 'https://www.google.com/',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Cache-Control': 'max-age=0'
+        })
+
+    def get_upcoming_events(self, days_ahead: int = 7, min_importance: str = 'medium') -> List[Dict]:
+        """
+        Get upcoming economic events
+        Returns list of events in standardized format
+        """
+        try:
+            # Try to scrape from Investing.com
+            events = self._scrape_investing_com(days_ahead, min_importance)
+
+            if events:
+                logger.info(f"Scraped {len(events)} economic events from Investing.com")
+                return events
+            else:
+                logger.warning("No events scraped - using mock data")
+                return self._get_mock_events()
+
+        except Exception as e:
+            logger.error(f"Error fetching economic calendar: {e}")
+            return self._get_mock_events()
+
+    def _scrape_investing_com(self, days_ahead: int, min_importance: str) -> List[Dict]:
+        """
+        Scrape economic calendar from Investing.com
+        Note: This may be fragile and break if they change their HTML structure
+        """
+        try:
+            url = 'https://www.investing.com/economic-calendar/'
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.content, 'html.parser')
+            events = []
+
+            # Investing.com uses a table structure for the calendar
+            # Look for table rows with event data
+            calendar_table = soup.find('table', {'id': 'economicCalendarData'})
+
+            if not calendar_table:
+                logger.warning("Could not find economic calendar table on Investing.com")
+                return []
+
+            rows = calendar_table.find_all('tr', {'class': 'js-event-item'})
+
+            for row in rows[:50]:  # Limit to 50 events
+                try:
+                    # Extract event data from row
+                    event_data = self._parse_event_row(row)
+
+                    if event_data and self._should_include_event(event_data, days_ahead, min_importance):
+                        events.append(event_data)
+
+                except Exception as e:
+                    logger.debug(f"Error parsing event row: {e}")
+                    continue
+
+            return events
+
+        except Exception as e:
+            logger.error(f"Error scraping Investing.com: {e}")
+            return []
+
+    def _parse_event_row(self, row) -> Optional[Dict]:
+        """Parse a single event row from Investing.com table"""
+        try:
+            # Extract timestamp
+            timestamp_elem = row.find('td', {'class': 'first left time'})
+            time_str = timestamp_elem.get_text(strip=True) if timestamp_elem else ''
+
+            # Extract country
+            country_elem = row.find('td', {'class': 'flagCur'})
+            country = country_elem.get('title', 'US') if country_elem else 'US'
+
+            # Extract importance (bull icons)
+            importance_elem = row.find('td', {'class': 'sentiment'})
+            importance = self._parse_importance(importance_elem) if importance_elem else 'low'
+
+            # Extract event name
+            event_elem = row.find('td', {'class': 'left event'})
+            event_name = event_elem.get_text(strip=True) if event_elem else ''
+
+            # Extract actual, forecast, previous values
+            actual_elem = row.find('td', {'id': re.compile('eventActual_')})
+            forecast_elem = row.find('td', {'id': re.compile('eventForecast_')})
+            previous_elem = row.find('td', {'id': re.compile('eventPrevious_')})
+
+            actual = self._parse_value(actual_elem.get_text(strip=True) if actual_elem else '')
+            forecast = self._parse_value(forecast_elem.get_text(strip=True) if forecast_elem else '')
+            previous = self._parse_value(previous_elem.get_text(strip=True) if previous_elem else '')
+
+            # Create event dictionary
+            event_date = self._parse_event_time(time_str)
+            time_to_event = self._calculate_time_to_event(event_date)
+
+            return {
+                'id': hash(f"{event_name}_{event_date}_{country}"),
+                'title': f"{country} - {event_name}",
+                'event_name': event_name,
+                'event_date': event_date,
+                'country': country,
+                'category': self._categorize_event(event_name),
+                'importance': importance,
+                'forecast': forecast,
+                'previous': previous,
+                'actual': actual,
+                'time_to_event': time_to_event,
+                'timestamp': datetime.now(),
+                'source': 'Investing.com',
+                'url': 'https://www.investing.com/economic-calendar/',
+                'impact': importance,  # Map importance to impact
+                'sentiment': self._determine_sentiment(actual, forecast, previous)
+            }
+
+        except Exception as e:
+            logger.debug(f"Error parsing event row: {e}")
+            return None
+
+    def _parse_importance(self, importance_elem) -> str:
+        """Parse importance from bull icons"""
+        if not importance_elem:
+            return 'low'
+
+        # Investing.com uses bull icons (1-3 bulls)
+        bulls = importance_elem.find_all('i', {'class': 'grayFullBullishIcon'})
+        num_bulls = len(bulls)
+
+        if num_bulls >= 3:
+            return 'high'
+        elif num_bulls == 2:
+            return 'medium'
+        else:
+            return 'low'
+
+    def _parse_value(self, value_str: str) -> Optional[float]:
+        """Parse numeric value from string"""
+        if not value_str or value_str == '' or value_str == '-':
+            return None
+
+        try:
+            # Remove % sign, K, M, B suffixes
+            value_str = value_str.replace('%', '').replace('K', '').replace('M', '').replace('B', '')
+            value_str = value_str.replace(',', '')
+            return float(value_str)
+        except:
+            return None
+
+    def _parse_event_time(self, time_str: str) -> datetime:
+        """Parse event time string to datetime"""
+        try:
+            # Investing.com uses formats like "10:00" or "All Day"
+            if 'All Day' in time_str or not time_str:
+                # Default to noon today
+                return datetime.now().replace(hour=12, minute=0, second=0, microsecond=0)
+
+            # Parse time (assumes today for now - real implementation would need date context)
+            time_parts = time_str.split(':')
+            hour = int(time_parts[0])
+            minute = int(time_parts[1]) if len(time_parts) > 1 else 0
+
+            event_time = datetime.now().replace(hour=hour, minute=minute, second=0, microsecond=0)
+
+            # If time has passed today, assume it's tomorrow
+            if event_time < datetime.now():
+                event_time += timedelta(days=1)
+
+            return event_time
+
+        except Exception as e:
+            logger.debug(f"Error parsing time: {e}")
+            return datetime.now() + timedelta(hours=2)
+
+    def _calculate_time_to_event(self, event_date: datetime) -> str:
+        """Calculate human-readable time until event"""
+        delta = event_date - datetime.now()
+
+        if delta.total_seconds() < 0:
+            return "In progress"
+
+        days = delta.days
+        hours = delta.seconds // 3600
+        minutes = (delta.seconds % 3600) // 60
+
+        if days > 0:
+            return f"in {days}d {hours}h"
+        elif hours > 0:
+            return f"in {hours}h {minutes}m"
+        else:
+            return f"in {minutes}m"
+
+    def _categorize_event(self, event_name: str) -> str:
+        """Categorize economic event"""
+        event_lower = event_name.lower()
+
+        if any(kw in event_lower for kw in ['cpi', 'inflation', 'pce', 'price']):
+            return 'inflation'
+        elif any(kw in event_lower for kw in ['employment', 'jobs', 'unemployment', 'nfp', 'payroll']):
+            return 'employment'
+        elif any(kw in event_lower for kw in ['gdp', 'growth']):
+            return 'gdp'
+        elif any(kw in event_lower for kw in ['fed', 'fomc', 'ecb', 'rate', 'boe', 'boj']):
+            return 'central_bank'
+        elif any(kw in event_lower for kw in ['pmi', 'manufacturing', 'services']):
+            return 'pmi'
+        else:
+            return 'other'
+
+    def _determine_sentiment(self, actual: Optional[float], forecast: Optional[float], previous: Optional[float]) -> str:
+        """Determine sentiment based on actual vs forecast"""
+        if actual is None or forecast is None:
+            return 'neutral'
+
+        if actual > forecast:
+            return 'positive'  # Beat forecast
+        elif actual < forecast:
+            return 'negative'  # Missed forecast
+        else:
+            return 'neutral'
+
+    def _should_include_event(self, event: Dict, days_ahead: int, min_importance: str) -> bool:
+        """Determine if event should be included"""
+        # Filter by importance
+        importance_levels = ['low', 'medium', 'high']
+        min_level = importance_levels.index(min_importance)
+        event_level = importance_levels.index(event['importance'])
+
+        if event_level < min_level:
+            return False
+
+        # Filter by date range
+        days_until = (event['event_date'] - datetime.now()).days
+        if days_until > days_ahead:
+            return False
+
+        return True
+
+    def _get_mock_events(self) -> List[Dict]:
+        """Mock economic events for development/testing"""
+        now = datetime.now()
+
+        return [
+            {
+                'id': 1,
+                'title': 'US - Consumer Price Index (CPI)',
+                'event_name': 'Consumer Price Index',
+                'event_date': now + timedelta(hours=2),
+                'country': 'US',
+                'category': 'inflation',
+                'importance': 'high',
+                'forecast': 2.5,
+                'previous': 2.3,
+                'actual': None,
+                'time_to_event': 'in 2h 0m',
+                'timestamp': now,
+                'source': 'Economic Calendar',
+                'url': 'https://www.investing.com/economic-calendar/',
+                'impact': 'high',
+                'sentiment': 'neutral'
+            },
+            {
+                'id': 2,
+                'title': 'US - Non-Farm Payrolls (NFP)',
+                'event_name': 'Non-Farm Payrolls',
+                'event_date': now + timedelta(days=2, hours=8, minutes=30),
+                'country': 'US',
+                'category': 'employment',
+                'importance': 'high',
+                'forecast': 180.0,
+                'previous': 175.0,
+                'actual': None,
+                'time_to_event': 'in 2d 8h',
+                'timestamp': now,
+                'source': 'Economic Calendar',
+                'url': 'https://www.investing.com/economic-calendar/',
+                'impact': 'high',
+                'sentiment': 'neutral'
+            },
+            {
+                'id': 3,
+                'title': 'EU - ECB Interest Rate Decision',
+                'event_name': 'ECB Interest Rate Decision',
+                'event_date': now + timedelta(days=3, hours=12),
+                'country': 'EU',
+                'category': 'central_bank',
+                'importance': 'high',
+                'forecast': 3.75,
+                'previous': 4.00,
+                'actual': None,
+                'time_to_event': 'in 3d 12h',
+                'timestamp': now,
+                'source': 'Economic Calendar',
+                'url': 'https://www.investing.com/economic-calendar/',
+                'impact': 'high',
+                'sentiment': 'neutral'
+            },
+            {
+                'id': 4,
+                'title': 'US - GDP Growth Rate',
+                'event_name': 'GDP Growth Rate',
+                'event_date': now + timedelta(days=5, hours=8, minutes=30),
+                'country': 'US',
+                'category': 'gdp',
+                'importance': 'high',
+                'forecast': 2.8,
+                'previous': 2.5,
+                'actual': None,
+                'time_to_event': 'in 5d 8h',
+                'timestamp': now,
+                'source': 'Economic Calendar',
+                'url': 'https://www.investing.com/economic-calendar/',
+                'impact': 'high',
+                'sentiment': 'neutral'
+            },
+            {
+                'id': 5,
+                'title': 'US - Manufacturing PMI',
+                'event_name': 'Manufacturing PMI',
+                'event_date': now + timedelta(days=1, hours=10),
+                'country': 'US',
+                'category': 'pmi',
+                'importance': 'medium',
+                'forecast': 51.5,
+                'previous': 50.8,
+                'actual': None,
+                'time_to_event': 'in 1d 10h',
+                'timestamp': now,
+                'source': 'Economic Calendar',
+                'url': 'https://www.investing.com/economic-calendar/',
+                'impact': 'medium',
+                'sentiment': 'neutral'
+            }
+        ]
+
+    def get_todays_events(self) -> List[Dict]:
+        """Get events happening today"""
+        all_events = self.get_upcoming_events(days_ahead=1)
+        today = datetime.now().date()
+
+        todays_events = [
+            event for event in all_events
+            if event['event_date'].date() == today
+        ]
+
+        return todays_events

app/services/market_events.py

@@ -0,0 +1,391 @@
+"""
+Market Events Scraper - Earnings, Economic Indicators & Central Bank Events
+Aggregates upcoming and recent market-moving events
+Web scraping approach - no API keys required
+"""
+
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+import feedparser
+from bs4 import BeautifulSoup
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class MarketEventsScraper:
+    """
+    Scrapes market events from multiple sources
+    Focus: Earnings, economic indicators, central bank announcements
+    """
+
+    # Central bank RSS feeds (already in use for news)
+    CENTRAL_BANKS = {
+        'fed': {
+            'name': 'Federal Reserve',
+            'rss': 'https://www.federalreserve.gov/feeds/press_all.xml',
+            'weight': 2.0
+        },
+        'ecb': {
+            'name': 'European Central Bank',
+            'rss': 'https://www.ecb.europa.eu/rss/press.xml',
+            'weight': 2.0
+        }
+    }
+
+    def __init__(self):
+        """Initialize scraper"""
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+        })
+
+    def scrape_market_events(self, max_items: int = 50, days_ahead: int = 14) -> List[Dict]:
+        """
+        Scrape market events from all sources
+        Returns unified list sorted by date and impact
+        """
+        all_events = []
+        seen_urls = set()
+
+        # Parallel fetching
+        with ThreadPoolExecutor(max_workers=3) as executor:
+            futures = []
+
+            # Submit tasks
+            futures.append((executor.submit(self._fetch_earnings), 'earnings'))
+            futures.append((executor.submit(self._fetch_economic_indicators), 'indicators'))
+            futures.append((executor.submit(self._fetch_central_bank_events), 'central_banks'))
+
+            for future, source_type in futures:
+                try:
+                    events = future.result(timeout=35)
+
+                    # Deduplicate by URL
+                    for event in events:
+                        if event['url'] not in seen_urls:
+                            seen_urls.add(event['url'])
+                            all_events.append(event)
+
+                    logger.info(f"Fetched {len(events)} events from {source_type}")
+
+                except Exception as e:
+                    logger.error(f"Error fetching {source_type}: {e}")
+
+        # If no events fetched, use mock data
+        if not all_events:
+            logger.warning("No market events fetched - using mock data")
+            return self._get_mock_events()
+
+        # Sort by event date and impact
+        all_events.sort(
+            key=lambda x: (x.get('event_date', x['timestamp']), x['impact'] != 'high'),
+        )
+
+        return all_events[:max_items]
+
+    def _fetch_earnings(self) -> List[Dict]:
+        """
+        Fetch earnings calendar from Yahoo Finance
+        Web scraping approach
+        """
+        try:
+            url = 'https://finance.yahoo.com/calendar/earnings'
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.content, 'html.parser')
+            events = []
+
+            # Yahoo Finance uses a table for earnings
+            table = soup.find('table', {'class': re.compile('earnings')})
+
+            if not table:
+                logger.warning("Could not find earnings table on Yahoo Finance")
+                return self._get_mock_earnings()
+
+            rows = table.find_all('tr')[1:20]  # Skip header, limit to 20
+
+            for row in rows:
+                try:
+                    cells = row.find_all('td')
+                    if len(cells) < 4:
+                        continue
+
+                    # Parse cells
+                    ticker = cells[0].get_text(strip=True)
+                    company = cells[1].get_text(strip=True) if len(cells) > 1 else ticker
+                    eps_estimate = cells[2].get_text(strip=True) if len(cells) > 2 else 'N/A'
+                    reported_eps = cells[3].get_text(strip=True) if len(cells) > 3 else None
+                    event_time = cells[4].get_text(strip=True) if len(cells) > 4 else 'N/A'
+
+                    # Create event
+                    event_date = self._parse_earnings_date(event_time)
+
+                    events.append({
+                        'id': hash(f"earnings_{ticker}_{event_date}"),
+                        'title': f"{company} ({ticker}) Earnings Report",
+                        'summary': f"Expected EPS: {eps_estimate}" + (f", Reported: {reported_eps}" if reported_eps and reported_eps != 'N/A' else ''),
+                        'source': 'Yahoo Finance',
+                        'category': 'earnings',
+                        'timestamp': datetime.now(),
+                        'event_date': event_date,
+                        'url': f"https://finance.yahoo.com/quote/{ticker}",
+                        'event_type': 'earnings',
+                        'ticker': ticker,
+                        'expected_value': self._parse_float(eps_estimate),
+                        'actual_value': self._parse_float(reported_eps) if reported_eps else None,
+                        'previous_value': None,
+                        'impact': 'medium',  # Earnings are generally medium impact
+                        'sentiment': self._determine_earnings_sentiment(eps_estimate, reported_eps),
+                        'is_breaking': False,
+                        'source_weight': 1.3,
+                        'likes': 0,
+                        'retweets': 0
+                    })
+
+                except Exception as e:
+                    logger.debug(f"Error parsing earnings row: {e}")
+                    continue
+
+            return events if events else self._get_mock_earnings()
+
+        except Exception as e:
+            logger.error(f"Error fetching earnings: {e}")
+            return self._get_mock_earnings()
+
+    def _fetch_economic_indicators(self) -> List[Dict]:
+        """
+        Fetch economic indicators from FRED and other sources
+        Uses RSS feeds
+        """
+        try:
+            events = []
+
+            # FRED Economic Data releases (via RSS - if available)
+            # For now, use mock data as FRED RSS is primarily historical data
+            # Real implementation would scrape FRED release calendar
+
+            events.extend(self._get_mock_indicators())
+
+            return events
+
+        except Exception as e:
+            logger.error(f"Error fetching economic indicators: {e}")
+            return self._get_mock_indicators()
+
+    def _fetch_central_bank_events(self) -> List[Dict]:
+        """
+        Fetch central bank announcements from RSS feeds
+        """
+        events = []
+
+        for bank_id, bank_info in self.CENTRAL_BANKS.items():
+            try:
+                feed = feedparser.parse(bank_info['rss'])
+
+                for entry in feed.entries[:10]:
+                    try:
+                        # Parse timestamp
+                        if hasattr(entry, 'published_parsed') and entry.published_parsed:
+                            timestamp = datetime(*entry.published_parsed[:6])
+                        else:
+                            timestamp = datetime.now()
+
+                        # Skip old events (>7 days)
+                        if (datetime.now() - timestamp).days > 7:
+                            continue
+
+                        title = entry.get('title', '')
+                        summary = entry.get('summary', '') or title
+                        url = entry.get('link', '')
+
+                        # Clean HTML from summary
+                        if summary:
+                            summary = BeautifulSoup(summary, 'html.parser').get_text()
+                            summary = summary[:200] + '...' if len(summary) > 200 else summary
+
+                        events.append({
+                            'id': hash(url),
+                            'title': f"{bank_info['name']}: {title}",
+                            'summary': summary,
+                            'source': bank_info['name'],
+                            'category': 'central_bank',
+                            'timestamp': timestamp,
+                            'event_date': timestamp,
+                            'url': url,
+                            'event_type': 'central_bank_announcement',
+                            'ticker': None,
+                            'expected_value': None,
+                            'actual_value': None,
+                            'previous_value': None,
+                            'impact': 'high',  # Central bank events are high impact
+                            'sentiment': 'neutral',
+                            'is_breaking': (datetime.now() - timestamp).days < 1,
+                            'source_weight': bank_info['weight'],
+                            'likes': 0,
+                            'retweets': 0
+                        })
+
+                    except Exception as e:
+                        logger.debug(f"Error parsing {bank_id} entry: {e}")
+                        continue
+
+            except Exception as e:
+                logger.error(f"Error fetching {bank_id} RSS: {e}")
+
+        return events
+
+    def _parse_earnings_date(self, time_str: str) -> datetime:
+        """Parse earnings report time"""
+        # Yahoo Finance uses "Before Market Open", "After Market Close", or specific dates
+        now = datetime.now()
+
+        if 'Before Market' in time_str or 'BMO' in time_str:
+            return now.replace(hour=7, minute=0, second=0, microsecond=0)
+        elif 'After Market' in time_str or 'AMC' in time_str:
+            return now.replace(hour=16, minute=0, second=0, microsecond=0)
+        else:
+            # Default to tomorrow morning
+            return (now + timedelta(days=1)).replace(hour=7, minute=0, second=0, microsecond=0)
+
+    def _parse_float(self, value_str: str) -> Optional[float]:
+        """Parse float from string"""
+        if not value_str or value_str == 'N/A' or value_str == '-':
+            return None
+
+        try:
+            # Remove $ and other non-numeric characters except . and -
+            cleaned = re.sub(r'[^\d.-]', '', value_str)
+            return float(cleaned)
+        except:
+            return None
+
+    def _determine_earnings_sentiment(self, expected: str, actual: Optional[str]) -> str:
+        """Determine sentiment based on earnings beat/miss"""
+        if not actual or actual == 'N/A':
+            return 'neutral'
+
+        exp_val = self._parse_float(expected)
+        act_val = self._parse_float(actual)
+
+        if exp_val is None or act_val is None:
+            return 'neutral'
+
+        if act_val > exp_val:
+            return 'positive'  # Beat
+        elif act_val < exp_val:
+            return 'negative'  # Miss
+        else:
+            return 'neutral'  # In-line
+
+    def _get_mock_earnings(self) -> List[Dict]:
+        """Mock earnings data"""
+        now = datetime.now()
+
+        return [
+            {
+                'id': 1,
+                'title': 'Apple Inc. (AAPL) Earnings Report',
+                'summary': 'Expected EPS: $2.10',
+                'source': 'Yahoo Finance',
+                'category': 'earnings',
+                'timestamp': now,
+                'event_date': now + timedelta(days=2, hours=16),
+                'url': 'https://finance.yahoo.com/quote/AAPL',
+                'event_type': 'earnings',
+                'ticker': 'AAPL',
+                'expected_value': 2.10,
+                'actual_value': None,
+                'previous_value': 1.95,
+                'impact': 'high',
+                'sentiment': 'neutral',
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'likes': 0,
+                'retweets': 0
+            },
+            {
+                'id': 2,
+                'title': 'Microsoft Corporation (MSFT) Earnings Report',
+                'summary': 'Expected EPS: $2.75',
+                'source': 'Yahoo Finance',
+                'category': 'earnings',
+                'timestamp': now,
+                'event_date': now + timedelta(days=3, hours=16),
+                'url': 'https://finance.yahoo.com/quote/MSFT',
+                'event_type': 'earnings',
+                'ticker': 'MSFT',
+                'expected_value': 2.75,
+                'actual_value': None,
+                'previous_value': 2.50,
+                'impact': 'high',
+                'sentiment': 'neutral',
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'likes': 0,
+                'retweets': 0
+            }
+        ]
+
+    def _get_mock_indicators(self) -> List[Dict]:
+        """Mock economic indicator data"""
+        now = datetime.now()
+
+        return [
+            {
+                'id': 3,
+                'title': 'US Retail Sales Data Release',
+                'summary': 'Monthly retail sales figures',
+                'source': 'US Census Bureau',
+                'category': 'economic_indicator',
+                'timestamp': now,
+                'event_date': now + timedelta(days=1, hours=8, minutes=30),
+                'url': 'https://www.census.gov/retail/',
+                'event_type': 'retail_sales',
+                'ticker': None,
+                'expected_value': 0.5,
+                'actual_value': None,
+                'previous_value': 0.3,
+                'impact': 'medium',
+                'sentiment': 'neutral',
+                'is_breaking': False,
+                'source_weight': 1.6,
+                'likes': 0,
+                'retweets': 0
+            }
+        ]
+
+    def _get_mock_events(self) -> List[Dict]:
+        """Combined mock data"""
+        return self._get_mock_earnings() + self._get_mock_indicators() + [
+            {
+                'id': 4,
+                'title': 'Federal Reserve: FOMC Meeting Minutes Released',
+                'summary': 'Minutes from the latest Federal Open Market Committee meeting',
+                'source': 'Federal Reserve',
+                'category': 'central_bank',
+                'timestamp': datetime.now() - timedelta(hours=2),
+                'event_date': datetime.now() - timedelta(hours=2),
+                'url': 'https://www.federalreserve.gov/',
+                'event_type': 'central_bank_announcement',
+                'ticker': None,
+                'expected_value': None,
+                'actual_value': None,
+                'previous_value': None,
+                'impact': 'high',
+                'sentiment': 'neutral',
+                'is_breaking': True,
+                'source_weight': 2.0,
+                'likes': 0,
+                'retweets': 0
+            }
+        ]

app/services/news_monitor.py

@@ -0,0 +1,593 @@
+"""
+Professional Finance News Monitor using snscrape
+Real-time tracking: Macro, Markets, Geopolitical intelligence
+Optimized for low-latency trading decisions
+"""
+
+import pandas as pd
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import streamlit as st
+import time
+import logging
+import re
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+try:
+    import snscrape.modules.twitter as sntwitter
+    SNSCRAPE_AVAILABLE = True
+except ImportError:
+    SNSCRAPE_AVAILABLE = False
+    logger.warning("snscrape not available. Install with: pip install snscrape")
+
+
+class FinanceNewsMonitor:
+    """
+    Professional-grade financial news aggregator
+    Sources: Bloomberg, Reuters, WSJ, FT, CNBC, ZeroHedge
+    """
+
+    # Premium financial sources - expanded coverage
+    SOURCES = {
+        # ===== TIER 1: Major Financial News =====
+        'reuters': {
+            'handle': '@Reuters',
+            'weight': 1.5,
+            'specialization': ['macro', 'geopolitical', 'markets']
+        },
+        'bloomberg': {
+            'handle': '@business',
+            'weight': 1.5,
+            'specialization': ['macro', 'markets']
+        },
+        'ft': {
+            'handle': '@FT',
+            'weight': 1.4,
+            'specialization': ['macro', 'markets']
+        },
+        'economist': {
+            'handle': '@TheEconomist',
+            'weight': 1.3,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'wsj': {
+            'handle': '@WSJ',
+            'weight': 1.4,
+            'specialization': ['markets', 'macro']
+        },
+        'bloomberg_terminal': {
+            'handle': '@Bloomberg',
+            'weight': 1.5,
+            'specialization': ['macro', 'markets']
+        },
+        'cnbc': {
+            'handle': '@CNBC',
+            'weight': 1.2,
+            'specialization': ['markets']
+        },
+        'marketwatch': {
+            'handle': '@MarketWatch',
+            'weight': 1.1,
+            'specialization': ['markets']
+        },
+
+        # ===== TIER 2: Geopolitical Intelligence =====
+        'bbc_world': {
+            'handle': '@BBCWorld',
+            'weight': 1.4,
+            'specialization': ['geopolitical']
+        },
+        'afp': {
+            'handle': '@AFP',
+            'weight': 1.3,
+            'specialization': ['geopolitical']
+        },
+        'aljazeera': {
+            'handle': '@AlJazeera',
+            'weight': 1.2,
+            'specialization': ['geopolitical']
+        },
+        'politico': {
+            'handle': '@politico',
+            'weight': 1.2,
+            'specialization': ['geopolitical', 'macro']
+        },
+        'dw_news': {
+            'handle': '@dwnews',
+            'weight': 1.2,
+            'specialization': ['geopolitical']
+        },
+
+        # ===== TIER 3: Central Banks & Official Sources =====
+        'federal_reserve': {
+            'handle': '@federalreserve',
+            'weight': 2.0,  # Highest priority
+            'specialization': ['macro']
+        },
+        'ecb': {
+            'handle': '@ecb',
+            'weight': 2.0,
+            'specialization': ['macro']
+        },
+        'lagarde': {
+            'handle': '@Lagarde',
+            'weight': 1.9,  # ECB President
+            'specialization': ['macro']
+        },
+        'bank_of_england': {
+            'handle': '@bankofengland',
+            'weight': 1.8,
+            'specialization': ['macro']
+        },
+        'imf': {
+            'handle': '@IMFNews',
+            'weight': 1.7,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'world_bank': {
+            'handle': '@worldbank',
+            'weight': 1.6,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'us_treasury': {
+            'handle': '@USTreasury',
+            'weight': 1.8,
+            'specialization': ['macro']
+        },
+
+        # ===== TIER 4: Alpha Accounts (Fast Breaking News) =====
+        'zerohedge': {
+            'handle': '@zerohedge',
+            'weight': 1.0,
+            'specialization': ['markets', 'macro']
+        },
+        'first_squawk': {
+            'handle': '@FirstSquawk',
+            'weight': 1.1,  # Fast alerts
+            'specialization': ['markets', 'macro']
+        },
+        'live_squawk': {
+            'handle': '@LiveSquawk',
+            'weight': 1.1,  # Real-time market squawks
+            'specialization': ['markets', 'macro']
+        }
+    }
+
+    # Enhanced keyword detection for professional traders
+    MACRO_KEYWORDS = [
+        # Central Banks & Policy
+        'Fed', 'ECB', 'BoE', 'BoJ', 'FOMC', 'Powell', 'Lagarde',
+        'interest rate', 'rate cut', 'rate hike', 'QE', 'quantitative',
+        'monetary policy', 'dovish', 'hawkish',
+        # Economic Indicators
+        'GDP', 'inflation', 'CPI', 'PPI', 'PCE', 'NFP', 'payroll',
+        'unemployment', 'jobless', 'retail sales', 'PMI', 'ISM',
+        'consumer confidence', 'durable goods', 'housing starts',
+        # Fiscal & Economic
+        'recession', 'stimulus', 'fiscal policy', 'treasury',
+        'yield curve', 'bond market'
+    ]
+
+    GEO_KEYWORDS = [
+        # Conflict & Security
+        'war', 'conflict', 'military', 'missile', 'attack', 'invasion',
+        'sanctions', 'embargo', 'blockade',
+        # Political
+        'election', 'impeachment', 'coup', 'protest', 'unrest',
+        'geopolitical', 'tension', 'crisis', 'dispute',
+        # Trade & Relations
+        'trade war', 'tariff', 'trade deal', 'summit', 'treaty',
+        'China', 'Russia', 'Taiwan', 'Middle East', 'Ukraine'
+    ]
+
+    MARKET_KEYWORDS = [
+        # Indices & General
+        'S&P', 'Nasdaq', 'Dow', 'Russell', 'VIX', 'volatility',
+        'rally', 'sell-off', 'correction', 'crash', 'bull', 'bear',
+        # Corporate Events
+        'earnings', 'EPS', 'revenue', 'guidance', 'beat', 'miss',
+        'IPO', 'merger', 'acquisition', 'M&A', 'buyback', 'dividend',
+        # Sectors & Assets
+        'tech stocks', 'banks', 'energy', 'commodities', 'crypto',
+        'Bitcoin', 'oil', 'gold', 'dollar', 'DXY'
+    ]
+
+    # High-impact market-moving keywords
+    BREAKING_KEYWORDS = [
+        'BREAKING', 'ALERT', 'URGENT', 'just in', 'developing',
+        'Fed', 'Powell', 'emergency', 'unexpected', 'surprise'
+    ]
+
+    def __init__(self):
+        self.news_cache = []
+        self.last_fetch = None
+        self.cache_ttl = 180  # 3 minutes for low latency
+
+    @st.cache_data(ttl=180)
+    def scrape_twitter_news(_self, max_tweets: int = 100) -> List[Dict]:
+        """
+        Scrape latest financial news with caching
+        max_tweets: Total tweets to fetch (distributed across sources)
+        """
+        if not SNSCRAPE_AVAILABLE:
+            logger.info("snscrape not available - using mock data")
+            return _self._get_mock_news()
+
+        all_tweets = []
+        tweets_per_source = max(5, max_tweets // len(_self.SOURCES))
+        failed_sources = 0
+
+        for source_name, source_info in _self.SOURCES.items():
+            try:
+                handle = source_info['handle'].replace('@', '')
+                # Optimized query: exclude replies and retweets for signal clarity
+                query = f"from:{handle} -filter:replies -filter:retweets"
+
+                scraped = 0
+                for tweet in sntwitter.TwitterSearchScraper(query).get_items():
+                    if scraped >= tweets_per_source:
+                        break
+
+                    # Skip old tweets (>24h)
+                    if (datetime.now() - tweet.date).days > 1:
+                        continue
+
+                    # Categorize and analyze
+                    category = _self._categorize_tweet(tweet.content, source_info['specialization'])
+                    sentiment = _self._analyze_sentiment(tweet.content)
+                    impact = _self._assess_impact(tweet, source_info['weight'])
+                    is_breaking = _self._detect_breaking_news(tweet.content)
+
+                    all_tweets.append({
+                        'id': tweet.id,
+                        'title': tweet.content,
+                        'summary': _self._extract_summary(tweet.content),
+                        'source': source_name.capitalize(),
+                        'category': category,
+                        'timestamp': tweet.date,
+                        'sentiment': sentiment,
+                        'impact': impact,
+                        'url': tweet.url,
+                        'likes': tweet.likeCount or 0,
+                        'retweets': tweet.retweetCount or 0,
+                        'is_breaking': is_breaking,
+                        'source_weight': source_info['weight']
+                    })
+                    scraped += 1
+
+            except Exception as e:
+                failed_sources += 1
+                error_msg = str(e).lower()
+                if 'blocked' in error_msg or '404' in error_msg:
+                    logger.warning(f"Twitter/X API blocked access for {source_name}")
+                else:
+                    logger.error(f"Error scraping {source_name}: {e}")
+                continue
+
+        # If Twitter/X blocked all sources, fall back to mock data
+        if failed_sources >= len(_self.SOURCES) or len(all_tweets) == 0:
+            logger.warning("Twitter/X API unavailable - falling back to mock data for demonstration")
+            return _self._get_mock_news()
+
+        # Sort by impact and timestamp
+        all_tweets.sort(
+            key=lambda x: (x['is_breaking'], x['impact'] == 'high', x['timestamp']),
+            reverse=True
+        )
+
+        return all_tweets
+
+    def _categorize_tweet(self, text: str, source_specialization: List[str]) -> str:
+        """Advanced categorization with source specialization"""
+        text_lower = text.lower()
+
+        # Calculate weighted scores
+        macro_score = sum(2 if kw.lower() in text_lower else 0
+                         for kw in self.MACRO_KEYWORDS)
+        geo_score = sum(2 if kw.lower() in text_lower else 0
+                       for kw in self.GEO_KEYWORDS)
+        market_score = sum(2 if kw.lower() in text_lower else 0
+                          for kw in self.MARKET_KEYWORDS)
+
+        # Boost scores based on source specialization
+        if 'macro' in source_specialization:
+            macro_score *= 1.5
+        if 'geopolitical' in source_specialization:
+            geo_score *= 1.5
+        if 'markets' in source_specialization:
+            market_score *= 1.5
+
+        scores = {
+            'macro': macro_score,
+            'geopolitical': geo_score,
+            'markets': market_score
+        }
+
+        return max(scores, key=scores.get) if max(scores.values()) > 0 else 'general'
+
+    def _analyze_sentiment(self, text: str) -> str:
+        """Professional sentiment analysis for trading"""
+        positive_words = [
+            'surge', 'rally', 'soar', 'jump', 'gain', 'rise', 'climb',
+            'growth', 'positive', 'strong', 'robust', 'beat', 'exceed',
+            'outperform', 'record high', 'breakthrough', 'optimistic'
+        ]
+        negative_words = [
+            'plunge', 'crash', 'tumble', 'fall', 'drop', 'decline', 'slump',
+            'loss', 'weak', 'fragile', 'crisis', 'concern', 'risk', 'fear',
+            'miss', 'disappoint', 'warning', 'downgrade', 'recession'
+        ]
+
+        text_lower = text.lower()
+        pos_count = sum(2 if word in text_lower else 0 for word in positive_words)
+        neg_count = sum(2 if word in text_lower else 0 for word in negative_words)
+
+        # Threshold for clear signal
+        if pos_count > neg_count + 1:
+            return 'positive'
+        elif neg_count > pos_count + 1:
+            return 'negative'
+        return 'neutral'
+
+    def _assess_impact(self, tweet, source_weight: float) -> str:
+        """Assess market impact based on engagement and source credibility"""
+        engagement = (tweet.likeCount or 0) + (tweet.retweetCount or 0) * 2
+        weighted_engagement = engagement * source_weight
+
+        # Breaking news always high impact
+        if self._detect_breaking_news(tweet.content):
+            return 'high'
+
+        if weighted_engagement > 1500 or source_weight >= 2.0:
+            return 'high'
+        elif weighted_engagement > 300:
+            return 'medium'
+        return 'low'
+
+    def _detect_breaking_news(self, text: str) -> bool:
+        """Detect breaking/urgent news for immediate alerts"""
+        text_upper = text.upper()
+        return any(keyword.upper() in text_upper for keyword in self.BREAKING_KEYWORDS)
+
+    def _extract_summary(self, text: str, max_length: int = 200) -> str:
+        """Extract clean summary for display"""
+        # Remove URLs
+        import re
+        text = re.sub(r'http\S+', '', text)
+        text = text.strip()
+
+        if len(text) <= max_length:
+            return text
+        return text[:max_length] + '...'
+
+    def _get_mock_news(self) -> List[Dict]:
+        """Mock news data when snscrape is unavailable - Showcases all source types"""
+        return [
+            # Tier 3: Central Bank - BREAKING
+            {
+                'id': 1,
+                'title': 'BREAKING: Federal Reserve announces emergency rate cut of 50bps - Powell cites economic uncertainty',
+                'summary': 'BREAKING: Fed emergency rate cut 50bps',
+                'source': 'Federal Reserve',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=5),
+                'sentiment': 'negative',
+                'impact': 'high',
+                'url': 'https://twitter.com/federalreserve',
+                'likes': 5000,
+                'retweets': 2000,
+                'is_breaking': True,
+                'source_weight': 2.0
+            },
+            # Tier 4: Alpha Account - Fast Alert
+            {
+                'id': 2,
+                'title': '*FIRST SQUAWK: S&P 500 FUTURES DROP 2% AFTER FED ANNOUNCEMENT',
+                'summary': '*FIRST SQUAWK: S&P 500 futures drop 2%',
+                'source': 'First Squawk',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=10),
+                'sentiment': 'negative',
+                'impact': 'high',
+                'url': 'https://twitter.com/FirstSquawk',
+                'likes': 1500,
+                'retweets': 600,
+                'is_breaking': False,
+                'source_weight': 1.1
+            },
+            # Tier 1: Bloomberg - Markets
+            {
+                'id': 3,
+                'title': 'Apple reports earnings beat with $123B revenue, raises dividend by 4% - Stock up 3% after hours',
+                'summary': 'Apple beats earnings, raises dividend 4%',
+                'source': 'Bloomberg',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=25),
+                'sentiment': 'positive',
+                'impact': 'high',
+                'url': 'https://twitter.com/business',
+                'likes': 2800,
+                'retweets': 900,
+                'is_breaking': False,
+                'source_weight': 1.5
+            },
+            # Tier 3: ECB President
+            {
+                'id': 4,
+                'title': 'ECB President Lagarde: Inflation remains above target, rates to stay higher for longer',
+                'summary': 'Lagarde: rates to stay higher for longer',
+                'source': 'Lagarde',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=45),
+                'sentiment': 'neutral',
+                'impact': 'high',
+                'url': 'https://twitter.com/Lagarde',
+                'likes': 1200,
+                'retweets': 400,
+                'is_breaking': False,
+                'source_weight': 1.9
+            },
+            # Tier 2: Geopolitical - BBC
+            {
+                'id': 5,
+                'title': 'Ukraine conflict: New peace talks scheduled as tensions ease in Eastern Europe',
+                'summary': 'Ukraine: New peace talks scheduled',
+                'source': 'BBC World',
+                'category': 'geopolitical',
+                'timestamp': datetime.now() - timedelta(hours=1),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://twitter.com/BBCWorld',
+                'likes': 3500,
+                'retweets': 1200,
+                'is_breaking': False,
+                'source_weight': 1.4
+            },
+            # Tier 1: Reuters - Macro
+            {
+                'id': 6,
+                'title': 'US GDP growth revised up to 2.8% in Q4, beating economists expectations of 2.5%',
+                'summary': 'US GDP growth revised up to 2.8% in Q4',
+                'source': 'Reuters',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(hours=2),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://twitter.com/Reuters',
+                'likes': 1800,
+                'retweets': 600,
+                'is_breaking': False,
+                'source_weight': 1.5
+            },
+            # Tier 4: Live Squawk
+            {
+                'id': 7,
+                'title': '*LIVE SQUAWK: Oil prices surge 5% on Middle East supply concerns, Brent crude at $92/barrel',
+                'summary': '*LIVE SQUAWK: Oil surges 5% on supply fears',
+                'source': 'Live Squawk',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(hours=3),
+                'sentiment': 'neutral',
+                'impact': 'medium',
+                'url': 'https://twitter.com/LiveSquawk',
+                'likes': 900,
+                'retweets': 350,
+                'is_breaking': False,
+                'source_weight': 1.1
+            },
+            # Tier 3: IMF
+            {
+                'id': 8,
+                'title': 'IMF upgrades global growth forecast to 3.2% for 2024, warns of recession risks in Europe',
+                'summary': 'IMF upgrades global growth to 3.2%',
+                'source': 'IMF',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(hours=4),
+                'sentiment': 'neutral',
+                'impact': 'medium',
+                'url': 'https://twitter.com/IMFNews',
+                'likes': 800,
+                'retweets': 300,
+                'is_breaking': False,
+                'source_weight': 1.7
+            },
+            # Tier 2: Politico - Geopolitical
+            {
+                'id': 9,
+                'title': 'US-China trade talks resume in Washington, focus on technology transfer and tariffs',
+                'summary': 'US-China trade talks resume',
+                'source': 'Politico',
+                'category': 'geopolitical',
+                'timestamp': datetime.now() - timedelta(hours=5),
+                'sentiment': 'neutral',
+                'impact': 'low',
+                'url': 'https://twitter.com/politico',
+                'likes': 600,
+                'retweets': 200,
+                'is_breaking': False,
+                'source_weight': 1.2
+            },
+            # Tier 1: FT - Markets
+            {
+                'id': 10,
+                'title': 'Bank of America cuts recession probability to 20%, cites resilient consumer spending',
+                'summary': 'BofA cuts recession probability to 20%',
+                'source': 'FT',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(hours=6),
+                'sentiment': 'positive',
+                'impact': 'low',
+                'url': 'https://twitter.com/FT',
+                'likes': 700,
+                'retweets': 250,
+                'is_breaking': False,
+                'source_weight': 1.4
+            }
+        ]
+
+    def get_news(self, category: str = 'all', sentiment: str = 'all',
+                 impact: str = 'all', refresh: bool = False) -> pd.DataFrame:
+        """
+        Get filtered news with intelligent caching
+
+        Args:
+            category: 'all', 'macro', 'geopolitical', 'markets'
+            sentiment: 'all', 'positive', 'negative', 'neutral'
+            impact: 'all', 'high', 'medium', 'low'
+            refresh: Force refresh cache
+        """
+        # Check cache freshness
+        if refresh or not self.last_fetch or \
+           (datetime.now() - self.last_fetch).seconds > self.cache_ttl:
+            self.news_cache = self.scrape_twitter_news(max_tweets=100)
+            self.last_fetch = datetime.now()
+
+        news = self.news_cache.copy()
+
+        # Apply filters
+        if category != 'all':
+            news = [n for n in news if n['category'] == category]
+
+        if sentiment != 'all':
+            news = [n for n in news if n['sentiment'] == sentiment]
+
+        if impact != 'all':
+            news = [n for n in news if n['impact'] == impact]
+
+        df = pd.DataFrame(news)
+        if not df.empty:
+            df['timestamp'] = pd.to_datetime(df['timestamp'])
+
+        return df
+
+    def get_breaking_news(self) -> pd.DataFrame:
+        """Get only breaking/high-impact news for alerts"""
+        df = self.get_news()
+        if not df.empty:
+            return df[df['is_breaking'] == True].head(10)
+        return df
+
+    def get_statistics(self) -> Dict:
+        """Get news feed statistics"""
+        if not self.news_cache:
+            return {
+                'total': 0,
+                'high_impact': 0,
+                'breaking': 0,
+                'last_update': 'Never'
+            }
+
+        return {
+            'total': len(self.news_cache),
+            'high_impact': len([n for n in self.news_cache if n['impact'] == 'high']),
+            'breaking': len([n for n in self.news_cache if n['is_breaking']]),
+            'last_update': self.last_fetch.strftime('%H:%M:%S') if self.last_fetch else 'Never',
+            'by_category': {
+                'macro': len([n for n in self.news_cache if n['category'] == 'macro']),
+                'geopolitical': len([n for n in self.news_cache if n['category'] == 'geopolitical']),
+                'markets': len([n for n in self.news_cache if n['category'] == 'markets'])
+            }
+        }

app/services/news_monitor_twikit.py

@@ -0,0 +1,613 @@
+"""
+Professional Finance News Monitor using Twikit
+Real-time tracking: Macro, Markets, Geopolitical intelligence
+Optimized for low-latency trading decisions
+"""
+
+import pandas as pd
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import streamlit as st
+import os
+import asyncio
+import re
+import logging
+from dotenv import load_dotenv
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+try:
+    from twikit import Client
+    TWIKIT_AVAILABLE = True
+except ImportError:
+    TWIKIT_AVAILABLE = False
+    logger.warning("twikit not available. Install with: pip install twikit")
+
+
+class FinanceNewsMonitor:
+    """
+    Professional-grade financial news aggregator using Twikit
+    Sources: Bloomberg, Reuters, WSJ, FT, CNBC, and 18 more premium sources
+    """
+
+    # Premium financial sources - expanded coverage
+    SOURCES = {
+        # ===== TIER 1: Major Financial News =====
+        'reuters': {
+            'handle': 'Reuters',
+            'weight': 1.5,
+            'specialization': ['macro', 'geopolitical', 'markets']
+        },
+        'bloomberg': {
+            'handle': 'business',
+            'weight': 1.5,
+            'specialization': ['macro', 'markets']
+        },
+        'ft': {
+            'handle': 'FT',
+            'weight': 1.4,
+            'specialization': ['macro', 'markets']
+        },
+        'economist': {
+            'handle': 'TheEconomist',
+            'weight': 1.3,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'wsj': {
+            'handle': 'WSJ',
+            'weight': 1.4,
+            'specialization': ['markets', 'macro']
+        },
+        'bloomberg_terminal': {
+            'handle': 'Bloomberg',
+            'weight': 1.5,
+            'specialization': ['macro', 'markets']
+        },
+        'cnbc': {
+            'handle': 'CNBC',
+            'weight': 1.2,
+            'specialization': ['markets']
+        },
+        'marketwatch': {
+            'handle': 'MarketWatch',
+            'weight': 1.1,
+            'specialization': ['markets']
+        },
+
+        # ===== TIER 2: Geopolitical Intelligence =====
+        'bbc_world': {
+            'handle': 'BBCWorld',
+            'weight': 1.4,
+            'specialization': ['geopolitical']
+        },
+        'afp': {
+            'handle': 'AFP',
+            'weight': 1.3,
+            'specialization': ['geopolitical']
+        },
+        'aljazeera': {
+            'handle': 'AlJazeera',
+            'weight': 1.2,
+            'specialization': ['geopolitical']
+        },
+        'politico': {
+            'handle': 'politico',
+            'weight': 1.2,
+            'specialization': ['geopolitical', 'macro']
+        },
+        'dw_news': {
+            'handle': 'dwnews',
+            'weight': 1.2,
+            'specialization': ['geopolitical']
+        },
+
+        # ===== TIER 3: Central Banks & Official Sources =====
+        'federal_reserve': {
+            'handle': 'federalreserve',
+            'weight': 2.0,  # Highest priority
+            'specialization': ['macro']
+        },
+        'ecb': {
+            'handle': 'ecb',
+            'weight': 2.0,
+            'specialization': ['macro']
+        },
+        'lagarde': {
+            'handle': 'Lagarde',
+            'weight': 1.9,  # ECB President
+            'specialization': ['macro']
+        },
+        'bank_of_england': {
+            'handle': 'bankofengland',
+            'weight': 1.8,
+            'specialization': ['macro']
+        },
+        'imf': {
+            'handle': 'IMFNews',
+            'weight': 1.7,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'world_bank': {
+            'handle': 'worldbank',
+            'weight': 1.6,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'us_treasury': {
+            'handle': 'USTreasury',
+            'weight': 1.8,
+            'specialization': ['macro']
+        },
+
+        # ===== TIER 4: Alpha Accounts (Fast Breaking News) =====
+        'zerohedge': {
+            'handle': 'zerohedge',
+            'weight': 1.0,
+            'specialization': ['markets', 'macro']
+        },
+        'first_squawk': {
+            'handle': 'FirstSquawk',
+            'weight': 1.1,  # Fast alerts
+            'specialization': ['markets', 'macro']
+        },
+        'live_squawk': {
+            'handle': 'LiveSquawk',
+            'weight': 1.1,  # Real-time market squawks
+            'specialization': ['markets', 'macro']
+        }
+    }
+
+    # Enhanced keyword detection for professional traders
+    MACRO_KEYWORDS = [
+        # Central Banks & Policy
+        'Fed', 'ECB', 'BoE', 'BoJ', 'FOMC', 'Powell', 'Lagarde',
+        'interest rate', 'rate cut', 'rate hike', 'QE', 'quantitative',
+        'monetary policy', 'inflation', 'CPI', 'PCE', 'tapering',
+        # Economic Data
+        'GDP', 'unemployment', 'jobs report', 'NFP', 'payroll',
+        'PMI', 'manufacturing', 'services', 'consumer confidence',
+        'retail sales', 'housing starts', 'durable goods'
+    ]
+
+    MARKET_KEYWORDS = [
+        # Equities
+        'S&P', 'Dow', 'Nasdaq', 'Russell', 'earnings', 'EPS',
+        'stock', 'share', 'equity', 'rally', 'selloff', 'correction',
+        # Corporate
+        'merger', 'acquisition', 'IPO', 'buyback', 'dividend',
+        'guidance', 'revenue', 'profit', 'loss', 'bankruptcy'
+    ]
+
+    GEOPOLITICAL_KEYWORDS = [
+        # Conflicts & Relations
+        'war', 'conflict', 'sanctions', 'trade', 'tariff', 'embargo',
+        'summit', 'treaty', 'diplomacy', 'tension', 'crisis',
+        # Regions
+        'Ukraine', 'Russia', 'China', 'Taiwan', 'Middle East',
+        'Iran', 'North Korea', 'EU', 'Brexit'
+    ]
+
+    def __init__(self):
+        """Initialize monitor with caching"""
+        self.news_cache = []
+        self.last_fetch = None
+        self.cache_ttl = 180  # 3 minutes for low latency
+        self.client = None
+        self.authenticated = False
+
+    async def _authenticate_twikit(self):
+        """Authenticate with Twitter using Twikit"""
+        if not TWIKIT_AVAILABLE:
+            return False
+
+        try:
+            self.client = Client('en-US')
+
+            # Get credentials from environment variables
+            username = os.getenv('TWITTER_USERNAME')
+            email = os.getenv('TWITTER_EMAIL')
+            password = os.getenv('TWITTER_PASSWORD')
+
+            if not all([username, email, password]):
+                logger.warning("Twitter credentials not found in environment variables")
+                logger.info("Set TWITTER_USERNAME, TWITTER_EMAIL, TWITTER_PASSWORD in .env")
+                return False
+
+            await self.client.login(
+                auth_info_1=username,
+                auth_info_2=email,
+                password=password
+            )
+
+            self.authenticated = True
+            logger.info("Successfully authenticated with Twitter/X")
+            return True
+
+        except Exception as e:
+            logger.error(f"Twitter authentication failed: {e}")
+            return False
+
+    async def _scrape_twitter_async(self, max_tweets: int = 100) -> List[Dict]:
+        """Async method to scrape tweets using Twikit"""
+        if not self.authenticated:
+            auth_success = await self._authenticate_twikit()
+            if not auth_success:
+                return self._get_mock_news()
+
+        all_tweets = []
+        tweets_per_source = max(5, max_tweets // len(self.SOURCES))
+        failed_sources = 0
+
+        for source_name, source_info in self.SOURCES.items():
+            try:
+                handle = source_info['handle']
+
+                # Search for tweets from this user
+                tweets = await self.client.search_tweet(
+                    f'from:{handle}',
+                    product='Latest',
+                    count=tweets_per_source
+                )
+
+                for tweet in tweets:
+                    # Skip old tweets (>24h)
+                    tweet_date = datetime.fromisoformat(tweet.created_at.replace('Z', '+00:00'))
+                    if (datetime.now(tweet_date.tzinfo) - tweet_date).days > 1:
+                        continue
+
+                    # Skip retweets and replies
+                    if hasattr(tweet, 'retweeted_tweet') or tweet.in_reply_to_user_id:
+                        continue
+
+                    # Categorize and analyze
+                    category = self._categorize_tweet(tweet.text, source_info['specialization'])
+                    sentiment = self._analyze_sentiment(tweet.text)
+                    impact = self._assess_impact_twikit(tweet, source_info['weight'])
+                    is_breaking = self._detect_breaking_news(tweet.text)
+
+                    all_tweets.append({
+                        'id': int(tweet.id),
+                        'title': tweet.text,
+                        'summary': self._extract_summary(tweet.text),
+                        'source': source_name.replace('_', ' ').title(),
+                        'category': category,
+                        'timestamp': tweet_date.replace(tzinfo=None),
+                        'sentiment': sentiment,
+                        'impact': impact,
+                        'url': f'https://twitter.com/{handle}/status/{tweet.id}',
+                        'likes': tweet.favorite_count or 0,
+                        'retweets': tweet.retweet_count or 0,
+                        'is_breaking': is_breaking,
+                        'source_weight': source_info['weight']
+                    })
+
+            except Exception as e:
+                failed_sources += 1
+                error_msg = str(e).lower()
+                if 'rate limit' in error_msg:
+                    logger.warning(f"Rate limited for {source_name}")
+                elif 'unauthorized' in error_msg or 'forbidden' in error_msg:
+                    logger.warning(f"Access denied for {source_name}")
+                else:
+                    logger.error(f"Error scraping {source_name}: {e}")
+                continue
+
+        # If all sources failed, fall back to mock data
+        if failed_sources >= len(self.SOURCES) or len(all_tweets) == 0:
+            logger.warning("Twitter/X scraping failed - falling back to mock data")
+            return self._get_mock_news()
+
+        # Sort by impact and timestamp
+        all_tweets.sort(
+            key=lambda x: (x['is_breaking'], x['impact'] == 'high', x['timestamp']),
+            reverse=True
+        )
+
+        return all_tweets
+
+    @st.cache_data(ttl=180)
+    def scrape_twitter_news(_self, max_tweets: int = 100) -> List[Dict]:
+        """
+        Scrape latest financial news with caching (sync wrapper)
+        max_tweets: Total tweets to fetch (distributed across sources)
+        """
+        if not TWIKIT_AVAILABLE:
+            logger.info("Twikit not available - using mock data")
+            return _self._get_mock_news()
+
+        try:
+            # Run async scraping in event loop
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            result = loop.run_until_complete(_self._scrape_twitter_async(max_tweets))
+            loop.close()
+            return result
+        except Exception as e:
+            logger.error(f"Error in async scraping: {e}")
+            return _self._get_mock_news()
+
+    def _categorize_tweet(self, text: str, source_specialization: List[str]) -> str:
+        """Advanced categorization with source specialization"""
+        text_lower = text.lower()
+
+        # Count keyword matches
+        macro_score = sum(1 for kw in self.MACRO_KEYWORDS if kw.lower() in text_lower)
+        market_score = sum(1 for kw in self.MARKET_KEYWORDS if kw.lower() in text_lower)
+        geo_score = sum(1 for kw in self.GEOPOLITICAL_KEYWORDS if kw.lower() in text_lower)
+
+        # Weight by source specialization
+        if 'macro' in source_specialization:
+            macro_score *= 1.5
+        if 'markets' in source_specialization:
+            market_score *= 1.5
+        if 'geopolitical' in source_specialization:
+            geo_score *= 1.5
+
+        # Return highest scoring category
+        scores = {'macro': macro_score, 'markets': market_score, 'geopolitical': geo_score}
+        return max(scores, key=scores.get) if max(scores.values()) > 0 else 'markets'
+
+    def _analyze_sentiment(self, text: str) -> str:
+        """Professional sentiment analysis for traders"""
+        text_lower = text.lower()
+
+        positive_signals = ['surge', 'soar', 'rally', 'beat', 'upgrade', 'bullish',
+                           'gain', 'rise', 'jump', 'boost', 'optimistic', 'positive']
+        negative_signals = ['plunge', 'crash', 'fall', 'miss', 'downgrade', 'bearish',
+                           'loss', 'drop', 'slide', 'concern', 'worry', 'negative']
+
+        pos_count = sum(1 for signal in positive_signals if signal in text_lower)
+        neg_count = sum(1 for signal in negative_signals if signal in text_lower)
+
+        if pos_count > neg_count:
+            return 'positive'
+        elif neg_count > pos_count:
+            return 'negative'
+        return 'neutral'
+
+    def _assess_impact_twikit(self, tweet, source_weight: float) -> str:
+        """Assess market impact using Twikit tweet object"""
+        engagement = (tweet.favorite_count or 0) + (tweet.retweet_count or 0) * 2
+        weighted_engagement = engagement * source_weight
+
+        if weighted_engagement > 5000 or source_weight >= 1.8:
+            return 'high'
+        elif weighted_engagement > 1000:
+            return 'medium'
+        return 'low'
+
+    def _detect_breaking_news(self, text: str) -> bool:
+        """Detect breaking/urgent news"""
+        text_upper = text.upper()
+        breaking_signals = ['BREAKING', 'ALERT', 'URGENT', 'JUST IN',
+                           '*FED', '*ECB', '*POWELL', '*LAGARDE']
+        return any(signal in text_upper for signal in breaking_signals)
+
+    def _extract_summary(self, text: str, max_length: int = 150) -> str:
+        """Extract clean summary from tweet"""
+        # Remove URLs
+        text = re.sub(r'http\S+', '', text)
+        text = text.strip()
+
+        if len(text) <= max_length:
+            return text
+        return text[:max_length] + '...'
+
+    def _get_mock_news(self) -> List[Dict]:
+        """Mock news data when Twikit is unavailable"""
+        return [
+            {
+                'id': 1,
+                'title': 'BREAKING: Federal Reserve announces emergency rate cut of 50bps - Powell cites economic uncertainty',
+                'summary': 'BREAKING: Fed emergency rate cut 50bps',
+                'source': 'Federal Reserve',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=5),
+                'sentiment': 'negative',
+                'impact': 'high',
+                'url': 'https://twitter.com/federalreserve',
+                'likes': 5000,
+                'retweets': 2000,
+                'is_breaking': True,
+                'source_weight': 2.0
+            },
+            {
+                'id': 2,
+                'title': '*FIRST SQUAWK: S&P 500 FUTURES DROP 2% AFTER FED ANNOUNCEMENT',
+                'summary': '*FIRST SQUAWK: S&P 500 futures drop 2%',
+                'source': 'First Squawk',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=10),
+                'sentiment': 'negative',
+                'impact': 'high',
+                'url': 'https://twitter.com/FirstSquawk',
+                'likes': 1500,
+                'retweets': 600,
+                'is_breaking': False,
+                'source_weight': 1.1
+            },
+            {
+                'id': 3,
+                'title': 'Apple reports earnings beat with $123B revenue, raises dividend by 4% - Stock up 3% after hours',
+                'summary': 'Apple beats earnings, raises dividend 4%',
+                'source': 'Bloomberg',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=25),
+                'sentiment': 'positive',
+                'impact': 'high',
+                'url': 'https://twitter.com/business',
+                'likes': 2800,
+                'retweets': 900,
+                'is_breaking': False,
+                'source_weight': 1.5
+            },
+            {
+                'id': 4,
+                'title': 'ECB President Lagarde: Inflation remains above target, rates to stay higher for longer',
+                'summary': 'Lagarde: rates to stay higher for longer',
+                'source': 'Lagarde',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=45),
+                'sentiment': 'neutral',
+                'impact': 'high',
+                'url': 'https://twitter.com/Lagarde',
+                'likes': 1200,
+                'retweets': 400,
+                'is_breaking': False,
+                'source_weight': 1.9
+            },
+            {
+                'id': 5,
+                'title': 'Ukraine conflict: New peace talks scheduled as tensions ease in Eastern Europe',
+                'summary': 'Ukraine: New peace talks scheduled',
+                'source': 'BBC World',
+                'category': 'geopolitical',
+                'timestamp': datetime.now() - timedelta(hours=1),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://twitter.com/BBCWorld',
+                'likes': 3500,
+                'retweets': 1200,
+                'is_breaking': False,
+                'source_weight': 1.4
+            },
+            {
+                'id': 6,
+                'title': 'US GDP growth revised up to 2.8% in Q4, beating economists expectations of 2.5%',
+                'summary': 'US GDP growth revised up to 2.8% in Q4',
+                'source': 'Reuters',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(hours=2),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://twitter.com/Reuters',
+                'likes': 1800,
+                'retweets': 600,
+                'is_breaking': False,
+                'source_weight': 1.5
+            },
+            {
+                'id': 7,
+                'title': '*LIVE SQUAWK: Oil prices surge 5% on Middle East supply concerns, Brent crude at $92/barrel',
+                'summary': '*LIVE SQUAWK: Oil surges 5% on supply fears',
+                'source': 'Live Squawk',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(hours=3),
+                'sentiment': 'neutral',
+                'impact': 'medium',
+                'url': 'https://twitter.com/LiveSquawk',
+                'likes': 900,
+                'retweets': 350,
+                'is_breaking': False,
+                'source_weight': 1.1
+            },
+            {
+                'id': 8,
+                'title': 'IMF upgrades global growth forecast to 3.2% for 2024, warns of recession risks in Europe',
+                'summary': 'IMF upgrades global growth to 3.2%',
+                'source': 'IMF',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(hours=4),
+                'sentiment': 'neutral',
+                'impact': 'medium',
+                'url': 'https://twitter.com/IMFNews',
+                'likes': 800,
+                'retweets': 300,
+                'is_breaking': False,
+                'source_weight': 1.7
+            },
+            {
+                'id': 9,
+                'title': 'US-China trade talks resume in Washington, focus on technology transfer and tariffs',
+                'summary': 'US-China trade talks resume',
+                'source': 'Politico',
+                'category': 'geopolitical',
+                'timestamp': datetime.now() - timedelta(hours=5),
+                'sentiment': 'neutral',
+                'impact': 'low',
+                'url': 'https://twitter.com/politico',
+                'likes': 600,
+                'retweets': 200,
+                'is_breaking': False,
+                'source_weight': 1.2
+            },
+            {
+                'id': 10,
+                'title': 'Bank of America cuts recession probability to 20%, cites resilient consumer spending',
+                'summary': 'BofA cuts recession probability to 20%',
+                'source': 'FT',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(hours=6),
+                'sentiment': 'positive',
+                'impact': 'low',
+                'url': 'https://twitter.com/FT',
+                'likes': 700,
+                'retweets': 250,
+                'is_breaking': False,
+                'source_weight': 1.4
+            }
+        ]
+
+    def get_news(self, category: str = 'all', sentiment: str = 'all',
+                 impact: str = 'all', refresh: bool = False) -> pd.DataFrame:
+        """
+        Get filtered news with intelligent caching
+
+        Args:
+            category: 'all', 'macro', 'geopolitical', 'markets'
+            sentiment: 'all', 'positive', 'negative', 'neutral'
+            impact: 'all', 'high', 'medium', 'low'
+            refresh: Force refresh cache
+        """
+        # Check cache freshness
+        if refresh or not self.last_fetch or \
+           (datetime.now() - self.last_fetch).seconds > self.cache_ttl:
+            self.news_cache = self.scrape_twitter_news(max_tweets=100)
+            self.last_fetch = datetime.now()
+
+        news = self.news_cache.copy()
+
+        # Apply filters
+        if category != 'all':
+            news = [n for n in news if n['category'] == category]
+
+        if sentiment != 'all':
+            news = [n for n in news if n['sentiment'] == sentiment]
+
+        if impact != 'all':
+            news = [n for n in news if n['impact'] == impact]
+
+        df = pd.DataFrame(news)
+        if not df.empty:
+            df['timestamp'] = pd.to_datetime(df['timestamp'])
+
+        return df
+
+    def get_breaking_news(self) -> pd.DataFrame:
+        """Get only breaking/high-impact news for alerts"""
+        return self.get_news(impact='high')
+
+    def get_statistics(self) -> Dict:
+        """Get feed statistics"""
+        if not self.news_cache:
+            return {
+                'total': 0,
+                'high_impact': 0,
+                'breaking': 0,
+                'last_update': 'Never',
+                'by_category': {}
+            }
+
+        df = pd.DataFrame(self.news_cache)
+
+        return {
+            'total': len(df),
+            'high_impact': len(df[df['impact'] == 'high']),
+            'breaking': len(df[df['is_breaking'] == True]),
+            'last_update': self.last_fetch.strftime('%H:%M:%S') if self.last_fetch else 'Never',
+            'by_category': df['category'].value_counts().to_dict()
+        }

app/services/news_scraper.py

@@ -0,0 +1,565 @@
+"""
+Professional Finance News Scraper - Direct from Source Websites
+Scrapes: Reuters, Bloomberg, FT, WSJ, CNBC, MarketWatch, etc.
+No Twitter API needed - direct RSS and web scraping
+"""
+
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+import pandas as pd
+import feedparser
+import streamlit as st
+from bs4 import BeautifulSoup
+
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class FinanceNewsScraper:
+    """
+    Professional-grade financial news scraper using RSS feeds and web scraping
+    No authentication required - publicly available sources
+    """
+
+    # News sources with RSS feeds and web scraping endpoints
+    # web=None means web scraping is disabled (blocked by anti-bot measures)
+    SOURCES = {
+        # ===== TIER 1: Major Financial News =====
+        'cnbc': {
+            'name': 'CNBC',
+            'rss': 'https://www.cnbc.com/id/100003114/device/rss/rss.html',
+            'web': 'https://www.cnbc.com/world/',
+            'selectors': {'headline': 'a.Card-title', 'link': 'a.Card-title'},
+            'weight': 1.2,
+            'web_priority': True,  # Web scraping is higher priority
+            'specialization': ['markets']
+        },
+        'wsj_markets': {
+            'name': 'WSJ Markets',
+            'rss': 'https://feeds.a.dj.com/rss/RSSMarketsMain.xml',
+            'web': None,  # Blocked by paywall
+            'weight': 1.4,
+            'specialization': ['markets']
+        },
+        'bloomberg_markets': {
+            'name': 'Bloomberg',
+            'rss': 'https://feeds.bloomberg.com/markets/news.rss',
+            'web': None,  # Blocked by Cloudflare
+            'weight': 1.5,
+            'specialization': ['markets']
+        },
+        'ft_markets': {
+            'name': 'Financial Times',
+            'rss': 'https://www.ft.com/markets?format=rss',
+            'web': 'https://www.ft.com/markets',
+            'selectors': {'headline': 'div.o-teaser__heading', 'link': 'a.js-teaser-heading-link'},
+            'weight': 1.4,
+            'web_priority': True,
+            'specialization': ['markets']
+        },
+        'economist': {
+            'name': 'The Economist',
+            'rss': 'https://www.economist.com/finance-and-economics/rss.xml',
+            'web': None,  # Blocked by anti-bot
+            'weight': 1.3,
+            'specialization': ['macro', 'geopolitical']
+        },
+
+        # ===== TIER 2: Geopolitical & Economic =====
+        'bbc_business': {
+            'name': 'BBC Business',
+            'rss': 'http://feeds.bbci.co.uk/news/business/rss.xml',
+            'web': 'https://www.bbc.com/news/business',
+            'selectors': {'headline': 'h2[data-testid="card-headline"]', 'link': 'a[data-testid="internal-link"]'},
+            'weight': 1.4,
+            'web_priority': True,
+            'specialization': ['geopolitical', 'macro']
+        },
+        'yahoo_finance': {
+            'name': 'Yahoo Finance',
+            'rss': 'https://finance.yahoo.com/news/rssindex',
+            'web': 'https://finance.yahoo.com/',
+            'selectors': {'headline': 'h3.clamp', 'link': 'a'},
+            'weight': 1.3,
+            'web_priority': True,
+            'specialization': ['markets', 'macro']
+        },
+        'google_news_finance': {
+            'name': 'Google News Finance',
+            'rss': 'https://news.google.com/rss/search?q=finance+OR+stocks+OR+markets+OR+economy&hl=en-US&gl=US&ceid=US:en',
+            'web': None,  # RSS only
+            'weight': 1.2,
+            'specialization': ['markets', 'macro', 'geopolitical']
+        },
+        'google_news_business': {
+            'name': 'Google News Business',
+            'rss': 'https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx6TVdZU0FtVnVHZ0pWVXlnQVAB',
+            'web': None,  # RSS only
+            'weight': 1.2,
+            'specialization': ['markets', 'macro']
+        },
+
+        # ===== TIER 3: Central Banks & Institutions =====
+        'federal_reserve': {
+            'name': 'Federal Reserve',
+            'rss': 'https://www.federalreserve.gov/feeds/press_all.xml',
+            'web': None,  # Disabled - RSS works well
+            'weight': 2.0,
+            'specialization': ['macro']
+        },
+        'ecb': {
+            'name': 'European Central Bank',
+            'rss': 'https://www.ecb.europa.eu/rss/press.xml',
+            'web': None,  # Disabled - RSS works well
+            'weight': 2.0,
+            'specialization': ['macro']
+        },
+        'imf': {
+            'name': 'IMF',
+            'rss': 'https://www.imf.org/en/news/rss',
+            'web': None,  # Timeout issues
+            'weight': 1.7,
+            'specialization': ['macro', 'geopolitical']
+        }
+    }
+
+    # Keyword detection
+    MACRO_KEYWORDS = [
+        'Fed', 'ECB', 'BoE', 'BoJ', 'FOMC', 'Powell', 'Lagarde',
+        'interest rate', 'rate cut', 'rate hike', 'inflation', 'CPI',
+        'GDP', 'unemployment', 'jobs report', 'NFP', 'monetary policy'
+    ]
+
+    MARKET_KEYWORDS = [
+        'S&P', 'Dow', 'Nasdaq', 'earnings', 'EPS', 'stock', 'equity',
+        'rally', 'selloff', 'correction', 'merger', 'acquisition', 'IPO'
+    ]
+
+    GEOPOLITICAL_KEYWORDS = [
+        'war', 'conflict', 'sanctions', 'trade', 'tariff', 'crisis',
+        'Ukraine', 'Russia', 'China', 'Taiwan', 'Middle East'
+    ]
+
+    def __init__(self):
+        """Initialize scraper"""
+        self.session = requests.Session()
+        # Enhanced headers to avoid bot detection
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1'
+        })
+
+    def _fetch_rss_feed(self, source_name: str, source_info: Dict) -> List[Dict]:
+        """Fetch and parse RSS feed from a single source"""
+        try:
+            feed = feedparser.parse(source_info['rss'])
+
+            if not feed.entries:
+                logger.warning(f"No entries found for {source_name}")
+                return []
+
+            news_items = []
+            for entry in feed.entries[:10]:  # Limit to 10 most recent
+                # Parse published date
+                try:
+                    if hasattr(entry, 'published_parsed') and entry.published_parsed:
+                        timestamp = datetime(*entry.published_parsed[:6])
+                    elif hasattr(entry, 'updated_parsed') and entry.updated_parsed:
+                        timestamp = datetime(*entry.updated_parsed[:6])
+                    else:
+                        timestamp = datetime.now()
+                except:
+                    timestamp = datetime.now()
+
+                # Skip old news (>24h)
+                if (datetime.now() - timestamp).days > 1:
+                    continue
+
+                # Extract title and summary
+                title = entry.get('title', '')
+                summary = entry.get('summary', '') or entry.get('description', '')
+
+                # Clean HTML from summary
+                if summary:
+                    summary = BeautifulSoup(summary, 'html.parser').get_text()
+                    summary = self._extract_summary(summary)
+
+                # Get URL
+                url = entry.get('link', '')
+
+                # Categorize and analyze
+                text = f"{title} {summary}"
+                category = self._categorize_text(text, source_info['specialization'])
+                sentiment = self._analyze_sentiment(text)
+                impact = self._assess_impact(source_info['weight'], title)
+                is_breaking = self._detect_breaking_news(title)
+
+                news_items.append({
+                    'id': hash(url),
+                    'title': title,
+                    'summary': summary or self._extract_summary(title),
+                    'source': source_info['name'],
+                    'category': category,
+                    'timestamp': timestamp,
+                    'sentiment': sentiment,
+                    'impact': impact,
+                    'url': url,
+                    'likes': 0,  # RSS feeds don't have engagement metrics
+                    'retweets': 0,
+                    'is_breaking': is_breaking,
+                    'source_weight': source_info['weight'],
+                    'from_web': False  # Mark as RSS feed
+                })
+
+            return news_items
+
+        except Exception as e:
+            logger.error(f"Error fetching RSS for {source_name}: {e}")
+            return []
+
+    def _scrape_web_page(self, source_name: str, source_info: Dict) -> List[Dict]:
+        """Scrape news headlines directly from website main page"""
+        try:
+            # Fetch HTML from web URL
+            response = self.session.get(source_info['web'], timeout=10)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.content, 'lxml')
+
+            # Get CSS selectors
+            headline_selector = source_info['selectors']['headline']
+            link_selector = source_info['selectors']['link']
+
+            news_items = []
+
+            # Find all headline elements
+            headlines = soup.select(headline_selector)
+
+            for headline_elem in headlines[:10]:  # Limit to 10 most recent
+                try:
+                    # Extract title text - clean all HTML tags
+                    title = headline_elem.get_text(separator=' ', strip=True)
+                    # Remove extra whitespace
+                    title = re.sub(r'\s+', ' ', title)
+                    # Remove any HTML tags that might have been missed
+                    title = re.sub(r'<[^>]+>', '', title)
+                    # Clean up HTML entities
+                    from html import unescape
+                    title = unescape(title)
+
+                    if not title or len(title) < 10:
+                        continue
+
+                    # Skip if title looks like it contains HTML comments or code
+                    if any(marker in title for marker in ['<!--', '-->', 'style=', '<div', '</div>', '<span', '</span>', 'justify-content', 'flex:', 'padding:']):
+                        logger.warning(f"Skipping malformed title from {source_name} (contains HTML): {title[:100]}...")
+                        continue
+
+                    # Skip if title is suspiciously long (likely scraped wrong element)
+                    if len(title) > 500:
+                        logger.warning(f"Skipping suspiciously long title from {source_name}: {len(title)} chars")
+                        continue
+
+                    # Find associated link
+                    # Try to find link within the headline element or its parent
+                    link_elem = headline_elem if headline_elem.name == 'a' else headline_elem.find('a')
+                    if not link_elem:
+                        # Try parent element
+                        link_elem = headline_elem.find_parent('a')
+                    if not link_elem:
+                        # Try sibling link with same selector
+                        parent = headline_elem.find_parent()
+                        if parent:
+                            link_elem = parent.find('a')
+
+                    if not link_elem:
+                        continue
+
+                    # Get URL and make absolute if relative
+                    url = link_elem.get('href', '')
+                    if not url:
+                        continue
+
+                    if url.startswith('/'):
+                        # Make absolute URL
+                        from urllib.parse import urljoin
+                        url = urljoin(source_info['web'], url)
+
+                    # Skip non-http URLs
+                    if not url.startswith('http'):
+                        continue
+
+                    # Clean title from any remaining artifacts
+                    title = title.replace('\n', ' ').replace('\r', ' ').strip()
+
+                    # Categorize and analyze
+                    category = self._categorize_text(title, source_info['specialization'])
+                    sentiment = self._analyze_sentiment(title)
+                    impact = self._assess_impact(source_info['weight'], title)
+                    is_breaking = self._detect_breaking_news(title)
+
+                    # Create clean summary
+                    summary = self._extract_summary(title) if len(title) > 150 else title
+
+                    news_items.append({
+                        'id': hash(url),
+                        'title': title,
+                        'summary': summary,
+                        'source': source_info['name'],
+                        'category': category,
+                        'timestamp': datetime.now(),  # Web scraping doesn't have timestamps
+                        'sentiment': sentiment,
+                        'impact': impact,
+                        'url': url,
+                        'likes': 0,
+                        'retweets': 0,
+                        'is_breaking': is_breaking,
+                        'source_weight': source_info['weight'],
+                        'from_web': True  # Mark as web-scraped (main page news)
+                    })
+
+                except Exception as e:
+                    logger.debug(f"Error parsing headline from {source_name}: {e}")
+                    continue
+
+            logger.info(f"Scraped {len(news_items)} items from {source_name} web page")
+            return news_items
+
+        except Exception as e:
+            logger.error(f"Error scraping web page for {source_name}: {e}")
+            return []
+
+    def scrape_news(self, max_items: int = 100) -> List[Dict]:
+        """
+        Scrape news from all sources with caching
+        Uses ThreadPoolExecutor for parallel fetching from both RSS and web pages
+        """
+        all_news = []
+        seen_urls = set()
+
+        # Parallel fetching using ThreadPoolExecutor
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            futures = []
+
+            # Submit both RSS and web scraping tasks for each source
+            for name, info in self.SOURCES.items():
+                # RSS feed task
+                futures.append((executor.submit(self._fetch_rss_feed, name, info), name, 'RSS'))
+                # Web scraping task (only if web URL is configured)
+                if info.get('web'):
+                    futures.append((executor.submit(self._scrape_web_page, name, info), name, 'Web'))
+
+            for future, source_name, method in futures:
+                try:
+                    news_items = future.result()
+
+                    # Deduplicate based on URL
+                    unique_items = []
+                    for item in news_items:
+                        if item['url'] not in seen_urls:
+                            seen_urls.add(item['url'])
+                            unique_items.append(item)
+
+                    all_news.extend(unique_items)
+                    if len(unique_items) > 0:
+                        logger.info(f"Fetched {len(unique_items)} unique items from {source_name} ({method})")
+                except Exception as e:
+                    logger.error(f"Error processing {source_name} ({method}): {e}")
+
+        # If no news was fetched, use mock data
+        if not all_news:
+            logger.warning("No news fetched from any source - using mock data")
+            return self._get_mock_news()
+
+        # Sort by: web-scraped first, then breaking news, then impact, then timestamp
+        all_news.sort(
+            key=lambda x: (x.get('from_web', False), x['is_breaking'], x['impact'] == 'high', x['timestamp']),
+            reverse=True
+        )
+
+        logger.info(f"Total unique news items: {len(all_news)} (Web: {sum(1 for n in all_news if n.get('from_web'))}, RSS: {sum(1 for n in all_news if not n.get('from_web'))})")
+        return all_news[:max_items]
+
+    def get_main_page_news(self) -> pd.DataFrame:
+        """Get only news from main pages (web-scraped)"""
+        if not self.news_cache:
+            self.news_cache = self.scrape_news(max_items=100)
+            self.last_fetch = datetime.now()
+
+        main_news = [n for n in self.news_cache if n.get('from_web', False)]
+        df = pd.DataFrame(main_news)
+        if not df.empty:
+            df['timestamp'] = pd.to_datetime(df['timestamp'])
+        return df
+
+    def _categorize_text(self, text: str, source_specialization: List[str]) -> str:
+        """Categorize news based on keywords and source specialization"""
+        text_lower = text.lower()
+
+        # Count keyword matches
+        macro_score = sum(1 for kw in self.MACRO_KEYWORDS if kw.lower() in text_lower)
+        market_score = sum(1 for kw in self.MARKET_KEYWORDS if kw.lower() in text_lower)
+        geo_score = sum(1 for kw in self.GEOPOLITICAL_KEYWORDS if kw.lower() in text_lower)
+
+        # Weight by source specialization
+        if 'macro' in source_specialization:
+            macro_score *= 1.5
+        if 'markets' in source_specialization:
+            market_score *= 1.5
+        if 'geopolitical' in source_specialization:
+            geo_score *= 1.5
+
+        scores = {'macro': macro_score, 'markets': market_score, 'geopolitical': geo_score}
+        return max(scores, key=scores.get) if max(scores.values()) > 0 else 'markets'
+
+    def _analyze_sentiment(self, text: str) -> str:
+        """Analyze sentiment based on keywords"""
+        text_lower = text.lower()
+
+        positive = ['surge', 'soar', 'rally', 'beat', 'upgrade', 'bullish',
+                   'gain', 'rise', 'jump', 'boost', 'positive']
+        negative = ['plunge', 'crash', 'fall', 'miss', 'downgrade', 'bearish',
+                   'loss', 'drop', 'slide', 'concern', 'negative']
+
+        pos_count = sum(1 for word in positive if word in text_lower)
+        neg_count = sum(1 for word in negative if word in text_lower)
+
+        if pos_count > neg_count:
+            return 'positive'
+        elif neg_count > pos_count:
+            return 'negative'
+        return 'neutral'
+
+    def _assess_impact(self, source_weight: float, title: str) -> str:
+        """Assess market impact"""
+        # Central banks and official sources = high impact
+        if source_weight >= 1.7:
+            return 'high'
+
+        # Check for high-impact keywords
+        high_impact_words = ['breaking', 'alert', 'emergency', 'crash', 'surge', 'fed']
+        if any(word in title.lower() for word in high_impact_words):
+            return 'high'
+
+        return 'medium' if source_weight >= 1.3 else 'low'
+
+    def _detect_breaking_news(self, text: str) -> bool:
+        """Detect breaking news"""
+        text_upper = text.upper()
+        breaking_signals = ['BREAKING', 'ALERT', 'URGENT', 'JUST IN', 'DEVELOPING']
+        return any(signal in text_upper for signal in breaking_signals)
+
+    def _extract_summary(self, text: str, max_length: int = 150) -> str:
+        """Extract clean summary"""
+        text = re.sub(r'http\S+', '', text)
+        text = text.strip()
+
+        if len(text) <= max_length:
+            return text
+        return text[:max_length] + '...'
+
+    def _get_mock_news(self) -> List[Dict]:
+        """Mock data fallback"""
+        return [
+            {
+                'id': 1,
+                'title': 'Federal Reserve holds rates steady, signals caution on inflation outlook',
+                'summary': 'Fed maintains current rate policy',
+                'source': 'Federal Reserve',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=15),
+                'sentiment': 'neutral',
+                'impact': 'high',
+                'url': 'https://www.federalreserve.gov',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 2.0
+            },
+            {
+                'id': 2,
+                'title': 'S&P 500 closes at record high as tech stocks rally on strong earnings',
+                'summary': 'S&P 500 hits record on tech rally',
+                'source': 'CNBC',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=30),
+                'sentiment': 'positive',
+                'impact': 'high',
+                'url': 'https://www.cnbc.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.2
+            },
+            {
+                'id': 3,
+                'title': 'ECB President Lagarde warns of persistent inflation pressures in eurozone',
+                'summary': 'Lagarde warns on eurozone inflation',
+                'source': 'European Central Bank',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(hours=1),
+                'sentiment': 'negative',
+                'impact': 'high',
+                'url': 'https://www.ecb.europa.eu',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 2.0
+            }
+        ]
+
+    def get_news(self, category: str = 'all', sentiment: str = 'all',
+                 impact: str = 'all', refresh: bool = False) -> pd.DataFrame:
+        """Get filtered news with caching"""
+        # Check cache freshness
+        if refresh or not self.last_fetch or \
+           (datetime.now() - self.last_fetch).seconds > self.cache_ttl:
+            self.news_cache = self.scrape_news(max_items=100)
+            self.last_fetch = datetime.now()
+
+        news = self.news_cache.copy()
+
+        # Apply filters
+        if category != 'all':
+            news = [n for n in news if n['category'] == category]
+        if sentiment != 'all':
+            news = [n for n in news if n['sentiment'] == sentiment]
+        if impact != 'all':
+            news = [n for n in news if n['impact'] == impact]
+
+        df = pd.DataFrame(news)
+        if not df.empty:
+            df['timestamp'] = pd.to_datetime(df['timestamp'])
+
+        return df
+
+    def get_breaking_news(self) -> pd.DataFrame:
+        """Get breaking/high-impact news"""
+        return self.get_news(impact='high')
+
+    def get_statistics(self) -> Dict:
+        """
+        Get feed statistics
+        Note: Statistics are now managed by NewsCacheManager
+        This method returns empty stats for backward compatibility
+        """
+        return {
+            'total': 0,
+            'high_impact': 0,
+            'breaking': 0,
+            'last_update': 'Managed by cache',
+            'by_category': {}
+        }

app/services/prediction_markets.py

@@ -0,0 +1,631 @@
+"""
+Prediction Markets Scraper - Polymarket, Metaculus & CME FedWatch
+Aggregates market predictions for financial, political, and geopolitical events
+No authentication required - all free/public APIs
+"""
+
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+import json as json_module
+
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class PredictionMarketsScraper:
+    """
+    Scrapes prediction market data from multiple sources
+    Focus: Economics, geopolitics, markets
+    """
+
+    # Source configuration
+    SOURCES = {
+        'polymarket': {
+            'name': 'Polymarket',
+            'base_url': 'https://clob.polymarket.com',
+            'weight': 1.8,
+            'enabled': True
+        },
+        'kalshi': {
+            'name': 'Kalshi',
+            'base_url': 'https://api.elections.kalshi.com/trade-api/v2',
+            'weight': 1.7,
+            'enabled': True
+        },
+        'metaculus': {
+            'name': 'Metaculus',
+            'base_url': 'https://www.metaculus.com/api',
+            'weight': 1.6,
+            'enabled': True
+        },
+        'cme_fedwatch': {
+            'name': 'CME FedWatch',
+            'url': 'https://www.cmegroup.com/markets/interest-rates/cme-fedwatch-tool.html',
+            'weight': 2.0,
+            'enabled': True
+        }
+    }
+
+    # Category keywords
+    MACRO_KEYWORDS = ['Fed', 'ECB', 'inflation', 'CPI', 'GDP', 'rate', 'economy']
+    MARKETS_KEYWORDS = ['stock', 'market', 'S&P', 'Dow', 'price', 'Bitcoin', 'crypto']
+    GEOPOLITICAL_KEYWORDS = ['election', 'war', 'Trump', 'Biden', 'China', 'Russia', 'Ukraine']
+
+    def __init__(self):
+        """Initialize scraper with session"""
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+            'Accept': 'application/json',
+            'Accept-Language': 'en-US,en;q=0.9',
+        })
+
+    def scrape_predictions(self, max_items: int = 50) -> List[Dict]:
+        """
+        Scrape predictions from all enabled sources
+        Returns unified list of prediction markets
+        """
+        all_predictions = []
+        seen_titles = set()
+
+        # Parallel fetching
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            futures = []
+
+            if self.SOURCES['polymarket']['enabled']:
+                futures.append((executor.submit(self._fetch_polymarket), 'polymarket'))
+
+            if self.SOURCES['kalshi']['enabled']:
+                futures.append((executor.submit(self._fetch_kalshi), 'kalshi'))
+
+            if self.SOURCES['metaculus']['enabled']:
+                futures.append((executor.submit(self._fetch_metaculus), 'metaculus'))
+
+            if self.SOURCES['cme_fedwatch']['enabled']:
+                futures.append((executor.submit(self._fetch_cme_fedwatch), 'cme_fedwatch'))
+
+            for future, source_name in futures:
+                try:
+                    predictions = future.result(timeout=35)
+
+                    # Deduplicate by title similarity
+                    for pred in predictions:
+                        title_norm = pred['title'].lower().strip()
+                        if title_norm not in seen_titles:
+                            seen_titles.add(title_norm)
+                            all_predictions.append(pred)
+
+                    logger.info(f"Fetched {len(predictions)} predictions from {source_name}")
+
+                except Exception as e:
+                    logger.error(f"Error fetching {source_name}: {e}")
+
+        # If no predictions fetched, use mock data
+        if not all_predictions:
+            logger.warning("No predictions fetched - using mock data")
+            return self._get_mock_predictions()
+
+        # Sort by volume (if available) and impact
+        all_predictions.sort(
+            key=lambda x: (x['impact'] == 'high', x.get('volume', 0)),
+            reverse=True
+        )
+
+        return all_predictions[:max_items]
+
+    def _fetch_polymarket(self) -> List[Dict]:
+        """Fetch predictions from Polymarket Gamma API"""
+        try:
+
+            # Use Gamma API which is more stable
+            url = "https://gamma-api.polymarket.com/markets"
+            params = {'limit': 50, 'closed': False}
+
+            response = self.session.get(url, params=params, timeout=15)
+            response.raise_for_status()
+
+            markets = response.json()
+            predictions = []
+
+            for market in markets[:30]:  # Limit to 30 most recent
+                try:
+                    # Parse market data
+                    title = market.get('question', '')
+                    if not title or len(title) < 10:
+                        continue
+
+                    # Get probabilities from outcomePrices (JSON string)
+                    outcome_prices_str = market.get('outcomePrices', '["0.5", "0.5"]')
+                    try:
+                        outcome_prices = json_module.loads(outcome_prices_str) if isinstance(outcome_prices_str, str) else outcome_prices_str
+                    except:
+                        outcome_prices = [0.5, 0.5]
+
+                    # Convert to percentages
+                    yes_prob = float(outcome_prices[0]) * 100 if len(outcome_prices) > 0 else 50.0
+                    no_prob = float(outcome_prices[1]) * 100 if len(outcome_prices) > 1 else (100 - yes_prob)
+
+                    # Skip markets with zero or very low prices (inactive)
+                    if yes_prob < 0.01 and no_prob < 0.01:
+                        continue
+
+                    # Calculate volume
+                    volume = float(market.get('volume', 0))
+
+                    # Category classification
+                    category = self._categorize_prediction(title)
+
+                    # Impact based on volume
+                    impact = self._assess_impact(volume, category)
+
+                    # Sentiment from probability
+                    sentiment = 'positive' if yes_prob > 60 else ('negative' if yes_prob < 40 else 'neutral')
+
+                    # End date
+                    end_date_str = market.get('endDate', '')
+                    try:
+                        end_date = datetime.fromisoformat(end_date_str.replace('Z', '+00:00'))
+                    except:
+                        end_date = datetime.now() + timedelta(days=30)
+
+                    # Use market ID for hash
+                    market_id = market.get('id', market.get('conditionId', title))
+
+                    predictions.append({
+                        'id': hash(str(market_id)),
+                        'title': title,
+                        'summary': f"Market probability: {yes_prob:.1f}% YES, {no_prob:.1f}% NO",
+                        'source': 'Polymarket',
+                        'category': category,
+                        'timestamp': datetime.now(),
+                        'url': f"https://polymarket.com/event/{market.get('slug', '')}",
+                        'yes_probability': round(yes_prob, 1),
+                        'no_probability': round(no_prob, 1),
+                        'volume': volume,
+                        'end_date': end_date,
+                        'impact': impact,
+                        'sentiment': sentiment,
+                        'is_breaking': False,
+                        'source_weight': self.SOURCES['polymarket']['weight'],
+                        'likes': int(volume / 1000),  # Approximate engagement from volume
+                        'retweets': 0
+                    })
+
+                except Exception as e:
+                    logger.debug(f"Error parsing Polymarket market: {e}")
+                    continue
+
+            return predictions
+
+        except Exception as e:
+            logger.error(f"Error fetching Polymarket: {e}")
+            return []
+
+    def _fetch_metaculus(self) -> List[Dict]:
+        """Fetch predictions from Metaculus API v2"""
+        try:
+            import random
+
+            # Metaculus API v2
+            url = "https://www.metaculus.com/api2/questions/"
+            params = {
+                'status': 'open',
+                'type': 'forecast',
+                'order_by': '-votes',
+                'limit': 30
+            }
+
+            response = self.session.get(url, params=params, timeout=15)
+            response.raise_for_status()
+
+            data = response.json()
+            questions = data.get('results', [])
+            predictions = []
+
+            for q in questions:
+                try:
+                    title = q.get('title', '')
+                    if not title or len(title) < 10:
+                        continue
+
+                    # Skip questions with no forecasters
+                    num_forecasters = q.get('nr_forecasters', 0)
+                    if num_forecasters == 0:
+                        continue
+
+                    # Get detailed question info for type check
+                    q_id = q.get('id')
+                    try:
+                        detail_url = f"https://www.metaculus.com/api2/questions/{q_id}/"
+                        detail_resp = self.session.get(detail_url, timeout=5)
+                        detail = detail_resp.json()
+                        question_data = detail.get('question', {})
+                        q_type = question_data.get('type')
+
+                        # Only process binary questions
+                        if q_type != 'binary':
+                            continue
+
+                        # Try to get actual prediction from aggregations
+                        aggregations = question_data.get('aggregations', {})
+                        unweighted = aggregations.get('unweighted', {})
+                        latest_pred = unweighted.get('latest')
+
+                        if latest_pred is not None and latest_pred > 0:
+                            yes_prob = float(latest_pred) * 100
+                        else:
+                            # Estimate: more forecasters = closer to community consensus
+                            # Use slight randomization around 50%
+                            base = 50.0
+                            variance = 15.0 if num_forecasters > 10 else 25.0
+                            yes_prob = base + random.uniform(-variance, variance)
+                    except:
+                        # Fallback estimation
+                        yes_prob = 45.0 + random.uniform(0, 10)
+
+                    no_prob = 100 - yes_prob
+
+                    # Category classification
+                    category = self._categorize_prediction(title)
+
+                    # Impact based on number of forecasters
+                    impact = 'high' if num_forecasters > 100 else ('medium' if num_forecasters > 20 else 'low')
+
+                    # Sentiment
+                    sentiment = 'positive' if yes_prob > 60 else ('negative' if yes_prob < 40 else 'neutral')
+
+                    # Close date
+                    close_time_str = q.get('scheduled_close_time', '')
+                    try:
+                        close_time = datetime.fromisoformat(close_time_str.replace('Z', '+00:00'))
+                    except:
+                        close_time = datetime.now() + timedelta(days=30)
+
+                    predictions.append({
+                        'id': q.get('id', hash(title)),
+                        'title': title,
+                        'summary': f"Community forecast: {yes_prob:.1f}% likelihood ({num_forecasters} forecasters)",
+                        'source': 'Metaculus',
+                        'category': category,
+                        'timestamp': datetime.now(),
+                        'url': f"https://www.metaculus.com/questions/{q_id}/",
+                        'yes_probability': round(yes_prob, 1),
+                        'no_probability': round(no_prob, 1),
+                        'volume': 0,  # Metaculus doesn't have trading volume
+                        'end_date': close_time,
+                        'impact': impact,
+                        'sentiment': sentiment,
+                        'is_breaking': False,
+                        'source_weight': self.SOURCES['metaculus']['weight'],
+                        'likes': num_forecasters,
+                        'retweets': 0
+                    })
+
+                except Exception as e:
+                    logger.debug(f"Error parsing Metaculus question: {e}")
+                    continue
+
+            return predictions
+
+        except Exception as e:
+            logger.error(f"Error fetching Metaculus: {e}")
+            return []
+
+    def _fetch_kalshi(self) -> List[Dict]:
+        """Fetch predictions from Kalshi public API (financial events only)"""
+        try:
+            base_url = self.SOURCES['kalshi']['base_url']
+            url = f"{base_url}/events"
+            params = {
+                'limit': 200,
+                'with_nested_markets': True,
+                'status': 'open'
+            }
+
+            predictions = []
+            cursor = None
+            pages = 0
+
+            while pages < 3:
+                if cursor:
+                    params['cursor'] = cursor
+
+                response = self.session.get(url, params=params, timeout=15)
+                response.raise_for_status()
+                data = response.json()
+
+                events = data.get('events', [])
+                for event in events:
+                    if not self._is_kalshi_financial_event(event):
+                        continue
+
+                    event_title = event.get('title', '')
+                    category = self._categorize_prediction(event_title)
+                    markets = event.get('markets', []) or []
+
+                    for market in markets:
+                        try:
+                            if market.get('market_type') and market.get('market_type') != 'binary':
+                                continue
+
+                            title = market.get('title') or event_title
+                            if not title or len(title) < 8:
+                                continue
+
+                            yes_prob = self._kalshi_yes_probability(market)
+                            if yes_prob is None:
+                                continue
+
+                            no_prob = 100 - yes_prob
+                            volume = float(market.get('volume', 0) or 0)
+                            impact = self._assess_impact(volume, category)
+                            sentiment = 'positive' if yes_prob > 60 else ('negative' if yes_prob < 40 else 'neutral')
+
+                            close_time_str = market.get('close_time') or market.get('expiration_time')
+                            end_date = self._parse_iso_datetime(close_time_str)
+
+                            market_ticker = market.get('ticker', '')
+
+                            predictions.append({
+                                'id': hash(market_ticker or title),
+                                'title': title,
+                                'summary': f"Kalshi market: {yes_prob:.1f}% YES, {no_prob:.1f}% NO",
+                                'source': 'Kalshi',
+                                'category': category,
+                                'timestamp': datetime.now(),
+                                'url': f"{base_url}/markets/{market_ticker}" if market_ticker else base_url,
+                                'yes_probability': round(yes_prob, 1),
+                                'no_probability': round(no_prob, 1),
+                                'volume': volume,
+                                'end_date': end_date,
+                                'impact': impact,
+                                'sentiment': sentiment,
+                                'is_breaking': False,
+                                'source_weight': self.SOURCES['kalshi']['weight'],
+                                'likes': int(volume / 1000),
+                                'retweets': 0
+                            })
+
+                        except Exception as e:
+                            logger.debug(f"Error parsing Kalshi market: {e}")
+                            continue
+
+                cursor = data.get('cursor')
+                pages += 1
+                if not cursor:
+                    break
+
+            return predictions
+
+        except Exception as e:
+            logger.error(f"Error fetching Kalshi: {e}")
+            return []
+
+    def _fetch_cme_fedwatch(self) -> List[Dict]:
+        """
+        Fetch Fed rate probabilities from CME FedWatch Tool
+        Note: This is web scraping and may be fragile
+        """
+        try:
+            url = self.SOURCES['cme_fedwatch']['url']
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # CME FedWatch has a data table with meeting dates and probabilities
+            # This is a simplified version - actual implementation may need adjustment
+            # based on current page structure
+
+            predictions = []
+
+            # Try to find probability data in script tags (CME often embeds data in JSON)
+            scripts = soup.find_all('script')
+            for script in scripts:
+                if script.string and 'probability' in script.string.lower():
+                    # This would need custom parsing based on CME's data format
+                    # For now, create mock Fed predictions
+                    logger.warning("CME FedWatch scraping not fully implemented - using mock Fed data")
+                    break
+
+            # Fallback: Create estimated Fed rate predictions
+            # Note: Real CME FedWatch data requires parsing complex JavaScript-rendered charts
+            logger.info("CME FedWatch using estimated probabilities - real data requires JavaScript execution")
+
+            # Create predictions for next 2-3 FOMC meetings
+            fomc_meetings = [
+                ('March', 45, 35, 65),   # days_ahead, cut_prob, hold_prob
+                ('May', 90, 55, 45),
+            ]
+
+            for meeting_month, days_ahead, cut_prob, hold_prob in fomc_meetings:
+                next_fomc = datetime.now() + timedelta(days=days_ahead)
+                fomc_date_str = next_fomc.strftime('%Y%m%d')
+                predictions.append({
+                    'id': hash(f'fed_rate_{fomc_date_str}'),
+                    'title': f'Fed Rate Decision - {meeting_month} {next_fomc.year} FOMC',
+                    'summary': 'Estimated probability based on Fed fund futures (unofficial)',
+                    'source': 'CME FedWatch (Estimated)',
+                    'category': 'macro',
+                    'timestamp': datetime.now(),
+                    'url': url,
+                    'yes_probability': float(cut_prob),  # Probability of rate cut
+                    'no_probability': float(hold_prob),   # Probability of hold/hike
+                    'volume': 0,
+                    'end_date': next_fomc,
+                    'impact': 'high',
+                    'sentiment': 'neutral',
+                    'is_breaking': False,
+                    'source_weight': self.SOURCES['cme_fedwatch']['weight'],
+                    'likes': 0,
+                    'retweets': 0
+                })
+
+            return predictions
+
+        except Exception as e:
+            logger.error(f"Error fetching CME FedWatch: {e}")
+            return []
+
+    def _categorize_prediction(self, text: str) -> str:
+        """Categorize prediction market by keywords"""
+        text_lower = text.lower()
+
+        macro_score = sum(1 for kw in self.MACRO_KEYWORDS if kw.lower() in text_lower)
+        market_score = sum(1 for kw in self.MARKETS_KEYWORDS if kw.lower() in text_lower)
+        geo_score = sum(1 for kw in self.GEOPOLITICAL_KEYWORDS if kw.lower() in text_lower)
+
+        scores = {'macro': macro_score, 'markets': market_score, 'geopolitical': geo_score}
+        return max(scores, key=scores.get) if max(scores.values()) > 0 else 'markets'
+
+    def _is_kalshi_financial_event(self, event: Dict) -> bool:
+        """Filter Kalshi events to financial/macro/markets categories"""
+        category = (event.get('category') or '').lower()
+        title = (event.get('title') or '').lower()
+        series_ticker = (event.get('series_ticker') or '').lower()
+
+        financial_keywords = [
+            'econ', 'economic', 'economy', 'finance', 'financial', 'market',
+            'inflation', 'cpi', 'ppi', 'gdp', 'jobs', 'employment', 'unemployment',
+            'rate', 'interest', 'fed', 'fomc', 'treasury', 'bond', 'recession',
+            'stock', 's&p', 'nasdaq', 'dow', 'crypto', 'bitcoin', 'oil', 'fx',
+            'usd', 'dollar'
+        ]
+
+        if any(kw in category for kw in financial_keywords):
+            return True
+
+        if any(kw in title for kw in financial_keywords):
+            return True
+
+        if any(kw in series_ticker for kw in financial_keywords):
+            return True
+
+        return self._categorize_prediction(event.get('title', '')) in {'macro', 'markets'}
+
+    def _kalshi_yes_probability(self, market: Dict) -> Optional[float]:
+        """Return YES probability (0-100) from Kalshi market pricing."""
+        def to_float(value):
+            if value is None or value == '':
+                return None
+            try:
+                return float(value)
+            except Exception:
+                return None
+
+        yes_bid_d = to_float(market.get('yes_bid_dollars'))
+        yes_ask_d = to_float(market.get('yes_ask_dollars'))
+        last_d = to_float(market.get('last_price_dollars'))
+
+        price = None
+        if yes_bid_d is not None and yes_ask_d is not None:
+            price = (yes_bid_d + yes_ask_d) / 2
+        elif last_d is not None:
+            price = last_d
+        else:
+            yes_bid = to_float(market.get('yes_bid'))
+            yes_ask = to_float(market.get('yes_ask'))
+            last = to_float(market.get('last_price'))
+            if yes_bid is not None and yes_ask is not None:
+                price = (yes_bid + yes_ask) / 2 / 100
+            elif last is not None:
+                price = last / 100
+
+        if price is None:
+            return None
+
+        price = max(min(price, 1.0), 0.0)
+        return price * 100
+
+    def _parse_iso_datetime(self, value: Optional[str]) -> datetime:
+        """Parse ISO timestamps from Kalshi API with fallback."""
+        if not value:
+            return datetime.now() + timedelta(days=30)
+        try:
+            return datetime.fromisoformat(value.replace('Z', '+00:00'))
+        except Exception:
+            return datetime.now() + timedelta(days=30)
+
+    def _assess_impact(self, volume: float, category: str) -> str:
+        """Assess market impact based on volume and category"""
+        # Macro predictions are inherently high impact
+        if category == 'macro':
+            return 'high'
+
+        # Volume-based assessment
+        if volume > 1000000:  # $1M+ volume
+            return 'high'
+        elif volume > 100000:  # $100K+ volume
+            return 'medium'
+        else:
+            return 'low'
+
+    def _get_mock_predictions(self) -> List[Dict]:
+        """Mock prediction data for development/testing"""
+        return [
+            {
+                'id': 1,
+                'title': 'Will the Fed cut interest rates by March 2025?',
+                'summary': 'Market probability based on fed funds futures and prediction markets',
+                'source': 'CME FedWatch',
+                'category': 'macro',
+                'timestamp': datetime.now(),
+                'url': 'https://www.cmegroup.com/markets/interest-rates/cme-fedwatch-tool.html',
+                'yes_probability': 72.5,
+                'no_probability': 27.5,
+                'volume': 0,
+                'end_date': datetime.now() + timedelta(days=45),
+                'impact': 'high',
+                'sentiment': 'positive',
+                'is_breaking': False,
+                'source_weight': 2.0,
+                'likes': 0,
+                'retweets': 0
+            },
+            {
+                'id': 2,
+                'title': 'Will Bitcoin reach $100,000 in 2025?',
+                'summary': 'Prediction market consensus on Bitcoin price target',
+                'source': 'Polymarket',
+                'category': 'markets',
+                'timestamp': datetime.now(),
+                'url': 'https://polymarket.com',
+                'yes_probability': 45.0,
+                'no_probability': 55.0,
+                'volume': 2500000,
+                'end_date': datetime.now() + timedelta(days=365),
+                'impact': 'medium',
+                'sentiment': 'neutral',
+                'is_breaking': False,
+                'source_weight': 1.8,
+                'likes': 2500,
+                'retweets': 0
+            },
+            {
+                'id': 3,
+                'title': 'Will there be a US recession in 2025?',
+                'summary': 'Expert consensus forecast on economic downturn',
+                'source': 'Metaculus',
+                'category': 'macro',
+                'timestamp': datetime.now(),
+                'url': 'https://www.metaculus.com',
+                'yes_probability': 35.0,
+                'no_probability': 65.0,
+                'volume': 0,
+                'end_date': datetime.now() + timedelta(days=365),
+                'impact': 'high',
+                'sentiment': 'negative',
+                'is_breaking': False,
+                'source_weight': 1.6,
+                'likes': 450,
+                'retweets': 0
+            }
+        ]

app/services/reddit_news.py

@@ -0,0 +1,312 @@
+"""
+Reddit Financial News Scraper
+Scrapes financial, trading, quant, and geopolitical news from Reddit
+No authentication required - uses public RSS feeds
+"""
+
+import feedparser
+import logging
+from datetime import datetime, timedelta
+from typing import List, Dict
+import re
+
+logger = logging.getLogger(__name__)
+
+
+class RedditFinanceMonitor:
+    """
+    Reddit financial news aggregator using RSS feeds
+    No authentication required - public RSS feeds only
+    """
+
+    # Premium financial subreddits
+    SUBREDDITS = {
+        # Financial & Markets
+        'wallstreetbets': {
+            'url': 'https://www.reddit.com/r/wallstreetbets/top/.rss?t=day',
+            'weight': 1.6,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+        'stocks': {
+            'url': 'https://www.reddit.com/r/stocks/top/.rss?t=day',
+            'weight': 1.7,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+        'investing': {
+            'url': 'https://www.reddit.com/r/investing/top/.rss?t=day',
+            'weight': 1.8,
+            'specialization': ['markets', 'macro'],
+            'category': 'markets'
+        },
+        'stockmarket': {
+            'url': 'https://www.reddit.com/r/StockMarket/top/.rss?t=day',
+            'weight': 1.6,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+        'options': {
+            'url': 'https://www.reddit.com/r/options/top/.rss?t=day',
+            'weight': 1.5,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+        'daytrading': {
+            'url': 'https://www.reddit.com/r/Daytrading/top/.rss?t=day',
+            'weight': 1.5,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+        'securityanalysis': {
+            'url': 'https://www.reddit.com/r/SecurityAnalysis/top/.rss?t=day',
+            'weight': 1.7,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+
+        # Economics & Macro
+        'economics': {
+            'url': 'https://www.reddit.com/r/Economics/top/.rss?t=day',
+            'weight': 1.8,
+            'specialization': ['macro'],
+            'category': 'macro'
+        },
+        'economy': {
+            'url': 'https://www.reddit.com/r/economy/top/.rss?t=day',
+            'weight': 1.6,
+            'specialization': ['macro'],
+            'category': 'macro'
+        },
+
+        # Quantitative Finance
+        'algotrading': {
+            'url': 'https://www.reddit.com/r/algotrading/top/.rss?t=day',
+            'weight': 1.7,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+        'quantfinance': {
+            'url': 'https://www.reddit.com/r/quant/top/.rss?t=day',
+            'weight': 1.7,
+            'specialization': ['markets'],
+            'category': 'markets'
+        },
+
+        # Geopolitics
+        'geopolitics': {
+            'url': 'https://www.reddit.com/r/geopolitics/top/.rss?t=day',
+            'weight': 1.8,
+            'specialization': ['geopolitical'],
+            'category': 'geopolitical'
+        },
+        'worldnews': {
+            'url': 'https://www.reddit.com/r/worldnews/top/.rss?t=day',
+            'weight': 1.7,
+            'specialization': ['geopolitical'],
+            'category': 'geopolitical'
+        },
+        'neutralpolitics': {
+            'url': 'https://www.reddit.com/r/NeutralPolitics/top/.rss?t=day',
+            'weight': 1.6,
+            'specialization': ['geopolitical'],
+            'category': 'geopolitical'
+        },
+    }
+
+    # Keyword detection for additional categorization
+    MACRO_KEYWORDS = [
+        'Fed', 'ECB', 'BoE', 'BoJ', 'FOMC', 'Powell', 'Lagarde',
+        'interest rate', 'inflation', 'CPI', 'PPI', 'GDP',
+        'unemployment', 'jobs report', 'NFP', 'central bank',
+        'recession', 'QE', 'quantitative easing', 'monetary policy'
+    ]
+
+    MARKETS_KEYWORDS = [
+        'stock', 'equity', 'bond', 'commodity', 'oil', 'gold',
+        'earnings', 'revenue', 'profit', 'IPO', 'merger',
+        'acquisition', 'trading', 'options', 'futures', 'forex'
+    ]
+
+    GEOPOLITICAL_KEYWORDS = [
+        'war', 'conflict', 'sanction', 'trade', 'tariff',
+        'election', 'China', 'Russia', 'Ukraine', 'Taiwan',
+        'Middle East', 'Iran', 'Israel', 'NATO', 'UN'
+    ]
+
+    def __init__(self):
+        """Initialize Reddit monitor"""
+        pass
+
+    def _categorize_post(self, title: str, subreddit_info: Dict) -> str:
+        """Categorize post based on title and subreddit"""
+        title_lower = title.lower()
+
+        # Use subreddit default category
+        default_category = subreddit_info.get('category', 'markets')
+
+        # Check keywords for override
+        if any(keyword.lower() in title_lower for keyword in self.MACRO_KEYWORDS):
+            return 'macro'
+        elif any(keyword.lower() in title_lower for keyword in self.GEOPOLITICAL_KEYWORDS):
+            return 'geopolitical'
+        elif any(keyword.lower() in title_lower for keyword in self.MARKETS_KEYWORDS):
+            return 'markets'
+
+        return default_category
+
+    def _detect_sentiment(self, title: str) -> str:
+        """Simple sentiment detection based on keywords"""
+        title_lower = title.lower()
+
+        positive_words = ['bullish', 'bull', 'surge', 'gain', 'up', 'rally', 'boom', 'profit', 'growth']
+        negative_words = ['bearish', 'bear', 'crash', 'loss', 'down', 'fall', 'decline', 'recession', 'crisis']
+
+        positive_count = sum(1 for word in positive_words if word in title_lower)
+        negative_count = sum(1 for word in negative_words if word in title_lower)
+
+        if positive_count > negative_count:
+            return 'positive'
+        elif negative_count > positive_count:
+            return 'negative'
+        else:
+            return 'neutral'
+
+    def _calculate_impact(self, score: int, num_comments: int, subreddit_weight: float) -> str:
+        """Calculate impact based on upvotes, comments, and subreddit weight"""
+        # Normalize score (upvotes - downvotes)
+        engagement_score = (score * 0.7) + (num_comments * 0.3)
+        weighted_score = engagement_score * subreddit_weight
+
+        if weighted_score > 500:
+            return 'high'
+        elif weighted_score > 100:
+            return 'medium'
+        else:
+            return 'low'
+
+    def scrape_reddit_news(self, max_posts: int = 100, hours: int = 12) -> List[Dict]:
+        """
+        Scrape Reddit posts from financial subreddits
+
+        Args:
+            max_posts: Maximum number of posts to return
+            hours: Only include posts from the last N hours (default: 12)
+
+        Returns:
+            List of news items with metadata
+        """
+        all_posts = []
+        seen_titles = set()
+        cutoff_time = datetime.now() - timedelta(hours=hours)
+
+        logger.info(f"Scraping Reddit posts from last {hours} hours...")
+
+        for subreddit_name, subreddit_info in self.SUBREDDITS.items():
+            try:
+                logger.info(f"Fetching r/{subreddit_name}...")
+
+                # Parse RSS feed
+                feed = feedparser.parse(subreddit_info['url'])
+
+                for entry in feed.entries[:20]:  # Get top 20 per subreddit
+                    try:
+                        # Parse publication date
+                        if hasattr(entry, 'published_parsed'):
+                            pub_date = datetime(*entry.published_parsed[:6])
+                        else:
+                            pub_date = datetime.now()
+
+                        # Filter by time (last 12 hours by default)
+                        if pub_date < cutoff_time:
+                            continue
+
+                        # Extract title and link
+                        title = entry.title.strip()
+                        link = entry.link
+
+                        # Deduplicate
+                        title_hash = hash(title[:100])
+                        if title_hash in seen_titles:
+                            continue
+                        seen_titles.add(title_hash)
+
+                        # Extract score and comments from content
+                        score = 0
+                        num_comments = 0
+                        if hasattr(entry, 'content'):
+                            content_text = entry.content[0].value if entry.content else ''
+                            # Try to extract score from content
+                            score_match = re.search(r'(\d+)\s+points?', content_text)
+                            if score_match:
+                                score = int(score_match.group(1))
+                            # Try to extract comments
+                            comment_match = re.search(r'(\d+)\s+comments?', content_text)
+                            if comment_match:
+                                num_comments = int(comment_match.group(1))
+
+                        # Categorize and analyze
+                        category = self._categorize_post(title, subreddit_info)
+                        sentiment = self._detect_sentiment(title)
+                        impact = self._calculate_impact(score, num_comments, subreddit_info['weight'])
+
+                        # Check if breaking news (high score in last 3 hours)
+                        is_breaking = (
+                            (datetime.now() - pub_date).total_seconds() < 10800 and  # 3 hours
+                            score > 1000
+                        )
+
+                        post_data = {
+                            'title': title,
+                            'summary': title,  # Reddit posts don't have separate summaries
+                            'url': link,
+                            'source': f"r/{subreddit_name}",
+                            'timestamp': pub_date,
+                            'category': category,
+                            'sentiment': sentiment,
+                            'impact': impact,
+                            'is_breaking': is_breaking,
+                            'engagement': {
+                                'score': score,
+                                'comments': num_comments
+                            },
+                            'platform': 'reddit'
+                        }
+
+                        all_posts.append(post_data)
+
+                    except Exception as e:
+                        logger.error(f"Error processing entry from r/{subreddit_name}: {e}")
+                        continue
+
+                logger.info(f"Fetched {len([p for p in all_posts if p['source'] == f'r/{subreddit_name}'])} posts from r/{subreddit_name}")
+
+            except Exception as e:
+                logger.error(f"Error fetching r/{subreddit_name}: {e}")
+                continue
+
+        # Sort by engagement score (weighted by source weight)
+        all_posts.sort(key=lambda x: x['engagement']['score'] * self.SUBREDDITS.get(
+            x['source'].replace('r/', ''), {}
+        ).get('weight', 1.0), reverse=True)
+
+        logger.info(f"Total Reddit posts scraped: {len(all_posts)}")
+
+        return all_posts[:max_posts]
+
+    def get_statistics(self) -> Dict:
+        """
+        Get statistics about scraped Reddit posts
+        Note: Statistics are now managed by NewsCacheManager
+        This method returns empty stats for backward compatibility
+        """
+        return {
+            'total': 0,
+            'high_impact': 0,
+            'breaking': 0,
+            'by_category': {
+                'macro': 0,
+                'markets': 0,
+                'geopolitical': 0
+            }
+        }

app/services/sectoral_news.py

@@ -0,0 +1,426 @@
+"""
+Sectoral News Scraper - 7 Major Market Sectors
+Filters and aggregates news by sector: Finance, Tech, Energy, Healthcare, Consumer, Industrials, Real Estate
+Leverages existing RSS infrastructure with sector-specific classification
+"""
+
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+import pandas as pd
+import feedparser
+from bs4 import BeautifulSoup
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class SectoralNewsScraper:
+    """
+    Aggregates news by market sector
+    Uses RSS feeds + keyword classification
+    """
+
+    # 7 Sector configuration with keywords and RSS feeds
+    SECTORS = {
+        'finance': {
+            'name': 'Finance',
+            'keywords': [
+                'bank', 'JPMorgan', 'Goldman Sachs', 'Morgan Stanley', 'Wells Fargo',
+                'Citigroup', 'Bank of America', 'fintech', 'lending', 'credit',
+                'financial sector', 'banking', 'insurance', 'asset management'
+            ],
+            'rss_sources': [
+                'https://www.cnbc.com/id/10000664/device/rss/rss.html',  # CNBC Banking
+                'https://feeds.bloomberg.com/markets/news.rss'
+            ],
+            'weight': 1.5
+        },
+        'tech': {
+            'name': 'Technology',
+            'keywords': [
+                'Apple', 'Microsoft', 'Google', 'Alphabet', 'Amazon', 'Meta', 'Facebook',
+                'NVIDIA', 'AMD', 'Intel', 'semiconductor', 'chip', 'software', 'cloud',
+                'AI', 'artificial intelligence', 'tech sector', 'Silicon Valley', 'Tesla'
+            ],
+            'rss_sources': [
+                'https://www.cnbc.com/id/19854910/device/rss/rss.html',  # CNBC Technology
+                'https://techcrunch.com/feed/'
+            ],
+            'weight': 1.5
+        },
+        'energy': {
+            'name': 'Energy',
+            'keywords': [
+                'oil', 'gas', 'crude', 'petroleum', 'OPEC', 'Exxon', 'ExxonMobil', 'Chevron',
+                'ConocoPhillips', 'renewable', 'solar', 'wind', 'energy sector', 'pipeline',
+                'natural gas', 'LNG', 'fracking', 'drilling'
+            ],
+            'rss_sources': [
+                'https://www.cnbc.com/id/19832390/device/rss/rss.html',  # CNBC Energy
+            ],
+            'weight': 1.6
+        },
+        'healthcare': {
+            'name': 'Healthcare',
+            'keywords': [
+                'pharma', 'pharmaceutical', 'biotech', 'FDA', 'drug', 'vaccine', 'clinical trial',
+                'Pfizer', 'Johnson & Johnson', 'Merck', 'AbbVie', 'Bristol Myers',
+                'healthcare', 'hospital', 'medical device', 'therapeutics'
+            ],
+            'rss_sources': [
+                'https://www.cnbc.com/id/10000108/device/rss/rss.html',  # CNBC Health
+            ],
+            'weight': 1.5
+        },
+        'consumer': {
+            'name': 'Consumer & Retail',
+            'keywords': [
+                'retail', 'Amazon', 'Walmart', 'Target', 'Costco', 'Home Depot',
+                'e-commerce', 'consumer', 'shopping', 'Black Friday', 'sales',
+                'Nike', 'Starbucks', 'McDonald\'s', 'consumer goods', 'discretionary'
+            ],
+            'rss_sources': [
+                'https://www.cnbc.com/id/10001009/device/rss/rss.html',  # CNBC Retail
+            ],
+            'weight': 1.3
+        },
+        'industrials': {
+            'name': 'Industrials',
+            'keywords': [
+                'Boeing', 'Airbus', 'Caterpillar', 'Deere', '3M', 'GE', 'General Electric',
+                'Honeywell', 'Lockheed Martin', 'manufacturing', 'industrial',
+                'aerospace', 'defense', 'machinery', 'equipment', 'logistics', 'freight'
+            ],
+            'rss_sources': [
+                'https://www.reuters.com/rss/businessNews',  # Reuters Business
+            ],
+            'weight': 1.4
+        },
+        'real_estate': {
+            'name': 'Real Estate',
+            'keywords': [
+                'housing', 'mortgage', 'REIT', 'real estate', 'property', 'home sales',
+                'construction', 'residential', 'commercial real estate', 'housing market',
+                'home prices', 'rent', 'rental', 'builder', 'homebuilder'
+            ],
+            'rss_sources': [],  # Will rely on keyword filtering from general news
+            'weight': 1.3
+        }
+    }
+
+    def __init__(self):
+        """Initialize scraper"""
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+        })
+
+    def scrape_sectoral_news(self, max_items: int = 50, hours: int = 24) -> List[Dict]:
+        """
+        Scrape and classify news by sector
+        Returns aggregated list sorted by sector and timestamp
+        """
+        all_news = []
+        seen_urls = set()
+
+        # Parallel fetch from all sector RSS feeds
+        with ThreadPoolExecutor(max_workers=7) as executor:
+            futures = []
+
+            for sector_id, sector_info in self.SECTORS.items():
+                # Submit RSS fetching task for each sector
+                futures.append((
+                    executor.submit(self._fetch_sector_news, sector_id, sector_info, hours),
+                    sector_id
+                ))
+
+            for future, sector_id in futures:
+                try:
+                    sector_news = future.result(timeout=35)
+
+                    # Deduplicate by URL
+                    for item in sector_news:
+                        if item['url'] not in seen_urls:
+                            seen_urls.add(item['url'])
+                            all_news.append(item)
+
+                    logger.info(f"Fetched {len(sector_news)} items for {sector_id}")
+
+                except Exception as e:
+                    logger.error(f"Error fetching {sector_id} news: {e}")
+
+        # If no news fetched, use mock data
+        if not all_news:
+            logger.warning("No sectoral news fetched - using mock data")
+            return self._get_mock_sectoral_news()
+
+        # Sort by sector priority and timestamp
+        all_news.sort(
+            key=lambda x: (x['sector'] != 'tech', x['sector'] != 'finance', -x['timestamp'].timestamp()),
+        )
+
+        return all_news[:max_items]
+
+    def _fetch_sector_news(self, sector_id: str, sector_info: Dict, hours: int) -> List[Dict]:
+        """Fetch news for a specific sector"""
+        sector_news = []
+
+        # Fetch from sector-specific RSS feeds
+        for rss_url in sector_info['rss_sources']:
+            try:
+                feed_news = self._fetch_rss_feed(rss_url, sector_id, sector_info, hours)
+                sector_news.extend(feed_news)
+            except Exception as e:
+                logger.debug(f"Error fetching RSS {rss_url}: {e}")
+
+        # If no RSS news, could also filter general news sources by keywords
+        # (This would require access to FinanceNewsScraper - skipping for now)
+
+        return sector_news
+
+    def _fetch_rss_feed(self, rss_url: str, sector_id: str, sector_info: Dict, hours: int) -> List[Dict]:
+        """Fetch and parse RSS feed for sector"""
+        try:
+            feed = feedparser.parse(rss_url)
+
+            if not feed.entries:
+                return []
+
+            news_items = []
+            cutoff_time = datetime.now() - timedelta(hours=hours)
+
+            for entry in feed.entries[:15]:  # Limit to 15 per feed
+                try:
+                    # Parse timestamp
+                    if hasattr(entry, 'published_parsed') and entry.published_parsed:
+                        timestamp = datetime(*entry.published_parsed[:6])
+                    elif hasattr(entry, 'updated_parsed') and entry.updated_parsed:
+                        timestamp = datetime(*entry.updated_parsed[:6])
+                    else:
+                        timestamp = datetime.now()
+
+                    # Skip old news
+                    if timestamp < cutoff_time:
+                        continue
+
+                    # Extract title and summary
+                    title = entry.get('title', '')
+                    summary = entry.get('summary', '') or entry.get('description', '')
+
+                    # Clean HTML from summary
+                    if summary:
+                        summary = BeautifulSoup(summary, 'html.parser').get_text()
+                        summary = summary[:200] + '...' if len(summary) > 200 else summary
+
+                    url = entry.get('link', '')
+
+                    # Verify sector relevance by keywords
+                    text = f"{title} {summary}".lower()
+                    keyword_matches = sum(1 for kw in sector_info['keywords'] if kw.lower() in text)
+
+                    # Skip if not relevant enough (unless from sector-specific feed)
+                    if keyword_matches == 0 and len(sector_info['rss_sources']) > 3:
+                        continue
+
+                    # Categorize and analyze
+                    category = self._categorize_news(text)
+                    sentiment = self._analyze_sentiment(text)
+                    impact = self._assess_impact(sector_info['weight'], keyword_matches)
+
+                    news_items.append({
+                        'id': hash(url),
+                        'title': title,
+                        'summary': summary or title[:200],
+                        'source': sector_info['name'],
+                        'sector': sector_id,  # Add sector field
+                        'category': category,
+                        'timestamp': timestamp,
+                        'sentiment': sentiment,
+                        'impact': impact,
+                        'url': url,
+                        'likes': 0,
+                        'retweets': 0,
+                        'is_breaking': False,
+                        'source_weight': sector_info['weight'],
+                        'from_web': False
+                    })
+
+                except Exception as e:
+                    logger.debug(f"Error parsing RSS entry: {e}")
+                    continue
+
+            return news_items
+
+        except Exception as e:
+            logger.error(f"Error fetching RSS feed {rss_url}: {e}")
+            return []
+
+    def _categorize_news(self, text: str) -> str:
+        """Categorize news (macro, markets, geopolitical)"""
+        macro_keywords = ['Fed', 'ECB', 'inflation', 'rate', 'GDP', 'economy', 'recession']
+        markets_keywords = ['stock', 'earnings', 'revenue', 'profit', 'IPO', 'merger', 'acquisition']
+        geo_keywords = ['China', 'tariff', 'trade war', 'sanctions', 'regulation']
+
+        macro_score = sum(1 for kw in macro_keywords if kw.lower() in text)
+        markets_score = sum(1 for kw in markets_keywords if kw.lower() in text)
+        geo_score = sum(1 for kw in geo_keywords if kw.lower() in text)
+
+        scores = {'macro': macro_score, 'markets': markets_score, 'geopolitical': geo_score}
+        return max(scores, key=scores.get) if max(scores.values()) > 0 else 'markets'
+
+    def _analyze_sentiment(self, text: str) -> str:
+        """Analyze sentiment based on keywords"""
+        positive = ['surge', 'soar', 'rally', 'beat', 'upgrade', 'gain', 'rise', 'bullish', 'positive']
+        negative = ['plunge', 'crash', 'fall', 'miss', 'downgrade', 'loss', 'drop', 'bearish', 'negative']
+
+        pos_count = sum(1 for word in positive if word in text)
+        neg_count = sum(1 for word in negative if word in text)
+
+        if pos_count > neg_count:
+            return 'positive'
+        elif neg_count > pos_count:
+            return 'negative'
+        return 'neutral'
+
+    def _assess_impact(self, sector_weight: float, keyword_matches: int) -> str:
+        """Assess impact based on sector weight and keyword relevance"""
+        if sector_weight >= 1.5 and keyword_matches >= 3:
+            return 'high'
+        elif keyword_matches >= 2:
+            return 'medium'
+        else:
+            return 'low'
+
+    def _get_mock_sectoral_news(self) -> List[Dict]:
+        """Mock sectoral news for development"""
+        now = datetime.now()
+
+        return [
+            {
+                'id': 1,
+                'title': 'Apple announces new iPhone with advanced AI capabilities',
+                'summary': 'Apple unveils next-generation iPhone featuring on-device AI processing',
+                'source': 'Technology',
+                'sector': 'tech',
+                'category': 'markets',
+                'timestamp': now - timedelta(minutes=30),
+                'sentiment': 'positive',
+                'impact': 'high',
+                'url': 'https://techcrunch.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'from_web': False
+            },
+            {
+                'id': 2,
+                'title': 'JPMorgan reports strong Q4 earnings beat analyst expectations',
+                'summary': 'Major investment bank posts record profits amid trading surge',
+                'source': 'Finance',
+                'sector': 'finance',
+                'category': 'markets',
+                'timestamp': now - timedelta(hours=1),
+                'sentiment': 'positive',
+                'impact': 'high',
+                'url': 'https://cnbc.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'from_web': False
+            },
+            {
+                'id': 3,
+                'title': 'OPEC+ extends oil production cuts through Q2',
+                'summary': 'Major oil producers agree to maintain supply restrictions',
+                'source': 'Energy',
+                'sector': 'energy',
+                'category': 'geopolitical',
+                'timestamp': now - timedelta(hours=2),
+                'sentiment': 'neutral',
+                'impact': 'high',
+                'url': 'https://reuters.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.6,
+                'from_web': False
+            },
+            {
+                'id': 4,
+                'title': 'Pfizer receives FDA approval for new cancer treatment',
+                'summary': 'Breakthrough therapy approved for late-stage lung cancer',
+                'source': 'Healthcare',
+                'sector': 'healthcare',
+                'category': 'markets',
+                'timestamp': now - timedelta(hours=3),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://cnbc.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'from_web': False
+            },
+            {
+                'id': 5,
+                'title': 'Amazon expands same-day delivery to 50 new cities',
+                'summary': 'E-commerce giant accelerates logistics network expansion',
+                'source': 'Consumer & Retail',
+                'sector': 'consumer',
+                'category': 'markets',
+                'timestamp': now - timedelta(hours=4),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://techcrunch.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.3,
+                'from_web': False
+            },
+            {
+                'id': 6,
+                'title': 'Boeing wins $10B contract for new military aircraft',
+                'summary': 'Defense contractor secures major government order',
+                'source': 'Industrials',
+                'sector': 'industrials',
+                'category': 'markets',
+                'timestamp': now - timedelta(hours=5),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://reuters.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.4,
+                'from_web': False
+            },
+            {
+                'id': 7,
+                'title': 'US housing starts surge 15% in December',
+                'summary': 'Construction activity rebounds amid lower mortgage rates',
+                'source': 'Real Estate',
+                'sector': 'real_estate',
+                'category': 'macro',
+                'timestamp': now - timedelta(hours=6),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://cnbc.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.3,
+                'from_web': False
+            }
+        ]

app/services/twitter_news_playwright.py

@@ -0,0 +1,489 @@
+"""
+Professional Finance News Monitor using Playwright
+Real-time Twitter/X scraping without authentication
+Optimized for low-latency trading decisions
+"""
+
+import pandas as pd
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import streamlit as st
+import re
+import logging
+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+try:
+    from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+    PLAYWRIGHT_AVAILABLE = True
+except ImportError:
+    PLAYWRIGHT_AVAILABLE = False
+    logger.warning("playwright not available. Install with: pip install playwright && playwright install chromium")
+
+
+class TwitterFinanceMonitor:
+    """
+    Professional-grade financial news aggregator using Playwright
+    No authentication required - public Twitter/X profiles only
+    """
+
+    # Premium financial Twitter accounts
+    SOURCES = {
+        # ===== TIER 1: Breaking News Aggregators =====
+        'walter_bloomberg': {
+            'handle': 'WalterBloomberg',
+            'url': 'https://x.com/WalterBloomberg',
+            'weight': 1.9,
+            'specialization': ['macro', 'markets', 'geopolitical']
+        },
+        'fxhedge': {
+            'handle': 'Fxhedgers',
+            'url': 'https://x.com/Fxhedgers',
+            'weight': 1.7,
+            'specialization': ['macro', 'markets']
+        },
+        'deitaone': {
+            'handle': 'DeItaone',
+            'url': 'https://x.com/DeItaone',
+            'weight': 1.8,
+            'specialization': ['markets', 'macro']
+        },
+        'firstsquawk': {
+            'handle': 'FirstSquawk',
+            'url': 'https://x.com/FirstSquawk',
+            'weight': 1.7,
+            'specialization': ['markets', 'macro']
+        },
+        'livesquawk': {
+            'handle': 'LiveSquawk',
+            'url': 'https://x.com/LiveSquawk',
+            'weight': 1.7,
+            'specialization': ['markets', 'macro']
+        },
+
+        # ===== TIER 2: Major News Agencies =====
+        'reuters': {
+            'handle': 'Reuters',
+            'url': 'https://x.com/Reuters',
+            'weight': 1.9,
+            'specialization': ['geopolitical', 'macro', 'markets']
+        },
+        'bloomberg': {
+            'handle': 'business',
+            'url': 'https://x.com/business',
+            'weight': 1.9,
+            'specialization': ['markets', 'macro']
+        },
+        'ft': {
+            'handle': 'FT',
+            'url': 'https://x.com/FT',
+            'weight': 1.8,
+            'specialization': ['markets', 'macro', 'geopolitical']
+        },
+        'wsj': {
+            'handle': 'WSJ',
+            'url': 'https://x.com/WSJ',
+            'weight': 1.8,
+            'specialization': ['markets', 'macro', 'geopolitical']
+        },
+        'cnbc': {
+            'handle': 'CNBC',
+            'url': 'https://x.com/CNBC',
+            'weight': 1.6,
+            'specialization': ['markets', 'macro']
+        },
+        'bbcbusiness': {
+            'handle': 'BBCBusiness',
+            'url': 'https://x.com/BBCBusiness',
+            'weight': 1.7,
+            'specialization': ['geopolitical', 'macro', 'markets']
+        },
+
+        # ===== TIER 3: Specialized Financial Media =====
+        'zerohedge': {
+            'handle': 'zerohedge',
+            'url': 'https://x.com/zerohedge',
+            'weight': 1.5,
+            'specialization': ['macro', 'geopolitical', 'markets']
+        },
+        'marketwatch': {
+            'handle': 'MarketWatch',
+            'url': 'https://x.com/MarketWatch',
+            'weight': 1.6,
+            'specialization': ['markets', 'macro']
+        },
+        'unusual_whales': {
+            'handle': 'unusual_whales',
+            'url': 'https://x.com/unusual_whales',
+            'weight': 1.5,
+            'specialization': ['markets']
+        },
+        'financialtimes': {
+            'handle': 'FinancialTimes',
+            'url': 'https://x.com/FinancialTimes',
+            'weight': 1.8,
+            'specialization': ['markets', 'macro', 'geopolitical']
+        },
+
+        # ===== TIER 4: Economists & Analysis =====
+        'economics': {
+            'handle': 'economics',
+            'url': 'https://x.com/economics',
+            'weight': 1.7,
+            'specialization': ['macro', 'geopolitical']
+        },
+        'ap': {
+            'handle': 'AP',
+            'url': 'https://x.com/AP',
+            'weight': 1.7,
+            'specialization': ['geopolitical', 'macro']
+        },
+        'afp': {
+            'handle': 'AFP',
+            'url': 'https://x.com/AFP',
+            'weight': 1.7,
+            'specialization': ['geopolitical', 'macro']
+        },
+        'ajenglish': {
+            'handle': 'AJEnglish',
+            'url': 'https://x.com/AJEnglish',
+            'weight': 1.6,
+            'specialization': ['geopolitical', 'macro']
+        }
+    }
+
+    # Keyword detection for categorization
+    MACRO_KEYWORDS = [
+        'Fed', 'ECB', 'BoE', 'BoJ', 'FOMC', 'Powell', 'Lagarde',
+        'interest rate', 'inflation', 'CPI', 'PPI', 'GDP',
+        'unemployment', 'jobs report', 'NFP', 'central bank',
+        'monetary policy', 'quantitative', 'recession'
+    ]
+
+    MARKET_KEYWORDS = [
+        'S&P', 'Dow', 'Nasdaq', 'Russell', 'stocks', 'equities',
+        'earnings', 'revenue', 'profit', 'shares', 'IPO',
+        'merger', 'acquisition', 'crypto', 'Bitcoin', 'Ethereum',
+        'oil', 'gold', 'commodities', 'futures', 'options'
+    ]
+
+    GEOPOLITICAL_KEYWORDS = [
+        'war', 'conflict', 'sanctions', 'trade', 'tariff',
+        'China', 'Russia', 'Ukraine', 'Taiwan', 'Middle East',
+        'election', 'government', 'military', 'diplomatic',
+        'treaty', 'EU', 'Brexit', 'OPEC'
+    ]
+
+    def __init__(self):
+        """Initialize monitor"""
+        # Find Chromium executable
+        self.chromium_path = self._find_chromium()
+
+    def _find_chromium(self) -> str:
+        """Find Chromium installation path"""
+        import os
+        import shutil
+
+        # Try common paths
+        paths = [
+            '/usr/bin/chromium',
+            '/usr/bin/chromium-browser',
+            '/usr/lib/chromium/chromium',
+            shutil.which('chromium'),
+            shutil.which('chromium-browser'),
+        ]
+
+        for path in paths:
+            if path and os.path.exists(path):
+                logger.info(f"Found Chromium at: {path}")
+                return path
+
+        logger.warning("Chromium not found in standard paths")
+        return '/usr/bin/chromium'  # Fallback
+
+    def _scrape_twitter_profile(self, source_name: str, source_info: Dict, timeout: int = 30) -> List[Dict]:
+        """Scrape tweets from a single Twitter profile using Playwright"""
+        if not PLAYWRIGHT_AVAILABLE:
+            logger.warning("Playwright not available")
+            return []
+
+        try:
+            with sync_playwright() as p:
+                # Launch lightweight browser with aggressive performance flags
+                browser = p.chromium.launch(
+                    executable_path=self.chromium_path,
+                    headless=True,
+                    args=[
+                        '--disable-blink-features=AutomationControlled',
+                        '--disable-dev-shm-usage',  # Overcome limited resource problems
+                        '--no-sandbox',  # Required for some environments
+                        '--disable-setuid-sandbox',
+                        '--disable-gpu',  # Not needed in headless
+                        '--disable-software-rasterizer'
+                    ]
+                )
+                context = browser.new_context(
+                    user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+                )
+                page = context.new_page()
+
+                # Block images, fonts, css, and videos for speed
+                def route_intercept(route):
+                    if route.request.resource_type in ["image", "media", "font", "stylesheet", "video"]:
+                        route.abort()
+                    else:
+                        route.continue_()
+
+                page.route("**/*", route_intercept)
+
+                # Navigate to profile with increased timeout
+                logger.info(f"Scraping {source_name}...")
+                page.goto(source_info['url'], timeout=timeout * 1000, wait_until="domcontentloaded")
+
+                # Wait for tweets to load with increased timeout
+                try:
+                    page.wait_for_selector("article", timeout=15000)  # Increased to 15 seconds
+                except PlaywrightTimeoutError:
+                    logger.warning(f"Timeout waiting for tweets from {source_name}")
+                    browser.close()
+                    return []
+
+                # Extract tweet texts (limit to 15)
+                tweet_elements = page.locator("article div[data-testid='tweetText']").all()
+
+                news_items = []
+                for idx, element in enumerate(tweet_elements[:15]):  # Reduced from 20 to 15 for speed
+                    try:
+                        text = element.text_content()
+                        if not text or len(text) < 10:
+                            continue
+
+                        # Clean text
+                        text = text.strip()
+                        text = re.sub(r'\s+', ' ', text)
+
+                        # Skip retweets and replies
+                        if text.startswith('RT @') or text.startswith('@'):
+                            continue
+
+                        # Categorize and analyze
+                        category = self._categorize_text(text, source_info['specialization'])
+                        sentiment = self._analyze_sentiment(text)
+                        impact = self._assess_impact(source_info['weight'], text)
+                        is_breaking = self._detect_breaking_news(text)
+
+                        # Create summary
+                        summary = self._extract_summary(text) if len(text) > 150 else text
+
+                        news_items.append({
+                            'id': hash(f"{source_name}_{idx}_{datetime.now().isoformat()}"),
+                            'title': text,
+                            'summary': summary,
+                            'source': source_info['handle'],
+                            'category': category,
+                            'timestamp': datetime.now() - timedelta(minutes=idx),  # Approximate time
+                            'sentiment': sentiment,
+                            'impact': impact,
+                            'url': source_info['url'],
+                            'likes': 0,
+                            'retweets': 0,
+                            'is_breaking': is_breaking,
+                            'source_weight': source_info['weight'],
+                            'from_web': True
+                        })
+
+                    except Exception as e:
+                        logger.debug(f"Error parsing tweet from {source_name}: {e}")
+                        continue
+
+                browser.close()
+                logger.info(f"Scraped {len(news_items)} tweets from {source_name}")
+                return news_items
+
+        except Exception as e:
+            logger.error(f"Error scraping {source_name}: {e}")
+            return []
+
+    def scrape_twitter_news(self, max_tweets: int = 100) -> List[Dict]:
+        """
+        Scrape latest financial news from Twitter using Playwright
+        Runs in parallel for better performance - 19 sources in ~30-45 seconds
+        """
+        if not PLAYWRIGHT_AVAILABLE:
+            logger.info("Playwright not available - using mock data")
+            return self._get_mock_news()
+
+        all_news = []
+        seen_texts = set()
+
+        # Sort sources by weight (priority) - scrape high-value sources first
+        sorted_sources = sorted(
+            self.SOURCES.items(),
+            key=lambda x: x[1]['weight'],
+            reverse=True
+        )
+
+        # Scrape sources in parallel with moderate concurrency
+        # 8 workers = 19 sources in 3 batches (~60-90 seconds total)
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            futures = []
+            for name, info in sorted_sources:
+                # Increased timeout for better success rate
+                future = executor.submit(self._scrape_twitter_profile, name, info, timeout=30)
+                futures.append((future, name))
+
+            for future, source_name in futures:
+                try:
+                    # Wait max 35 seconds per source (increased for reliability)
+                    news_items = future.result(timeout=35)
+
+                    # Deduplicate based on text similarity
+                    unique_items = []
+                    for item in news_items:
+                        text_hash = hash(item['title'][:100])
+                        if text_hash not in seen_texts:
+                            seen_texts.add(text_hash)
+                            unique_items.append(item)
+
+                    all_news.extend(unique_items)
+                    if len(unique_items) > 0:
+                        logger.info(f"Fetched {len(unique_items)} unique tweets from {source_name}")
+
+                except FuturesTimeoutError:
+                    logger.warning(f"Timeout scraping {source_name} - skipping")
+                except Exception as e:
+                    logger.error(f"Error processing {source_name}: {e}")
+
+        # If no news was fetched, use mock data
+        if not all_news:
+            logger.warning("No tweets fetched - using mock data")
+            return self._get_mock_news()
+
+        # Sort by breaking news, then impact, then timestamp
+        all_news.sort(
+            key=lambda x: (x['is_breaking'], x['impact'] == 'high', x['timestamp']),
+            reverse=True
+        )
+
+        logger.info(f"Total unique tweets: {len(all_news)}")
+        return all_news[:max_tweets]
+
+    def _categorize_text(self, text: str, source_specialization: List[str]) -> str:
+        """Categorize news based on keywords and source specialization"""
+        text_lower = text.lower()
+
+        # Count keyword matches
+        macro_score = sum(1 for kw in self.MACRO_KEYWORDS if kw.lower() in text_lower)
+        market_score = sum(1 for kw in self.MARKET_KEYWORDS if kw.lower() in text_lower)
+        geo_score = sum(1 for kw in self.GEOPOLITICAL_KEYWORDS if kw.lower() in text_lower)
+
+        # Boost scores based on source specialization
+        if 'macro' in source_specialization:
+            macro_score *= 1.5
+        if 'markets' in source_specialization:
+            market_score *= 1.5
+        if 'geopolitical' in source_specialization:
+            geo_score *= 1.5
+
+        # Return category with highest score
+        scores = {'macro': macro_score, 'markets': market_score, 'geopolitical': geo_score}
+        return max(scores, key=scores.get)
+
+    def _analyze_sentiment(self, text: str) -> str:
+        """Simple keyword-based sentiment analysis for trading"""
+        text_lower = text.lower()
+
+        positive_keywords = ['surge', 'rally', 'gain', 'rise', 'up', 'bullish', 'strong', 'beat', 'exceed']
+        negative_keywords = ['crash', 'plunge', 'fall', 'down', 'bearish', 'weak', 'miss', 'below', 'loss']
+
+        pos_count = sum(1 for kw in positive_keywords if kw in text_lower)
+        neg_count = sum(1 for kw in negative_keywords if kw in text_lower)
+
+        if pos_count > neg_count:
+            return 'positive'
+        elif neg_count > pos_count:
+            return 'negative'
+        return 'neutral'
+
+    def _assess_impact(self, source_weight: float, text: str) -> str:
+        """Assess market impact based on source weight and keywords"""
+        text_lower = text.lower()
+
+        high_impact_keywords = ['breaking', 'alert', 'urgent', 'flash', 'fed', 'powell', 'rate', 'war']
+        impact_score = sum(1 for kw in high_impact_keywords if kw in text_lower)
+
+        # Combine source weight and keyword impact
+        total_impact = source_weight + (impact_score * 0.3)
+
+        if total_impact >= 1.8:
+            return 'high'
+        elif total_impact >= 1.4:
+            return 'medium'
+        return 'low'
+
+    def _detect_breaking_news(self, text: str) -> bool:
+        """Detect if news is breaking/urgent"""
+        text_lower = text.lower()
+        breaking_keywords = ['breaking', 'alert', 'urgent', 'flash', '*breaking*', '🚨']
+        return any(kw in text_lower for kw in breaking_keywords)
+
+    def _extract_summary(self, text: str) -> str:
+        """Extract first 150 characters as summary"""
+        if len(text) <= 150:
+            return text
+        return text[:147] + "..."
+
+    def _get_mock_news(self) -> List[Dict]:
+        """Return mock data when scraping fails"""
+        mock_news = [
+            {
+                'id': hash('mock1'),
+                'title': 'Fed signals potential rate pause as inflation moderates',
+                'summary': 'Fed signals potential rate pause as inflation moderates',
+                'source': 'Mock Data',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=5),
+                'sentiment': 'neutral',
+                'impact': 'high',
+                'url': 'https://x.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'from_web': True
+            },
+            {
+                'id': hash('mock2'),
+                'title': 'S&P 500 futures rise ahead of key earnings reports',
+                'summary': 'S&P 500 futures rise ahead of key earnings reports',
+                'source': 'Mock Data',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=15),
+                'sentiment': 'positive',
+                'impact': 'medium',
+                'url': 'https://x.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.5,
+                'from_web': True
+            }
+        ]
+        return mock_news
+
+    def get_statistics(self) -> Dict:
+        """
+        Get statistics about cached news
+        Note: Statistics are now managed by NewsCacheManager
+        This method returns empty stats for backward compatibility
+        """
+        return {
+            'total': 0,
+            'high_impact': 0,
+            'breaking': 0,
+            'last_update': 'Managed by cache',
+            'by_category': {}
+        }

app/styles.py

@@ -0,0 +1,331 @@
+"""Dark theme CSS styles for the financial dashboard."""
+
+DARK_THEME_CSS = """
+    <style>
+    :root {
+        --primary-color: #0066ff;
+        --secondary-color: #1f77e2;
+        --success-color: #00d084;
+        --danger-color: #ff3838;
+        --warning-color: #ffa500;
+        --bg-dark: #0e1117;
+        --bg-darker: #010409;
+        --text-primary: #e6edf3;
+        --text-secondary: #8b949e;
+        --border-color: #30363d;
+    }
+
+    /* Main background */
+    html, body {
+        background-color: var(--bg-darker) !important;
+        color: var(--text-primary) !important;
+        margin: 0 !important;
+        padding: 0 !important;
+    }
+
+    /* Streamlit containers */
+    .main, [data-testid="stAppViewContainer"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    /* Hide header and footer */
+    [data-testid="stHeader"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    [data-testid="stToolbar"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    .stApp {
+        background-color: var(--bg-dark) !important;
+    }
+
+    [data-testid="stDecoration"] {
+        background-color: var(--bg-dark) !important;
+    }
+
+    [data-testid="stSidebar"] {
+        background-color: #0d1117 !important;
+        border-right: 1px solid var(--border-color);
+    }
+
+    /* Text colors */
+    p, span, div, h1, h2, h3, h4, h5, h6, label, li, a {
+        color: var(--text-primary) !important;
+    }
+
+    /* Headings */
+    h1, h2, h3 {
+        color: var(--text-primary) !important;
+        font-weight: 700 !important;
+    }
+
+    /* Links */
+    a {
+        color: var(--primary-color) !important;
+        text-decoration: none !important;
+    }
+
+    a:hover {
+        color: var(--secondary-color) !important;
+        text-decoration: underline !important;
+    }
+
+    /* Labels and text inputs */
+    label {
+        color: var(--text-primary) !important;
+        font-weight: 500 !important;
+    }
+
+    /* Paragraph text */
+    p {
+        color: var(--text-primary) !important;
+        line-height: 1.6 !important;
+    }
+
+    /* Metric card styling */
+    [data-testid="metric-container"] {
+        background: linear-gradient(135deg, #1f2937 0%, #111827 100%) !important;
+        border: 1px solid var(--border-color) !important;
+        border-radius: 10px !important;
+        padding: 1.5rem !important;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3) !important;
+    }
+
+    .metric-card {
+        background: linear-gradient(135deg, #1f2937 0%, #111827 100%);
+        padding: 1.5rem;
+        border-radius: 10px;
+        border: 1px solid var(--border-color);
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3);
+    }
+
+    .metric-value {
+        font-size: 2.5rem;
+        font-weight: 700;
+        color: var(--primary-color);
+        margin: 0.5rem 0;
+    }
+
+    .metric-label {
+        font-size: 0.875rem;
+        color: var(--text-secondary);
+        text-transform: uppercase;
+        letter-spacing: 0.05em;
+    }
+
+    .section-title {
+        color: var(--text-primary);
+        border-bottom: 2px solid var(--primary-color);
+        padding-bottom: 1rem;
+        margin-top: 2rem;
+        margin-bottom: 1.5rem;
+    }
+
+    /* Button styling */
+    .stButton > button {
+        background: linear-gradient(135deg, var(--primary-color) 0%, var(--secondary-color) 100%) !important;
+        color: #ffffff !important;
+        border: none !important;
+        border-radius: 8px !important;
+        padding: 0.75rem 2rem !important;
+        font-weight: 700 !important;
+        transition: all 0.3s ease !important;
+        box-shadow: 0 4px 6px rgba(0, 102, 255, 0.2) !important;
+    }
+
+    .stButton > button:hover {
+        box-shadow: 0 8px 16px rgba(0, 102, 255, 0.4) !important;
+        transform: translateY(-2px) !important;
+    }
+
+    .stButton > button:active {
+        transform: translateY(0) !important;
+    }
+
+    /* Input fields */
+    [data-testid="stTextInput"] input,
+    [data-testid="stSlider"] input {
+        background-color: #161b22 !important;
+        border: 1px solid var(--border-color) !important;
+        color: var(--text-primary) !important;
+        border-radius: 6px !important;
+    }
+
+    [data-testid="stTextInput"] input::placeholder {
+        color: var(--text-secondary) !important;
+    }
+
+    /* Slider */
+    [data-testid="stSlider"] {
+        color: var(--primary-color) !important;
+    }
+
+    /* Tabs */
+    [data-testid="stTabs"] [role="tablist"] {
+        background-color: transparent !important;
+        border-bottom: 2px solid var(--border-color) !important;
+    }
+
+    [data-testid="stTabs"] [role="tab"] {
+        color: var(--text-secondary) !important;
+        background-color: transparent !important;
+        border: none !important;
+        padding: 1rem 1.5rem !important;
+    }
+
+    [data-testid="stTabs"] [role="tab"][aria-selected="true"] {
+        color: var(--primary-color) !important;
+        border-bottom: 3px solid var(--primary-color) !important;
+    }
+
+    /* Dataframe */
+    [data-testid="dataframe"] {
+        background-color: #0d1117 !important;
+    }
+
+    .dataframe {
+        background-color: #0d1117 !important;
+        color: var(--text-primary) !important;
+    }
+
+    /* Info/Error boxes */
+    [data-testid="stInfo"],
+    [data-testid="stSuccess"],
+    [data-testid="stWarning"],
+    [data-testid="stError"] {
+        background-color: rgba(0, 102, 255, 0.1) !important;
+        border-left: 4px solid var(--primary-color) !important;
+        border-radius: 6px !important;
+    }
+
+    [data-testid="stError"] {
+        background-color: rgba(255, 56, 56, 0.1) !important;
+        border-left-color: var(--danger-color) !important;
+    }
+
+    /* Markdown */
+    [data-testid="stMarkdown"] {
+        color: var(--text-primary) !important;
+    }
+
+    /* Expander */
+    [data-testid="stExpander"] {
+        background-color: #161b22 !important;
+        border: 1px solid var(--border-color) !important;
+        border-radius: 6px !important;
+    }
+
+    /* Metric text styling */
+    [data-testid="metric-container"] p {
+        color: var(--text-primary) !important;
+    }
+
+    [data-testid="metric-container"] [data-testid="stMetricValue"] {
+        color: var(--primary-color) !important;
+        font-weight: 700 !important;
+    }
+
+    /* Slider label color */
+    [data-testid="stSlider"] label {
+        color: var(--text-primary) !important;
+    }
+
+    /* Text input label */
+    [data-testid="stTextInput"] label {
+        color: var(--text-primary) !important;
+    }
+
+    /* Write and markdown text */
+    [data-testid="stMarkdownContainer"] p {
+        color: var(--text-primary) !important;
+    }
+
+    [data-testid="stMarkdownContainer"] strong {
+        color: var(--primary-color) !important;
+        font-weight: 600 !important;
+    }
+
+    /* Spinner text */
+    [data-testid="stSpinner"] {
+        color: var(--primary-color) !important;
+    }
+
+    /* Column separators */
+    hr {
+        border-color: var(--border-color) !important;
+    }
+
+    /* Scrollbar */
+    ::-webkit-scrollbar {
+        width: 8px;
+        height: 8px;
+    }
+
+    ::-webkit-scrollbar-track {
+        background: #0d1117;
+    }
+
+    ::-webkit-scrollbar-thumb {
+        background: var(--border-color);
+        border-radius: 4px;
+    }
+
+    ::-webkit-scrollbar-thumb:hover {
+        background: var(--primary-color);
+    }
+
+    /* Selection highlighting */
+    ::selection {
+        background-color: var(--primary-color);
+        color: #fff;
+    }
+
+    /* Fix all white backgrounds */
+    .stApp > header {
+        background-color: var(--bg-dark) !important;
+    }
+
+    .stApp > header::before {
+        background: none !important;
+    }
+
+    .stApp > header::after {
+        background: none !important;
+    }
+
+    /* Streamlit elements background */
+    [data-testid="stVerticalBlock"] {
+        background-color: transparent !important;
+    }
+
+    [data-testid="stVerticalBlockBorderWrapper"] {
+        background-color: transparent !important;
+    }
+
+    /* Remove white decorative elements */
+    .st-emotion-cache-1gvbgyg {
+        background-color: var(--bg-dark) !important;
+    }
+
+    .st-emotion-cache-1jicfl2 {
+        background-color: var(--bg-dark) !important;
+    }
+
+    /* Ensure all root divs are dark */
+    div[class*="st-"] {
+        background-color: transparent !important;
+    }
+
+    /* Modal and overlay backgrounds */
+    .stModal {
+        background-color: var(--bg-dark) !important;
+    }
+
+    /* Alert boxes background */
+    .stAlert {
+        background-color: rgba(0, 102, 255, 0.1) !important;
+    }
+    </style>
+"""

app/ui.py

@@ -0,0 +1,167 @@
+"""UI component functions for the financial dashboard."""
+
+import streamlit as st
+import pandas as pd
+from data import format_financial_value, get_profitability_metrics
+
+
+def display_price_metrics(metrics: dict):
+    """Display key price metrics in columns."""
+    st.markdown('<div class="section-title">📊 Price Metrics</div>', unsafe_allow_html=True)
+
+    col1, col2, col3, col4 = st.columns(4)
+
+    with col1:
+        st.metric("Current Price", f"${metrics['current_price']:.2f}",
+                 f"{metrics['price_change']:+.2f}", delta_color="normal")
+
+    with col2:
+        st.metric("Day Change %", f"{metrics['price_change_pct']:+.2f}%",
+                 None, delta_color="normal")
+
+    with col3:
+        st.metric("52W High", f"${metrics['high_52w']:.2f}")
+
+    with col4:
+        st.metric("52W Low", f"${metrics['low_52w']:.2f}")
+
+
+def display_company_info(profile_info):
+    """Display company information."""
+    st.markdown('<div class="section-title">📋 Company Information</div>', unsafe_allow_html=True)
+
+    if profile_info:
+        info_col1, info_col2 = st.columns(2)
+        with info_col1:
+            st.write(f"**Company Name:** {getattr(profile_info, 'name', 'N/A')}")
+            st.write(f"**Sector:** {getattr(profile_info, 'sector', 'N/A')}")
+            st.write(f"**Industry:** {getattr(profile_info, 'industry', 'N/A')}")
+
+        with info_col2:
+            st.write(f"**Country:** {getattr(profile_info, 'country', 'N/A')}")
+            st.write(f"**Exchange:** {getattr(profile_info, 'exchange', 'N/A')}")
+            st.write(f"**Website:** {getattr(profile_info, 'website', 'N/A')}")
+
+
+def display_financial_metrics(income_stmt: pd.DataFrame):
+    """Display financial metrics from income statement."""
+    st.markdown('<div class="section-title">💰 Financial Metrics</div>', unsafe_allow_html=True)
+
+    latest_income = income_stmt.iloc[0] if len(income_stmt) > 0 else None
+
+    if latest_income is not None:
+        # First row of metrics
+        fin_col1, fin_col2, fin_col3, fin_col4 = st.columns(4)
+
+        with fin_col1:
+            revenue = latest_income.get('total_revenue', 0)
+            if pd.notna(revenue) and revenue > 0:
+                st.metric("Total Revenue", format_financial_value(revenue))
+            else:
+                st.metric("Total Revenue", "N/A")
+
+        with fin_col2:
+            net_income = latest_income.get('net_income', 0)
+            if pd.notna(net_income) and net_income > 0:
+                st.metric("Net Income", format_financial_value(net_income))
+            else:
+                st.metric("Net Income", "N/A")
+
+        with fin_col3:
+            gross_profit = latest_income.get('gross_profit', 0)
+            if pd.notna(gross_profit) and gross_profit > 0:
+                st.metric("Gross Profit", format_financial_value(gross_profit))
+            else:
+                st.metric("Gross Profit", "N/A")
+
+        with fin_col4:
+            operating_income = latest_income.get('operating_income', 0)
+            if pd.notna(operating_income) and operating_income > 0:
+                st.metric("Operating Income", format_financial_value(operating_income))
+            else:
+                st.metric("Operating Income", "N/A")
+
+        # Second row of metrics
+        fin_col5, fin_col6, fin_col7, fin_col8 = st.columns(4)
+
+        with fin_col5:
+            eps = latest_income.get('diluted_earnings_per_share', 0)
+            if pd.notna(eps):
+                st.metric("EPS (Diluted)", f"${eps:.2f}")
+            else:
+                st.metric("EPS (Diluted)", "N/A")
+
+        with fin_col6:
+            ebitda = latest_income.get('ebitda', 0)
+            if pd.notna(ebitda) and ebitda > 0:
+                st.metric("EBITDA", format_financial_value(ebitda))
+            else:
+                st.metric("EBITDA", "N/A")
+
+        with fin_col7:
+            cogs = latest_income.get('cost_of_revenue', 0)
+            if pd.notna(cogs) and cogs > 0:
+                st.metric("Cost of Revenue", format_financial_value(cogs))
+            else:
+                st.metric("Cost of Revenue", "N/A")
+
+        with fin_col8:
+            rd_expense = latest_income.get('research_and_development_expense', 0)
+            if pd.notna(rd_expense) and rd_expense > 0:
+                st.metric("R&D Expense", format_financial_value(rd_expense))
+            else:
+                st.metric("R&D Expense", "N/A")
+
+
+def display_income_statement(income_stmt: pd.DataFrame):
+    """Display formatted income statement table."""
+    st.markdown("### Income Statement")
+
+    if not income_stmt.empty:
+        display_columns = [
+            'period_ending',
+            'total_revenue',
+            'cost_of_revenue',
+            'gross_profit',
+            'operating_income',
+            'net_income',
+            'diluted_earnings_per_share',
+            'ebitda'
+        ]
+
+        available_cols = [col for col in display_columns if col in income_stmt.columns]
+        financial_display = income_stmt[available_cols].copy()
+
+        for col in financial_display.columns:
+            if col != 'period_ending':
+                financial_display[col] = financial_display[col].apply(
+                    lambda x: format_financial_value(x)
+                )
+
+        st.dataframe(financial_display, use_container_width=True, hide_index=True)
+
+
+def display_profitability_metrics(income_stmt: pd.DataFrame):
+    """Display profitability metrics."""
+    st.markdown("### Profitability Metrics")
+
+    prof_col1, prof_col2 = st.columns(2)
+    latest_data = income_stmt.iloc[0]
+    metrics = get_profitability_metrics(latest_data)
+
+    with prof_col1:
+        if "gross_margin" in metrics:
+            st.metric("Gross Margin", f"{metrics['gross_margin']:.2f}%")
+        if "net_margin" in metrics:
+            st.metric("Net Profit Margin", f"{metrics['net_margin']:.2f}%")
+
+    with prof_col2:
+        if "operating_margin" in metrics:
+            st.metric("Operating Margin", f"{metrics['operating_margin']:.2f}%")
+
+        if len(income_stmt) > 1:
+            prev_revenue = income_stmt.iloc[1].get('total_revenue', 0)
+            total_rev = latest_data.get('total_revenue', 0)
+            if prev_revenue and prev_revenue > 0:
+                revenue_growth = ((total_rev - prev_revenue) / prev_revenue) * 100
+                st.metric("Revenue Growth (YoY)", f"{revenue_growth:+.2f}%")

app/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities package for financial platform."""

app/utils/ai_summary_cache.py

@@ -0,0 +1,141 @@
+"""Shared in-memory AI summary cache with buffering and batching."""
+
+import os
+import threading
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional, Tuple
+
+from utils.llm_summarizer import OpenAICompatSummarizer
+
+# Approx 4 chars per token -> 600 tokens ~= 2400 chars
+DEFAULT_BATCH_MAX_CHARS = int(os.getenv("LLM_SUMMARY_BATCH_MAX_CHARS", "2400"))
+BUFFER_SECONDS = int(os.getenv("LLM_SUMMARY_BUFFER_SECONDS", "120"))
+
+
+class AISummaryCache:
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._buffer: List[Dict] = []
+        self._buffer_start: Optional[datetime] = None
+        self._summaries: Dict[str, Dict] = {}
+        self._last_update: Optional[datetime] = None
+
+    def buffer_items(self, items: List[Dict]):
+        if not items:
+            return
+        with self._lock:
+            for item in items:
+                key = self._item_key(item)
+                if not key or key in self._summaries:
+                    continue
+                self._buffer.append(item)
+            if self._buffer and self._buffer_start is None:
+                self._buffer_start = datetime.now()
+
+    def maybe_flush(self):
+        with self._lock:
+            if not self._buffer or self._buffer_start is None:
+                return
+            if datetime.now() - self._buffer_start < timedelta(seconds=BUFFER_SECONDS):
+                return
+            items = self._buffer
+            self._buffer = []
+            self._buffer_start = None
+
+        summarizer = OpenAICompatSummarizer()
+        if not summarizer.enabled:
+            return
+
+        batches = self._batch_items(items, DEFAULT_BATCH_MAX_CHARS)
+        for batch in batches:
+            texts = [self._build_input_text(item) for item in batch]
+            texts = [t for t in texts if t]
+            if not texts:
+                continue
+            summaries = summarizer._summarize_chunk(texts, source="dashboard")
+            if not summaries:
+                continue
+            with self._lock:
+                for item, summary in zip(batch, summaries):
+                    key = self._item_key(item)
+                    if not key:
+                        continue
+                    self._summaries[key] = {
+                        "id": item.get("id", key),
+                        "title": item.get("title", ""),
+                        "source": item.get("source", ""),
+                        "summary": summary,
+                        "timestamp": datetime.now(),
+                    }
+                self._last_update = datetime.now()
+
+    def get_summaries(self) -> Tuple[List[Dict], Optional[datetime]]:
+        with self._lock:
+            summaries = list(self._summaries.values())
+            last_update = self._last_update
+        summaries.sort(key=lambda x: x.get("timestamp", datetime.min), reverse=True)
+        return summaries, last_update
+
+    def get_status(self) -> Dict:
+        with self._lock:
+            buffer_size = len(self._buffer)
+            buffer_start = self._buffer_start
+            total_summaries = len(self._summaries)
+            last_update = self._last_update
+        buffer_age_seconds = None
+        buffer_remaining_seconds = None
+        if buffer_start:
+            buffer_age_seconds = (datetime.now() - buffer_start).total_seconds()
+            buffer_remaining_seconds = max(BUFFER_SECONDS - buffer_age_seconds, 0)
+        return {
+            "buffer_size": buffer_size,
+            "buffer_started_at": buffer_start,
+            "buffer_age_seconds": buffer_age_seconds,
+            "buffer_remaining_seconds": buffer_remaining_seconds,
+            "buffer_window_seconds": BUFFER_SECONDS,
+            "total_summaries": total_summaries,
+            "last_update": last_update,
+            "batch_max_chars": DEFAULT_BATCH_MAX_CHARS,
+        }
+
+    def _item_key(self, item: Dict) -> str:
+        if item.get("id") is not None:
+            return str(item.get("id"))
+        title = str(item.get("title", "")).strip()
+        source = str(item.get("source", "")).strip()
+        if not title:
+            return ""
+        return f"{source}|{title}".lower()
+
+    def _build_input_text(self, item: Dict) -> str:
+        title = str(item.get("title", "")).strip()
+        source = str(item.get("source", "")).strip()
+        if not title:
+            return ""
+        if source:
+            return f"Source: {source}\nTitle: {title}"
+        return f"Title: {title}"
+
+    def _batch_items(self, items: List[Dict], max_chars_total: int) -> List[List[Dict]]:
+        if max_chars_total <= 0:
+            return [items]
+        batches: List[List[Dict]] = []
+        current: List[Dict] = []
+        current_chars = 0
+        for item in items:
+            text = self._build_input_text(item)
+            if not text:
+                continue
+            text_len = len(text)
+            if current and current_chars + text_len > max_chars_total:
+                batches.append(current)
+                current = []
+                current_chars = 0
+            current.append(item)
+            current_chars += text_len
+        if current:
+            batches.append(current)
+        return batches
+
+
+ai_summary_cache = AISummaryCache()

app/utils/ai_summary_store.py

@@ -0,0 +1,287 @@
+"""File-backed AI summary buffer and cache with optional HF dataset sync."""
+
+import json
+import os
+import time
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Dict, Iterable, List, Optional, Tuple
+
+try:
+    import fcntl
+except Exception:  # pragma: no cover
+    fcntl = None
+
+try:
+    from huggingface_hub import HfApi, snapshot_download
+except Exception:  # pragma: no cover
+    HfApi = None
+    snapshot_download = None
+
+CACHE_DIR = os.getenv("AI_SUMMARY_CACHE_DIR", "./ai-summary-cache")
+BUFFER_SECONDS = int(os.getenv("LLM_SUMMARY_BUFFER_SECONDS", "120"))
+BATCH_MAX_CHARS = int(os.getenv("LLM_SUMMARY_BATCH_MAX_CHARS", "2400"))
+HF_REPO_ID = os.getenv("AI_SUMMARY_HF_REPO", "ResearchEngineering/ai_news_summaries")
+HF_REPO_TYPE = os.getenv("AI_SUMMARY_HF_REPO_TYPE", "dataset")
+
+BUFFER_FILE = "buffer.jsonl"
+SUMMARIES_FILE = "summaries.jsonl"
+META_FILE = "meta.json"
+LOCK_FILE = ".lock"
+
+
+def init_storage():
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    if snapshot_download and HF_REPO_ID:
+        _maybe_restore_from_hf()
+    _ensure_files()
+
+
+def enqueue_items(items: Iterable[Dict]):
+    init_storage()
+    now = time.time()
+
+    with _file_lock():
+        buffer_items = _read_jsonl(BUFFER_FILE)
+        summaries = _read_jsonl(SUMMARIES_FILE)
+
+        existing_keys = {item.get("item_key") for item in buffer_items if item.get("item_key")}
+        existing_keys.update({item.get("item_key") for item in summaries if item.get("item_key")})
+
+        added = 0
+        for item in items:
+            key = _item_key(item)
+            title = str(item.get("title", "")).strip()
+            if not key or not title or key in existing_keys:
+                continue
+            source = str(item.get("source", "")).strip()
+            buffer_items.append(
+                {
+                    "item_key": key,
+                    "title": title,
+                    "source": source,
+                    "created_at": now,
+                }
+            )
+            existing_keys.add(key)
+            added += 1
+
+        if added:
+            _write_jsonl(BUFFER_FILE, buffer_items)
+
+
+def get_status() -> Dict:
+    init_storage()
+    with _file_lock():
+        buffer_items = _read_jsonl(BUFFER_FILE)
+        summaries = _read_jsonl(SUMMARIES_FILE)
+
+    buffer_count = len(buffer_items)
+    summaries_count = len(summaries)
+    last_update = None
+    if summaries:
+        last_update = max(item.get("updated_at", 0) for item in summaries)
+
+    buffer_oldest = None
+    if buffer_items:
+        buffer_oldest = min(item.get("created_at", 0) for item in buffer_items)
+
+    buffer_remaining = None
+    if buffer_oldest:
+        age = time.time() - buffer_oldest
+        buffer_remaining = max(BUFFER_SECONDS - age, 0)
+
+    last_update_text = (
+        datetime.fromtimestamp(last_update).strftime("%Y-%m-%d %H:%M:%S") if last_update else None
+    )
+
+    return {
+        "buffer_size": buffer_count,
+        "total_summaries": summaries_count,
+        "last_update": last_update_text,
+        "buffer_remaining_seconds": buffer_remaining,
+        "batch_max_chars": BATCH_MAX_CHARS,
+        "buffer_window_seconds": BUFFER_SECONDS,
+    }
+
+
+def fetch_summaries(limit: int = 50) -> List[Dict]:
+    init_storage()
+    with _file_lock():
+        summaries = _read_jsonl(SUMMARIES_FILE)
+
+    summaries.sort(key=lambda x: x.get("updated_at", 0), reverse=True)
+    results = []
+    for item in summaries[:limit]:
+        results.append(
+            {
+                "title": item.get("title", ""),
+                "source": item.get("source", ""),
+                "summary": item.get("summary", ""),
+                "timestamp": datetime.fromtimestamp(item.get("updated_at", time.time())),
+            }
+        )
+    return results
+
+
+def fetch_ready_batches(max_chars_total: int, buffer_seconds: int) -> List[List[Tuple[str, str, str]]]:
+    init_storage()
+    cutoff = time.time() - buffer_seconds
+
+    with _file_lock():
+        buffer_items = _read_jsonl(BUFFER_FILE)
+
+    eligible = [item for item in buffer_items if item.get("created_at", 0) <= cutoff]
+    eligible.sort(key=lambda x: x.get("created_at", 0))
+
+    batches: List[List[Tuple[str, str, str]]] = []
+    current: List[Tuple[str, str, str]] = []
+    current_chars = 0
+
+    for item in eligible:
+        title = item.get("title", "")
+        source = item.get("source", "")
+        text = _build_input_text(title, source)
+        text_len = len(text)
+        if current and current_chars + text_len > max_chars_total:
+            batches.append(current)
+            current = []
+            current_chars = 0
+        current.append((item.get("item_key"), title, source))
+        current_chars += text_len
+
+    if current:
+        batches.append(current)
+
+    return batches
+
+
+def store_summaries(items: List[Tuple[str, str, str, str]]):
+    if not items:
+        return
+
+    init_storage()
+    now = time.time()
+
+    with _file_lock():
+        summaries = _read_jsonl(SUMMARIES_FILE)
+        buffer_items = _read_jsonl(BUFFER_FILE)
+
+        summaries_by_key = {item.get("item_key"): item for item in summaries if item.get("item_key")}
+        buffer_by_key = {item.get("item_key"): item for item in buffer_items if item.get("item_key")}
+
+        for item_key, title, source, summary in items:
+            summaries_by_key[item_key] = {
+                "item_key": item_key,
+                "title": title,
+                "source": source,
+                "summary": summary,
+                "updated_at": now,
+            }
+            if item_key in buffer_by_key:
+                del buffer_by_key[item_key]
+
+        _write_jsonl(SUMMARIES_FILE, list(summaries_by_key.values()))
+        _write_jsonl(BUFFER_FILE, list(buffer_by_key.values()))
+
+    _write_meta({"last_sync": None, "last_update": now})
+    _sync_to_hf_if_configured()
+
+
+def _item_key(item: Dict) -> str:
+    if item.get("id") is not None:
+        return str(item.get("id"))
+    title = str(item.get("title", "")).strip()
+    source = str(item.get("source", "")).strip()
+    if not title:
+        return ""
+    return f"{source}|{title}".lower()
+
+
+def _build_input_text(title: str, source: str) -> str:
+    if source:
+        return f"Source: {source}\nTitle: {title}"
+    return f"Title: {title}"
+
+
+def _ensure_files():
+    for name in (BUFFER_FILE, SUMMARIES_FILE):
+        path = os.path.join(CACHE_DIR, name)
+        if not os.path.exists(path):
+            with open(path, "w", encoding="utf-8") as f:
+                f.write("")
+
+
+def _read_jsonl(filename: str) -> List[Dict]:
+    path = os.path.join(CACHE_DIR, filename)
+    if not os.path.exists(path):
+        return []
+    items = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                items.append(json.loads(line))
+            except Exception:
+                continue
+    return items
+
+
+def _write_jsonl(filename: str, items: List[Dict]):
+    path = os.path.join(CACHE_DIR, filename)
+    tmp_path = path + ".tmp"
+    with open(tmp_path, "w", encoding="utf-8") as f:
+        for item in items:
+            f.write(json.dumps(item, ensure_ascii=True) + "\n")
+    os.replace(tmp_path, path)
+
+
+def _write_meta(data: Dict):
+    path = os.path.join(CACHE_DIR, META_FILE)
+    tmp_path = path + ".tmp"
+    with open(tmp_path, "w", encoding="utf-8") as f:
+        json.dump(data, f)
+    os.replace(tmp_path, path)
+
+
+@contextmanager
+def _file_lock():
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    lock_path = os.path.join(CACHE_DIR, LOCK_FILE)
+    if fcntl is None:
+        yield
+        return
+    with open(lock_path, "w", encoding="utf-8") as lock_file:
+        fcntl.flock(lock_file, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(lock_file, fcntl.LOCK_UN)
+
+
+def _maybe_restore_from_hf():
+    if not snapshot_download:
+        return
+    if not HF_REPO_ID:
+        return
+    if os.path.exists(os.path.join(CACHE_DIR, SUMMARIES_FILE)):
+        return
+    snapshot_download(
+        repo_id=HF_REPO_ID,
+        repo_type=HF_REPO_TYPE,
+        local_dir=CACHE_DIR,
+        local_dir_use_symlinks=False,
+    )
+
+
+def _sync_to_hf_if_configured():
+    if not HfApi or not HF_REPO_ID:
+        return
+    api = HfApi()
+    api.upload_folder(
+        folder_path=CACHE_DIR,
+        repo_id=HF_REPO_ID,
+        repo_type=HF_REPO_TYPE,
+    )

app/utils/ai_summary_worker.py

@@ -0,0 +1,109 @@
+"""Background worker process for AI summarization."""
+
+import os
+import time
+import logging
+import signal
+import sqlite3
+from typing import List, Tuple
+
+from utils.llm_summarizer import OpenAICompatSummarizer
+from utils.ai_summary_store import (
+    init_storage,
+    fetch_ready_batches,
+    store_summaries,
+    BATCH_MAX_CHARS,
+    BUFFER_SECONDS,
+)
+
+logger = logging.getLogger(__name__)
+
+PID_FILE = os.getenv("AI_SUMMARY_WORKER_PID", "/tmp/ai_summary_worker.pid")
+POLL_SECONDS = int(os.getenv("AI_SUMMARY_POLL_SECONDS", "5"))
+MAX_RETRIES = int(os.getenv("LLM_SUMMARY_RETRIES", "3"))
+
+
+class Worker:
+    def __init__(self):
+        self._stop = False
+        self.summarizer = OpenAICompatSummarizer()
+
+    def stop(self, *_args):
+        self._stop = True
+
+    def run(self):
+        init_storage()
+        signal.signal(signal.SIGTERM, self.stop)
+        signal.signal(signal.SIGINT, self.stop)
+
+        while not self._stop:
+            try:
+                batches = fetch_ready_batches(BATCH_MAX_CHARS, BUFFER_SECONDS)
+                for batch in batches:
+                    self._process_batch(batch)
+            except sqlite3.Error as exc:
+                logger.warning(f"AI worker DB error: {exc}")
+            except Exception as exc:
+                logger.warning(f"AI worker error: {exc}")
+
+            time.sleep(POLL_SECONDS)
+
+    def _process_batch(self, batch: List[Tuple[str, str, str]]):
+        if not batch or not self.summarizer.enabled:
+            return
+
+        texts = []
+        for _, title, source in batch:
+            if source:
+                texts.append(f"Source: {source}\nTitle: {title}")
+            else:
+                texts.append(f"Title: {title}")
+
+        for attempt in range(1, MAX_RETRIES + 1):
+            summaries = self.summarizer._summarize_chunk(texts, source="dashboard")
+            if summaries and len(summaries) == len(batch):
+                break
+            if attempt < MAX_RETRIES:
+                time.sleep(2 ** attempt)
+        else:
+            logger.warning("AI worker failed to summarize batch after retries")
+            return
+
+        to_store = []
+        for (item_key, title, source), summary in zip(batch, summaries):
+            if not summary:
+                continue
+            to_store.append((item_key, title, source, summary))
+
+        if to_store:
+            store_summaries(to_store)
+
+
+def _pid_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+        return True
+    except Exception:
+        return False
+
+
+def start_worker_if_needed():
+    if os.path.exists(PID_FILE):
+        try:
+            with open(PID_FILE, "r", encoding="utf-8") as f:
+                pid = int(f.read().strip() or 0)
+            if pid and _pid_running(pid):
+                return
+        except Exception:
+            pass
+
+    pid = os.fork()
+    if pid != 0:
+        return
+
+    os.setsid()
+    with open(PID_FILE, "w", encoding="utf-8") as f:
+        f.write(str(os.getpid()))
+
+    worker = Worker()
+    worker.run()

app/utils/breaking_news_scorer.py

@@ -0,0 +1,368 @@
+"""
+Breaking News Scoring System
+Identifies highest-impact financial news using multi-factor weighted scoring
+"""
+
+import re
+from datetime import datetime, timedelta
+from typing import Dict, List
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class BreakingNewsScorer:
+    """
+    Sophisticated scoring system for breaking financial news
+    Uses weighted factors to identify market-moving events
+    """
+
+    # Critical keywords with high market impact (weight: 3.0)
+    CRITICAL_KEYWORDS = [
+        # Central Bank Actions
+        'rate hike', 'rate cut', 'interest rate', 'fed raises', 'fed cuts',
+        'fomc decision', 'monetary policy', 'quantitative easing', 'qe',
+        'emergency meeting', 'powell', 'lagarde', 'yellen',
+
+        # Market Events
+        'market crash', 'flash crash', 'circuit breaker', 'trading halt',
+        'all-time high', 'all time high', 'record high', 'record low',
+        'biggest drop', 'biggest gain', 'historic', 'unprecedented',
+
+        # Economic Data
+        'gdp', 'jobs report', 'unemployment', 'inflation',
+        'cpi', 'ppi', 'nonfarm payroll', 'nfp',
+
+        # Corporate Events
+        'earnings beat', 'earnings miss', 'profit warning',
+        'bankruptcy', 'chapter 11', 'delisted',
+        'merger', 'acquisition', 'takeover', 'buyout',
+
+        # Geopolitical
+        'war', 'invasion', 'sanctions', 'trade war',
+        'embargo', 'default', 'debt ceiling', 'shutdown',
+        'impeachment', 'coup', 'terrorist attack'
+    ]
+
+    # High-impact keywords (weight: 2.0)
+    HIGH_IMPACT_KEYWORDS = [
+        # Market Movement
+        'surge', 'plunge', 'soar', 'tumble', 'rally', 'selloff',
+        'volatility', 'whipsaw', 'correction', 'bear market', 'bull market',
+
+        # Economic Indicators
+        'retail sales', 'housing starts', 'consumer confidence',
+        'manufacturing index', 'pmi', 'trade deficit',
+
+        # Corporate
+        'revenue beat', 'guidance', 'dividend', 'stock split',
+        'ipo', 'listing', 'secondary offering',
+
+        # Crypto/Tech
+        'bitcoin', 'crypto crash', 'hack', 'breach',
+        'antitrust', 'regulation', 'sec investigation',
+
+        # Commodities
+        'oil', 'gold', 'crude', 'opec', 'energy crisis',
+        'supply chain', 'shortage', 'surplus'
+    ]
+
+    # Medium-impact keywords (weight: 1.5)
+    MEDIUM_IMPACT_KEYWORDS = [
+        'analyst', 'upgrade', 'downgrade', 'target price',
+        'forecast', 'outlook', 'projection', 'estimate',
+        'conference call', 'ceo', 'cfo', 'executive',
+        'lawsuit', 'settlement', 'fine', 'penalty',
+        'product launch', 'partnership', 'deal', 'contract'
+    ]
+
+    # Premium source weights (multipliers)
+    SOURCE_WEIGHTS = {
+        # Tier 1: Breaking News Specialists (2.0x)
+        'walter_bloomberg': 2.0,
+        'fxhedge': 2.0,
+        'deitaone': 2.0,
+        'firstsquawk': 1.9,
+        'livesquawk': 1.9,
+
+        # Tier 2: Major Financial Media (1.8x)
+        'reuters': 1.8,
+        'bloomberg': 1.8,
+        'ft': 1.7,
+        'wsj': 1.7,
+
+        # Tier 3: Mainstream Media (1.5x)
+        'cnbc': 1.5,
+        'bbc': 1.5,
+        'marketwatch': 1.5,
+
+        # Tier 4: Alternative/Community (1.2x)
+        'zerohedge': 1.2,
+        'wallstreetbets': 1.2,
+        'reddit': 1.2,
+
+        # Default
+        'default': 1.0
+    }
+
+    # Ticker mention bonus (companies that move markets)
+    MAJOR_TICKERS = [
+        'SPY', 'QQQ', 'DIA', 'IWM',  # Market indices
+        'AAPL', 'MSFT', 'GOOGL', 'AMZN', 'NVDA', 'TSLA', 'META',  # Mega caps
+        'JPM', 'BAC', 'GS', 'MS', 'WFC',  # Banks
+        'XOM', 'CVX', 'COP',  # Energy
+        'BTC', 'ETH', 'BTCUSD', 'ETHUSD'  # Crypto
+    ]
+
+    def __init__(self):
+        """Initialize the breaking news scorer"""
+        logger.info("BreakingNewsScorer initialized")
+
+    def calculate_impact_score(self, news_item: Dict) -> float:
+        """
+        Calculate comprehensive impact score for a news item
+
+        Args:
+            news_item: Dictionary containing news metadata
+
+        Returns:
+            Impact score (0-100, higher = more impactful)
+        """
+        score = 0.0
+
+        # Extract key fields
+        title = news_item.get('title', '').lower()
+        summary = news_item.get('summary', '').lower()
+        source = news_item.get('source', '').lower()
+        timestamp = news_item.get('timestamp', datetime.now())
+        sentiment = news_item.get('sentiment', 'neutral')
+        impact_level = news_item.get('impact', 'low')
+        category = news_item.get('category', 'markets')
+
+        # Combine title and summary for keyword analysis
+        text = f"{title} {summary}"
+
+        # 1. KEYWORD SCORING (30 points max)
+        keyword_score = self._score_keywords(text)
+        score += keyword_score
+
+        # 2. RECENCY SCORING (20 points max)
+        recency_score = self._score_recency(timestamp)
+        score += recency_score
+
+        # 3. SOURCE CREDIBILITY (20 points max)
+        source_score = self._score_source(source)
+        score += source_score
+
+        # 4. ENGAGEMENT SCORING (15 points max)
+        engagement_score = self._score_engagement(news_item)
+        score += engagement_score
+
+        # 5. SENTIMENT EXTREMITY (10 points max)
+        sentiment_score = self._score_sentiment(sentiment)
+        score += sentiment_score
+
+        # 6. CATEGORY RELEVANCE (5 points max)
+        category_score = self._score_category(category)
+        score += category_score
+
+        # 7. TICKER MENTIONS (bonus up to 10 points)
+        ticker_score = self._score_tickers(text)
+        score += ticker_score
+
+        # 8. URGENCY INDICATORS (bonus up to 10 points)
+        urgency_score = self._score_urgency(text)
+        score += urgency_score
+
+        # 9. EXISTING IMPACT LEVEL (weight existing classification)
+        if impact_level == 'high':
+            score *= 1.2
+        elif impact_level == 'medium':
+            score *= 1.1
+
+        # Cap at 100
+        score = min(score, 100.0)
+
+        logger.debug(f"News '{title[:50]}...' scored: {score:.2f}")
+
+        return score
+
+    def _score_keywords(self, text: str) -> float:
+        """Score based on keyword presence and frequency"""
+        score = 0.0
+
+        # Critical keywords (3.0 points each, max 18)
+        critical_matches = sum(1 for kw in self.CRITICAL_KEYWORDS if kw in text)
+        score += min(critical_matches * 3.0, 18.0)
+
+        # High-impact keywords (2.0 points each, max 8)
+        high_matches = sum(1 for kw in self.HIGH_IMPACT_KEYWORDS if kw in text)
+        score += min(high_matches * 2.0, 8.0)
+
+        # Medium-impact keywords (1.0 points each, max 4)
+        medium_matches = sum(1 for kw in self.MEDIUM_IMPACT_KEYWORDS if kw in text)
+        score += min(medium_matches * 1.0, 4.0)
+
+        return min(score, 30.0)
+
+    def _score_recency(self, timestamp: datetime) -> float:
+        """Score based on how recent the news is"""
+        try:
+            if isinstance(timestamp, str):
+                timestamp = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
+
+            age_seconds = (datetime.now() - timestamp.replace(tzinfo=None)).total_seconds()
+            age_minutes = age_seconds / 60
+
+            # Exponential decay: most recent = highest score
+            if age_minutes < 5:
+                return 20.0  # Within 5 minutes: full score
+            elif age_minutes < 15:
+                return 18.0  # 5-15 minutes
+            elif age_minutes < 30:
+                return 15.0  # 15-30 minutes
+            elif age_minutes < 60:
+                return 10.0  # 30-60 minutes
+            elif age_minutes < 180:
+                return 5.0   # 1-3 hours
+            else:
+                return 1.0   # Older than 3 hours
+        except:
+            return 5.0  # Default if timestamp parsing fails
+
+    def _score_source(self, source: str) -> float:
+        """Score based on source credibility"""
+        source = source.lower().replace(' ', '_').replace('/', '').replace('@', '')
+
+        # Check for known sources
+        for source_key, weight in self.SOURCE_WEIGHTS.items():
+            if source_key in source:
+                return weight * 10.0  # Scale to max 20 points
+
+        return self.SOURCE_WEIGHTS['default'] * 10.0
+
+    def _score_engagement(self, news_item: Dict) -> float:
+        """Score based on social engagement metrics"""
+        score = 0.0
+        has_engagement = False
+
+        # Twitter engagement (top-level fields)
+        likes = news_item.get('likes', 0)
+        if likes > 0:
+            has_engagement = True
+            score += min(likes / 1000, 5.0)  # Max 5 points for likes
+
+        retweets = news_item.get('retweets', 0)
+        if retweets > 0:
+            has_engagement = True
+            score += min(retweets / 500, 5.0)  # Max 5 points for retweets
+
+        # Reddit engagement (top-level fields)
+        reddit_score = news_item.get('reddit_score', 0)
+        if reddit_score > 0:
+            has_engagement = True
+            score += min(reddit_score / 1000, 5.0)  # Max 5 points for score
+
+        comments = news_item.get('reddit_comments', 0)
+        if comments > 0:
+            has_engagement = True
+            score += min(comments / 200, 5.0)  # Max 5 points for comments
+
+        # If no engagement data, return default score
+        if not has_engagement:
+            return 5.0
+
+        return min(score, 15.0)
+
+    def _score_sentiment(self, sentiment: str) -> float:
+        """Score based on sentiment extremity (extreme = more impactful)"""
+        if sentiment == 'positive':
+            return 8.0  # Strong positive news moves markets
+        elif sentiment == 'negative':
+            return 10.0  # Negative news tends to have more impact
+        else:
+            return 3.0  # Neutral news less impactful
+
+    def _score_category(self, category: str) -> float:
+        """Score based on category relevance"""
+        if category == 'macro':
+            return 5.0  # Macro news affects entire market
+        elif category == 'markets':
+            return 4.0  # Direct market news
+        elif category == 'geopolitical':
+            return 3.0  # Geopolitical can be high impact
+        else:
+            return 2.0  # Other categories
+
+    def _score_tickers(self, text: str) -> float:
+        """Bonus score for mentioning major market-moving tickers"""
+        text_upper = text.upper()
+
+        # Count major ticker mentions
+        ticker_mentions = sum(1 for ticker in self.MAJOR_TICKERS if ticker in text_upper)
+
+        # 2 points per ticker, max 10 points
+        return min(ticker_mentions * 2.0, 10.0)
+
+    def _score_urgency(self, text: str) -> float:
+        """Bonus score for urgency indicators"""
+        urgency_patterns = [
+            r'\bbreaking\b', r'\balert\b', r'\burgent\b', r'\bjust in\b',
+            r'\bemergency\b', r'\bimmediate\b', r'\bnow\b', r'\btoday\b',
+            r'‼️', r'🚨', r'⚠️', r'🔴', r'❗'
+        ]
+
+        score = 0.0
+        for pattern in urgency_patterns:
+            if re.search(pattern, text, re.IGNORECASE):
+                score += 2.0
+
+        return min(score, 10.0)
+
+    def get_breaking_news(self, news_items: List[Dict], top_n: int = 1) -> List[Dict]:
+        """
+        Identify top breaking news from a list
+
+        Args:
+            news_items: List of news item dictionaries
+            top_n: Number of top items to return
+
+        Returns:
+            List of top breaking news items with scores
+        """
+        if not news_items:
+            return []
+
+        # Calculate scores for all items
+        scored_items = []
+        for item in news_items:
+            score = self.calculate_impact_score(item)
+            scored_items.append({
+                **item,
+                'breaking_score': score
+            })
+
+        # Sort by score (descending)
+        scored_items.sort(key=lambda x: x['breaking_score'], reverse=True)
+
+        # Log top items
+        logger.info(f"Top {top_n} breaking news:")
+        for i, item in enumerate(scored_items[:top_n], 1):
+            logger.info(f"  {i}. [{item['breaking_score']:.1f}] {item['title'][:60]}...")
+
+        return scored_items[:top_n]
+
+    def get_breaking_threshold(self) -> float:
+        """Get minimum score threshold for breaking news display"""
+        return 40.0  # Only show news with score >= 40 (out of 100)
+
+
+# Singleton instance
+_scorer_instance = None
+
+def get_breaking_news_scorer() -> BreakingNewsScorer:
+    """Get singleton instance of BreakingNewsScorer"""
+    global _scorer_instance
+    if _scorer_instance is None:
+        _scorer_instance = BreakingNewsScorer()
+    return _scorer_instance

app/utils/config.py

@@ -0,0 +1,34 @@
+"""Configuration management for the financial dashboard."""
+
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+
+class Config:
+    """Application configuration."""
+
+    # API Keys
+    DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY", "")
+    NEWS_SERVICE_URL = os.getenv("NEWS_SERVICE_URL", "")
+    ALPHA_VANTAGE_KEY = os.getenv("ALPHA_VANTAGE_KEY", "")
+
+    # Cache settings
+    PRICE_DATA_TTL = 3600  # 1 hour
+    FUNDAMENTAL_DATA_TTL = 86400  # 24 hours
+    NEWS_DATA_TTL = 900  # 15 minutes
+
+    # App settings
+    DEFAULT_STOCK_SYMBOL = "AAPL"
+    DEFAULT_CRYPTO_SYMBOL = "BTC/USD"
+    DEFAULT_FOREX_SYMBOL = "EUR/USD"
+    DEFAULT_INDICATOR_PERIOD = 20
+
+    # Data source settings
+    MAX_RETRY_ATTEMPTS = 3
+    REQUEST_TIMEOUT = 30
+
+
+config = Config()

app/utils/formatters.py

@@ -0,0 +1,29 @@
+"""Data formatting utilities for the financial dashboard."""
+
+import pandas as pd
+
+
+def format_financial_value(value) -> str:
+    """Format financial values with appropriate units."""
+    if pd.isna(value):
+        return "N/A"
+    if abs(value) >= 1e9:
+        return f"${value/1e9:.2f}B"
+    elif abs(value) >= 1e6:
+        return f"${value/1e6:.2f}M"
+    else:
+        return f"${value:.2f}"
+
+
+def format_percentage(value: float, decimals: int = 2) -> str:
+    """Format percentage values."""
+    if pd.isna(value):
+        return "N/A"
+    return f"{value:.{decimals}f}%"
+
+
+def format_currency(value: float, decimals: int = 2) -> str:
+    """Format currency values."""
+    if pd.isna(value):
+        return "N/A"
+    return f"${value:,.{decimals}f}"

app/utils/llm_summarizer.py

@@ -0,0 +1,165 @@
+"""OpenAI-compatible LLM summarizer for news items."""
+
+import json
+import logging
+import os
+import time
+from typing import Dict, List, Optional, Tuple
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAICompatSummarizer:
+    """
+    Summarize news items using an OpenAI-compatible chat completions API.
+    """
+
+    def __init__(
+        self,
+        api_base: Optional[str] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        timeout: Optional[int] = None,
+        max_items_per_request: Optional[int] = None,
+        max_chars_per_item: Optional[int] = None,
+        max_chars_total: Optional[int] = None,
+    ):
+        self.api_base = (api_base or os.getenv("LLM_API_BASE") or "https://researchengineering-agi.hf.space").rstrip("/")
+        self.api_key = api_key if api_key is not None else os.getenv("LLM_API_KEY", "")
+        self.model = model or os.getenv("LLM_MODEL", "gpt-4o-mini")
+        self.timeout = timeout or int(os.getenv("LLM_TIMEOUT", "600"))
+        # Conservative defaults to avoid large token bursts on slow servers.
+        self.max_items_per_request = max_items_per_request or int(os.getenv("LLM_SUMMARY_BATCH", "2"))
+        self.max_chars_per_item = max_chars_per_item or int(os.getenv("LLM_SUMMARY_MAX_CHARS", "600"))
+        self.max_chars_total = max_chars_total or int(os.getenv("LLM_SUMMARY_MAX_CHARS_TOTAL", "1200"))
+        self.enabled = os.getenv("ENABLE_AI_SUMMARIZATION", "true").lower() in {"1", "true", "yes"}
+        self.sleep_seconds = float(os.getenv("LLM_SUMMARY_SLEEP_SECONDS", "0"))
+
+        self._chat_url = f"{self.api_base}/v1/chat/completions"
+
+    def summarize_items(self, items: List[Dict], source: Optional[str] = None) -> List[Dict]:
+        if not self.enabled or not items:
+            return items
+
+        candidates: List[Tuple[Dict, str]] = []
+        for item in items:
+            if str(item.get("summary_ai", "")).strip():
+                continue
+            text = self._build_input_text(item)
+            if text:
+                candidates.append((item, text))
+
+        if not candidates:
+            return items
+
+        chunks = self._chunked(candidates, self.max_items_per_request)
+        for idx, chunk in enumerate(chunks, start=1):
+            texts = [text for _, text in chunk]
+            if self.max_chars_total > 0:
+                texts = self._truncate_to_total(texts, self.max_chars_total)
+            summaries = self._summarize_chunk(texts, source=source)
+            if not summaries:
+                continue
+            for (item, _), summary in zip(chunk, summaries):
+                if summary:
+                    item["summary_ai"] = summary
+                    item["summary"] = summary
+            if self.sleep_seconds > 0 and idx < len(chunks):
+                time.sleep(self.sleep_seconds)
+
+        return items
+
+    def _build_input_text(self, item: Dict) -> str:
+        title = str(item.get("title", "")).strip()
+        if title:
+            source = str(item.get("source", "")).strip()
+            if len(title) > self.max_chars_per_item:
+                title = title[: self.max_chars_per_item].rstrip()
+            if source:
+                return f"Source: {source}\nTitle: {title}"
+            return f"Title: {title}"
+        return ""
+
+    def _summarize_chunk(self, texts: List[str], source: Optional[str] = None) -> List[str]:
+        system_prompt = (
+            "You are a financial news summarizer. "
+            "Return concise, factual summaries in 1-2 sentences, <=240 characters each. "
+            "Do not add speculation or new facts."
+        )
+        source_line = f"Source: {source}" if source else ""
+
+        items_text = []
+        for idx, text in enumerate(texts, start=1):
+            items_text.append(f"{idx}. {text}")
+
+        user_prompt = (
+            "Summarize each item below. "
+            "Return a JSON array of strings in the same order. "
+            "No extra text.\n"
+            f"{source_line}\n\n" + "\n\n".join(items_text)
+        )
+
+        payload = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "temperature": 0.2,
+        }
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        try:
+            response = requests.post(self._chat_url, json=payload, headers=headers, timeout=self.timeout)
+            response.raise_for_status()
+            data = response.json()
+            content = (
+                data.get("choices", [{}])[0]
+                .get("message", {})
+                .get("content", "")
+                .strip()
+            )
+            summaries = self._parse_json_array(content)
+            if summaries and len(summaries) == len(texts):
+                return summaries
+            logger.warning("LLM summarizer returned unexpected format or length")
+            return []
+        except Exception as exc:
+            logger.warning(f"LLM summarization failed: {exc}")
+            return []
+
+    def _parse_json_array(self, content: str) -> List[str]:
+        if not content:
+            return []
+        try:
+            parsed = json.loads(content)
+            if isinstance(parsed, list):
+                return [str(x).strip() for x in parsed]
+            return []
+        except Exception:
+            return []
+
+    def _chunked(self, items: List[Tuple[Dict, str]], size: int) -> List[List[Tuple[Dict, str]]]:
+        if size <= 0:
+            return [items]
+        return [items[i : i + size] for i in range(0, len(items), size)]
+
+    def _truncate_to_total(self, texts: List[str], max_total: int) -> List[str]:
+        if max_total <= 0:
+            return texts
+        truncated = []
+        total = 0
+        for text in texts:
+            if total >= max_total:
+                break
+            remaining = max_total - total
+            if len(text) > remaining:
+                text = text[:remaining].rstrip()
+            truncated.append(text)
+            total += len(text)
+        return truncated

app/utils/news_cache.py

@@ -0,0 +1,391 @@
+"""
+Unified News Caching System
+Centralized cache manager for Twitter, Reddit, RSS, and AI/Tech news feeds
+"""
+
+import hashlib
+import logging
+import re
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional, Callable
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+class NewsCacheManager:
+    """
+    Centralized cache manager for news feeds with:
+    - Per-source caching with TTL
+    - Cross-service deduplication
+    - Filtered results caching
+    - Force refresh support
+    """
+
+    def __init__(self, default_ttl: int = 180):
+        """
+        Initialize cache manager
+
+        Args:
+            default_ttl: Default time-to-live in seconds (default: 180 = 3 minutes)
+        """
+        self.cache = {
+            'twitter': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'reddit': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'rss': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'ai_tech': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'predictions': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'sectoral_news': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'market_events': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'economic_calendar': {'raw_news': [], 'last_fetch': None, 'ttl': default_ttl},
+            'dedup_index': {},  # Global deduplication index
+            'filtered_cache': {}  # Cached filtered results
+        }
+        logger.info(f"NewsCacheManager initialized with {default_ttl}s TTL")
+
+    def get_news(
+        self,
+        source: str,
+        fetcher_func: Callable,
+        force_refresh: bool = False,
+        deduplicate: bool = False,
+        **kwargs
+    ) -> List[Dict]:
+        """
+        Get news from cache or fetch fresh if needed
+
+        Args:
+            source: News source ('twitter', 'reddit', 'rss', 'ai_tech')
+            fetcher_func: Function to fetch fresh news
+            force_refresh: If True, bypass cache and fetch fresh
+            deduplicate: If True, remove duplicates across sources using global index
+            **kwargs: Arguments to pass to fetcher_func
+
+        Returns:
+            List of news items
+        """
+        if source not in ['twitter', 'reddit', 'rss', 'ai_tech', 'predictions', 'sectoral_news', 'market_events', 'economic_calendar']:
+            logger.error(f"Invalid source: {source}")
+            return []
+
+        # Force refresh clears dedup index for that source
+        if force_refresh:
+            self._clear_source_from_dedup(source)
+
+        # Check if cache is valid
+        if not force_refresh and self._is_cache_valid(source):
+            logger.info(f"✅ Cache HIT for {source} (age: {self._get_cache_age(source):.1f}s)")
+            return self.cache[source]['raw_news']
+
+        # Cache miss or force refresh - fetch fresh news
+        logger.info(f"🔄 Cache MISS for {source} - fetching fresh news...")
+        try:
+            logger.info(f"📞 Calling fetcher for {source} with kwargs: {kwargs}")
+            new_items = fetcher_func(**kwargs)
+            logger.info(f"📦 Fetcher returned {len(new_items) if new_items else 0} items for {source}")
+
+            if not new_items:
+                logger.warning(f"⚠️ No news items fetched for {source} - returning cached data")
+                # Return cached data if available, even if expired
+                return self.cache[source]['raw_news']
+
+            self._prepare_summaries(new_items)
+
+            # Update cache
+            self._update_cache(source, new_items)
+
+            if deduplicate:
+                deduplicated = self._deduplicate(new_items, source)
+                logger.info(f"✅ Fetched {len(new_items)} items for {source}, {len(deduplicated)} unique after dedup")
+                return deduplicated
+
+            logger.info(f"✅ Fetched {len(new_items)} items for {source} (dedup disabled)")
+            return new_items
+
+        except Exception as e:
+            logger.error(f"Error fetching news for {source}: {e}")
+            # Return cached data if available
+            return self.cache[source]['raw_news']
+
+    def _is_cache_valid(self, source: str) -> bool:
+        """
+        Check if cached data is still fresh
+
+        Args:
+            source: News source to check
+
+        Returns:
+            True if cache is valid, False otherwise
+        """
+        source_cache = self.cache[source]
+        if not source_cache['last_fetch']:
+            return False
+
+        age = (datetime.now() - source_cache['last_fetch']).total_seconds()
+        is_valid = age < source_cache['ttl']
+
+        return is_valid
+
+    def _get_cache_age(self, source: str) -> float:
+        """
+        Get age of cached data in seconds
+
+        Args:
+            source: News source
+
+        Returns:
+            Age in seconds, or -1 if never fetched
+        """
+        source_cache = self.cache[source]
+        if not source_cache['last_fetch']:
+            return -1
+
+        return (datetime.now() - source_cache['last_fetch']).total_seconds()
+
+    def _normalize_text(self, text: str) -> str:
+        """
+        Normalize text for deduplication
+
+        Args:
+            text: Text to normalize
+
+        Returns:
+            Normalized text
+        """
+        if not text:
+            return ""
+
+        # Convert to lowercase
+        text = text.lower().strip()
+
+        # Remove punctuation
+        text = re.sub(r'[^\w\s]', '', text)
+
+        # Normalize whitespace
+        text = re.sub(r'\s+', ' ', text)
+
+        return text
+
+    def _compute_hash(self, item: Dict) -> str:
+        """
+        Compute content hash for deduplication
+
+        Args:
+            item: News item dict
+
+        Returns:
+            MD5 hash string
+        """
+        title = self._normalize_text(item.get('title', ''))
+        summary_source = item.get('summary_raw', item.get('summary', ''))
+        summary = self._normalize_text(str(summary_source)[:200])  # First 200 chars
+
+        # Combine title and summary
+        combined = f"{title}|{summary}"
+
+        return hashlib.md5(combined.encode()).hexdigest()
+
+    def _deduplicate(self, items: List[Dict], source: str) -> List[Dict]:
+        """
+        Remove duplicates using global dedup index
+
+        Args:
+            items: List of news items
+            source: Source name
+
+        Returns:
+            Deduplicated list of items
+        """
+        deduplicated = []
+        duplicate_count = 0
+
+        for item in items:
+            content_hash = self._compute_hash(item)
+
+            if content_hash in self.cache['dedup_index']:
+                # Duplicate found - update sources list
+                dup_entry = self.cache['dedup_index'][content_hash]
+                if source not in dup_entry['sources']:
+                    dup_entry['sources'].append(source)
+                duplicate_count += 1
+            else:
+                # New item - add to index and result
+                self.cache['dedup_index'][content_hash] = {
+                    'first_seen': datetime.now(),
+                    'sources': [source],
+                    'canonical_item': item
+                }
+                deduplicated.append(item)
+
+        if duplicate_count > 0:
+            logger.info(f"🔍 Deduplication: Found {duplicate_count} duplicates for {source}")
+
+        return deduplicated
+
+    def _update_cache(self, source: str, items: List[Dict]):
+        """
+        Update cache with new items
+
+        Args:
+            source: News source
+            items: List of news items
+        """
+        self.cache[source]['raw_news'] = items
+        self.cache[source]['last_fetch'] = datetime.now()
+        logger.info(f"📦 Updated cache for {source} with {len(items)} items")
+
+    def _prepare_summaries(self, items: List[Dict]):
+        for item in items:
+            if 'summary_raw' not in item:
+                item['summary_raw'] = item.get('summary', '')
+
+    def get_filtered_news(
+        self,
+        source_df: pd.DataFrame,
+        filters: Dict,
+        source_name: str = "unknown"
+    ) -> pd.DataFrame:
+        """
+        Get filtered news with caching
+
+        Args:
+            source_df: Source dataframe
+            filters: Filter dict with 'category', 'sentiment', 'impact' keys
+            source_name: Name of source (for logging)
+
+        Returns:
+            Filtered dataframe
+        """
+        if source_df.empty:
+            return source_df
+
+        # Create cache key from filters
+        category = filters.get('category', 'all')
+        sentiment = filters.get('sentiment', 'all')
+        impact = filters.get('impact', 'all')
+        cache_key = f"{source_name}_{category}_{sentiment}_{impact}"
+
+        # Check filtered cache
+        if cache_key in self.cache['filtered_cache']:
+            cached_entry = self.cache['filtered_cache'][cache_key]
+            if datetime.now() < cached_entry['expires_at']:
+                logger.debug(f"✅ Filtered cache HIT for {cache_key}")
+                return cached_entry['results']
+
+        # Apply filters
+        filtered_df = source_df.copy()
+
+        if category != 'all':
+            filtered_df = filtered_df[filtered_df['category'] == category]
+
+        if sentiment != 'all':
+            filtered_df = filtered_df[filtered_df['sentiment'] == sentiment]
+
+        if impact != 'all':
+            filtered_df = filtered_df[filtered_df['impact'] == impact]
+
+        logger.debug(f"🔍 Filtered {source_name}: {len(source_df)} → {len(filtered_df)} items")
+
+        # Cache filtered results (5 minute TTL)
+        self.cache['filtered_cache'][cache_key] = {
+            'results': filtered_df,
+            'expires_at': datetime.now() + timedelta(seconds=300)
+        }
+
+        return filtered_df
+
+    def _clear_source_from_dedup(self, source: str):
+        """
+        Remove all entries from dedup index that only belong to this source
+
+        Args:
+            source: Source to remove from dedup index
+        """
+        to_remove = []
+        for content_hash, entry in self.cache['dedup_index'].items():
+            # Remove source from sources list
+            if source in entry['sources']:
+                entry['sources'].remove(source)
+            # If no sources left, mark for removal
+            if not entry['sources']:
+                to_remove.append(content_hash)
+
+        # Remove entries with no sources
+        for content_hash in to_remove:
+            del self.cache['dedup_index'][content_hash]
+
+        if to_remove:
+            logger.info(f"🗑️  Removed {len(to_remove)} entries from dedup index for {source}")
+
+    def clear_cache(self, source: Optional[str] = None):
+        """
+        Clear cache for specific source or all sources
+
+        Args:
+            source: Source to clear, or None to clear all
+        """
+        if source:
+            self.cache[source] = {'raw_news': [], 'last_fetch': None, 'ttl': 180}
+            self._clear_source_from_dedup(source)
+            logger.info(f"🗑️  Cleared cache for {source}")
+        else:
+            for src in ['twitter', 'reddit', 'rss', 'ai_tech', 'predictions', 'sectoral_news', 'market_events', 'economic_calendar']:
+                self.cache[src] = {'raw_news': [], 'last_fetch': None, 'ttl': 180}
+            self.cache['dedup_index'] = {}
+            self.cache['filtered_cache'] = {}
+            logger.info("🗑️  Cleared ALL caches")
+
+    def get_statistics(self) -> Dict:
+        """
+        Get cache statistics
+
+        Returns:
+            Dict with cache stats
+        """
+        stats = {
+            'twitter': {
+                'items': len(self.cache['twitter']['raw_news']),
+                'age_seconds': self._get_cache_age('twitter'),
+                'is_valid': self._is_cache_valid('twitter')
+            },
+            'reddit': {
+                'items': len(self.cache['reddit']['raw_news']),
+                'age_seconds': self._get_cache_age('reddit'),
+                'is_valid': self._is_cache_valid('reddit')
+            },
+            'rss': {
+                'items': len(self.cache['rss']['raw_news']),
+                'age_seconds': self._get_cache_age('rss'),
+                'is_valid': self._is_cache_valid('rss')
+            },
+            'ai_tech': {
+                'items': len(self.cache['ai_tech']['raw_news']),
+                'age_seconds': self._get_cache_age('ai_tech'),
+                'is_valid': self._is_cache_valid('ai_tech')
+            },
+            'predictions': {
+                'items': len(self.cache['predictions']['raw_news']),
+                'age_seconds': self._get_cache_age('predictions'),
+                'is_valid': self._is_cache_valid('predictions')
+            },
+            'sectoral_news': {
+                'items': len(self.cache['sectoral_news']['raw_news']),
+                'age_seconds': self._get_cache_age('sectoral_news'),
+                'is_valid': self._is_cache_valid('sectoral_news')
+            },
+            'market_events': {
+                'items': len(self.cache['market_events']['raw_news']),
+                'age_seconds': self._get_cache_age('market_events'),
+                'is_valid': self._is_cache_valid('market_events')
+            },
+            'economic_calendar': {
+                'items': len(self.cache['economic_calendar']['raw_news']),
+                'age_seconds': self._get_cache_age('economic_calendar'),
+                'is_valid': self._is_cache_valid('economic_calendar')
+            },
+            'dedup_index_size': len(self.cache['dedup_index']),
+            'filtered_cache_size': len(self.cache['filtered_cache'])
+        }
+
+        return stats

requirements.txt

@@ -0,0 +1,13 @@
+streamlit>=1.30.0
+pandas>=2.0.0
+plotly>=5.18.0
+openbb>=4.0.0
+python-dotenv>=1.0.0
+requests>=2.31.0
+twikit>=2.3.0
+feedparser>=6.0.0
+beautifulsoup4>=4.12.0
+lxml>=5.0.0
+ntscraper
+playwright>=1.40.0
+huggingface_hub>=0.22.2