add news scrapper

app/pages/05_Dashboard.py

@@ -18,13 +18,17 @@ from components.news import (
     display_breaking_news_banner
 )
 
-# Try to import Twikit version first, fall back to old version
+# Try to import RSS scraper first (most reliable), fall back to Twikit, then old snscrape
 try:
-    from services.news_monitor_twikit import FinanceNewsMonitor
-    USING_TWIKIT = True
+    from services.news_scraper import FinanceNewsScraper as FinanceNewsMonitor
+    NEWS_SOURCE = "RSS Feeds"
 except ImportError:
-    from services.news_monitor import FinanceNewsMonitor
-    USING_TWIKIT = False
+    try:
+        from services.news_monitor_twikit import FinanceNewsMonitor
+        NEWS_SOURCE = "Twikit"
+    except ImportError:
+        from services.news_monitor import FinanceNewsMonitor
+        NEWS_SOURCE = "snscrape"
 
 
 # ---- Page Configuration ----
@@ -210,7 +214,7 @@ st.markdown("""
 - Breaking news (🔴) indicates urgent market-moving information
 - Check engagement metrics (likes + retweets) for news importance
 
-**Data Source:** Live tweets from premium financial news sources via Twikit
+**Data Source:** Dual-mode scraping - RSS feeds + direct web page parsing from Reuters, Bloomberg, FT, WSJ, CNBC, Fed, ECB and more
 **Update Frequency:** 3-minute cache for low-latency delivery
-**Authentication:** Requires Twitter/X account credentials in .env file
+**No Authentication Required:** Public sources - works out of the box
 """)

app/services/news_scraper.py

@@ -0,0 +1,528 @@
+"""
+Professional Finance News Scraper - Direct from Source Websites
+Scrapes: Reuters, Bloomberg, FT, WSJ, CNBC, MarketWatch, etc.
+No Twitter API needed - direct RSS and web scraping
+"""
+
+import pandas as pd
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional
+import streamlit as st
+import logging
+import re
+import feedparser
+import requests
+from bs4 import BeautifulSoup
+from concurrent.futures import ThreadPoolExecutor
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class FinanceNewsScraper:
+    """
+    Professional-grade financial news scraper using RSS feeds and web scraping
+    No authentication required - publicly available sources
+    """
+
+    # News sources with RSS feeds and web scraping endpoints
+    SOURCES = {
+        # ===== TIER 1: Major Financial News (RSS + Web Scraping) =====
+        'reuters_business': {
+            'name': 'Reuters Business',
+            'rss': 'https://www.reutersagency.com/feed/?taxonomy=best-topics&post_type=best',
+            'web': 'https://www.reuters.com/business/',
+            'selectors': {'headline': 'h3.text__text__1FZLe', 'link': 'a.text__text__1FZLe'},
+            'weight': 1.5,
+            'specialization': ['macro', 'markets']
+        },
+        'reuters_markets': {
+            'name': 'Reuters Markets',
+            'rss': 'https://www.reutersagency.com/feed/?best-sectors=business-finance&post_type=best',
+            'web': 'https://www.reuters.com/markets/',
+            'selectors': {'headline': 'h3', 'link': 'a[data-testid="Heading"]'},
+            'weight': 1.5,
+            'specialization': ['markets']
+        },
+        'cnbc': {
+            'name': 'CNBC',
+            'rss': 'https://www.cnbc.com/id/100003114/device/rss/rss.html',
+            'web': 'https://www.cnbc.com/world/',
+            'selectors': {'headline': 'a.Card-title', 'link': 'a.Card-title'},
+            'weight': 1.2,
+            'specialization': ['markets']
+        },
+        'marketwatch': {
+            'name': 'MarketWatch',
+            'rss': 'https://www.marketwatch.com/rss/topstories',
+            'web': 'https://www.marketwatch.com/',
+            'selectors': {'headline': 'h3.article__headline', 'link': 'a.link'},
+            'weight': 1.1,
+            'specialization': ['markets']
+        },
+        'ft_markets': {
+            'name': 'Financial Times',
+            'rss': 'https://www.ft.com/markets?format=rss',
+            'web': 'https://www.ft.com/markets',
+            'selectors': {'headline': 'div.o-teaser__heading', 'link': 'a.js-teaser-heading-link'},
+            'weight': 1.4,
+            'specialization': ['markets']
+        },
+        'wsj_markets': {
+            'name': 'WSJ Markets',
+            'rss': 'https://feeds.a.dj.com/rss/RSSMarketsMain.xml',
+            'web': 'https://www.wsj.com/news/markets',
+            'selectors': {'headline': 'h3.WSJTheme--headline', 'link': 'a'},
+            'weight': 1.4,
+            'specialization': ['markets']
+        },
+        'economist': {
+            'name': 'The Economist',
+            'rss': 'https://www.economist.com/finance-and-economics/rss.xml',
+            'web': 'https://www.economist.com/finance-and-economics',
+            'selectors': {'headline': 'span._headline', 'link': 'a'},
+            'weight': 1.3,
+            'specialization': ['macro', 'geopolitical']
+        },
+
+        # ===== TIER 2: Geopolitical & Economic =====
+        'bbc_business': {
+            'name': 'BBC Business',
+            'rss': 'http://feeds.bbci.co.uk/news/business/rss.xml',
+            'web': 'https://www.bbc.com/news/business',
+            'selectors': {'headline': 'h3', 'link': 'a[data-testid="internal-link"]'},
+            'weight': 1.4,
+            'specialization': ['geopolitical', 'macro']
+        },
+        'bloomberg_markets': {
+            'name': 'Bloomberg',
+            'rss': 'https://www.bloomberg.com/feed/podcast/etf-report.xml',
+            'web': 'https://www.bloomberg.com/markets',
+            'selectors': {'headline': 'div.single-story-module__headline', 'link': 'a'},
+            'weight': 1.5,
+            'specialization': ['markets']
+        },
+
+        # ===== TIER 3: Central Banks (RSS + Web) =====
+        'federal_reserve': {
+            'name': 'Federal Reserve',
+            'rss': 'https://www.federalreserve.gov/feeds/press_all.xml',
+            'web': 'https://www.federalreserve.gov/newsevents/pressreleases.htm',
+            'selectors': {'headline': 'div.row', 'link': 'a'},
+            'weight': 2.0,
+            'specialization': ['macro']
+        },
+        'ecb': {
+            'name': 'European Central Bank',
+            'rss': 'https://www.ecb.europa.eu/rss/press.xml',
+            'web': 'https://www.ecb.europa.eu/press/pr/date/html/index.en.html',
+            'selectors': {'headline': 'dt', 'link': 'a'},
+            'weight': 2.0,
+            'specialization': ['macro']
+        },
+        'imf': {
+            'name': 'IMF',
+            'rss': 'https://www.imf.org/en/News/rss?language_id=1',
+            'web': 'https://www.imf.org/en/News',
+            'selectors': {'headline': 'h3', 'link': 'a'},
+            'weight': 1.7,
+            'specialization': ['macro', 'geopolitical']
+        }
+    }
+
+    # Keyword detection
+    MACRO_KEYWORDS = [
+        'Fed', 'ECB', 'BoE', 'BoJ', 'FOMC', 'Powell', 'Lagarde',
+        'interest rate', 'rate cut', 'rate hike', 'inflation', 'CPI',
+        'GDP', 'unemployment', 'jobs report', 'NFP', 'monetary policy'
+    ]
+
+    MARKET_KEYWORDS = [
+        'S&P', 'Dow', 'Nasdaq', 'earnings', 'EPS', 'stock', 'equity',
+        'rally', 'selloff', 'correction', 'merger', 'acquisition', 'IPO'
+    ]
+
+    GEOPOLITICAL_KEYWORDS = [
+        'war', 'conflict', 'sanctions', 'trade', 'tariff', 'crisis',
+        'Ukraine', 'Russia', 'China', 'Taiwan', 'Middle East'
+    ]
+
+    def __init__(self):
+        """Initialize scraper with caching"""
+        self.news_cache = []
+        self.last_fetch = None
+        self.cache_ttl = 180  # 3 minutes
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        })
+
+    def _fetch_rss_feed(self, source_name: str, source_info: Dict) -> List[Dict]:
+        """Fetch and parse RSS feed from a single source"""
+        try:
+            feed = feedparser.parse(source_info['rss'])
+
+            if not feed.entries:
+                logger.warning(f"No entries found for {source_name}")
+                return []
+
+            news_items = []
+            for entry in feed.entries[:10]:  # Limit to 10 most recent
+                # Parse published date
+                try:
+                    if hasattr(entry, 'published_parsed') and entry.published_parsed:
+                        timestamp = datetime(*entry.published_parsed[:6])
+                    elif hasattr(entry, 'updated_parsed') and entry.updated_parsed:
+                        timestamp = datetime(*entry.updated_parsed[:6])
+                    else:
+                        timestamp = datetime.now()
+                except:
+                    timestamp = datetime.now()
+
+                # Skip old news (>24h)
+                if (datetime.now() - timestamp).days > 1:
+                    continue
+
+                # Extract title and summary
+                title = entry.get('title', '')
+                summary = entry.get('summary', '') or entry.get('description', '')
+
+                # Clean HTML from summary
+                if summary:
+                    summary = BeautifulSoup(summary, 'html.parser').get_text()
+                    summary = self._extract_summary(summary)
+
+                # Get URL
+                url = entry.get('link', '')
+
+                # Categorize and analyze
+                text = f"{title} {summary}"
+                category = self._categorize_text(text, source_info['specialization'])
+                sentiment = self._analyze_sentiment(text)
+                impact = self._assess_impact(source_info['weight'], title)
+                is_breaking = self._detect_breaking_news(title)
+
+                news_items.append({
+                    'id': hash(url),
+                    'title': title,
+                    'summary': summary or self._extract_summary(title),
+                    'source': source_info['name'],
+                    'category': category,
+                    'timestamp': timestamp,
+                    'sentiment': sentiment,
+                    'impact': impact,
+                    'url': url,
+                    'likes': 0,  # RSS feeds don't have engagement metrics
+                    'retweets': 0,
+                    'is_breaking': is_breaking,
+                    'source_weight': source_info['weight']
+                })
+
+            return news_items
+
+        except Exception as e:
+            logger.error(f"Error fetching RSS for {source_name}: {e}")
+            return []
+
+    def _scrape_web_page(self, source_name: str, source_info: Dict) -> List[Dict]:
+        """Scrape news headlines directly from website main page"""
+        try:
+            # Fetch HTML from web URL
+            response = self.session.get(source_info['web'], timeout=10)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.content, 'lxml')
+
+            # Get CSS selectors
+            headline_selector = source_info['selectors']['headline']
+            link_selector = source_info['selectors']['link']
+
+            news_items = []
+
+            # Find all headline elements
+            headlines = soup.select(headline_selector)
+
+            for headline_elem in headlines[:10]:  # Limit to 10 most recent
+                try:
+                    # Extract title text
+                    title = headline_elem.get_text(strip=True)
+                    if not title or len(title) < 10:
+                        continue
+
+                    # Find associated link
+                    # Try to find link within the headline element or its parent
+                    link_elem = headline_elem if headline_elem.name == 'a' else headline_elem.find('a')
+                    if not link_elem:
+                        # Try parent element
+                        link_elem = headline_elem.find_parent('a')
+                    if not link_elem:
+                        # Try sibling link with same selector
+                        parent = headline_elem.find_parent()
+                        if parent:
+                            link_elem = parent.find('a')
+
+                    if not link_elem:
+                        continue
+
+                    # Get URL and make absolute if relative
+                    url = link_elem.get('href', '')
+                    if not url:
+                        continue
+
+                    if url.startswith('/'):
+                        # Make absolute URL
+                        from urllib.parse import urljoin
+                        url = urljoin(source_info['web'], url)
+
+                    # Skip non-http URLs
+                    if not url.startswith('http'):
+                        continue
+
+                    # Categorize and analyze
+                    category = self._categorize_text(title, source_info['specialization'])
+                    sentiment = self._analyze_sentiment(title)
+                    impact = self._assess_impact(source_info['weight'], title)
+                    is_breaking = self._detect_breaking_news(title)
+
+                    news_items.append({
+                        'id': hash(url),
+                        'title': title,
+                        'summary': self._extract_summary(title),
+                        'source': source_info['name'],
+                        'category': category,
+                        'timestamp': datetime.now(),  # Web scraping doesn't have timestamps
+                        'sentiment': sentiment,
+                        'impact': impact,
+                        'url': url,
+                        'likes': 0,
+                        'retweets': 0,
+                        'is_breaking': is_breaking,
+                        'source_weight': source_info['weight']
+                    })
+
+                except Exception as e:
+                    logger.debug(f"Error parsing headline from {source_name}: {e}")
+                    continue
+
+            logger.info(f"Scraped {len(news_items)} items from {source_name} web page")
+            return news_items
+
+        except Exception as e:
+            logger.error(f"Error scraping web page for {source_name}: {e}")
+            return []
+
+    @st.cache_data(ttl=180)
+    def scrape_news(_self, max_items: int = 100) -> List[Dict]:
+        """
+        Scrape news from all sources with caching
+        Uses ThreadPoolExecutor for parallel fetching from both RSS and web pages
+        """
+        all_news = []
+        seen_urls = set()
+
+        # Parallel fetching using ThreadPoolExecutor
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            futures = []
+
+            # Submit both RSS and web scraping tasks for each source
+            for name, info in _self.SOURCES.items():
+                # RSS feed task
+                futures.append((executor.submit(_self._fetch_rss_feed, name, info), name, 'RSS'))
+                # Web scraping task
+                futures.append((executor.submit(_self._scrape_web_page, name, info), name, 'Web'))
+
+            for future, source_name, method in futures:
+                try:
+                    news_items = future.result()
+
+                    # Deduplicate based on URL
+                    unique_items = []
+                    for item in news_items:
+                        if item['url'] not in seen_urls:
+                            seen_urls.add(item['url'])
+                            unique_items.append(item)
+
+                    all_news.extend(unique_items)
+                    logger.info(f"Fetched {len(unique_items)} unique items from {source_name} ({method})")
+                except Exception as e:
+                    logger.error(f"Error processing {source_name} ({method}): {e}")
+
+        # If no news was fetched, use mock data
+        if not all_news:
+            logger.warning("No news fetched from any source - using mock data")
+            return _self._get_mock_news()
+
+        # Sort by breaking news, impact, and timestamp
+        all_news.sort(
+            key=lambda x: (x['is_breaking'], x['impact'] == 'high', x['timestamp']),
+            reverse=True
+        )
+
+        logger.info(f"Total unique news items: {len(all_news)}")
+        return all_news[:max_items]
+
+    def _categorize_text(self, text: str, source_specialization: List[str]) -> str:
+        """Categorize news based on keywords and source specialization"""
+        text_lower = text.lower()
+
+        # Count keyword matches
+        macro_score = sum(1 for kw in self.MACRO_KEYWORDS if kw.lower() in text_lower)
+        market_score = sum(1 for kw in self.MARKET_KEYWORDS if kw.lower() in text_lower)
+        geo_score = sum(1 for kw in self.GEOPOLITICAL_KEYWORDS if kw.lower() in text_lower)
+
+        # Weight by source specialization
+        if 'macro' in source_specialization:
+            macro_score *= 1.5
+        if 'markets' in source_specialization:
+            market_score *= 1.5
+        if 'geopolitical' in source_specialization:
+            geo_score *= 1.5
+
+        scores = {'macro': macro_score, 'markets': market_score, 'geopolitical': geo_score}
+        return max(scores, key=scores.get) if max(scores.values()) > 0 else 'markets'
+
+    def _analyze_sentiment(self, text: str) -> str:
+        """Analyze sentiment based on keywords"""
+        text_lower = text.lower()
+
+        positive = ['surge', 'soar', 'rally', 'beat', 'upgrade', 'bullish',
+                   'gain', 'rise', 'jump', 'boost', 'positive']
+        negative = ['plunge', 'crash', 'fall', 'miss', 'downgrade', 'bearish',
+                   'loss', 'drop', 'slide', 'concern', 'negative']
+
+        pos_count = sum(1 for word in positive if word in text_lower)
+        neg_count = sum(1 for word in negative if word in text_lower)
+
+        if pos_count > neg_count:
+            return 'positive'
+        elif neg_count > pos_count:
+            return 'negative'
+        return 'neutral'
+
+    def _assess_impact(self, source_weight: float, title: str) -> str:
+        """Assess market impact"""
+        # Central banks and official sources = high impact
+        if source_weight >= 1.7:
+            return 'high'
+
+        # Check for high-impact keywords
+        high_impact_words = ['breaking', 'alert', 'emergency', 'crash', 'surge', 'fed']
+        if any(word in title.lower() for word in high_impact_words):
+            return 'high'
+
+        return 'medium' if source_weight >= 1.3 else 'low'
+
+    def _detect_breaking_news(self, text: str) -> bool:
+        """Detect breaking news"""
+        text_upper = text.upper()
+        breaking_signals = ['BREAKING', 'ALERT', 'URGENT', 'JUST IN', 'DEVELOPING']
+        return any(signal in text_upper for signal in breaking_signals)
+
+    def _extract_summary(self, text: str, max_length: int = 150) -> str:
+        """Extract clean summary"""
+        text = re.sub(r'http\S+', '', text)
+        text = text.strip()
+
+        if len(text) <= max_length:
+            return text
+        return text[:max_length] + '...'
+
+    def _get_mock_news(self) -> List[Dict]:
+        """Mock data fallback"""
+        return [
+            {
+                'id': 1,
+                'title': 'Federal Reserve holds rates steady, signals caution on inflation outlook',
+                'summary': 'Fed maintains current rate policy',
+                'source': 'Federal Reserve',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(minutes=15),
+                'sentiment': 'neutral',
+                'impact': 'high',
+                'url': 'https://www.federalreserve.gov',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 2.0
+            },
+            {
+                'id': 2,
+                'title': 'S&P 500 closes at record high as tech stocks rally on strong earnings',
+                'summary': 'S&P 500 hits record on tech rally',
+                'source': 'CNBC',
+                'category': 'markets',
+                'timestamp': datetime.now() - timedelta(minutes=30),
+                'sentiment': 'positive',
+                'impact': 'high',
+                'url': 'https://www.cnbc.com',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 1.2
+            },
+            {
+                'id': 3,
+                'title': 'ECB President Lagarde warns of persistent inflation pressures in eurozone',
+                'summary': 'Lagarde warns on eurozone inflation',
+                'source': 'European Central Bank',
+                'category': 'macro',
+                'timestamp': datetime.now() - timedelta(hours=1),
+                'sentiment': 'negative',
+                'impact': 'high',
+                'url': 'https://www.ecb.europa.eu',
+                'likes': 0,
+                'retweets': 0,
+                'is_breaking': False,
+                'source_weight': 2.0
+            }
+        ]
+
+    def get_news(self, category: str = 'all', sentiment: str = 'all',
+                 impact: str = 'all', refresh: bool = False) -> pd.DataFrame:
+        """Get filtered news with caching"""
+        # Check cache freshness
+        if refresh or not self.last_fetch or \
+           (datetime.now() - self.last_fetch).seconds > self.cache_ttl:
+            self.news_cache = self.scrape_news(max_items=100)
+            self.last_fetch = datetime.now()
+
+        news = self.news_cache.copy()
+
+        # Apply filters
+        if category != 'all':
+            news = [n for n in news if n['category'] == category]
+        if sentiment != 'all':
+            news = [n for n in news if n['sentiment'] == sentiment]
+        if impact != 'all':
+            news = [n for n in news if n['impact'] == impact]
+
+        df = pd.DataFrame(news)
+        if not df.empty:
+            df['timestamp'] = pd.to_datetime(df['timestamp'])
+
+        return df
+
+    def get_breaking_news(self) -> pd.DataFrame:
+        """Get breaking/high-impact news"""
+        return self.get_news(impact='high')
+
+    def get_statistics(self) -> Dict:
+        """Get feed statistics"""
+        if not self.news_cache:
+            return {
+                'total': 0,
+                'high_impact': 0,
+                'breaking': 0,
+                'last_update': 'Never',
+                'by_category': {}
+            }
+
+        df = pd.DataFrame(self.news_cache)
+        return {
+            'total': len(df),
+            'high_impact': len(df[df['impact'] == 'high']),
+            'breaking': len(df[df['is_breaking'] == True]),
+            'last_update': self.last_fetch.strftime('%H:%M:%S') if self.last_fetch else 'Never',
+            'by_category': df['category'].value_counts().to_dict()
+        }

requirements.txt

@@ -5,3 +5,6 @@ openbb>=4.0.0
 python-dotenv>=1.0.0
 requests>=2.31.0
 twikit>=2.3.0
+feedparser>=6.0.0
+beautifulsoup4>=4.12.0
+lxml>=5.0.0