data_fetcher.py

"""
data_fetcher.py — Fetches REAL article content (not website taglines).
Key fix: extracts full paragraph text from pages, filters out metadata garbage.
"""
import requests, feedparser, random, re, time, json
from datetime import datetime
from collections import deque
from urllib.parse import quote_plus
from html import unescape

try:
    from bs4 import BeautifulSoup
    BS4 = True
except ImportError:
    BS4 = False

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.9',
}

# ── GARBAGE PATTERNS — these are website UI text, not real content ─────────────
GARBAGE_PATTERNS = re.compile(
    r'(sign up|subscribe|click here|breaking news|stay informed|'
    r'all rights reserved|privacy policy|cookie|advertisement|'
    r'newsletter|follow us|share this|read more|learn more|'
    r'top stories|latest news from|everything you need to know|'
    r'has everything|stay up to date|get the latest|'
    r'news articles today|news today ap|ap news|'
    r'live science|science news,|sciencedaily|'
    r'^\s*\d+\s*$|^[\w\s]{1,15}:?\s*$)',   # very short / numeric only
    re.IGNORECASE
)

RSS_FEEDS = {
    'technology': [
        'https://feeds.arstechnica.com/arstechnica/index',
        'https://www.wired.com/feed/rss',
        'https://hnrss.org/frontpage',
        'https://www.theverge.com/rss/index.xml',
        'https://dev.to/feed',
        'https://thenextweb.com/feed/',
    ],
    'science': [
        'https://www.sciencedaily.com/rss/all.xml',
        'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
        'http://export.arxiv.org/rss/cs.AI',
        'https://phys.org/rss-feed/breaking/',
    ],
    'world': [
        'https://feeds.bbci.co.uk/news/world/rss.xml',
        'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
        'https://www.aljazeera.com/xml/rss/all.xml',
        'https://feeds.npr.org/1004/rss.xml',
        'https://www.theguardian.com/world/rss',
    ],
    'sports':        ['https://feeds.bbci.co.uk/sport/rss.xml',
                      'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml'],
    'business':      ['https://feeds.bbci.co.uk/news/business/rss.xml',
                      'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
                      'https://www.theguardian.com/business/rss'],
    'health':        ['https://feeds.bbci.co.uk/news/health/rss.xml',
                      'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml'],
    'entertainment': ['https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml',
                      'https://variety.com/feed/'],
    'ai':            ['http://export.arxiv.org/rss/cs.AI',
                      'http://export.arxiv.org/rss/cs.LG',
                      'https://hnrss.org/frontpage'],
}

REDDIT_SUBS = [
    ('technology',    'technology'),
    ('science',       'science'),
    ('world',         'worldnews'),
    ('sports',        'sports'),
    ('business',      'business'),
    ('health',        'Health'),
    ('entertainment', 'movies'),
    ('ai',            'MachineLearning'),
    ('ai',            'artificial'),
    ('technology',    'programming'),
]

WIKI_TOPICS = [
    ('science',     ['Physics','Chemistry','Biology','Astronomy','Mathematics','Evolution','Genetics']),
    ('technology',  ['Artificial intelligence','Machine learning','Computer science','Internet','Robotics']),
    ('world',       ['Climate change','Democracy','United Nations','Geopolitics','Economics']),
    ('health',      ['Medicine','Vaccine','Cancer','Nutrition','Mental health','COVID-19']),
    ('ai',          ['Neural network','Deep learning','Natural language processing','GPT','Transformer model']),
    ('business',    ['Stock market','Cryptocurrency','Inflation','Supply chain','Startup']),
    ('entertainment',['Film','Music','Video game','Television','Streaming']),
    ('sports',      ['Football','Basketball','Olympic Games','Tennis','Cricket']),
]

HN_TOP = 'https://hacker-news.firebaseio.com/v0/topstories.json'
HN_ITEM = 'https://hacker-news.firebaseio.com/v0/item/{}.json'

# ── UTILS ────────────────────────────────────────────────────────────────────
def clean(text, max_chars=800):
    if not text: return ''
    text = unescape(str(text))
    text = re.sub(r'&#?[a-zA-Z0-9]+;', ' ', text)
    text = re.sub(r'<[^>]+>', ' ', text)
    text = re.sub(r'http\S+', '', text)
    text = re.sub(r'[^\w\s.,!?;:\'\-–—]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text[:max_chars]

def is_garbage(text):
    """Return True if text is a website tagline/nav text, not real content."""
    if len(text) < 40: return True
    if GARBAGE_PATTERNS.search(text): return True
    # If it's just a title with no real sentence, skip
    words = text.split()
    if len(words) < 8: return True
    return False

def make_item(text, category, source, extra=None):
    text = clean(text)
    if not text or is_garbage(text): return None
    return {'text': text, 'category': category, 'source': source,
            'timestamp': datetime.utcnow().isoformat(), **(extra or {})}

def extract_article_text(url, max_chars=600):
    """Fetch a URL and extract real paragraph text using BeautifulSoup."""
    if not BS4: return ''
    try:
        r = requests.get(url, headers=HEADERS, timeout=8)
        soup = BeautifulSoup(r.text, 'html.parser')
        # Remove nav, header, footer, ads
        for tag in soup(['nav','header','footer','script','style','aside',
                         'figure','form','button','iframe']):
            tag.decompose()
        # Get paragraphs
        paras = soup.find_all('p')
        text = ' '.join(p.get_text(' ', strip=True) for p in paras if len(p.get_text()) > 60)
        return clean(text, max_chars)
    except Exception:
        return ''

# ── FETCHERS ─────────────────────────────────────────────────────────────────
def fetch_rss(category, url):
    items = []
    try:
        feed = feedparser.parse(url)
        for entry in feed.entries[:12]:
            title   = entry.get('title', '')
            summary = entry.get('summary', entry.get('description', ''))
            # Prefer summary if it has real content (>100 chars)
            body = summary if len(summary) > 100 else ''
            text = f"{title}. {body}".strip()
            item = make_item(text, category, 'rss', {'feed': url.split('/')[2]})
            if item: items.append(item)
    except Exception: pass
    return items

def fetch_reddit(category, subreddit):
    items = []
    try:
        url = f"https://www.reddit.com/r/{subreddit}/top.json?limit=20&t=day"
        r   = requests.get(url, headers=HEADERS, timeout=10)
        r.raise_for_status()
        for post in r.json().get('data',{}).get('children',[]):
            d = post.get('data',{})
            title    = d.get('title','')
            selftext = d.get('selftext','')
            # Reddit selftext often has real content
            text = f"{title}. {selftext}" if len(selftext) > 80 else title
            item = make_item(text, category, 'reddit',
                           {'subreddit': subreddit, 'score': d.get('score',0)})
            if item: items.append(item)
    except Exception: pass
    return items

def fetch_wikipedia(topic, category):
    """Fetch real Wikipedia article content — actual knowledge, not taglines."""
    try:
        url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{quote_plus(topic)}"
        r   = requests.get(url, headers=HEADERS, timeout=10)
        r.raise_for_status()
        data  = r.json()
        title = data.get('title','')
        extract = data.get('extract','')  # Wikipedia gives full intro paragraph
        if len(extract) < 80: return None
        text = f"{title}: {extract}"
        return make_item(text, category, 'wikipedia', {'title': title})
    except Exception: return None

def fetch_hackernews(n=6):
    items = []
    try:
        ids    = requests.get(HN_TOP, headers=HEADERS, timeout=8).json()[:40]
        chosen = random.sample(ids, min(n, len(ids)))
        for sid in chosen:
            try:
                story = requests.get(HN_ITEM.format(sid), headers=HEADERS, timeout=5).json()
                title = story.get('title','')
                text_body = story.get('text','')
                text = f"{title}. {text_body}" if len(text_body)>60 else title
                item = make_item(text, 'technology', 'hackernews',
                               {'score': story.get('score',0)})
                if item: items.append(item)
                time.sleep(0.05)
            except Exception: continue
    except Exception: pass
    return items

# ── MAIN FETCHER ─────────────────────────────────────────────────────────────
class DataFetcher:
    def __init__(self):
        self.total_fetched = 0
        self.source_counts = {'rss':0,'reddit':0,'wikipedia':0,'hackernews':0}
        self.recent_items  = deque(maxlen=100)
        self.log           = deque(maxlen=300)
        self._wiki_idx     = 0

    def _log(self, msg):
        ts = datetime.utcnow().strftime('%H:%M:%S')
        self.log.appendleft(f"[{ts}] {msg}")

    def fetch_round(self):
        all_items = []

        # 1. RSS — 2 random feeds
        for _ in range(2):
            cat = random.choice(list(RSS_FEEDS.keys()))
            url = random.choice(RSS_FEEDS[cat])
            items = fetch_rss(cat, url)
            all_items.extend(items)
            self.source_counts['rss'] += len(items)
            self._log(f"📰 RSS [{cat.upper()}] +{len(items)} ← {url.split('/')[2]}")

        # 2. Reddit
        cat, sub = random.choice(REDDIT_SUBS)
        items = fetch_reddit(cat, sub)
        all_items.extend(items)
        self.source_counts['reddit'] += len(items)
        self._log(f"🟠 Reddit [r/{sub}] +{len(items)}")

        # 3. Wikipedia — rotate through topics (REAL knowledge content)
        flat = [(cat, t) for cat, topics in WIKI_TOPICS for t in topics]
        cat, topic = flat[self._wiki_idx % len(flat)]
        self._wiki_idx += 1
        item = fetch_wikipedia(topic, cat)
        if item:
            all_items.append(item)
            self.source_counts['wikipedia'] += 1
            self._log(f"📖 Wikipedia: {topic}")

        # 4. HackerNews (every other round)
        if random.random() < 0.5:
            items = fetch_hackernews(5)
            all_items.extend(items)
            self.source_counts['hackernews'] += len(items)
            self._log(f"💻 HackerNews +{len(items)}")

        for item in all_items:
            self.recent_items.appendleft(item)
        self.total_fetched += len(all_items)
        self._log(f"✅ Round done — +{len(all_items)} | total {self.total_fetched}")
        return all_items

    def get_stats(self):
        return {'total_fetched': self.total_fetched,
                'sources': dict(self.source_counts),
                'recent_log': list(self.log)[:30]}

    def get_recent_items(self, n=20):
        return list(self.recent_items)[:n]