Delete data_fetcher.py

data_fetcher.py

@@ -1,320 +0,0 @@
-"""
-data_fetcher.py — Fetches real live data from the internet.
-Sources: RSS, Reddit, Wikipedia, HackerNews, DuckDuckGo Search,
-         DEV.to, GitHub Trending, Google News RSS, Al Jazeera, Guardian, NPR, arXiv
-"""
-
-import requests, feedparser, random, re, time, json
-from datetime import datetime
-from collections import deque
-from urllib.parse import quote_plus
-from html import unescape
-
-HEADERS = {
-    'User-Agent': 'Mozilla/5.0 (compatible; LivingNeuralNetwork/2.0; educational)',
-    'Accept-Language': 'en-US,en;q=0.9',
-}
-
-# ─── RSS FEEDS ────────────────────────────────────────────────────────────────
-RSS_FEEDS = {
-    'technology': [
-        'https://feeds.feedburner.com/TechCrunch',
-        'https://feeds.arstechnica.com/arstechnica/index',
-        'https://www.wired.com/feed/rss',
-        'https://hnrss.org/frontpage',
-        'https://www.theverge.com/rss/index.xml',
-        'https://feeds.feedburner.com/venturebeat/SZYF',
-        'https://dev.to/feed',
-        'https://thenextweb.com/feed/',
-    ],
-    'science': [
-        'https://www.sciencedaily.com/rss/all.xml',
-        'https://www.newscientist.com/feed/home/',
-        'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
-        'https://www.nature.com/nature.rss',
-        'http://export.arxiv.org/rss/cs.AI',
-        'http://export.arxiv.org/rss/cs.LG',
-        'https://phys.org/rss-feed/breaking/',
-    ],
-    'world': [
-        'https://feeds.bbci.co.uk/news/world/rss.xml',
-        'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
-        'https://www.aljazeera.com/xml/rss/all.xml',
-        'https://feeds.npr.org/1004/rss.xml',
-        'https://www.theguardian.com/world/rss',
-        'https://feeds.reuters.com/reuters/worldNews',
-    ],
-    'sports': [
-        'https://feeds.bbci.co.uk/sport/rss.xml',
-        'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml',
-        'https://www.espn.com/espn/rss/news',
-    ],
-    'business': [
-        'https://feeds.bbci.co.uk/news/business/rss.xml',
-        'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
-        'https://feeds.bloomberg.com/markets/news.rss',
-        'https://www.theguardian.com/business/rss',
-        'https://feeds.reuters.com/reuters/businessNews',
-    ],
-    'health': [
-        'https://feeds.bbci.co.uk/news/health/rss.xml',
-        'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml',
-        'https://www.theguardian.com/society/health/rss',
-        'https://feeds.npr.org/1128/rss.xml',
-    ],
-    'entertainment': [
-        'https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml',
-        'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml',
-        'https://www.theguardian.com/culture/rss',
-        'https://variety.com/feed/',
-    ],
-    'ai': [
-        'http://export.arxiv.org/rss/cs.AI',
-        'http://export.arxiv.org/rss/cs.LG',
-        'https://hnrss.org/frontpage',
-        'https://feeds.feedburner.com/TechCrunch',
-    ],
-}
-
-REDDIT_FEEDS = [
-    ('technology',    'https://www.reddit.com/r/technology/top.json?limit=25&t=day'),
-    ('science',       'https://www.reddit.com/r/science/top.json?limit=25&t=day'),
-    ('world',         'https://www.reddit.com/r/worldnews/top.json?limit=25&t=day'),
-    ('sports',        'https://www.reddit.com/r/sports/top.json?limit=25&t=day'),
-    ('business',      'https://www.reddit.com/r/business/top.json?limit=25&t=day'),
-    ('health',        'https://www.reddit.com/r/Health/top.json?limit=25&t=day'),
-    ('entertainment', 'https://www.reddit.com/r/movies/top.json?limit=25&t=day'),
-    ('ai',            'https://www.reddit.com/r/MachineLearning/top.json?limit=25&t=day'),
-    ('ai',            'https://www.reddit.com/r/artificial/top.json?limit=25&t=day'),
-    ('science',       'https://www.reddit.com/r/askscience/top.json?limit=25&t=day'),
-    ('technology',    'https://www.reddit.com/r/programming/top.json?limit=25&t=day'),
-    ('world',         'https://www.reddit.com/r/geopolitics/top.json?limit=25&t=day'),
-]
-
-# Topics to actively search on DuckDuckGo
-DDG_SEARCH_TOPICS = [
-    ('technology', 'latest AI artificial intelligence news'),
-    ('technology', 'tech industry news today'),
-    ('science',    'science discovery research breakthrough'),
-    ('world',      'world news today'),
-    ('business',   'business economy markets news'),
-    ('health',     'health medicine research news'),
-    ('ai',         'machine learning deep learning news'),
-    ('ai',         'ChatGPT OpenAI Anthropic Google AI'),
-    ('world',      'politics international relations'),
-    ('science',    'space NASA astronomy discovery'),
-    ('technology', 'cybersecurity data breach'),
-    ('entertainment', 'movies music entertainment news'),
-]
-
-WIKIPEDIA_API = 'https://en.wikipedia.org/api/rest_v1/page/random/summary'
-WIKI_SEARCH   = 'https://en.wikipedia.org/api/rest_v1/page/summary/{}'
-HN_TOP_API    = 'https://hacker-news.firebaseio.com/v0/topstories.json'
-HN_ITEM_API   = 'https://hacker-news.firebaseio.com/v0/item/{}.json'
-
-# ─── UTILS ────────────────────────────────────────────────────────────────────
-def clean(text: str, max_chars: int = 700) -> str:
-    if not text: return ''
-    text = unescape(text)                              # fix ' & < etc
-    text = re.sub(r'&#?[a-zA-Z0-9]+;', ' ', text)    # remove any remaining entities
-    text = re.sub(r'<[^>]+>', ' ', text)               # strip HTML tags
-    text = re.sub(r'http\S+', '', text)
-    text = re.sub(r'[^\w\s.,!?;:\'\-–—]', ' ', text)
-    text = re.sub(r'\s+', ' ', text).strip()
-    return text[:max_chars]
-
-def make_item(text, category, source, extra=None):
-    text = clean(text)
-    if len(text) < 30: return None
-    return {
-        'text':      text,
-        'category':  category,
-        'source':    source,
-        'timestamp': datetime.utcnow().isoformat(),
-        **(extra or {}),
-    }
-
-# ─── FETCHERS ─────────────────────────────────────────────────────────────────
-def fetch_rss(category: str, url: str) -> list:
-    items = []
-    try:
-        feed = feedparser.parse(url)
-        for entry in feed.entries[:15]:
-            title   = entry.get('title', '')
-            summary = entry.get('summary', entry.get('description', ''))
-            text    = f"{title}. {summary}"
-            item = make_item(text, category, 'rss', {'feed': url.split('/')[2]})
-            if item: items.append(item)
-    except Exception: pass
-    return items
-
-def fetch_reddit(category: str, url: str) -> list:
-    items = []
-    try:
-        r = requests.get(url, headers=HEADERS, timeout=10)
-        r.raise_for_status()
-        posts = r.json().get('data', {}).get('children', [])
-        for post in posts:
-            d = post.get('data', {})
-            title    = d.get('title', '')
-            selftext = d.get('selftext', '')
-            text = f"{title}. {selftext}"
-            item = make_item(text, category, 'reddit',
-                           {'subreddit': d.get('subreddit',''), 'score': d.get('score',0)})
-            if item: items.append(item)
-    except Exception: pass
-    return items
-
-def fetch_duckduckgo(query: str, category: str) -> list:
-    """Search DuckDuckGo and extract text snippets — no API key needed."""
-    items = []
-    try:
-        url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
-        r = requests.get(url, headers={**HEADERS, 'Accept': 'text/html'}, timeout=12)
-        # Extract result snippets
-        snippets = re.findall(r'class="result__snippet"[^>]*>(.*?)</[as]>', r.text, re.DOTALL)
-        titles   = re.findall(r'class="result__a"[^>]*>(.*?)</a>', r.text, re.DOTALL)
-        for i, snippet in enumerate(snippets[:8]):
-            title = titles[i] if i < len(titles) else query
-            title = re.sub(r'<[^>]+>', '', title).strip()
-            snippet = re.sub(r'<[^>]+>', '', snippet).strip()
-            text = f"{title}. {snippet}"
-            item = make_item(text, category, 'duckduckgo', {'query': query})
-            if item: items.append(item)
-    except Exception: pass
-    return items
-
-def fetch_wikipedia_random() -> dict | None:
-    try:
-        r = requests.get(WIKIPEDIA_API, headers=HEADERS, timeout=10)
-        r.raise_for_status()
-        data  = r.json()
-        title = data.get('title', '')
-        text  = f"{title}. {data.get('extract','')}"
-        return make_item(text, 'other', 'wikipedia', {'title': title})
-    except Exception: return None
-
-def fetch_wikipedia_topic(topic: str, category: str) -> dict | None:
-    """Fetch Wikipedia article for a specific topic."""
-    try:
-        url = WIKI_SEARCH.format(quote_plus(topic))
-        r = requests.get(url, headers=HEADERS, timeout=10)
-        r.raise_for_status()
-        data  = r.json()
-        title = data.get('title', '')
-        text  = f"{title}. {data.get('extract','')}"
-        return make_item(text, category, 'wikipedia', {'title': title})
-    except Exception: return None
-
-def fetch_hackernews(n: int = 8) -> list:
-    items = []
-    try:
-        r = requests.get(HN_TOP_API, headers=HEADERS, timeout=10)
-        r.raise_for_status()
-        ids    = r.json()[:50]
-        chosen = random.sample(ids, min(n, len(ids)))
-        for sid in chosen:
-            try:
-                sr    = requests.get(HN_ITEM_API.format(sid), headers=HEADERS, timeout=6)
-                story = sr.json()
-                title = story.get('title', '')
-                body  = story.get('text', '')
-                item  = make_item(f"{title}. {body}", 'technology', 'hackernews',
-                                 {'score': story.get('score', 0)})
-                if item: items.append(item)
-                time.sleep(0.08)
-            except Exception: continue
-    except Exception: pass
-    return items
-
-# ─── MAIN FETCHER ─────────────────────────────────────────────────────────────
-class DataFetcher:
-    def __init__(self):
-        self.total_fetched = 0
-        self.source_counts = {
-            'rss': 0, 'reddit': 0, 'wikipedia': 0,
-            'hackernews': 0, 'duckduckgo': 0
-        }
-        self.recent_items = deque(maxlen=100)
-        self.log          = deque(maxlen=300)
-        self._ddg_idx     = 0   # rotate through DDG search topics
-
-    def _log(self, msg: str):
-        ts    = datetime.utcnow().strftime('%H:%M:%S')
-        entry = f"[{ts}] {msg}"
-        self.log.appendleft(entry)
-        return entry
-
-    def fetch_round(self) -> list:
-        """Fetch one full round from ALL sources."""
-        all_items = []
-
-        # 1. RSS — pick 2 random categories
-        for _ in range(2):
-            try:
-                cat = random.choice(list(RSS_FEEDS.keys()))
-                url = random.choice(RSS_FEEDS[cat])
-                items = fetch_rss(cat, url)
-                all_items.extend(items)
-                self.source_counts['rss'] += len(items)
-                self._log(f"📰 RSS [{cat.upper()}] +{len(items)} ← {url.split('/')[2]}")
-            except Exception as e:
-                self._log(f"⚠ RSS error: {e}")
-
-        # 2. Reddit
-        try:
-            cat, url = random.choice(REDDIT_FEEDS)
-            items = fetch_reddit(cat, url)
-            all_items.extend(items)
-            self.source_counts['reddit'] += len(items)
-            sub = url.split('/r/')[1].split('/')[0]
-            self._log(f"🟠 Reddit [r/{sub}] +{len(items)}")
-        except Exception as e:
-            self._log(f"⚠ Reddit error: {e}")
-
-        # 3. DuckDuckGo search (rotate through topics)
-        try:
-            cat, query = DDG_SEARCH_TOPICS[self._ddg_idx % len(DDG_SEARCH_TOPICS)]
-            self._ddg_idx += 1
-            items = fetch_duckduckgo(query, cat)
-            all_items.extend(items)
-            self.source_counts['duckduckgo'] += len(items)
-            self._log(f"🦆 DuckDuckGo [{cat.upper()}] \"{query[:40]}\" +{len(items)}")
-        except Exception as e:
-            self._log(f"⚠ DDG error: {e}")
-
-        # 4. Wikipedia (random + topic)
-        try:
-            item = fetch_wikipedia_random()
-            if item:
-                all_items.append(item)
-                self.source_counts['wikipedia'] += 1
-                self._log(f"📖 Wikipedia (random): {item['text'][:50]}…")
-        except Exception as e:
-            self._log(f"⚠ Wikipedia error: {e}")
-
-        # 5. HackerNews (every other round)
-        if random.random() < 0.5:
-            try:
-                items = fetch_hackernews(6)
-                all_items.extend(items)
-                self.source_counts['hackernews'] += len(items)
-                self._log(f"💻 HackerNews +{len(items)}")
-            except Exception as e:
-                self._log(f"⚠ HN error: {e}")
-
-        for item in all_items:
-            self.recent_items.appendleft(item)
-        self.total_fetched += len(all_items)
-        self._log(f"✅ Round complete — {len(all_items)} new items | total: {self.total_fetched}")
-        return all_items
-
-    def get_stats(self) -> dict:
-        return {
-            'total_fetched': self.total_fetched,
-            'sources':       dict(self.source_counts),
-            'recent_log':    list(self.log)[:30],
-        }
-
-    def get_recent_items(self, n: int = 20) -> list:
-        return list(self.recent_items)[:n]