Deploy NewsApp

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim AS builder
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+FROM python:3.12-slim
+WORKDIR /app
+RUN addgroup --system app && adduser --system --group app
+COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY . .
+USER app
+EXPOSE 7860
+ENV FLASK_ENV=production \
+    PORT=7860 \
+    HOST=0.0.0.0 \
+    LOG_LEVEL=INFO \
+    PYTHONUNBUFFERED=1
+CMD ["gunicorn", "-w", "2", "-b", "0.0.0.0:7860", "--timeout", "300", "--access-logfile", "-", "--error-logfile", "-", "webapp:app"]

config.py

@@ -0,0 +1,100 @@
+import os
+from dataclasses import dataclass, field
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+@dataclass
+class Config:
+    reddit_client_id: str = os.getenv("REDDIT_CLIENT_ID", "")
+    reddit_client_secret: str = os.getenv("REDDIT_CLIENT_SECRET", "")
+    reddit_user_agent: str = os.getenv("REDDIT_USER_AGENT", "newsapp/1.0")
+
+    rss_feeds: list[str] = field(default_factory=lambda: [
+        # ---- Geopolitical ----
+        "https://feeds.bbci.co.uk/news/world/rss.xml",
+        "https://rss.nytimes.com/services/xml/rss/nyt/World.xml",
+        "https://rss.nytimes.com/services/xml/rss/nyt/Politics.xml",
+        "https://feeds.npr.org/1001/rss.xml",
+        "https://www.aljazeera.com/xml/rss/all.xml",
+        "https://www.theguardian.com/world/rss",
+        # ---- World Health ----
+        "https://feeds.bbci.co.uk/news/health/rss.xml",
+        "https://rss.nytimes.com/services/xml/rss/nyt/Science.xml",
+        "https://www.statnews.com/feed/",
+        "https://www.sciencedaily.com/rss/all.xml",
+        # ---- Tech ----
+        "https://feeds.bbci.co.uk/news/technology/rss.xml",
+        "https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
+        "https://techcrunch.com/feed/",
+        "https://www.wired.com/feed/rss",
+        "https://www.theverge.com/rss/index.xml",
+        "https://arstechnica.com/feed/",
+        # ---- Cybersecurity ----
+        "https://feeds.feedburner.com/TheHackerNews",
+        "https://krebsonsecurity.com/feed/",
+        "https://www.bleepingcomputer.com/feed/",
+        "https://threatpost.com/feed/",
+        "https://therecord.media/feed/",
+        # ---- Funny / Weird ----
+        "https://www.theonion.com/rss",
+        "https://www.reddit.com/r/nottheonion/.rss",
+        "https://www.thedailymash.co.uk/feed",
+        "https://babylonbee.com/feed",
+        # ---- Gaming ----
+        "https://feeds.ign.com/ign/all",
+        "https://www.eurogamer.net/feed",
+        "https://www.pcgamer.com/rss/",
+        "https://www.kotaku.com/rss",
+        "https://www.gamespot.com/feeds/news/",
+        "https://www.polygon.com/rss/index.xml",
+        # ---- Movies ----
+        "https://variety.com/feed/",
+        "https://www.hollywoodreporter.com/feed/",
+        "https://deadline.com/feed/",
+        "https://screenrant.com/feed/",
+        # ---- Arab World ----
+        "https://www.arabnews.com/rss.xml",
+        "https://www.middleeasteye.net/rss",
+        "https://www.newarab.com/rss.xml",
+        "https://www.france24.com/en/middle-east/rss",
+        # ---- Tunisia ----
+        "https://www.tunisiaonlinenews.com/feed/",
+        "https://northafricapost.com/feed/",
+        "https://www.africanews.com/feed/",
+    ])
+
+    # Fallback subreddits (used for --source reddit or --source rss)
+    news_subreddits: list[str] = field(default_factory=lambda: [
+        "worldnews", "news", "politics", "science", "technology",
+        "UpliftingNews", "economy", "geopolitics",
+    ])
+
+    posts_per_subreddit: int = 3
+
+    weight_popularity: float = 0.20
+    weight_trustworthiness: float = 0.35
+    weight_coverage: float = 0.30
+    weight_recency: float = 0.15
+
+    user_interests: list[str] = field(default_factory=lambda: [
+        "artificial intelligence", "climate change", "health",
+        "economy", "space exploration", "cybersecurity",
+        "democracy", "science",
+    ])
+
+    use_local_models: bool = False
+    summarization_model: str = "t5-small"
+    embedding_model: str = "all-MiniLM-L6-v2"
+
+    similarity_threshold: float = 0.70
+
+    port: int = int(os.getenv("PORT", "5050"))
+    host: str = os.getenv("HOST", "0.0.0.0")
+    log_level: str = os.getenv("LOG_LEVEL", "INFO")
+    flask_env: str = os.getenv("FLASK_ENV", "production")
+    cors_origins: str = os.getenv("CORS_ORIGINS", "*")
+    refresh_hour: int = int(os.getenv("REFRESH_HOUR", "4"))
+    refresh_minute: int = int(os.getenv("REFRESH_MINUTE", "0"))
+    refresh_timezone: str = os.getenv("REFRESH_TIMEZONE", "Africa/Tunis")

main.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import sys
+import time
+
+from config import Config
+from src.models import Article, NewsItem
+from src.extractor import ArticleExtractor
+from src.analyzer import NewsAnalyzer
+from src.aggregator import NewsAggregator
+from src.presenter import NewsPresenter
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="OneNews — RSS news aggregator")
+    parser.add_argument("--demo", action="store_true", help="Use sample data (no internet)")
+    parser.add_argument("--source", choices=["feeds", "hn", "reddit"], default=None,
+                        help="Data source (default: RSS feeds)")
+    parser.add_argument("--models", action="store_true", help="Enable local ML models (slower)")
+    parser.add_argument("--subreddits", nargs="+", default=None, help="Override subreddits")
+    parser.add_argument("--limit", type=int, default=None, help="Posts per subreddit")
+    return parser.parse_args()
+
+
+def run_pipeline(items: list[NewsItem], cfg: Config, skip_extraction: bool = False):
+    if not skip_extraction:
+        logger.info("═══  Extracting article content  ═══")
+        extractor = ArticleExtractor()
+        for i, item in enumerate(items, 1):
+            article = extractor.extract(item.post.url)
+            if article:
+                logger.info("  [%2d/%d] %-60s ✓  (%s)", i, len(items), item.post.title[:60], article.source_domain)
+            else:
+                article = Article(
+                    url=item.post.url,
+                    title=item.post.title,
+                    text=item.post.title,
+                    source_domain=item.post.source_domain or "reddit.com",
+                    extraction_success=False,
+                    image_url=item.post.image_url,
+                )
+                logger.info("  [%2d/%d] %-60s ✗  (title only)", i, len(items), item.post.title[:60])
+            item.article = article
+    else:
+        logger.info("═══  Extraction skipped (articles already loaded)  ═══")
+
+    logger.info("═══  Analysing articles  ═══")
+    analyzer = NewsAnalyzer(cfg)
+    for i, item in enumerate(items, 1):
+        item.analysis = analyzer.analyze(item.article)
+        cat = item.analysis.category
+        topics = ", ".join(item.analysis.topics) if item.analysis.topics else "(none)"
+        logger.info("  [%2d/%d] %-14s topic: %-30s trust: %s", i, len(items), cat, topics, f"{item.analysis.trustworthiness_score:.0%}")
+
+    logger.info("═══  Clustering & ranking  ═══")
+    aggregator = NewsAggregator(cfg)
+    clusters = aggregator.cluster_news(items)
+    logger.info("  → %d story clusters found", len(clusters))
+
+    return clusters
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(levelname).1s %(message)s", stream=sys.stderr)
+    args = parse_args()
+    cfg = Config()
+
+    if args.models:
+        cfg.use_local_models = True
+        logging.getLogger().setLevel(logging.DEBUG)
+    if args.subreddits:
+        cfg.news_subreddits = args.subreddits
+    if args.limit:
+        cfg.posts_per_subreddit = args.limit
+
+    total_start = time.perf_counter()
+
+    source = args.source or "feeds"
+
+    if args.demo:
+        from src.mockdata import generate_demo_items
+        print("═══  Loading demo data  ═══")
+        items = generate_demo_items()
+        print(f"  → {len(items)} sample articles loaded\n")
+        clusters = run_pipeline(items, cfg, skip_extraction=True)
+        n_posts = len(items)
+
+    elif source == "hn":
+        from src.hn_scraper import HackerNewsScraper
+        print("═══  Scraping Hacker News  ═══")
+        scraper = HackerNewsScraper(cfg)
+        posts = scraper.fetch_posts()
+        n_posts = len(posts)
+        print(f"  → {n_posts} posts collected\n")
+        if not posts:
+            sys.exit(1)
+        items = [NewsItem(post=p) for p in posts]
+        clusters = run_pipeline(items, cfg)
+
+    elif source == "reddit":
+        from src.scraper import RedditScraper
+        print("═══  Scraping Reddit  ═══")
+        scraper = RedditScraper(cfg)
+        posts = scraper.fetch_posts()
+        n_posts = len(posts)
+        print(f"  → {n_posts} posts collected\n")
+        if not posts:
+            sys.exit(1)
+        items = [NewsItem(post=p) for p in posts]
+        clusters = run_pipeline(items, cfg)
+
+    else:
+        from src.rss_feed_scraper import RSSFeedScraper
+        print("═══  Fetching RSS news feeds  ═══")
+        scraper = RSSFeedScraper(cfg)
+        posts = scraper.fetch_posts()
+        n_posts = len(posts)
+        print(f"  → {n_posts} posts collected\n")
+        if not posts:
+            sys.exit(1)
+        items = [NewsItem(post=p) for p in posts]
+        clusters = run_pipeline(items, cfg)
+
+    elapsed = time.perf_counter() - total_start
+    print(f"\n  Done in {elapsed:.1f}s — {n_posts} posts, {len(clusters)} clusters\n")
+
+
+if __name__ == "__main__":
+    main()

refresh_task.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""
+Standalone script for PythonAnywhere scheduled tasks.
+Run daily at 3:00 UTC (4:00 AM Tunisian time).
+
+Usage:
+    python3 /home/YOUR_USER/newsapp/refresh_task.py
+"""
+
+import json
+import logging
+import os
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+
+logging.basicConfig(level=logging.INFO, format="%(levelname).1s %(message)s", stream=sys.stderr)
+logger = logging.getLogger(__name__)
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from config import Config
+from main import run_pipeline
+from src.models import NewsItem
+from src.rss_feed_scraper import RSSFeedScraper
+
+cfg = Config()
+
+CACHE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache_data.json")
+
+CATEGORIES = ["Geopolitical", "World Health", "Tech", "Cybersecurity", "Funny/Weird", "Gaming", "Movies", "Arab World", "Tunisia"]
+
+
+def serialize_clusters(clusters):
+    by_cat = defaultdict(list)
+    results = []
+    for c in clusters:
+        cat = "General"
+        if c.articles and c.articles[0].analysis:
+            ca = c.articles[0].analysis.category
+            cat = ca if ca in CATEGORIES else "General"
+
+        html = {
+            "category": cat,
+            "topic": c.topic,
+            "score": round(c.final_score, 2),
+            "trust": f"{c.avg_trustworthiness:.0%}",
+            "coverage": c.total_coverage,
+            "image_url": c.image_url,
+            "top_post_url": c.top_post_url,
+            "articles": [
+                {
+                    "title": a.post.title,
+                    "domain": a.article.source_domain if a.article else "",
+                    "summary": a.analysis.summary if a.analysis else "",
+                    "topics": a.analysis.topics if a.analysis else [],
+                    "trust": f"{a.analysis.trustworthiness_score:.0%}" if a.analysis else "",
+                    "score": a.post.score,
+                    "comments": a.post.num_comments,
+                    "url": a.post.url,
+                    "image": a.article.image_url if a.article else a.post.image_url,
+                    "published": a.post.published,
+                    "published_iso": a.post.published_iso,
+                }
+                for a in c.articles[:5]
+            ],
+        }
+        by_cat[cat].append(html)
+
+        api = {
+            "id": f"cluster-{id(c)}",
+            "topic": c.topic,
+            "category": cat,
+            "final_score": round(c.final_score, 2),
+            "avg_trustworthiness": round(c.avg_trustworthiness, 2),
+            "total_coverage": c.total_coverage,
+            "image_url": c.image_url,
+            "top_post_url": c.top_post_url,
+            "articles": [
+                {
+                    "title": a.post.title,
+                    "url": a.post.url,
+                    "domain": a.article.source_domain if a.article else "",
+                    "summary": a.analysis.summary if a.analysis else "",
+                    "topics": a.analysis.topics if a.analysis else [],
+                    "trust": round(a.analysis.trustworthiness_score, 2) if a.analysis else 0,
+                    "score": a.post.score,
+                    "comments": a.post.num_comments,
+                    "image": a.article.image_url if a.article else a.post.image_url,
+                    "published": a.post.published,
+                    "published_iso": a.post.published_iso,
+                }
+                for a in c.articles[:5]
+            ],
+        }
+        results.append(api)
+
+    data = {
+        "by_cat": dict(by_cat),
+        "results": results,
+        "status": f"ok — {len(clusters)} clusters",
+        "last_refresh": datetime.now(timezone.utc).isoformat(),
+    }
+    with open(CACHE_FILE, "w") as f:
+        json.dump(data, f)
+    logger.info("Cache written to %s (%d clusters)", CACHE_FILE, len(clusters))
+
+
+def main():
+    logger.info("Starting refresh task...")
+    scraper = RSSFeedScraper(cfg)
+    posts = scraper.fetch_posts()
+    items = [NewsItem(post=p) for p in posts]
+    clusters = run_pipeline(items, cfg)
+    serialize_clusters(clusters)
+    logger.info("Refresh task done — %d clusters", len(clusters))
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,22 @@
+# Core
+flask>=3.0.0
+flask-cors>=4.0.0
+gunicorn>=21.2.0
+feedparser>=6.0.0
+requests>=2.28.0
+python-dotenv>=1.0.0
+trafilatura>=1.6.0
+beautifulsoup4>=4.12.0
+apscheduler>=3.10.0
+
+# Reddit (optional — only needed for Reddit source)
+praw>=7.7.0
+
+# NLP / GenAI (optional — program falls back to rule-based if missing)
+# transformers>=4.30.0
+# sentence-transformers>=2.2.0
+# torch>=2.0.0
+# numpy>=1.24.0
+
+# Install with: pip install -r requirements.txt
+# For ML models: pip install transformers sentence-transformers torch numpy

src/aggregator.py

@@ -0,0 +1,156 @@
+import logging
+from collections import Counter
+from typing import Optional
+
+from .models import NewsCluster, NewsItem
+
+logger = logging.getLogger(__name__)
+
+
+class NewsAggregator:
+    def __init__(self, config):
+        self.config = config
+        self._encoder = None
+        self._setup_encoder()
+
+    def _setup_encoder(self):
+        if not self.config.use_local_models:
+            logger.info("Local models disabled — using keyword similarity")
+            return
+        try:
+            from sentence_transformers import SentenceTransformer
+            model_name = f"sentence-transformers/{self.config.embedding_model}"
+            logger.info("Loading embedding model: %s ...", model_name)
+            self._encoder = SentenceTransformer(model_name)
+        except ImportError:
+            logger.warning("sentence-transformers not available — using keyword fallback")
+        except Exception as exc:
+            logger.warning("Embedding model failed: %s", exc)
+
+    def compute_similarity(self, a: str, b: str) -> float:
+        if self._encoder:
+            emb_a = self._encoder.encode(a, normalize_embeddings=True)
+            emb_b = self._encoder.encode(b, normalize_embeddings=True)
+            return float(emb_a @ emb_b)
+        return self._keyword_overlap(a, b)
+
+    @staticmethod
+    def _keyword_overlap(a: str, b: str) -> float:
+        words_a = set(a.lower().split())
+        words_b = set(b.lower().split())
+        if not words_a or not words_b:
+            return 0.0
+        common = words_a & words_b
+        return len(common) / max(len(words_a), len(words_b))
+
+    def cluster_news(self, items: list[NewsItem]) -> list[NewsCluster]:
+        clusters: list[list[NewsItem]] = []
+
+        for item in items:
+            if not item.analysis or not item.article:
+                continue
+
+            text = f"{item.post.title} {item.analysis.summary}"
+            placed = False
+
+            for cluster in clusters:
+                rep = cluster[0]
+                if not rep.analysis:
+                    continue
+                rep_text = f"{rep.post.title} {rep.analysis.summary}"
+                if self.compute_similarity(text, rep_text) >= self.config.similarity_threshold:
+                    cluster.append(item)
+                    placed = True
+                    break
+
+            if not placed:
+                clusters.append([item])
+
+        return self._rank_clusters(clusters)
+
+    def _rank_clusters(self, raw: list[list[NewsItem]]) -> list[NewsCluster]:
+        scored = []
+        for group in raw:
+            if not group:
+                continue
+
+            topic = self._main_topic(group)
+
+            # Highest scoring post represents the cluster
+            best = max(group, key=lambda x: x.post.score)
+
+            avg_trust = sum(
+                it.analysis.trustworthiness_score for it in group if it.analysis
+            ) / max(len(group), 1)
+
+            avg_pop = sum(it.post.score for it in group) / max(len(group), 1)
+
+            # Pick the first non-empty image
+            image_url = ""
+            for it in group:
+                src = it.article.image_url if it.article else ""
+                if src:
+                    image_url = src
+                    break
+                if it.post.image_url:
+                    image_url = it.post.image_url
+                    break
+
+            cluster_score = self._cluster_score(group, avg_trust)
+
+            cluster = NewsCluster(
+                topic=topic,
+                articles=group,
+                total_coverage=len(group),
+                avg_trustworthiness=avg_trust,
+                avg_popularity=avg_pop,
+                top_post_url=best.post.url,
+                final_score=cluster_score,
+                image_url=image_url,
+            )
+            scored.append(cluster)
+
+        return sorted(scored, key=lambda c: c.final_score, reverse=True)
+
+    def _cluster_score(self, group: list[NewsItem], avg_trust: float) -> float:
+        scores = []
+        for item in group:
+            s = avg_trust * 0.50
+
+            # Content quality: longer articles score higher
+            if item.article and item.article.text:
+                title_len = len(item.article.title or "")
+                text_len = len(item.article.text)
+                if text_len > title_len * 3:
+                    s += 0.15
+                elif text_len > title_len * 1.5:
+                    s += 0.08
+
+            # Successfully extracted vs title-only
+            if item.article and item.article.extraction_success:
+                s += 0.10
+            else:
+                s -= 0.10
+
+            # More topics = richer article
+            if item.analysis and item.analysis.topics:
+                s += min(len(item.analysis.topics) * 0.06, 0.18)
+
+            # Having a category means we actually understood it
+            if item.analysis and item.analysis.category != "General":
+                s += 0.05
+
+            scores.append(max(0.05, min(1.0, s)))
+
+        return sum(scores) / max(len(scores), 1)
+
+    @staticmethod
+    def _main_topic(cluster: list[NewsItem]) -> str:
+        counter: Counter[str] = Counter()
+        for item in cluster:
+            if item.analysis:
+                for t in item.analysis.topics:
+                    counter[t] += 1
+        if counter:
+            return counter.most_common(1)[0][0]
+        return (cluster[0].post.title or "")[:60]

src/analyzer.py

@@ -0,0 +1,410 @@
+import html
+import logging
+import re
+from typing import Optional
+
+from .models import Analysis, Article
+
+logger = logging.getLogger(__name__)
+
+RELIABLE_DOMAINS = {
+    "reuters.com": 0.20, "apnews.com": 0.20, "bbc.com": 0.15,
+    "bbc.co.uk": 0.15, "npr.org": 0.12, "wsj.com": 0.12,
+    "economist.com": 0.15, "nature.com": 0.20, "science.org": 0.18,
+    "sciencedaily.com": 0.14, "theguardian.com": 0.08, "nytimes.com": 0.10,
+    "washingtonpost.com": 0.10, "ft.com": 0.14, "bloomberg.com": 0.12,
+}
+
+UNRELIABLE_DOMAINS = {
+    "infowars.com": -0.30, "breitbart.com": -0.20, "dailymail.co.uk": -0.12,
+    "theonion.com": -0.25, "naturalnews.com": -0.30, "zerohedge.com": -0.15,
+}
+
+CLICKBAIT_PATTERNS = [
+    r"you won'?t believe", r"shocked?", r"gobsmacked",
+    r"this is what happens", r"number \d+ will",
+    r"here'?s why", r"what happens next",
+    r"blown away", r"mind.?blowing",
+]
+
+OPINION_MARKERS = [
+    r"\bi think\b", r"\bin my opinion\b", r"\bpersonally\b",
+    r"\bi believe\b", r"\bclearly\b", r"\bobviously\b",
+    r"\bin my view\b", r"\bit seems\b", r"\bi feel\b",
+]
+
+LEFT_KEYWORDS = ["progressive", "equality", "social justice", "climate crisis",
+                 "marginalized", "systemic", "privilege", "inequality"]
+RIGHT_KEYWORDS = ["deregulation", "tax cuts", "free market", "traditional",
+                  "sovereignty", "patriot", "heritage", "small government"]
+
+TOPIC_MAP: dict[str, list[str]] = {
+    "artificial intelligence": [r"\bai\b", r"\bartificial intelligence\b",
+                                r"\bmachine learning\b", r"\bgpt\b", r"\bllm\b",
+                                r"\bneural network", r"\bdeep learning\b"],
+    "climate change": [r"\bclimate\b", r"\bglobal warming\b", r"\bemissions\b",
+                       r"\bcarbon\b", r"\brenewable\b", r"\bsolar\b", r"\bwind turbine",
+                       r"\bheatwave\b", r"\bextreme weather\b", r"\bheat wave\b"],
+    "health": [r"\bhealth\b", r"\bcovid\b", r"\bvaccine\b", r"\bdisease\b",
+               r"\bhospital\b", r"\bmedical\b", r"\bcancer\b", r"\bdrug\b",
+               r"\bod\b", r"\bpandemic\b", r"\bpatient\b", r"\bsurgery\b",
+               r"\bdoctor\b", r"\bnurse\b", r"\btreatment\b", r"\btherapy\b",
+               r"\bdementia\b", r"\bdiabetes\b", r"\bobesity\b", r"\bmental health\b",
+               r"\babortion\b", r"\bpregnant\b", r"\bmedicine\b", r"\bclinical\b",
+               r"\bsymptom\b", r"\bheat\b", r"\brabies\b", r"\bfever\b"],
+    "economy": [r"\beconomy\b", r"\binflation\b", r"\bgdp\b", r"\binterest rate\b",
+                r"\brecession\b", r"\bunemployment\b", r"\bmarket\b", r"\btariff\b",
+                r"\btrade war\b", r"\bdebt\b", r"\bstock\b", r"\bprice\b", r"\bcost\b",
+                r"\bfinancial\b"],
+    "space": [r"\bspace\b", r"\bnasa\b", r"\bspacex\b", r"\bmars\b", r"\brocket\b",
+              r"\bastronaut\b", r"\bgalaxy\b", r"\bplanet\b", r"\borgbit\b",
+              r"\bstellar\b", r"\bcosmic\b"],
+    "cybersecurity": [r"\bcyber\b", r"\bhack", r"\bsecurity breach\b",
+                      r"\bdata breach\b", r"\bransomware\b", r"\bmalware\b",
+                      r"\bphishing\b", r"\bzero.day\b", r"\bfirewall\b",
+                      r"\bencryption\b", r"\bCVE\b", r"\bexploit\b",
+                      r"\bbotnet\b", r"\bDDoS\b", r"\bvulnerability\b", r"\bfraud\b"],
+    "politics": [r"\belection\b", r"\bvot(?:e|ing|er)\b", r"\bcongress\b",
+                 r"\bparliament\b", r"\bsenate\b", r"\bpresident\b",
+                 r"\bgovern(?:ment|or)\b", r"\bGOP\b", r"\bDemocrat\b",
+                 r"\brepublican\b", r"\bpolitician\b", r"\bcandidate\b",
+                 r"\bambassador\b", r"\bdiplomat\b", r"\bsanction\b",
+                 r"\btreaty\b", r"\bembassy\b", r"\bminister\b", r"\bregime\b",
+                 r"\blegislat\b", r"\bpolicy\b", r"\bfederal\b"],
+    "science": [r"\bscien(?:ce|tist|tists|tific)\b", r"\bresearch\b", r"\bstudy\b",
+                r"\bdiscovery\b", r"\bgenome\b", r"\bquantum\b", r"\bparticle\b",
+                r"\bevolution\b", r"\bexperiment\b", r"\bjournal\b", r"\blab\b",
+                r"\bDNA\b", r"\bgene\b", r"\bprotein\b", r"\bbiolog\b",
+                r"\bchemical\b", r"\bphysics\b"],
+    "technology": [r"\btech\b", r"\bsoftware\b", r"\bhardware\b", r"\bchip\b",
+                   r"\bsemiconductor\b", r"\bapp\b", r"\balgorithm\b",
+                   r"\bcomputer\b", r"\brobot\b", r"\bgaming\b", r"\bvideo game\b",
+                   r"\bconsole\b", r"\bmobile\b", r"\bphone\b", r"\blaptop\b",
+                   r"\bsmartphone\b", r"\bgadget\b", r"\bstartup\b",
+                   r"\bplatform\b", r"\bdeveloper\b", r"\bcode\b", r"\bprogramming\b",
+                   r"\bdigital\b", r"\bcloud\b", r"\bdevice\b", r"\bsmart\b",
+                   r"\bIoT\b", r"\bOS\b", r"\bWindows\b", r"\bAndroid\b", r"\biOS\b",
+                   r"\bPlayStation\b", r"\bapp\b", r"\bAI\b", r"\bA\.I",
+                   r"\bEV\b", r"\belectric vehicle\b", r"\bgadget\b",
+                   r"\btechlash\b"],
+    "sports": [r"\bsport\b", r"\bfootball\b", r"\bsoccer\b", r"\bbasketball\b",
+               r"\btennis\b", r"\bworld cup\b", r"\bolympic\b"],
+    "education": [r"\beducation\b", r"\bschool\b", r"\buniversity\b",
+                  r"\bstudent\b", r"\bteacher\b", r"\bcollege\b", r"\bcampus\b"],
+    "immigration": [r"\bimmigra(?:nt|tion)\b", r"\bborder\b", r"\basylum\b",
+                    r"\brefugee\b", r"\bdeport\b", r"\bvisa\b"],
+    "energy": [r"\boil\b", r"\bgas\b", r"\bnuclear\b", r"\benergy\b",
+               r"\bpower plant\b", r"\brenewable\b", r"\bfossil fuel\b"],
+    "world": [r"\bwar\b", r"\bmilitary\b", r"\binvasion\b", r"\bsanction\b",
+              r"\bforeign\b", r"\bdiplomat\b", r"\btreaty\b", r"\bconflict\b",
+              r"\bearthquake\b", r"\bflood\b", r"\bdisaster\b", r"\bpresident\b",
+              r"\bprime minister\b", r"\bgeopolitic\b", r"\balliance\b",
+              r"\bmilitant\b", r"\bguerrilla\b", r"\bceasefire\b", r"\bterrorism\b",
+              r"\bUkraine\b", r"\bRussia\b", r"\bChina\b", r"\bIran\b",
+              r"\batomic\b", r"\bnuclear\b", r"\bmissile\b", r"\bdrone\b",
+              r"\battack\b", r"\bstrike\b", r"\bbomb\b", r"\btroop\b",
+              r"\bsoldier\b", r"\bmissile\b", r"\bdefence\b", r"\bdefense\b",
+              r"\bNATO\b", r"\bUN\b", r"\bICC\b", r"\bintelligence\b",
+              r"\bVatican\b", r"\bCatholic\b"],
+    "funny": [r"\bfunny\b", r"\bjoke\b", r"\bhumor\b", r"\bcomedy\b",
+              r"\bsatire\b", r"\bparody\b", r"\blol\b", r"\bwtf\b",
+              r"\babsurd\b", r"\bridiculous\b", r"\bhilarious\b", r"\bcomic\b",
+              r"\blaugh\b", r"\bclown\b"],
+    "weird": [r"\bweird\b", r"\bstrange\b", r"\bbizarre\b", r"\boddb?all\b",
+              r"\bpeculiar\b", r"\bunusual\b", r"\bodd\b", r"\bunbelievable\b",
+              r"\bsurreal\b", r"\bunconventional\b", r"\bwtf\b"],
+    "onion": [r"\bonion\b", r"\btheonion\b"],
+    "gaming": [r"\bgam(?:e|ing|er|ers)\b", r"\besport\b", r"\bplaystation\b",
+               r"\bxbox\b", r"\bnintendo\b", r"\bsteam\b", r"\bconsole\b",
+               r"\bgta\b", r"\bgrand theft auto\b", r"\bfortnite\b",
+               r"\bminecraft\b", r"\bvalorant\b", r"\bvideogame\b",
+               r"\bvideo game\b"],
+    "movies": [r"\bmovie\b", r"\bfilm\b", r"\bcinema\b", r"\bHollywood\b",
+               r"\bbox office\b", r"\bblockbuster\b", r"\bOscar\b",
+               r"\bactor\b", r"\bactress\b", r"\bscreenplay\b",
+               r"\bdirector\b", r"\bNetflix\b", r"\bDisney\+\b",
+               r"\bHBO\b", r"\breboot\b", r"\bsequel\b", r"\bprequel\b",
+               r"\bIMAX\b", r"\banimation\b"],
+    "tunisia": [r"\bTunisia\b", r"\bTunis\b", r"\bCarthage\b",
+                r"\bSousse\b", r"\bSfax\b"],
+    "arab_world": [r"\barab\b", r"\bgulf\b", r"\bmiddle east\b",
+                   r"\bsaudi\b", r"\bQatar\b", r"\bUAE\b", r"\bDubai\b",
+                   r"\bAbu Dhabi\b", r"\bDoha\b", r"\bRiyadh\b",
+                   r"\bPalestin\b", r"\bGaza\b", r"\bWest Bank\b",
+                   r"\bLeban\b", r"\bBeirut\b", r"\bBaghdad\b",
+                   r"\bCairo\b", r"\bEgypt\b", r"\bSyria\b",
+                   r"\bYemen\b", r"\bAmman\b", r"\bJordan\b",
+                   r"\bOman\b", r"\bKuwait\b", r"\bBahrain\b",
+                   r"\bUnrwa\b", r"\bHezbollah\b", r"\bHouthi\b",
+                   r"\bOPEC\b", r"\bMENA\b"],
+}
+
+TOPIC_TO_CATEGORY: dict[str, str] = {
+    "politics": "Geopolitical",
+    "world": "Geopolitical",
+    "immigration": "Geopolitical",
+    "economy": "Geopolitical",
+    "energy": "Geopolitical",
+    "health": "World Health",
+    "science": "World Health",
+    "technology": "Tech",
+    "artificial intelligence": "Tech",
+    "space": "Tech",
+    "cybersecurity": "Cybersecurity",
+    "funny": "Funny/Weird",
+    "weird": "Funny/Weird",
+    "onion": "Funny/Weird",
+    "sports": "Funny/Weird",
+    "education": "Geopolitical",
+    "climate change": "Geopolitical",
+    "gaming": "Gaming",
+    "movies": "Movies",
+    "tunisia": "Tunisia",
+    "arab_world": "Arab World",
+}
+
+FACTUAL_KEYWORDS = [
+    r"\breport\b", r"\baccording to\b", r"\bsource said\b", r"\bstated\b",
+    r"\bstudy found\b", r"\bdata show\b", r"\bofficial said\b",
+    r"\bresearch suggests\b", r"\bthe study\b", r"\bsurvey\b",
+]
+
+
+class NewsAnalyzer:
+    def __init__(self, config):
+        self.config = config
+        self._summariser = None
+        self._classifier = None
+        self._setup_models()
+
+    def _setup_models(self):
+        if not self.config.use_local_models:
+            logger.info("Local models disabled — using rule-based analysis")
+            return
+        try:
+            from transformers import pipeline
+            logger.info("Loading summariser: %s ...", self.config.summarization_model)
+            self._summariser = pipeline(
+                "summarization",
+                model=self.config.summarization_model,
+                tokenizer=self.config.summarization_model,
+            )
+            logger.info("Loading zero-shot classifier ...")
+            self._classifier = pipeline(
+                "zero-shot-classification",
+                model="typeform/distilbert-base-uncased-mnli",
+            )
+        except ImportError:
+            logger.warning("transformers not available — using rule-based analysis")
+        except Exception as exc:
+            logger.warning("Model loading failed: %s — using rule-based", exc)
+
+    def analyze(self, article: Article) -> Analysis:
+        summary = self._summarise(article)
+        topics = self._classify_topics(article, summary)
+        trust = self._assess_trustworthiness(article)
+        is_opinion = self._detect_opinion(article.text or "")
+        leaning = self._detect_political_leaning(article.text or "")
+
+        category = self._map_category(topics)
+
+        return Analysis(
+            summary=summary,
+            topics=topics,
+            trustworthiness_score=trust,
+            is_opinion=is_opinion,
+            political_leaning=leaning,
+            category=category,
+        )
+
+    def _summarise(self, article: Article) -> str:
+        title = article.title or ""
+        text = article.text or ""
+
+        if self._summariser:
+            try:
+                input_text = text[:1024]
+                out = self._summariser(input_text, max_length=130, min_length=30,
+                                       do_sample=False)
+                return out[0]["summary_text"]
+            except Exception as exc:
+                logger.debug("Summariser failed: %s", exc)
+
+        body = self._strip_metadata(text)
+        body = self._strip_title_line(body, title)
+
+        if not body or len(body) < len(title) * 1.5:
+            return ""
+
+        title_norm = self._norm(title)
+        sentences = re.split(r"(?<=[.!?])\s+", body.strip())
+        selected = []
+        for s in sentences:
+            s = s.strip()
+            if not s:
+                continue
+            if self._is_title_like(s, title_norm):
+                continue
+            selected.append(s)
+            if len(selected) >= 2:
+                break
+        return " ".join(selected) if selected else ""
+
+    @staticmethod
+    def _norm(s: str) -> str:
+        return re.sub(r"\s+", " ", html.unescape(s).lower().strip()).rstrip(".")
+
+    @staticmethod
+    def _is_title_like(sentence: str, title_norm: str) -> bool:
+        s_norm = re.sub(r"\s+", " ", sentence.lower().strip()).rstrip(".")
+        if s_norm == title_norm:
+            return True
+        words_s = set(s_norm.split())
+        words_t = set(title_norm.split())
+        if not words_s or not words_t:
+            return False
+        short, long = (words_s, words_t) if len(words_s) < len(words_t) else (words_t, words_s)
+        overlap = len(short & long) / max(len(short), len(long))
+        return overlap > 0.7
+
+    @staticmethod
+    def _strip_title_line(text: str, title: str) -> str:
+        lines = text.split("\n")
+        if not lines:
+            return text
+        first = lines[0].strip()
+        if not first:
+            return "\n".join(lines[1:]).strip()
+        if len(first) < 150 and not re.search(r"[.!?]$", first):
+            return "\n".join(lines[1:]).strip()
+        return text
+
+    @staticmethod
+    def _strip_metadata(text: str) -> str:
+        lines = text.split("\n")
+        cleaned = []
+        for line in lines:
+            clean = line.strip()
+            if re.match(r"^\s*[—\-] (Published|Updated|BBC News|Image|Copyright)", clean, re.IGNORECASE):
+                continue
+            cleaned.append(line)
+        return "\n".join(cleaned).strip()
+
+    def _classify_topics(self, article: Article, summary: str) -> list[str]:
+        if self._classifier:
+            try:
+                text = f"{article.title} {summary}" if summary else article.title
+                candidates = self.config.user_interests + ["other"]
+                result = self._classifier(text[:512], candidates)
+                return [
+                    label for label, score in zip(result["labels"], result["scores"])
+                    if score > 0.25
+                ]
+            except Exception as exc:
+                logger.debug("Classifier failed: %s", exc)
+
+        return self._keyword_topic_match(article, summary)
+
+    DOMAIN_TOPICS: dict[str, str] = {
+        "krebsonsecurity.com": "cybersecurity",
+        "bleepingcomputer.com": "cybersecurity",
+        "theonion.com": "onion",
+        "ign.com": "gaming",
+        "eurogamer.net": "gaming",
+        "pcgamer.com": "gaming",
+        "rockpapershotgun.com": "gaming",
+        "kotaku.com": "gaming",
+        "gamespot.com": "gaming",
+        "arabnews.com": "arab_world",
+        "middleeasteye.net": "arab_world",
+        "thenationalnews.com": "arab_world",
+        "newarab.com": "arab_world",
+        "therecord.media": "cybersecurity",
+        "threatpost.com": "cybersecurity",
+        "thedailymash.co.uk": "funny",
+        "babylonbee.com": "onion",
+        "polygon.com": "gaming",
+        "variety.com": "movies",
+        "hollywoodreporter.com": "movies",
+        "deadline.com": "movies",
+        "screenrant.com": "movies",
+        "tunisiaonlinenews.com": "tunisia",
+    }
+
+    def _keyword_topic_match(self, article: Article, summary: str) -> list[str]:
+        text = f"{article.title} {summary}" if summary else article.title
+        found = []
+        for topic, patterns in TOPIC_MAP.items():
+            if any(re.search(p, text, re.IGNORECASE) for p in patterns):
+                found.append(topic)
+        domain = re.sub(r"^www\.", "", (article.source_domain or ""))
+        mapped = self.DOMAIN_TOPICS.get(domain)
+        # Only use domain fallback if keyword matching found nothing
+        if mapped and mapped not in found:
+            if not found:
+                found.append(mapped)
+        return found
+
+    def _assess_trustworthiness(self, article: Article) -> float:
+        score = 0.40
+
+        domain = article.source_domain or ""
+        clean_domain = re.sub(r"^www\.", "", domain)
+        score += RELIABLE_DOMAINS.get(clean_domain, 0.0)
+        score += UNRELIABLE_DOMAINS.get(clean_domain, 0.0)
+
+        text = article.text or ""
+        title = article.title or ""
+        if article.extraction_success is False:
+            score -= 0.15
+        elif len(text) > len(title) * 3:
+            score += 0.05
+
+        word_count = len(text.split())
+        if word_count > 200:
+            score += 0.10
+        elif word_count > 100:
+            score += 0.05
+        elif word_count > 50:
+            score += 0.02
+
+        factual_count = sum(1 for p in FACTUAL_KEYWORDS if re.search(p, text, re.IGNORECASE))
+        score += min(factual_count * 0.02, 0.08)
+
+        if any(re.search(p, title, re.IGNORECASE) for p in CLICKBAIT_PATTERNS):
+            score -= 0.20
+
+        opinion_count = sum(1 for p in OPINION_MARKERS if re.search(p, text, re.IGNORECASE))
+        score -= opinion_count * 0.05
+
+        return max(0.05, min(1.0, score))
+
+    @staticmethod
+    def _map_category(topics: list[str]) -> str:
+        priority = ["onion", "funny", "weird", "cybersecurity", "gaming",
+                     "technology", "artificial intelligence", "health", "science",
+                     "tunisia", "arab_world", "world", "politics", "immigration", "economy",
+                     "energy", "education", "climate change", "space", "sports", "movies"]
+        topic_set = {t.lower() for t in topics}
+        for p in priority:
+            if p in topic_set:
+                mapped = TOPIC_TO_CATEGORY.get(p)
+                if mapped:
+                    return mapped
+        return "General"
+
+    @staticmethod
+    def _detect_opinion(text: str) -> bool:
+        count = sum(1 for p in OPINION_MARKERS if re.search(p, text.lower()))
+        return count >= 3
+
+    @staticmethod
+    def _detect_political_leaning(text: str) -> str:
+        text_lower = text.lower()
+        left = sum(1 for k in LEFT_KEYWORDS if k in text_lower)
+        right = sum(1 for k in RIGHT_KEYWORDS if k in text_lower)
+        if left > right + 1:
+            return "left-leaning"
+        if right > left + 1:
+            return "right-leaning"
+        return "centrist"

src/extractor.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+from .models import Article
+
+logger = logging.getLogger(__name__)
+
+
+class ArticleExtractor:
+    def __init__(self):
+        self._trafilatura = None
+        self._init_backend()
+
+    def _init_backend(self):
+        try:
+            import trafilatura
+            self._trafilatura = trafilatura
+        except ImportError:
+            logger.info("trafilatura not installed — using fallback extractor")
+
+    def extract(self, url: str) -> Optional[Article]:
+        domain = urlparse(url).netloc
+        try:
+            if self._trafilatura:
+                return self._extract_trafilatura(url, domain)
+            return self._extract_fallback(url, domain)
+        except Exception as exc:
+            logger.debug("Extraction failed for %s: %s", url, exc)
+            return None
+
+    def _extract_trafilatura(self, url: str, domain: str) -> Optional[Article]:
+        downloaded = self._trafilatura.fetch_url(url)
+        if not downloaded:
+            return None
+        text = self._trafilatura.extract(downloaded)
+        if not text:
+            return None
+        title = self._extract_title_meta(downloaded) or ""
+        image = self._extract_og_image(downloaded)
+        return Article(url=url, title=title, text=text, source_domain=domain, image_url=image)
+
+    def _extract_fallback(self, url: str, domain: str) -> Optional[Article]:
+        resp = requests.get(url, headers={"User-Agent": "newsapp/1.0"}, timeout=15)
+        resp.raise_for_status()
+        soup = BeautifulSoup(resp.text, "html.parser")
+
+        title = ""
+        if soup.title:
+            title = soup.title.get_text(strip=True)
+
+        image = self._extract_og_image_soup(soup)
+
+        paragraphs = soup.find_all("p")
+        text = "\n\n".join(p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True))
+        if not text:
+            return None
+        return Article(url=url, title=title, text=text, source_domain=domain, image_url=image)
+
+    @staticmethod
+    def _extract_title_meta(html: str) -> Optional[str]:
+        import html as html_mod
+        import re
+        m = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
+        return html_mod.unescape(m.group(1).strip()) if m else None
+
+    @staticmethod
+    def _extract_og_image(html: str) -> str:
+        import re
+        m = re.search(
+            r'<meta\s+[^>]*property=["\']og:image["\'][^>]*content=["\']([^"\']+)["\']',
+            html, re.IGNORECASE,
+        )
+        if m:
+            return m.group(1)
+        m = re.search(
+            r'<meta\s+[^>]*content=["\']([^"\']+)["\'][^>]*property=["\']og:image["\']',
+            html, re.IGNORECASE,
+        )
+        return m.group(1) if m else ""
+
+    @staticmethod
+    def _extract_og_image_soup(soup) -> str:
+        for prop in ("og:image", "twitter:image"):
+            tag = soup.find("meta", property=prop) or soup.find("meta", attrs={"name": prop})
+            if tag and tag.get("content"):
+                return tag["content"]
+        return ""
+
+
+import requests
+from bs4 import BeautifulSoup

src/hn_scraper.py

@@ -0,0 +1,48 @@
+import logging
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
+import requests
+
+from .models import RedditPost
+
+logger = logging.getLogger(__name__)
+
+API_BASE = "https://hacker-news.firebaseio.com/v0"
+
+
+class HackerNewsScraper:
+    def __init__(self, config):
+        self.config = config
+
+    def fetch_posts(self) -> list[RedditPost]:
+        resp = requests.get(f"{API_BASE}/topstories.json", timeout=15)
+        resp.raise_for_status()
+        all_ids = resp.json()
+
+        limit = min(self.config.posts_per_subreddit * 3, 50)
+        posts = []
+        for story_id in all_ids[:limit]:
+            try:
+                detail = requests.get(f"{API_BASE}/item/{story_id}.json", timeout=10)
+                detail.raise_for_status()
+                data = detail.json()
+                if not data or data.get("type") != "story":
+                    continue
+                url = data.get("url") or f"https://news.ycombinator.com/item?id={story_id}"
+                posts.append(RedditPost(
+                    id=f"hn_{story_id}",
+                    title=data.get("title", ""),
+                    url=url,
+                    subreddit="hackernews",
+                    score=data.get("score", 0),
+                    num_comments=data.get("descendants", 0),
+                    source_domain=urlparse(url).netloc,
+                ))
+            except Exception as exc:
+                logger.debug("Failed to fetch HN item %s: %s", story_id, exc)
+
+        posts.sort(key=lambda p: p.score, reverse=True)
+        posts = posts[: self.config.posts_per_subreddit]
+        logger.info("Fetched %d posts from Hacker News", len(posts))
+        return posts

src/mockdata.py

@@ -0,0 +1,189 @@
+import random
+import time
+
+from .models import Analysis, Article, NewsItem, RedditPost
+
+_MOCK_ARTICLES = [
+    {
+        "title": "New AI model achieves breakthrough in protein folding prediction",
+        "text": (
+            "Researchers at DeepMind and several universities have announced a major breakthrough "
+            "in protein folding prediction using a new deep learning architecture. The model, called "
+            "AlphaFold-Next, is able to predict protein structures with accuracy approaching "
+            "experimental methods. This advancement could accelerate drug discovery and our "
+            "understanding of diseases. The team trained the model on a dataset of over 100,000 "
+            "known protein structures and used a novel attention mechanism to capture long-range "
+            "interactions between amino acids. Early tests show the model generalises well to "
+            "previously unseen protein families."
+        ),
+        "domain": "nature.com",
+        "subreddit": "science",
+        "score": 5420,
+        "comments": 342,
+    },
+    {
+        "title": "WHO declares new global health emergency as novel virus spreads across continents",
+        "text": (
+            "The World Health Organization has declared a Public Health Emergency of International "
+            "Concern as a novel respiratory virus continues to spread rapidly across multiple "
+            "continents. The virus, which emerged in Southeast Asia, has been detected in 15 "
+            "countries so far. Health officials are implementing containment measures including "
+            "travel restrictions and increased surveillance. The WHO is coordinating with national "
+            "health agencies to ensure a rapid response. Vaccines are expected to begin clinical "
+            "trials within six months, according to the Director-General."
+        ),
+        "domain": "reuters.com",
+        "subreddit": "worldnews",
+        "score": 8210,
+        "comments": 2801,
+    },
+    {
+        "title": "New study links ultra-processed foods to increased cancer risk",
+        "text": (
+            "A comprehensive study published in The Lancet has found a significant correlation "
+            "between consumption of ultra-processed foods and increased risk of developing certain "
+            "types of cancer. The study followed over 200,000 participants for 15 years and "
+            "controlled for lifestyle factors such as smoking and exercise. Researchers found that "
+            "participants who consumed the highest levels of ultra-processed foods had a 23% higher "
+            "risk of developing colorectal cancer. The findings add to growing evidence that dietary "
+            "patterns play a crucial role in cancer prevention."
+        ),
+        "domain": "bbc.com",
+        "subreddit": "health",
+        "score": 3890,
+        "comments": 567,
+    },
+    {
+        "title": "SpaceX successfully launches satellite constellation for global internet coverage",
+        "text": (
+            "SpaceX has successfully launched another batch of 60 Starlink satellites, bringing the "
+            "total constellation size to over 5,000. The Falcon 9 rocket lifted off from Cape "
+            "Canaveral and successfully deployed the satellites in low Earth orbit. This expansion "
+            "will bring high-speed internet access to previously unserved rural areas across the "
+            "globe. The company plans to increase the constellation to 12,000 satellites within "
+            "the next three years, with initial tests showing latency as low as 20 milliseconds."
+        ),
+        "domain": "reuters.com",
+        "subreddit": "technology",
+        "score": 4560,
+        "comments": 890,
+    },
+    {
+        "title": "Federal Reserve signals interest rate cut as inflation continues to cool",
+        "text": (
+            "The Federal Reserve has signalled it may cut interest rates at its next meeting as "
+            "inflation continues to trend downward toward the 2% target. Recent economic data "
+            "shows consumer prices rose only 2.3% year-over-year, down from a peak of 9.1% two "
+            "years ago. Fed Chair Jerome Powell stated that while progress has been made, the "
+            "committee would remain data-dependent. Markets responded positively, with the S&P "
+            "500 rising 1.2% on the news. Economists expect a quarter-point cut in September."
+        ),
+        "domain": "wsj.com",
+        "subreddit": "economy",
+        "score": 3200,
+        "comments": 1200,
+    },
+    {
+        "title": "Global climate summit reaches historic agreement on fossil fuel phase-out",
+        "text": (
+            "Nearly 200 nations have reached a landmark agreement to phase out fossil fuels at the "
+            "UN Climate Summit in Dubai. The agreement sets a timeline for reducing coal, oil, and "
+            "gas production, with developed nations committing to faster reductions. Developing "
+            "countries will receive financial support through a new climate fund worth $100 billion "
+            "annually. Environmental groups have cautiously welcomed the deal while noting that "
+            "the timeline may need to accelerate to meet Paris Agreement targets. The agreement "
+            "marks the first time all nations have explicitly committed to transitioning away from "
+            "fossil fuels."
+        ),
+        "domain": "theguardian.com",
+        "subreddit": "worldnews",
+        "score": 9500,
+        "comments": 3400,
+    },
+    {
+        "title": "Cybersecurity researchers discover zero-day exploit affecting billions of devices",
+        "text": (
+            "Security researchers have discovered a critical vulnerability in a widely-used "
+            "networking library that affects an estimated 3 billion devices worldwide. The "
+            "zero-day exploit, dubbed 'PacketStorm', allows remote code execution without user "
+            "interaction. Major technology companies including Google, Apple, and Microsoft have "
+            "released emergency patches. Users are strongly advised to update their devices "
+            "immediately. The vulnerability has been present in the codebase for over a decade "
+            "and was discovered during a routine security audit."
+        ),
+        "domain": "reuters.com",
+        "subreddit": "technology",
+        "score": 6700,
+        "comments": 1500,
+    },
+    {
+        "title": "Unprecedented heatwave breaks temperature records across Europe",
+        "text": (
+            "An unprecedented heatwave is sweeping across Europe, with temperatures exceeding "
+            "45°C in several countries. Multiple heat records have been broken, including the "
+            "all-time high for the United Kingdom at 42.3°C. Authorities have issued red alerts "
+            "and are urging residents to stay indoors. The extreme weather has been linked to "
+            "climate change by leading meteorological agencies. Hospitals are reporting increased "
+            "admissions for heat-related illnesses, and transport networks have been disrupted "
+            "due to heat-damaged infrastructure."
+        ),
+        "domain": "bbc.com",
+        "subreddit": "worldnews",
+        "score": 7800,
+        "comments": 2100,
+    },
+    {
+        "title": "New CRISPR therapy shows promising results in clinical trial for sickle cell disease",
+        "text": (
+            "A groundbreaking CRISPR-based gene therapy has shown remarkable results in a Phase 3 "
+            "clinical trial for sickle cell disease. Out of 45 patients, 42 showed complete "
+            "remission of symptoms 12 months after treatment. The therapy, developed by Vertex "
+            "Pharmaceuticals, uses CRISPR-Cas9 to edit the patient's own stem cells, correcting "
+            "the genetic mutation responsible for the disease. The FDA has granted breakthrough "
+            "therapy designation, potentially accelerating approval. This marks one of the first "
+            "successful CRISPR-based treatments for a genetic blood disorder."
+        ),
+        "domain": "nature.com",
+        "subreddit": "science",
+        "score": 5100,
+        "comments": 450,
+    },
+    {
+        "title": "AI regulation bill passes Senate with bipartisan support",
+        "text": (
+            "The US Senate has passed a landmark artificial intelligence regulation bill with "
+            "significant bipartisan support. The legislation requires AI companies to conduct "
+            "safety testing before releasing powerful models, establish transparency requirements, "
+            "and create a new federal agency to oversee AI development. The bill was co-sponsored "
+            "by senators from both parties and represents one of the most comprehensive AI "
+            "governance frameworks in the world. Tech companies have expressed mixed reactions, "
+            "with some supporting the clarity while others worry about innovation impact."
+        ),
+        "domain": "nytimes.com",
+        "subreddit": "politics",
+        "score": 4300,
+        "comments": 1800,
+    },
+]
+
+
+def generate_demo_items() -> list[NewsItem]:
+    items = []
+    for art in _MOCK_ARTICLES:
+        post = RedditPost(
+            id=f"mock_{random.randint(100000, 999999)}",
+            title=art["title"],
+            url=f"https://{art['domain']}/article/{random.randint(1000, 9999)}",
+            subreddit=art["subreddit"],
+            score=art["score"],
+            num_comments=art["comments"],
+            source_domain=art["domain"],
+        )
+        article = Article(
+            url=post.url,
+            title=art["title"],
+            text=art["text"],
+            source_domain=art["domain"],
+        )
+        items.append(NewsItem(post=post, article=article))
+    return items

src/models.py

@@ -0,0 +1,56 @@
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class RedditPost:
+    id: str
+    title: str
+    url: str
+    subreddit: str
+    score: int
+    num_comments: int
+    source_domain: str = ""
+    image_url: str = ""
+    published: str = ""
+    published_iso: str = ""
+
+
+@dataclass
+class Article:
+    url: str
+    title: str
+    text: str
+    source_domain: str
+    extraction_success: bool = True
+    image_url: str = ""
+
+
+@dataclass
+class Analysis:
+    summary: str
+    topics: list[str]
+    trustworthiness_score: float
+    is_opinion: bool
+    political_leaning: str = "centrist"
+    category: str = "General"
+
+
+@dataclass
+class NewsItem:
+    post: RedditPost
+    article: Optional[Article] = None
+    analysis: Optional[Analysis] = None
+    final_score: float = 0.0
+
+
+@dataclass
+class NewsCluster:
+    topic: str
+    articles: list[NewsItem]
+    total_coverage: int
+    avg_trustworthiness: float
+    avg_popularity: float
+    top_post_url: str
+    final_score: float = 0.0
+    image_url: str = ""

src/presenter.py

@@ -0,0 +1,48 @@
+from collections import defaultdict
+
+from .models import NewsCluster
+
+
+class NewsPresenter:
+    CATEGORIES = ["Geopolitical", "World Health", "Tech", "Cybersecurity", "Funny/Weird", "Gaming", "Movies", "Arab World", "Tunisia"]
+
+    @staticmethod
+    def display(clusters: list[NewsCluster], top_n: int = 10):
+        if not clusters:
+            print("\n  No news stories matched your interests this round.")
+            return
+
+        by_cat: dict[str, list[NewsCluster]] = defaultdict(list)
+        for c in clusters:
+            cat = "General"
+            if c.articles and c.articles[0].analysis:
+                ca = c.articles[0].analysis.category
+                cat = ca if ca in NewsPresenter.CATEGORIES else "General"
+            by_cat[cat].append(c)
+
+        print("╔" + "═" * 78 + "╗")
+        print("║  📰  NEWS DIGEST  —  Top Stories  ║".center(80))
+        print("╚" + "═" * 78 + "╝")
+
+        for cat in NewsPresenter.CATEGORIES:
+            items = by_cat.get(cat, [])
+            items.sort(key=lambda x: (x.articles[0].post.published_iso or "", x.final_score), reverse=True)
+            items = items[:top_n]
+            if not items:
+                continue
+            print(f"\n  ══ {cat} ({len(items)}) ══\n")
+            for i, cluster in enumerate(items, 1):
+                print(f"  #{i:<2} [{cluster.topic:<30}]  "
+                      f"Score: {cluster.final_score:.2f}  "
+                      f"Trust: {cluster.avg_trustworthiness:.0%}")
+                item = cluster.articles[0]
+                title = item.post.title[:72] + "…" if len(item.post.title) > 72 else item.post.title
+                print(f"       {title}")
+                if item.post.published:
+                    print(f"       📅 {item.post.published}")
+                if item.analysis and item.analysis.summary:
+                    short = item.analysis.summary[:72] + "…" if len(item.analysis.summary) > 72 else item.analysis.summary
+                    print(f"       → {short}")
+                print()
+
+        print("▔" * 80)

src/rss_feed_scraper.py

@@ -0,0 +1,164 @@
+import logging
+import time
+from email.utils import parsedate_to_datetime
+from urllib.parse import urlparse, urlencode, parse_qs
+
+import feedparser
+import requests
+
+from .models import RedditPost
+
+logger = logging.getLogger(__name__)
+
+TRACKING_PARAMS = {"at_medium", "at_campaign", "ref", "utm_source", "utm_medium", "utm_campaign"}
+
+DEFAULT_FEEDS = [
+    # Geopolitical
+    "https://feeds.bbci.co.uk/news/world/rss.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/World.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/Politics.xml",
+    "https://feeds.npr.org/1001/rss.xml",
+    "https://www.aljazeera.com/xml/rss/all.xml",
+    "https://www.theguardian.com/world/rss",
+    # World Health
+    "https://feeds.bbci.co.uk/news/health/rss.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/Science.xml",
+    "https://www.statnews.com/feed/",
+    "https://www.sciencedaily.com/rss/all.xml",
+    # Tech
+    "https://feeds.bbci.co.uk/news/technology/rss.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
+    "https://techcrunch.com/feed/",
+    "https://www.wired.com/feed/rss",
+    "https://www.theverge.com/rss/index.xml",
+    "https://arstechnica.com/feed/",
+    # Cybersecurity
+    "https://feeds.feedburner.com/TheHackerNews",
+    "https://krebsonsecurity.com/feed/",
+    "https://www.bleepingcomputer.com/feed/",
+    "https://threatpost.com/feed/",
+    "https://therecord.media/feed/",
+    # Funny / Weird
+    "https://www.theonion.com/rss",
+    "https://www.reddit.com/r/nottheonion/.rss",
+    "https://www.thedailymash.co.uk/feed",
+    "https://babylonbee.com/feed",
+    # Gaming
+    "https://feeds.ign.com/ign/all",
+    "https://www.eurogamer.net/feed",
+    "https://www.pcgamer.com/rss/",
+    "https://www.kotaku.com/rss",
+    "https://www.gamespot.com/feeds/news/",
+    "https://www.polygon.com/rss/index.xml",
+    # Movies
+    "https://variety.com/feed/",
+    "https://www.hollywoodreporter.com/feed/",
+    "https://deadline.com/feed/",
+    "https://screenrant.com/feed/",
+    # Arab World
+    "https://www.arabnews.com/rss.xml",
+    "https://www.middleeasteye.net/rss",
+    "https://www.newarab.com/rss.xml",
+    "https://www.france24.com/en/middle-east/rss",
+    # Tunisia
+    "https://www.tunisiaonlinenews.com/feed/",
+    "https://northafricapost.com/feed/",
+    "https://www.africanews.com/feed/",
+]
+
+
+class RSSFeedScraper:
+    def __init__(self, config):
+        self.config = config
+        self.feeds = getattr(config, "rss_feeds", DEFAULT_FEEDS)
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": "Mozilla/5.0 (compatible; newsapp/1.0)",
+        })
+
+    def fetch_posts(self) -> list[RedditPost]:
+        seen_titles = set()
+        posts = []
+        for url in self.feeds:
+            logger.info("Fetching RSS: %s", url)
+            try:
+                if "reddit.com" in url:
+                    time.sleep(2.5)
+                resp = self.session.get(url, timeout=15)
+                resp.raise_for_status()
+                feed = feedparser.parse(resp.content)
+                for entry in feed.entries[: self.config.posts_per_subreddit]:
+                    post = self._entry_to_post(entry, url)
+                    if post and post.title not in seen_titles:
+                        seen_titles.add(post.title)
+                        posts.append(post)
+            except Exception as exc:
+                logger.warning("Failed RSS %s: %s", url, exc)
+            time.sleep(1.0)
+        return posts
+
+    @staticmethod
+    def _clean_url(url: str) -> str:
+        parsed = urlparse(url)
+        if not parsed.query:
+            return url
+        params = parse_qs(parsed.query)
+        clean = {k: v[0] for k, v in params.items() if k.lower() not in TRACKING_PARAMS}
+        if not clean:
+            return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+        return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{urlencode(clean)}"
+
+    @staticmethod
+    def _extract_image(entry) -> str:
+        for key in ("media_content", "media_thumbnail"):
+            items = entry.get(key) or []
+            for item in items:
+                url = item.get("url", "")
+                if url:
+                    return url
+        for link in entry.get("links", []):
+            if link.get("rel") == "enclosure" and "image" in link.get("type", ""):
+                return link.get("href", "")
+        return ""
+
+    def _entry_to_post(self, entry, feed_url: str) -> RedditPost | None:
+        title = entry.get("title", "")
+        if not title:
+            return None
+        link = self._clean_url(entry.get("link", ""))
+        domain = urlparse(link).netloc or urlparse(feed_url).netloc
+        image_url = self._extract_image(entry)
+        published, published_iso = self._format_date(entry)
+        return RedditPost(
+            id=entry.get("id", entry.get("guid", link)),
+            title=title,
+            url=link,
+            subreddit=domain,
+            score=0, num_comments=0,
+            source_domain=domain,
+            image_url=image_url,
+            published=published,
+            published_iso=published_iso,
+        )
+
+    @staticmethod
+    def _format_date(entry) -> tuple[str, str]:
+        raw = entry.get("published") or entry.get("updated") or ""
+        if not raw:
+            return ("", "")
+        try:
+            dt = parsedate_to_datetime(raw)
+            display = dt.strftime("%d %b %Y")
+            iso = dt.strftime("%Y-%m-%d")
+            return (display, iso)
+        except Exception:
+            if len(raw) >= 10 and raw[4] == "-" and raw[7] == "-":
+                iso = raw[:10]
+                from datetime import datetime
+                try:
+                    dt = datetime.strptime(iso, "%Y-%m-%d")
+                    display = dt.strftime("%d %b %Y")
+                except Exception:
+                    display = iso
+                return (display, iso)
+            return (raw[:16], "")

src/scraper.py

@@ -0,0 +1,65 @@
+import logging
+from urllib.parse import urlparse
+
+from .models import RedditPost
+
+logger = logging.getLogger(__name__)
+
+
+class RedditScraper:
+    def __init__(self, config):
+        self.config = config
+        self._praw = None
+        self._init_praw()
+
+    def _init_praw(self):
+        cid = self.config.reddit_client_id
+        secret = self.config.reddit_client_secret
+        if not cid or not secret:
+            raise ValueError(
+                "Reddit API credentials not found.\n"
+                "  Set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env, or\n"
+                "  use python main.py --demo to run with sample data."
+            )
+        try:
+            import praw
+            self._praw = praw.Reddit(
+                client_id=cid,
+                client_secret=secret,
+                user_agent=self.config.reddit_user_agent,
+            )
+            logger.info("Reddit API initialised via PRAW")
+        except ImportError:
+            raise ImportError("praw is required. Install with: pip install praw")
+
+    def fetch_posts(self) -> list[RedditPost]:
+        seen = set()
+        posts = []
+        for sub in self.config.news_subreddits:
+            logger.info("Fetching r/%s ...", sub)
+            try:
+                batch = self._fetch_subreddit(sub)
+                for p in batch:
+                    if p.id not in seen:
+                        seen.add(p.id)
+                        posts.append(p)
+            except Exception as exc:
+                logger.error("Failed to fetch r/%s: %s", sub, exc)
+        return posts
+
+    def _fetch_subreddit(self, subreddit: str) -> list[RedditPost]:
+        results = []
+        sub = self._praw.subreddit(subreddit)
+        for submission in sub.hot(limit=self.config.posts_per_subreddit):
+            if submission.is_self:
+                continue
+            results.append(RedditPost(
+                id=submission.id,
+                title=submission.title,
+                url=submission.url,
+                subreddit=subreddit.lower(),
+                score=submission.score,
+                num_comments=submission.num_comments,
+                source_domain=urlparse(submission.url).netloc,
+            ))
+        return results

src/src/aggregator.py

@@ -0,0 +1,156 @@
+import logging
+from collections import Counter
+from typing import Optional
+
+from .models import NewsCluster, NewsItem
+
+logger = logging.getLogger(__name__)
+
+
+class NewsAggregator:
+    def __init__(self, config):
+        self.config = config
+        self._encoder = None
+        self._setup_encoder()
+
+    def _setup_encoder(self):
+        if not self.config.use_local_models:
+            logger.info("Local models disabled — using keyword similarity")
+            return
+        try:
+            from sentence_transformers import SentenceTransformer
+            model_name = f"sentence-transformers/{self.config.embedding_model}"
+            logger.info("Loading embedding model: %s ...", model_name)
+            self._encoder = SentenceTransformer(model_name)
+        except ImportError:
+            logger.warning("sentence-transformers not available — using keyword fallback")
+        except Exception as exc:
+            logger.warning("Embedding model failed: %s", exc)
+
+    def compute_similarity(self, a: str, b: str) -> float:
+        if self._encoder:
+            emb_a = self._encoder.encode(a, normalize_embeddings=True)
+            emb_b = self._encoder.encode(b, normalize_embeddings=True)
+            return float(emb_a @ emb_b)
+        return self._keyword_overlap(a, b)
+
+    @staticmethod
+    def _keyword_overlap(a: str, b: str) -> float:
+        words_a = set(a.lower().split())
+        words_b = set(b.lower().split())
+        if not words_a or not words_b:
+            return 0.0
+        common = words_a & words_b
+        return len(common) / max(len(words_a), len(words_b))
+
+    def cluster_news(self, items: list[NewsItem]) -> list[NewsCluster]:
+        clusters: list[list[NewsItem]] = []
+
+        for item in items:
+            if not item.analysis or not item.article:
+                continue
+
+            text = f"{item.post.title} {item.analysis.summary}"
+            placed = False
+
+            for cluster in clusters:
+                rep = cluster[0]
+                if not rep.analysis:
+                    continue
+                rep_text = f"{rep.post.title} {rep.analysis.summary}"
+                if self.compute_similarity(text, rep_text) >= self.config.similarity_threshold:
+                    cluster.append(item)
+                    placed = True
+                    break
+
+            if not placed:
+                clusters.append([item])
+
+        return self._rank_clusters(clusters)
+
+    def _rank_clusters(self, raw: list[list[NewsItem]]) -> list[NewsCluster]:
+        scored = []
+        for group in raw:
+            if not group:
+                continue
+
+            topic = self._main_topic(group)
+
+            # Highest scoring post represents the cluster
+            best = max(group, key=lambda x: x.post.score)
+
+            avg_trust = sum(
+                it.analysis.trustworthiness_score for it in group if it.analysis
+            ) / max(len(group), 1)
+
+            avg_pop = sum(it.post.score for it in group) / max(len(group), 1)
+
+            # Pick the first non-empty image
+            image_url = ""
+            for it in group:
+                src = it.article.image_url if it.article else ""
+                if src:
+                    image_url = src
+                    break
+                if it.post.image_url:
+                    image_url = it.post.image_url
+                    break
+
+            cluster_score = self._cluster_score(group, avg_trust)
+
+            cluster = NewsCluster(
+                topic=topic,
+                articles=group,
+                total_coverage=len(group),
+                avg_trustworthiness=avg_trust,
+                avg_popularity=avg_pop,
+                top_post_url=best.post.url,
+                final_score=cluster_score,
+                image_url=image_url,
+            )
+            scored.append(cluster)
+
+        return sorted(scored, key=lambda c: c.final_score, reverse=True)
+
+    def _cluster_score(self, group: list[NewsItem], avg_trust: float) -> float:
+        scores = []
+        for item in group:
+            s = avg_trust * 0.50
+
+            # Content quality: longer articles score higher
+            if item.article and item.article.text:
+                title_len = len(item.article.title or "")
+                text_len = len(item.article.text)
+                if text_len > title_len * 3:
+                    s += 0.15
+                elif text_len > title_len * 1.5:
+                    s += 0.08
+
+            # Successfully extracted vs title-only
+            if item.article and item.article.extraction_success:
+                s += 0.10
+            else:
+                s -= 0.10
+
+            # More topics = richer article
+            if item.analysis and item.analysis.topics:
+                s += min(len(item.analysis.topics) * 0.06, 0.18)
+
+            # Having a category means we actually understood it
+            if item.analysis and item.analysis.category != "General":
+                s += 0.05
+
+            scores.append(max(0.05, min(1.0, s)))
+
+        return sum(scores) / max(len(scores), 1)
+
+    @staticmethod
+    def _main_topic(cluster: list[NewsItem]) -> str:
+        counter: Counter[str] = Counter()
+        for item in cluster:
+            if item.analysis:
+                for t in item.analysis.topics:
+                    counter[t] += 1
+        if counter:
+            return counter.most_common(1)[0][0]
+        return (cluster[0].post.title or "")[:60]

src/src/analyzer.py

@@ -0,0 +1,410 @@
+import html
+import logging
+import re
+from typing import Optional
+
+from .models import Analysis, Article
+
+logger = logging.getLogger(__name__)
+
+RELIABLE_DOMAINS = {
+    "reuters.com": 0.20, "apnews.com": 0.20, "bbc.com": 0.15,
+    "bbc.co.uk": 0.15, "npr.org": 0.12, "wsj.com": 0.12,
+    "economist.com": 0.15, "nature.com": 0.20, "science.org": 0.18,
+    "sciencedaily.com": 0.14, "theguardian.com": 0.08, "nytimes.com": 0.10,
+    "washingtonpost.com": 0.10, "ft.com": 0.14, "bloomberg.com": 0.12,
+}
+
+UNRELIABLE_DOMAINS = {
+    "infowars.com": -0.30, "breitbart.com": -0.20, "dailymail.co.uk": -0.12,
+    "theonion.com": -0.25, "naturalnews.com": -0.30, "zerohedge.com": -0.15,
+}
+
+CLICKBAIT_PATTERNS = [
+    r"you won'?t believe", r"shocked?", r"gobsmacked",
+    r"this is what happens", r"number \d+ will",
+    r"here'?s why", r"what happens next",
+    r"blown away", r"mind.?blowing",
+]
+
+OPINION_MARKERS = [
+    r"\bi think\b", r"\bin my opinion\b", r"\bpersonally\b",
+    r"\bi believe\b", r"\bclearly\b", r"\bobviously\b",
+    r"\bin my view\b", r"\bit seems\b", r"\bi feel\b",
+]
+
+LEFT_KEYWORDS = ["progressive", "equality", "social justice", "climate crisis",
+                 "marginalized", "systemic", "privilege", "inequality"]
+RIGHT_KEYWORDS = ["deregulation", "tax cuts", "free market", "traditional",
+                  "sovereignty", "patriot", "heritage", "small government"]
+
+TOPIC_MAP: dict[str, list[str]] = {
+    "artificial intelligence": [r"\bai\b", r"\bartificial intelligence\b",
+                                r"\bmachine learning\b", r"\bgpt\b", r"\bllm\b",
+                                r"\bneural network", r"\bdeep learning\b"],
+    "climate change": [r"\bclimate\b", r"\bglobal warming\b", r"\bemissions\b",
+                       r"\bcarbon\b", r"\brenewable\b", r"\bsolar\b", r"\bwind turbine",
+                       r"\bheatwave\b", r"\bextreme weather\b", r"\bheat wave\b"],
+    "health": [r"\bhealth\b", r"\bcovid\b", r"\bvaccine\b", r"\bdisease\b",
+               r"\bhospital\b", r"\bmedical\b", r"\bcancer\b", r"\bdrug\b",
+               r"\bod\b", r"\bpandemic\b", r"\bpatient\b", r"\bsurgery\b",
+               r"\bdoctor\b", r"\bnurse\b", r"\btreatment\b", r"\btherapy\b",
+               r"\bdementia\b", r"\bdiabetes\b", r"\bobesity\b", r"\bmental health\b",
+               r"\babortion\b", r"\bpregnant\b", r"\bmedicine\b", r"\bclinical\b",
+               r"\bsymptom\b", r"\bheat\b", r"\brabies\b", r"\bfever\b"],
+    "economy": [r"\beconomy\b", r"\binflation\b", r"\bgdp\b", r"\binterest rate\b",
+                r"\brecession\b", r"\bunemployment\b", r"\bmarket\b", r"\btariff\b",
+                r"\btrade war\b", r"\bdebt\b", r"\bstock\b", r"\bprice\b", r"\bcost\b",
+                r"\bfinancial\b"],
+    "space": [r"\bspace\b", r"\bnasa\b", r"\bspacex\b", r"\bmars\b", r"\brocket\b",
+              r"\bastronaut\b", r"\bgalaxy\b", r"\bplanet\b", r"\borgbit\b",
+              r"\bstellar\b", r"\bcosmic\b"],
+    "cybersecurity": [r"\bcyber\b", r"\bhack", r"\bsecurity breach\b",
+                      r"\bdata breach\b", r"\bransomware\b", r"\bmalware\b",
+                      r"\bphishing\b", r"\bzero.day\b", r"\bfirewall\b",
+                      r"\bencryption\b", r"\bCVE\b", r"\bexploit\b",
+                      r"\bbotnet\b", r"\bDDoS\b", r"\bvulnerability\b", r"\bfraud\b"],
+    "politics": [r"\belection\b", r"\bvot(?:e|ing|er)\b", r"\bcongress\b",
+                 r"\bparliament\b", r"\bsenate\b", r"\bpresident\b",
+                 r"\bgovern(?:ment|or)\b", r"\bGOP\b", r"\bDemocrat\b",
+                 r"\brepublican\b", r"\bpolitician\b", r"\bcandidate\b",
+                 r"\bambassador\b", r"\bdiplomat\b", r"\bsanction\b",
+                 r"\btreaty\b", r"\bembassy\b", r"\bminister\b", r"\bregime\b",
+                 r"\blegislat\b", r"\bpolicy\b", r"\bfederal\b"],
+    "science": [r"\bscien(?:ce|tist|tists|tific)\b", r"\bresearch\b", r"\bstudy\b",
+                r"\bdiscovery\b", r"\bgenome\b", r"\bquantum\b", r"\bparticle\b",
+                r"\bevolution\b", r"\bexperiment\b", r"\bjournal\b", r"\blab\b",
+                r"\bDNA\b", r"\bgene\b", r"\bprotein\b", r"\bbiolog\b",
+                r"\bchemical\b", r"\bphysics\b"],
+    "technology": [r"\btech\b", r"\bsoftware\b", r"\bhardware\b", r"\bchip\b",
+                   r"\bsemiconductor\b", r"\bapp\b", r"\balgorithm\b",
+                   r"\bcomputer\b", r"\brobot\b", r"\bgaming\b", r"\bvideo game\b",
+                   r"\bconsole\b", r"\bmobile\b", r"\bphone\b", r"\blaptop\b",
+                   r"\bsmartphone\b", r"\bgadget\b", r"\bstartup\b",
+                   r"\bplatform\b", r"\bdeveloper\b", r"\bcode\b", r"\bprogramming\b",
+                   r"\bdigital\b", r"\bcloud\b", r"\bdevice\b", r"\bsmart\b",
+                   r"\bIoT\b", r"\bOS\b", r"\bWindows\b", r"\bAndroid\b", r"\biOS\b",
+                   r"\bPlayStation\b", r"\bapp\b", r"\bAI\b", r"\bA\.I",
+                   r"\bEV\b", r"\belectric vehicle\b", r"\bgadget\b",
+                   r"\btechlash\b"],
+    "sports": [r"\bsport\b", r"\bfootball\b", r"\bsoccer\b", r"\bbasketball\b",
+               r"\btennis\b", r"\bworld cup\b", r"\bolympic\b"],
+    "education": [r"\beducation\b", r"\bschool\b", r"\buniversity\b",
+                  r"\bstudent\b", r"\bteacher\b", r"\bcollege\b", r"\bcampus\b"],
+    "immigration": [r"\bimmigra(?:nt|tion)\b", r"\bborder\b", r"\basylum\b",
+                    r"\brefugee\b", r"\bdeport\b", r"\bvisa\b"],
+    "energy": [r"\boil\b", r"\bgas\b", r"\bnuclear\b", r"\benergy\b",
+               r"\bpower plant\b", r"\brenewable\b", r"\bfossil fuel\b"],
+    "world": [r"\bwar\b", r"\bmilitary\b", r"\binvasion\b", r"\bsanction\b",
+              r"\bforeign\b", r"\bdiplomat\b", r"\btreaty\b", r"\bconflict\b",
+              r"\bearthquake\b", r"\bflood\b", r"\bdisaster\b", r"\bpresident\b",
+              r"\bprime minister\b", r"\bgeopolitic\b", r"\balliance\b",
+              r"\bmilitant\b", r"\bguerrilla\b", r"\bceasefire\b", r"\bterrorism\b",
+              r"\bUkraine\b", r"\bRussia\b", r"\bChina\b", r"\bIran\b",
+              r"\batomic\b", r"\bnuclear\b", r"\bmissile\b", r"\bdrone\b",
+              r"\battack\b", r"\bstrike\b", r"\bbomb\b", r"\btroop\b",
+              r"\bsoldier\b", r"\bmissile\b", r"\bdefence\b", r"\bdefense\b",
+              r"\bNATO\b", r"\bUN\b", r"\bICC\b", r"\bintelligence\b",
+              r"\bVatican\b", r"\bCatholic\b"],
+    "funny": [r"\bfunny\b", r"\bjoke\b", r"\bhumor\b", r"\bcomedy\b",
+              r"\bsatire\b", r"\bparody\b", r"\blol\b", r"\bwtf\b",
+              r"\babsurd\b", r"\bridiculous\b", r"\bhilarious\b", r"\bcomic\b",
+              r"\blaugh\b", r"\bclown\b"],
+    "weird": [r"\bweird\b", r"\bstrange\b", r"\bbizarre\b", r"\boddb?all\b",
+              r"\bpeculiar\b", r"\bunusual\b", r"\bodd\b", r"\bunbelievable\b",
+              r"\bsurreal\b", r"\bunconventional\b", r"\bwtf\b"],
+    "onion": [r"\bonion\b", r"\btheonion\b"],
+    "gaming": [r"\bgam(?:e|ing|er|ers)\b", r"\besport\b", r"\bplaystation\b",
+               r"\bxbox\b", r"\bnintendo\b", r"\bsteam\b", r"\bconsole\b",
+               r"\bgta\b", r"\bgrand theft auto\b", r"\bfortnite\b",
+               r"\bminecraft\b", r"\bvalorant\b", r"\bvideogame\b",
+               r"\bvideo game\b"],
+    "movies": [r"\bmovie\b", r"\bfilm\b", r"\bcinema\b", r"\bHollywood\b",
+               r"\bbox office\b", r"\bblockbuster\b", r"\bOscar\b",
+               r"\bactor\b", r"\bactress\b", r"\bscreenplay\b",
+               r"\bdirector\b", r"\bNetflix\b", r"\bDisney\+\b",
+               r"\bHBO\b", r"\breboot\b", r"\bsequel\b", r"\bprequel\b",
+               r"\bIMAX\b", r"\banimation\b"],
+    "tunisia": [r"\bTunisia\b", r"\bTunis\b", r"\bCarthage\b",
+                r"\bSousse\b", r"\bSfax\b"],
+    "arab_world": [r"\barab\b", r"\bgulf\b", r"\bmiddle east\b",
+                   r"\bsaudi\b", r"\bQatar\b", r"\bUAE\b", r"\bDubai\b",
+                   r"\bAbu Dhabi\b", r"\bDoha\b", r"\bRiyadh\b",
+                   r"\bPalestin\b", r"\bGaza\b", r"\bWest Bank\b",
+                   r"\bLeban\b", r"\bBeirut\b", r"\bBaghdad\b",
+                   r"\bCairo\b", r"\bEgypt\b", r"\bSyria\b",
+                   r"\bYemen\b", r"\bAmman\b", r"\bJordan\b",
+                   r"\bOman\b", r"\bKuwait\b", r"\bBahrain\b",
+                   r"\bUnrwa\b", r"\bHezbollah\b", r"\bHouthi\b",
+                   r"\bOPEC\b", r"\bMENA\b"],
+}
+
+TOPIC_TO_CATEGORY: dict[str, str] = {
+    "politics": "Geopolitical",
+    "world": "Geopolitical",
+    "immigration": "Geopolitical",
+    "economy": "Geopolitical",
+    "energy": "Geopolitical",
+    "health": "World Health",
+    "science": "World Health",
+    "technology": "Tech",
+    "artificial intelligence": "Tech",
+    "space": "Tech",
+    "cybersecurity": "Cybersecurity",
+    "funny": "Funny/Weird",
+    "weird": "Funny/Weird",
+    "onion": "Funny/Weird",
+    "sports": "Funny/Weird",
+    "education": "Geopolitical",
+    "climate change": "Geopolitical",
+    "gaming": "Gaming",
+    "movies": "Movies",
+    "tunisia": "Tunisia",
+    "arab_world": "Arab World",
+}
+
+FACTUAL_KEYWORDS = [
+    r"\breport\b", r"\baccording to\b", r"\bsource said\b", r"\bstated\b",
+    r"\bstudy found\b", r"\bdata show\b", r"\bofficial said\b",
+    r"\bresearch suggests\b", r"\bthe study\b", r"\bsurvey\b",
+]
+
+
+class NewsAnalyzer:
+    def __init__(self, config):
+        self.config = config
+        self._summariser = None
+        self._classifier = None
+        self._setup_models()
+
+    def _setup_models(self):
+        if not self.config.use_local_models:
+            logger.info("Local models disabled — using rule-based analysis")
+            return
+        try:
+            from transformers import pipeline
+            logger.info("Loading summariser: %s ...", self.config.summarization_model)
+            self._summariser = pipeline(
+                "summarization",
+                model=self.config.summarization_model,
+                tokenizer=self.config.summarization_model,
+            )
+            logger.info("Loading zero-shot classifier ...")
+            self._classifier = pipeline(
+                "zero-shot-classification",
+                model="typeform/distilbert-base-uncased-mnli",
+            )
+        except ImportError:
+            logger.warning("transformers not available — using rule-based analysis")
+        except Exception as exc:
+            logger.warning("Model loading failed: %s — using rule-based", exc)
+
+    def analyze(self, article: Article) -> Analysis:
+        summary = self._summarise(article)
+        topics = self._classify_topics(article, summary)
+        trust = self._assess_trustworthiness(article)
+        is_opinion = self._detect_opinion(article.text or "")
+        leaning = self._detect_political_leaning(article.text or "")
+
+        category = self._map_category(topics)
+
+        return Analysis(
+            summary=summary,
+            topics=topics,
+            trustworthiness_score=trust,
+            is_opinion=is_opinion,
+            political_leaning=leaning,
+            category=category,
+        )
+
+    def _summarise(self, article: Article) -> str:
+        title = article.title or ""
+        text = article.text or ""
+
+        if self._summariser:
+            try:
+                input_text = text[:1024]
+                out = self._summariser(input_text, max_length=130, min_length=30,
+                                       do_sample=False)
+                return out[0]["summary_text"]
+            except Exception as exc:
+                logger.debug("Summariser failed: %s", exc)
+
+        body = self._strip_metadata(text)
+        body = self._strip_title_line(body, title)
+
+        if not body or len(body) < len(title) * 1.5:
+            return ""
+
+        title_norm = self._norm(title)
+        sentences = re.split(r"(?<=[.!?])\s+", body.strip())
+        selected = []
+        for s in sentences:
+            s = s.strip()
+            if not s:
+                continue
+            if self._is_title_like(s, title_norm):
+                continue
+            selected.append(s)
+            if len(selected) >= 2:
+                break
+        return " ".join(selected) if selected else ""
+
+    @staticmethod
+    def _norm(s: str) -> str:
+        return re.sub(r"\s+", " ", html.unescape(s).lower().strip()).rstrip(".")
+
+    @staticmethod
+    def _is_title_like(sentence: str, title_norm: str) -> bool:
+        s_norm = re.sub(r"\s+", " ", sentence.lower().strip()).rstrip(".")
+        if s_norm == title_norm:
+            return True
+        words_s = set(s_norm.split())
+        words_t = set(title_norm.split())
+        if not words_s or not words_t:
+            return False
+        short, long = (words_s, words_t) if len(words_s) < len(words_t) else (words_t, words_s)
+        overlap = len(short & long) / max(len(short), len(long))
+        return overlap > 0.7
+
+    @staticmethod
+    def _strip_title_line(text: str, title: str) -> str:
+        lines = text.split("\n")
+        if not lines:
+            return text
+        first = lines[0].strip()
+        if not first:
+            return "\n".join(lines[1:]).strip()
+        if len(first) < 150 and not re.search(r"[.!?]$", first):
+            return "\n".join(lines[1:]).strip()
+        return text
+
+    @staticmethod
+    def _strip_metadata(text: str) -> str:
+        lines = text.split("\n")
+        cleaned = []
+        for line in lines:
+            clean = line.strip()
+            if re.match(r"^\s*[—\-] (Published|Updated|BBC News|Image|Copyright)", clean, re.IGNORECASE):
+                continue
+            cleaned.append(line)
+        return "\n".join(cleaned).strip()
+
+    def _classify_topics(self, article: Article, summary: str) -> list[str]:
+        if self._classifier:
+            try:
+                text = f"{article.title} {summary}" if summary else article.title
+                candidates = self.config.user_interests + ["other"]
+                result = self._classifier(text[:512], candidates)
+                return [
+                    label for label, score in zip(result["labels"], result["scores"])
+                    if score > 0.25
+                ]
+            except Exception as exc:
+                logger.debug("Classifier failed: %s", exc)
+
+        return self._keyword_topic_match(article, summary)
+
+    DOMAIN_TOPICS: dict[str, str] = {
+        "krebsonsecurity.com": "cybersecurity",
+        "bleepingcomputer.com": "cybersecurity",
+        "theonion.com": "onion",
+        "ign.com": "gaming",
+        "eurogamer.net": "gaming",
+        "pcgamer.com": "gaming",
+        "rockpapershotgun.com": "gaming",
+        "kotaku.com": "gaming",
+        "gamespot.com": "gaming",
+        "arabnews.com": "arab_world",
+        "middleeasteye.net": "arab_world",
+        "thenationalnews.com": "arab_world",
+        "newarab.com": "arab_world",
+        "therecord.media": "cybersecurity",
+        "threatpost.com": "cybersecurity",
+        "thedailymash.co.uk": "funny",
+        "babylonbee.com": "onion",
+        "polygon.com": "gaming",
+        "variety.com": "movies",
+        "hollywoodreporter.com": "movies",
+        "deadline.com": "movies",
+        "screenrant.com": "movies",
+        "tunisiaonlinenews.com": "tunisia",
+    }
+
+    def _keyword_topic_match(self, article: Article, summary: str) -> list[str]:
+        text = f"{article.title} {summary}" if summary else article.title
+        found = []
+        for topic, patterns in TOPIC_MAP.items():
+            if any(re.search(p, text, re.IGNORECASE) for p in patterns):
+                found.append(topic)
+        domain = re.sub(r"^www\.", "", (article.source_domain or ""))
+        mapped = self.DOMAIN_TOPICS.get(domain)
+        # Only use domain fallback if keyword matching found nothing
+        if mapped and mapped not in found:
+            if not found:
+                found.append(mapped)
+        return found
+
+    def _assess_trustworthiness(self, article: Article) -> float:
+        score = 0.40
+
+        domain = article.source_domain or ""
+        clean_domain = re.sub(r"^www\.", "", domain)
+        score += RELIABLE_DOMAINS.get(clean_domain, 0.0)
+        score += UNRELIABLE_DOMAINS.get(clean_domain, 0.0)
+
+        text = article.text or ""
+        title = article.title or ""
+        if article.extraction_success is False:
+            score -= 0.15
+        elif len(text) > len(title) * 3:
+            score += 0.05
+
+        word_count = len(text.split())
+        if word_count > 200:
+            score += 0.10
+        elif word_count > 100:
+            score += 0.05
+        elif word_count > 50:
+            score += 0.02
+
+        factual_count = sum(1 for p in FACTUAL_KEYWORDS if re.search(p, text, re.IGNORECASE))
+        score += min(factual_count * 0.02, 0.08)
+
+        if any(re.search(p, title, re.IGNORECASE) for p in CLICKBAIT_PATTERNS):
+            score -= 0.20
+
+        opinion_count = sum(1 for p in OPINION_MARKERS if re.search(p, text, re.IGNORECASE))
+        score -= opinion_count * 0.05
+
+        return max(0.05, min(1.0, score))
+
+    @staticmethod
+    def _map_category(topics: list[str]) -> str:
+        priority = ["onion", "funny", "weird", "cybersecurity", "gaming",
+                     "technology", "artificial intelligence", "health", "science",
+                     "tunisia", "arab_world", "world", "politics", "immigration", "economy",
+                     "energy", "education", "climate change", "space", "sports", "movies"]
+        topic_set = {t.lower() for t in topics}
+        for p in priority:
+            if p in topic_set:
+                mapped = TOPIC_TO_CATEGORY.get(p)
+                if mapped:
+                    return mapped
+        return "General"
+
+    @staticmethod
+    def _detect_opinion(text: str) -> bool:
+        count = sum(1 for p in OPINION_MARKERS if re.search(p, text.lower()))
+        return count >= 3
+
+    @staticmethod
+    def _detect_political_leaning(text: str) -> str:
+        text_lower = text.lower()
+        left = sum(1 for k in LEFT_KEYWORDS if k in text_lower)
+        right = sum(1 for k in RIGHT_KEYWORDS if k in text_lower)
+        if left > right + 1:
+            return "left-leaning"
+        if right > left + 1:
+            return "right-leaning"
+        return "centrist"

src/src/extractor.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+from .models import Article
+
+logger = logging.getLogger(__name__)
+
+
+class ArticleExtractor:
+    def __init__(self):
+        self._trafilatura = None
+        self._init_backend()
+
+    def _init_backend(self):
+        try:
+            import trafilatura
+            self._trafilatura = trafilatura
+        except ImportError:
+            logger.info("trafilatura not installed — using fallback extractor")
+
+    def extract(self, url: str) -> Optional[Article]:
+        domain = urlparse(url).netloc
+        try:
+            if self._trafilatura:
+                return self._extract_trafilatura(url, domain)
+            return self._extract_fallback(url, domain)
+        except Exception as exc:
+            logger.debug("Extraction failed for %s: %s", url, exc)
+            return None
+
+    def _extract_trafilatura(self, url: str, domain: str) -> Optional[Article]:
+        downloaded = self._trafilatura.fetch_url(url)
+        if not downloaded:
+            return None
+        text = self._trafilatura.extract(downloaded)
+        if not text:
+            return None
+        title = self._extract_title_meta(downloaded) or ""
+        image = self._extract_og_image(downloaded)
+        return Article(url=url, title=title, text=text, source_domain=domain, image_url=image)
+
+    def _extract_fallback(self, url: str, domain: str) -> Optional[Article]:
+        resp = requests.get(url, headers={"User-Agent": "newsapp/1.0"}, timeout=15)
+        resp.raise_for_status()
+        soup = BeautifulSoup(resp.text, "html.parser")
+
+        title = ""
+        if soup.title:
+            title = soup.title.get_text(strip=True)
+
+        image = self._extract_og_image_soup(soup)
+
+        paragraphs = soup.find_all("p")
+        text = "\n\n".join(p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True))
+        if not text:
+            return None
+        return Article(url=url, title=title, text=text, source_domain=domain, image_url=image)
+
+    @staticmethod
+    def _extract_title_meta(html: str) -> Optional[str]:
+        import html as html_mod
+        import re
+        m = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
+        return html_mod.unescape(m.group(1).strip()) if m else None
+
+    @staticmethod
+    def _extract_og_image(html: str) -> str:
+        import re
+        m = re.search(
+            r'<meta\s+[^>]*property=["\']og:image["\'][^>]*content=["\']([^"\']+)["\']',
+            html, re.IGNORECASE,
+        )
+        if m:
+            return m.group(1)
+        m = re.search(
+            r'<meta\s+[^>]*content=["\']([^"\']+)["\'][^>]*property=["\']og:image["\']',
+            html, re.IGNORECASE,
+        )
+        return m.group(1) if m else ""
+
+    @staticmethod
+    def _extract_og_image_soup(soup) -> str:
+        for prop in ("og:image", "twitter:image"):
+            tag = soup.find("meta", property=prop) or soup.find("meta", attrs={"name": prop})
+            if tag and tag.get("content"):
+                return tag["content"]
+        return ""
+
+
+import requests
+from bs4 import BeautifulSoup

src/src/hn_scraper.py

@@ -0,0 +1,48 @@
+import logging
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
+import requests
+
+from .models import RedditPost
+
+logger = logging.getLogger(__name__)
+
+API_BASE = "https://hacker-news.firebaseio.com/v0"
+
+
+class HackerNewsScraper:
+    def __init__(self, config):
+        self.config = config
+
+    def fetch_posts(self) -> list[RedditPost]:
+        resp = requests.get(f"{API_BASE}/topstories.json", timeout=15)
+        resp.raise_for_status()
+        all_ids = resp.json()
+
+        limit = min(self.config.posts_per_subreddit * 3, 50)
+        posts = []
+        for story_id in all_ids[:limit]:
+            try:
+                detail = requests.get(f"{API_BASE}/item/{story_id}.json", timeout=10)
+                detail.raise_for_status()
+                data = detail.json()
+                if not data or data.get("type") != "story":
+                    continue
+                url = data.get("url") or f"https://news.ycombinator.com/item?id={story_id}"
+                posts.append(RedditPost(
+                    id=f"hn_{story_id}",
+                    title=data.get("title", ""),
+                    url=url,
+                    subreddit="hackernews",
+                    score=data.get("score", 0),
+                    num_comments=data.get("descendants", 0),
+                    source_domain=urlparse(url).netloc,
+                ))
+            except Exception as exc:
+                logger.debug("Failed to fetch HN item %s: %s", story_id, exc)
+
+        posts.sort(key=lambda p: p.score, reverse=True)
+        posts = posts[: self.config.posts_per_subreddit]
+        logger.info("Fetched %d posts from Hacker News", len(posts))
+        return posts

src/src/mockdata.py

@@ -0,0 +1,189 @@
+import random
+import time
+
+from .models import Analysis, Article, NewsItem, RedditPost
+
+_MOCK_ARTICLES = [
+    {
+        "title": "New AI model achieves breakthrough in protein folding prediction",
+        "text": (
+            "Researchers at DeepMind and several universities have announced a major breakthrough "
+            "in protein folding prediction using a new deep learning architecture. The model, called "
+            "AlphaFold-Next, is able to predict protein structures with accuracy approaching "
+            "experimental methods. This advancement could accelerate drug discovery and our "
+            "understanding of diseases. The team trained the model on a dataset of over 100,000 "
+            "known protein structures and used a novel attention mechanism to capture long-range "
+            "interactions between amino acids. Early tests show the model generalises well to "
+            "previously unseen protein families."
+        ),
+        "domain": "nature.com",
+        "subreddit": "science",
+        "score": 5420,
+        "comments": 342,
+    },
+    {
+        "title": "WHO declares new global health emergency as novel virus spreads across continents",
+        "text": (
+            "The World Health Organization has declared a Public Health Emergency of International "
+            "Concern as a novel respiratory virus continues to spread rapidly across multiple "
+            "continents. The virus, which emerged in Southeast Asia, has been detected in 15 "
+            "countries so far. Health officials are implementing containment measures including "
+            "travel restrictions and increased surveillance. The WHO is coordinating with national "
+            "health agencies to ensure a rapid response. Vaccines are expected to begin clinical "
+            "trials within six months, according to the Director-General."
+        ),
+        "domain": "reuters.com",
+        "subreddit": "worldnews",
+        "score": 8210,
+        "comments": 2801,
+    },
+    {
+        "title": "New study links ultra-processed foods to increased cancer risk",
+        "text": (
+            "A comprehensive study published in The Lancet has found a significant correlation "
+            "between consumption of ultra-processed foods and increased risk of developing certain "
+            "types of cancer. The study followed over 200,000 participants for 15 years and "
+            "controlled for lifestyle factors such as smoking and exercise. Researchers found that "
+            "participants who consumed the highest levels of ultra-processed foods had a 23% higher "
+            "risk of developing colorectal cancer. The findings add to growing evidence that dietary "
+            "patterns play a crucial role in cancer prevention."
+        ),
+        "domain": "bbc.com",
+        "subreddit": "health",
+        "score": 3890,
+        "comments": 567,
+    },
+    {
+        "title": "SpaceX successfully launches satellite constellation for global internet coverage",
+        "text": (
+            "SpaceX has successfully launched another batch of 60 Starlink satellites, bringing the "
+            "total constellation size to over 5,000. The Falcon 9 rocket lifted off from Cape "
+            "Canaveral and successfully deployed the satellites in low Earth orbit. This expansion "
+            "will bring high-speed internet access to previously unserved rural areas across the "
+            "globe. The company plans to increase the constellation to 12,000 satellites within "
+            "the next three years, with initial tests showing latency as low as 20 milliseconds."
+        ),
+        "domain": "reuters.com",
+        "subreddit": "technology",
+        "score": 4560,
+        "comments": 890,
+    },
+    {
+        "title": "Federal Reserve signals interest rate cut as inflation continues to cool",
+        "text": (
+            "The Federal Reserve has signalled it may cut interest rates at its next meeting as "
+            "inflation continues to trend downward toward the 2% target. Recent economic data "
+            "shows consumer prices rose only 2.3% year-over-year, down from a peak of 9.1% two "
+            "years ago. Fed Chair Jerome Powell stated that while progress has been made, the "
+            "committee would remain data-dependent. Markets responded positively, with the S&P "
+            "500 rising 1.2% on the news. Economists expect a quarter-point cut in September."
+        ),
+        "domain": "wsj.com",
+        "subreddit": "economy",
+        "score": 3200,
+        "comments": 1200,
+    },
+    {
+        "title": "Global climate summit reaches historic agreement on fossil fuel phase-out",
+        "text": (
+            "Nearly 200 nations have reached a landmark agreement to phase out fossil fuels at the "
+            "UN Climate Summit in Dubai. The agreement sets a timeline for reducing coal, oil, and "
+            "gas production, with developed nations committing to faster reductions. Developing "
+            "countries will receive financial support through a new climate fund worth $100 billion "
+            "annually. Environmental groups have cautiously welcomed the deal while noting that "
+            "the timeline may need to accelerate to meet Paris Agreement targets. The agreement "
+            "marks the first time all nations have explicitly committed to transitioning away from "
+            "fossil fuels."
+        ),
+        "domain": "theguardian.com",
+        "subreddit": "worldnews",
+        "score": 9500,
+        "comments": 3400,
+    },
+    {
+        "title": "Cybersecurity researchers discover zero-day exploit affecting billions of devices",
+        "text": (
+            "Security researchers have discovered a critical vulnerability in a widely-used "
+            "networking library that affects an estimated 3 billion devices worldwide. The "
+            "zero-day exploit, dubbed 'PacketStorm', allows remote code execution without user "
+            "interaction. Major technology companies including Google, Apple, and Microsoft have "
+            "released emergency patches. Users are strongly advised to update their devices "
+            "immediately. The vulnerability has been present in the codebase for over a decade "
+            "and was discovered during a routine security audit."
+        ),
+        "domain": "reuters.com",
+        "subreddit": "technology",
+        "score": 6700,
+        "comments": 1500,
+    },
+    {
+        "title": "Unprecedented heatwave breaks temperature records across Europe",
+        "text": (
+            "An unprecedented heatwave is sweeping across Europe, with temperatures exceeding "
+            "45°C in several countries. Multiple heat records have been broken, including the "
+            "all-time high for the United Kingdom at 42.3°C. Authorities have issued red alerts "
+            "and are urging residents to stay indoors. The extreme weather has been linked to "
+            "climate change by leading meteorological agencies. Hospitals are reporting increased "
+            "admissions for heat-related illnesses, and transport networks have been disrupted "
+            "due to heat-damaged infrastructure."
+        ),
+        "domain": "bbc.com",
+        "subreddit": "worldnews",
+        "score": 7800,
+        "comments": 2100,
+    },
+    {
+        "title": "New CRISPR therapy shows promising results in clinical trial for sickle cell disease",
+        "text": (
+            "A groundbreaking CRISPR-based gene therapy has shown remarkable results in a Phase 3 "
+            "clinical trial for sickle cell disease. Out of 45 patients, 42 showed complete "
+            "remission of symptoms 12 months after treatment. The therapy, developed by Vertex "
+            "Pharmaceuticals, uses CRISPR-Cas9 to edit the patient's own stem cells, correcting "
+            "the genetic mutation responsible for the disease. The FDA has granted breakthrough "
+            "therapy designation, potentially accelerating approval. This marks one of the first "
+            "successful CRISPR-based treatments for a genetic blood disorder."
+        ),
+        "domain": "nature.com",
+        "subreddit": "science",
+        "score": 5100,
+        "comments": 450,
+    },
+    {
+        "title": "AI regulation bill passes Senate with bipartisan support",
+        "text": (
+            "The US Senate has passed a landmark artificial intelligence regulation bill with "
+            "significant bipartisan support. The legislation requires AI companies to conduct "
+            "safety testing before releasing powerful models, establish transparency requirements, "
+            "and create a new federal agency to oversee AI development. The bill was co-sponsored "
+            "by senators from both parties and represents one of the most comprehensive AI "
+            "governance frameworks in the world. Tech companies have expressed mixed reactions, "
+            "with some supporting the clarity while others worry about innovation impact."
+        ),
+        "domain": "nytimes.com",
+        "subreddit": "politics",
+        "score": 4300,
+        "comments": 1800,
+    },
+]
+
+
+def generate_demo_items() -> list[NewsItem]:
+    items = []
+    for art in _MOCK_ARTICLES:
+        post = RedditPost(
+            id=f"mock_{random.randint(100000, 999999)}",
+            title=art["title"],
+            url=f"https://{art['domain']}/article/{random.randint(1000, 9999)}",
+            subreddit=art["subreddit"],
+            score=art["score"],
+            num_comments=art["comments"],
+            source_domain=art["domain"],
+        )
+        article = Article(
+            url=post.url,
+            title=art["title"],
+            text=art["text"],
+            source_domain=art["domain"],
+        )
+        items.append(NewsItem(post=post, article=article))
+    return items

src/src/models.py

@@ -0,0 +1,56 @@
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class RedditPost:
+    id: str
+    title: str
+    url: str
+    subreddit: str
+    score: int
+    num_comments: int
+    source_domain: str = ""
+    image_url: str = ""
+    published: str = ""
+    published_iso: str = ""
+
+
+@dataclass
+class Article:
+    url: str
+    title: str
+    text: str
+    source_domain: str
+    extraction_success: bool = True
+    image_url: str = ""
+
+
+@dataclass
+class Analysis:
+    summary: str
+    topics: list[str]
+    trustworthiness_score: float
+    is_opinion: bool
+    political_leaning: str = "centrist"
+    category: str = "General"
+
+
+@dataclass
+class NewsItem:
+    post: RedditPost
+    article: Optional[Article] = None
+    analysis: Optional[Analysis] = None
+    final_score: float = 0.0
+
+
+@dataclass
+class NewsCluster:
+    topic: str
+    articles: list[NewsItem]
+    total_coverage: int
+    avg_trustworthiness: float
+    avg_popularity: float
+    top_post_url: str
+    final_score: float = 0.0
+    image_url: str = ""

src/src/presenter.py

@@ -0,0 +1,48 @@
+from collections import defaultdict
+
+from .models import NewsCluster
+
+
+class NewsPresenter:
+    CATEGORIES = ["Geopolitical", "World Health", "Tech", "Cybersecurity", "Funny/Weird", "Gaming", "Movies", "Arab World", "Tunisia"]
+
+    @staticmethod
+    def display(clusters: list[NewsCluster], top_n: int = 10):
+        if not clusters:
+            print("\n  No news stories matched your interests this round.")
+            return
+
+        by_cat: dict[str, list[NewsCluster]] = defaultdict(list)
+        for c in clusters:
+            cat = "General"
+            if c.articles and c.articles[0].analysis:
+                ca = c.articles[0].analysis.category
+                cat = ca if ca in NewsPresenter.CATEGORIES else "General"
+            by_cat[cat].append(c)
+
+        print("╔" + "═" * 78 + "╗")
+        print("║  📰  NEWS DIGEST  —  Top Stories  ║".center(80))
+        print("╚" + "═" * 78 + "╝")
+
+        for cat in NewsPresenter.CATEGORIES:
+            items = by_cat.get(cat, [])
+            items.sort(key=lambda x: (x.articles[0].post.published_iso or "", x.final_score), reverse=True)
+            items = items[:top_n]
+            if not items:
+                continue
+            print(f"\n  ══ {cat} ({len(items)}) ══\n")
+            for i, cluster in enumerate(items, 1):
+                print(f"  #{i:<2} [{cluster.topic:<30}]  "
+                      f"Score: {cluster.final_score:.2f}  "
+                      f"Trust: {cluster.avg_trustworthiness:.0%}")
+                item = cluster.articles[0]
+                title = item.post.title[:72] + "…" if len(item.post.title) > 72 else item.post.title
+                print(f"       {title}")
+                if item.post.published:
+                    print(f"       📅 {item.post.published}")
+                if item.analysis and item.analysis.summary:
+                    short = item.analysis.summary[:72] + "…" if len(item.analysis.summary) > 72 else item.analysis.summary
+                    print(f"       → {short}")
+                print()
+
+        print("▔" * 80)

src/src/rss_feed_scraper.py

@@ -0,0 +1,164 @@
+import logging
+import time
+from email.utils import parsedate_to_datetime
+from urllib.parse import urlparse, urlencode, parse_qs
+
+import feedparser
+import requests
+
+from .models import RedditPost
+
+logger = logging.getLogger(__name__)
+
+TRACKING_PARAMS = {"at_medium", "at_campaign", "ref", "utm_source", "utm_medium", "utm_campaign"}
+
+DEFAULT_FEEDS = [
+    # Geopolitical
+    "https://feeds.bbci.co.uk/news/world/rss.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/World.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/Politics.xml",
+    "https://feeds.npr.org/1001/rss.xml",
+    "https://www.aljazeera.com/xml/rss/all.xml",
+    "https://www.theguardian.com/world/rss",
+    # World Health
+    "https://feeds.bbci.co.uk/news/health/rss.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/Science.xml",
+    "https://www.statnews.com/feed/",
+    "https://www.sciencedaily.com/rss/all.xml",
+    # Tech
+    "https://feeds.bbci.co.uk/news/technology/rss.xml",
+    "https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
+    "https://techcrunch.com/feed/",
+    "https://www.wired.com/feed/rss",
+    "https://www.theverge.com/rss/index.xml",
+    "https://arstechnica.com/feed/",
+    # Cybersecurity
+    "https://feeds.feedburner.com/TheHackerNews",
+    "https://krebsonsecurity.com/feed/",
+    "https://www.bleepingcomputer.com/feed/",
+    "https://threatpost.com/feed/",
+    "https://therecord.media/feed/",
+    # Funny / Weird
+    "https://www.theonion.com/rss",
+    "https://www.reddit.com/r/nottheonion/.rss",
+    "https://www.thedailymash.co.uk/feed",
+    "https://babylonbee.com/feed",
+    # Gaming
+    "https://feeds.ign.com/ign/all",
+    "https://www.eurogamer.net/feed",
+    "https://www.pcgamer.com/rss/",
+    "https://www.kotaku.com/rss",
+    "https://www.gamespot.com/feeds/news/",
+    "https://www.polygon.com/rss/index.xml",
+    # Movies
+    "https://variety.com/feed/",
+    "https://www.hollywoodreporter.com/feed/",
+    "https://deadline.com/feed/",
+    "https://screenrant.com/feed/",
+    # Arab World
+    "https://www.arabnews.com/rss.xml",
+    "https://www.middleeasteye.net/rss",
+    "https://www.newarab.com/rss.xml",
+    "https://www.france24.com/en/middle-east/rss",
+    # Tunisia
+    "https://www.tunisiaonlinenews.com/feed/",
+    "https://northafricapost.com/feed/",
+    "https://www.africanews.com/feed/",
+]
+
+
+class RSSFeedScraper:
+    def __init__(self, config):
+        self.config = config
+        self.feeds = getattr(config, "rss_feeds", DEFAULT_FEEDS)
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": "Mozilla/5.0 (compatible; newsapp/1.0)",
+        })
+
+    def fetch_posts(self) -> list[RedditPost]:
+        seen_titles = set()
+        posts = []
+        for url in self.feeds:
+            logger.info("Fetching RSS: %s", url)
+            try:
+                if "reddit.com" in url:
+                    time.sleep(2.5)
+                resp = self.session.get(url, timeout=15)
+                resp.raise_for_status()
+                feed = feedparser.parse(resp.content)
+                for entry in feed.entries[: self.config.posts_per_subreddit]:
+                    post = self._entry_to_post(entry, url)
+                    if post and post.title not in seen_titles:
+                        seen_titles.add(post.title)
+                        posts.append(post)
+            except Exception as exc:
+                logger.warning("Failed RSS %s: %s", url, exc)
+            time.sleep(1.0)
+        return posts
+
+    @staticmethod
+    def _clean_url(url: str) -> str:
+        parsed = urlparse(url)
+        if not parsed.query:
+            return url
+        params = parse_qs(parsed.query)
+        clean = {k: v[0] for k, v in params.items() if k.lower() not in TRACKING_PARAMS}
+        if not clean:
+            return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+        return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{urlencode(clean)}"
+
+    @staticmethod
+    def _extract_image(entry) -> str:
+        for key in ("media_content", "media_thumbnail"):
+            items = entry.get(key) or []
+            for item in items:
+                url = item.get("url", "")
+                if url:
+                    return url
+        for link in entry.get("links", []):
+            if link.get("rel") == "enclosure" and "image" in link.get("type", ""):
+                return link.get("href", "")
+        return ""
+
+    def _entry_to_post(self, entry, feed_url: str) -> RedditPost | None:
+        title = entry.get("title", "")
+        if not title:
+            return None
+        link = self._clean_url(entry.get("link", ""))
+        domain = urlparse(link).netloc or urlparse(feed_url).netloc
+        image_url = self._extract_image(entry)
+        published, published_iso = self._format_date(entry)
+        return RedditPost(
+            id=entry.get("id", entry.get("guid", link)),
+            title=title,
+            url=link,
+            subreddit=domain,
+            score=0, num_comments=0,
+            source_domain=domain,
+            image_url=image_url,
+            published=published,
+            published_iso=published_iso,
+        )
+
+    @staticmethod
+    def _format_date(entry) -> tuple[str, str]:
+        raw = entry.get("published") or entry.get("updated") or ""
+        if not raw:
+            return ("", "")
+        try:
+            dt = parsedate_to_datetime(raw)
+            display = dt.strftime("%d %b %Y")
+            iso = dt.strftime("%Y-%m-%d")
+            return (display, iso)
+        except Exception:
+            if len(raw) >= 10 and raw[4] == "-" and raw[7] == "-":
+                iso = raw[:10]
+                from datetime import datetime
+                try:
+                    dt = datetime.strptime(iso, "%Y-%m-%d")
+                    display = dt.strftime("%d %b %Y")
+                except Exception:
+                    display = iso
+                return (display, iso)
+            return (raw[:16], "")

src/src/scraper.py

@@ -0,0 +1,65 @@
+import logging
+from urllib.parse import urlparse
+
+from .models import RedditPost
+
+logger = logging.getLogger(__name__)
+
+
+class RedditScraper:
+    def __init__(self, config):
+        self.config = config
+        self._praw = None
+        self._init_praw()
+
+    def _init_praw(self):
+        cid = self.config.reddit_client_id
+        secret = self.config.reddit_client_secret
+        if not cid or not secret:
+            raise ValueError(
+                "Reddit API credentials not found.\n"
+                "  Set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env, or\n"
+                "  use python main.py --demo to run with sample data."
+            )
+        try:
+            import praw
+            self._praw = praw.Reddit(
+                client_id=cid,
+                client_secret=secret,
+                user_agent=self.config.reddit_user_agent,
+            )
+            logger.info("Reddit API initialised via PRAW")
+        except ImportError:
+            raise ImportError("praw is required. Install with: pip install praw")
+
+    def fetch_posts(self) -> list[RedditPost]:
+        seen = set()
+        posts = []
+        for sub in self.config.news_subreddits:
+            logger.info("Fetching r/%s ...", sub)
+            try:
+                batch = self._fetch_subreddit(sub)
+                for p in batch:
+                    if p.id not in seen:
+                        seen.add(p.id)
+                        posts.append(p)
+            except Exception as exc:
+                logger.error("Failed to fetch r/%s: %s", sub, exc)
+        return posts
+
+    def _fetch_subreddit(self, subreddit: str) -> list[RedditPost]:
+        results = []
+        sub = self._praw.subreddit(subreddit)
+        for submission in sub.hot(limit=self.config.posts_per_subreddit):
+            if submission.is_self:
+                continue
+            results.append(RedditPost(
+                id=submission.id,
+                title=submission.title,
+                url=submission.url,
+                subreddit=subreddit.lower(),
+                score=submission.score,
+                num_comments=submission.num_comments,
+                source_domain=urlparse(submission.url).netloc,
+            ))
+        return results

templates/index.html

@@ -0,0 +1,289 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>News Digest</title>
+<link href="https://fonts.googleapis.com/css2?family=Inter:opsz@14..32&display=swap" rel="stylesheet">
+<style>
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+body {
+  font-family: 'Inter', -apple-system, sans-serif;
+  background: #0f172a;
+  color: #e2e8f0;
+  min-height: 100vh;
+}
+.header {
+  background: linear-gradient(135deg, #1e293b, #0f172a);
+  border-bottom: 1px solid #1e293b;
+  padding: 20px 32px;
+  position: sticky;
+  top: 0;
+  z-index: 50;
+  backdrop-filter: blur(12px);
+}
+.header-inner {
+  max-width: 1400px;
+  margin: 0 auto;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  flex-wrap: wrap;
+  gap: 12px;
+}
+.header h1 {
+  font-size: 22px;
+  font-weight: 700;
+  background: linear-gradient(135deg, #60a5fa, #a78bfa);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+}
+.header span { font-size: 13px; color: #64748b; }
+.status { color: #64748b; font-size: 13px; display: flex; align-items: center; gap: 8px; }
+.status-dot { width: 8px; height: 8px; border-radius: 50%; background: #22c55e; display: inline-block; }
+.container { max-width: 1400px; margin: 0 auto; padding: 20px 32px; }
+
+/* Tabs */
+.tabs { display: flex; gap: 4px; margin-bottom: 24px; border-bottom: 1px solid #1e293b; overflow-x: auto; }
+.tab {
+  padding: 10px 20px;
+  border: none;
+  background: none;
+  color: #64748b;
+  font-size: 14px;
+  font-weight: 500;
+  cursor: pointer;
+  border-bottom: 2px solid transparent;
+  transition: all .15s;
+  white-space: nowrap;
+}
+.tab:hover { color: #94a3b8; }
+.tab.active { color: #60a5fa; border-bottom-color: #60a5fa; }
+.tab-count { font-size: 11px; color: #475569; margin-left: 4px; }
+
+.tab-content { display: none; }
+.tab-content.active { display: block; }
+
+.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 16px; }
+.card {
+  background: #1e293b;
+  border-radius: 14px;
+  overflow: hidden;
+  border: 1px solid #334155;
+  transition: transform .2s, box-shadow .2s;
+  display: flex;
+  flex-direction: column;
+}
+.card:hover { transform: translateY(-2px); box-shadow: 0 8px 24px rgba(0,0,0,.3); }
+.card-image {
+  width: 100%;
+  height: 180px;
+  object-fit: cover;
+  background: #334155;
+  display: block;
+}
+.card-image-placeholder {
+  width: 100%;
+  height: 180px;
+  background: linear-gradient(135deg, #334155, #1e293b);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  font-size: 40px;
+  color: #475569;
+}
+.card-body { padding: 14px 18px 18px; flex: 1; display: flex; flex-direction: column; }
+.card-rank {
+  font-size: 10px;
+  font-weight: 700;
+  color: #475569;
+  letter-spacing: .05em;
+  margin-bottom: 4px;
+}
+.card-topic {
+  font-size: 10px;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: .08em;
+  color: #60a5fa;
+  margin-bottom: 6px;
+}
+.card-title {
+  font-size: 15px;
+  font-weight: 600;
+  line-height: 1.4;
+  margin-bottom: 6px;
+  display: -webkit-box;
+  -webkit-line-clamp: 2;
+  -webkit-box-orient: vertical;
+  overflow: hidden;
+}
+.card-summary {
+  font-size: 13px;
+  color: #94a3b8;
+  line-height: 1.5;
+  margin-bottom: 10px;
+  display: -webkit-box;
+  -webkit-line-clamp: 2;
+  -webkit-box-orient: vertical;
+  overflow: hidden;
+  flex: 1;
+}
+.card-meta { display: flex; align-items: center; gap: 10px; font-size: 12px; color: #64748b; margin-bottom: 10px; flex-wrap: wrap; }
+.card-meta .source { color: #94a3b8; }
+.score-bar-wrap { height: 3px; background: #334155; border-radius: 3px; margin-bottom: 3px; }
+.score-bar-fill { height: 100%; border-radius: 3px; background: linear-gradient(90deg, #60a5fa, #a78bfa); transition: width .5s; }
+.score-label { font-size: 10px; color: #64748b; display: flex; justify-content: space-between; }
+.card-actions { display: flex; gap: 6px; margin-top: 6px; }
+.btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  padding: 6px 14px;
+  border-radius: 8px;
+  font-size: 12px;
+  font-weight: 500;
+  text-decoration: none;
+  transition: background .15s;
+}
+.btn-primary { background: #3b82f6; color: #fff; }
+.btn-primary:hover { background: #2563eb; }
+.btn-secondary { background: #334155; color: #94a3b8; }
+.btn-secondary:hover { background: #475569; color: #e2e8f0; }
+.card-date {
+  margin-left: auto;
+  font-size: 12px;
+  font-weight: 700;
+  color: #c4b5fd;
+  background: rgba(167, 139, 250, 0.12);
+  padding: 3px 10px;
+  border-radius: 6px;
+  letter-spacing: .03em;
+  border: 1px solid rgba(167, 139, 250, 0.2);
+}
+.card-topics { display: flex; flex-wrap: wrap; gap: 4px; margin-bottom: 10px; }
+.topic-tag { font-size: 10px; padding: 2px 7px; border-radius: 5px; background: #334155; color: #94a3b8; }
+
+.date-group { margin-bottom: 16px; }
+.date-header {
+  font-size: 12px;
+  font-weight: 700;
+  color: #818cf8;
+  text-transform: uppercase;
+  letter-spacing: .08em;
+  margin-bottom: 10px;
+  padding-bottom: 6px;
+  border-bottom: 1px solid #334155;
+}
+
+.empty { text-align: center; padding: 60px 20px; color: #64748b; }
+.empty h2 { font-size: 18px; margin-bottom: 6px; }
+
+@media (max-width: 640px) {
+  .header { padding: 14px; }
+  .container { padding: 14px; }
+  .grid { grid-template-columns: 1fr; }
+}
+</style>
+</head>
+<body>
+<div class="header">
+  <div class="header-inner">
+    <div>
+      <h1>News Digest</h1>
+      <span>{{ total }} stories across {{ categories|length }} topics</span>
+    </div>
+    <div class="status">
+      <span class="status-dot"></span>
+      {{ status }}
+    </div>
+  </div>
+</div>
+
+<div class="container">
+  {% if total == 0 %}
+  <div class="empty">
+    <h2>No news yet</h2>
+    <p>Run <code>python webapp.py</code> to fetch and analyze articles</p>
+  </div>
+  {% else %}
+   <div class="tabs" id="tabs">
+     {% for cat in categories %}
+     <button class="tab {% if loop.first %}active{% endif %}" data-tab="{{ cat | replace(' ', '_') | replace('/', '_') }}">
+       {{ cat }} <span class="tab-count">({{ tab_counts[cat] }})</span>
+     </button>
+     {% endfor %}
+   </div>
+
+   {% for cat in categories %}
+   <div class="tab-content {% if loop.first %}active{% endif %}" id="tab-{{ cat | replace(' ', '_') | replace('/', '_') }}">
+     {% set date_groups = grouped_by_date[cat] %}
+     {% if date_groups %}
+       {% for group in date_groups %}
+       <div class="date-group">
+         <div class="date-header">{{ group.date }}</div>
+         <div class="grid">
+         {% for c in group['items'] %}
+         <div class="card">
+           {% if c.image_url %}
+           <img class="card-image" src="{{ c.image_url }}" alt="" loading="lazy" onerror="this.style.display='none'">
+           {% else %}
+           <div class="card-image-placeholder">📰</div>
+           {% endif %}
+           <div class="card-body">
+             <div class="card-rank">#{{ loop.index }}</div>
+             <div class="card-topic">{{ c.topic }}</div>
+             <div class="card-title">{{ c.articles[0].title }}</div>
+             {% if c.articles[0].summary %}
+             <div class="card-summary">{{ c.articles[0].summary[:160] }}{% if c.articles[0].summary|length > 160 %}…{% endif %}</div>
+             {% endif %}
+             <div class="card-topics">
+               {% for t in c.articles[0].topics[:3] %}
+               <span class="topic-tag">{{ t }}</span>
+               {% endfor %}
+             </div>
+             <div class="card-meta">
+               <span class="source">{{ c.articles[0].domain }}</span>
+               <span>🛡️ {{ c.trust }}</span>
+               {% if c.coverage > 1 %}<span>📰 {{ c.coverage }}</span>{% endif %}
+             </div>
+             <div class="score-bar-wrap">
+               <div class="score-bar-fill" style="width:{{ c.score * 100 }}%"></div>
+             </div>
+             <div class="score-label">
+               <span>Relevance</span>
+               <span>{{ c.score }}</span>
+             </div>
+             <div class="card-actions">
+               <a href="{{ c.top_post_url }}" class="btn btn-primary" target="_blank">Read</a>
+               <a href="{{ c.top_post_url }}" class="btn btn-secondary" target="_blank">Open</a>
+               {% if c.articles[0].published %}
+               <span class="card-date">{{ c.articles[0].published }}</span>
+               {% endif %}
+             </div>
+           </div>
+         </div>
+         {% endfor %}
+         </div>
+       </div>
+       {% endfor %}
+     {% else %}
+     <div class="empty"><h2>No {{ cat }} stories found</h2></div>
+     {% endif %}
+   </div>
+   {% endfor %}
+  {% endif %}
+</div>
+
+<script>
+document.querySelectorAll('.tab').forEach(tab => {
+  tab.addEventListener('click', () => {
+    document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
+    document.querySelectorAll('.tab-content').forEach(tc => tc.classList.remove('active'));
+    tab.classList.add('active');
+    document.getElementById('tab-' + tab.dataset.tab).classList.add('active');
+  });
+});
+</script>
+</body>
+</html>

templates/templates/index.html

@@ -0,0 +1,289 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>News Digest</title>
+<link href="https://fonts.googleapis.com/css2?family=Inter:opsz@14..32&display=swap" rel="stylesheet">
+<style>
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+body {
+  font-family: 'Inter', -apple-system, sans-serif;
+  background: #0f172a;
+  color: #e2e8f0;
+  min-height: 100vh;
+}
+.header {
+  background: linear-gradient(135deg, #1e293b, #0f172a);
+  border-bottom: 1px solid #1e293b;
+  padding: 20px 32px;
+  position: sticky;
+  top: 0;
+  z-index: 50;
+  backdrop-filter: blur(12px);
+}
+.header-inner {
+  max-width: 1400px;
+  margin: 0 auto;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  flex-wrap: wrap;
+  gap: 12px;
+}
+.header h1 {
+  font-size: 22px;
+  font-weight: 700;
+  background: linear-gradient(135deg, #60a5fa, #a78bfa);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+}
+.header span { font-size: 13px; color: #64748b; }
+.status { color: #64748b; font-size: 13px; display: flex; align-items: center; gap: 8px; }
+.status-dot { width: 8px; height: 8px; border-radius: 50%; background: #22c55e; display: inline-block; }
+.container { max-width: 1400px; margin: 0 auto; padding: 20px 32px; }
+
+/* Tabs */
+.tabs { display: flex; gap: 4px; margin-bottom: 24px; border-bottom: 1px solid #1e293b; overflow-x: auto; }
+.tab {
+  padding: 10px 20px;
+  border: none;
+  background: none;
+  color: #64748b;
+  font-size: 14px;
+  font-weight: 500;
+  cursor: pointer;
+  border-bottom: 2px solid transparent;
+  transition: all .15s;
+  white-space: nowrap;
+}
+.tab:hover { color: #94a3b8; }
+.tab.active { color: #60a5fa; border-bottom-color: #60a5fa; }
+.tab-count { font-size: 11px; color: #475569; margin-left: 4px; }
+
+.tab-content { display: none; }
+.tab-content.active { display: block; }
+
+.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 16px; }
+.card {
+  background: #1e293b;
+  border-radius: 14px;
+  overflow: hidden;
+  border: 1px solid #334155;
+  transition: transform .2s, box-shadow .2s;
+  display: flex;
+  flex-direction: column;
+}
+.card:hover { transform: translateY(-2px); box-shadow: 0 8px 24px rgba(0,0,0,.3); }
+.card-image {
+  width: 100%;
+  height: 180px;
+  object-fit: cover;
+  background: #334155;
+  display: block;
+}
+.card-image-placeholder {
+  width: 100%;
+  height: 180px;
+  background: linear-gradient(135deg, #334155, #1e293b);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  font-size: 40px;
+  color: #475569;
+}
+.card-body { padding: 14px 18px 18px; flex: 1; display: flex; flex-direction: column; }
+.card-rank {
+  font-size: 10px;
+  font-weight: 700;
+  color: #475569;
+  letter-spacing: .05em;
+  margin-bottom: 4px;
+}
+.card-topic {
+  font-size: 10px;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: .08em;
+  color: #60a5fa;
+  margin-bottom: 6px;
+}
+.card-title {
+  font-size: 15px;
+  font-weight: 600;
+  line-height: 1.4;
+  margin-bottom: 6px;
+  display: -webkit-box;
+  -webkit-line-clamp: 2;
+  -webkit-box-orient: vertical;
+  overflow: hidden;
+}
+.card-summary {
+  font-size: 13px;
+  color: #94a3b8;
+  line-height: 1.5;
+  margin-bottom: 10px;
+  display: -webkit-box;
+  -webkit-line-clamp: 2;
+  -webkit-box-orient: vertical;
+  overflow: hidden;
+  flex: 1;
+}
+.card-meta { display: flex; align-items: center; gap: 10px; font-size: 12px; color: #64748b; margin-bottom: 10px; flex-wrap: wrap; }
+.card-meta .source { color: #94a3b8; }
+.score-bar-wrap { height: 3px; background: #334155; border-radius: 3px; margin-bottom: 3px; }
+.score-bar-fill { height: 100%; border-radius: 3px; background: linear-gradient(90deg, #60a5fa, #a78bfa); transition: width .5s; }
+.score-label { font-size: 10px; color: #64748b; display: flex; justify-content: space-between; }
+.card-actions { display: flex; gap: 6px; margin-top: 6px; }
+.btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  padding: 6px 14px;
+  border-radius: 8px;
+  font-size: 12px;
+  font-weight: 500;
+  text-decoration: none;
+  transition: background .15s;
+}
+.btn-primary { background: #3b82f6; color: #fff; }
+.btn-primary:hover { background: #2563eb; }
+.btn-secondary { background: #334155; color: #94a3b8; }
+.btn-secondary:hover { background: #475569; color: #e2e8f0; }
+.card-date {
+  margin-left: auto;
+  font-size: 12px;
+  font-weight: 700;
+  color: #c4b5fd;
+  background: rgba(167, 139, 250, 0.12);
+  padding: 3px 10px;
+  border-radius: 6px;
+  letter-spacing: .03em;
+  border: 1px solid rgba(167, 139, 250, 0.2);
+}
+.card-topics { display: flex; flex-wrap: wrap; gap: 4px; margin-bottom: 10px; }
+.topic-tag { font-size: 10px; padding: 2px 7px; border-radius: 5px; background: #334155; color: #94a3b8; }
+
+.date-group { margin-bottom: 16px; }
+.date-header {
+  font-size: 12px;
+  font-weight: 700;
+  color: #818cf8;
+  text-transform: uppercase;
+  letter-spacing: .08em;
+  margin-bottom: 10px;
+  padding-bottom: 6px;
+  border-bottom: 1px solid #334155;
+}
+
+.empty { text-align: center; padding: 60px 20px; color: #64748b; }
+.empty h2 { font-size: 18px; margin-bottom: 6px; }
+
+@media (max-width: 640px) {
+  .header { padding: 14px; }
+  .container { padding: 14px; }
+  .grid { grid-template-columns: 1fr; }
+}
+</style>
+</head>
+<body>
+<div class="header">
+  <div class="header-inner">
+    <div>
+      <h1>News Digest</h1>
+      <span>{{ total }} stories across {{ categories|length }} topics</span>
+    </div>
+    <div class="status">
+      <span class="status-dot"></span>
+      {{ status }}
+    </div>
+  </div>
+</div>
+
+<div class="container">
+  {% if total == 0 %}
+  <div class="empty">
+    <h2>No news yet</h2>
+    <p>Run <code>python webapp.py</code> to fetch and analyze articles</p>
+  </div>
+  {% else %}
+   <div class="tabs" id="tabs">
+     {% for cat in categories %}
+     <button class="tab {% if loop.first %}active{% endif %}" data-tab="{{ cat | replace(' ', '_') | replace('/', '_') }}">
+       {{ cat }} <span class="tab-count">({{ tab_counts[cat] }})</span>
+     </button>
+     {% endfor %}
+   </div>
+
+   {% for cat in categories %}
+   <div class="tab-content {% if loop.first %}active{% endif %}" id="tab-{{ cat | replace(' ', '_') | replace('/', '_') }}">
+     {% set date_groups = grouped_by_date[cat] %}
+     {% if date_groups %}
+       {% for group in date_groups %}
+       <div class="date-group">
+         <div class="date-header">{{ group.date }}</div>
+         <div class="grid">
+         {% for c in group['items'] %}
+         <div class="card">
+           {% if c.image_url %}
+           <img class="card-image" src="{{ c.image_url }}" alt="" loading="lazy" onerror="this.style.display='none'">
+           {% else %}
+           <div class="card-image-placeholder">📰</div>
+           {% endif %}
+           <div class="card-body">
+             <div class="card-rank">#{{ loop.index }}</div>
+             <div class="card-topic">{{ c.topic }}</div>
+             <div class="card-title">{{ c.articles[0].title }}</div>
+             {% if c.articles[0].summary %}
+             <div class="card-summary">{{ c.articles[0].summary[:160] }}{% if c.articles[0].summary|length > 160 %}…{% endif %}</div>
+             {% endif %}
+             <div class="card-topics">
+               {% for t in c.articles[0].topics[:3] %}
+               <span class="topic-tag">{{ t }}</span>
+               {% endfor %}
+             </div>
+             <div class="card-meta">
+               <span class="source">{{ c.articles[0].domain }}</span>
+               <span>🛡️ {{ c.trust }}</span>
+               {% if c.coverage > 1 %}<span>📰 {{ c.coverage }}</span>{% endif %}
+             </div>
+             <div class="score-bar-wrap">
+               <div class="score-bar-fill" style="width:{{ c.score * 100 }}%"></div>
+             </div>
+             <div class="score-label">
+               <span>Relevance</span>
+               <span>{{ c.score }}</span>
+             </div>
+             <div class="card-actions">
+               <a href="{{ c.top_post_url }}" class="btn btn-primary" target="_blank">Read</a>
+               <a href="{{ c.top_post_url }}" class="btn btn-secondary" target="_blank">Open</a>
+               {% if c.articles[0].published %}
+               <span class="card-date">{{ c.articles[0].published }}</span>
+               {% endif %}
+             </div>
+           </div>
+         </div>
+         {% endfor %}
+         </div>
+       </div>
+       {% endfor %}
+     {% else %}
+     <div class="empty"><h2>No {{ cat }} stories found</h2></div>
+     {% endif %}
+   </div>
+   {% endfor %}
+  {% endif %}
+</div>
+
+<script>
+document.querySelectorAll('.tab').forEach(tab => {
+  tab.addEventListener('click', () => {
+    document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
+    document.querySelectorAll('.tab-content').forEach(tc => tc.classList.remove('active'));
+    tab.classList.add('active');
+    document.getElementById('tab-' + tab.dataset.tab).classList.add('active');
+  });
+});
+</script>
+</body>
+</html>

webapp.py

@@ -0,0 +1,282 @@
+import json
+import logging
+import os
+import sys
+import threading
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from flask import Flask, jsonify, render_template, request
+from flask_cors import CORS
+
+from config import Config
+from main import run_pipeline
+from src.models import NewsItem
+from src.rss_feed_scraper import RSSFeedScraper
+
+logging.basicConfig(level=logging.INFO, format="%(levelname).1s %(message)s", stream=sys.stderr)
+logger = logging.getLogger(__name__)
+
+cfg = Config()
+
+app = Flask(__name__)
+app.config["TEMPLATES_AUTO_RELOAD"] = True
+CORS(app, origins=cfg.cors_origins.split(",") if cfg.cors_origins != "*" else "*")
+
+cache_lock = threading.Lock()
+
+CACHE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache_data.json")
+
+CATEGORIES = ["Geopolitical", "World Health", "Tech", "Cybersecurity", "Funny/Weird", "Gaming", "Movies", "Arab World", "Tunisia"]
+
+cached_by_cat: dict[str, list[dict]] = defaultdict(list)
+cached_results: list[dict] = []
+cached_status = "starting"
+cached_last_refresh = None
+
+
+def _cluster_to_html_dict(c):
+    category = "General"
+    if c.articles and c.articles[0].analysis:
+        cat = c.articles[0].analysis.category
+        category = cat if cat in CATEGORIES else "General"
+    return {
+        "category": category,
+        "topic": c.topic,
+        "score": round(c.final_score, 2),
+        "trust": f"{c.avg_trustworthiness:.0%}",
+        "coverage": c.total_coverage,
+        "image_url": c.image_url,
+        "top_post_url": c.top_post_url,
+        "articles": [
+            {
+                "title": a.post.title,
+                "domain": a.article.source_domain if a.article else "",
+                "summary": a.analysis.summary if a.analysis else "",
+                "topics": a.analysis.topics if a.analysis else [],
+                "trust": f"{a.analysis.trustworthiness_score:.0%}" if a.analysis else "",
+                "score": a.post.score,
+                "comments": a.post.num_comments,
+                "url": a.post.url,
+                "image": a.article.image_url if a.article else a.post.image_url,
+                "published": a.post.published,
+                "published_iso": a.post.published_iso,
+            }
+            for a in c.articles[:5]
+        ],
+    }
+
+
+def _cluster_to_api_dict(c):
+    cat = "General"
+    if c.articles and c.articles[0].analysis:
+        ca = c.articles[0].analysis.category
+        cat = ca if ca in CATEGORIES else "General"
+    return {
+        "id": f"cluster-{id(c)}",
+        "topic": c.topic,
+        "category": cat,
+        "final_score": round(c.final_score, 2),
+        "avg_trustworthiness": round(c.avg_trustworthiness, 2),
+        "total_coverage": c.total_coverage,
+        "image_url": c.image_url,
+        "top_post_url": c.top_post_url,
+        "articles": [
+            {
+                "title": a.post.title,
+                "url": a.post.url,
+                "domain": a.article.source_domain if a.article else "",
+                "summary": a.analysis.summary if a.analysis else "",
+                "topics": a.analysis.topics if a.analysis else [],
+                "trust": round(a.analysis.trustworthiness_score, 2) if a.analysis else 0,
+                "score": a.post.score,
+                "comments": a.post.num_comments,
+                "image": a.article.image_url if a.article else a.post.image_url,
+                "published": a.post.published,
+                "published_iso": a.post.published_iso,
+            }
+            for a in c.articles[:5]
+        ],
+    }
+
+
+def refresh_data():
+    global cached_by_cat, cached_results, cached_status, cached_last_refresh
+    with cache_lock:
+        cached_status = "running"
+    try:
+        scraper = RSSFeedScraper(cfg)
+        posts = scraper.fetch_posts()
+        items = [NewsItem(post=p) for p in posts]
+        new_clusters = run_pipeline(items, cfg)
+        html_dicts = [_cluster_to_html_dict(c) for c in new_clusters]
+        api_dicts = [_cluster_to_api_dict(c) for c in new_clusters]
+        by_cat: dict[str, list[dict]] = defaultdict(list)
+        for d in html_dicts:
+            by_cat[d["category"]].append(d)
+        with cache_lock:
+            cached_by_cat = by_cat
+            cached_results = api_dicts
+            cached_last_refresh = datetime.now(timezone.utc)
+            cached_status = f"ok — {len(new_clusters)} clusters from {len(items)} articles"
+        _save_cache(by_cat, api_dicts, cached_status, cached_last_refresh)
+        logger.info("Refresh complete: %s", cached_status)
+    except Exception as exc:
+        with cache_lock:
+            cached_status = f"error: {exc}"
+        logger.error("Pipeline failed: %s", exc)
+
+
+def _save_cache(by_cat, results, status, last_refresh):
+    try:
+        data = {
+            "by_cat": {k: v for k, v in by_cat.items()},
+            "results": results,
+            "status": status,
+            "last_refresh": last_refresh.isoformat() if last_refresh else None,
+        }
+        with open(CACHE_FILE, "w") as f:
+            json.dump(data, f)
+    except Exception as exc:
+        logger.warning("Failed to write cache file: %s", exc)
+
+
+def _load_cache():
+    try:
+        with open(CACHE_FILE) as f:
+            data = json.load(f)
+        return data
+    except (FileNotFoundError, json.JSONDecodeError):
+        return None
+
+
+# ---- Web UI ----
+
+@app.route("/")
+def index():
+    with cache_lock:
+        bc = cached_by_cat
+        status = cached_status
+    grouped_by_date = {}
+    for cat in CATEGORIES:
+        items = bc.get(cat, [])
+        items.sort(key=lambda x: (x["articles"][0]["published_iso"] or "", x["score"]), reverse=True)
+        date_groups = []
+        seen_date = None
+        for item in items:
+            d = item["articles"][0]["published"] or "Unknown"
+            if d != seen_date:
+                date_groups.append({"date": d, "items": []})
+                seen_date = d
+            date_groups[-1]["items"].append(item)
+        grouped_by_date[cat] = date_groups
+
+    total = sum(len(g["items"]) for groups in grouped_by_date.values() for g in groups)
+    tab_counts = {cat: sum(len(g["items"]) for g in grouped_by_date[cat]) for cat in CATEGORIES}
+    return render_template("index.html", grouped_by_date=grouped_by_date, categories=CATEGORIES, status=status, total=total, tab_counts=tab_counts)
+
+
+# ---- REST API ----
+
+@app.route("/api/v1/status")
+def api_status():
+    with cache_lock:
+        status = cached_status
+        lr = cached_last_refresh
+        n_clusters = len(cached_results)
+        n_articles = sum(c["total_coverage"] for c in cached_results)
+    return jsonify({
+        "status": "ok",
+        "clusters": n_clusters,
+        "total_articles": n_articles,
+        "last_refresh": lr.isoformat() if lr else None,
+        "pipeline_status": status,
+    })
+
+
+@app.route("/api/v1/posts")
+def api_posts():
+    category = request.args.get("category")
+    topic = request.args.get("topic")
+    limit = request.args.get("limit", type=int)
+    min_score = request.args.get("min_score", type=float)
+
+    with cache_lock:
+        all_results = list(cached_results)
+
+    filtered = all_results
+    if category:
+        filtered = [r for r in filtered if r["category"].lower() == category.lower()]
+    if topic:
+        filtered = [r for r in filtered if topic.lower() in [t.lower() for t in r.get("articles", [])[:1]]]
+    if min_score is not None:
+        filtered = [r for r in filtered if r["final_score"] >= min_score]
+    if limit:
+        filtered = filtered[:limit]
+
+    return jsonify({"clusters": filtered, "meta": {"total": len(filtered)}})
+
+
+@app.route("/api/v1/categories")
+def api_categories():
+    return jsonify({"categories": CATEGORIES})
+
+
+@app.route("/api/v1/refresh", methods=["POST"])
+def api_refresh():
+    threading.Thread(target=refresh_data, daemon=True).start()
+    return jsonify({"status": "accepted", "message": "refresh started"}), 202
+
+
+@app.route("/health")
+def health():
+    with cache_lock:
+        ok = "error" not in cached_status
+    if ok:
+        return jsonify({"status": "healthy"}), 200
+    return jsonify({"status": "unhealthy", "message": cached_status}), 503
+
+
+# ---- Load cached data on startup (for WSGI) ----
+
+_cache_data = _load_cache()
+if _cache_data:
+    cached_by_cat = defaultdict(list, _cache_data.get("by_cat", {}))
+    cached_results = _cache_data.get("results", [])
+    cached_status = _cache_data.get("status", "idle")
+    lr = _cache_data.get("last_refresh")
+    if lr:
+        try:
+            cached_last_refresh = datetime.fromisoformat(lr)
+        except Exception:
+            pass
+    logger.info("Loaded cached data: %s", cached_status)
+
+# ---- Scheduled refresh for long-running processes ----
+
+def _start_scheduler():
+    try:
+        from apscheduler.schedulers.background import BackgroundScheduler
+        scheduler = BackgroundScheduler(daemon=True)
+        scheduler.add_job(
+            refresh_data,
+            trigger="cron",
+            hour=cfg.refresh_hour,
+            minute=cfg.refresh_minute,
+            timezone=cfg.refresh_timezone,
+            id="daily_refresh",
+            name="Daily news refresh",
+            replace_existing=True,
+        )
+        scheduler.start()
+        logger.info("Scheduler started — daily refresh at %02d:%02d %s", cfg.refresh_hour, cfg.refresh_minute, cfg.refresh_timezone)
+    except ImportError:
+        logger.warning("APScheduler not available — scheduled refresh disabled. Use PythonAnywhere tasks instead.")
+
+
+# ---- Entry points ----
+
+if __name__ == "__main__":
+    _start_scheduler()
+    threading.Thread(target=refresh_data, daemon=True).start()
+    app.run(host=cfg.host, port=cfg.port, debug=(cfg.flask_env != "production"))