main.py

#!/usr/bin/env python3

import argparse
import logging
import sys
import time

from config import Config
from src.models import Article, NewsItem
from src.extractor import ArticleExtractor
from src.analyzer import NewsAnalyzer
from src.aggregator import NewsAggregator
from src.presenter import NewsPresenter

logger = logging.getLogger(__name__)

TRANSLATE_DOMAINS = {
    "nawaat.org", "www.nawaat.org",
    "tunisienumerique.com", "www.tunisienumerique.com",
    "lapresse.tn", "www.lapresse.tn",
    "webmanagercenter.com", "www.webmanagercenter.com",
    "directinfo.webmanagercenter.com",
    "tuniscope.com", "www.tuniscope.com",
}

_translator = None


def _translate_text(text: str, target: str = "en") -> str:
    if not text or len(text.strip()) < 3:
        return text
    global _translator
    try:
        if _translator is None:
            from googletrans import Translator
            _translator = Translator()
        return _translator.translate(text[:2000], dest=target).text
    except Exception:
        return text


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="OneNews — RSS news aggregator")
    parser.add_argument("--demo", action="store_true", help="Use sample data (no internet)")
    parser.add_argument("--source", choices=["feeds", "hn", "reddit"], default=None,
                        help="Data source (default: RSS feeds)")
    parser.add_argument("--models", action="store_true", help="Enable local ML models (slower)")
    parser.add_argument("--subreddits", nargs="+", default=None, help="Override subreddits")
    parser.add_argument("--limit", type=int, default=None, help="Posts per subreddit")
    return parser.parse_args()


def run_pipeline(items: list[NewsItem], cfg: Config, skip_extraction: bool = False):
    if not skip_extraction:
        logger.info("═══  Extracting article content  ═══")
        extractor = ArticleExtractor()
        for i, item in enumerate(items, 1):
            article = extractor.extract(item.post.url)
            if article:
                logger.info("  [%2d/%d] %-60s ✓  (%s)", i, len(items), item.post.title[:60], article.source_domain)
            else:
                article = Article(
                    url=item.post.url,
                    title=item.post.title,
                    text=item.post.title,
                    source_domain=item.post.source_domain or "reddit.com",
                    extraction_success=False,
                    image_url=item.post.image_url,
                    published=item.post.published,
                    published_iso=item.post.published_iso,
                )
                logger.info("  [%2d/%d] %-60s ✗  (title only)", i, len(items), item.post.title[:60])
            item.article = article
    else:
        logger.info("═══  Extraction skipped (articles already loaded)  ═══")

    logger.info("═══  Translating Arab/Tunisian articles  ═══")
    for item in items:
        art = item.article
        post = item.post
        if not art or not post or not post.source_domain:
            continue
        domain = post.source_domain.lower()
        if domain not in TRANSLATE_DOMAINS:
            continue
        t_title = _translate_text(art.title)
        if t_title and t_title != art.title:
            logger.info("  Title: %s → %s", art.title[:50], t_title[:50])
            art.title = t_title
        t_text = _translate_text(art.text)
        if t_text and t_text != art.text:
            art.text = t_text

    logger.info("═══  Analysing articles  ═══")
    analyzer = NewsAnalyzer(cfg)
    for i, item in enumerate(items, 1):
        item.analysis = analyzer.analyze(item.article)
        cat = item.analysis.category
        topics = ", ".join(item.analysis.topics) if item.analysis.topics else "(none)"
        logger.info("  [%2d/%d] %-14s topic: %-30s trust: %s", i, len(items), cat, topics, f"{item.analysis.trustworthiness_score:.0%}")

    logger.info("═══  Clustering & ranking  ═══")
    aggregator = NewsAggregator(cfg)
    clusters = aggregator.cluster_news(items)
    logger.info("  → %d story clusters found", len(clusters))

    return clusters


def main():
    logging.basicConfig(level=logging.INFO, format="%(levelname).1s %(message)s", stream=sys.stderr)
    args = parse_args()
    cfg = Config()

    if args.models:
        cfg.use_local_models = True
        logging.getLogger().setLevel(logging.DEBUG)
    if args.subreddits:
        cfg.news_subreddits = args.subreddits
    if args.limit:
        cfg.posts_per_subreddit = args.limit

    total_start = time.perf_counter()

    source = args.source or "feeds"

    if args.demo:
        from src.mockdata import generate_demo_items
        print("═══  Loading demo data  ═══")
        items = generate_demo_items()
        print(f"  → {len(items)} sample articles loaded\n")
        clusters = run_pipeline(items, cfg, skip_extraction=True)
        n_posts = len(items)

    elif source == "hn":
        from src.hn_scraper import HackerNewsScraper
        print("═══  Scraping Hacker News  ═══")
        scraper = HackerNewsScraper(cfg)
        posts = scraper.fetch_posts()
        n_posts = len(posts)
        print(f"  → {n_posts} posts collected\n")
        if not posts:
            sys.exit(1)
        items = [NewsItem(post=p) for p in posts]
        clusters = run_pipeline(items, cfg)

    elif source == "reddit":
        from src.scraper import RedditScraper
        print("═══  Scraping Reddit  ═══")
        scraper = RedditScraper(cfg)
        posts = scraper.fetch_posts()
        n_posts = len(posts)
        print(f"  → {n_posts} posts collected\n")
        if not posts:
            sys.exit(1)
        items = [NewsItem(post=p) for p in posts]
        clusters = run_pipeline(items, cfg)

    else:
        from src.rss_feed_scraper import RSSFeedScraper
        from src.html_scraper import HTMLSiteScraper
        print("═══  Fetching RSS news feeds  ═══")
        rss = RSSFeedScraper(cfg).fetch_posts()
        print(f"  → {len(rss)} RSS posts collected")
        print("═══  Scraping HTML sites  ═══")
        html_posts = HTMLSiteScraper(cfg).fetch_posts()
        print(f"  → {len(html_posts)} HTML posts collected")
        posts = rss + html_posts
        n_posts = len(posts)
        if not posts:
            sys.exit(1)
        items = [NewsItem(post=p) for p in posts]
        clusters = run_pipeline(items, cfg)

    elapsed = time.perf_counter() - total_start
    print(f"\n  Done in {elapsed:.1f}s — {n_posts} posts, {len(clusters)} clusters\n")


if __name__ == "__main__":
    main()