checker/external/core.py

import sys
import os
import logging
import threading
import time

# Add project root to path
sys.path.insert(
    0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
)

from checker.external.local_search import find_related_articles
from checker.external.web_search import (
    search_google_news_multilingual,
    extract_search_query,
)
from checker.external.scraper import fetch_all_news
from db.database import insert_articles, init_db, get_article_count

logger = logging.getLogger(__name__)

# ── Auto-scrape configuration ───────────────────────────────────────
SCRAPE_INTERVAL_HOURS = int(os.environ.get("SCRAPE_INTERVAL_HOURS", 6))
_scrape_thread = None
_scrape_lock = threading.Lock()


def _background_scrape(repeat=True):
    """Run the scraper in the background. Optionally repeats on a schedule."""
    while True:
        try:
            logger.info("🔄 Background scrape starting...")
            news = fetch_all_news()
            if news:
                new_count = insert_articles(news)
                total = get_article_count()
                logger.info(
                    "✅ Background scrape complete: %d fetched, %d new, %d total in DB",
                    len(news), new_count, total,
                )
            else:
                logger.info("Background scrape: no articles fetched")
        except Exception as exc:
            logger.error("Background scrape failed: %s", exc, exc_info=True)

        if not repeat:
            break
        # Sleep until next scrape
        logger.info("Next scrape in %d hours", SCRAPE_INTERVAL_HOURS)
        time.sleep(SCRAPE_INTERVAL_HOURS * 3600)


def start_background_scraper():
    """Start the periodic background scraper thread (if not already running)."""
    global _scrape_thread
    with _scrape_lock:
        if _scrape_thread is not None and _scrape_thread.is_alive():
            logger.info("Background scraper already running")
            return
        _scrape_thread = threading.Thread(
            target=_background_scrape,
            kwargs={"repeat": True},
            daemon=True,
            name="news-scraper",
        )
        _scrape_thread.start()
        logger.info("🚀 Background scraper thread started (every %dh)", SCRAPE_INTERVAL_HOURS)


def trigger_scrape_once():
    """Trigger a one-off scrape in a background thread (non-blocking)."""
    t = threading.Thread(
        target=_background_scrape,
        kwargs={"repeat": False},
        daemon=True,
        name="news-scraper-oneoff",
    )
    t.start()
    return t


# ── Language detection + translation ─────────────────────────────────
_translator = None
_detector_available = False

try:
    from langdetect import detect as _detect_lang
    _detector_available = True
except ImportError:
    logger.warning("langdetect not installed — language detection disabled")
    _detect_lang = None

try:
    from deep_translator import GoogleTranslator
    _translator_available = True
except ImportError:
    logger.warning("deep-translator not installed — translation disabled")
    GoogleTranslator = None
    _translator_available = False


def detect_language(text: str) -> str:
    """Detect the language of the input text. Returns ISO 639-1 code."""
    if not _detector_available or not text or len(text.strip()) < 20:
        return "en"
    try:
        lang = _detect_lang(text)
        # langdetect returns 'tl' for Tagalog/Filipino, 'ceb' for Cebuano
        return lang
    except Exception:
        return "en"


def translate_to_english(text: str, source_lang: str) -> str:
    """Translate text to English for ML processing."""
    if source_lang == "en" or not _translator_available or not GoogleTranslator:
        return text
    import sys, io
    try:
        # Suppress deep_translator's "Backend with N concurrent workers." progress
        # messages so they don't interleave with our print() output in the console.
        _old_stdout = sys.stdout
        _old_stderr = sys.stderr
        sys.stdout = io.StringIO()
        sys.stderr = io.StringIO()
        try:
            translated = GoogleTranslator(source=source_lang, target="en").translate(text)
        finally:
            sys.stdout = _old_stdout
            sys.stderr = _old_stderr

        if translated:
            logger.info("Translated %s → en (%d chars)", source_lang, len(text))
            return translated
        return text
    except Exception as exc:
        logger.warning("Translation failed (%s→en): %s", source_lang, exc)
        return text


class ExternalChecker:
    def __init__(self):
        """Initialize the external checker, ensure DBs are ready, and start auto-scraper."""
        init_db()
        # Start periodic background scraping
        start_background_scraper()

    def check_claim(self, claim_text, translated_text=None):
        """
        Check a claim against the local database and external web search.

        Args:
            claim_text (str): The claim or article text to verify (original language).
            translated_text (str, optional): English translation for search boost.

        Returns:
            dict: Structured results containing:
                - "db_results": List of dicts from local DB search.
                - "web_results": List of dicts from Google News search.
                - "verdict": Preliminary verdict string.
                - "top_score": Highest semantic similarity score (float) or 0.0.
        """
        if not claim_text or not claim_text.strip():
            return {
                "db_results": [],
                "web_results": [],
                "verdict": "NO INPUT",
                "top_score": 0.0,
            }

        # 1. Search local database (use translated text if available for better matching)
        search_text = translated_text if translated_text else claim_text
        db_results = find_related_articles(search_text)
        top_score = db_results[0]["similarity"] if db_results else 0.0
        has_strong_db_match = top_score >= 0.55

        # 2. Multilingual web search — searches in English + Filipino + Cebuano editions.
        # When we have an English translation, make it the PRIMARY query: Google News
        # indexes primarily in English, so an English query produces far more relevant
        # hits for Philippine news stories even when the input is Cebuano/Tagalog.
        # The original-language query is passed as `translated_query` (i.e. supplementary).
        if translated_text:
            # Primary query: from English translation (better Google News coverage)
            query = extract_search_query(translated_text)
            # Supplementary query: from original language (catches Cebuano sources)
            translated_query = extract_search_query(claim_text)
        else:
            query = extract_search_query(claim_text)
            translated_query = None
        web_results = search_google_news_multilingual(
            query,
            translated_query=translated_query,
            max_results=10,
        )

        # 3. Determine verdict
        if has_strong_db_match:
            verdict = "VERIFIED"
        elif web_results:
            verdict = "COVERAGE_FOUND_ONLINE"
        elif db_results:  # Weak match in DB
            verdict = "WEAK_MATCH"
        else:
            verdict = "UNVERIFIED"

        return {
            "db_results": db_results,
            "web_results": web_results,
            "verdict": verdict,
            "top_score": top_score,
        }

    def update_database(self):
        """
        Trigger the scraper to fetch new articles and update the local database.

        Returns:
            int: Number of new articles added.
        """
        logger.info("Manual database update triggered")
        news = fetch_all_news()
        if news:
            new_count = insert_articles(news)
            logger.info("Added %d new articles.", new_count)
            return new_count
        return 0