import sys import os import logging import threading import time # Add project root to path sys.path.insert( 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) from checker.external.local_search import find_related_articles from checker.external.web_search import ( search_google_news_multilingual, extract_search_query, ) from checker.external.scraper import fetch_all_news from db.database import insert_articles, init_db, get_article_count logger = logging.getLogger(__name__) # ── Auto-scrape configuration ─────────────────────────────────────── SCRAPE_INTERVAL_HOURS = int(os.environ.get("SCRAPE_INTERVAL_HOURS", 6)) _scrape_thread = None _scrape_lock = threading.Lock() def _background_scrape(repeat=True): """Run the scraper in the background. Optionally repeats on a schedule.""" while True: try: logger.info("🔄 Background scrape starting...") news = fetch_all_news() if news: new_count = insert_articles(news) total = get_article_count() logger.info( "✅ Background scrape complete: %d fetched, %d new, %d total in DB", len(news), new_count, total, ) else: logger.info("Background scrape: no articles fetched") except Exception as exc: logger.error("Background scrape failed: %s", exc, exc_info=True) if not repeat: break # Sleep until next scrape logger.info("Next scrape in %d hours", SCRAPE_INTERVAL_HOURS) time.sleep(SCRAPE_INTERVAL_HOURS * 3600) def start_background_scraper(): """Start the periodic background scraper thread (if not already running).""" global _scrape_thread with _scrape_lock: if _scrape_thread is not None and _scrape_thread.is_alive(): logger.info("Background scraper already running") return _scrape_thread = threading.Thread( target=_background_scrape, kwargs={"repeat": True}, daemon=True, name="news-scraper", ) _scrape_thread.start() logger.info("🚀 Background scraper thread started (every %dh)", SCRAPE_INTERVAL_HOURS) def trigger_scrape_once(): """Trigger a one-off scrape in a background thread (non-blocking).""" t = threading.Thread( target=_background_scrape, kwargs={"repeat": False}, daemon=True, name="news-scraper-oneoff", ) t.start() return t # ── Language detection + translation ───────────────────────────────── _translator = None _detector_available = False try: from langdetect import detect as _detect_lang _detector_available = True except ImportError: logger.warning("langdetect not installed — language detection disabled") _detect_lang = None try: from deep_translator import GoogleTranslator _translator_available = True except ImportError: logger.warning("deep-translator not installed — translation disabled") GoogleTranslator = None _translator_available = False def detect_language(text: str) -> str: """Detect the language of the input text. Returns ISO 639-1 code.""" if not _detector_available or not text or len(text.strip()) < 20: return "en" try: lang = _detect_lang(text) # langdetect returns 'tl' for Tagalog/Filipino, 'ceb' for Cebuano return lang except Exception: return "en" def translate_to_english(text: str, source_lang: str) -> str: """Translate text to English for ML processing.""" if source_lang == "en" or not _translator_available or not GoogleTranslator: return text import sys, io try: # Suppress deep_translator's "Backend with N concurrent workers." progress # messages so they don't interleave with our print() output in the console. _old_stdout = sys.stdout _old_stderr = sys.stderr sys.stdout = io.StringIO() sys.stderr = io.StringIO() try: translated = GoogleTranslator(source=source_lang, target="en").translate(text) finally: sys.stdout = _old_stdout sys.stderr = _old_stderr if translated: logger.info("Translated %s → en (%d chars)", source_lang, len(text)) return translated return text except Exception as exc: logger.warning("Translation failed (%s→en): %s", source_lang, exc) return text class ExternalChecker: def __init__(self): """Initialize the external checker, ensure DBs are ready, and start auto-scraper.""" init_db() # Start periodic background scraping start_background_scraper() def check_claim(self, claim_text, translated_text=None): """ Check a claim against the local database and external web search. Args: claim_text (str): The claim or article text to verify (original language). translated_text (str, optional): English translation for search boost. Returns: dict: Structured results containing: - "db_results": List of dicts from local DB search. - "web_results": List of dicts from Google News search. - "verdict": Preliminary verdict string. - "top_score": Highest semantic similarity score (float) or 0.0. """ if not claim_text or not claim_text.strip(): return { "db_results": [], "web_results": [], "verdict": "NO INPUT", "top_score": 0.0, } # 1. Search local database (use translated text if available for better matching) search_text = translated_text if translated_text else claim_text db_results = find_related_articles(search_text) top_score = db_results[0]["similarity"] if db_results else 0.0 has_strong_db_match = top_score >= 0.55 # 2. Multilingual web search — searches in English + Filipino + Cebuano editions. # When we have an English translation, make it the PRIMARY query: Google News # indexes primarily in English, so an English query produces far more relevant # hits for Philippine news stories even when the input is Cebuano/Tagalog. # The original-language query is passed as `translated_query` (i.e. supplementary). if translated_text: # Primary query: from English translation (better Google News coverage) query = extract_search_query(translated_text) # Supplementary query: from original language (catches Cebuano sources) translated_query = extract_search_query(claim_text) else: query = extract_search_query(claim_text) translated_query = None web_results = search_google_news_multilingual( query, translated_query=translated_query, max_results=10, ) # 3. Determine verdict if has_strong_db_match: verdict = "VERIFIED" elif web_results: verdict = "COVERAGE_FOUND_ONLINE" elif db_results: # Weak match in DB verdict = "WEAK_MATCH" else: verdict = "UNVERIFIED" return { "db_results": db_results, "web_results": web_results, "verdict": verdict, "top_score": top_score, } def update_database(self): """ Trigger the scraper to fetch new articles and update the local database. Returns: int: Number of new articles added. """ logger.info("Manual database update triggered") news = fetch_all_news() if news: new_count = insert_articles(news) logger.info("Added %d new articles.", new_count) return new_count return 0