backend/train.py

"""
Internal Fact Checker — Training Script (Fake News Detection)
==============================================================
Trains a Random Forest classifier on multiple Filipino/Cebuano news datasets
to classify news articles as Real or Fake.

Datasets:
  1. jcblaise/fake_news_filipino (local CSV)
  2. Philippine Fake News Corpus (local CSV)
  3. josephimperial/CebuaNER (HuggingFace — Cebuano news, treated as Real)

Enhanced with:
- Hybrid feature matrix (TF-IDF + MiniLM embeddings + stylometric)
- MiniLM multilingual embeddings (384-dim semantic features)
- 25 stylometric features (incl. subjectivity, caps ratio, exclamation density)
- K-Fold Cross-Validation (k=5)
- Tuned hyperparameters (n_estimators=500, max_depth=20, class_weight=balanced)

Usage:
    python backend/train.py
"""

import sys
import os
import json
import time
import re
import numpy as np

# Add project root to path
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, PROJECT_ROOT)

import pandas as pd
import joblib
from textblob import TextBlob
import textstat
from scipy.sparse import hstack, csr_matrix
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics import classification_report, accuracy_score, confusion_matrix
from sentence_transformers import SentenceTransformer


# ── Paths ──
DATA_MODELS_DIR = os.path.join(PROJECT_ROOT, "data_models")


# ── MiniLM Model (lazy-loaded singleton) ──
MINILM_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
_minilm_model = None


def get_minilm_model():
    """Load the multilingual MiniLM model (cached after first call)."""
    global _minilm_model
    if _minilm_model is None:
        print("  Loading MiniLM model...")
        _minilm_model = SentenceTransformer(MINILM_MODEL_NAME)
    return _minilm_model


# ───────────────────────────────────────────────────────────
# Text Cleaning
# ───────────────────────────────────────────────────────────


def clean_text(text):
    """Basic text cleaning for Filipino news articles."""
    if not text or not isinstance(text, str):
        return ""
    text = re.sub(r"<[^>]+>", " ", text)  # HTML tags
    text = re.sub(r"https?://\S+", " ", text)  # URLs
    text = re.sub(r"\s+", " ", text)  # Extra whitespace
    return text.strip()


# ───────────────────────────────────────────────────────────
# Stylometric Feature Extraction
# ───────────────────────────────────────────────────────────


# ── Word lists for linguistic features ──
FIRST_PERSON_PRONOUNS = {
    # English
    "i",
    "me",
    "my",
    "mine",
    "myself",
    "we",
    "us",
    "our",
    "ours",
    "ourselves",
    # Filipino
    "ako",
    "ko",
    "akin",
    "aking",
    "natin",
    "atin",
    "namin",
    "amin",
    "tayo",
    "kami",
    "ta",
}

AUXILIARY_VERBS = {
    # English
    "have",
    "has",
    "had",
    "do",
    "does",
    "did",
    "will",
    "would",
    "shall",
    "should",
    "may",
    "might",
    "can",
    "could",
    "must",
    "am",
    "is",
    "are",
    "was",
    "were",
    "be",
    "been",
    "being",
    # Filipino
    "ay",
    "dapat",
    "mayroon",
    "meron",
    "maaari",
    "pwede",
    "kailangan",
}

ANALYTICAL_WORDS = {
    # English articles + prepositions
    "the",
    "a",
    "an",
    "of",
    "in",
    "on",
    "at",
    "to",
    "for",
    "with",
    "by",
    "from",
    "about",
    "between",
    "through",
    "during",
    "before",
    "after",
    # Filipino
    "ang",
    "ng",
    "sa",
    "mga",
    "nang",
    "para",
    "tungkol",
    "mula",
}

CERTAINTY_WORDS = {
    # English
    "always",
    "never",
    "absolutely",
    "definitely",
    "certainly",
    "undoubtedly",
    "clearly",
    "obviously",
    "without doubt",
    "guaranteed",
    "proven",
    "fact",
    "undeniable",
    "indisputable",
    "every",
    "all",
    # Filipino
    "palagi",
    "sigurado",
    "tiyak",
    "talaga",
    "totoo",
    "lagi",
    "walang duda",
}

TENTATIVE_WORDS = {
    # English
    "perhaps",
    "maybe",
    "possibly",
    "might",
    "could",
    "likely",
    "unlikely",
    "suggests",
    "appears",
    "seems",
    "allegedly",
    "reportedly",
    "according",
    "probable",
    "approximately",
    "estimated",
    # Filipino
    "siguro",
    "marahil",
    "maaaring",
    "mukhang",
    "parang",
    "umano",
    "diumano",
}

CLOUT_WORDS = {
    # English — authority and dominance markers
    "must",
    "demand",
    "require",
    "order",
    "command",
    "insist",
    "decree",
    "mandate",
    "authority",
    "power",
    "control",
    "dominant",
    "superior",
    "we must",
    "you must",
    # Filipino
    "kailangan",
    "dapat",
    "utos",
    "kapangyarihan",
    "kontrol",
    "mando",
}

PAST_FOCUS_WORDS = {
    "talked",
    "did",
    "ago",
    "said",
    "was",
    "were",
    "had",
    "went",
    "told",
    "noon",
    "nakaraan",
    "dati",
    "kahapon",
}

PRESENT_FOCUS_WORDS = {
    "now",
    "is",
    "today",
    "are",
    "being",
    "currently",
    "ongoing",
    "ngayon",
    "kasalukuyan",
}

FUTURE_FOCUS_WORDS = {
    "soon",
    "will",
    "may",
    "shall",
    "going",
    "plan",
    "expect",
    "tomorrow",
    "bukas",
    "darating",
    "magiging",
    "gagawin",
}


def extract_stylometric_features(text):
    """Extract linguistic style features from text.

    Features (25 total — Hybrid Feature Set):
        1.  exclamation_density    — Exclamation marks per word
        2.  question_count         — Number of '?' characters
        3.  caps_ratio             — Uppercase Ratio (ALL CAPS words / total words)
        4.  avg_sentence_length    — Average number of words per sentence
        5.  punctuation_density    — Punctuation chars per 100 chars
        6.  token_count            — Total word count (metadata feature)
        7.  unique_word_ratio      — Unique words / total words (vocabulary richness)
        8.  avg_word_length        — Average word length in characters
        9.  subjectivity           — TextBlob subjectivity score (0=objective, 1=subjective)
       10.  flesch_reading_ease    — Flesch Reading Ease (higher = easier to read)
       11.  flesch_kincaid_grade   — Flesch-Kincaid Grade Level
       12.  coleman_liau_index     — Coleman-Liau Index
       13.  ari                    — Automated Readability Index
       14.  first_person_ratio     — First-person pronouns / total words
       15.  auxiliary_verb_ratio   — Auxiliary/linking verbs / total words
       16.  gunning_fog_index      — Gunning Fog readability index
       17.  analytical_thinking    — Articles + prepositions / total words
       18.  certainty_score        — High-certainty words / total words
       19.  tentative_score        — Hedge/tentative words / total words
       20.  clout_score            — Dominance/authority words / total words
       21.  comma_period_density   — (commas + periods) per 100 chars
       22.  informal_punct_density — (parentheses + dashes + ellipses) per 100 chars
       23.  past_focus_ratio       — Past-tense / historical keywords / total words
       24.  present_focus_ratio    — Present-tense keywords / total words
       25.  future_focus_ratio     — Future-tense keywords / total words

    Returns:
        List of 25 float values.
    """
    if not text or not isinstance(text, str):
        return [0.0] * 25

    words = text.split()
    token_count = len(words)
    if token_count == 0:
        return [0.0] * 25

    words_lower = [w.lower() for w in words]
    text_len = len(text)

    # 1. Exclamation Mark Density (per word)
    exclamation_density = text.count("!") / token_count

    # 2. Question mark count
    question_count = text.count("?")

    # 3. Uppercase Ratio (ALL CAPS words with 2+ chars / total words)
    caps_words = sum(1 for w in words if len(w) >= 2 and w.isupper())
    caps_ratio = caps_words / token_count

    # 4. Average sentence length
    sentences = re.split(r"[.!?]+", text)
    sentences = [s.strip() for s in sentences if s.strip()]
    avg_sentence_length = (
        sum(len(s.split()) for s in sentences) / len(sentences)
        if sentences
        else token_count
    )

    # 5. Punctuation density (per 100 chars)
    punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0

    # 6. Token Count (article length — real news is typically longer)
    # (already computed as token_count)

    # 7. Unique word ratio (vocabulary richness)
    unique_words = len(set(words_lower))
    unique_word_ratio = unique_words / token_count

    # 8. Average word length
    avg_word_length = sum(len(w) for w in words) / token_count

    # 9. Subjectivity Score (TextBlob: 0=objective, 1=subjective)
    try:
        subjectivity = TextBlob(text).sentiment.subjectivity
    except Exception:
        subjectivity = 0.0

    # 10-13. Readability Scores (textstat)
    try:
        flesch_reading_ease = textstat.flesch_reading_ease(text)
        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
        coleman_liau_index = textstat.coleman_liau_index(text)
        ari = textstat.automated_readability_index(text)
    except Exception:
        flesch_reading_ease = 0.0
        flesch_kincaid_grade = 0.0
        coleman_liau_index = 0.0
        ari = 0.0

    # 14. First-person pronoun ratio
    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
    first_person_ratio = first_person_count / token_count

    # 15. Auxiliary verb ratio
    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
    auxiliary_verb_ratio = aux_count / token_count

    # 16. Gunning Fog Index
    try:
        gunning_fog_index = textstat.gunning_fog(text)
    except Exception:
        gunning_fog_index = 0.0

    # 17. Analytical thinking (articles + prepositions ratio)
    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
    analytical_thinking = analytical_count / token_count

    # 18. Certainty score
    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
    certainty_score = certainty_count / token_count

    # 19. Tentative score
    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
    tentative_score = tentative_count / token_count

    # 20. Clout score (dominance/authority markers)
    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
    clout_score = clout_count / token_count

    # 21. Comma + period density (per 100 chars)
    comma_period_count = text.count(",") + text.count(".")
    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0

    # 22. Informal punctuation density — parentheses, dashes, ellipses (per 100 chars)
    informal_count = (
        text.count("(")
        + text.count(")")
        + text.count("—")
        + text.count("–")
        + text.count("-")
        + text.count("...")
        + text.count("…")
    )
    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0

    # 23. Past focus ratio
    past_count = sum(1 for w in words_lower if w in PAST_FOCUS_WORDS)
    past_focus_ratio = past_count / token_count

    # 24. Present focus ratio
    present_count = sum(1 for w in words_lower if w in PRESENT_FOCUS_WORDS)
    present_focus_ratio = present_count / token_count

    # 25. Future focus ratio
    future_count = sum(1 for w in words_lower if w in FUTURE_FOCUS_WORDS)
    future_focus_ratio = future_count / token_count

    return [
        float(exclamation_density),
        float(question_count),
        float(caps_ratio),
        float(avg_sentence_length),
        float(punctuation_density),
        float(token_count),
        float(unique_word_ratio),
        float(avg_word_length),
        float(subjectivity),
        float(flesch_reading_ease),
        float(flesch_kincaid_grade),
        float(coleman_liau_index),
        float(ari),
        float(first_person_ratio),
        float(auxiliary_verb_ratio),
        float(gunning_fog_index),
        float(analytical_thinking),
        float(certainty_score),
        float(tentative_score),
        float(clout_score),
        float(comma_period_density),
        float(informal_punct_density),
        float(past_focus_ratio),
        float(present_focus_ratio),
        float(future_focus_ratio),
    ]


STYLOMETRIC_FEATURE_NAMES = [
    "exclamation_density",
    "question_count",
    "caps_ratio",
    "avg_sentence_length",
    "punctuation_density",
    "token_count",
    "unique_word_ratio",
    "avg_word_length",
    "subjectivity",
    "flesch_reading_ease",
    "flesch_kincaid_grade",
    "coleman_liau_index",
    "ari",
    "first_person_ratio",
    "auxiliary_verb_ratio",
    "gunning_fog_index",
    "analytical_thinking",
    "certainty_score",
    "tentative_score",
    "clout_score",
    "comma_period_density",
    "informal_punct_density",
    "past_focus_ratio",
    "present_focus_ratio",
    "future_focus_ratio",
]


# ───────────────────────────────────────────────────────────
# Language Detection (optional — used with --tagalog-only / --cebuano-only)
# ───────────────────────────────────────────────────────────


def _detect_lang(text: str) -> str:
    """Detect ISO language code for a text snippet.

    Returns 'tl', 'ceb', 'en', etc.  Falls back to a heuristic when
    langdetect is not installed or fails on a given sample.
    """
    _TAGALOG_MARKERS = {
        "ang",
        "ng",
        "mga",
        "sa",
        "na",
        "ay",
        "at",
        "hindi",
        "ako",
        "siya",
        "nila",
        "niya",
        "ito",
        "iyon",
        "kung",
        "para",
        "nang",
        "din",
        "rin",
        "kaya",
        "pero",
        "dahil",
        "ayon",
        "noon",
        "ngayon",
        "dito",
        "doon",
        "sinabi",
        "araw",
        "taon",
        "buwan",
    }
    _CEBUANO_MARKERS = {
        "ug",
        "nga",
        "si",
        "nag",
        "mao",
        "kang",
        "usab",
        "man",
        "dayon",
        "gyud",
        "kaayo",
        "lang",
        "pud",
        "adto",
        "kini",
        "sila",
        "niadtong",
        "gitawag",
        "giingon",
        "matud",
        "nasayran",
        "gidakop",
    }
    if not text or not isinstance(text, str) or len(text.split()) < 5:
        return "unknown"
    try:
        from langdetect import detect, LangDetectException
        from langdetect import DetectorFactory

        DetectorFactory.seed = 42
        return detect(text[:400])
    except Exception:
        pass
    # Heuristic fallback
    words = set(text.lower().split())
    tl_hits = len(words & _TAGALOG_MARKERS)
    ceb_hits = len(words & _CEBUANO_MARKERS)
    if tl_hits == 0 and ceb_hits == 0:
        return "unknown"
    return "tl" if tl_hits >= ceb_hits else "ceb"


def filter_tagalog(df: "pd.DataFrame", text_col: str = "article") -> "pd.DataFrame":
    """Return only rows whose text is classified as Tagalog/Filipino."""
    print("  Detecting languages (this may take a minute for large datasets)...")
    langs = df[text_col].apply(lambda t: _detect_lang(str(t)))
    mask = langs.isin({"tl", "fil"})
    tl_count = mask.sum()
    total = len(df)
    print(
        f"  Language filter: keeping {tl_count:,} / {total:,} articles "
        f"detected as Tagalog/Filipino ({tl_count/total*100:.1f}%)"
    )
    return df[mask].reset_index(drop=True)


def filter_cebuano(df: "pd.DataFrame", text_col: str = "article") -> "pd.DataFrame":
    """Return only rows whose text is classified as Cebuano."""
    print("  Detecting languages (filtering for Cebuano)...")
    langs = df[text_col].apply(lambda t: _detect_lang(str(t)))
    mask = langs.isin({"ceb"})
    ceb_count = mask.sum()
    total = len(df)
    print(
        f"  Language filter: keeping {ceb_count:,} / {total:,} articles "
        f"detected as Cebuano ({ceb_count/total*100:.1f}%)"
    )
    return df[mask].reset_index(drop=True)


# ───────────────────────────────────────────────────────────
# Machine Translation Augmentation
# ───────────────────────────────────────────────────────────

_TRANSLATION_CACHE_PATH = os.path.join(
    PROJECT_ROOT, "data", "raw", "translation_cache.json"
)


def _load_translation_cache() -> dict:
    if os.path.exists(_TRANSLATION_CACHE_PATH):
        try:
            with open(_TRANSLATION_CACHE_PATH, "r", encoding="utf-8") as f:
                return json.load(f)
        except Exception:
            pass
    return {}


def _save_translation_cache(cache: dict) -> None:
    os.makedirs(os.path.dirname(_TRANSLATION_CACHE_PATH), exist_ok=True)
    with open(_TRANSLATION_CACHE_PATH, "w", encoding="utf-8") as f:
        json.dump(cache, f, ensure_ascii=False, indent=2)


def _translate_texts(texts: list, target_lang: str, source_lang: str = "auto") -> list:
    """Translate a list of texts using Google Translate (free tier via deep-translator).

    Results are cached to disk at data/raw/translation_cache.json so translation
    only runs once; subsequent calls read from cache instantly.

    Args:
        texts:       List of source texts.
        target_lang: ISO code for the target language ('ceb', 'tl', 'en', ...).
        source_lang: ISO code for the source language (default: 'auto'-detect).

    Returns:
        List of translated strings (same length as input; failures return originals).
    """
    try:
        from deep_translator import GoogleTranslator
    except ImportError:
        print("  Translation: 'deep-translator' not installed — returning originals.")
        print("      Install with: pip install deep-translator")
        return texts

    cache = _load_translation_cache()
    results = []
    new_translations = 0
    MAX_RETRIES = 3

    for i, text in enumerate(texts):
        # Truncate to 4500 chars (Google Translate free cap is 5000)
        snippet = str(text)[:4500]
        cache_key = f"{target_lang}::{hash(snippet)}"

        if cache_key in cache:
            results.append(cache[cache_key])
            continue

        # Progress indicator every 50 new items
        if new_translations % 50 == 0 and new_translations > 0:
            print(
                f"    ... {new_translations} translated so far ({i}/{len(texts)} items processed)"
            )

        # Retry with exponential backoff to handle free-tier rate limits
        translated = None
        for attempt in range(MAX_RETRIES):
            try:
                translated = GoogleTranslator(
                    source=source_lang, target=target_lang
                ).translate(snippet)
                break  # success
            except Exception as exc:
                wait = 2**attempt  # 1s, 2s, 4s
                if attempt < MAX_RETRIES - 1:
                    time.sleep(wait)
                else:
                    print(
                        f"    [translate] Item {i} failed after {MAX_RETRIES} retries: {exc} — keeping original."
                    )

        if translated:
            results.append(translated)
            cache[cache_key] = translated
            new_translations += 1
        else:
            results.append(snippet)  # fallback to original

        # Small delay between requests to avoid hitting rate limits
        time.sleep(0.3)

        # Save cache incrementally every 50 new translations
        if new_translations > 0 and new_translations % 50 == 0:
            _save_translation_cache(cache)

    if new_translations > 0:
        _save_translation_cache(cache)
        print(f"    Translated {new_translations} new items (cache updated).")
    else:
        print(f"    All {len(texts)} translations served from cache.")

    return results


def _get_fake_jcblaise_texts() -> list:
    """Return the raw article texts labeled as Fake (label=1) from the jcblaise CSV."""
    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
    if not os.path.exists(csv1):
        return []
    df = pd.read_csv(csv1, skiprows=1)
    if "article" not in df.columns or "label" not in df.columns:
        return []
    # label column is StringDtype (git conflict markers embedded) — coerce to int
    df["label"] = pd.to_numeric(df["label"], errors="coerce")
    df = df.dropna(subset=["label"])
    df["label"] = df["label"].astype(int)
    df = df[df["label"] == 1][["article"]].dropna()
    df = df[~df["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
    return df["article"].astype(str).tolist()


def _augment_fake_news_with_translation(target_lang: str) -> "pd.DataFrame | None":
    """Generate translated fake-news articles for a target language.

    On the first run, translates source articles and saves them to:
        data/raw/augmented_tl_fakes.csv   (Tagalog)
        data/raw/augmented_ceb_fakes.csv  (Cebuano)

    On all subsequent runs, loads the CSV directly — no translation,
    no network calls, instant start.

    Strategy:
      - Tagalog ('tl') : translate English "Not Credible" articles from the
                         Philippine Corpus (up to 500) → Tagalog fakes.
      - Cebuano ('ceb'): translate jcblaise Tagalog fake articles → Cebuano.

    Returns:
        pd.DataFrame with columns ['article', 'label'] (label=1, Fake) or None.
    """
    lang_name = {"tl": "Tagalog", "ceb": "Cebuano"}.get(target_lang, target_lang)

    # ── Fast path: load pre-saved CSV if it exists ──
    csv_out = os.path.join(
        PROJECT_ROOT, "data", "raw", f"augmented_{target_lang}_fakes.csv"
    )
    if os.path.exists(csv_out):
        df_cached = pd.read_csv(csv_out)
        if "article" in df_cached.columns and not df_cached.empty:
            print(
                f"  [aug] Loaded pre-saved {lang_name} fakes from cache: "
                f"{len(df_cached)} articles  ({os.path.basename(csv_out)})"
            )
            df_cached["label"] = 1
            return df_cached[["article", "label"]]

    # ── Slow path: translate, then save ──
    if target_lang == "ceb":
        # Source: all jcblaise Tagalog fake articles
        texts = _get_fake_jcblaise_texts()
        if not texts:
            print("  [aug] No source fake texts found for Cebuano augmentation.")
            return None
        print(
            f"  [aug] Translating {len(texts)} jcblaise fake articles → Cebuano "
            f"(will save to {os.path.basename(csv_out)} for future runs)..."
        )
        translated = _translate_texts(texts, target_lang="ceb", source_lang="tl")

    elif target_lang == "tl":
        # Source: English "Not Credible" articles from Philippine Corpus (up to 500)
        csv2 = os.path.join(
            PROJECT_ROOT,
            "data",
            "raw",
            "philippine_corpus",
            "Philippine Fake News Corpus.csv",
        )
        if not os.path.exists(csv2):
            print(
                "  [aug] Philippine Corpus not found — skipping Tagalog augmentation."
            )
            return None
        df2 = pd.read_csv(csv2, skiprows=1).rename(columns={"Content": "article"})
        df2["label_raw"] = df2.get("Label", "")
        df2 = df2[df2["label_raw"] == "Not Credible"][["article"]].dropna()
        df2 = df2[~df2["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
        texts = df2["article"].astype(str).tolist()[:2000]
        if not texts:
            return None
        print(
            f"  [aug] Translating {len(texts)} English fake articles → Tagalog "
            f"(will save to {os.path.basename(csv_out)} for future runs)..."
        )
        translated = _translate_texts(texts, target_lang="tl", source_lang="en")
    else:
        print(f"  [aug] Unsupported target language '{target_lang}' — skipping.")
        return None

    translated = [t for t in translated if t and len(str(t).split()) >= 5]
    if not translated:
        print("  [aug] No translated texts after filtering.")
        return None

    df_aug = pd.DataFrame({"article": translated, "label": 1})  # label=1 → Fake

    # ── Save to CSV for future runs ──
    os.makedirs(os.path.dirname(csv_out), exist_ok=True)
    df_aug[["article"]].to_csv(csv_out, index=False)
    print(f"  [aug] Saved {len(df_aug)} {lang_name} fakes → {csv_out}")

    return df_aug


def _load_cebuaner_as_dataframe():
    """Load josephimperial/CebuaNER from HuggingFace and reconstruct article texts.

    CebuaNER is a token-level NER dataset compiled from credible Cebuano news
    sources (Yes the Best Dumaguete, Filipinas Bisaya, Sunstar Cebu).  There
    are no fake-news labels — every entry is treated as Real (label=0).

    The dataset has 188 k+ token rows.  We reconstruct sentences by joining
    each row's token list, then group consecutive sentences into ~paragraph-
    sized chunks (≥ 30 tokens) so each chunk resembles a short news excerpt.

    Returns:
        pd.DataFrame with columns ['article', 'label'] or None on failure.
    """
    try:
        from datasets import load_dataset  # optional dep — only needed for training
    except ImportError:
        print("  [3] 'datasets' library not installed — skipping CebuaNER.")
        print("      Install with: pip install datasets")
        return None

    print("  [3] Downloading josephimperial/CebuaNER from HuggingFace...")
    try:
        ds = load_dataset("josephimperial/CebuaNER")
    except Exception as exc:
        print(f"  [3] Failed to load CebuaNER: {exc}")
        return None

    # Combine all splits (train / validation / test) into one list of sentences
    sentences = []
    for split_name, split_data in ds.items():
        for row in split_data:
            # CebuaNER schema: {'text': str}  — one sentence per row
            text = row.get("text") or " ".join(
                row.get("tokens") or row.get("words") or []
            )
            if text and text.strip():
                sentences.append(text.strip())

    if not sentences:
        print("  [3] CebuaNER: no token rows found, skipping.")
        return None

    # Group sentences into article-sized chunks (target ≥ 100 tokens).
    # Using 100 instead of 30 ensures each chunk resembles a proper news
    # excerpt rather than a short sentence fragment — reducing the model's
    # tendency to classify based on chunk length alone.
    MIN_CHUNK_TOKENS = 100
    articles = []
    buffer = []
    buffer_tokens = 0
    for sent in sentences:
        buffer.append(sent)
        buffer_tokens += len(sent.split())
        if buffer_tokens >= MIN_CHUNK_TOKENS:
            articles.append(" ".join(buffer))
            buffer = []
            buffer_tokens = 0
    if buffer and buffer_tokens >= 30:  # flush remaining only if non-trivial
        articles.append(" ".join(buffer))

    df3 = pd.DataFrame({"article": articles, "label": 0})  # label=0 → Real
    print(f"  [3] josephimperial/CebuaNER: {len(df3)} reconstructed article chunks")
    return df3


def _load_balitanlp_as_dataframe(max_articles: int = 10_000) -> "pd.DataFrame | None":
    """Load LanceBunag/BalitaNLP from HuggingFace (streaming, no-image config).

    BalitaNLP is a Filipino/Tagalog news dataset with 351k real news articles
    scraped from credible Philippine news outlets.  There are no fake-news
    labels — every entry is treated as Real (label=0).

    The `no-image` config is used to avoid downloading the 40 GB image variant.
    Streaming is used to cap how many articles are loaded.

    Fields used:
      - title : Article headline
      - body  : List of paragraph strings (joined with double newline)

    Args:
      max_articles: Maximum number of articles to load (default 10,000).

    Returns:
      pd.DataFrame with columns ['article', 'label'] or None on failure.
    """
    try:
        from datasets import load_dataset  # optional dep
    except ImportError:
        print("  [4] 'datasets' library not installed — skipping BalitaNLP.")
        return None

    print(
        f"  [4] Streaming LanceBunag/BalitaNLP (no-image, up to {max_articles:,} articles)..."
    )
    try:
        ds = load_dataset(
            "LanceBunag/BalitaNLP",
            "no-image",
            split="train",
            streaming=True,
        )
    except Exception as exc:
        print(f"  [4] Failed to load BalitaNLP: {exc}")
        return None

    articles = []
    for row in ds:
        title = str(row.get("title") or "").strip()
        body = row.get("body") or []
        if isinstance(body, list):
            body_text = "\n\n".join(p for p in body if p and p.strip())
        else:
            body_text = str(body).strip()

        # Combine title + body for a richer text representation
        full_text = f"{title}\n\n{body_text}".strip() if title else body_text
        if full_text and len(full_text.split()) >= 10:
            articles.append(full_text)

        if len(articles) >= max_articles:
            break

    if not articles:
        print("  [4] BalitaNLP: no articles loaded, skipping.")
        return None

    df4 = pd.DataFrame({"article": articles, "label": 0})  # label=0 → Real
    print(f"  [4] LanceBunag/BalitaNLP: {len(df4):,} articles (Tagalog real news)")
    return df4


def load_fake_news_dataset(
    tagalog_only: bool = False,
    cebuano_only: bool = False,
):
    """Load and merge fake news datasets.

    Merges (depending on mode):
      1. jcblaise/fake_news_filipino (local CSV)
      2. Philippine Fake News Corpus    (local CSV)
      3. josephimperial/CebuaNER        (HuggingFace)
      4. LanceBunag/BalitaNLP           (HuggingFace, streaming)
      + MT-augmented fake news          (disk-cached translations)

    Args:
      tagalog_only:  If True, load only Tagalog/Filipino data + augmented fakes.
      cebuano_only:  If True, load only Cebuano data + translated Cebuano fakes.
                     (mutually exclusive with tagalog_only)

    Deduplicates by a fingerprint of article length + first 200 chars.
    """
    print("Loading datasets...")

    frames = []

    if cebuano_only:
        # ── Cebuano-only mode ──────────────────────────────────────────────
        # Real: translated jcblaise reals (augmented_ceb_reals.csv)
        # Fake: translated jcblaise fakes (augmented_ceb_fakes.csv)
        #
        # CebuaNER reconstructed token chunks do NOT resemble real news, so
        # the model trained on them classifies all actual Cebuano news as fake.
        # Using translated jcblaise real articles gives proper news-quality text.
        print("  Mode: CEBUANO-ONLY")

        # ── Fake class: MT-translated jcblaise fakes → Cebuano ──
        df_ceb_fake = _augment_fake_news_with_translation(target_lang="ceb")
        ceb_fake_count = len(df_ceb_fake) if df_ceb_fake is not None else 0

        # ── Real class: MT-translated jcblaise reals → Cebuano ──
        ceb_reals_csv = os.path.join(
            PROJECT_ROOT, "data", "raw", "augmented_ceb_reals.csv"
        )
        if os.path.exists(ceb_reals_csv):
            df_ceb_real = pd.read_csv(ceb_reals_csv)
            if "article" in df_ceb_real.columns and not df_ceb_real.empty:
                df_ceb_real = df_ceb_real[["article"]].dropna().copy()
                df_ceb_real = df_ceb_real[df_ceb_real["article"].str.split().str.len() >= 5]
                df_ceb_real["label"] = 0  # Real
                # Balance: cap to match fake count
                if ceb_fake_count > 0 and len(df_ceb_real) > ceb_fake_count:
                    df_ceb_real = df_ceb_real.sample(
                        n=ceb_fake_count, random_state=42
                    ).reset_index(drop=True)
                print(
                    f"  [real] augmented_ceb_reals.csv: {len(df_ceb_real)} "
                    f"translated Cebuano real articles"
                )
                frames.append(df_ceb_real)
            else:
                print("  [WARN] augmented_ceb_reals.csv exists but has no 'article' column")
        else:
            # Fallback: CebuaNER (not ideal, but better than nothing)
            print(
                "  [WARN] augmented_ceb_reals.csv not found — "
                "falling back to CebuaNER (run backend/translate_ceb_reals.py first!)"
            )
            df3 = _load_cebuaner_as_dataframe()
            if df3 is not None:
                if ceb_fake_count > 0 and len(df3) > ceb_fake_count:
                    df3 = df3.sample(
                        n=ceb_fake_count, random_state=42
                    ).reset_index(drop=True)
                frames.append(df3)

        if df_ceb_fake is not None:
            frames.append(df_ceb_fake)

    elif tagalog_only:
        # ── Tagalog-only mode ──────────────────────────────────────────────
        # Fake priority: satire_facebook.csv first (real Filipino social media
        # fake news), then augmented_tl_fakes.csv to fill the quota.
        # Real news: BalitaNLP capped to total fake count (undersampling).
        print("  Mode: TAGALOG-ONLY (per-dataset filtering, undersampled)")

        # [1] jcblaise — labeled Filipino fake-news corpus, load ALL rows
        csv1 = os.path.join(
            PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv"
        )
        if os.path.exists(csv1):
            df1 = pd.read_csv(csv1)[["article", "label"]].copy()
            df1 = df1[
                ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
            ]
            # label is StringDtype due to git conflict markers — coerce to int
            df1["label"] = pd.to_numeric(df1["label"], errors="coerce")
            df1 = df1.dropna(subset=["label"])
            df1["label"] = df1["label"].astype(int)
            print(
                f"  [1] jcblaise/fake_news_filipino: {len(df1)} articles "
                f"(Real={int((df1['label']==0).sum())}, Fake={int((df1['label']==1).sum())})"
            )
            frames.append(df1)
        else:
            print("  [1] jcblaise not found, skipping.")

        # [2] Philippine Corpus — Credible (Real) rows only, language-filtered to Tagalog.
        #     Not Credible (Fake) rows are English; MT augmentation provides
        #     translated Tagalog fakes instead (see [aug] below).
        csv2 = os.path.join(
            PROJECT_ROOT,
            "data",
            "raw",
            "philippine_corpus",
            "Philippine Fake News Corpus.csv",
        )
        if os.path.exists(csv2):
            df2_raw = pd.read_csv(csv2, skiprows=1).rename(
                columns={"Content": "article"}
            )
            df2_real = (
                df2_raw[df2_raw["Label"] == "Credible"][["article"]].dropna().copy()
            )
            df2_real = df2_real[
                ~df2_real["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
            ]
            before_filter = len(df2_real)
            print("  [2] Filtering Philippine Corpus (Credible) to Tagalog...")
            langs = df2_real["article"].apply(lambda t: _detect_lang(str(t)))
            df2_real = df2_real[langs.isin({"tl", "fil"})].copy()
            df2_real["label"] = 0
            print(
                f"  [2] Philippine Corpus Tagalog Real: "
                f"{len(df2_real)} / {before_filter} articles kept"
            )
            if not df2_real.empty:
                frames.append(df2_real)
        else:
            print("  [2] Philippine Corpus not found, skipping.")

        # [3] CebuaNER — skip (Cebuano, not Tagalog)
        print("  [3] josephimperial/CebuaNER: skipped (Tagalog-only mode)")

        # [sat] PRIORITY: satire_facebook.csv — real Filipino social-media fake news.
        #       Loaded FIRST so it is always included in the fake quota.
        tl_satire_count = 0
        csv_satire = os.path.join(PROJECT_ROOT, "data", "raw", "satire_facebook.csv")
        if os.path.exists(csv_satire):
            df_sat = pd.read_csv(csv_satire)
            if "article" in df_sat.columns:
                df_sat = df_sat[["article"]].dropna().copy()
                df_sat = df_sat[df_sat["article"].str.split().str.len() >= 5]
                df_sat["label"] = 1  # Fake/Satire
                tl_satire_count = len(df_sat)
                print(
                    f"  [sat] satire_facebook.csv (PRIORITY): "
                    f"{tl_satire_count} Tagalog satire posts (all Fake)"
                )
                frames.append(df_sat)
        else:
            print("  [sat] satire_facebook.csv not found — skipping priority satire.")

        # [aug] MT-translated Tagalog fakes — fill remaining quota after satire.
        df_tl_fake = _augment_fake_news_with_translation(target_lang="tl")
        if df_tl_fake is not None:
            frames.append(df_tl_fake)
        tl_aug_count = len(df_tl_fake) if df_tl_fake is not None else 0
        total_tl_fake = tl_satire_count + tl_aug_count
        print(f"  Total Tagalog fakes available: {total_tl_fake} ({tl_satire_count} satire + {tl_aug_count} augmented)")  # noqa: E501

        # [4] BalitaNLP — Tagalog real news (no undersampling; oversampling balances at preprocess step)
        df4 = _load_balitanlp_as_dataframe()
        if df4 is not None:
            frames.append(df4)

    else:
        # ── Mixed mode: equal-sized buckets per language × label ──
        # Target: 1500 each for English fake, English real,
        #         Tagalog fake, Tagalog real,
        #         Cebuano fake, Cebuano real  →  9 000 total
        N_PER_BUCKET = 1500
        print(f"  Mode: MIXED (capping each language/label bucket to {N_PER_BUCKET})")

        # ── [A] English fake & real — Philippine Fake News Corpus ──────────
        csv2 = os.path.join(
            PROJECT_ROOT,
            "data",
            "raw",
            "philippine_corpus",
            "Philippine Fake News Corpus.csv",
        )
        if os.path.exists(csv2):
            df2_raw = pd.read_csv(csv2, skiprows=1).rename(columns={"Content": "article"})
            df2_raw["label"] = df2_raw["Label"].map({"Credible": 0, "Not Credible": 1})
            df2_raw = df2_raw[["article", "label"]].dropna().copy()
            df2_raw = df2_raw[
                ~df2_raw["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
            ]

            # Language-detect to isolate English articles
            print("  [A] Detecting language of Philippine Corpus articles (English filter)...")
            df2_langs = df2_raw["article"].apply(lambda t: _detect_lang(str(t)))
            df2_en = df2_raw[df2_langs == "en"].copy()
            print(f"  [A] Philippine Corpus: {len(df2_en)} English articles detected")

            # English Fake (label=1)
            df_en_fake = df2_en[df2_en["label"] == 1]
            if len(df_en_fake) > N_PER_BUCKET:
                df_en_fake = df_en_fake.sample(n=N_PER_BUCKET, random_state=42)
            df_en_fake = df_en_fake.reset_index(drop=True)
            print(f"  [A] English Fake:  {len(df_en_fake)} articles (target {N_PER_BUCKET})")
            if not df_en_fake.empty:
                frames.append(df_en_fake)

            # English Real (label=0)
            df_en_real = df2_en[df2_en["label"] == 0]
            if len(df_en_real) > N_PER_BUCKET:
                df_en_real = df_en_real.sample(n=N_PER_BUCKET, random_state=42)
            df_en_real = df_en_real.reset_index(drop=True)
            print(f"  [A] English Real:  {len(df_en_real)} articles (target {N_PER_BUCKET})")
            if not df_en_real.empty:
                frames.append(df_en_real)
        else:
            print("  [A] Philippine Fake News Corpus not found — skipping English buckets.")

        # ── [B] Tagalog fake & real — jcblaise + augmented_tl_fakes ────────
        csv1 = os.path.join(
            PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv"
        )
        if os.path.exists(csv1):
            df1 = pd.read_csv(csv1)[["article", "label"]].copy()
            df1 = df1[
                ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
            ]
            df1["label"] = pd.to_numeric(df1["label"], errors="coerce")
            df1 = df1.dropna(subset=["label"])
            df1["label"] = df1["label"].astype(int)
            print(
                f"  [B] jcblaise/fake_news_filipino: {len(df1)} articles "
                f"(Real={int((df1['label']==0).sum())}, Fake={int((df1['label']==1).sum())})"
            )

            # Tagalog Real (label=0) — cap to N_PER_BUCKET
            df_tl_real = df1[df1["label"] == 0].copy()
            if len(df_tl_real) > N_PER_BUCKET:
                df_tl_real = df_tl_real.sample(n=N_PER_BUCKET, random_state=42)
            df_tl_real = df_tl_real.reset_index(drop=True)
            print(f"  [B] Tagalog Real:  {len(df_tl_real)} articles (target {N_PER_BUCKET})")
            if not df_tl_real.empty:
                frames.append(df_tl_real)

            # Tagalog Fake (label=1) — PRIORITY: satire_facebook.csv first,
            # then jcblaise label=1, then augmented_tl_fakes to fill remaining quota.
            # Total capped at N_PER_BUCKET.

            # [sat] PRIORITY — real Filipino social-media satire/fake posts
            csv_satire = os.path.join(PROJECT_ROOT, "data", "raw", "satire_facebook.csv")
            df_tl_satire = pd.DataFrame(columns=["article", "label"])
            if os.path.exists(csv_satire):
                df_sat_raw = pd.read_csv(csv_satire)
                if "article" in df_sat_raw.columns:
                    df_sat_raw = df_sat_raw[["article"]].dropna().copy()
                    df_sat_raw = df_sat_raw[
                        df_sat_raw["article"].str.split().str.len() >= 5
                    ]
                    df_sat_raw["label"] = 1
                    df_tl_satire = df_sat_raw[["article", "label"]]
                    print(
                        f"  [B] satire_facebook.csv (PRIORITY): "
                        f"{len(df_tl_satire)} Tagalog satire posts (all Fake)"
                    )
            else:
                print("  [B] satire_facebook.csv not found — skipping satire priority.")

            satire_count = len(df_tl_satire)
            remaining = max(0, N_PER_BUCKET - satire_count)

            # Fill remaining quota: jcblaise fakes first
            df_tl_fake_jcb = df1[df1["label"] == 1].copy()
            jcb_fake_count = len(df_tl_fake_jcb)

            # Then augmented_tl_fakes
            csv_tl_aug = os.path.join(PROJECT_ROOT, "data", "raw", "augmented_tl_fakes.csv")
            if os.path.exists(csv_tl_aug):
                df_tl_aug = pd.read_csv(csv_tl_aug)
                if "article" in df_tl_aug.columns and not df_tl_aug.empty:
                    df_tl_aug["label"] = 1
                    df_tl_aug = df_tl_aug[["article", "label"]].dropna()
                    print(f"  [B] augmented_tl_fakes.csv: {len(df_tl_aug)} articles")
                else:
                    df_tl_aug = pd.DataFrame(columns=["article", "label"])
            else:
                df_tl_aug = pd.DataFrame(columns=["article", "label"])

            # Combine filler sources and take up to `remaining` rows
            df_tl_filler = pd.concat(
                [df_tl_fake_jcb, df_tl_aug], ignore_index=True
            ).drop_duplicates(subset=["article"])
            if len(df_tl_filler) > remaining:
                df_tl_filler = df_tl_filler.sample(n=remaining, random_state=42)

            # Final Tagalog fake bucket: satire (priority) + filler
            df_tl_fake_all = pd.concat(
                [df_tl_satire, df_tl_filler], ignore_index=True
            ).drop_duplicates(subset=["article"]).reset_index(drop=True)

            print(
                f"  [B] Tagalog Fake:  {len(df_tl_fake_all)} articles "
                f"(satire: {satire_count}, jcblaise: {jcb_fake_count}, "
                f"augmented: {len(df_tl_aug)}, target {N_PER_BUCKET})"
            )
            if not df_tl_fake_all.empty:
                frames.append(df_tl_fake_all)
        else:
            print("  [B] jcblaise dataset not found — skipping Tagalog buckets.")

        # ── [C] Cebuano fake & real ──────────────────────────────────────────
        # Cebuano Real — CebuaNER reconstructed article chunks
        df3 = _load_cebuaner_as_dataframe()
        if df3 is not None:
            if len(df3) > N_PER_BUCKET:
                df3 = df3.sample(n=N_PER_BUCKET, random_state=42)
            df3 = df3.reset_index(drop=True)
            print(f"  [C] Cebuano Real:  {len(df3)} articles (target {N_PER_BUCKET})")
            frames.append(df3)
        else:
            print("  [C] CebuaNER not available — skipping Cebuano real bucket.")

        # Cebuano Fake — pre-translated augmented_ceb_fakes.csv
        csv_ceb_aug = os.path.join(PROJECT_ROOT, "data", "raw", "augmented_ceb_fakes.csv")
        if os.path.exists(csv_ceb_aug):
            df_ceb_fake = pd.read_csv(csv_ceb_aug)
            if "article" in df_ceb_fake.columns and not df_ceb_fake.empty:
                df_ceb_fake["label"] = 1
                df_ceb_fake = df_ceb_fake[["article", "label"]].dropna()
                if len(df_ceb_fake) > N_PER_BUCKET:
                    df_ceb_fake = df_ceb_fake.sample(n=N_PER_BUCKET, random_state=42)
                df_ceb_fake = df_ceb_fake.reset_index(drop=True)
                print(f"  [C] Cebuano Fake:  {len(df_ceb_fake)} articles (target {N_PER_BUCKET})")
                frames.append(df_ceb_fake)
        else:
            print("  [C] augmented_ceb_fakes.csv not found — skipping Cebuano fake bucket.")

    # [5] Facebook satire posts — loading notes:
    #     In mixed mode:       loaded inside bucket [B] as PRIORITY Tagalog fakes.
    #     In tagalog_only mode: loaded above as [sat] PRIORITY.
    #     In cebuano_only mode: not applicable (Tagalog/Filipino content).
    # No additional satire loading needed here — it is handled per-mode above.


    if not frames:
        raise FileNotFoundError(
            "No datasets found! Place at least one dataset in data/raw/."
        )

    # ── Merge and deduplicate ──

    df = pd.concat(frames, ignore_index=True)
    df = df.dropna(subset=["article"]).copy()
    df = df[df["article"].str.len() > 0].copy()

    # ── Language filter (cebuano_only only — tagalog_only uses per-dataset filters above) ──
    if cebuano_only:
        pass  # CebuaNER + MT-translated fakes are already Cebuano by construction

    # Deduplicate by fingerprint (length + first 200 chars)
    before = len(df)
    df["_fingerprint"] = df["article"].apply(lambda x: f"{len(str(x))}_{str(x)[:200]}")
    df = df.drop_duplicates(subset=["_fingerprint"]).drop(columns=["_fingerprint"])
    after = len(df)
    if before > after:
        print(f"  Removed {before - after} duplicate articles")

    label_map = {0: "Real", 1: "Fake"}
    df["label_name"] = df["label"].map(label_map)

    print(f"\n  MERGED TOTAL: {len(df)} articles")
    print(f"  Distribution:\n{df['label_name'].value_counts().to_string()}")
    return df


# ───────────────────────────────────────────────────────────
# Preprocessing
# ───────────────────────────────────────────────────────────


def preprocess(df, undersample=False, oversample=True):
    """Clean text and prepare features.

    Balancing strategy (applied in order of preference):
      1. oversample=True  — RandomOverSampler duplicates minority-class (Fake) rows
                            until classes are equal. Used for mixed-language mode
                            where the real:fake ratio can be large.
      2. undersample=True — downsample the majority class (legacy fallback).
      Both: oversample is applied first, then undersample (rarely needed).
      Neither (language-specific modes): real news is already capped to match fake
                            count at load time, so no resampling is needed here.

    class_weight='balanced' is ALSO set on the RandomForest, so even without
    resampling the model still penalises fake-news misclassification more.
    """
    print("\nPreprocessing...")
    df = df.copy()
    df = df.dropna(subset=["article"]).copy()
    df = df[df["article"].str.len() > 0].copy()
    print(f"  After dropping empty rows: {len(df)} articles")

    if len(df) == 0:
        raise ValueError("No valid articles remaining after filtering!")

    counts_before = df["label"].value_counts()
    print(
        f"  Class distribution before balancing: "
        f"Real={counts_before.get(0, 0)}, Fake={counts_before.get(1, 0)}"
    )

    print("  Cleaning text...")
    df.loc[:, "article_clean"] = df["article"].apply(clean_text)

    texts = df["article_clean"].tolist()
    labels = df["label"].tolist()

    # ── Step 1: Oversampling (preferred) ──────────────────────────────────────
    if oversample:
        try:
            from imblearn.over_sampling import RandomOverSampler

            ros = RandomOverSampler(random_state=42)
            # RandomOverSampler needs a 2-D feature array; use text indices as proxy
            idx = [[i] for i in range(len(texts))]
            idx_res, labels_res = ros.fit_resample(idx, labels)
            texts = [texts[i[0]] for i in idx_res]
            labels = list(labels_res)

            counts_after = {l: labels.count(l) for l in set(labels)}
            print(
                f"  Class distribution after oversampling:  "
                f"Real={counts_after.get(0, 0)}, Fake={counts_after.get(1, 0)}"
            )
            print(f"  Total samples after oversampling: {len(texts)}")
        except ImportError:
            print(
                "  [WARNING] imbalanced-learn not installed — skipping oversampling.\n"
                "            Install with: pip install imbalanced-learn"
            )

    # ── Step 2: Undersampling (legacy fallback) ───────────────────────────────
    if undersample:
        import random
        label_to_texts = {0: [], 1: []}
        for t, l in zip(texts, labels):
            label_to_texts[l].append(t)
        minority_count = min(len(v) for v in label_to_texts.values())
        texts_balanced, labels_balanced = [], []
        for label, txts in label_to_texts.items():
            sample = random.sample(txts, minority_count) if len(txts) > minority_count else txts
            texts_balanced.extend(sample)
            labels_balanced.extend([label] * len(sample))
        # Shuffle
        combined = list(zip(texts_balanced, labels_balanced))
        random.seed(42)
        random.shuffle(combined)
        texts, labels = zip(*combined)
        texts, labels = list(texts), list(labels)
        print(
            f"  Class distribution after undersampling: "
            f"Real={labels.count(0)}, Fake={labels.count(1)}"
        )

    return texts, labels



# ───────────────────────────────────────────────────────────
# Build Hybrid Feature Matrix
# ───────────────────────────────────────────────────────────


def build_features(texts, tfidf=None, scaler=None, svd=None, fit=False):
    """Build hybrid feature matrix: TF-IDF (SVD-reduced) + MiniLM embeddings + stylometric.

    Args:
        texts: List of cleaned text strings.
        tfidf: TfidfVectorizer instance (created if None and fit=True).
        scaler: StandardScaler instance (created if None and fit=True).
        svd: TruncatedSVD instance (created if None and fit=True).
        fit: Whether to fit the transformers (True for training data).

    Returns:
        Tuple of (feature_matrix, tfidf, scaler, svd)
    """
    # TF-IDF features (trigrams capture more fake-news-specific phrases)
    if fit:
        tfidf = TfidfVectorizer(
            max_features=10000,
            ngram_range=(1, 3),
            min_df=2,
            max_df=0.95,
            sublinear_tf=True,
        )
        X_tfidf = tfidf.fit_transform(texts)
        svd = TruncatedSVD(n_components=300, random_state=42)
        X_tfidf_svd = svd.fit_transform(X_tfidf)
    else:
        X_tfidf = tfidf.transform(texts)
        X_tfidf_svd = svd.transform(X_tfidf)

    # MiniLM embeddings (384-dim semantic features)
    print("    Encoding texts with MiniLM...")
    minilm = get_minilm_model()
    embeddings = minilm.encode(texts, show_progress_bar=False, batch_size=64)

    # Stylometric features
    print(
        f"    Extracting stylometric features ({len(STYLOMETRIC_FEATURE_NAMES)} features)..."
    )
    stylo_data = np.array([extract_stylometric_features(t) for t in texts])

    if fit:
        scaler = StandardScaler()
        stylo_scaled = scaler.fit_transform(stylo_data)
    else:
        stylo_scaled = scaler.transform(stylo_data)

    # Combine: TF-IDF/SVD (dense -> sparse) + MiniLM (dense -> sparse) + stylometric (dense -> sparse)
    X_combined = hstack(
        [
            csr_matrix(X_tfidf_svd),
            csr_matrix(embeddings),
            csr_matrix(stylo_scaled),
        ]
    )

    return X_combined, tfidf, scaler, svd


# ───────────────────────────────────────────────────────────
# Training
# ───────────────────────────────────────────────────────────


def train_model(X_texts, y_labels, max_depth=None, min_samples_leaf=3):
    """Train a Random Forest with hybrid features and cross-validation.

    Args:
        max_depth (int): Maximum tree depth. Use lower values (8-10) for small or
            homogeneous datasets (e.g. Cebuano) to prevent memorizing source-format
            artifacts instead of genuine fake-news signals.
        min_samples_leaf (int): Minimum samples at a leaf. Higher values (5+) add
            regularization and reduce overfitting on small datasets.
    """
    label_names = ["Real", "Fake"]
    print(f"  Hyperparameters: max_depth={max_depth}, min_samples_leaf={min_samples_leaf}")

    # ── Split data (80/10/10) ──
    print("\nSplitting data (80/10/10)...")
    X_train, X_temp, y_train, y_temp = train_test_split(
        X_texts, y_labels, test_size=0.2, random_state=42, stratify=y_labels
    )
    X_val, X_test, y_val, y_test = train_test_split(
        X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp
    )
    print(f"  Train: {len(X_train)} | Val: {len(X_val)} | Test: {len(X_test)}")

    # ── Build Hybrid Features ──
    print("\nBuilding hybrid features (TF-IDF + MiniLM + stylometric)...")
    print("  Fitting on training data...")
    X_train_feat, tfidf, scaler, svd = build_features(X_train, fit=True)
    n_svd = svd.n_components
    n_minilm = 384
    n_stylo = len(STYLOMETRIC_FEATURE_NAMES)
    print(f"  TF-IDF vocabulary: {len(tfidf.vocabulary_)} → SVD components: {n_svd}")
    print(
        f"  Total feature count: {X_train_feat.shape[1]} "
        f"(TF-IDF/SVD: {n_svd}"
        f" + MiniLM: {n_minilm}"
        f" + Stylometric: {n_stylo})"
    )

    print("  Transforming validation & test...")
    X_val_feat, _, _, _ = build_features(X_val, tfidf=tfidf, scaler=scaler, svd=svd, fit=False)
    X_test_feat, _, _, _ = build_features(X_test, tfidf=tfidf, scaler=scaler, svd=svd, fit=False)

    # ── K-Fold Cross-Validation ──
    print("\nRunning 5-Fold Cross-Validation on training set...")
    rf_cv = RandomForestClassifier(
        n_estimators=500,
        max_depth=max_depth,
        max_features=0.15,
        min_samples_split=5,
        min_samples_leaf=min_samples_leaf,
        class_weight="balanced",
        n_jobs=-1,
        random_state=42,
    )
    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
    cv_scores = cross_val_score(rf_cv, X_train_feat, y_train, cv=cv, scoring="accuracy")
    print(f"  CV Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std():.4f})")
    print(f"  Fold scores: {[f'{s:.4f}' for s in cv_scores]}")

    # ── Train Final Model ──
    print("\nTraining final Random Forest...")
    start_time = time.time()
    rf = RandomForestClassifier(
        n_estimators=500,
        max_depth=max_depth,
        max_features=0.15,
        min_samples_split=5,
        min_samples_leaf=min_samples_leaf,
        class_weight="balanced",
        n_jobs=-1,
        random_state=42,
        verbose=1,
    )
    rf.fit(X_train_feat, y_train)
    train_time = time.time() - start_time
    print(f"  Training completed in {train_time:.1f}s")

    # ── Evaluate on validation set ──
    print("\n" + "=" * 60)
    print("  VALIDATION SET RESULTS")
    print("=" * 60)
    y_val_pred = rf.predict(X_val_feat)
    val_acc = accuracy_score(y_val, y_val_pred)
    print(f"  Accuracy: {val_acc:.4f}")
    print(
        classification_report(
            y_val,
            y_val_pred,
            labels=[0, 1],
            target_names=label_names,
            zero_division=0,
        )
    )

    # ── Evaluate on test set ──
    print("=" * 60)
    print("  TEST SET RESULTS")
    print("=" * 60)
    y_test_pred = rf.predict(X_test_feat)
    test_acc = accuracy_score(y_test, y_test_pred)
    print(f"  Accuracy: {test_acc:.4f}")
    print(
        classification_report(
            y_test,
            y_test_pred,
            labels=[0, 1],
            target_names=label_names,
            zero_division=0,
        )
    )

    cm = confusion_matrix(y_test, y_test_pred)
    print("  Confusion Matrix:")
    print(f"  Labels: {label_names}")
    print(cm)

    # ── Feature Importance (top stylometric) ──
    print("\n  Stylometric Feature Importance:")
    stylo_start = X_train_feat.shape[1] - len(STYLOMETRIC_FEATURE_NAMES)
    importances = rf.feature_importances_[stylo_start:]
    for name, imp in sorted(
        zip(STYLOMETRIC_FEATURE_NAMES, importances), key=lambda x: -x[1]
    ):
        bar = "#" * int(imp * 500)
        print(f"    {name:<25} {imp:.6f} {bar}")

    return (
        rf,
        tfidf,
        scaler,
        svd,
        {
            "cv_mean": float(cv_scores.mean()),
            "cv_std": float(cv_scores.std()),
            "val_accuracy": float(val_acc),
            "test_accuracy": float(test_acc),
        },
    )


# ───────────────────────────────────────────────────────────
# Save Artifacts
# ───────────────────────────────────────────────────────────


def save_artifacts(model, vectorizer, scaler, svd, metrics, lang_suffix=""):
    """Save trained model, vectorizer, scaler, and SVD to disk.

    Args:
        lang_suffix: Empty string for the mixed/default model, or '_tagalog' /
                     '_cebuano' when training a language-specific sub-model.
    """
    os.makedirs(DATA_MODELS_DIR, exist_ok=True)

    if lang_suffix:
        model_path = os.path.join(DATA_MODELS_DIR, f"rf_fakenews{lang_suffix}.pkl")
        vectorizer_path = os.path.join(
            DATA_MODELS_DIR, f"tfidf_fakenews{lang_suffix}.pkl"
        )
        scaler_path = os.path.join(DATA_MODELS_DIR, f"stylo_scaler{lang_suffix}.pkl")
        svd_path = os.path.join(DATA_MODELS_DIR, f"tfidf_svd{lang_suffix}.pkl")
    else:
        model_path = os.path.join(DATA_MODELS_DIR, "rf_fakenews_model.pkl")
        vectorizer_path = os.path.join(DATA_MODELS_DIR, "tfidf_fakenews.pkl")
        scaler_path = os.path.join(DATA_MODELS_DIR, "stylo_scaler.pkl")
        svd_path = os.path.join(DATA_MODELS_DIR, "tfidf_svd.pkl")

    joblib.dump(model, model_path)
    joblib.dump(vectorizer, vectorizer_path)
    joblib.dump(scaler, scaler_path)
    joblib.dump(svd, svd_path)

    print(f"\n  Model saved to:      {model_path}")
    print(f"  Vectorizer saved to: {vectorizer_path}")
    print(f"  Scaler saved to:     {scaler_path}")
    print(f"  SVD saved to:        {svd_path}")
    print(f"\n  CV Accuracy:   {metrics['cv_mean']:.4f} (+/- {metrics['cv_std']:.4f})")
    print(f"  Val Accuracy:  {metrics['val_accuracy']:.4f}")
    print(f"  Test Accuracy: {metrics['test_accuracy']:.4f}")


# ───────────────────────────────────────────────────────────
# Main
# ───────────────────────────────────────────────────────────


def main():
    import argparse

    parser = argparse.ArgumentParser(description="Train the fake-news detection model.")
    parser.add_argument(
        "--tagalog-only",
        action="store_true",
        default=False,
        help="Filter training data to Tagalog/Filipino articles only "
        "(skips CebuaNER; loads BalitaNLP + MT-augmented fake news).",
    )
    parser.add_argument(
        "--cebuano-only",
        action="store_true",
        default=False,
        help="Train on Cebuano data only: CebuaNER as Real + "
        "MT-translated Cebuano fake news from jcblaise.",
    )
    args = parser.parse_args()

    if args.tagalog_only and args.cebuano_only:
        print("ERROR: --tagalog-only and --cebuano-only are mutually exclusive.")
        sys.exit(1)

    print("=" * 60)
    print("  FAKE NEWS DETECTOR — Enhanced Model Training")
    if args.tagalog_only:
        print("  MODE: Tagalog/Filipino articles only")
    elif args.cebuano_only:
        print("  MODE: Cebuano articles only")
    print("=" * 60)

    # Decide filename suffix for language-specific sub-models
    if args.tagalog_only:
        lang_suffix = "_tagalog"
    elif args.cebuano_only:
        lang_suffix = "_cebuano"
    else:
        lang_suffix = ""  # mixed/default model

    # 1. Load data
    df = load_fake_news_dataset(
        tagalog_only=args.tagalog_only,
        cebuano_only=args.cebuano_only,
    )

    # ── Guard: require both classes before proceeding ─────────────────────────
    n_classes = df["label"].nunique()
    if n_classes < 2:
        present = df["label"].unique().tolist()
        print("\n" + "=" * 60)
        print("  ⚠️  TRAINING ABORTED — Only 1 class found in dataset!")
        print("=" * 60)
        print(f"  Classes present: {present}  (need both 0=Real and 1=Fake)")
        print()
        print("  This usually means the local CSV datasets are missing.")
        print("  Required files for mixed mode:")
        print("    • data/raw/fakenews/fakenews/full.csv      (Tagalog fake/real)")
        print("    • data/raw/philippine_corpus/Philippine Fake News Corpus.csv  (English)")
        print("    • data/raw/augmented_ceb_fakes.csv         (Cebuano fake)")
        print("    • data/raw/augmented_tl_fakes.csv          (Tagalog fake)")
        print()
        print("  On HuggingFace Spaces: upload pre-trained model .pkl files")
        print("  to data_models/ instead of relying on auto-train.")
        print("=" * 60)
        sys.exit(0)  # exit 0 so start.sh allows the server to still boot
    # ─────────────────────────────────────────────────────────────────────────

    # Language-specific modes previously undersampled real news at load time;
    # now we load more real news and oversample the minority (fake) class instead,
    # which gives the model more diverse real-news examples to learn from.
    X_texts, y_labels = preprocess(df, undersample=False, oversample=True)

    # 3. Train & evaluate
    # Cebuano-only: reduce model complexity to prevent memorizing source-format
    # artifacts (machine-translated fakes vs. native CebuaNER text). Lower
    # max_depth forces the model to use weaker, more-generalizable signals.
    if args.cebuano_only:
        model, vectorizer, scaler, svd, metrics = train_model(
            X_texts, y_labels,
            max_depth=8,
            min_samples_leaf=5,
        )
    else:
        model, vectorizer, scaler, svd, metrics = train_model(X_texts, y_labels)

    # 4. Save
    print("\n" + "=" * 60)
    print("  SAVING ARTIFACTS")
    print("=" * 60)
    save_artifacts(model, vectorizer, scaler, svd, metrics, lang_suffix=lang_suffix)

    print("\n" + "=" * 60)
    print("  TRAINING COMPLETE!")
    print("=" * 60)


if __name__ == "__main__":
    main()