checker/internal/core.py

"""
Internal Checker — ML-based fake news detection + bias analysis
================================================================
Combines:
1. Fake News Validation: Random Forest with hybrid features
   (TF-IDF + MiniLM embeddings + stylometric)
2. Bias Analysis: Heuristic-based political leaning + subjectivity scoring
"""

import os
import re
import numpy as np
import joblib
from scipy.sparse import hstack, csr_matrix
from textblob import TextBlob
import textstat
from sentence_transformers import SentenceTransformer
from checker.internal.bias_analyzer import analyze_bias
from checker.internal.structure_analyzer import analyze_structure


# Paths
PROJECT_ROOT = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
DATA_MODELS_DIR = os.path.join(PROJECT_ROOT, "data_models")


# ── MiniLM Model (lazy-loaded singleton) ──
MINILM_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
_minilm_model = None


def get_minilm_model():
    """Load the multilingual MiniLM model (cached after first call)."""
    global _minilm_model
    if _minilm_model is None:
        _minilm_model = SentenceTransformer(MINILM_MODEL_NAME)
    return _minilm_model


def clean_text(text):
    """Clean text to match the training preprocessing."""
    if not text or not isinstance(text, str):
        return ""
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"https?://\S+", " ", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip()


# ── Word lists for linguistic features ──
FIRST_PERSON_PRONOUNS = {
    "i",
    "me",
    "my",
    "mine",
    "myself",
    "we",
    "us",
    "our",
    "ours",
    "ourselves",
    "ako",
    "ko",
    "akin",
    "aking",
    "natin",
    "atin",
    "namin",
    "amin",
    "tayo",
    "kami",
    "ta",
}

AUXILIARY_VERBS = {
    "have",
    "has",
    "had",
    "do",
    "does",
    "did",
    "will",
    "would",
    "shall",
    "should",
    "may",
    "might",
    "can",
    "could",
    "must",
    "am",
    "is",
    "are",
    "was",
    "were",
    "be",
    "been",
    "being",
    "ay",
    "dapat",
    "mayroon",
    "meron",
    "maaari",
    "pwede",
    "kailangan",
}

ANALYTICAL_WORDS = {
    "the",
    "a",
    "an",
    "of",
    "in",
    "on",
    "at",
    "to",
    "for",
    "with",
    "by",
    "from",
    "about",
    "between",
    "through",
    "during",
    "before",
    "after",
    "ang",
    "ng",
    "sa",
    "mga",
    "nang",
    "para",
    "tungkol",
    "mula",
}

CERTAINTY_WORDS = {
    "always",
    "never",
    "absolutely",
    "definitely",
    "certainly",
    "undoubtedly",
    "clearly",
    "obviously",
    "without doubt",
    "guaranteed",
    "proven",
    "fact",
    "undeniable",
    "indisputable",
    "every",
    "all",
    "palagi",
    "sigurado",
    "tiyak",
    "talaga",
    "totoo",
    "lagi",
    "walang duda",
}

TENTATIVE_WORDS = {
    "perhaps",
    "maybe",
    "possibly",
    "might",
    "could",
    "likely",
    "unlikely",
    "suggests",
    "appears",
    "seems",
    "allegedly",
    "reportedly",
    "according",
    "probable",
    "approximately",
    "estimated",
    "siguro",
    "marahil",
    "maaaring",
    "mukhang",
    "parang",
    "umano",
    "diumano",
}

CLOUT_WORDS = {
    "must",
    "demand",
    "require",
    "order",
    "command",
    "insist",
    "decree",
    "mandate",
    "authority",
    "power",
    "control",
    "dominant",
    "superior",
    "we must",
    "you must",
    "kailangan",
    "dapat",
    "utos",
    "kapangyarihan",
    "kontrol",
    "mando",
}

PAST_FOCUS_WORDS = {
    "talked",
    "did",
    "ago",
    "said",
    "was",
    "were",
    "had",
    "went",
    "told",
    "noon",
    "nakaraan",
    "dati",
    "kahapon",
}

PRESENT_FOCUS_WORDS = {
    "now",
    "is",
    "today",
    "are",
    "being",
    "currently",
    "ongoing",
    "ngayon",
    "kasalukuyan",
}

FUTURE_FOCUS_WORDS = {
    "soon",
    "will",
    "may",
    "shall",
    "going",
    "plan",
    "expect",
    "tomorrow",
    "bukas",
    "darating",
    "magiging",
    "gagawin",
}


def extract_stylometric_features(text):
    """Extract the same 25 stylometric features used during training."""
    if not text or not isinstance(text, str):
        return [0.0] * 25

    words = text.split()
    token_count = len(words)
    if token_count == 0:
        return [0.0] * 25

    words_lower = [w.lower() for w in words]
    text_len = len(text)

    exclamation_density = text.count("!") / token_count
    question_count = text.count("?")

    caps_words = sum(1 for w in words if len(w) >= 2 and w.isupper())
    caps_ratio = caps_words / token_count

    sentences = re.split(r"[.!?]+", text)
    sentences = [s.strip() for s in sentences if s.strip()]
    avg_sentence_length = (
        sum(len(s.split()) for s in sentences) / len(sentences)
        if sentences
        else token_count
    )

    punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0

    unique_words = len(set(words_lower))
    unique_word_ratio = unique_words / token_count

    avg_word_length = sum(len(w) for w in words) / token_count

    try:
        subjectivity = TextBlob(text).sentiment.subjectivity
    except Exception:
        subjectivity = 0.0

    try:
        flesch_reading_ease = textstat.flesch_reading_ease(text)
        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
        coleman_liau_index = textstat.coleman_liau_index(text)
        ari = textstat.automated_readability_index(text)
    except Exception:
        flesch_reading_ease = 0.0
        flesch_kincaid_grade = 0.0
        coleman_liau_index = 0.0
        ari = 0.0

    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
    first_person_ratio = first_person_count / token_count

    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
    auxiliary_verb_ratio = aux_count / token_count

    try:
        gunning_fog_index = textstat.gunning_fog(text)
    except Exception:
        gunning_fog_index = 0.0

    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
    analytical_thinking = analytical_count / token_count

    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
    certainty_score = certainty_count / token_count

    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
    tentative_score = tentative_count / token_count

    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
    clout_score = clout_count / token_count

    comma_period_count = text.count(",") + text.count(".")
    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0

    informal_count = (
        text.count("(")
        + text.count(")")
        + text.count("—")
        + text.count("–")
        + text.count("-")
        + text.count("...")
        + text.count("…")
    )
    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0

    # 23. Past focus ratio
    past_count = sum(1 for w in words_lower if w in PAST_FOCUS_WORDS)
    past_focus_ratio = past_count / token_count

    # 24. Present focus ratio
    present_count = sum(1 for w in words_lower if w in PRESENT_FOCUS_WORDS)
    present_focus_ratio = present_count / token_count

    # 25. Future focus ratio
    future_count = sum(1 for w in words_lower if w in FUTURE_FOCUS_WORDS)
    future_focus_ratio = future_count / token_count

    return [
        float(exclamation_density),
        float(question_count),
        float(caps_ratio),
        float(avg_sentence_length),
        float(punctuation_density),
        float(token_count),
        float(unique_word_ratio),
        float(avg_word_length),
        float(subjectivity),
        float(flesch_reading_ease),
        float(flesch_kincaid_grade),
        float(coleman_liau_index),
        float(ari),
        float(first_person_ratio),
        float(auxiliary_verb_ratio),
        float(gunning_fog_index),
        float(analytical_thinking),
        float(certainty_score),
        float(tentative_score),
        float(clout_score),
        float(comma_period_density),
        float(informal_punct_density),
        float(past_focus_ratio),
        float(present_focus_ratio),
        float(future_focus_ratio),
    ]


class InternalChecker:
    """Dual-output internal checker: News Validation + Bias Analysis.

    Language routing:
      - Detects the language of the input article (Tagalog / Cebuano / other).
      - If a language-specific sub-model exists on disk, uses it automatically.
          tl / fil  →  rf_fakenews_tagalog.pkl
          ceb       →  rf_fakenews_cebuano.pkl
      - Falls back to the mixed model (rf_fakenews_model.pkl) for English or
        when no language sub-model has been trained yet.
    """

    # Lazy-loaded cache: suffix → (model, vectorizer, scaler, svd) or None
    _sub_model_cache: dict = {}

    def __init__(self):
        # Always load the mixed / default model as the base fallback
        model_path = os.path.join(DATA_MODELS_DIR, "rf_fakenews_model.pkl")
        vectorizer_path = os.path.join(DATA_MODELS_DIR, "tfidf_fakenews.pkl")
        scaler_path = os.path.join(DATA_MODELS_DIR, "stylo_scaler.pkl")
        svd_path = os.path.join(DATA_MODELS_DIR, "tfidf_svd.pkl")

        if not all(
            os.path.exists(p) for p in [model_path, vectorizer_path, scaler_path, svd_path]
        ):
            raise FileNotFoundError(
                "Model artifacts not found. Run 'python backend/train.py' first."
            )

        self.model = joblib.load(model_path)
        self.vectorizer = joblib.load(vectorizer_path)
        self.scaler = joblib.load(scaler_path)
        self.svd = joblib.load(svd_path)

    # ── Sub-model helpers ─────────────────────────────────────────────

    # Exclusive Cebuano function words / particles that rarely appear in Tagalog.
    # Three or more hits in the text strongly suggest the input is Cebuano.
    _CEBUANO_MARKERS = frozenset({
        "ug", "nga", "kang", "gikan", "aron", "usab", "pud", "dili",
        "matud", "atong", "nato", "bisan", "samtang", "apan", "siya",
        "niya", "nila", "nako", "namo", "ninyo", "imong", "akong",
        "niini", "niining", "kadto", "niadto", "kini", "kana",
        "giingon", "miingon", "matud", "nagkanayon",
    })

    @classmethod
    def _detect_lang(cls, text: str) -> str:
        """Return ISO language code, with a Cebuano correction heuristic.

        langdetect often confuses Cebuano with Tagalog because both are
        Filipino languages sharing many root words.  When langdetect returns
        'tl' but the text contains 3+ exclusive Cebuano markers, we override
        to 'ceb' so the correct sub-model is used.
        """
        try:
            from langdetect import detect
            raw = detect(text[:1000])
        except Exception:
            return "en"

        # Cebuano correction: if langdetect says Tagalog but the text has
        # enough Cebuano-exclusive markers, reclassify as Cebuano.
        if raw in ("tl", "fil"):
            words_lower = set(text.lower().split())
            hits = words_lower & cls._CEBUANO_MARKERS
            if len(hits) >= 3:
                return "ceb"

        return raw

    def _load_sub_model(self, suffix: str):
        """Lazy-load a language-specific sub-model; cache result."""
        if suffix in self._sub_model_cache:
            return self._sub_model_cache[suffix]

        m = os.path.join(DATA_MODELS_DIR, f"rf_fakenews{suffix}.pkl")
        v = os.path.join(DATA_MODELS_DIR, f"tfidf_fakenews{suffix}.pkl")
        s = os.path.join(DATA_MODELS_DIR, f"stylo_scaler{suffix}.pkl")
        d = os.path.join(DATA_MODELS_DIR, f"tfidf_svd{suffix}.pkl")

        if all(os.path.exists(p) for p in [m, v, s, d]):
            sub = (joblib.load(m), joblib.load(v), joblib.load(s), joblib.load(d))
        else:
            sub = None  # not trained yet — will fall back to mixed model

        self._sub_model_cache[suffix] = sub
        return sub

    def _get_components_for(self, lang: str):
        """Return (model, vectorizer, scaler, svd) for the detected language."""
        lang_to_suffix = {"tl": "_tagalog", "fil": "_tagalog", "ceb": "_cebuano"}
        suffix = lang_to_suffix.get(lang)
        if suffix:
            sub = self._load_sub_model(suffix)
            if sub is not None:
                return sub
        return self.model, self.vectorizer, self.scaler, self.svd  # mixed fallback

    # ── Translation-bias detection ──────────────────────────────────────
    # The training data has machine-translated text ONLY in the Fake class,
    # which causes the model to learn "translation artifacts → Fake".
    # This heuristic detects MT-characteristic patterns and applies a
    # confidence recalibration to avoid false positives on translated
    # real articles.

    # Common English loanwords that appear in machine-translated Filipino/Cebuano
    _MT_ENGLISH_LOANWORDS = frozenset({
        "government", "president", "security", "energy", "department",
        "foreign", "affairs", "agreement", "exemption", "transit",
        "international", "diplomatic", "critical", "alliance", "defense",
        "national", "emergency", "strait", "passage", "toll",
        "administration", "bilateral", "coalition", "economy",
        "infrastructure", "legislation", "memorandum", "protocol",
        "resolution", "sanctions", "sovereignty", "territorial",
        "vaccination", "quarantine", "pandemic", "inflation",
        "investment", "subsidy", "regulation", "amendment",
    })

    # Filipino/Cebuano function words that co-occur with English loanwords in MT
    _FILIPINO_FUNCTION = frozenset({
        "ang", "ng", "sa", "mga", "na", "ay", "at", "ni", "si", "kay",
        "para", "dahil", "matapos", "bilang", "kung", "pero",
        "nga", "ug", "og", "kang", "alang", "aron", "tungod",
    })

    @classmethod
    def _has_translation_artifacts(cls, text: str, lang: str) -> bool:
        """Detect if text shows machine-translation characteristics.

        MT-translated Filipino/Cebuano text typically has:
        - High density of English loanwords (untranslated terms)
        - Mixed with Filipino/Cebuano function words
        - This pattern does NOT occur in natively-written news
        """
        if lang not in ("tl", "fil", "ceb"):
            return False

        words_lower = set(text.lower().split())
        total = len(text.split())
        if total < 20:
            return False

        eng_hits = len(words_lower & cls._MT_ENGLISH_LOANWORDS)
        fil_hits = len(words_lower & cls._FILIPINO_FUNCTION)

        # MT signature: significant English loanwords + Filipino function words
        # Native Filipino news rarely has 4+ English technical/political terms
        eng_ratio = eng_hits / total if total > 0 else 0
        return eng_hits >= 4 and fil_hits >= 3 and eng_ratio > 0.02

    # ── Public interface ──────────────────────────────────────────────

    def check(self, text, pattern_deviation=None):
        """Run validation, bias analysis, and journalism structure check.

        Args:
            text (str): Article text to analyze.
            pattern_deviation (dict|None): Pre-computed pattern deviation result
                from compare_to_external_pattern(). When provided, it is passed
                directly into analyze_bias() so deviation from reputable sources
                acts as a primary bias signal rather than a post-hoc annotation.
        """
        # Detect language once; reuse it for both validation and bias analysis
        # so the SVO translation excerpt and Filipino lexicon score use the
        # correct source language without a redundant detect() call.
        lang = self._detect_lang(text)
        return {
            "validation": self._validate(text),
            "bias": analyze_bias(text, pattern_deviation=pattern_deviation, lang=lang),
            "structure": analyze_structure(text),
        }

    def _validate(self, text):
        """Predict whether the text is Real, Fake, or Uncertain."""
        cleaned = clean_text(text)
        if not cleaned:
            return {"verdict": "Uncertain", "confidence": 0.0, "probabilities": {}}

        # Detect language → pick best available model
        lang = self._detect_lang(cleaned)
        model, vectorizer, scaler, svd = self._get_components_for(lang)

        # Build hybrid features (same pipeline as training)
        X_tfidf = vectorizer.transform([cleaned])
        X_tfidf_svd = svd.transform(X_tfidf)
        minilm = get_minilm_model()
        embedding = minilm.encode([cleaned])
        stylo_raw = np.array([extract_stylometric_features(cleaned)])
        stylo_scaled = scaler.transform(stylo_raw)
        X_combined = hstack([csr_matrix(X_tfidf_svd), csr_matrix(embedding), csr_matrix(stylo_scaled)])

        # Predict
        proba = model.predict_proba(X_combined)[0]
        classes = model.classes_
        prob_map = {int(cls): float(p) for cls, p in zip(classes, proba)}
        prob_real = prob_map.get(0, 0.0)
        prob_fake = prob_map.get(1, 0.0)

        # ── Translation-bias calibration ──────────────────────────────
        # If the text shows machine-translation artifacts and the model
        # predicts Fake, the prediction may be driven by translation
        # patterns rather than genuine fake-news signals.  Apply a mild
        # dampening to the fake probability.
        is_translated = self._has_translation_artifacts(cleaned, lang)
        if is_translated and prob_fake > prob_real:
            # Dampen: pull fake probability toward 0.5 (uncertainty)
            # The translation bias is systematic (translated text exists ONLY
            # in the Fake training class), so a strong correction is justified.
            # Factor 0.25 means: 83% fake → 58% (Uncertain), 95% → 61% (weak Fake)
            # This lets external source verification determine the final verdict.
            dampening = 0.25
            prob_fake_adj = 0.5 + (prob_fake - 0.5) * dampening
            prob_real_adj = 1.0 - prob_fake_adj
            prob_fake = prob_fake_adj
            prob_real = prob_real_adj

        # Verdict with uncertainty threshold
        CONFIDENCE_THRESHOLD = 0.60
        if prob_real >= CONFIDENCE_THRESHOLD:
            verdict = "Real"
            confidence = prob_real
        elif prob_fake >= CONFIDENCE_THRESHOLD:
            verdict = "Fake"
            confidence = prob_fake
        else:
            verdict = "Uncertain"
            confidence = max(prob_real, prob_fake)

        lang_label = {"tl": "Tagalog", "fil": "Tagalog", "ceb": "Cebuano"}.get(
            lang, "English/Other"
        )

        result = {
            "verdict": verdict,
            "confidence": confidence,
            "probabilities": {"Real": prob_real, "Fake": prob_fake},
            "detected_lang": lang_label,
        }

        # Add a note if translation artifacts were detected
        if is_translated:
            result["translation_note"] = (
                "This text appears to be machine-translated. "
                "ML confidence has been calibrated to account for "
                "translation artifacts that may affect the prediction."
            )

        return result