""" Internal Fact Checker — Training Script (Fake News Detection) ============================================================== Trains a Random Forest classifier on multiple Filipino/Cebuano news datasets to classify news articles as Real or Fake. Datasets: 1. jcblaise/fake_news_filipino (local CSV) 2. Philippine Fake News Corpus (local CSV) 3. josephimperial/CebuaNER (HuggingFace — Cebuano news, treated as Real) Enhanced with: - Hybrid feature matrix (TF-IDF + MiniLM embeddings + stylometric) - MiniLM multilingual embeddings (384-dim semantic features) - 25 stylometric features (incl. subjectivity, caps ratio, exclamation density) - K-Fold Cross-Validation (k=5) - Tuned hyperparameters (n_estimators=500, max_depth=20, class_weight=balanced) Usage: python backend/train.py """ import sys import os import json import time import re import numpy as np # Add project root to path PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, PROJECT_ROOT) import pandas as pd import joblib from textblob import TextBlob import textstat from scipy.sparse import hstack, csr_matrix from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.ensemble import RandomForestClassifier from sklearn.preprocessing import StandardScaler from sklearn.decomposition import TruncatedSVD from sklearn.metrics import classification_report, accuracy_score, confusion_matrix from sentence_transformers import SentenceTransformer # ── Paths ── DATA_MODELS_DIR = os.path.join(PROJECT_ROOT, "data_models") # ── MiniLM Model (lazy-loaded singleton) ── MINILM_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2" _minilm_model = None def get_minilm_model(): """Load the multilingual MiniLM model (cached after first call).""" global _minilm_model if _minilm_model is None: print(" Loading MiniLM model...") _minilm_model = SentenceTransformer(MINILM_MODEL_NAME) return _minilm_model # ─────────────────────────────────────────────────────────── # Text Cleaning # ─────────────────────────────────────────────────────────── def clean_text(text): """Basic text cleaning for Filipino news articles.""" if not text or not isinstance(text, str): return "" text = re.sub(r"<[^>]+>", " ", text) # HTML tags text = re.sub(r"https?://\S+", " ", text) # URLs text = re.sub(r"\s+", " ", text) # Extra whitespace return text.strip() # ─────────────────────────────────────────────────────────── # Stylometric Feature Extraction # ─────────────────────────────────────────────────────────── # ── Word lists for linguistic features ── FIRST_PERSON_PRONOUNS = { # English "i", "me", "my", "mine", "myself", "we", "us", "our", "ours", "ourselves", # Filipino "ako", "ko", "akin", "aking", "natin", "atin", "namin", "amin", "tayo", "kami", "ta", } AUXILIARY_VERBS = { # English "have", "has", "had", "do", "does", "did", "will", "would", "shall", "should", "may", "might", "can", "could", "must", "am", "is", "are", "was", "were", "be", "been", "being", # Filipino "ay", "dapat", "mayroon", "meron", "maaari", "pwede", "kailangan", } ANALYTICAL_WORDS = { # English articles + prepositions "the", "a", "an", "of", "in", "on", "at", "to", "for", "with", "by", "from", "about", "between", "through", "during", "before", "after", # Filipino "ang", "ng", "sa", "mga", "nang", "para", "tungkol", "mula", } CERTAINTY_WORDS = { # English "always", "never", "absolutely", "definitely", "certainly", "undoubtedly", "clearly", "obviously", "without doubt", "guaranteed", "proven", "fact", "undeniable", "indisputable", "every", "all", # Filipino "palagi", "sigurado", "tiyak", "talaga", "totoo", "lagi", "walang duda", } TENTATIVE_WORDS = { # English "perhaps", "maybe", "possibly", "might", "could", "likely", "unlikely", "suggests", "appears", "seems", "allegedly", "reportedly", "according", "probable", "approximately", "estimated", # Filipino "siguro", "marahil", "maaaring", "mukhang", "parang", "umano", "diumano", } CLOUT_WORDS = { # English — authority and dominance markers "must", "demand", "require", "order", "command", "insist", "decree", "mandate", "authority", "power", "control", "dominant", "superior", "we must", "you must", # Filipino "kailangan", "dapat", "utos", "kapangyarihan", "kontrol", "mando", } PAST_FOCUS_WORDS = { "talked", "did", "ago", "said", "was", "were", "had", "went", "told", "noon", "nakaraan", "dati", "kahapon", } PRESENT_FOCUS_WORDS = { "now", "is", "today", "are", "being", "currently", "ongoing", "ngayon", "kasalukuyan", } FUTURE_FOCUS_WORDS = { "soon", "will", "may", "shall", "going", "plan", "expect", "tomorrow", "bukas", "darating", "magiging", "gagawin", } def extract_stylometric_features(text): """Extract linguistic style features from text. Features (25 total — Hybrid Feature Set): 1. exclamation_density — Exclamation marks per word 2. question_count — Number of '?' characters 3. caps_ratio — Uppercase Ratio (ALL CAPS words / total words) 4. avg_sentence_length — Average number of words per sentence 5. punctuation_density — Punctuation chars per 100 chars 6. token_count — Total word count (metadata feature) 7. unique_word_ratio — Unique words / total words (vocabulary richness) 8. avg_word_length — Average word length in characters 9. subjectivity — TextBlob subjectivity score (0=objective, 1=subjective) 10. flesch_reading_ease — Flesch Reading Ease (higher = easier to read) 11. flesch_kincaid_grade — Flesch-Kincaid Grade Level 12. coleman_liau_index — Coleman-Liau Index 13. ari — Automated Readability Index 14. first_person_ratio — First-person pronouns / total words 15. auxiliary_verb_ratio — Auxiliary/linking verbs / total words 16. gunning_fog_index — Gunning Fog readability index 17. analytical_thinking — Articles + prepositions / total words 18. certainty_score — High-certainty words / total words 19. tentative_score — Hedge/tentative words / total words 20. clout_score — Dominance/authority words / total words 21. comma_period_density — (commas + periods) per 100 chars 22. informal_punct_density — (parentheses + dashes + ellipses) per 100 chars 23. past_focus_ratio — Past-tense / historical keywords / total words 24. present_focus_ratio — Present-tense keywords / total words 25. future_focus_ratio — Future-tense keywords / total words Returns: List of 25 float values. """ if not text or not isinstance(text, str): return [0.0] * 25 words = text.split() token_count = len(words) if token_count == 0: return [0.0] * 25 words_lower = [w.lower() for w in words] text_len = len(text) # 1. Exclamation Mark Density (per word) exclamation_density = text.count("!") / token_count # 2. Question mark count question_count = text.count("?") # 3. Uppercase Ratio (ALL CAPS words with 2+ chars / total words) caps_words = sum(1 for w in words if len(w) >= 2 and w.isupper()) caps_ratio = caps_words / token_count # 4. Average sentence length sentences = re.split(r"[.!?]+", text) sentences = [s.strip() for s in sentences if s.strip()] avg_sentence_length = ( sum(len(s.split()) for s in sentences) / len(sentences) if sentences else token_count ) # 5. Punctuation density (per 100 chars) punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...") punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0 # 6. Token Count (article length — real news is typically longer) # (already computed as token_count) # 7. Unique word ratio (vocabulary richness) unique_words = len(set(words_lower)) unique_word_ratio = unique_words / token_count # 8. Average word length avg_word_length = sum(len(w) for w in words) / token_count # 9. Subjectivity Score (TextBlob: 0=objective, 1=subjective) try: subjectivity = TextBlob(text).sentiment.subjectivity except Exception: subjectivity = 0.0 # 10-13. Readability Scores (textstat) try: flesch_reading_ease = textstat.flesch_reading_ease(text) flesch_kincaid_grade = textstat.flesch_kincaid_grade(text) coleman_liau_index = textstat.coleman_liau_index(text) ari = textstat.automated_readability_index(text) except Exception: flesch_reading_ease = 0.0 flesch_kincaid_grade = 0.0 coleman_liau_index = 0.0 ari = 0.0 # 14. First-person pronoun ratio first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS) first_person_ratio = first_person_count / token_count # 15. Auxiliary verb ratio aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS) auxiliary_verb_ratio = aux_count / token_count # 16. Gunning Fog Index try: gunning_fog_index = textstat.gunning_fog(text) except Exception: gunning_fog_index = 0.0 # 17. Analytical thinking (articles + prepositions ratio) analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS) analytical_thinking = analytical_count / token_count # 18. Certainty score certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS) certainty_score = certainty_count / token_count # 19. Tentative score tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS) tentative_score = tentative_count / token_count # 20. Clout score (dominance/authority markers) clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS) clout_score = clout_count / token_count # 21. Comma + period density (per 100 chars) comma_period_count = text.count(",") + text.count(".") comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0 # 22. Informal punctuation density — parentheses, dashes, ellipses (per 100 chars) informal_count = ( text.count("(") + text.count(")") + text.count("—") + text.count("–") + text.count("-") + text.count("...") + text.count("…") ) informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0 # 23. Past focus ratio past_count = sum(1 for w in words_lower if w in PAST_FOCUS_WORDS) past_focus_ratio = past_count / token_count # 24. Present focus ratio present_count = sum(1 for w in words_lower if w in PRESENT_FOCUS_WORDS) present_focus_ratio = present_count / token_count # 25. Future focus ratio future_count = sum(1 for w in words_lower if w in FUTURE_FOCUS_WORDS) future_focus_ratio = future_count / token_count return [ float(exclamation_density), float(question_count), float(caps_ratio), float(avg_sentence_length), float(punctuation_density), float(token_count), float(unique_word_ratio), float(avg_word_length), float(subjectivity), float(flesch_reading_ease), float(flesch_kincaid_grade), float(coleman_liau_index), float(ari), float(first_person_ratio), float(auxiliary_verb_ratio), float(gunning_fog_index), float(analytical_thinking), float(certainty_score), float(tentative_score), float(clout_score), float(comma_period_density), float(informal_punct_density), float(past_focus_ratio), float(present_focus_ratio), float(future_focus_ratio), ] STYLOMETRIC_FEATURE_NAMES = [ "exclamation_density", "question_count", "caps_ratio", "avg_sentence_length", "punctuation_density", "token_count", "unique_word_ratio", "avg_word_length", "subjectivity", "flesch_reading_ease", "flesch_kincaid_grade", "coleman_liau_index", "ari", "first_person_ratio", "auxiliary_verb_ratio", "gunning_fog_index", "analytical_thinking", "certainty_score", "tentative_score", "clout_score", "comma_period_density", "informal_punct_density", "past_focus_ratio", "present_focus_ratio", "future_focus_ratio", ] # ─────────────────────────────────────────────────────────── # Language Detection (optional — used with --tagalog-only / --cebuano-only) # ─────────────────────────────────────────────────────────── def _detect_lang(text: str) -> str: """Detect ISO language code for a text snippet. Returns 'tl', 'ceb', 'en', etc. Falls back to a heuristic when langdetect is not installed or fails on a given sample. """ _TAGALOG_MARKERS = { "ang", "ng", "mga", "sa", "na", "ay", "at", "hindi", "ako", "siya", "nila", "niya", "ito", "iyon", "kung", "para", "nang", "din", "rin", "kaya", "pero", "dahil", "ayon", "noon", "ngayon", "dito", "doon", "sinabi", "araw", "taon", "buwan", } _CEBUANO_MARKERS = { "ug", "nga", "si", "nag", "mao", "kang", "usab", "man", "dayon", "gyud", "kaayo", "lang", "pud", "adto", "kini", "sila", "niadtong", "gitawag", "giingon", "matud", "nasayran", "gidakop", } if not text or not isinstance(text, str) or len(text.split()) < 5: return "unknown" try: from langdetect import detect, LangDetectException from langdetect import DetectorFactory DetectorFactory.seed = 42 return detect(text[:400]) except Exception: pass # Heuristic fallback words = set(text.lower().split()) tl_hits = len(words & _TAGALOG_MARKERS) ceb_hits = len(words & _CEBUANO_MARKERS) if tl_hits == 0 and ceb_hits == 0: return "unknown" return "tl" if tl_hits >= ceb_hits else "ceb" def filter_tagalog(df: "pd.DataFrame", text_col: str = "article") -> "pd.DataFrame": """Return only rows whose text is classified as Tagalog/Filipino.""" print(" Detecting languages (this may take a minute for large datasets)...") langs = df[text_col].apply(lambda t: _detect_lang(str(t))) mask = langs.isin({"tl", "fil"}) tl_count = mask.sum() total = len(df) print( f" Language filter: keeping {tl_count:,} / {total:,} articles " f"detected as Tagalog/Filipino ({tl_count/total*100:.1f}%)" ) return df[mask].reset_index(drop=True) def filter_cebuano(df: "pd.DataFrame", text_col: str = "article") -> "pd.DataFrame": """Return only rows whose text is classified as Cebuano.""" print(" Detecting languages (filtering for Cebuano)...") langs = df[text_col].apply(lambda t: _detect_lang(str(t))) mask = langs.isin({"ceb"}) ceb_count = mask.sum() total = len(df) print( f" Language filter: keeping {ceb_count:,} / {total:,} articles " f"detected as Cebuano ({ceb_count/total*100:.1f}%)" ) return df[mask].reset_index(drop=True) # ─────────────────────────────────────────────────────────── # Machine Translation Augmentation # ─────────────────────────────────────────────────────────── _TRANSLATION_CACHE_PATH = os.path.join( PROJECT_ROOT, "data", "raw", "translation_cache.json" ) def _load_translation_cache() -> dict: if os.path.exists(_TRANSLATION_CACHE_PATH): try: with open(_TRANSLATION_CACHE_PATH, "r", encoding="utf-8") as f: return json.load(f) except Exception: pass return {} def _save_translation_cache(cache: dict) -> None: os.makedirs(os.path.dirname(_TRANSLATION_CACHE_PATH), exist_ok=True) with open(_TRANSLATION_CACHE_PATH, "w", encoding="utf-8") as f: json.dump(cache, f, ensure_ascii=False, indent=2) def _translate_texts(texts: list, target_lang: str, source_lang: str = "auto") -> list: """Translate a list of texts using Google Translate (free tier via deep-translator). Results are cached to disk at data/raw/translation_cache.json so translation only runs once; subsequent calls read from cache instantly. Args: texts: List of source texts. target_lang: ISO code for the target language ('ceb', 'tl', 'en', ...). source_lang: ISO code for the source language (default: 'auto'-detect). Returns: List of translated strings (same length as input; failures return originals). """ try: from deep_translator import GoogleTranslator except ImportError: print(" Translation: 'deep-translator' not installed — returning originals.") print(" Install with: pip install deep-translator") return texts cache = _load_translation_cache() results = [] new_translations = 0 MAX_RETRIES = 3 for i, text in enumerate(texts): # Truncate to 4500 chars (Google Translate free cap is 5000) snippet = str(text)[:4500] cache_key = f"{target_lang}::{hash(snippet)}" if cache_key in cache: results.append(cache[cache_key]) continue # Progress indicator every 50 new items if new_translations % 50 == 0 and new_translations > 0: print( f" ... {new_translations} translated so far ({i}/{len(texts)} items processed)" ) # Retry with exponential backoff to handle free-tier rate limits translated = None for attempt in range(MAX_RETRIES): try: translated = GoogleTranslator( source=source_lang, target=target_lang ).translate(snippet) break # success except Exception as exc: wait = 2**attempt # 1s, 2s, 4s if attempt < MAX_RETRIES - 1: time.sleep(wait) else: print( f" [translate] Item {i} failed after {MAX_RETRIES} retries: {exc} — keeping original." ) if translated: results.append(translated) cache[cache_key] = translated new_translations += 1 else: results.append(snippet) # fallback to original # Small delay between requests to avoid hitting rate limits time.sleep(0.3) # Save cache incrementally every 50 new translations if new_translations > 0 and new_translations % 50 == 0: _save_translation_cache(cache) if new_translations > 0: _save_translation_cache(cache) print(f" Translated {new_translations} new items (cache updated).") else: print(f" All {len(texts)} translations served from cache.") return results def _get_fake_jcblaise_texts() -> list: """Return the raw article texts labeled as Fake (label=1) from the jcblaise CSV.""" csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv") if not os.path.exists(csv1): return [] df = pd.read_csv(csv1, skiprows=1) if "article" not in df.columns or "label" not in df.columns: return [] # label column is StringDtype (git conflict markers embedded) — coerce to int df["label"] = pd.to_numeric(df["label"], errors="coerce") df = df.dropna(subset=["label"]) df["label"] = df["label"].astype(int) df = df[df["label"] == 1][["article"]].dropna() df = df[~df["article"].astype(str).str.startswith(("=======", ">>>>>>>"))] return df["article"].astype(str).tolist() def _augment_fake_news_with_translation(target_lang: str) -> "pd.DataFrame | None": """Generate translated fake-news articles for a target language. On the first run, translates source articles and saves them to: data/raw/augmented_tl_fakes.csv (Tagalog) data/raw/augmented_ceb_fakes.csv (Cebuano) On all subsequent runs, loads the CSV directly — no translation, no network calls, instant start. Strategy: - Tagalog ('tl') : translate English "Not Credible" articles from the Philippine Corpus (up to 500) → Tagalog fakes. - Cebuano ('ceb'): translate jcblaise Tagalog fake articles → Cebuano. Returns: pd.DataFrame with columns ['article', 'label'] (label=1, Fake) or None. """ lang_name = {"tl": "Tagalog", "ceb": "Cebuano"}.get(target_lang, target_lang) # ── Fast path: load pre-saved CSV if it exists ── csv_out = os.path.join( PROJECT_ROOT, "data", "raw", f"augmented_{target_lang}_fakes.csv" ) if os.path.exists(csv_out): df_cached = pd.read_csv(csv_out) if "article" in df_cached.columns and not df_cached.empty: print( f" [aug] Loaded pre-saved {lang_name} fakes from cache: " f"{len(df_cached)} articles ({os.path.basename(csv_out)})" ) df_cached["label"] = 1 return df_cached[["article", "label"]] # ── Slow path: translate, then save ── if target_lang == "ceb": # Source: all jcblaise Tagalog fake articles texts = _get_fake_jcblaise_texts() if not texts: print(" [aug] No source fake texts found for Cebuano augmentation.") return None print( f" [aug] Translating {len(texts)} jcblaise fake articles → Cebuano " f"(will save to {os.path.basename(csv_out)} for future runs)..." ) translated = _translate_texts(texts, target_lang="ceb", source_lang="tl") elif target_lang == "tl": # Source: English "Not Credible" articles from Philippine Corpus (up to 500) csv2 = os.path.join( PROJECT_ROOT, "data", "raw", "philippine_corpus", "Philippine Fake News Corpus.csv", ) if not os.path.exists(csv2): print( " [aug] Philippine Corpus not found — skipping Tagalog augmentation." ) return None df2 = pd.read_csv(csv2, skiprows=1).rename(columns={"Content": "article"}) df2["label_raw"] = df2.get("Label", "") df2 = df2[df2["label_raw"] == "Not Credible"][["article"]].dropna() df2 = df2[~df2["article"].astype(str).str.startswith(("=======", ">>>>>>>"))] texts = df2["article"].astype(str).tolist()[:2000] if not texts: return None print( f" [aug] Translating {len(texts)} English fake articles → Tagalog " f"(will save to {os.path.basename(csv_out)} for future runs)..." ) translated = _translate_texts(texts, target_lang="tl", source_lang="en") else: print(f" [aug] Unsupported target language '{target_lang}' — skipping.") return None translated = [t for t in translated if t and len(str(t).split()) >= 5] if not translated: print(" [aug] No translated texts after filtering.") return None df_aug = pd.DataFrame({"article": translated, "label": 1}) # label=1 → Fake # ── Save to CSV for future runs ── os.makedirs(os.path.dirname(csv_out), exist_ok=True) df_aug[["article"]].to_csv(csv_out, index=False) print(f" [aug] Saved {len(df_aug)} {lang_name} fakes → {csv_out}") return df_aug def _load_cebuaner_as_dataframe(): """Load josephimperial/CebuaNER from HuggingFace and reconstruct article texts. CebuaNER is a token-level NER dataset compiled from credible Cebuano news sources (Yes the Best Dumaguete, Filipinas Bisaya, Sunstar Cebu). There are no fake-news labels — every entry is treated as Real (label=0). The dataset has 188 k+ token rows. We reconstruct sentences by joining each row's token list, then group consecutive sentences into ~paragraph- sized chunks (≥ 30 tokens) so each chunk resembles a short news excerpt. Returns: pd.DataFrame with columns ['article', 'label'] or None on failure. """ try: from datasets import load_dataset # optional dep — only needed for training except ImportError: print(" [3] 'datasets' library not installed — skipping CebuaNER.") print(" Install with: pip install datasets") return None print(" [3] Downloading josephimperial/CebuaNER from HuggingFace...") try: ds = load_dataset("josephimperial/CebuaNER") except Exception as exc: print(f" [3] Failed to load CebuaNER: {exc}") return None # Combine all splits (train / validation / test) into one list of sentences sentences = [] for split_name, split_data in ds.items(): for row in split_data: # CebuaNER schema: {'text': str} — one sentence per row text = row.get("text") or " ".join( row.get("tokens") or row.get("words") or [] ) if text and text.strip(): sentences.append(text.strip()) if not sentences: print(" [3] CebuaNER: no token rows found, skipping.") return None # Group sentences into article-sized chunks (target ≥ 100 tokens). # Using 100 instead of 30 ensures each chunk resembles a proper news # excerpt rather than a short sentence fragment — reducing the model's # tendency to classify based on chunk length alone. MIN_CHUNK_TOKENS = 100 articles = [] buffer = [] buffer_tokens = 0 for sent in sentences: buffer.append(sent) buffer_tokens += len(sent.split()) if buffer_tokens >= MIN_CHUNK_TOKENS: articles.append(" ".join(buffer)) buffer = [] buffer_tokens = 0 if buffer and buffer_tokens >= 30: # flush remaining only if non-trivial articles.append(" ".join(buffer)) df3 = pd.DataFrame({"article": articles, "label": 0}) # label=0 → Real print(f" [3] josephimperial/CebuaNER: {len(df3)} reconstructed article chunks") return df3 def _load_balitanlp_as_dataframe(max_articles: int = 10_000) -> "pd.DataFrame | None": """Load LanceBunag/BalitaNLP from HuggingFace (streaming, no-image config). BalitaNLP is a Filipino/Tagalog news dataset with 351k real news articles scraped from credible Philippine news outlets. There are no fake-news labels — every entry is treated as Real (label=0). The `no-image` config is used to avoid downloading the 40 GB image variant. Streaming is used to cap how many articles are loaded. Fields used: - title : Article headline - body : List of paragraph strings (joined with double newline) Args: max_articles: Maximum number of articles to load (default 10,000). Returns: pd.DataFrame with columns ['article', 'label'] or None on failure. """ try: from datasets import load_dataset # optional dep except ImportError: print(" [4] 'datasets' library not installed — skipping BalitaNLP.") return None print( f" [4] Streaming LanceBunag/BalitaNLP (no-image, up to {max_articles:,} articles)..." ) try: ds = load_dataset( "LanceBunag/BalitaNLP", "no-image", split="train", streaming=True, ) except Exception as exc: print(f" [4] Failed to load BalitaNLP: {exc}") return None articles = [] for row in ds: title = str(row.get("title") or "").strip() body = row.get("body") or [] if isinstance(body, list): body_text = "\n\n".join(p for p in body if p and p.strip()) else: body_text = str(body).strip() # Combine title + body for a richer text representation full_text = f"{title}\n\n{body_text}".strip() if title else body_text if full_text and len(full_text.split()) >= 10: articles.append(full_text) if len(articles) >= max_articles: break if not articles: print(" [4] BalitaNLP: no articles loaded, skipping.") return None df4 = pd.DataFrame({"article": articles, "label": 0}) # label=0 → Real print(f" [4] LanceBunag/BalitaNLP: {len(df4):,} articles (Tagalog real news)") return df4 def load_fake_news_dataset( tagalog_only: bool = False, cebuano_only: bool = False, ): """Load and merge fake news datasets. Merges (depending on mode): 1. jcblaise/fake_news_filipino (local CSV) 2. Philippine Fake News Corpus (local CSV) 3. josephimperial/CebuaNER (HuggingFace) 4. LanceBunag/BalitaNLP (HuggingFace, streaming) + MT-augmented fake news (disk-cached translations) Args: tagalog_only: If True, load only Tagalog/Filipino data + augmented fakes. cebuano_only: If True, load only Cebuano data + translated Cebuano fakes. (mutually exclusive with tagalog_only) Deduplicates by a fingerprint of article length + first 200 chars. """ print("Loading datasets...") frames = [] if cebuano_only: # ── Cebuano-only mode ────────────────────────────────────────────── # Real: translated jcblaise reals (augmented_ceb_reals.csv) # Fake: translated jcblaise fakes (augmented_ceb_fakes.csv) # # CebuaNER reconstructed token chunks do NOT resemble real news, so # the model trained on them classifies all actual Cebuano news as fake. # Using translated jcblaise real articles gives proper news-quality text. print(" Mode: CEBUANO-ONLY") # ── Fake class: MT-translated jcblaise fakes → Cebuano ── df_ceb_fake = _augment_fake_news_with_translation(target_lang="ceb") ceb_fake_count = len(df_ceb_fake) if df_ceb_fake is not None else 0 # ── Real class: MT-translated jcblaise reals → Cebuano ── ceb_reals_csv = os.path.join( PROJECT_ROOT, "data", "raw", "augmented_ceb_reals.csv" ) if os.path.exists(ceb_reals_csv): df_ceb_real = pd.read_csv(ceb_reals_csv) if "article" in df_ceb_real.columns and not df_ceb_real.empty: df_ceb_real = df_ceb_real[["article"]].dropna().copy() df_ceb_real = df_ceb_real[df_ceb_real["article"].str.split().str.len() >= 5] df_ceb_real["label"] = 0 # Real # Balance: cap to match fake count if ceb_fake_count > 0 and len(df_ceb_real) > ceb_fake_count: df_ceb_real = df_ceb_real.sample( n=ceb_fake_count, random_state=42 ).reset_index(drop=True) print( f" [real] augmented_ceb_reals.csv: {len(df_ceb_real)} " f"translated Cebuano real articles" ) frames.append(df_ceb_real) else: print(" [WARN] augmented_ceb_reals.csv exists but has no 'article' column") else: # Fallback: CebuaNER (not ideal, but better than nothing) print( " [WARN] augmented_ceb_reals.csv not found — " "falling back to CebuaNER (run backend/translate_ceb_reals.py first!)" ) df3 = _load_cebuaner_as_dataframe() if df3 is not None: if ceb_fake_count > 0 and len(df3) > ceb_fake_count: df3 = df3.sample( n=ceb_fake_count, random_state=42 ).reset_index(drop=True) frames.append(df3) if df_ceb_fake is not None: frames.append(df_ceb_fake) elif tagalog_only: # ── Tagalog-only mode ────────────────────────────────────────────── # Fake priority: satire_facebook.csv first (real Filipino social media # fake news), then augmented_tl_fakes.csv to fill the quota. # Real news: BalitaNLP capped to total fake count (undersampling). print(" Mode: TAGALOG-ONLY (per-dataset filtering, undersampled)") # [1] jcblaise — labeled Filipino fake-news corpus, load ALL rows csv1 = os.path.join( PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv" ) if os.path.exists(csv1): df1 = pd.read_csv(csv1)[["article", "label"]].copy() df1 = df1[ ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>")) ] # label is StringDtype due to git conflict markers — coerce to int df1["label"] = pd.to_numeric(df1["label"], errors="coerce") df1 = df1.dropna(subset=["label"]) df1["label"] = df1["label"].astype(int) print( f" [1] jcblaise/fake_news_filipino: {len(df1)} articles " f"(Real={int((df1['label']==0).sum())}, Fake={int((df1['label']==1).sum())})" ) frames.append(df1) else: print(" [1] jcblaise not found, skipping.") # [2] Philippine Corpus — Credible (Real) rows only, language-filtered to Tagalog. # Not Credible (Fake) rows are English; MT augmentation provides # translated Tagalog fakes instead (see [aug] below). csv2 = os.path.join( PROJECT_ROOT, "data", "raw", "philippine_corpus", "Philippine Fake News Corpus.csv", ) if os.path.exists(csv2): df2_raw = pd.read_csv(csv2, skiprows=1).rename( columns={"Content": "article"} ) df2_real = ( df2_raw[df2_raw["Label"] == "Credible"][["article"]].dropna().copy() ) df2_real = df2_real[ ~df2_real["article"].astype(str).str.startswith(("=======", ">>>>>>>")) ] before_filter = len(df2_real) print(" [2] Filtering Philippine Corpus (Credible) to Tagalog...") langs = df2_real["article"].apply(lambda t: _detect_lang(str(t))) df2_real = df2_real[langs.isin({"tl", "fil"})].copy() df2_real["label"] = 0 print( f" [2] Philippine Corpus Tagalog Real: " f"{len(df2_real)} / {before_filter} articles kept" ) if not df2_real.empty: frames.append(df2_real) else: print(" [2] Philippine Corpus not found, skipping.") # [3] CebuaNER — skip (Cebuano, not Tagalog) print(" [3] josephimperial/CebuaNER: skipped (Tagalog-only mode)") # [sat] PRIORITY: satire_facebook.csv — real Filipino social-media fake news. # Loaded FIRST so it is always included in the fake quota. tl_satire_count = 0 csv_satire = os.path.join(PROJECT_ROOT, "data", "raw", "satire_facebook.csv") if os.path.exists(csv_satire): df_sat = pd.read_csv(csv_satire) if "article" in df_sat.columns: df_sat = df_sat[["article"]].dropna().copy() df_sat = df_sat[df_sat["article"].str.split().str.len() >= 5] df_sat["label"] = 1 # Fake/Satire tl_satire_count = len(df_sat) print( f" [sat] satire_facebook.csv (PRIORITY): " f"{tl_satire_count} Tagalog satire posts (all Fake)" ) frames.append(df_sat) else: print(" [sat] satire_facebook.csv not found — skipping priority satire.") # [aug] MT-translated Tagalog fakes — fill remaining quota after satire. df_tl_fake = _augment_fake_news_with_translation(target_lang="tl") if df_tl_fake is not None: frames.append(df_tl_fake) tl_aug_count = len(df_tl_fake) if df_tl_fake is not None else 0 total_tl_fake = tl_satire_count + tl_aug_count print(f" Total Tagalog fakes available: {total_tl_fake} ({tl_satire_count} satire + {tl_aug_count} augmented)") # noqa: E501 # [4] BalitaNLP — Tagalog real news (no undersampling; oversampling balances at preprocess step) df4 = _load_balitanlp_as_dataframe() if df4 is not None: frames.append(df4) else: # ── Mixed mode: equal-sized buckets per language × label ── # Target: 1500 each for English fake, English real, # Tagalog fake, Tagalog real, # Cebuano fake, Cebuano real → 9 000 total N_PER_BUCKET = 1500 print(f" Mode: MIXED (capping each language/label bucket to {N_PER_BUCKET})") # ── [A] English fake & real — Philippine Fake News Corpus ────────── csv2 = os.path.join( PROJECT_ROOT, "data", "raw", "philippine_corpus", "Philippine Fake News Corpus.csv", ) if os.path.exists(csv2): df2_raw = pd.read_csv(csv2, skiprows=1).rename(columns={"Content": "article"}) df2_raw["label"] = df2_raw["Label"].map({"Credible": 0, "Not Credible": 1}) df2_raw = df2_raw[["article", "label"]].dropna().copy() df2_raw = df2_raw[ ~df2_raw["article"].astype(str).str.startswith(("=======", ">>>>>>>")) ] # Language-detect to isolate English articles print(" [A] Detecting language of Philippine Corpus articles (English filter)...") df2_langs = df2_raw["article"].apply(lambda t: _detect_lang(str(t))) df2_en = df2_raw[df2_langs == "en"].copy() print(f" [A] Philippine Corpus: {len(df2_en)} English articles detected") # English Fake (label=1) df_en_fake = df2_en[df2_en["label"] == 1] if len(df_en_fake) > N_PER_BUCKET: df_en_fake = df_en_fake.sample(n=N_PER_BUCKET, random_state=42) df_en_fake = df_en_fake.reset_index(drop=True) print(f" [A] English Fake: {len(df_en_fake)} articles (target {N_PER_BUCKET})") if not df_en_fake.empty: frames.append(df_en_fake) # English Real (label=0) df_en_real = df2_en[df2_en["label"] == 0] if len(df_en_real) > N_PER_BUCKET: df_en_real = df_en_real.sample(n=N_PER_BUCKET, random_state=42) df_en_real = df_en_real.reset_index(drop=True) print(f" [A] English Real: {len(df_en_real)} articles (target {N_PER_BUCKET})") if not df_en_real.empty: frames.append(df_en_real) else: print(" [A] Philippine Fake News Corpus not found — skipping English buckets.") # ── [B] Tagalog fake & real — jcblaise + augmented_tl_fakes ──────── csv1 = os.path.join( PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv" ) if os.path.exists(csv1): df1 = pd.read_csv(csv1)[["article", "label"]].copy() df1 = df1[ ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>")) ] df1["label"] = pd.to_numeric(df1["label"], errors="coerce") df1 = df1.dropna(subset=["label"]) df1["label"] = df1["label"].astype(int) print( f" [B] jcblaise/fake_news_filipino: {len(df1)} articles " f"(Real={int((df1['label']==0).sum())}, Fake={int((df1['label']==1).sum())})" ) # Tagalog Real (label=0) — cap to N_PER_BUCKET df_tl_real = df1[df1["label"] == 0].copy() if len(df_tl_real) > N_PER_BUCKET: df_tl_real = df_tl_real.sample(n=N_PER_BUCKET, random_state=42) df_tl_real = df_tl_real.reset_index(drop=True) print(f" [B] Tagalog Real: {len(df_tl_real)} articles (target {N_PER_BUCKET})") if not df_tl_real.empty: frames.append(df_tl_real) # Tagalog Fake (label=1) — PRIORITY: satire_facebook.csv first, # then jcblaise label=1, then augmented_tl_fakes to fill remaining quota. # Total capped at N_PER_BUCKET. # [sat] PRIORITY — real Filipino social-media satire/fake posts csv_satire = os.path.join(PROJECT_ROOT, "data", "raw", "satire_facebook.csv") df_tl_satire = pd.DataFrame(columns=["article", "label"]) if os.path.exists(csv_satire): df_sat_raw = pd.read_csv(csv_satire) if "article" in df_sat_raw.columns: df_sat_raw = df_sat_raw[["article"]].dropna().copy() df_sat_raw = df_sat_raw[ df_sat_raw["article"].str.split().str.len() >= 5 ] df_sat_raw["label"] = 1 df_tl_satire = df_sat_raw[["article", "label"]] print( f" [B] satire_facebook.csv (PRIORITY): " f"{len(df_tl_satire)} Tagalog satire posts (all Fake)" ) else: print(" [B] satire_facebook.csv not found — skipping satire priority.") satire_count = len(df_tl_satire) remaining = max(0, N_PER_BUCKET - satire_count) # Fill remaining quota: jcblaise fakes first df_tl_fake_jcb = df1[df1["label"] == 1].copy() jcb_fake_count = len(df_tl_fake_jcb) # Then augmented_tl_fakes csv_tl_aug = os.path.join(PROJECT_ROOT, "data", "raw", "augmented_tl_fakes.csv") if os.path.exists(csv_tl_aug): df_tl_aug = pd.read_csv(csv_tl_aug) if "article" in df_tl_aug.columns and not df_tl_aug.empty: df_tl_aug["label"] = 1 df_tl_aug = df_tl_aug[["article", "label"]].dropna() print(f" [B] augmented_tl_fakes.csv: {len(df_tl_aug)} articles") else: df_tl_aug = pd.DataFrame(columns=["article", "label"]) else: df_tl_aug = pd.DataFrame(columns=["article", "label"]) # Combine filler sources and take up to `remaining` rows df_tl_filler = pd.concat( [df_tl_fake_jcb, df_tl_aug], ignore_index=True ).drop_duplicates(subset=["article"]) if len(df_tl_filler) > remaining: df_tl_filler = df_tl_filler.sample(n=remaining, random_state=42) # Final Tagalog fake bucket: satire (priority) + filler df_tl_fake_all = pd.concat( [df_tl_satire, df_tl_filler], ignore_index=True ).drop_duplicates(subset=["article"]).reset_index(drop=True) print( f" [B] Tagalog Fake: {len(df_tl_fake_all)} articles " f"(satire: {satire_count}, jcblaise: {jcb_fake_count}, " f"augmented: {len(df_tl_aug)}, target {N_PER_BUCKET})" ) if not df_tl_fake_all.empty: frames.append(df_tl_fake_all) else: print(" [B] jcblaise dataset not found — skipping Tagalog buckets.") # ── [C] Cebuano fake & real ────────────────────────────────────────── # Cebuano Real — CebuaNER reconstructed article chunks df3 = _load_cebuaner_as_dataframe() if df3 is not None: if len(df3) > N_PER_BUCKET: df3 = df3.sample(n=N_PER_BUCKET, random_state=42) df3 = df3.reset_index(drop=True) print(f" [C] Cebuano Real: {len(df3)} articles (target {N_PER_BUCKET})") frames.append(df3) else: print(" [C] CebuaNER not available — skipping Cebuano real bucket.") # Cebuano Fake — pre-translated augmented_ceb_fakes.csv csv_ceb_aug = os.path.join(PROJECT_ROOT, "data", "raw", "augmented_ceb_fakes.csv") if os.path.exists(csv_ceb_aug): df_ceb_fake = pd.read_csv(csv_ceb_aug) if "article" in df_ceb_fake.columns and not df_ceb_fake.empty: df_ceb_fake["label"] = 1 df_ceb_fake = df_ceb_fake[["article", "label"]].dropna() if len(df_ceb_fake) > N_PER_BUCKET: df_ceb_fake = df_ceb_fake.sample(n=N_PER_BUCKET, random_state=42) df_ceb_fake = df_ceb_fake.reset_index(drop=True) print(f" [C] Cebuano Fake: {len(df_ceb_fake)} articles (target {N_PER_BUCKET})") frames.append(df_ceb_fake) else: print(" [C] augmented_ceb_fakes.csv not found — skipping Cebuano fake bucket.") # [5] Facebook satire posts — loading notes: # In mixed mode: loaded inside bucket [B] as PRIORITY Tagalog fakes. # In tagalog_only mode: loaded above as [sat] PRIORITY. # In cebuano_only mode: not applicable (Tagalog/Filipino content). # No additional satire loading needed here — it is handled per-mode above. if not frames: raise FileNotFoundError( "No datasets found! Place at least one dataset in data/raw/." ) # ── Merge and deduplicate ── df = pd.concat(frames, ignore_index=True) df = df.dropna(subset=["article"]).copy() df = df[df["article"].str.len() > 0].copy() # ── Language filter (cebuano_only only — tagalog_only uses per-dataset filters above) ── if cebuano_only: pass # CebuaNER + MT-translated fakes are already Cebuano by construction # Deduplicate by fingerprint (length + first 200 chars) before = len(df) df["_fingerprint"] = df["article"].apply(lambda x: f"{len(str(x))}_{str(x)[:200]}") df = df.drop_duplicates(subset=["_fingerprint"]).drop(columns=["_fingerprint"]) after = len(df) if before > after: print(f" Removed {before - after} duplicate articles") label_map = {0: "Real", 1: "Fake"} df["label_name"] = df["label"].map(label_map) print(f"\n MERGED TOTAL: {len(df)} articles") print(f" Distribution:\n{df['label_name'].value_counts().to_string()}") return df # ─────────────────────────────────────────────────────────── # Preprocessing # ─────────────────────────────────────────────────────────── def preprocess(df, undersample=False, oversample=True): """Clean text and prepare features. Balancing strategy (applied in order of preference): 1. oversample=True — RandomOverSampler duplicates minority-class (Fake) rows until classes are equal. Used for mixed-language mode where the real:fake ratio can be large. 2. undersample=True — downsample the majority class (legacy fallback). Both: oversample is applied first, then undersample (rarely needed). Neither (language-specific modes): real news is already capped to match fake count at load time, so no resampling is needed here. class_weight='balanced' is ALSO set on the RandomForest, so even without resampling the model still penalises fake-news misclassification more. """ print("\nPreprocessing...") df = df.copy() df = df.dropna(subset=["article"]).copy() df = df[df["article"].str.len() > 0].copy() print(f" After dropping empty rows: {len(df)} articles") if len(df) == 0: raise ValueError("No valid articles remaining after filtering!") counts_before = df["label"].value_counts() print( f" Class distribution before balancing: " f"Real={counts_before.get(0, 0)}, Fake={counts_before.get(1, 0)}" ) print(" Cleaning text...") df.loc[:, "article_clean"] = df["article"].apply(clean_text) texts = df["article_clean"].tolist() labels = df["label"].tolist() # ── Step 1: Oversampling (preferred) ────────────────────────────────────── if oversample: try: from imblearn.over_sampling import RandomOverSampler ros = RandomOverSampler(random_state=42) # RandomOverSampler needs a 2-D feature array; use text indices as proxy idx = [[i] for i in range(len(texts))] idx_res, labels_res = ros.fit_resample(idx, labels) texts = [texts[i[0]] for i in idx_res] labels = list(labels_res) counts_after = {l: labels.count(l) for l in set(labels)} print( f" Class distribution after oversampling: " f"Real={counts_after.get(0, 0)}, Fake={counts_after.get(1, 0)}" ) print(f" Total samples after oversampling: {len(texts)}") except ImportError: print( " [WARNING] imbalanced-learn not installed — skipping oversampling.\n" " Install with: pip install imbalanced-learn" ) # ── Step 2: Undersampling (legacy fallback) ─────────────────────────────── if undersample: import random label_to_texts = {0: [], 1: []} for t, l in zip(texts, labels): label_to_texts[l].append(t) minority_count = min(len(v) for v in label_to_texts.values()) texts_balanced, labels_balanced = [], [] for label, txts in label_to_texts.items(): sample = random.sample(txts, minority_count) if len(txts) > minority_count else txts texts_balanced.extend(sample) labels_balanced.extend([label] * len(sample)) # Shuffle combined = list(zip(texts_balanced, labels_balanced)) random.seed(42) random.shuffle(combined) texts, labels = zip(*combined) texts, labels = list(texts), list(labels) print( f" Class distribution after undersampling: " f"Real={labels.count(0)}, Fake={labels.count(1)}" ) return texts, labels # ─────────────────────────────────────────────────────────── # Build Hybrid Feature Matrix # ─────────────────────────────────────────────────────────── def build_features(texts, tfidf=None, scaler=None, svd=None, fit=False): """Build hybrid feature matrix: TF-IDF (SVD-reduced) + MiniLM embeddings + stylometric. Args: texts: List of cleaned text strings. tfidf: TfidfVectorizer instance (created if None and fit=True). scaler: StandardScaler instance (created if None and fit=True). svd: TruncatedSVD instance (created if None and fit=True). fit: Whether to fit the transformers (True for training data). Returns: Tuple of (feature_matrix, tfidf, scaler, svd) """ # TF-IDF features (trigrams capture more fake-news-specific phrases) if fit: tfidf = TfidfVectorizer( max_features=10000, ngram_range=(1, 3), min_df=2, max_df=0.95, sublinear_tf=True, ) X_tfidf = tfidf.fit_transform(texts) svd = TruncatedSVD(n_components=300, random_state=42) X_tfidf_svd = svd.fit_transform(X_tfidf) else: X_tfidf = tfidf.transform(texts) X_tfidf_svd = svd.transform(X_tfidf) # MiniLM embeddings (384-dim semantic features) print(" Encoding texts with MiniLM...") minilm = get_minilm_model() embeddings = minilm.encode(texts, show_progress_bar=False, batch_size=64) # Stylometric features print( f" Extracting stylometric features ({len(STYLOMETRIC_FEATURE_NAMES)} features)..." ) stylo_data = np.array([extract_stylometric_features(t) for t in texts]) if fit: scaler = StandardScaler() stylo_scaled = scaler.fit_transform(stylo_data) else: stylo_scaled = scaler.transform(stylo_data) # Combine: TF-IDF/SVD (dense -> sparse) + MiniLM (dense -> sparse) + stylometric (dense -> sparse) X_combined = hstack( [ csr_matrix(X_tfidf_svd), csr_matrix(embeddings), csr_matrix(stylo_scaled), ] ) return X_combined, tfidf, scaler, svd # ─────────────────────────────────────────────────────────── # Training # ─────────────────────────────────────────────────────────── def train_model(X_texts, y_labels, max_depth=None, min_samples_leaf=3): """Train a Random Forest with hybrid features and cross-validation. Args: max_depth (int): Maximum tree depth. Use lower values (8-10) for small or homogeneous datasets (e.g. Cebuano) to prevent memorizing source-format artifacts instead of genuine fake-news signals. min_samples_leaf (int): Minimum samples at a leaf. Higher values (5+) add regularization and reduce overfitting on small datasets. """ label_names = ["Real", "Fake"] print(f" Hyperparameters: max_depth={max_depth}, min_samples_leaf={min_samples_leaf}") # ── Split data (80/10/10) ── print("\nSplitting data (80/10/10)...") X_train, X_temp, y_train, y_temp = train_test_split( X_texts, y_labels, test_size=0.2, random_state=42, stratify=y_labels ) X_val, X_test, y_val, y_test = train_test_split( X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp ) print(f" Train: {len(X_train)} | Val: {len(X_val)} | Test: {len(X_test)}") # ── Build Hybrid Features ── print("\nBuilding hybrid features (TF-IDF + MiniLM + stylometric)...") print(" Fitting on training data...") X_train_feat, tfidf, scaler, svd = build_features(X_train, fit=True) n_svd = svd.n_components n_minilm = 384 n_stylo = len(STYLOMETRIC_FEATURE_NAMES) print(f" TF-IDF vocabulary: {len(tfidf.vocabulary_)} → SVD components: {n_svd}") print( f" Total feature count: {X_train_feat.shape[1]} " f"(TF-IDF/SVD: {n_svd}" f" + MiniLM: {n_minilm}" f" + Stylometric: {n_stylo})" ) print(" Transforming validation & test...") X_val_feat, _, _, _ = build_features(X_val, tfidf=tfidf, scaler=scaler, svd=svd, fit=False) X_test_feat, _, _, _ = build_features(X_test, tfidf=tfidf, scaler=scaler, svd=svd, fit=False) # ── K-Fold Cross-Validation ── print("\nRunning 5-Fold Cross-Validation on training set...") rf_cv = RandomForestClassifier( n_estimators=500, max_depth=max_depth, max_features=0.15, min_samples_split=5, min_samples_leaf=min_samples_leaf, class_weight="balanced", n_jobs=-1, random_state=42, ) cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) cv_scores = cross_val_score(rf_cv, X_train_feat, y_train, cv=cv, scoring="accuracy") print(f" CV Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std():.4f})") print(f" Fold scores: {[f'{s:.4f}' for s in cv_scores]}") # ── Train Final Model ── print("\nTraining final Random Forest...") start_time = time.time() rf = RandomForestClassifier( n_estimators=500, max_depth=max_depth, max_features=0.15, min_samples_split=5, min_samples_leaf=min_samples_leaf, class_weight="balanced", n_jobs=-1, random_state=42, verbose=1, ) rf.fit(X_train_feat, y_train) train_time = time.time() - start_time print(f" Training completed in {train_time:.1f}s") # ── Evaluate on validation set ── print("\n" + "=" * 60) print(" VALIDATION SET RESULTS") print("=" * 60) y_val_pred = rf.predict(X_val_feat) val_acc = accuracy_score(y_val, y_val_pred) print(f" Accuracy: {val_acc:.4f}") print( classification_report( y_val, y_val_pred, labels=[0, 1], target_names=label_names, zero_division=0, ) ) # ── Evaluate on test set ── print("=" * 60) print(" TEST SET RESULTS") print("=" * 60) y_test_pred = rf.predict(X_test_feat) test_acc = accuracy_score(y_test, y_test_pred) print(f" Accuracy: {test_acc:.4f}") print( classification_report( y_test, y_test_pred, labels=[0, 1], target_names=label_names, zero_division=0, ) ) cm = confusion_matrix(y_test, y_test_pred) print(" Confusion Matrix:") print(f" Labels: {label_names}") print(cm) # ── Feature Importance (top stylometric) ── print("\n Stylometric Feature Importance:") stylo_start = X_train_feat.shape[1] - len(STYLOMETRIC_FEATURE_NAMES) importances = rf.feature_importances_[stylo_start:] for name, imp in sorted( zip(STYLOMETRIC_FEATURE_NAMES, importances), key=lambda x: -x[1] ): bar = "#" * int(imp * 500) print(f" {name:<25} {imp:.6f} {bar}") return ( rf, tfidf, scaler, svd, { "cv_mean": float(cv_scores.mean()), "cv_std": float(cv_scores.std()), "val_accuracy": float(val_acc), "test_accuracy": float(test_acc), }, ) # ─────────────────────────────────────────────────────────── # Save Artifacts # ─────────────────────────────────────────────────────────── def save_artifacts(model, vectorizer, scaler, svd, metrics, lang_suffix=""): """Save trained model, vectorizer, scaler, and SVD to disk. Args: lang_suffix: Empty string for the mixed/default model, or '_tagalog' / '_cebuano' when training a language-specific sub-model. """ os.makedirs(DATA_MODELS_DIR, exist_ok=True) if lang_suffix: model_path = os.path.join(DATA_MODELS_DIR, f"rf_fakenews{lang_suffix}.pkl") vectorizer_path = os.path.join( DATA_MODELS_DIR, f"tfidf_fakenews{lang_suffix}.pkl" ) scaler_path = os.path.join(DATA_MODELS_DIR, f"stylo_scaler{lang_suffix}.pkl") svd_path = os.path.join(DATA_MODELS_DIR, f"tfidf_svd{lang_suffix}.pkl") else: model_path = os.path.join(DATA_MODELS_DIR, "rf_fakenews_model.pkl") vectorizer_path = os.path.join(DATA_MODELS_DIR, "tfidf_fakenews.pkl") scaler_path = os.path.join(DATA_MODELS_DIR, "stylo_scaler.pkl") svd_path = os.path.join(DATA_MODELS_DIR, "tfidf_svd.pkl") joblib.dump(model, model_path) joblib.dump(vectorizer, vectorizer_path) joblib.dump(scaler, scaler_path) joblib.dump(svd, svd_path) print(f"\n Model saved to: {model_path}") print(f" Vectorizer saved to: {vectorizer_path}") print(f" Scaler saved to: {scaler_path}") print(f" SVD saved to: {svd_path}") print(f"\n CV Accuracy: {metrics['cv_mean']:.4f} (+/- {metrics['cv_std']:.4f})") print(f" Val Accuracy: {metrics['val_accuracy']:.4f}") print(f" Test Accuracy: {metrics['test_accuracy']:.4f}") # ─────────────────────────────────────────────────────────── # Main # ─────────────────────────────────────────────────────────── def main(): import argparse parser = argparse.ArgumentParser(description="Train the fake-news detection model.") parser.add_argument( "--tagalog-only", action="store_true", default=False, help="Filter training data to Tagalog/Filipino articles only " "(skips CebuaNER; loads BalitaNLP + MT-augmented fake news).", ) parser.add_argument( "--cebuano-only", action="store_true", default=False, help="Train on Cebuano data only: CebuaNER as Real + " "MT-translated Cebuano fake news from jcblaise.", ) args = parser.parse_args() if args.tagalog_only and args.cebuano_only: print("ERROR: --tagalog-only and --cebuano-only are mutually exclusive.") sys.exit(1) print("=" * 60) print(" FAKE NEWS DETECTOR — Enhanced Model Training") if args.tagalog_only: print(" MODE: Tagalog/Filipino articles only") elif args.cebuano_only: print(" MODE: Cebuano articles only") print("=" * 60) # Decide filename suffix for language-specific sub-models if args.tagalog_only: lang_suffix = "_tagalog" elif args.cebuano_only: lang_suffix = "_cebuano" else: lang_suffix = "" # mixed/default model # 1. Load data df = load_fake_news_dataset( tagalog_only=args.tagalog_only, cebuano_only=args.cebuano_only, ) # ── Guard: require both classes before proceeding ───────────────────────── n_classes = df["label"].nunique() if n_classes < 2: present = df["label"].unique().tolist() print("\n" + "=" * 60) print(" ⚠️ TRAINING ABORTED — Only 1 class found in dataset!") print("=" * 60) print(f" Classes present: {present} (need both 0=Real and 1=Fake)") print() print(" This usually means the local CSV datasets are missing.") print(" Required files for mixed mode:") print(" • data/raw/fakenews/fakenews/full.csv (Tagalog fake/real)") print(" • data/raw/philippine_corpus/Philippine Fake News Corpus.csv (English)") print(" • data/raw/augmented_ceb_fakes.csv (Cebuano fake)") print(" • data/raw/augmented_tl_fakes.csv (Tagalog fake)") print() print(" On HuggingFace Spaces: upload pre-trained model .pkl files") print(" to data_models/ instead of relying on auto-train.") print("=" * 60) sys.exit(0) # exit 0 so start.sh allows the server to still boot # ───────────────────────────────────────────────────────────────────────── # Language-specific modes previously undersampled real news at load time; # now we load more real news and oversample the minority (fake) class instead, # which gives the model more diverse real-news examples to learn from. X_texts, y_labels = preprocess(df, undersample=False, oversample=True) # 3. Train & evaluate # Cebuano-only: reduce model complexity to prevent memorizing source-format # artifacts (machine-translated fakes vs. native CebuaNER text). Lower # max_depth forces the model to use weaker, more-generalizable signals. if args.cebuano_only: model, vectorizer, scaler, svd, metrics = train_model( X_texts, y_labels, max_depth=8, min_samples_leaf=5, ) else: model, vectorizer, scaler, svd, metrics = train_model(X_texts, y_labels) # 4. Save print("\n" + "=" * 60) print(" SAVING ARTIFACTS") print("=" * 60) save_artifacts(model, vectorizer, scaler, svd, metrics, lang_suffix=lang_suffix) print("\n" + "=" * 60) print(" TRAINING COMPLETE!") print("=" * 60) if __name__ == "__main__": main()