Upload folder using huggingface_hub

.gitattributes

@@ -1,10 +1,46 @@
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.csv filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.db filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.jpg filter=lfs diff=lfs merge=lfs -text
-*.jpeg filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text
-*.PNG filter=lfs diff=lfs merge=lfs -text
-*.ico filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+check_app/android/.gradle/8.14/checksums/sha1-checksums.bin filter=lfs diff=lfs merge=lfs -text
+check_app/android/.gradle/8.14/executionHistory/executionHistory.bin filter=lfs diff=lfs merge=lfs -text
+check_app/android/.gradle/8.14/fileHashes/fileHashes.bin filter=lfs diff=lfs merge=lfs -text
+check_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png filter=lfs diff=lfs merge=lfs -text
+check_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png filter=lfs diff=lfs merge=lfs -text
+check_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png filter=lfs diff=lfs merge=lfs -text
+check_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png filter=lfs diff=lfs merge=lfs -text
+check_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png filter=lfs diff=lfs merge=lfs -text
+check_app/assets/images/app_logo_dark.jpg filter=lfs diff=lfs merge=lfs -text
+check_app/assets/images/app_logo_light.jpg filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-1024x1024@1x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@1x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@3x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@1x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@3x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@1x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@3x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@3x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@1x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-83.5x83.5@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@2x.png filter=lfs diff=lfs merge=lfs -text
+check_app/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@3x.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png filter=lfs diff=lfs merge=lfs -text
+check_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png filter=lfs diff=lfs merge=lfs -text
+check_app/web/favicon.png filter=lfs diff=lfs merge=lfs -text
+check_app/web/icons/Icon-192.png filter=lfs diff=lfs merge=lfs -text
+check_app/web/icons/Icon-512.png filter=lfs diff=lfs merge=lfs -text
+check_app/web/icons/Icon-maskable-192.png filter=lfs diff=lfs merge=lfs -text
+check_app/web/icons/Icon-maskable-512.png filter=lfs diff=lfs merge=lfs -text
+check_app/windows/runner/resources/app_icon.ico filter=lfs diff=lfs merge=lfs -text
+Icons/648286034_929277936354011_247561233351299912_n.jpg filter=lfs diff=lfs merge=lfs -text
+Icons/648528435_1489135242834045_123474419086154405_n.jpg filter=lfs diff=lfs merge=lfs -text
+Icons/IMG_7342.PNG filter=lfs diff=lfs merge=lfs -text
+Icons/IMG_7343.PNG filter=lfs diff=lfs merge=lfs -text

.github/workflows/scrape.yml

@@ -1,34 +1,34 @@
-name: Daily News Scraper
-
-on:
-  schedule:
-    # Runs every day at 00:00 UTC (8:00 AM PHT)
-    - cron: '0 0 * * *'
-  workflow_dispatch:  # Allows manual trigger from GitHub UI
-
-jobs:
-  scrape:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: pip install feedparser
-
-      - name: Run scraper
-        run: python scraper/news_scraper.py
-
-      - name: Commit updated database
-        run: |
-          git config user.name "GitHub Actions Bot"
-          git config user.email "actions@github.com"
-          git add data/news.db
-          git diff --staged --quiet || git commit -m "Auto-update: scraped news articles $(date -u +'%Y-%m-%d')"
-          git push
+name: Daily News Scraper
+
+on:
+  schedule:
+    # Runs every day at 00:00 UTC (8:00 AM PHT)
+    - cron: '0 0 * * *'
+  workflow_dispatch:  # Allows manual trigger from GitHub UI
+
+jobs:
+  scrape:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: pip install feedparser
+
+      - name: Run scraper
+        run: python scraper/news_scraper.py
+
+      - name: Commit updated database
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "actions@github.com"
+          git add data/news.db
+          git diff --staged --quiet || git commit -m "Auto-update: scraped news articles $(date -u +'%Y-%m-%d')"
+          git push

.gitignore

@@ -1,35 +1,46 @@
-# Virtual environment
-venv/
-.venv/
-
-# Python
-__pycache__/
-*.pyc
-*.pyo
-*.egg-info/
-dist/
-build/
-
-# IDE
-.vscode/
-.idea/
-
-# OS
-.DS_Store
-Thumbs.db
-
-# Environment
-.env
-
-# Large data files — too big for GitHub LFS free tier
-# Download datasets separately / run train.py to regenerate
-data/raw/
-data/embeddings.npz
-.pyre_configuration
-
-# Trained model binaries — regenerate with python backend/train.py
-data_models/*.pkl
-
-# Augmented translation caches — regenerate with train.py
-data/augmented_tl_fakes.csv
-data/augmented_ceb_fakes.csv
+# Virtual environment
+venv/
+.venv/
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Environment
+.env
+
+# Large data files — too big for GitHub LFS free tier
+# Download datasets separately / run train.py to regenerate
+data/raw/
+data/embeddings.npz
+.pyre_configuration
+
+# Trained model binaries — tracked via Git LFS (see .gitattributes)
+# Do NOT gitignore these: the HF Space needs them to start without retraining.
+# data_models/*.pkl
+
+# Augmented translation caches — regenerate with train.py
+data/augmented_tl_fakes.csv
+data/augmented_ceb_fakes.csv
+
+# Temp deployment/sync scripts & artifacts (not needed in repo)
+dummy.bin
+fix_hf_models.py
+hf_start.sh
+hf_tfidf.pkl
+restore_remaining_models.py
+sync_all_hf_models.py
+sync_source_to_hf.py
+commitchecker.txt

DEPLOYMENT.md

@@ -1,71 +1,71 @@
-# Deployment Guide
-
-## ⚠️ Required Step After Every Fresh Deploy / Clone
-
-The ML model files (`.pkl`) are **not stored in the repository** (binary files, too large for GitHub).  
-You must generate them **once on the server** after the first clone.
-
-### Step 1 — Install dependencies
-```bash
-pip install -r requirements.txt
-```
-
-### Step 2 — Train the main model
-```bash
-python backend/train.py
-```
-This will generate the following files in `data_models/`:
-- `rf_fakenews_model.pkl`
-- `tfidf_fakenews.pkl`
-- `stylo_scaler.pkl`
-
-⏱ Takes approximately **10–20 minutes** depending on server specs.  
-📦 Requires ~**2 GB RAM** during training.
-
-### Step 3 — (Optional) Train the Tagalog sub-model
-```bash
-python backend/train.py --tagalog-only
-```
-
-### Step 4 — Start the app normally
-```bash
-# Whatever your usual start command is, e.g.:
-python app.py
-# or
-uvicorn app:app --host 0.0.0.0 --port 8000
-```
-
----
-
-## After the First Deploy
-
-**You do NOT need to retrain on every redeploy.**  
-The `.pkl` files persist on the server. Future `git pull` + restart is enough.
-
-Only retrain if:
-- The server is wiped / re-imaged
-- `backend/train.py` was updated with new features
-- Model files were accidentally deleted
-
----
-
-## Checking if Models Exist
-
-```bash
-ls data_models/*.pkl
-```
-
-Expected output (6 files):
-```
-data_models/rf_fakenews_model.pkl
-data_models/rf_fakenews_tagalog.pkl
-data_models/rf_fakenews_cebuano.pkl
-data_models/tfidf_fakenews.pkl
-data_models/tfidf_fakenews_tagalog.pkl
-data_models/tfidf_fakenews_cebuano.pkl
-data_models/stylo_scaler.pkl
-data_models/stylo_scaler_tagalog.pkl
-data_models/stylo_scaler_cebuano.pkl
-```
-
-If any are missing → run `python backend/train.py`.
+# Deployment Guide
+
+## ⚠️ Required Step After Every Fresh Deploy / Clone
+
+The ML model files (`.pkl`) are **not stored in the repository** (binary files, too large for GitHub).  
+You must generate them **once on the server** after the first clone.
+
+### Step 1 — Install dependencies
+```bash
+pip install -r requirements.txt
+```
+
+### Step 2 — Train the main model
+```bash
+python backend/train.py
+```
+This will generate the following files in `data_models/`:
+- `rf_fakenews_model.pkl`
+- `tfidf_fakenews.pkl`
+- `stylo_scaler.pkl`
+
+⏱ Takes approximately **10–20 minutes** depending on server specs.  
+📦 Requires ~**2 GB RAM** during training.
+
+### Step 3 — (Optional) Train the Tagalog sub-model
+```bash
+python backend/train.py --tagalog-only
+```
+
+### Step 4 — Start the app normally
+```bash
+# Whatever your usual start command is, e.g.:
+python app.py
+# or
+uvicorn app:app --host 0.0.0.0 --port 8000
+```
+
+---
+
+## After the First Deploy
+
+**You do NOT need to retrain on every redeploy.**  
+The `.pkl` files persist on the server. Future `git pull` + restart is enough.
+
+Only retrain if:
+- The server is wiped / re-imaged
+- `backend/train.py` was updated with new features
+- Model files were accidentally deleted
+
+---
+
+## Checking if Models Exist
+
+```bash
+ls data_models/*.pkl
+```
+
+Expected output (6 files):
+```
+data_models/rf_fakenews_model.pkl
+data_models/rf_fakenews_tagalog.pkl
+data_models/rf_fakenews_cebuano.pkl
+data_models/tfidf_fakenews.pkl
+data_models/tfidf_fakenews_tagalog.pkl
+data_models/tfidf_fakenews_cebuano.pkl
+data_models/stylo_scaler.pkl
+data_models/stylo_scaler_tagalog.pkl
+data_models/stylo_scaler_cebuano.pkl
+```
+
+If any are missing → run `python backend/train.py`.

Dockerfile

@@ -1,5 +1,5 @@
 # Use the official Python base image
-FROM python:3.10
+FROM python:3.10-slim
 
 # Set the working directory for root installations
 WORKDIR /setup
@@ -7,20 +7,31 @@ WORKDIR /setup
 # Copy requirements first to leverage Docker cache
 COPY ./requirements.txt /setup/requirements.txt
 
-# Install python packages
-RUN pip install --no-cache-dir --upgrade pip
-RUN pip install --no-cache-dir -r /setup/requirements.txt
+# Install python packages (single RUN to reduce layers)
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r /setup/requirements.txt
+
+# Pre-download the MiniLM model during BUILD so it's baked into the image.
+# This saves ~2-3 minutes on every cold start.
+ENV HF_HOME=/setup/hf_cache
+RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')"
 
 # Hugging Face Spaces require running as a non-root user for security.
 # Set up a new user named "user" with user ID 1000
 RUN useradd -m -u 1000 user
 
+# Move the cached model to the user's home so it's accessible after user switch
+RUN mkdir -p /home/user/.cache && \
+    cp -r /setup/hf_cache /home/user/.cache/huggingface && \
+    chown -R user:user /home/user/.cache
+
 # Switch to the "user" user
 USER user
 
 # Set home to the user's home directory
 ENV HOME=/home/user \
-    PATH=/home/user/.local/bin:$PATH
+    PATH=/home/user/.local/bin:$PATH \
+    HF_HOME=/home/user/.cache/huggingface
 
 # Set the working directory to the user's home directory
 WORKDIR $HOME/app
@@ -28,10 +39,12 @@ WORKDIR $HOME/app
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
 COPY --chown=user . $HOME/app
 
+# Make the startup script executable
+RUN chmod +x start.sh
+
 # HuggingFace Spaces requires port 7860
 ENV PORT=7860
 EXPOSE 7860
 
-# Command to run the Uvicorn server
-CMD uvicorn api.main:app --host 0.0.0.0 --port $PORT
-
+# Startup: auto-train models if missing, then start Uvicorn
+CMD ["bash", "start.sh"]

add_cebuano_data.py

@@ -0,0 +1,329 @@
+"""
+Cebuano Fake News Data Entry Tool
+===================================
+Manually input Cebuano fake news articles into
+data/raw/augmented_ceb_fakes.csv for model training.
+
+Usage:
+    python add_cebuano_data.py
+
+Features:
+    - Interactive CLI with menu options
+    - Paste single-line or multi-line articles
+    - Preview entries before saving
+    - View existing dataset stats
+    - Duplicate detection (warns if near-identical text exists)
+"""
+
+import csv
+import os
+import sys
+import textwrap
+from datetime import datetime
+
+# ── Paths ──
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+CSV_PATH = os.path.join(SCRIPT_DIR, "data", "raw", "augmented_ceb_fakes.csv")
+BACKUP_DIR = os.path.join(SCRIPT_DIR, "data", "raw", ".backups")
+
+
+def _count_existing_rows() -> int:
+    """Count how many articles are already in the CSV."""
+    if not os.path.exists(CSV_PATH):
+        return 0
+    with open(CSV_PATH, "r", encoding="utf-8") as f:
+        reader = csv.reader(f)
+        header = next(reader, None)
+        return sum(1 for _ in reader)
+
+
+def _load_existing_articles() -> list[str]:
+    """Load all existing article texts (for duplicate checking)."""
+    articles = []
+    if not os.path.exists(CSV_PATH):
+        return articles
+    with open(CSV_PATH, "r", encoding="utf-8") as f:
+        reader = csv.reader(f)
+        next(reader, None)  # skip header
+        for row in reader:
+            if row:
+                articles.append(row[0].strip().lower())
+    return articles
+
+
+def _is_duplicate(new_text: str, existing: list[str], threshold: int = 50) -> bool:
+    """Check if the first `threshold` characters match any existing article."""
+    snippet = new_text.strip().lower()[:threshold]
+    if len(snippet) < 20:
+        return False
+    for art in existing:
+        if art[:threshold] == snippet:
+            return True
+    return False
+
+
+def _backup_csv():
+    """Create a timestamped backup of the current CSV."""
+    if not os.path.exists(CSV_PATH):
+        return
+    os.makedirs(BACKUP_DIR, exist_ok=True)
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    backup_path = os.path.join(BACKUP_DIR, f"augmented_ceb_fakes_{ts}.csv")
+    with open(CSV_PATH, "r", encoding="utf-8") as src:
+        with open(backup_path, "w", encoding="utf-8", newline="") as dst:
+            dst.write(src.read())
+    print(f"  Backup saved: {os.path.relpath(backup_path, SCRIPT_DIR)}")
+
+
+def _ensure_csv_exists():
+    """Create the CSV with header if it doesn't exist yet."""
+    if not os.path.exists(CSV_PATH):
+        os.makedirs(os.path.dirname(CSV_PATH), exist_ok=True)
+        with open(CSV_PATH, "w", encoding="utf-8", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(["article"])
+        print(f"  Created new CSV: {CSV_PATH}")
+
+
+def _append_articles(articles: list[str]):
+    """Append a list of article texts to the CSV."""
+    with open(CSV_PATH, "a", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f)
+        for text in articles:
+            writer.writerow([text])
+
+
+def _preview_text(text: str, width: int = 80) -> str:
+    """Return a truncated preview of article text."""
+    preview = text.replace("\n", " ").strip()
+    if len(preview) > width:
+        return preview[:width] + "..."
+    return preview
+
+
+# ── CLI Display ──
+
+def print_header():
+    print()
+    print("=" * 60)
+    print("   CEBUANO FAKE NEWS — DATA ENTRY TOOL")
+    print("=" * 60)
+    print(f"  CSV: {os.path.relpath(CSV_PATH, SCRIPT_DIR)}")
+    print(f"  Existing articles: {_count_existing_rows():,}")
+    print("=" * 60)
+
+
+def print_menu():
+    print()
+    print("  [1] Add a single article")
+    print("  [2] Add multiple articles (batch mode)")
+    print("  [3] View dataset stats")
+    print("  [4] View last 5 entries")
+    print("  [5] Exit")
+    print()
+
+
+def input_single_article() -> str | None:
+    """Prompt the user to type/paste a single article.
+    
+    Supports multi-line input: type on multiple lines, then enter
+    a blank line to finish.
+    """
+    print()
+    print("  Paste or type the Cebuano fake news article below.")
+    print("  (Enter a blank line when done, or type 'cancel' to abort)")
+    print("  " + "-" * 50)
+
+    lines = []
+    while True:
+        try:
+            line = input("  > ")
+        except EOFError:
+            break
+        if line.strip().lower() == "cancel":
+            return None
+        if line.strip() == "" and lines:
+            break
+        lines.append(line)
+
+    text = " ".join(lines).strip()
+    if not text:
+        print("  (empty input — skipped)")
+        return None
+    return text
+
+
+def add_single_article():
+    """Menu option 1: add one article."""
+    _ensure_csv_exists()
+    existing = _load_existing_articles()
+    
+    text = input_single_article()
+    if text is None:
+        print("  Cancelled.")
+        return
+
+    # Duplicate check
+    if _is_duplicate(text, existing):
+        print()
+        print("  ⚠  WARNING: This article appears to be a duplicate!")
+        confirm = input("  Add anyway? (y/n): ").strip().lower()
+        if confirm != "y":
+            print("  Skipped.")
+            return
+
+    # Preview and confirm
+    print()
+    print("  Preview:")
+    print(f"    {_preview_text(text, 100)}")
+    print(f"    Length: {len(text):,} chars, {len(text.split()):,} words")
+    print()
+    confirm = input("  Save this article? (y/n): ").strip().lower()
+    if confirm != "y":
+        print("  Discarded.")
+        return
+
+    _append_articles([text])
+    print(f"  ✓ Article saved! Total articles now: {_count_existing_rows():,}")
+
+
+def add_batch_articles():
+    """Menu option 2: add multiple articles in batch."""
+    _ensure_csv_exists()
+    existing = _load_existing_articles()
+    
+    print()
+    print("  BATCH MODE — Enter one article per prompt.")
+    print("  Type 'done' when finished, 'cancel' to discard all.")
+    print()
+
+    pending: list[str] = []
+    article_num = 1
+
+    while True:
+        print(f"  --- Article #{article_num} ---")
+        text = input_single_article()
+        
+        if text is None:
+            # Check if user typed cancel
+            break
+            
+        if _is_duplicate(text, existing):
+            print("  ⚠  Possible duplicate — skipping this one.")
+            continue
+
+        pending.append(text)
+        print(f"  ✓ Queued ({len(pending)} pending)")
+        article_num += 1
+
+        cont = input("  Add another? (y/n/done): ").strip().lower()
+        if cont in ("n", "done"):
+            break
+
+    if not pending:
+        print("  No articles to save.")
+        return
+
+    # Preview all
+    print()
+    print(f"  === {len(pending)} articles ready to save ===")
+    for i, art in enumerate(pending, 1):
+        print(f"    {i}. {_preview_text(art, 70)}")
+    print()
+
+    confirm = input(f"  Save all {len(pending)} articles? (y/n): ").strip().lower()
+    if confirm != "y":
+        print("  Discarded all.")
+        return
+
+    _backup_csv()
+    _append_articles(pending)
+    print(f"  ✓ {len(pending)} articles saved! Total now: {_count_existing_rows():,}")
+
+
+def show_stats():
+    """Menu option 3: show dataset statistics."""
+    if not os.path.exists(CSV_PATH):
+        print("  CSV not found — no data yet.")
+        return
+
+    articles = _load_existing_articles()
+    total = len(articles)
+    if total == 0:
+        print("  Dataset is empty.")
+        return
+
+    word_counts = [len(a.split()) for a in articles]
+    char_counts = [len(a) for a in articles]
+
+    print()
+    print("  ╔══════════════════════════════════════╗")
+    print("  ║       DATASET STATISTICS             ║")
+    print("  ╠══════════════════════════════════════╣")
+    print(f"  ║  Total articles:    {total:>10,}      ║")
+    print(f"  ║  Avg words/article: {sum(word_counts)//total:>10,}      ║")
+    print(f"  ║  Min words:         {min(word_counts):>10,}      ║")
+    print(f"  ║  Max words:         {max(word_counts):>10,}      ║")
+    print(f"  ║  Avg chars/article: {sum(char_counts)//total:>10,}      ║")
+    print(f"  ║  CSV size:          {os.path.getsize(CSV_PATH):>10,} B   ║")
+    print("  ╚══════════════════════════════════════╝")
+    print()
+
+
+def show_last_entries():
+    """Menu option 4: show last 5 entries."""
+    if not os.path.exists(CSV_PATH):
+        print("  CSV not found — no data yet.")
+        return
+
+    articles = []
+    with open(CSV_PATH, "r", encoding="utf-8") as f:
+        reader = csv.reader(f)
+        next(reader, None)
+        for row in reader:
+            if row:
+                articles.append(row[0])
+
+    if not articles:
+        print("  No articles in dataset.")
+        return
+
+    last5 = articles[-5:]
+    print()
+    print(f"  Last {len(last5)} entries (of {len(articles)} total):")
+    print("  " + "-" * 50)
+    for i, art in enumerate(last5, len(articles) - len(last5) + 1):
+        print(f"    #{i}: {_preview_text(art, 70)}")
+    print()
+
+
+# ── Main Loop ──
+
+def main():
+    print_header()
+
+    while True:
+        print_menu()
+        try:
+            choice = input("  Choose an option [1-5]: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\n  Goodbye!")
+            break
+
+        if choice == "1":
+            add_single_article()
+        elif choice == "2":
+            add_batch_articles()
+        elif choice == "3":
+            show_stats()
+        elif choice == "4":
+            show_last_entries()
+        elif choice == "5":
+            print("  Goodbye!")
+            break
+        else:
+            print("  Invalid choice, try again.")
+
+
+if __name__ == "__main__":
+    main()

api/main.py

@@ -433,6 +433,36 @@ async def delete_account(
 
 
 # ── Fact-check endpoint ──────────────────────────────────────────────
+class AnalyzeStructureRequest(BaseModel):
+    text: str
+
+
+@app.post("/api/analyze-structure")
+async def api_analyze_structure(payload: AnalyzeStructureRequest) -> Dict[str, Any]:
+    """
+    Exposes the structural (inverted pyramid) analysis algorithm 
+    to the frontend Live Writing Guide without running full ML/URL checks.
+    """
+    if not payload.text or len(payload.text.strip()) < 30:
+        return {
+            "language": "unknown",
+            "formalism_score": 0,
+            "assessment": "Text too short for structure analysis.",
+            "recommendations": [],
+        }
+
+    from checker.internal.structure_analyzer import analyze_structure
+    try:
+        res = analyze_structure(payload.text)
+        return res
+    except Exception as exc:
+        logger.error("Error in analyze_structure: %s", exc)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Structure analysis failed: {exc}"
+        )
+
+
 @app.post("/api/check")
 async def check_article(
     payload: CheckRequest,

backend/analyze_languages.py

@@ -1,275 +1,275 @@
-"""
-backend/analyze_languages.py
-============================
-Analyzes the language distribution of every training dataset and prints a
-summary table broken down by dataset and language.
-
-Uses langdetect (already pulled in transitively; install with `pip install langdetect`
-if missing).
-
-Usage:
-    python backend/analyze_languages.py
-"""
-
-import sys
-import os
-import random
-from collections import Counter
-
-import pandas as pd
-
-PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, PROJECT_ROOT)
-
-# ── Try importing langdetect ──────────────────────────────────────────────────
-try:
-    from langdetect import detect, LangDetectException
-    from langdetect import DetectorFactory
-
-    DetectorFactory.seed = 42  # make detection deterministic
-    HAS_LANGDETECT = True
-except ImportError:
-    HAS_LANGDETECT = False
-    print("WARNING: langdetect not installed. Run: pip install langdetect")
-    print("         Falling back to heuristic detection only.\n")
-
-
-# ── Simple Tagalog heuristic (fast, used as a sanity check) ──────────────────
-_TAGALOG_MARKERS = {
-    "ang",
-    "ng",
-    "mga",
-    "sa",
-    "na",
-    "ay",
-    "at",
-    "hindi",
-    "ako",
-    "siya",
-    "nila",
-    "niya",
-    "ito",
-    "iyon",
-    "kung",
-    "para",
-    "nang",
-    "din",
-    "rin",
-    "kaya",
-    "pero",
-    "dahil",
-    "ayon",
-    "noon",
-    "ngayon",
-    "dito",
-    "doon",
-    "sinabi",
-    "sinasabi",
-    "nagpapatunay",
-    "araw",
-    "taon",
-    "buwan",
-}
-
-_CEBUANO_MARKERS = {
-    "ug",
-    "nga",
-    "ang",
-    "sa",
-    "si",
-    "nag",
-    "mao",
-    "kang",
-    "usab",
-    "man",
-    "dayon",
-    "gyud",
-    "kaayo",
-    "lang",
-    "pud",
-    "adto",
-    "kini",
-    "sila",
-    "niadtong",
-    "gitawag",
-    "giingon",
-    "matud",
-    "nasayran",
-    "gidakop",
-}
-
-
-def _heuristic_lang(text: str) -> str:
-    """Very rough heuristic: count Tagalog vs Cebuano marker hits."""
-    words = set(text.lower().split())
-    tl_hits = len(words & _TAGALOG_MARKERS)
-    ceb_hits = len(words & _CEBUANO_MARKERS)
-    if tl_hits == 0 and ceb_hits == 0:
-        return "unknown"
-    return "tl" if tl_hits >= ceb_hits else "ceb"
-
-
-def detect_lang(text: str) -> str:
-    """Detect language; falls back to heuristic if langdetect fails."""
-    if not text or not isinstance(text, str) or len(text.split()) < 5:
-        return "unknown"
-    if HAS_LANGDETECT:
-        try:
-            return detect(text[:500])  # only need a snippet
-        except LangDetectException:
-            pass
-    return _heuristic_lang(text)
-
-
-# ── Dataset loaders (mirrors train.py logic) ─────────────────────────────────
-
-
-def load_datasets_raw() -> list[tuple[str, pd.DataFrame]]:
-    """Return list of (name, df) pairs, df has columns: article, label."""
-    result = []
-
-    # 1. jcblaise/fake_news_filipino
-    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
-    if os.path.exists(csv1):
-        # The CSV has a `<<<<<<< HEAD` git conflict marker on line 1;
-        # skiprows=1 makes pandas treat the real header (line 2) as the header.
-        df1 = pd.read_csv(csv1, skiprows=1)
-        # Keep only rows where both columns are valid
-        if "article" in df1.columns and "label" in df1.columns:
-            df1 = df1[["article", "label"]].dropna()
-            # Drop any remaining git conflict marker rows
-            df1 = df1[
-                ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
-            ]
-        result.append(("jcblaise/fake_news_filipino", df1))
-        print(f"  [1] Loaded jcblaise: {len(df1)} articles")
-    else:
-        print(f"  [1] jcblaise not found at {csv1}, skipping.")
-
-    # 2. Philippine Fake News Corpus
-    csv2 = os.path.join(
-        PROJECT_ROOT,
-        "data",
-        "raw",
-        "philippine_corpus",
-        "Philippine Fake News Corpus.csv",
-    )
-    if os.path.exists(csv2):
-        # Same git conflict marker fix — skip line 1
-        df2 = pd.read_csv(csv2, skiprows=1)
-        df2 = df2.rename(columns={"Content": "article"})
-        df2["label"] = df2["Label"].map({"Credible": 0, "Not Credible": 1})
-        df2 = df2[["article", "label"]].dropna()
-        df2 = df2[~df2["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
-        result.append(("Philippine Fake News Corpus", df2))
-        print(f"  [2] Loaded Philippine Corpus: {len(df2)} articles")
-    else:
-        print(f"  [2] Philippine Corpus not found at {csv2}, skipping.")
-
-    # 3. CebuaNER — definitively Cebuano, no need to run detection on every row
-    try:
-        from datasets import load_dataset
-
-        print("  [3] Downloading CebuaNER...")
-        ds = load_dataset("josephimperial/CebuaNER")
-        sentences = []
-        for split_data in ds.values():
-            for row in split_data:
-                # CebuaNER schema: {'text': str}  — one sentence per row
-                text = row.get("text") or " ".join(
-                    row.get("tokens") or row.get("words") or []
-                )
-                if text and text.strip():
-                    sentences.append(text.strip())
-
-        MIN_CHUNK = 30
-        articles, buf, buf_tok = [], [], 0
-        for s in sentences:
-            buf.append(s)
-            buf_tok += len(s.split())
-            if buf_tok >= MIN_CHUNK:
-                articles.append(" ".join(buf))
-                buf, buf_tok = [], 0
-        if buf:
-            articles.append(" ".join(buf))
-
-        df3 = pd.DataFrame({"article": articles, "label": 0})
-        result.append(("josephimperial/CebuaNER", df3))
-        print(f"  [3] Loaded CebuaNER: {len(df3)} chunks")
-    except ImportError:
-        print("  [3] 'datasets' not installed, skipping CebuaNER.")
-    except Exception as exc:
-        print(f"  [3] CebuaNER error: {exc}")
-
-    return result
-
-
-# ── Main analysis ─────────────────────────────────────────────────────────────
-
-
-def analyze(sample_size: int = 500):
-    print("=" * 60)
-    print("  LANGUAGE DISTRIBUTION ANALYSIS")
-    print("=" * 60)
-    print("\nLoading datasets...\n")
-
-    datasets = load_datasets_raw()
-    if not datasets:
-        print("No datasets found.")
-        return
-
-    grand_total = 0
-    grand_tl = 0
-
-    for name, df in datasets:
-        total = len(df)
-        grand_total += total
-
-        # CebuaNER is definitively Cebuano — skip expensive detection
-        if "CebuaNER" in name:
-            lang_counts = Counter({"ceb": total})
-            print(f"\n  [{name}]")
-            print(f"    Total  : {total:,}")
-            print(
-                f"    ceb    : {total:,} (100.0%)  [source is Cebuano news by definition]"
-            )
-            continue
-
-        # Sample for speed on large datasets
-        if total > sample_size:
-            df_sample = df.sample(n=sample_size, random_state=42)
-            sampled = True
-        else:
-            df_sample = df
-            sampled = False
-
-        lang_counts: Counter = Counter()
-        for text in df_sample["article"]:
-            lang_counts[detect_lang(str(text))] += 1
-
-        # Scale up sample counts to full dataset size
-        if sampled:
-            scale = total / sample_size
-            lang_counts = Counter({k: int(v * scale) for k, v in lang_counts.items()})
-
-        tl_count = lang_counts.get("tl", 0) + lang_counts.get("fil", 0)
-        grand_tl += tl_count
-
-        print(f"\n  [{name}]")
-        print(
-            f"    Total  : {total:,}" + (" (estimate from sample)" if sampled else "")
-        )
-        for lang, cnt in lang_counts.most_common():
-            pct = cnt / total * 100
-            print(f"    {lang:<8}: {cnt:>6,}  ({pct:.1f}%)")
-
-    print("\n" + "=" * 60)
-    print(f"  GRAND TOTAL articles : {grand_total:,}")
-    print(f"  Estimated Tagalog    : {grand_tl:,}  ({grand_tl/grand_total*100:.1f}%)")
-    print("=" * 60)
-    print("\nNote: 'tl'=Tagalog/Filipino, 'ceb'=Cebuano, 'en'=English")
-    print("      langdetect may mis-classify short or code-switched texts.")
-
-
-if __name__ == "__main__":
-    analyze()
+"""
+backend/analyze_languages.py
+============================
+Analyzes the language distribution of every training dataset and prints a
+summary table broken down by dataset and language.
+
+Uses langdetect (already pulled in transitively; install with `pip install langdetect`
+if missing).
+
+Usage:
+    python backend/analyze_languages.py
+"""
+
+import sys
+import os
+import random
+from collections import Counter
+
+import pandas as pd
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_ROOT)
+
+# ── Try importing langdetect ──────────────────────────────────────────────────
+try:
+    from langdetect import detect, LangDetectException
+    from langdetect import DetectorFactory
+
+    DetectorFactory.seed = 42  # make detection deterministic
+    HAS_LANGDETECT = True
+except ImportError:
+    HAS_LANGDETECT = False
+    print("WARNING: langdetect not installed. Run: pip install langdetect")
+    print("         Falling back to heuristic detection only.\n")
+
+
+# ── Simple Tagalog heuristic (fast, used as a sanity check) ──────────────────
+_TAGALOG_MARKERS = {
+    "ang",
+    "ng",
+    "mga",
+    "sa",
+    "na",
+    "ay",
+    "at",
+    "hindi",
+    "ako",
+    "siya",
+    "nila",
+    "niya",
+    "ito",
+    "iyon",
+    "kung",
+    "para",
+    "nang",
+    "din",
+    "rin",
+    "kaya",
+    "pero",
+    "dahil",
+    "ayon",
+    "noon",
+    "ngayon",
+    "dito",
+    "doon",
+    "sinabi",
+    "sinasabi",
+    "nagpapatunay",
+    "araw",
+    "taon",
+    "buwan",
+}
+
+_CEBUANO_MARKERS = {
+    "ug",
+    "nga",
+    "ang",
+    "sa",
+    "si",
+    "nag",
+    "mao",
+    "kang",
+    "usab",
+    "man",
+    "dayon",
+    "gyud",
+    "kaayo",
+    "lang",
+    "pud",
+    "adto",
+    "kini",
+    "sila",
+    "niadtong",
+    "gitawag",
+    "giingon",
+    "matud",
+    "nasayran",
+    "gidakop",
+}
+
+
+def _heuristic_lang(text: str) -> str:
+    """Very rough heuristic: count Tagalog vs Cebuano marker hits."""
+    words = set(text.lower().split())
+    tl_hits = len(words & _TAGALOG_MARKERS)
+    ceb_hits = len(words & _CEBUANO_MARKERS)
+    if tl_hits == 0 and ceb_hits == 0:
+        return "unknown"
+    return "tl" if tl_hits >= ceb_hits else "ceb"
+
+
+def detect_lang(text: str) -> str:
+    """Detect language; falls back to heuristic if langdetect fails."""
+    if not text or not isinstance(text, str) or len(text.split()) < 5:
+        return "unknown"
+    if HAS_LANGDETECT:
+        try:
+            return detect(text[:500])  # only need a snippet
+        except LangDetectException:
+            pass
+    return _heuristic_lang(text)
+
+
+# ── Dataset loaders (mirrors train.py logic) ─────────────────────────────────
+
+
+def load_datasets_raw() -> list[tuple[str, pd.DataFrame]]:
+    """Return list of (name, df) pairs, df has columns: article, label."""
+    result = []
+
+    # 1. jcblaise/fake_news_filipino
+    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
+    if os.path.exists(csv1):
+        # The CSV has a `<<<<<<< HEAD` git conflict marker on line 1;
+        # skiprows=1 makes pandas treat the real header (line 2) as the header.
+        df1 = pd.read_csv(csv1, skiprows=1)
+        # Keep only rows where both columns are valid
+        if "article" in df1.columns and "label" in df1.columns:
+            df1 = df1[["article", "label"]].dropna()
+            # Drop any remaining git conflict marker rows
+            df1 = df1[
+                ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
+            ]
+        result.append(("jcblaise/fake_news_filipino", df1))
+        print(f"  [1] Loaded jcblaise: {len(df1)} articles")
+    else:
+        print(f"  [1] jcblaise not found at {csv1}, skipping.")
+
+    # 2. Philippine Fake News Corpus
+    csv2 = os.path.join(
+        PROJECT_ROOT,
+        "data",
+        "raw",
+        "philippine_corpus",
+        "Philippine Fake News Corpus.csv",
+    )
+    if os.path.exists(csv2):
+        # Same git conflict marker fix — skip line 1
+        df2 = pd.read_csv(csv2, skiprows=1)
+        df2 = df2.rename(columns={"Content": "article"})
+        df2["label"] = df2["Label"].map({"Credible": 0, "Not Credible": 1})
+        df2 = df2[["article", "label"]].dropna()
+        df2 = df2[~df2["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
+        result.append(("Philippine Fake News Corpus", df2))
+        print(f"  [2] Loaded Philippine Corpus: {len(df2)} articles")
+    else:
+        print(f"  [2] Philippine Corpus not found at {csv2}, skipping.")
+
+    # 3. CebuaNER — definitively Cebuano, no need to run detection on every row
+    try:
+        from datasets import load_dataset
+
+        print("  [3] Downloading CebuaNER...")
+        ds = load_dataset("josephimperial/CebuaNER")
+        sentences = []
+        for split_data in ds.values():
+            for row in split_data:
+                # CebuaNER schema: {'text': str}  — one sentence per row
+                text = row.get("text") or " ".join(
+                    row.get("tokens") or row.get("words") or []
+                )
+                if text and text.strip():
+                    sentences.append(text.strip())
+
+        MIN_CHUNK = 30
+        articles, buf, buf_tok = [], [], 0
+        for s in sentences:
+            buf.append(s)
+            buf_tok += len(s.split())
+            if buf_tok >= MIN_CHUNK:
+                articles.append(" ".join(buf))
+                buf, buf_tok = [], 0
+        if buf:
+            articles.append(" ".join(buf))
+
+        df3 = pd.DataFrame({"article": articles, "label": 0})
+        result.append(("josephimperial/CebuaNER", df3))
+        print(f"  [3] Loaded CebuaNER: {len(df3)} chunks")
+    except ImportError:
+        print("  [3] 'datasets' not installed, skipping CebuaNER.")
+    except Exception as exc:
+        print(f"  [3] CebuaNER error: {exc}")
+
+    return result
+
+
+# ── Main analysis ─────────────────────────────────────────────────────────────
+
+
+def analyze(sample_size: int = 500):
+    print("=" * 60)
+    print("  LANGUAGE DISTRIBUTION ANALYSIS")
+    print("=" * 60)
+    print("\nLoading datasets...\n")
+
+    datasets = load_datasets_raw()
+    if not datasets:
+        print("No datasets found.")
+        return
+
+    grand_total = 0
+    grand_tl = 0
+
+    for name, df in datasets:
+        total = len(df)
+        grand_total += total
+
+        # CebuaNER is definitively Cebuano — skip expensive detection
+        if "CebuaNER" in name:
+            lang_counts = Counter({"ceb": total})
+            print(f"\n  [{name}]")
+            print(f"    Total  : {total:,}")
+            print(
+                f"    ceb    : {total:,} (100.0%)  [source is Cebuano news by definition]"
+            )
+            continue
+
+        # Sample for speed on large datasets
+        if total > sample_size:
+            df_sample = df.sample(n=sample_size, random_state=42)
+            sampled = True
+        else:
+            df_sample = df
+            sampled = False
+
+        lang_counts: Counter = Counter()
+        for text in df_sample["article"]:
+            lang_counts[detect_lang(str(text))] += 1
+
+        # Scale up sample counts to full dataset size
+        if sampled:
+            scale = total / sample_size
+            lang_counts = Counter({k: int(v * scale) for k, v in lang_counts.items()})
+
+        tl_count = lang_counts.get("tl", 0) + lang_counts.get("fil", 0)
+        grand_tl += tl_count
+
+        print(f"\n  [{name}]")
+        print(
+            f"    Total  : {total:,}" + (" (estimate from sample)" if sampled else "")
+        )
+        for lang, cnt in lang_counts.most_common():
+            pct = cnt / total * 100
+            print(f"    {lang:<8}: {cnt:>6,}  ({pct:.1f}%)")
+
+    print("\n" + "=" * 60)
+    print(f"  GRAND TOTAL articles : {grand_total:,}")
+    print(f"  Estimated Tagalog    : {grand_tl:,}  ({grand_tl/grand_total*100:.1f}%)")
+    print("=" * 60)
+    print("\nNote: 'tl'=Tagalog/Filipino, 'ceb'=Cebuano, 'en'=English")
+    print("      langdetect may mis-classify short or code-switched texts.")
+
+
+if __name__ == "__main__":
+    analyze()

backend/benchmark.py

@@ -0,0 +1,538 @@
+"""
+Benchmark — Comparing Your RF Model Against Classic ML Baselines
+================================================================
+Trains and evaluates multiple fake-news classifiers on the SAME
+locked test split so the comparison is completely fair.
+
+Models compared
+---------------
+  1. Naive Bayes        (TF-IDF only)            — simple baseline
+  2. Logistic Regression (TF-IDF only)           — strong linear baseline
+  3. Linear SVM          (TF-IDF only)           — often best for text
+  4. Random Forest       (TF-IDF only)           — RF without embeddings
+  5. ★ YOUR MODEL ★   (TF-IDF + MiniLM + Stylo) — YOUR hybrid RF
+
+Metrics reported (per model, per class + weighted avg)
+-------------------------------------------------------
+  • Accuracy
+  • Precision  (Fake class)
+  • Recall     (Fake class)
+  • F1 Score   (weighted)
+  • AUC-ROC
+
+Output
+------
+  evaluation_results/benchmark_table.txt   — plaintext comparison table
+  evaluation_results/benchmark_chart.png   — bar chart (Accuracy + F1)
+  evaluation_results/benchmark_roc.png     — ROC curves for all models
+
+Usage
+-----
+  python backend/benchmark.py
+  python backend/benchmark.py --mode tagalog    # Tagalog dataset only
+  python backend/benchmark.py --mode cebuano   # Cebuano dataset only
+  python backend/benchmark.py --mode mixed      # All languages (default)
+"""
+
+import sys
+import os
+import re
+import time
+import argparse
+import warnings
+
+warnings.filterwarnings("ignore")
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_ROOT)
+
+import numpy as np
+import pandas as pd
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+
+from scipy.sparse import hstack, csr_matrix
+
+from sklearn.model_selection import train_test_split
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.preprocessing import StandardScaler
+from sklearn.pipeline import Pipeline
+
+# Classifiers
+from sklearn.naive_bayes import MultinomialNB
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import LinearSVC
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.calibration import CalibratedClassifierCV
+
+# Metrics
+from sklearn.metrics import (
+    accuracy_score,
+    classification_report,
+    confusion_matrix,
+    roc_auc_score,
+    roc_curve,
+    f1_score,
+    precision_score,
+    recall_score,
+)
+
+# Re-use helpers from train.py (keep feature extraction identical)
+from backend.train import (
+    load_fake_news_dataset,
+    preprocess,
+    clean_text,
+    extract_stylometric_features,
+    get_minilm_model,
+    STYLOMETRIC_FEATURE_NAMES,
+)
+
+# ── Paths ──────────────────────────────────────────────────────────────────
+OUTPUT_DIR = os.path.join(PROJECT_ROOT, "evaluation_results")
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# Feature builders
+# ───────────────────────────────────────────────────────────────────────────
+
+
+def build_tfidf_features(X_train, X_test):
+    """Plain TF-IDF (used by all baseline models)."""
+    tfidf = TfidfVectorizer(
+        max_features=15_000,
+        ngram_range=(1, 3),
+        min_df=2,
+        max_df=0.95,
+        sublinear_tf=True,
+    )
+    X_tr = tfidf.fit_transform(X_train)
+    X_te = tfidf.transform(X_test)
+    return X_tr, X_te, tfidf
+
+
+def build_hybrid_features(X_train, X_test):
+    """TF-IDF + MiniLM embeddings + stylometric (your full pipeline)."""
+    print("  [hybrid] Fitting TF-IDF …")
+    tfidf = TfidfVectorizer(
+        max_features=15_000,
+        ngram_range=(1, 3),
+        min_df=2,
+        max_df=0.95,
+        sublinear_tf=True,
+    )
+    X_tr_tfidf = tfidf.fit_transform(X_train)
+    X_te_tfidf = tfidf.transform(X_test)
+
+    print("  [hybrid] Encoding with MiniLM …")
+    minilm = get_minilm_model()
+    emb_train = minilm.encode(X_train, show_progress_bar=True, batch_size=64)
+    emb_test  = minilm.encode(X_test,  show_progress_bar=True, batch_size=64)
+
+    print(f"  [hybrid] Extracting {len(STYLOMETRIC_FEATURE_NAMES)} stylometric features …")
+    stylo_train = np.array([extract_stylometric_features(t) for t in X_train])
+    stylo_test  = np.array([extract_stylometric_features(t) for t in X_test])
+
+    scaler = StandardScaler()
+    stylo_train_sc = scaler.fit_transform(stylo_train)
+    stylo_test_sc  = scaler.transform(stylo_test)
+
+    X_tr = hstack([X_tr_tfidf, csr_matrix(emb_train), csr_matrix(stylo_train_sc)])
+    X_te = hstack([X_te_tfidf, csr_matrix(emb_test),  csr_matrix(stylo_test_sc)])
+
+    n_total = X_tr.shape[1]
+    print(
+        f"  [hybrid] Feature dimensions: {n_total} "
+        f"(TF-IDF: {X_tr_tfidf.shape[1]} + MiniLM: 384 + Stylo: {len(STYLOMETRIC_FEATURE_NAMES)})"
+    )
+    return X_tr, X_te
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# Compute metrics for one model
+# ───────────────────────────────────────────────────────────────────────────
+
+
+def evaluate(name, model, X_test, y_test, proba=None):
+    """Return a metrics dict for one fitted model."""
+    y_pred = model.predict(X_test)
+
+    acc         = accuracy_score(y_test, y_pred)
+    prec_fake   = precision_score(y_test, y_pred, pos_label=1, zero_division=0)
+    rec_fake    = recall_score(y_test, y_pred, pos_label=1, zero_division=0)
+    f1_weighted = f1_score(y_test, y_pred, average="weighted", zero_division=0)
+    f1_fake     = f1_score(y_test, y_pred, pos_label=1, zero_division=0)
+
+    # AUC-ROC (needs probability scores)
+    if proba is not None:
+        try:
+            auc = roc_auc_score(y_test, proba[:, 1])
+        except Exception:
+            auc = float("nan")
+    else:
+        auc = float("nan")
+
+    report = classification_report(
+        y_test, y_pred,
+        target_names=["Real", "Fake"],
+        zero_division=0,
+    )
+
+    return {
+        "name":         name,
+        "accuracy":     acc,
+        "precision":    prec_fake,
+        "recall":       rec_fake,
+        "f1_weighted":  f1_weighted,
+        "f1_fake":      f1_fake,
+        "auc":          auc,
+        "report":       report,
+        "y_pred":       y_pred,
+        "proba":        proba,
+    }
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# Bar chart
+# ───────────────────────────────────────────────────────────────────────────
+
+
+def plot_bar_chart(results, output_path):
+    """Side-by-side bar chart: Accuracy vs F1 (weighted) vs AUC-ROC."""
+    names     = [r["name"] for r in results]
+    accs      = [r["accuracy"]    for r in results]
+    f1s       = [r["f1_weighted"] for r in results]
+    aucs      = [r["auc"]         for r in results]
+
+    x     = np.arange(len(names))
+    width = 0.26
+
+    fig, ax = plt.subplots(figsize=(max(10, len(names) * 2.2), 6))
+
+    # Color highlight for YOUR model (last entry)
+    bar_colors_acc = ["#2196F3"] * (len(names) - 1) + ["#E91E63"]
+    bar_colors_f1  = ["#4CAF50"] * (len(names) - 1) + ["#FF5722"]
+    bar_colors_auc = ["#9C27B0"] * (len(names) - 1) + ["#FF9800"]
+
+    b1 = ax.bar(x - width, accs,  width, color=bar_colors_acc, edgecolor="black", lw=0.5, label="Accuracy")
+    b2 = ax.bar(x,          f1s,   width, color=bar_colors_f1,  edgecolor="black", lw=0.5, label="F1 Weighted")
+    b3 = ax.bar(x + width,  aucs,  width, color=bar_colors_auc, edgecolor="black", lw=0.5, label="AUC-ROC")
+
+    # Value labels
+    for bars in (b1, b2, b3):
+        for bar in bars:
+            h = bar.get_height()
+            if not np.isnan(h):
+                ax.text(
+                    bar.get_x() + bar.get_width() / 2,
+                    h + 0.005,
+                    f"{h:.3f}",
+                    ha="center", va="bottom", fontsize=7.5, fontweight="bold",
+                )
+
+    ax.set_xticks(x)
+    ax.set_xticklabels(names, rotation=12, ha="right", fontsize=10)
+    ax.set_ylim(0, 1.12)
+    ax.set_ylabel("Score", fontsize=12)
+    ax.set_title(
+        "Benchmark: Your RF Model vs. Classic ML Baselines\n"
+        "(Highlighted in pink/orange = Your Model)",
+        fontsize=13, fontweight="bold",
+    )
+    ax.axhline(y=0.80, color="gray", linestyle="--", alpha=0.4, linewidth=1)
+    ax.text(len(names) - 0.5, 0.805, "80% threshold", color="gray", fontsize=8)
+
+    patch_yours = mpatches.Patch(color="#E91E63", label="★ Your Hybrid RF (Accuracy)")
+    ax.legend(handles=[*ax.get_legend_handles_labels()[0][:3], patch_yours], fontsize=9)
+
+    plt.tight_layout()
+    fig.savefig(output_path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {output_path}")
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# ROC curve chart
+# ───────────────────────────────────────────────────────────────────────────
+
+
+def plot_roc_curves(results, y_test, output_path):
+    """Overlay ROC curves for all models that have probability scores."""
+    fig, ax = plt.subplots(figsize=(8, 6))
+
+    COLORS = [
+        "#2196F3", "#4CAF50", "#FF9800", "#9C27B0",
+        "#E91E63", "#00BCD4", "#F44336", "#8BC34A",
+    ]
+
+    for i, r in enumerate(results):
+        if r["proba"] is None or np.isnan(r["auc"]):
+            continue
+        fpr, tpr, _ = roc_curve(y_test, r["proba"][:, 1], pos_label=1)
+        lw = 2.5 if "★" in r["name"] else 1.5
+        dash = "-" if "★" in r["name"] else "--"
+        ax.plot(
+            fpr, tpr,
+            color=COLORS[i % len(COLORS)],
+            lw=lw, linestyle=dash,
+            label=f"{r['name']}  (AUC={r['auc']:.3f})",
+        )
+
+    ax.plot([0, 1], [0, 1], "k:", lw=1, label="Random (AUC=0.500)")
+    ax.set_xlim([0.0, 1.0])
+    ax.set_ylim([0.0, 1.05])
+    ax.set_xlabel("False Positive Rate", fontsize=12)
+    ax.set_ylabel("True Positive Rate", fontsize=12)
+    ax.set_title("ROC Curves — Fake-News Detection Benchmark", fontsize=13, fontweight="bold")
+    ax.legend(loc="lower right", fontsize=9)
+    ax.grid(alpha=0.3)
+
+    plt.tight_layout()
+    fig.savefig(output_path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {output_path}")
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# Plaintext summary table
+# ───────────────────────────────────────────────────────────────────────────
+
+
+def save_table(results, output_path, mode, n_train, n_test):
+    """Write a neatly formatted comparison table to disk and stdout."""
+    lines = []
+    sep   = "=" * 90
+
+    lines.append(sep)
+    lines.append("  BENCHMARK RESULTS — Fake-News Detection (Filipino / Cebuano)")
+    lines.append(f"  Mode: {mode.upper()}   |   Train: {n_train:,} samples   |   Test: {n_test:,} samples")
+    lines.append(sep)
+    lines.append("")
+
+    header = (
+        f"  {'Model':<35} {'Accuracy':>9} {'Prec(Fk)':>10} {'Rec(Fk)':>9} "
+        f"{'F1(Wtd)':>9} {'F1(Fk)':>8} {'AUC-ROC':>9}"
+    )
+    lines.append(header)
+    lines.append("  " + "-" * 88)
+
+    for r in results:
+        auc_str = f"{r['auc']:.4f}" if not np.isnan(r["auc"]) else "  N/A  "
+        marker  = "  ★" if "★" in r["name"] else "   "
+        row = (
+            f"{marker} {r['name']:<32} "
+            f"{r['accuracy']:>9.4f} "
+            f"{r['precision']:>10.4f} "
+            f"{r['recall']:>9.4f} "
+            f"{r['f1_weighted']:>9.4f} "
+            f"{r['f1_fake']:>8.4f} "
+            f"{auc_str:>9}"
+        )
+        lines.append(row)
+
+    lines.append("")
+    lines.append(sep)
+    lines.append("  DETAILED CLASSIFICATION REPORTS")
+    lines.append(sep)
+
+    for r in results:
+        lines.append("")
+        lines.append(f"  ── {r['name']} ──────────────────────────────────────────────")
+        for ln in r["report"].splitlines():
+            lines.append(f"  {ln}")
+
+    text = "\n".join(lines)
+    print(text)
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(text)
+    print(f"\n  Saved: {output_path}")
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# Main
+# ───────────────────────────────────────────────────────────────────────────
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Benchmark fake-news classifiers.")
+    parser.add_argument(
+        "--mode",
+        choices=["mixed", "tagalog", "cebuano"],
+        default="mixed",
+        help="Which language subset to benchmark on (default: mixed).",
+    )
+    parser.add_argument(
+        "--test-size",
+        type=float,
+        default=0.20,
+        help="Fraction of data to hold out as the locked test set (default: 0.20).",
+    )
+    parser.add_argument(
+        "--skip-minilm",
+        action="store_true",
+        default=False,
+        help="Skip your hybrid RF model (useful if MiniLM is too slow for a quick check).",
+    )
+    args = parser.parse_args()
+
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    print("=" * 60)
+    print("  FAKE-NEWS BENCHMARK")
+    print(f"  Mode: {args.mode.upper()} | Test size: {args.test_size:.0%}")
+    print("=" * 60)
+
+    # ── 1. Load & preprocess dataset ─────────────────────────────────────
+    tagalog_only = args.mode == "tagalog"
+    cebuano_only = args.mode == "cebuano"
+
+    df = load_fake_news_dataset(tagalog_only=tagalog_only, cebuano_only=cebuano_only)
+    X_all, y_all = preprocess(df, undersample=False, oversample=True)
+
+    # ── 2. Locked test split (same seed → reproducible) ──────────────────
+    print(f"\nCreating locked test split ({1 - args.test_size:.0%} train / {args.test_size:.0%} test) …")
+    X_train, X_test, y_train, y_test = train_test_split(
+        X_all, y_all,
+        test_size=args.test_size,
+        random_state=42,
+        stratify=y_all,
+    )
+    print(f"  Train: {len(X_train):,}  |  Test: {len(X_test):,}")
+    print(f"  Test distribution — Real: {y_test.count(0):,}, Fake: {y_test.count(1):,}")
+
+    y_train_arr = np.array(y_train)
+    y_test_arr  = np.array(y_test)
+
+    # ── 3. Build TF-IDF features (shared by baseline models) ─────────────
+    print("\nBuilding TF-IDF features for baseline models …")
+    X_tr_tfidf, X_te_tfidf, tfidf = build_tfidf_features(X_train, X_test)
+    print(f"  TF-IDF shape: {X_tr_tfidf.shape}")
+
+    # ── 4. Train and evaluate each model ─────────────────────────────────
+    results = []
+
+    # ── 4a. Naive Bayes (TF-IDF, no negative values — shift by min) ──────
+    print("\n[1/5] Naive Bayes (TF-IDF) …")
+    t0  = time.time()
+    nb  = MultinomialNB(alpha=1.0)
+    # MultinomialNB needs non-negative input; TF-IDF with sublinear_tf is ok
+    nb.fit(X_tr_tfidf, y_train_arr)
+    nb_proba = nb.predict_proba(X_te_tfidf)
+    elapsed  = time.time() - t0
+    res_nb   = evaluate("Naive Bayes (TF-IDF)", nb, X_te_tfidf, y_test_arr, nb_proba)
+    res_nb["train_time"] = elapsed
+    results.append(res_nb)
+    print(f"  Accuracy: {res_nb['accuracy']:.4f}  |  F1 Weighted: {res_nb['f1_weighted']:.4f}  |  Time: {elapsed:.1f}s")
+
+    # ── 4b. Logistic Regression (TF-IDF) ─────────────────────────────────
+    print("\n[2/5] Logistic Regression (TF-IDF) …")
+    t0  = time.time()
+    lr  = LogisticRegression(
+        max_iter=1000,
+        class_weight="balanced",
+        solver="saga",
+        C=1.0,
+        random_state=42,
+        n_jobs=-1,
+    )
+    lr.fit(X_tr_tfidf, y_train_arr)
+    lr_proba = lr.predict_proba(X_te_tfidf)
+    elapsed  = time.time() - t0
+    res_lr   = evaluate("Logistic Regression (TF-IDF)", lr, X_te_tfidf, y_test_arr, lr_proba)
+    res_lr["train_time"] = elapsed
+    results.append(res_lr)
+    print(f"  Accuracy: {res_lr['accuracy']:.4f}  |  F1 Weighted: {res_lr['f1_weighted']:.4f}  |  Time: {elapsed:.1f}s")
+
+    # ── 4c. Linear SVM (TF-IDF) ──────────────────────────────────────────
+    print("\n[3/5] Linear SVM (TF-IDF) …")
+    t0  = time.time()
+    svm = CalibratedClassifierCV(
+        LinearSVC(class_weight="balanced", max_iter=2000, random_state=42),
+        cv=3,
+    )
+    svm.fit(X_tr_tfidf, y_train_arr)
+    svm_proba = svm.predict_proba(X_te_tfidf)
+    elapsed   = time.time() - t0
+    res_svm   = evaluate("Linear SVM (TF-IDF)", svm, X_te_tfidf, y_test_arr, svm_proba)
+    res_svm["train_time"] = elapsed
+    results.append(res_svm)
+    print(f"  Accuracy: {res_svm['accuracy']:.4f}  |  F1 Weighted: {res_svm['f1_weighted']:.4f}  |  Time: {elapsed:.1f}s")
+
+    # ── 4d. Random Forest (TF-IDF only — no embeddings) ──────────────────
+    print("\n[4/5] Random Forest (TF-IDF only — no embeddings) …")
+    t0    = time.time()
+    rf_tf = RandomForestClassifier(
+        n_estimators=300,
+        max_depth=20,
+        min_samples_split=5,
+        min_samples_leaf=3,
+        class_weight="balanced",
+        n_jobs=-1,
+        random_state=42,
+    )
+    rf_tf.fit(X_tr_tfidf, y_train_arr)
+    rf_tf_proba = rf_tf.predict_proba(X_te_tfidf)
+    elapsed     = time.time() - t0
+    res_rf_tf   = evaluate("Random Forest (TF-IDF only)", rf_tf, X_te_tfidf, y_test_arr, rf_tf_proba)
+    res_rf_tf["train_time"] = elapsed
+    results.append(res_rf_tf)
+    print(f"  Accuracy: {res_rf_tf['accuracy']:.4f}  |  F1 Weighted: {res_rf_tf['f1_weighted']:.4f}  |  Time: {elapsed:.1f}s")
+
+    # ── 4e. YOUR Hybrid RF (TF-IDF + MiniLM + Stylometric) ───────────────
+    if not args.skip_minilm:
+        print("\n[5/5] ★ YOUR Hybrid RF (TF-IDF + MiniLM + Stylometric) …")
+        X_tr_hy, X_te_hy = build_hybrid_features(X_train, X_test)
+
+        t0    = time.time()
+        max_depth = 8 if cebuano_only else 20
+        rf_hy = RandomForestClassifier(
+            n_estimators=500,
+            max_depth=max_depth,
+            min_samples_split=5,
+            min_samples_leaf=3,
+            class_weight="balanced",
+            n_jobs=-1,
+            random_state=42,
+        )
+        rf_hy.fit(X_tr_hy, y_train_arr)
+        rf_hy_proba = rf_hy.predict_proba(X_te_hy)
+        elapsed     = time.time() - t0
+        res_rf_hy   = evaluate("★ Hybrid RF (TF-IDF + MiniLM + Stylo)", rf_hy, X_te_hy, y_test_arr, rf_hy_proba)
+        res_rf_hy["train_time"] = elapsed
+        results.append(res_rf_hy)
+        print(f"  Accuracy: {res_rf_hy['accuracy']:.4f}  |  F1 Weighted: {res_rf_hy['f1_weighted']:.4f}  |  Time: {elapsed:.1f}s")
+    else:
+        print("\n[5/5] Skipping Hybrid RF (--skip-minilm flag set).")
+
+    # ── 5. Output table ───────────────────────────────────────────────────
+    print("\n" + "=" * 60)
+    print("  BENCHMARK SUMMARY")
+    print("=" * 60)
+    table_path = os.path.join(OUTPUT_DIR, f"benchmark_table_{args.mode}.txt")
+    save_table(results, table_path, args.mode, len(X_train), len(X_test))
+
+    # ── 6. Bar chart ──────────────────────────────────────────────────────
+    chart_path = os.path.join(OUTPUT_DIR, f"benchmark_chart_{args.mode}.png")
+    plot_bar_chart(results, chart_path)
+
+    # ── 7. ROC curves ─────────────────────────────────────────────────────
+    roc_path = os.path.join(OUTPUT_DIR, f"benchmark_roc_{args.mode}.png")
+    plot_roc_curves(results, y_test_arr, roc_path)
+
+    # ── 8. Train time summary ─────────────────────────────────────────────
+    print("\n  Training times:")
+    for r in results:
+        t = r.get("train_time", 0)
+        print(f"    {r['name']:<45} {t:>6.1f}s")
+
+    print("\n" + "=" * 60)
+    print("  BENCHMARK COMPLETE")
+    print(f"  Results saved to: {OUTPUT_DIR}")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

backend/bias_words_report.txt

@@ -1,148 +1,148 @@
-=================================================================
-  CHI-SQUARED BIAS WORD MINER
-  Finds words most associated with Fake vs Real articles
-=================================================================
-
-Loaded 25664 articles
-Real: 16405 | Fake: 9259
-
-Building vocabulary from 25664 articles...
-Running chi-squared analysis...
-
-=================================================================
-  TOP 60 WORDS ASSOCIATED WITH *** FAKE NEWS ***
-  (Consider adding to SENSATIONAL_KEYWORDS or RIGHT/LEFT lists)
-=================================================================
-  ang                            chi2= 11772.6  fake_freq=2.1795  real_freq=0.6382
-  mga                            chi2=  7393.4  fake_freq=1.0763  real_freq=0.2492
-  miss                           chi2=  2917.1  fake_freq=0.2418  real_freq=0.0199
-  hindi                          chi2=  2627.4  fake_freq=0.3756  real_freq=0.0852
-  daw                            chi2=  2581.2  fake_freq=0.1800  real_freq=0.0065
-  adobo                          chi2=  2540.9  fake_freq=0.1594  real_freq=0.0012
-  lang                           chi2=  2520.2  fake_freq=0.2660  real_freq=0.0391
-  you                            chi2=  2501.5  fake_freq=1.1692  real_freq=0.5885
-  adobo chronicles               chi2=  2490.9  fake_freq=0.1521  real_freq=0.0001
-  chronicles                     chi2=  2472.7  fake_freq=0.1537  real_freq=0.0008
-  the adobo                      chi2=  2443.3  fake_freq=0.1489  real_freq=0.0000
-  sa mga                         chi2=  2399.6  fake_freq=0.2914  real_freq=0.0536
-  ito                            chi2=  2374.0  fake_freq=0.3101  real_freq=0.0630
-  rappler                        chi2=  2298.3  fake_freq=0.1879  real_freq=0.0148
-  sabi                           chi2=  2157.7  fake_freq=0.1847  real_freq=0.0168
-  media                          chi2=  2148.0  fake_freq=0.4887  real_freq=0.1679
-  kung                           chi2=  1897.0  fake_freq=0.2617  real_freq=0.0569
-  video                          chi2=  1894.0  fake_freq=0.2651  real_freq=0.0587
-  sabi ni                        chi2=  1842.0  fake_freq=0.1377  real_freq=0.0075
-  kay                            chi2=  1793.6  fake_freq=0.2103  real_freq=0.0368
-  naman                          chi2=  1775.7  fake_freq=0.2518  real_freq=0.0566
-  facebook                       chi2=  1758.7  fake_freq=0.2292  real_freq=0.0464
-  netizens                       chi2=  1748.0  fake_freq=0.1363  real_freq=0.0090
-  ang mga                        chi2=  1728.9  fake_freq=0.2514  real_freq=0.0582
-  pageant                        chi2=  1715.6  fake_freq=0.1253  real_freq=0.0060
-  your                           chi2=  1701.4  fake_freq=0.3996  real_freq=0.1403
-  ng mga                         chi2=  1582.1  fake_freq=0.2811  real_freq=0.0794
-  yung                           chi2=  1508.4  fake_freq=0.1792  real_freq=0.0319
-  social media                   chi2=  1503.9  fake_freq=0.2093  real_freq=0.0460
-  umano                          chi2=  1478.0  fake_freq=0.1161  real_freq=0.0079
-  kayo                           chi2=  1398.4  fake_freq=0.1054  real_freq=0.0060
-  leni                           chi2=  1397.2  fake_freq=0.1806  real_freq=0.0362
-  ako                            chi2=  1364.9  fake_freq=0.1684  real_freq=0.0317
-  comment                        chi2=  1353.9  fake_freq=0.2057  real_freq=0.0499
-  para                           chi2=  1319.4  fake_freq=0.2437  real_freq=0.0712
-  pangulong                      chi2=  1288.2  fake_freq=0.0925  real_freq=0.0040
-  catriona                       chi2=  1285.2  fake_freq=0.0797  real_freq=0.0004
-  thinkingpinoy                  chi2=  1268.6  fake_freq=0.0773  real_freq=0.0000
-  blog                           chi2=  1258.3  fake_freq=0.0961  real_freq=0.0058
-  dahil                          chi2=  1213.0  fake_freq=0.1816  real_freq=0.0433
-  isang                          chi2=  1210.0  fake_freq=0.2125  real_freq=0.0594
-  ayon                           chi2=  1204.6  fake_freq=0.1481  real_freq=0.0277
-  ilang                          chi2=  1176.7  fake_freq=0.1077  real_freq=0.0117
-  noynoy                         chi2=  1121.0  fake_freq=0.0794  real_freq=0.0032
-  ressa                          chi2=  1086.1  fake_freq=0.0760  real_freq=0.0028
-  com                            chi2=  1082.5  fake_freq=0.1124  real_freq=0.0160
-  din                            chi2=  1081.8  fake_freq=0.1438  real_freq=0.0299
-  bs aquino                      chi2=  1062.9  fake_freq=0.0650  real_freq=0.0001
-  natin                          chi2=  1035.5  fake_freq=0.1229  real_freq=0.0219
-  niya                           chi2=  1032.8  fake_freq=0.1660  real_freq=0.0427
-  http                           chi2=  1029.2  fake_freq=0.0703  real_freq=0.0021
-  grp                            chi2=  1010.0  fake_freq=0.0731  real_freq=0.0034
-  naging                         chi2=   993.2  fake_freq=0.0958  real_freq=0.0118
-  featured comment               chi2=   990.4  fake_freq=0.0604  real_freq=0.0000
-  is grp                         chi2=   988.7  fake_freq=0.0603  real_freq=0.0000
-  grp featured                   chi2=   983.3  fake_freq=0.0599  real_freq=0.0000
-  like                           chi2=   983.3  fake_freq=0.5211  real_freq=0.2752
-  news                           chi2=   980.1  fake_freq=0.3171  real_freq=0.1348
-  catriona gray                  chi2=   979.6  fake_freq=0.0599  real_freq=0.0001
-  yan                            chi2=   971.2  fake_freq=0.0989  real_freq=0.0136
-
-=================================================================
-  TOP 60 WORDS ASSOCIATED WITH *** REAL NEWS ***
-  (High-credibility markers ù useful context)
-=================================================================
-  said                           chi2= 30152.0  real_freq=5.3763  fake_freq=0.9817
-  said the                       chi2=  7892.5  real_freq=1.0162  fake_freq=0.0680
-  he said                        chi2=  7048.7  real_freq=1.0146  fake_freq=0.1094
-  of the                         chi2=  6391.5  real_freq=4.0884  fake_freq=2.1730
-  was                            chi2=  6127.3  real_freq=3.1958  fake_freq=1.5541
-  had                            chi2=  4985.5  real_freq=1.2013  fake_freq=0.3359
-  were                           chi2=  4869.7  real_freq=1.4602  fake_freq=0.5025
-  for                            chi2=  3693.8  real_freq=3.9857  fake_freq=2.5171
-  percent                        chi2=  3316.0  real_freq=0.4733  fake_freq=0.0496
-  said he                        chi2=  3054.9  real_freq=0.4525  fake_freq=0.0535
-  city                           chi2=  3015.1  real_freq=0.9589  fake_freq=0.3458
-  police                         chi2=  2951.7  real_freq=0.6889  fake_freq=0.1859
-  mr duterte                     chi2=  2937.8  real_freq=0.3344  fake_freq=0.0068
-  would                          chi2=  2682.3  real_freq=1.1915  fake_freq=0.5339
-  added                          chi2=  2527.3  real_freq=0.4712  fake_freq=0.0931
-  in the                         chi2=  2323.8  real_freq=2.2774  fake_freq=1.3999
-  government                     chi2=  2221.8  real_freq=0.9786  fake_freq=0.4364
-  from                           chi2=  2142.5  real_freq=1.9267  fake_freq=1.1543
-  court                          chi2=  2128.4  real_freq=0.5376  fake_freq=0.1580
-  said in                        chi2=  1914.3  real_freq=0.3323  fake_freq=0.0577
-  province                       chi2=  1836.7  real_freq=0.2501  fake_freq=0.0219
-  year                           chi2=  1725.1  real_freq=0.6088  fake_freq=0.2369
-  she said                       chi2=  1703.1  real_freq=0.2836  fake_freq=0.0451
-  sen                            chi2=  1646.4  real_freq=0.2895  fake_freq=0.0515
-  the department                 chi2=  1624.6  real_freq=0.2532  fake_freq=0.0345
-  also                           chi2=  1619.7  real_freq=1.1369  fake_freq=0.6263
-  for the                        chi2=  1617.7  real_freq=0.9776  fake_freq=0.5076
-  rep                            chi2=  1561.8  real_freq=0.2070  fake_freq=0.0161
-  the president                  chi2=  1555.6  real_freq=0.5825  fake_freq=0.2358
-  to the                         chi2=  1533.1  real_freq=1.4158  fake_freq=0.8552
-  department                     chi2=  1480.6  real_freq=0.3825  fake_freq=0.1150
-  he added                       chi2=  1479.9  real_freq=0.2401  fake_freq=0.0361
-  office                         chi2=  1426.7  real_freq=0.4416  fake_freq=0.1560
-  friday                         chi2=  1420.6  real_freq=0.2201  fake_freq=0.0295
-  with                           chi2=  1404.1  real_freq=2.2094  fake_freq=1.5269
-  officials                      chi2=  1398.2  real_freq=0.3345  fake_freq=0.0928
-  military                       chi2=  1392.7  real_freq=0.3024  fake_freq=0.0747
-  monday                         chi2=  1364.4  real_freq=0.2359  fake_freq=0.0406
-  tuesday                        chi2=  1357.9  real_freq=0.2077  fake_freq=0.0269
-  house                          chi2=  1340.9  real_freq=0.3684  fake_freq=0.1175
-  chief                          chi2=  1329.3  real_freq=0.4226  fake_freq=0.1524
-  committee                      chi2=  1317.5  real_freq=0.2536  fake_freq=0.0528
-  the government                 chi2=  1310.3  real_freq=0.3822  fake_freq=0.1285
-  the house                      chi2=  1290.2  real_freq=0.2091  fake_freq=0.0313
-  had been                       chi2=  1277.4  real_freq=0.1926  fake_freq=0.0240
-  on friday                      chi2=  1269.4  real_freq=0.1663  fake_freq=0.0122
-  thursday                       chi2=  1260.2  real_freq=0.1889  fake_freq=0.0231
-  peace                          chi2=  1233.9  real_freq=0.2489  fake_freq=0.0556
-  million                        chi2=  1231.2  real_freq=0.4199  fake_freq=0.1595
-  and the                        chi2=  1210.5  real_freq=0.7438  fake_freq=0.3889
-  at the                         chi2=  1192.8  real_freq=0.6397  fake_freq=0.3152
-  wednesday                      chi2=  1176.4  real_freq=0.1791  fake_freq=0.0229
-  department of                  chi2=  1141.5  real_freq=0.2878  fake_freq=0.0845
-  national                       chi2=  1125.5  real_freq=0.5410  fake_freq=0.2527
-  on monday                      chi2=  1123.8  real_freq=0.1721  fake_freq=0.0224
-  under                          chi2=  1114.4  real_freq=0.4066  fake_freq=0.1618
-  region                         chi2=  1113.0  real_freq=0.1859  fake_freq=0.0298
-  town                           chi2=  1102.8  real_freq=0.1554  fake_freq=0.0156
-  on tuesday                     chi2=  1101.4  real_freq=0.1529  fake_freq=0.0145
-  by the                         chi2=  1079.0  real_freq=0.7163  fake_freq=0.3863
-
-=================================================================
-  HOW TO USE THESE RESULTS:
-  1. Review the FAKE words list above
-  2. Add clearly sensational/biased words to SENSATIONAL_KEYWORDS
-     in checker/internal/bias_analyzer.py
-  3. Add political framing words to RIGHT/LEFT_LEANING_KEYWORDS
-=================================================================
+=================================================================
+  CHI-SQUARED BIAS WORD MINER
+  Finds words most associated with Fake vs Real articles
+=================================================================
+
+Loaded 25664 articles
+Real: 16405 | Fake: 9259
+
+Building vocabulary from 25664 articles...
+Running chi-squared analysis...
+
+=================================================================
+  TOP 60 WORDS ASSOCIATED WITH *** FAKE NEWS ***
+  (Consider adding to SENSATIONAL_KEYWORDS or RIGHT/LEFT lists)
+=================================================================
+  ang                            chi2= 11772.6  fake_freq=2.1795  real_freq=0.6382
+  mga                            chi2=  7393.4  fake_freq=1.0763  real_freq=0.2492
+  miss                           chi2=  2917.1  fake_freq=0.2418  real_freq=0.0199
+  hindi                          chi2=  2627.4  fake_freq=0.3756  real_freq=0.0852
+  daw                            chi2=  2581.2  fake_freq=0.1800  real_freq=0.0065
+  adobo                          chi2=  2540.9  fake_freq=0.1594  real_freq=0.0012
+  lang                           chi2=  2520.2  fake_freq=0.2660  real_freq=0.0391
+  you                            chi2=  2501.5  fake_freq=1.1692  real_freq=0.5885
+  adobo chronicles               chi2=  2490.9  fake_freq=0.1521  real_freq=0.0001
+  chronicles                     chi2=  2472.7  fake_freq=0.1537  real_freq=0.0008
+  the adobo                      chi2=  2443.3  fake_freq=0.1489  real_freq=0.0000
+  sa mga                         chi2=  2399.6  fake_freq=0.2914  real_freq=0.0536
+  ito                            chi2=  2374.0  fake_freq=0.3101  real_freq=0.0630
+  rappler                        chi2=  2298.3  fake_freq=0.1879  real_freq=0.0148
+  sabi                           chi2=  2157.7  fake_freq=0.1847  real_freq=0.0168
+  media                          chi2=  2148.0  fake_freq=0.4887  real_freq=0.1679
+  kung                           chi2=  1897.0  fake_freq=0.2617  real_freq=0.0569
+  video                          chi2=  1894.0  fake_freq=0.2651  real_freq=0.0587
+  sabi ni                        chi2=  1842.0  fake_freq=0.1377  real_freq=0.0075
+  kay                            chi2=  1793.6  fake_freq=0.2103  real_freq=0.0368
+  naman                          chi2=  1775.7  fake_freq=0.2518  real_freq=0.0566
+  facebook                       chi2=  1758.7  fake_freq=0.2292  real_freq=0.0464
+  netizens                       chi2=  1748.0  fake_freq=0.1363  real_freq=0.0090
+  ang mga                        chi2=  1728.9  fake_freq=0.2514  real_freq=0.0582
+  pageant                        chi2=  1715.6  fake_freq=0.1253  real_freq=0.0060
+  your                           chi2=  1701.4  fake_freq=0.3996  real_freq=0.1403
+  ng mga                         chi2=  1582.1  fake_freq=0.2811  real_freq=0.0794
+  yung                           chi2=  1508.4  fake_freq=0.1792  real_freq=0.0319
+  social media                   chi2=  1503.9  fake_freq=0.2093  real_freq=0.0460
+  umano                          chi2=  1478.0  fake_freq=0.1161  real_freq=0.0079
+  kayo                           chi2=  1398.4  fake_freq=0.1054  real_freq=0.0060
+  leni                           chi2=  1397.2  fake_freq=0.1806  real_freq=0.0362
+  ako                            chi2=  1364.9  fake_freq=0.1684  real_freq=0.0317
+  comment                        chi2=  1353.9  fake_freq=0.2057  real_freq=0.0499
+  para                           chi2=  1319.4  fake_freq=0.2437  real_freq=0.0712
+  pangulong                      chi2=  1288.2  fake_freq=0.0925  real_freq=0.0040
+  catriona                       chi2=  1285.2  fake_freq=0.0797  real_freq=0.0004
+  thinkingpinoy                  chi2=  1268.6  fake_freq=0.0773  real_freq=0.0000
+  blog                           chi2=  1258.3  fake_freq=0.0961  real_freq=0.0058
+  dahil                          chi2=  1213.0  fake_freq=0.1816  real_freq=0.0433
+  isang                          chi2=  1210.0  fake_freq=0.2125  real_freq=0.0594
+  ayon                           chi2=  1204.6  fake_freq=0.1481  real_freq=0.0277
+  ilang                          chi2=  1176.7  fake_freq=0.1077  real_freq=0.0117
+  noynoy                         chi2=  1121.0  fake_freq=0.0794  real_freq=0.0032
+  ressa                          chi2=  1086.1  fake_freq=0.0760  real_freq=0.0028
+  com                            chi2=  1082.5  fake_freq=0.1124  real_freq=0.0160
+  din                            chi2=  1081.8  fake_freq=0.1438  real_freq=0.0299
+  bs aquino                      chi2=  1062.9  fake_freq=0.0650  real_freq=0.0001
+  natin                          chi2=  1035.5  fake_freq=0.1229  real_freq=0.0219
+  niya                           chi2=  1032.8  fake_freq=0.1660  real_freq=0.0427
+  http                           chi2=  1029.2  fake_freq=0.0703  real_freq=0.0021
+  grp                            chi2=  1010.0  fake_freq=0.0731  real_freq=0.0034
+  naging                         chi2=   993.2  fake_freq=0.0958  real_freq=0.0118
+  featured comment               chi2=   990.4  fake_freq=0.0604  real_freq=0.0000
+  is grp                         chi2=   988.7  fake_freq=0.0603  real_freq=0.0000
+  grp featured                   chi2=   983.3  fake_freq=0.0599  real_freq=0.0000
+  like                           chi2=   983.3  fake_freq=0.5211  real_freq=0.2752
+  news                           chi2=   980.1  fake_freq=0.3171  real_freq=0.1348
+  catriona gray                  chi2=   979.6  fake_freq=0.0599  real_freq=0.0001
+  yan                            chi2=   971.2  fake_freq=0.0989  real_freq=0.0136
+
+=================================================================
+  TOP 60 WORDS ASSOCIATED WITH *** REAL NEWS ***
+  (High-credibility markers ù useful context)
+=================================================================
+  said                           chi2= 30152.0  real_freq=5.3763  fake_freq=0.9817
+  said the                       chi2=  7892.5  real_freq=1.0162  fake_freq=0.0680
+  he said                        chi2=  7048.7  real_freq=1.0146  fake_freq=0.1094
+  of the                         chi2=  6391.5  real_freq=4.0884  fake_freq=2.1730
+  was                            chi2=  6127.3  real_freq=3.1958  fake_freq=1.5541
+  had                            chi2=  4985.5  real_freq=1.2013  fake_freq=0.3359
+  were                           chi2=  4869.7  real_freq=1.4602  fake_freq=0.5025
+  for                            chi2=  3693.8  real_freq=3.9857  fake_freq=2.5171
+  percent                        chi2=  3316.0  real_freq=0.4733  fake_freq=0.0496
+  said he                        chi2=  3054.9  real_freq=0.4525  fake_freq=0.0535
+  city                           chi2=  3015.1  real_freq=0.9589  fake_freq=0.3458
+  police                         chi2=  2951.7  real_freq=0.6889  fake_freq=0.1859
+  mr duterte                     chi2=  2937.8  real_freq=0.3344  fake_freq=0.0068
+  would                          chi2=  2682.3  real_freq=1.1915  fake_freq=0.5339
+  added                          chi2=  2527.3  real_freq=0.4712  fake_freq=0.0931
+  in the                         chi2=  2323.8  real_freq=2.2774  fake_freq=1.3999
+  government                     chi2=  2221.8  real_freq=0.9786  fake_freq=0.4364
+  from                           chi2=  2142.5  real_freq=1.9267  fake_freq=1.1543
+  court                          chi2=  2128.4  real_freq=0.5376  fake_freq=0.1580
+  said in                        chi2=  1914.3  real_freq=0.3323  fake_freq=0.0577
+  province                       chi2=  1836.7  real_freq=0.2501  fake_freq=0.0219
+  year                           chi2=  1725.1  real_freq=0.6088  fake_freq=0.2369
+  she said                       chi2=  1703.1  real_freq=0.2836  fake_freq=0.0451
+  sen                            chi2=  1646.4  real_freq=0.2895  fake_freq=0.0515
+  the department                 chi2=  1624.6  real_freq=0.2532  fake_freq=0.0345
+  also                           chi2=  1619.7  real_freq=1.1369  fake_freq=0.6263
+  for the                        chi2=  1617.7  real_freq=0.9776  fake_freq=0.5076
+  rep                            chi2=  1561.8  real_freq=0.2070  fake_freq=0.0161
+  the president                  chi2=  1555.6  real_freq=0.5825  fake_freq=0.2358
+  to the                         chi2=  1533.1  real_freq=1.4158  fake_freq=0.8552
+  department                     chi2=  1480.6  real_freq=0.3825  fake_freq=0.1150
+  he added                       chi2=  1479.9  real_freq=0.2401  fake_freq=0.0361
+  office                         chi2=  1426.7  real_freq=0.4416  fake_freq=0.1560
+  friday                         chi2=  1420.6  real_freq=0.2201  fake_freq=0.0295
+  with                           chi2=  1404.1  real_freq=2.2094  fake_freq=1.5269
+  officials                      chi2=  1398.2  real_freq=0.3345  fake_freq=0.0928
+  military                       chi2=  1392.7  real_freq=0.3024  fake_freq=0.0747
+  monday                         chi2=  1364.4  real_freq=0.2359  fake_freq=0.0406
+  tuesday                        chi2=  1357.9  real_freq=0.2077  fake_freq=0.0269
+  house                          chi2=  1340.9  real_freq=0.3684  fake_freq=0.1175
+  chief                          chi2=  1329.3  real_freq=0.4226  fake_freq=0.1524
+  committee                      chi2=  1317.5  real_freq=0.2536  fake_freq=0.0528
+  the government                 chi2=  1310.3  real_freq=0.3822  fake_freq=0.1285
+  the house                      chi2=  1290.2  real_freq=0.2091  fake_freq=0.0313
+  had been                       chi2=  1277.4  real_freq=0.1926  fake_freq=0.0240
+  on friday                      chi2=  1269.4  real_freq=0.1663  fake_freq=0.0122
+  thursday                       chi2=  1260.2  real_freq=0.1889  fake_freq=0.0231
+  peace                          chi2=  1233.9  real_freq=0.2489  fake_freq=0.0556
+  million                        chi2=  1231.2  real_freq=0.4199  fake_freq=0.1595
+  and the                        chi2=  1210.5  real_freq=0.7438  fake_freq=0.3889
+  at the                         chi2=  1192.8  real_freq=0.6397  fake_freq=0.3152
+  wednesday                      chi2=  1176.4  real_freq=0.1791  fake_freq=0.0229
+  department of                  chi2=  1141.5  real_freq=0.2878  fake_freq=0.0845
+  national                       chi2=  1125.5  real_freq=0.5410  fake_freq=0.2527
+  on monday                      chi2=  1123.8  real_freq=0.1721  fake_freq=0.0224
+  under                          chi2=  1114.4  real_freq=0.4066  fake_freq=0.1618
+  region                         chi2=  1113.0  real_freq=0.1859  fake_freq=0.0298
+  town                           chi2=  1102.8  real_freq=0.1554  fake_freq=0.0156
+  on tuesday                     chi2=  1101.4  real_freq=0.1529  fake_freq=0.0145
+  by the                         chi2=  1079.0  real_freq=0.7163  fake_freq=0.3863
+
+=================================================================
+  HOW TO USE THESE RESULTS:
+  1. Review the FAKE words list above
+  2. Add clearly sensational/biased words to SENSATIONAL_KEYWORDS
+     in checker/internal/bias_analyzer.py
+  3. Add political framing words to RIGHT/LEFT_LEANING_KEYWORDS
+=================================================================

backend/evaluate_model.py

@@ -1,820 +1,820 @@
-"""
-Model Overfitting Evaluation Script
-=====================================
-Evaluates the Random Forest fake news classifier for overfitting by
-comparing Training vs. Testing performance.
-
-Split: 80% Train / 20% Test
-Metrics: classification_report, accuracy_score, confusion matrix plot
-Flag: Overfitting detected if Train Acc > 95% and Test Acc < 70%
-
-Usage:
-    python backend/evaluate_model.py
-"""
-
-import sys
-import os
-import re
-import time
-import numpy as np
-from textblob import TextBlob
-import textstat
-
-PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, PROJECT_ROOT)
-
-import pandas as pd
-import matplotlib
-
-matplotlib.use("Agg")  # Non-interactive backend for saving plots
-import matplotlib.pyplot as plt
-from scipy.sparse import hstack, csr_matrix
-from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.preprocessing import StandardScaler
-from sklearn.metrics import (
-    classification_report,
-    accuracy_score,
-    confusion_matrix,
-    ConfusionMatrixDisplay,
-)
-from sentence_transformers import SentenceTransformer
-
-
-# ── Paths ──
-DATA_MODELS_DIR = os.path.join(PROJECT_ROOT, "data_models")
-OUTPUT_DIR = os.path.join(PROJECT_ROOT, "evaluation_results")
-
-
-# ── MiniLM Model (lazy-loaded singleton) ──
-MINILM_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
-_minilm_model = None
-
-
-def get_minilm_model():
-    """Load the multilingual MiniLM model (cached after first call)."""
-    global _minilm_model
-    if _minilm_model is None:
-        print("  Loading MiniLM model...")
-        _minilm_model = SentenceTransformer(MINILM_MODEL_NAME)
-    return _minilm_model
-
-
-# ───────────────────────────────────────────────────────────
-# Text Cleaning (same as train.py)
-# ───────────────────────────────────────────────────────────
-
-
-def clean_text(text):
-    """Basic text cleaning for Filipino news articles."""
-    if not text or not isinstance(text, str):
-        return ""
-    text = re.sub(r"<[^>]+>", " ", text)
-    text = re.sub(r"https?://\S+", " ", text)
-    text = re.sub(r"\s+", " ", text)
-    return text.strip()
-
-
-# ───────────────────────────────────────────────────────────
-# Stylometric Features (same as train.py)
-# ───────────────────────────────────────────────────────────
-
-
-# ── Word lists for linguistic features ──
-FIRST_PERSON_PRONOUNS = {
-    "i",
-    "me",
-    "my",
-    "mine",
-    "myself",
-    "we",
-    "us",
-    "our",
-    "ours",
-    "ourselves",
-    "ako",
-    "ko",
-    "akin",
-    "aking",
-    "natin",
-    "atin",
-    "namin",
-    "amin",
-    "tayo",
-    "kami",
-    "ta",
-}
-
-AUXILIARY_VERBS = {
-    "have",
-    "has",
-    "had",
-    "do",
-    "does",
-    "did",
-    "will",
-    "would",
-    "shall",
-    "should",
-    "may",
-    "might",
-    "can",
-    "could",
-    "must",
-    "am",
-    "is",
-    "are",
-    "was",
-    "were",
-    "be",
-    "been",
-    "being",
-    "ay",
-    "dapat",
-    "mayroon",
-    "meron",
-    "maaari",
-    "pwede",
-    "kailangan",
-}
-
-ANALYTICAL_WORDS = {
-    "the",
-    "a",
-    "an",
-    "of",
-    "in",
-    "on",
-    "at",
-    "to",
-    "for",
-    "with",
-    "by",
-    "from",
-    "about",
-    "between",
-    "through",
-    "during",
-    "before",
-    "after",
-    "ang",
-    "ng",
-    "sa",
-    "mga",
-    "nang",
-    "para",
-    "tungkol",
-    "mula",
-}
-
-CERTAINTY_WORDS = {
-    "always",
-    "never",
-    "absolutely",
-    "definitely",
-    "certainly",
-    "undoubtedly",
-    "clearly",
-    "obviously",
-    "without doubt",
-    "guaranteed",
-    "proven",
-    "fact",
-    "undeniable",
-    "indisputable",
-    "every",
-    "all",
-    "palagi",
-    "sigurado",
-    "tiyak",
-    "talaga",
-    "totoo",
-    "lagi",
-    "walang duda",
-}
-
-TENTATIVE_WORDS = {
-    "perhaps",
-    "maybe",
-    "possibly",
-    "might",
-    "could",
-    "likely",
-    "unlikely",
-    "suggests",
-    "appears",
-    "seems",
-    "allegedly",
-    "reportedly",
-    "according",
-    "probable",
-    "approximately",
-    "estimated",
-    "siguro",
-    "marahil",
-    "maaaring",
-    "mukhang",
-    "parang",
-    "umano",
-    "diumano",
-}
-
-CLOUT_WORDS = {
-    "must",
-    "demand",
-    "require",
-    "order",
-    "command",
-    "insist",
-    "decree",
-    "mandate",
-    "authority",
-    "power",
-    "control",
-    "dominant",
-    "superior",
-    "we must",
-    "you must",
-    "kailangan",
-    "dapat",
-    "utos",
-    "kapangyarihan",
-    "kontrol",
-    "mando",
-}
-
-PAST_FOCUS_WORDS = {
-    "talked",
-    "did",
-    "ago",
-    "said",
-    "was",
-    "were",
-    "had",
-    "went",
-    "told",
-    "noon",
-    "nakaraan",
-    "dati",
-    "kahapon",
-}
-
-PRESENT_FOCUS_WORDS = {
-    "now",
-    "is",
-    "today",
-    "are",
-    "being",
-    "currently",
-    "ongoing",
-    "ngayon",
-    "kasalukuyan",
-}
-
-FUTURE_FOCUS_WORDS = {
-    "soon",
-    "will",
-    "may",
-    "shall",
-    "going",
-    "plan",
-    "expect",
-    "tomorrow",
-    "bukas",
-    "darating",
-    "magiging",
-    "gagawin",
-}
-
-
-def extract_stylometric_features(text):
-    """Extract 25 stylometric features from text (matches train.py)."""
-    if not text or not isinstance(text, str):
-        return [0.0] * 25
-
-    words = text.split()
-    token_count = len(words)
-    if token_count == 0:
-        return [0.0] * 25
-
-    words_lower = [w.lower() for w in words]
-    text_len = len(text)
-
-    exclamation_density = text.count("!") / token_count
-    question_count = text.count("?")
-
-    caps_words = sum(1 for w in words if len(w) >= 2 and w.isupper())
-    caps_ratio = caps_words / token_count
-
-    sentences = re.split(r"[.!?]+", text)
-    sentences = [s.strip() for s in sentences if s.strip()]
-    avg_sentence_length = (
-        sum(len(s.split()) for s in sentences) / len(sentences)
-        if sentences
-        else token_count
-    )
-
-    punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
-    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0
-
-    unique_words = len(set(words_lower))
-    unique_word_ratio = unique_words / token_count
-
-    avg_word_length = sum(len(w) for w in words) / token_count
-
-    try:
-        subjectivity = TextBlob(text).sentiment.subjectivity
-    except Exception:
-        subjectivity = 0.0
-
-    try:
-        flesch_reading_ease = textstat.flesch_reading_ease(text)
-        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
-        coleman_liau_index = textstat.coleman_liau_index(text)
-        ari = textstat.automated_readability_index(text)
-    except Exception:
-        flesch_reading_ease = 0.0
-        flesch_kincaid_grade = 0.0
-        coleman_liau_index = 0.0
-        ari = 0.0
-
-    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
-    first_person_ratio = first_person_count / token_count
-
-    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
-    auxiliary_verb_ratio = aux_count / token_count
-
-    try:
-        gunning_fog_index = textstat.gunning_fog(text)
-    except Exception:
-        gunning_fog_index = 0.0
-
-    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
-    analytical_thinking = analytical_count / token_count
-
-    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
-    certainty_score = certainty_count / token_count
-
-    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
-    tentative_score = tentative_count / token_count
-
-    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
-    clout_score = clout_count / token_count
-
-    comma_period_count = text.count(",") + text.count(".")
-    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0
-
-    informal_count = (
-        text.count("(")
-        + text.count(")")
-        + text.count("—")
-        + text.count("–")
-        + text.count("-")
-        + text.count("...")
-        + text.count("…")
-    )
-    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0
-
-    past_count = sum(1 for w in words_lower if w in PAST_FOCUS_WORDS)
-    past_focus_ratio = past_count / token_count
-
-    present_count = sum(1 for w in words_lower if w in PRESENT_FOCUS_WORDS)
-    present_focus_ratio = present_count / token_count
-
-    future_count = sum(1 for w in words_lower if w in FUTURE_FOCUS_WORDS)
-    future_focus_ratio = future_count / token_count
-
-    return [
-        float(exclamation_density),
-        float(question_count),
-        float(caps_ratio),
-        float(avg_sentence_length),
-        float(punctuation_density),
-        float(token_count),
-        float(unique_word_ratio),
-        float(avg_word_length),
-        float(subjectivity),
-        float(flesch_reading_ease),
-        float(flesch_kincaid_grade),
-        float(coleman_liau_index),
-        float(ari),
-        float(first_person_ratio),
-        float(auxiliary_verb_ratio),
-        float(gunning_fog_index),
-        float(analytical_thinking),
-        float(certainty_score),
-        float(tentative_score),
-        float(clout_score),
-        float(comma_period_density),
-        float(informal_punct_density),
-        float(past_focus_ratio),
-        float(present_focus_ratio),
-        float(future_focus_ratio),
-    ]
-
-
-STYLOMETRIC_FEATURE_NAMES = [
-    "exclamation_density",
-    "question_count",
-    "caps_ratio",
-    "avg_sentence_length",
-    "punctuation_density",
-    "token_count",
-    "unique_word_ratio",
-    "avg_word_length",
-    "subjectivity",
-    "flesch_reading_ease",
-    "flesch_kincaid_grade",
-    "coleman_liau_index",
-    "ari",
-    "first_person_ratio",
-    "auxiliary_verb_ratio",
-    "gunning_fog_index",
-    "analytical_thinking",
-    "certainty_score",
-    "tentative_score",
-    "clout_score",
-    "comma_period_density",
-    "informal_punct_density",
-    "past_focus_ratio",
-    "present_focus_ratio",
-    "future_focus_ratio",
-]
-
-
-# ───────────────────────────────────────────────────────────
-# Main Evaluation
-# ───────────────────────────────────────────────────────────
-
-
-def main():
-    os.makedirs(OUTPUT_DIR, exist_ok=True)
-    label_names = ["Real", "Fake"]
-
-    # ── 1. Load Dataset ──
-    print("=" * 60)
-    print("  MODEL OVERFITTING EVALUATION")
-    print("=" * 60)
-
-    csv_path = os.path.join(
-        PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv"
-    )
-    if not os.path.exists(csv_path):
-        print(f"ERROR: Dataset not found at {csv_path}")
-        return
-
-    df = pd.read_csv(csv_path)
-    print(f"\nDataset: jcblaise/fake_news_filipino")
-    print(f"Total articles: {len(df)}")
-    print(f"Distribution:")
-    print(f"  Real (0): {(df['label'] == 0).sum()}")
-    print(f"  Fake (1): {(df['label'] == 1).sum()}")
-
-    # ── 2. Preprocess ──
-    print("\nPreprocessing...")
-    df = df.dropna(subset=["article"]).copy()
-    df = df[df["article"].str.len() > 0].copy()
-    df.loc[:, "article_clean"] = df["article"].apply(clean_text)
-
-    X_texts = df["article_clean"].tolist()
-    y_labels = df["label"].tolist()
-    print(f"  Valid articles: {len(X_texts)}")
-
-    # ── 3. Split: 80% Train / 20% Test ──
-    print("\nSplitting data: 80% Train / 20% Test...")
-    X_train, X_test, y_train, y_test = train_test_split(
-        X_texts,
-        y_labels,
-        test_size=0.20,
-        random_state=42,
-        stratify=y_labels,
-    )
-    print(f"  Training set: {len(X_train)} articles")
-    print(f"  Testing set:  {len(X_test)} articles")
-
-    # ── 4. Build Hybrid Features ──
-    print("\nBuilding hybrid features (TF-IDF + MiniLM + stylometric)...")
-
-    # TF-IDF
-    tfidf = TfidfVectorizer(
-        max_features=15000,
-        ngram_range=(1, 2),
-        min_df=2,
-        max_df=0.95,
-        sublinear_tf=True,
-    )
-    X_train_tfidf = tfidf.fit_transform(X_train)
-    X_test_tfidf = tfidf.transform(X_test)
-
-    # MiniLM embeddings
-    print("  Encoding texts with MiniLM...")
-    minilm = get_minilm_model()
-    train_embeddings = minilm.encode(X_train, show_progress_bar=True, batch_size=64)
-    test_embeddings = minilm.encode(X_test, show_progress_bar=True, batch_size=64)
-
-    # Stylometric
-    print("  Extracting stylometric features...")
-    train_stylo = np.array([extract_stylometric_features(t) for t in X_train])
-    test_stylo = np.array([extract_stylometric_features(t) for t in X_test])
-
-    scaler = StandardScaler()
-    train_stylo_scaled = scaler.fit_transform(train_stylo)
-    test_stylo_scaled = scaler.transform(test_stylo)
-
-    # Combine
-    X_train_feat = hstack(
-        [X_train_tfidf, csr_matrix(train_embeddings), csr_matrix(train_stylo_scaled)]
-    )
-    X_test_feat = hstack(
-        [X_test_tfidf, csr_matrix(test_embeddings), csr_matrix(test_stylo_scaled)]
-    )
-
-    n_tfidf = X_train_tfidf.shape[1]
-    n_minilm = 384
-    n_stylo = len(STYLOMETRIC_FEATURE_NAMES)
-    print(
-        f"  Feature dimensions: {X_train_feat.shape[1]} "
-        f"(TF-IDF: {n_tfidf} + MiniLM: {n_minilm} + Stylometric: {n_stylo})"
-    )
-
-    # ── 5. Full 5-Fold Cross-Validation ──
-    print("\n" + "=" * 60)
-    print("  5-FOLD CROSS-VALIDATION (Full Dataset)")
-    print("=" * 60)
-
-    # Build features on entire dataset
-    print("\nBuilding features on full dataset...")
-    tfidf_full = TfidfVectorizer(
-        max_features=15000,
-        ngram_range=(1, 2),
-        min_df=2,
-        max_df=0.95,
-        sublinear_tf=True,
-    )
-    X_tfidf_full = tfidf_full.fit_transform(X_texts)
-
-    print("  Encoding full dataset with MiniLM...")
-    full_embeddings = minilm.encode(X_texts, show_progress_bar=True, batch_size=64)
-
-    stylo_full = np.array([extract_stylometric_features(t) for t in X_texts])
-    scaler_full = StandardScaler()
-    stylo_full_scaled = scaler_full.fit_transform(stylo_full)
-    X_full = hstack(
-        [X_tfidf_full, csr_matrix(full_embeddings), csr_matrix(stylo_full_scaled)]
-    )
-    y_full = np.array(y_labels)
-
-    print(f"  Total samples: {X_full.shape[0]}")
-    print(
-        f"  Feature dimensions: {X_full.shape[1]} "
-        f"(TF-IDF: {X_tfidf_full.shape[1]} + MiniLM: {n_minilm} + Stylometric: {n_stylo})"
-    )
-
-    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
-
-    fold_accuracies = []
-    fold_precisions = []
-    fold_recalls = []
-    fold_f1s = []
-    fold_train_accs = []
-    all_y_true = []
-    all_y_pred = []
-
-    for fold_idx, (train_idx, test_idx) in enumerate(cv.split(X_full, y_full), 1):
-        X_fold_train = X_full[train_idx]
-        X_fold_test = X_full[test_idx]
-        y_fold_train = y_full[train_idx]
-        y_fold_test = y_full[test_idx]
-
-        print(f"\n{'─' * 60}")
-        print(f"  FOLD {fold_idx}/5  (Train: {len(train_idx)}, Test: {len(test_idx)})")
-        print(f"{'─' * 60}")
-
-        rf_fold = RandomForestClassifier(
-            n_estimators=300,
-            max_depth=15,
-            min_samples_split=5,
-            min_samples_leaf=5,
-            class_weight="balanced",
-            n_jobs=-1,
-            random_state=42,
-        )
-        rf_fold.fit(X_fold_train, y_fold_train)
-
-        # Predictions
-        y_fold_train_pred = rf_fold.predict(X_fold_train)
-        y_fold_test_pred = rf_fold.predict(X_fold_test)
-
-        train_acc = accuracy_score(y_fold_train, y_fold_train_pred)
-        test_acc = accuracy_score(y_fold_test, y_fold_test_pred)
-
-        fold_train_accs.append(train_acc)
-        fold_accuracies.append(test_acc)
-
-        # Per-fold classification report
-        report = classification_report(
-            y_fold_test,
-            y_fold_test_pred,
-            target_names=label_names,
-            output_dict=True,
-        )
-        fold_precisions.append(report["weighted avg"]["precision"])
-        fold_recalls.append(report["weighted avg"]["recall"])
-        fold_f1s.append(report["weighted avg"]["f1-score"])
-
-        # Collect for final confusion matrix
-        all_y_true.extend(y_fold_test)
-        all_y_pred.extend(y_fold_test_pred)
-
-        print(f"  Train Accuracy: {train_acc:.4f} ({train_acc:.1%})")
-        print(f"  Test Accuracy:  {test_acc:.4f} ({test_acc:.1%})")
-        print(f"  Gap:            {train_acc - test_acc:.4f}")
-        print()
-        print(
-            classification_report(
-                y_fold_test, y_fold_test_pred, target_names=label_names
-            )
-        )
-
-    # ── 6. Cross-Fold Summary ──
-    fold_accuracies = np.array(fold_accuracies)
-    fold_train_accs = np.array(fold_train_accs)
-    fold_precisions = np.array(fold_precisions)
-    fold_recalls = np.array(fold_recalls)
-    fold_f1s = np.array(fold_f1s)
-    gaps = fold_train_accs - fold_accuracies
-
-    print("\n" + "=" * 60)
-    print("  CROSS-VALIDATION SUMMARY (5 Folds)")
-    print("=" * 60)
-
-    print(f"\n  Per-Fold Test Accuracies:")
-    for i, (ta, te) in enumerate(zip(fold_train_accs, fold_accuracies), 1):
-        print(f"    Fold {i}: Train {ta:.1%} | Test {te:.1%} | Gap {ta - te:.1%}")
-
-    print(
-        f"\n  Average Training Accuracy:  {fold_train_accs.mean():.4f} "
-        f"(+/- {fold_train_accs.std():.4f})"
-    )
-    print(
-        f"  Average Testing Accuracy:   {fold_accuracies.mean():.4f} "
-        f"(+/- {fold_accuracies.std():.4f})"
-    )
-    print(
-        f"  Average Precision:          {fold_precisions.mean():.4f} "
-        f"(+/- {fold_precisions.std():.4f})"
-    )
-    print(
-        f"  Average Recall:             {fold_recalls.mean():.4f} "
-        f"(+/- {fold_recalls.std():.4f})"
-    )
-    print(
-        f"  Average F1 Score:           {fold_f1s.mean():.4f} "
-        f"(+/- {fold_f1s.std():.4f})"
-    )
-    print(f"  Average Gap:                {gaps.mean():.4f} " f"(+/- {gaps.std():.4f})")
-
-    # ── 7. Consistency Check ──
-    print("\n" + "=" * 60)
-    print("  VERDICT CONSISTENCY & OVERFITTING ANALYSIS")
-    print("=" * 60)
-
-    avg_train = fold_train_accs.mean()
-    avg_test = fold_accuracies.mean()
-    avg_gap = gaps.mean()
-    acc_std = fold_accuracies.std()
-
-    if avg_train > 0.95 and avg_test < 0.70:
-        overfit_status = "OVERFITTING DETECTED"
-        print(f"\n  *** OVERFITTING DETECTED ***")
-        print(f"  Average training accuracy ({avg_train:.1%}) is much higher than")
-        print(f"  average testing accuracy ({avg_test:.1%}).")
-        print(f"  The model memorizes training data and fails to generalize.")
-    elif avg_gap > 0.10:
-        overfit_status = "MILD OVERFITTING"
-        print(f"\n  ** MILD OVERFITTING **")
-        print(f"  Average gap ({avg_gap:.1%}) exceeds 10%.")
-    else:
-        overfit_status = "NO OVERFITTING"
-        print(f"\n  NO OVERFITTING DETECTED")
-        print(f"  Average gap ({avg_gap:.1%}) is within acceptable range.")
-
-    if acc_std < 0.01:
-        consistency = "HIGHLY CONSISTENT"
-        print(f"  Verdict Consistency: HIGHLY CONSISTENT (std={acc_std:.4f})")
-        print(f"  Predictions are very stable across all 5 folds.")
-    elif acc_std < 0.03:
-        consistency = "CONSISTENT"
-        print(f"  Verdict Consistency: CONSISTENT (std={acc_std:.4f})")
-        print(f"  Minor variance across folds — acceptable for production.")
-    else:
-        consistency = "INCONSISTENT"
-        print(f"  Verdict Consistency: INCONSISTENT (std={acc_std:.4f})")
-        print(f"  High variance suggests model stability issues.")
-
-    # ── 8. Confusion Matrix (aggregated across all folds) ──
-    print("\n\nGenerating plots...")
-    cm = confusion_matrix(all_y_true, all_y_pred)
-    overall_acc = accuracy_score(all_y_true, all_y_pred)
-
-    fig, ax = plt.subplots(figsize=(8, 6))
-    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names)
-    disp.plot(ax=ax, cmap="Blues", values_format="d")
-
-    ax.set_title(
-        f"Confusion Matrix — Aggregated 5-Fold CV\n"
-        f"Overall Accuracy: {overall_acc:.1%} | {overfit_status}",
-        fontsize=14,
-        fontweight="bold",
-    )
-    ax.set_xlabel("Predicted Label", fontsize=12)
-    ax.set_ylabel("True Label", fontsize=12)
-
-    plt.tight_layout()
-    cm_path = os.path.join(OUTPUT_DIR, "confusion_matrix.png")
-    fig.savefig(cm_path, dpi=150, bbox_inches="tight")
-    print(f"  Saved: {cm_path}")
-
-    # ── 9. Per-Fold Accuracy Bar Chart ──
-    fig2, ax2 = plt.subplots(figsize=(10, 5))
-
-    x = np.arange(5)
-    width = 0.35
-    bars_train = ax2.bar(
-        x - width / 2,
-        fold_train_accs * 100,
-        width,
-        label="Training",
-        color="#2196F3",
-        edgecolor="black",
-        linewidth=0.5,
-    )
-    bars_test = ax2.bar(
-        x + width / 2,
-        fold_accuracies * 100,
-        width,
-        label="Testing",
-        color="#FF9800",
-        edgecolor="black",
-        linewidth=0.5,
-    )
-
-    for bar, val in zip(bars_train, fold_train_accs):
-        ax2.text(
-            bar.get_x() + bar.get_width() / 2,
-            bar.get_height() + 0.3,
-            f"{val:.1%}",
-            ha="center",
-            va="bottom",
-            fontsize=9,
-            fontweight="bold",
-        )
-    for bar, val in zip(bars_test, fold_accuracies):
-        ax2.text(
-            bar.get_x() + bar.get_width() / 2,
-            bar.get_height() + 0.3,
-            f"{val:.1%}",
-            ha="center",
-            va="bottom",
-            fontsize=9,
-            fontweight="bold",
-        )
-
-    ax2.set_xticks(x)
-    ax2.set_xticklabels([f"Fold {i}" for i in range(1, 6)])
-    ax2.set_ylim(0, 105)
-    ax2.set_ylabel("Accuracy (%)", fontsize=12)
-    ax2.set_title(
-        f"Per-Fold Accuracy Comparison\n"
-        f"Avg Test: {avg_test:.1%} (+/- {acc_std:.4f}) | {consistency}",
-        fontsize=14,
-        fontweight="bold",
-    )
-    ax2.legend(loc="lower right")
-    ax2.axhline(y=70, color="red", linestyle="--", alpha=0.5, label="70% threshold")
-
-    plt.tight_layout()
-    bar_path = os.path.join(OUTPUT_DIR, "accuracy_comparison.png")
-    fig2.savefig(bar_path, dpi=150, bbox_inches="tight")
-    print(f"  Saved: {bar_path}")
-
-    # ── Final Summary ──
-    print("\n" + "=" * 60)
-    print("  EVALUATION COMPLETE")
-    print("=" * 60)
-    print(f"  Dataset:               fake_news_filipino ({len(df)} articles)")
-    print(f"  Feature set:           {X_full.shape[1]} (TF-IDF + 9 stylometric)")
-    print(f"  Cross-Validation:      5-Fold Stratified")
-    print(f"  Avg Training Accuracy: {avg_train:.4f} (+/- {fold_train_accs.std():.4f})")
-    print(f"  Avg Testing Accuracy:  {avg_test:.4f} (+/- {acc_std:.4f})")
-    print(f"  Avg F1 Score:          {fold_f1s.mean():.4f} (+/- {fold_f1s.std():.4f})")
-    print(f"  Avg Gap:               {avg_gap:.4f}")
-    print(f"  Overfitting Status:    {overfit_status}")
-    print(f"  Verdict Consistency:   {consistency}")
-    print(f"  Plots saved to:        {OUTPUT_DIR}/")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()
+"""
+Model Overfitting Evaluation Script
+=====================================
+Evaluates the Random Forest fake news classifier for overfitting by
+comparing Training vs. Testing performance.
+
+Split: 80% Train / 20% Test
+Metrics: classification_report, accuracy_score, confusion matrix plot
+Flag: Overfitting detected if Train Acc > 95% and Test Acc < 70%
+
+Usage:
+    python backend/evaluate_model.py
+"""
+
+import sys
+import os
+import re
+import time
+import numpy as np
+from textblob import TextBlob
+import textstat
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_ROOT)
+
+import pandas as pd
+import matplotlib
+
+matplotlib.use("Agg")  # Non-interactive backend for saving plots
+import matplotlib.pyplot as plt
+from scipy.sparse import hstack, csr_matrix
+from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import StandardScaler
+from sklearn.metrics import (
+    classification_report,
+    accuracy_score,
+    confusion_matrix,
+    ConfusionMatrixDisplay,
+)
+from sentence_transformers import SentenceTransformer
+
+
+# ── Paths ──
+DATA_MODELS_DIR = os.path.join(PROJECT_ROOT, "data_models")
+OUTPUT_DIR = os.path.join(PROJECT_ROOT, "evaluation_results")
+
+
+# ── MiniLM Model (lazy-loaded singleton) ──
+MINILM_MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2"
+_minilm_model = None
+
+
+def get_minilm_model():
+    """Load the multilingual MiniLM model (cached after first call)."""
+    global _minilm_model
+    if _minilm_model is None:
+        print("  Loading MiniLM model...")
+        _minilm_model = SentenceTransformer(MINILM_MODEL_NAME)
+    return _minilm_model
+
+
+# ───────────────────────────────────────────────────────────
+# Text Cleaning (same as train.py)
+# ───────────────────────────────────────────────────────────
+
+
+def clean_text(text):
+    """Basic text cleaning for Filipino news articles."""
+    if not text or not isinstance(text, str):
+        return ""
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = re.sub(r"https?://\S+", " ", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+# ───────────────────────────────────────────────────────────
+# Stylometric Features (same as train.py)
+# ───────────────────────────────────────────────────────────
+
+
+# ── Word lists for linguistic features ──
+FIRST_PERSON_PRONOUNS = {
+    "i",
+    "me",
+    "my",
+    "mine",
+    "myself",
+    "we",
+    "us",
+    "our",
+    "ours",
+    "ourselves",
+    "ako",
+    "ko",
+    "akin",
+    "aking",
+    "natin",
+    "atin",
+    "namin",
+    "amin",
+    "tayo",
+    "kami",
+    "ta",
+}
+
+AUXILIARY_VERBS = {
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "shall",
+    "should",
+    "may",
+    "might",
+    "can",
+    "could",
+    "must",
+    "am",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "ay",
+    "dapat",
+    "mayroon",
+    "meron",
+    "maaari",
+    "pwede",
+    "kailangan",
+}
+
+ANALYTICAL_WORDS = {
+    "the",
+    "a",
+    "an",
+    "of",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "with",
+    "by",
+    "from",
+    "about",
+    "between",
+    "through",
+    "during",
+    "before",
+    "after",
+    "ang",
+    "ng",
+    "sa",
+    "mga",
+    "nang",
+    "para",
+    "tungkol",
+    "mula",
+}
+
+CERTAINTY_WORDS = {
+    "always",
+    "never",
+    "absolutely",
+    "definitely",
+    "certainly",
+    "undoubtedly",
+    "clearly",
+    "obviously",
+    "without doubt",
+    "guaranteed",
+    "proven",
+    "fact",
+    "undeniable",
+    "indisputable",
+    "every",
+    "all",
+    "palagi",
+    "sigurado",
+    "tiyak",
+    "talaga",
+    "totoo",
+    "lagi",
+    "walang duda",
+}
+
+TENTATIVE_WORDS = {
+    "perhaps",
+    "maybe",
+    "possibly",
+    "might",
+    "could",
+    "likely",
+    "unlikely",
+    "suggests",
+    "appears",
+    "seems",
+    "allegedly",
+    "reportedly",
+    "according",
+    "probable",
+    "approximately",
+    "estimated",
+    "siguro",
+    "marahil",
+    "maaaring",
+    "mukhang",
+    "parang",
+    "umano",
+    "diumano",
+}
+
+CLOUT_WORDS = {
+    "must",
+    "demand",
+    "require",
+    "order",
+    "command",
+    "insist",
+    "decree",
+    "mandate",
+    "authority",
+    "power",
+    "control",
+    "dominant",
+    "superior",
+    "we must",
+    "you must",
+    "kailangan",
+    "dapat",
+    "utos",
+    "kapangyarihan",
+    "kontrol",
+    "mando",
+}
+
+PAST_FOCUS_WORDS = {
+    "talked",
+    "did",
+    "ago",
+    "said",
+    "was",
+    "were",
+    "had",
+    "went",
+    "told",
+    "noon",
+    "nakaraan",
+    "dati",
+    "kahapon",
+}
+
+PRESENT_FOCUS_WORDS = {
+    "now",
+    "is",
+    "today",
+    "are",
+    "being",
+    "currently",
+    "ongoing",
+    "ngayon",
+    "kasalukuyan",
+}
+
+FUTURE_FOCUS_WORDS = {
+    "soon",
+    "will",
+    "may",
+    "shall",
+    "going",
+    "plan",
+    "expect",
+    "tomorrow",
+    "bukas",
+    "darating",
+    "magiging",
+    "gagawin",
+}
+
+
+def extract_stylometric_features(text):
+    """Extract 25 stylometric features from text (matches train.py)."""
+    if not text or not isinstance(text, str):
+        return [0.0] * 25
+
+    words = text.split()
+    token_count = len(words)
+    if token_count == 0:
+        return [0.0] * 25
+
+    words_lower = [w.lower() for w in words]
+    text_len = len(text)
+
+    exclamation_density = text.count("!") / token_count
+    question_count = text.count("?")
+
+    caps_words = sum(1 for w in words if len(w) >= 2 and w.isupper())
+    caps_ratio = caps_words / token_count
+
+    sentences = re.split(r"[.!?]+", text)
+    sentences = [s.strip() for s in sentences if s.strip()]
+    avg_sentence_length = (
+        sum(len(s.split()) for s in sentences) / len(sentences)
+        if sentences
+        else token_count
+    )
+
+    punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
+    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0
+
+    unique_words = len(set(words_lower))
+    unique_word_ratio = unique_words / token_count
+
+    avg_word_length = sum(len(w) for w in words) / token_count
+
+    try:
+        subjectivity = TextBlob(text).sentiment.subjectivity
+    except Exception:
+        subjectivity = 0.0
+
+    try:
+        flesch_reading_ease = textstat.flesch_reading_ease(text)
+        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
+        coleman_liau_index = textstat.coleman_liau_index(text)
+        ari = textstat.automated_readability_index(text)
+    except Exception:
+        flesch_reading_ease = 0.0
+        flesch_kincaid_grade = 0.0
+        coleman_liau_index = 0.0
+        ari = 0.0
+
+    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
+    first_person_ratio = first_person_count / token_count
+
+    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
+    auxiliary_verb_ratio = aux_count / token_count
+
+    try:
+        gunning_fog_index = textstat.gunning_fog(text)
+    except Exception:
+        gunning_fog_index = 0.0
+
+    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
+    analytical_thinking = analytical_count / token_count
+
+    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
+    certainty_score = certainty_count / token_count
+
+    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
+    tentative_score = tentative_count / token_count
+
+    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
+    clout_score = clout_count / token_count
+
+    comma_period_count = text.count(",") + text.count(".")
+    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0
+
+    informal_count = (
+        text.count("(")
+        + text.count(")")
+        + text.count("—")
+        + text.count("–")
+        + text.count("-")
+        + text.count("...")
+        + text.count("…")
+    )
+    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0
+
+    past_count = sum(1 for w in words_lower if w in PAST_FOCUS_WORDS)
+    past_focus_ratio = past_count / token_count
+
+    present_count = sum(1 for w in words_lower if w in PRESENT_FOCUS_WORDS)
+    present_focus_ratio = present_count / token_count
+
+    future_count = sum(1 for w in words_lower if w in FUTURE_FOCUS_WORDS)
+    future_focus_ratio = future_count / token_count
+
+    return [
+        float(exclamation_density),
+        float(question_count),
+        float(caps_ratio),
+        float(avg_sentence_length),
+        float(punctuation_density),
+        float(token_count),
+        float(unique_word_ratio),
+        float(avg_word_length),
+        float(subjectivity),
+        float(flesch_reading_ease),
+        float(flesch_kincaid_grade),
+        float(coleman_liau_index),
+        float(ari),
+        float(first_person_ratio),
+        float(auxiliary_verb_ratio),
+        float(gunning_fog_index),
+        float(analytical_thinking),
+        float(certainty_score),
+        float(tentative_score),
+        float(clout_score),
+        float(comma_period_density),
+        float(informal_punct_density),
+        float(past_focus_ratio),
+        float(present_focus_ratio),
+        float(future_focus_ratio),
+    ]
+
+
+STYLOMETRIC_FEATURE_NAMES = [
+    "exclamation_density",
+    "question_count",
+    "caps_ratio",
+    "avg_sentence_length",
+    "punctuation_density",
+    "token_count",
+    "unique_word_ratio",
+    "avg_word_length",
+    "subjectivity",
+    "flesch_reading_ease",
+    "flesch_kincaid_grade",
+    "coleman_liau_index",
+    "ari",
+    "first_person_ratio",
+    "auxiliary_verb_ratio",
+    "gunning_fog_index",
+    "analytical_thinking",
+    "certainty_score",
+    "tentative_score",
+    "clout_score",
+    "comma_period_density",
+    "informal_punct_density",
+    "past_focus_ratio",
+    "present_focus_ratio",
+    "future_focus_ratio",
+]
+
+
+# ───────────────────────────────────────────────────────────
+# Main Evaluation
+# ──��────────────────────────────────────────────────────────
+
+
+def main():
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+    label_names = ["Real", "Fake"]
+
+    # ── 1. Load Dataset ──
+    print("=" * 60)
+    print("  MODEL OVERFITTING EVALUATION")
+    print("=" * 60)
+
+    csv_path = os.path.join(
+        PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv"
+    )
+    if not os.path.exists(csv_path):
+        print(f"ERROR: Dataset not found at {csv_path}")
+        return
+
+    df = pd.read_csv(csv_path)
+    print(f"\nDataset: jcblaise/fake_news_filipino")
+    print(f"Total articles: {len(df)}")
+    print(f"Distribution:")
+    print(f"  Real (0): {(df['label'] == 0).sum()}")
+    print(f"  Fake (1): {(df['label'] == 1).sum()}")
+
+    # ── 2. Preprocess ──
+    print("\nPreprocessing...")
+    df = df.dropna(subset=["article"]).copy()
+    df = df[df["article"].str.len() > 0].copy()
+    df.loc[:, "article_clean"] = df["article"].apply(clean_text)
+
+    X_texts = df["article_clean"].tolist()
+    y_labels = df["label"].tolist()
+    print(f"  Valid articles: {len(X_texts)}")
+
+    # ── 3. Split: 80% Train / 20% Test ──
+    print("\nSplitting data: 80% Train / 20% Test...")
+    X_train, X_test, y_train, y_test = train_test_split(
+        X_texts,
+        y_labels,
+        test_size=0.20,
+        random_state=42,
+        stratify=y_labels,
+    )
+    print(f"  Training set: {len(X_train)} articles")
+    print(f"  Testing set:  {len(X_test)} articles")
+
+    # ── 4. Build Hybrid Features ──
+    print("\nBuilding hybrid features (TF-IDF + MiniLM + stylometric)...")
+
+    # TF-IDF
+    tfidf = TfidfVectorizer(
+        max_features=15000,
+        ngram_range=(1, 2),
+        min_df=2,
+        max_df=0.95,
+        sublinear_tf=True,
+    )
+    X_train_tfidf = tfidf.fit_transform(X_train)
+    X_test_tfidf = tfidf.transform(X_test)
+
+    # MiniLM embeddings
+    print("  Encoding texts with MiniLM...")
+    minilm = get_minilm_model()
+    train_embeddings = minilm.encode(X_train, show_progress_bar=True, batch_size=64)
+    test_embeddings = minilm.encode(X_test, show_progress_bar=True, batch_size=64)
+
+    # Stylometric
+    print("  Extracting stylometric features...")
+    train_stylo = np.array([extract_stylometric_features(t) for t in X_train])
+    test_stylo = np.array([extract_stylometric_features(t) for t in X_test])
+
+    scaler = StandardScaler()
+    train_stylo_scaled = scaler.fit_transform(train_stylo)
+    test_stylo_scaled = scaler.transform(test_stylo)
+
+    # Combine
+    X_train_feat = hstack(
+        [X_train_tfidf, csr_matrix(train_embeddings), csr_matrix(train_stylo_scaled)]
+    )
+    X_test_feat = hstack(
+        [X_test_tfidf, csr_matrix(test_embeddings), csr_matrix(test_stylo_scaled)]
+    )
+
+    n_tfidf = X_train_tfidf.shape[1]
+    n_minilm = 384
+    n_stylo = len(STYLOMETRIC_FEATURE_NAMES)
+    print(
+        f"  Feature dimensions: {X_train_feat.shape[1]} "
+        f"(TF-IDF: {n_tfidf} + MiniLM: {n_minilm} + Stylometric: {n_stylo})"
+    )
+
+    # ── 5. Full 5-Fold Cross-Validation ──
+    print("\n" + "=" * 60)
+    print("  5-FOLD CROSS-VALIDATION (Full Dataset)")
+    print("=" * 60)
+
+    # Build features on entire dataset
+    print("\nBuilding features on full dataset...")
+    tfidf_full = TfidfVectorizer(
+        max_features=15000,
+        ngram_range=(1, 2),
+        min_df=2,
+        max_df=0.95,
+        sublinear_tf=True,
+    )
+    X_tfidf_full = tfidf_full.fit_transform(X_texts)
+
+    print("  Encoding full dataset with MiniLM...")
+    full_embeddings = minilm.encode(X_texts, show_progress_bar=True, batch_size=64)
+
+    stylo_full = np.array([extract_stylometric_features(t) for t in X_texts])
+    scaler_full = StandardScaler()
+    stylo_full_scaled = scaler_full.fit_transform(stylo_full)
+    X_full = hstack(
+        [X_tfidf_full, csr_matrix(full_embeddings), csr_matrix(stylo_full_scaled)]
+    )
+    y_full = np.array(y_labels)
+
+    print(f"  Total samples: {X_full.shape[0]}")
+    print(
+        f"  Feature dimensions: {X_full.shape[1]} "
+        f"(TF-IDF: {X_tfidf_full.shape[1]} + MiniLM: {n_minilm} + Stylometric: {n_stylo})"
+    )
+
+    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+
+    fold_accuracies = []
+    fold_precisions = []
+    fold_recalls = []
+    fold_f1s = []
+    fold_train_accs = []
+    all_y_true = []
+    all_y_pred = []
+
+    for fold_idx, (train_idx, test_idx) in enumerate(cv.split(X_full, y_full), 1):
+        X_fold_train = X_full[train_idx]
+        X_fold_test = X_full[test_idx]
+        y_fold_train = y_full[train_idx]
+        y_fold_test = y_full[test_idx]
+
+        print(f"\n{'─' * 60}")
+        print(f"  FOLD {fold_idx}/5  (Train: {len(train_idx)}, Test: {len(test_idx)})")
+        print(f"{'─' * 60}")
+
+        rf_fold = RandomForestClassifier(
+            n_estimators=300,
+            max_depth=15,
+            min_samples_split=5,
+            min_samples_leaf=5,
+            class_weight="balanced",
+            n_jobs=-1,
+            random_state=42,
+        )
+        rf_fold.fit(X_fold_train, y_fold_train)
+
+        # Predictions
+        y_fold_train_pred = rf_fold.predict(X_fold_train)
+        y_fold_test_pred = rf_fold.predict(X_fold_test)
+
+        train_acc = accuracy_score(y_fold_train, y_fold_train_pred)
+        test_acc = accuracy_score(y_fold_test, y_fold_test_pred)
+
+        fold_train_accs.append(train_acc)
+        fold_accuracies.append(test_acc)
+
+        # Per-fold classification report
+        report = classification_report(
+            y_fold_test,
+            y_fold_test_pred,
+            target_names=label_names,
+            output_dict=True,
+        )
+        fold_precisions.append(report["weighted avg"]["precision"])
+        fold_recalls.append(report["weighted avg"]["recall"])
+        fold_f1s.append(report["weighted avg"]["f1-score"])
+
+        # Collect for final confusion matrix
+        all_y_true.extend(y_fold_test)
+        all_y_pred.extend(y_fold_test_pred)
+
+        print(f"  Train Accuracy: {train_acc:.4f} ({train_acc:.1%})")
+        print(f"  Test Accuracy:  {test_acc:.4f} ({test_acc:.1%})")
+        print(f"  Gap:            {train_acc - test_acc:.4f}")
+        print()
+        print(
+            classification_report(
+                y_fold_test, y_fold_test_pred, target_names=label_names
+            )
+        )
+
+    # ── 6. Cross-Fold Summary ──
+    fold_accuracies = np.array(fold_accuracies)
+    fold_train_accs = np.array(fold_train_accs)
+    fold_precisions = np.array(fold_precisions)
+    fold_recalls = np.array(fold_recalls)
+    fold_f1s = np.array(fold_f1s)
+    gaps = fold_train_accs - fold_accuracies
+
+    print("\n" + "=" * 60)
+    print("  CROSS-VALIDATION SUMMARY (5 Folds)")
+    print("=" * 60)
+
+    print(f"\n  Per-Fold Test Accuracies:")
+    for i, (ta, te) in enumerate(zip(fold_train_accs, fold_accuracies), 1):
+        print(f"    Fold {i}: Train {ta:.1%} | Test {te:.1%} | Gap {ta - te:.1%}")
+
+    print(
+        f"\n  Average Training Accuracy:  {fold_train_accs.mean():.4f} "
+        f"(+/- {fold_train_accs.std():.4f})"
+    )
+    print(
+        f"  Average Testing Accuracy:   {fold_accuracies.mean():.4f} "
+        f"(+/- {fold_accuracies.std():.4f})"
+    )
+    print(
+        f"  Average Precision:          {fold_precisions.mean():.4f} "
+        f"(+/- {fold_precisions.std():.4f})"
+    )
+    print(
+        f"  Average Recall:             {fold_recalls.mean():.4f} "
+        f"(+/- {fold_recalls.std():.4f})"
+    )
+    print(
+        f"  Average F1 Score:           {fold_f1s.mean():.4f} "
+        f"(+/- {fold_f1s.std():.4f})"
+    )
+    print(f"  Average Gap:                {gaps.mean():.4f} " f"(+/- {gaps.std():.4f})")
+
+    # ── 7. Consistency Check ──
+    print("\n" + "=" * 60)
+    print("  VERDICT CONSISTENCY & OVERFITTING ANALYSIS")
+    print("=" * 60)
+
+    avg_train = fold_train_accs.mean()
+    avg_test = fold_accuracies.mean()
+    avg_gap = gaps.mean()
+    acc_std = fold_accuracies.std()
+
+    if avg_train > 0.95 and avg_test < 0.70:
+        overfit_status = "OVERFITTING DETECTED"
+        print(f"\n  *** OVERFITTING DETECTED ***")
+        print(f"  Average training accuracy ({avg_train:.1%}) is much higher than")
+        print(f"  average testing accuracy ({avg_test:.1%}).")
+        print(f"  The model memorizes training data and fails to generalize.")
+    elif avg_gap > 0.10:
+        overfit_status = "MILD OVERFITTING"
+        print(f"\n  ** MILD OVERFITTING **")
+        print(f"  Average gap ({avg_gap:.1%}) exceeds 10%.")
+    else:
+        overfit_status = "NO OVERFITTING"
+        print(f"\n  NO OVERFITTING DETECTED")
+        print(f"  Average gap ({avg_gap:.1%}) is within acceptable range.")
+
+    if acc_std < 0.01:
+        consistency = "HIGHLY CONSISTENT"
+        print(f"  Verdict Consistency: HIGHLY CONSISTENT (std={acc_std:.4f})")
+        print(f"  Predictions are very stable across all 5 folds.")
+    elif acc_std < 0.03:
+        consistency = "CONSISTENT"
+        print(f"  Verdict Consistency: CONSISTENT (std={acc_std:.4f})")
+        print(f"  Minor variance across folds — acceptable for production.")
+    else:
+        consistency = "INCONSISTENT"
+        print(f"  Verdict Consistency: INCONSISTENT (std={acc_std:.4f})")
+        print(f"  High variance suggests model stability issues.")
+
+    # ── 8. Confusion Matrix (aggregated across all folds) ──
+    print("\n\nGenerating plots...")
+    cm = confusion_matrix(all_y_true, all_y_pred)
+    overall_acc = accuracy_score(all_y_true, all_y_pred)
+
+    fig, ax = plt.subplots(figsize=(8, 6))
+    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names)
+    disp.plot(ax=ax, cmap="Blues", values_format="d")
+
+    ax.set_title(
+        f"Confusion Matrix — Aggregated 5-Fold CV\n"
+        f"Overall Accuracy: {overall_acc:.1%} | {overfit_status}",
+        fontsize=14,
+        fontweight="bold",
+    )
+    ax.set_xlabel("Predicted Label", fontsize=12)
+    ax.set_ylabel("True Label", fontsize=12)
+
+    plt.tight_layout()
+    cm_path = os.path.join(OUTPUT_DIR, "confusion_matrix.png")
+    fig.savefig(cm_path, dpi=150, bbox_inches="tight")
+    print(f"  Saved: {cm_path}")
+
+    # ── 9. Per-Fold Accuracy Bar Chart ──
+    fig2, ax2 = plt.subplots(figsize=(10, 5))
+
+    x = np.arange(5)
+    width = 0.35
+    bars_train = ax2.bar(
+        x - width / 2,
+        fold_train_accs * 100,
+        width,
+        label="Training",
+        color="#2196F3",
+        edgecolor="black",
+        linewidth=0.5,
+    )
+    bars_test = ax2.bar(
+        x + width / 2,
+        fold_accuracies * 100,
+        width,
+        label="Testing",
+        color="#FF9800",
+        edgecolor="black",
+        linewidth=0.5,
+    )
+
+    for bar, val in zip(bars_train, fold_train_accs):
+        ax2.text(
+            bar.get_x() + bar.get_width() / 2,
+            bar.get_height() + 0.3,
+            f"{val:.1%}",
+            ha="center",
+            va="bottom",
+            fontsize=9,
+            fontweight="bold",
+        )
+    for bar, val in zip(bars_test, fold_accuracies):
+        ax2.text(
+            bar.get_x() + bar.get_width() / 2,
+            bar.get_height() + 0.3,
+            f"{val:.1%}",
+            ha="center",
+            va="bottom",
+            fontsize=9,
+            fontweight="bold",
+        )
+
+    ax2.set_xticks(x)
+    ax2.set_xticklabels([f"Fold {i}" for i in range(1, 6)])
+    ax2.set_ylim(0, 105)
+    ax2.set_ylabel("Accuracy (%)", fontsize=12)
+    ax2.set_title(
+        f"Per-Fold Accuracy Comparison\n"
+        f"Avg Test: {avg_test:.1%} (+/- {acc_std:.4f}) | {consistency}",
+        fontsize=14,
+        fontweight="bold",
+    )
+    ax2.legend(loc="lower right")
+    ax2.axhline(y=70, color="red", linestyle="--", alpha=0.5, label="70% threshold")
+
+    plt.tight_layout()
+    bar_path = os.path.join(OUTPUT_DIR, "accuracy_comparison.png")
+    fig2.savefig(bar_path, dpi=150, bbox_inches="tight")
+    print(f"  Saved: {bar_path}")
+
+    # ── Final Summary ──
+    print("\n" + "=" * 60)
+    print("  EVALUATION COMPLETE")
+    print("=" * 60)
+    print(f"  Dataset:               fake_news_filipino ({len(df)} articles)")
+    print(f"  Feature set:           {X_full.shape[1]} (TF-IDF + 9 stylometric)")
+    print(f"  Cross-Validation:      5-Fold Stratified")
+    print(f"  Avg Training Accuracy: {avg_train:.4f} (+/- {fold_train_accs.std():.4f})")
+    print(f"  Avg Testing Accuracy:  {avg_test:.4f} (+/- {acc_std:.4f})")
+    print(f"  Avg F1 Score:          {fold_f1s.mean():.4f} (+/- {fold_f1s.std():.4f})")
+    print(f"  Avg Gap:               {avg_gap:.4f}")
+    print(f"  Overfitting Status:    {overfit_status}")
+    print(f"  Verdict Consistency:   {consistency}")
+    print(f"  Plots saved to:        {OUTPUT_DIR}/")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

backend/mine_bias_words.py

@@ -1,194 +1,194 @@
-"""
-Chi-Squared Bias Word Miner
-============================
-Automatically discovers words most statistically associated with
-Fake vs. Real news in your training dataset using chi-squared analysis.
-
-Outputs:
-  1. Full scored table (word, chi2 score, fake freq, real freq)
-  2. Ready-to-paste Python lists for bias_analyzer.py
-
-Usage:
-    python backend/mine_bias_words.py
-"""
-
-import sys
-import os
-import pandas as pd
-from sklearn.feature_extraction.text import CountVectorizer
-from sklearn.feature_selection import chi2
-import numpy as np
-
-PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, PROJECT_ROOT)
-
-TOP_N = 60  # How many top words to show per class
-
-
-def load_data():
-    """Load the same datasets used in training."""
-    frames = []
-
-    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
-    if os.path.exists(csv1):
-        df1 = pd.read_csv(csv1, skiprows=1)[["article", "label"]].copy()
-        df1 = df1[~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
-        # label column may be StringDtype (git conflict markers) — coerce to int
-        df1["label"] = pd.to_numeric(df1["label"], errors="coerce")
-        df1 = df1.dropna(subset=["label"])
-        df1["label"] = df1["label"].astype(int)
-        frames.append(df1)
-        print(
-            f"  jcblaise: {len(df1)} articles (Real={int((df1.label==0).sum())}, Fake={int((df1.label==1).sum())})"
-        )
-
-    csv2 = os.path.join(
-        PROJECT_ROOT,
-        "data",
-        "raw",
-        "philippine_corpus",
-        "Philippine Fake News Corpus.csv",
-    )
-    if os.path.exists(csv2):
-        df2 = pd.read_csv(csv2, skiprows=1)
-        df2 = df2.rename(columns={"Content": "article"})
-        df2["label"] = df2["Label"].map({"Credible": 0, "Not Credible": 1})
-        df2 = df2[["article", "label"]].dropna().copy()
-        frames.append(df2)
-        print(
-            f"  PH Corpus: {len(df2)} articles (Real={int((df2.label==0).sum())}, Fake={int((df2.label==1).sum())})"
-        )
-
-    df = pd.concat(frames, ignore_index=True).dropna(subset=["article", "label"])
-    df = df[df["article"].astype(str).str.len() > 0].copy()
-    return df
-
-
-def mine_words(df, top_n=TOP_N):
-    texts = df["article"].fillna("").astype(str).tolist()
-    labels = df["label"].tolist()
-
-    print(f"\nBuilding vocabulary from {len(texts)} articles...")
-    vec = CountVectorizer(
-        max_features=20000,
-        ngram_range=(1, 2),
-        min_df=5,
-        max_df=0.85,
-        stop_words=None,  # keep Filipino/Tagalog words too
-    )
-    X = vec.fit_transform(texts)
-    feature_names = vec.get_feature_names_out()
-
-    print("Running chi-squared analysis...")
-    chi2_scores, _ = chi2(X, labels)
-    sorted_idx = np.argsort(chi2_scores)[::-1]
-
-    X_array = X.toarray()
-    real_mask = [l == 0 for l in labels]
-    fake_mask = [l == 1 for l in labels]
-    real_mean = X_array[real_mask].mean(axis=0)
-    fake_mean = X_array[fake_mask].mean(axis=0)
-
-    fake_words, real_words = [], []
-    for idx in sorted_idx[: top_n * 3]:
-        word = feature_names[idx]
-        score = chi2_scores[idx]
-        if word.isdigit() or len(word) <= 2:
-            continue
-        entry = (word, score, fake_mean[idx], real_mean[idx])
-        if fake_mean[idx] > real_mean[idx]:
-            fake_words.append(entry)
-        else:
-            real_words.append((word, score, real_mean[idx], fake_mean[idx]))
-        if len(fake_words) >= top_n and len(real_words) >= top_n:
-            break
-
-    return fake_words[:top_n], real_words[:top_n]
-
-
-def _py_list(words, var_name, comment):
-    """Format a word list as a ready-to-paste Python block."""
-    lines = [f"# {comment}", f"{var_name} = ["]
-    for i in range(0, len(words), 5):
-        chunk = words[i : i + 5]
-        lines.append("    " + ", ".join(f'"{w}"' for w in chunk) + ",")
-    lines.append("]")
-    return "\n".join(lines)
-
-
-def main():
-    print("=" * 65)
-    print("  CHI-SQUARED BIAS WORD MINER")
-    print("=" * 65)
-
-    df = load_data()
-    print(
-        f"\nTotal: {len(df)} articles | Real: {int((df.label==0).sum())} | Fake: {int((df.label==1).sum())}"
-    )
-
-    fake_words, real_words = mine_words(df)
-
-    # ── Scored table ───────────────────────────────────────────────────
-    print("\n" + "=" * 65)
-    print(f"  TOP {TOP_N} WORDS ASSOCIATED WITH *** FAKE NEWS ***")
-    print("=" * 65)
-    for word, score, f_avg, r_avg in fake_words:
-        bar = "█" * min(20, int(score / 200))
-        print(f"  {word:<28} χ²={score:>8.0f}  {bar}")
-
-    print("\n" + "=" * 65)
-    print(f"  TOP {TOP_N} WORDS ASSOCIATED WITH *** REAL NEWS ***")
-    print("=" * 65)
-    for word, score, r_avg, f_avg in real_words:
-        bar = "█" * min(20, int(score / 200))
-        print(f"  {word:<28} χ²={score:>8.0f}  {bar}")
-
-    # ── Ready-to-paste blocks ──────────────────────────────────────────
-    fake_terms = [w for w, *_ in fake_words]
-    real_terms = [w for w, *_ in real_words]
-
-    print("\n\n" + "=" * 65)
-    print("  ✂  COPY-PASTE INTO bias_analyzer.py")
-    print("=" * 65)
-
-    print(
-        "\n"
-        + _py_list(
-            fake_terms[:30],
-            "SENSATIONAL_ADDITIONS",
-            "Paste these into SENSATIONAL_KEYWORDS in bias_analyzer.py",
-        )
-    )
-
-    print(
-        "\n"
-        + _py_list(
-            fake_terms[30:],
-            "RIGHT_CANDIDATE_ADDITIONS",
-            "Review — move political terms to CONSERVATIVE_PRO / RIGHT_LEANING_KEYWORDS",
-        )
-    )
-
-    print(
-        "\n"
-        + _py_list(
-            real_terms[:30],
-            "EVIDENCE_ADDITIONS",
-            "Paste into EVIDENCE_BASED_MARKERS in bias_analyzer.py",
-        )
-    )
-
-    print("\n" + "=" * 65)
-    print("  TIPS:")
-    print("  • Words appearing in both tables are not discriminative — skip them")
-    print("  • Move politician names to CONSERVATIVE_PRO / LIBERAL_PRO instead")
-    print("  • For political faction slang, use this LLM prompt:")
-    print()
-    print('  "Give me 40 Filipino/Tagalog/Bisaya social media terms used by')
-    print("   (1) pro-Duterte DDS, (2) Kakampink/pro-Leni, (3) PBBM/pro-Marcos")
-    print('   supporters. Return three plain Python string lists."')
-    print("=" * 65)
-
-
-if __name__ == "__main__":
-    main()
+"""
+Chi-Squared Bias Word Miner
+============================
+Automatically discovers words most statistically associated with
+Fake vs. Real news in your training dataset using chi-squared analysis.
+
+Outputs:
+  1. Full scored table (word, chi2 score, fake freq, real freq)
+  2. Ready-to-paste Python lists for bias_analyzer.py
+
+Usage:
+    python backend/mine_bias_words.py
+"""
+
+import sys
+import os
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.feature_selection import chi2
+import numpy as np
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_ROOT)
+
+TOP_N = 60  # How many top words to show per class
+
+
+def load_data():
+    """Load the same datasets used in training."""
+    frames = []
+
+    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
+    if os.path.exists(csv1):
+        df1 = pd.read_csv(csv1, skiprows=1)[["article", "label"]].copy()
+        df1 = df1[~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
+        # label column may be StringDtype (git conflict markers) — coerce to int
+        df1["label"] = pd.to_numeric(df1["label"], errors="coerce")
+        df1 = df1.dropna(subset=["label"])
+        df1["label"] = df1["label"].astype(int)
+        frames.append(df1)
+        print(
+            f"  jcblaise: {len(df1)} articles (Real={int((df1.label==0).sum())}, Fake={int((df1.label==1).sum())})"
+        )
+
+    csv2 = os.path.join(
+        PROJECT_ROOT,
+        "data",
+        "raw",
+        "philippine_corpus",
+        "Philippine Fake News Corpus.csv",
+    )
+    if os.path.exists(csv2):
+        df2 = pd.read_csv(csv2, skiprows=1)
+        df2 = df2.rename(columns={"Content": "article"})
+        df2["label"] = df2["Label"].map({"Credible": 0, "Not Credible": 1})
+        df2 = df2[["article", "label"]].dropna().copy()
+        frames.append(df2)
+        print(
+            f"  PH Corpus: {len(df2)} articles (Real={int((df2.label==0).sum())}, Fake={int((df2.label==1).sum())})"
+        )
+
+    df = pd.concat(frames, ignore_index=True).dropna(subset=["article", "label"])
+    df = df[df["article"].astype(str).str.len() > 0].copy()
+    return df
+
+
+def mine_words(df, top_n=TOP_N):
+    texts = df["article"].fillna("").astype(str).tolist()
+    labels = df["label"].tolist()
+
+    print(f"\nBuilding vocabulary from {len(texts)} articles...")
+    vec = CountVectorizer(
+        max_features=20000,
+        ngram_range=(1, 2),
+        min_df=5,
+        max_df=0.85,
+        stop_words=None,  # keep Filipino/Tagalog words too
+    )
+    X = vec.fit_transform(texts)
+    feature_names = vec.get_feature_names_out()
+
+    print("Running chi-squared analysis...")
+    chi2_scores, _ = chi2(X, labels)
+    sorted_idx = np.argsort(chi2_scores)[::-1]
+
+    X_array = X.toarray()
+    real_mask = [l == 0 for l in labels]
+    fake_mask = [l == 1 for l in labels]
+    real_mean = X_array[real_mask].mean(axis=0)
+    fake_mean = X_array[fake_mask].mean(axis=0)
+
+    fake_words, real_words = [], []
+    for idx in sorted_idx[: top_n * 3]:
+        word = feature_names[idx]
+        score = chi2_scores[idx]
+        if word.isdigit() or len(word) <= 2:
+            continue
+        entry = (word, score, fake_mean[idx], real_mean[idx])
+        if fake_mean[idx] > real_mean[idx]:
+            fake_words.append(entry)
+        else:
+            real_words.append((word, score, real_mean[idx], fake_mean[idx]))
+        if len(fake_words) >= top_n and len(real_words) >= top_n:
+            break
+
+    return fake_words[:top_n], real_words[:top_n]
+
+
+def _py_list(words, var_name, comment):
+    """Format a word list as a ready-to-paste Python block."""
+    lines = [f"# {comment}", f"{var_name} = ["]
+    for i in range(0, len(words), 5):
+        chunk = words[i : i + 5]
+        lines.append("    " + ", ".join(f'"{w}"' for w in chunk) + ",")
+    lines.append("]")
+    return "\n".join(lines)
+
+
+def main():
+    print("=" * 65)
+    print("  CHI-SQUARED BIAS WORD MINER")
+    print("=" * 65)
+
+    df = load_data()
+    print(
+        f"\nTotal: {len(df)} articles | Real: {int((df.label==0).sum())} | Fake: {int((df.label==1).sum())}"
+    )
+
+    fake_words, real_words = mine_words(df)
+
+    # ── Scored table ───────────────────────────────────────────────────
+    print("\n" + "=" * 65)
+    print(f"  TOP {TOP_N} WORDS ASSOCIATED WITH *** FAKE NEWS ***")
+    print("=" * 65)
+    for word, score, f_avg, r_avg in fake_words:
+        bar = "█" * min(20, int(score / 200))
+        print(f"  {word:<28} χ²={score:>8.0f}  {bar}")
+
+    print("\n" + "=" * 65)
+    print(f"  TOP {TOP_N} WORDS ASSOCIATED WITH *** REAL NEWS ***")
+    print("=" * 65)
+    for word, score, r_avg, f_avg in real_words:
+        bar = "█" * min(20, int(score / 200))
+        print(f"  {word:<28} χ²={score:>8.0f}  {bar}")
+
+    # ── Ready-to-paste blocks ──────────────────────────────────────────
+    fake_terms = [w for w, *_ in fake_words]
+    real_terms = [w for w, *_ in real_words]
+
+    print("\n\n" + "=" * 65)
+    print("  ✂  COPY-PASTE INTO bias_analyzer.py")
+    print("=" * 65)
+
+    print(
+        "\n"
+        + _py_list(
+            fake_terms[:30],
+            "SENSATIONAL_ADDITIONS",
+            "Paste these into SENSATIONAL_KEYWORDS in bias_analyzer.py",
+        )
+    )
+
+    print(
+        "\n"
+        + _py_list(
+            fake_terms[30:],
+            "RIGHT_CANDIDATE_ADDITIONS",
+            "Review — move political terms to CONSERVATIVE_PRO / RIGHT_LEANING_KEYWORDS",
+        )
+    )
+
+    print(
+        "\n"
+        + _py_list(
+            real_terms[:30],
+            "EVIDENCE_ADDITIONS",
+            "Paste into EVIDENCE_BASED_MARKERS in bias_analyzer.py",
+        )
+    )
+
+    print("\n" + "=" * 65)
+    print("  TIPS:")
+    print("  • Words appearing in both tables are not discriminative — skip them")
+    print("  • Move politician names to CONSERVATIVE_PRO / LIBERAL_PRO instead")
+    print("  • For political faction slang, use this LLM prompt:")
+    print()
+    print('  "Give me 40 Filipino/Tagalog/Bisaya social media terms used by')
+    print("   (1) pro-Duterte DDS, (2) Kakampink/pro-Leni, (3) PBBM/pro-Marcos")
+    print('   supporters. Return three plain Python string lists."')
+    print("=" * 65)
+
+
+if __name__ == "__main__":
+    main()

backend/tune_rf.py

@@ -0,0 +1,333 @@
+"""
+Hybrid RF Hyperparameter Tuner
+================================
+Systematically tests different Random Forest configurations on
+the hybrid feature set (TF-IDF + MiniLM + Stylometric) to find
+the settings that beat the Linear SVM baseline (94.89% accuracy).
+
+Varies:
+  • tfidf_features  : number of TF-IDF vocabulary features (2000, 5000, 10000)
+  • max_depth       : RF tree depth (None=unlimited, 30, 50)
+  • max_features_rf : features sampled per split ('sqrt', 'log2', 0.15, 0.25)
+
+Output:
+  evaluation_results/tune_rf_results.txt   — all trial scores
+  evaluation_results/tune_rf_heatmap.png   — accuracy heatmap
+
+Usage:
+  python backend/tune_rf.py
+"""
+
+import sys, os, time, warnings
+warnings.filterwarnings("ignore")
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PROJECT_ROOT)
+
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from scipy.sparse import hstack, csr_matrix
+from sklearn.model_selection import train_test_split
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.preprocessing import StandardScaler
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score, f1_score, roc_auc_score
+
+from backend.train import (
+    load_fake_news_dataset,
+    preprocess,
+    extract_stylometric_features,
+    get_minilm_model,
+    STYLOMETRIC_FEATURE_NAMES,
+)
+
+OUTPUT_DIR = os.path.join(PROJECT_ROOT, "evaluation_results")
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+SVM_BASELINE_ACC = 0.9489   # target to beat
+SVM_BASELINE_F1  = 0.9489
+
+# ── Grid ──────────────────────────────────────────────────────────────────
+TFIDF_SIZES      = [2000, 5000, 10000]
+MAX_DEPTHS       = [None, 30, 50]
+MAX_FEATURES_RF  = ["sqrt", "log2", 0.15, 0.25]
+
+
+def build_hybrid(X_train, X_test, tfidf_size, feature_multiplier=1.0):
+    """Build TF-IDF + MiniLM + Stylometric features.
+    feature_multiplier scales the dense (MiniLM + Stylo) columns so they
+    are not drowned out by the sparse TF-IDF block when max_features is low.
+    """
+    tfidf = TfidfVectorizer(
+        max_features=tfidf_size,
+        ngram_range=(1, 3),
+        min_df=2,
+        max_df=0.95,
+        sublinear_tf=True,
+    )
+    X_tr_tfidf = tfidf.fit_transform(X_train)
+    X_te_tfidf = tfidf.transform(X_test)
+
+    minilm = get_minilm_model()
+    emb_tr = minilm.encode(X_train, show_progress_bar=False, batch_size=64)
+    emb_te = minilm.encode(X_test,  show_progress_bar=False, batch_size=64)
+
+    stylo_tr = np.array([extract_stylometric_features(t) for t in X_train])
+    stylo_te = np.array([extract_stylometric_features(t) for t in X_test])
+    scaler   = StandardScaler()
+    stylo_tr = scaler.fit_transform(stylo_tr)
+    stylo_te = scaler.transform(stylo_te)
+
+    # Scale dense features to give RF a fairer chance of picking them
+    dense_tr = np.hstack([emb_tr, stylo_tr]) * feature_multiplier
+    dense_te = np.hstack([emb_te, stylo_te]) * feature_multiplier
+
+    X_tr = hstack([X_tr_tfidf, csr_matrix(dense_tr)])
+    X_te = hstack([X_te_tfidf, csr_matrix(dense_te)])
+    return X_tr, X_te
+
+
+def main():
+    print("=" * 65)
+    print("  HYBRID RF HYPERPARAMETER TUNER")
+    print(f"  Target to beat: SVM Accuracy={SVM_BASELINE_ACC:.4f}  F1={SVM_BASELINE_F1:.4f}")
+    print("=" * 65)
+
+    # ── Load data once ───────────────────────────────────────────────────
+    print("\nLoading dataset (mixed mode) …")
+    df = load_fake_news_dataset()
+    X_all, y_all = preprocess(df, undersample=False, oversample=True)
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X_all, y_all, test_size=0.20, random_state=42, stratify=y_all
+    )
+    y_train_arr = np.array(y_train)
+    y_test_arr  = np.array(y_test)
+
+    print(f"  Train: {len(X_train):,}  |  Test: {len(X_test):,}")
+
+    # ── Encode MiniLM once for the LARGEST tfidf size run —————————————
+    # We re-encode per tfidf_size but cache embeddings since they don't change
+    print("\nPre-encoding texts with MiniLM (done once, reused across all trials) …")
+    minilm   = get_minilm_model()
+    emb_tr   = minilm.encode(X_train, show_progress_bar=True, batch_size=64)
+    emb_te   = minilm.encode(X_test,  show_progress_bar=True, batch_size=64)
+
+    stylo_tr = np.array([extract_stylometric_features(t) for t in X_train])
+    stylo_te = np.array([extract_stylometric_features(t) for t in X_test])
+    scaler   = StandardScaler()
+    stylo_tr = scaler.fit_transform(stylo_tr)
+    stylo_te = scaler.transform(stylo_te)
+
+    print(f"  MiniLM embeddings: {emb_tr.shape}  |  Stylometric: {stylo_tr.shape}")
+
+    # ── Grid search ──────────────────────────────────────────────────────
+    results = []
+    trial   = 0
+    total   = len(TFIDF_SIZES) * len(MAX_DEPTHS) * len(MAX_FEATURES_RF)
+
+    print(f"\nRunning {total} trials …\n")
+    print(f"  {'#':>3}  {'TFIDF':>6}  {'MaxDepth':>9}  {'MaxFeat':>8}  "
+          f"{'Accuracy':>9}  {'F1(Wtd)':>8}  {'AUC':>7}  {'Time':>6}  Beat?")
+    print("  " + "-" * 75)
+
+    best_acc  = 0.0
+    best_cfg  = {}
+
+    for tfidf_sz in TFIDF_SIZES:
+        # Build TF-IDF for this grid row (embeddings already cached)
+        tfidf    = TfidfVectorizer(
+            max_features=tfidf_sz,
+            ngram_range=(1, 3),
+            min_df=2,
+            max_df=0.95,
+            sublinear_tf=True,
+        )
+        X_tr_tfidf = tfidf.fit_transform(X_train)
+        X_te_tfidf = tfidf.transform(X_test)
+
+        # Combine with cached dense features
+        # Feature multiplier: scale dense block so RF sampling
+        # has a realistic chance of selecting it
+        # Multiplier = sqrt(tfidf_sz / n_dense) approximates equal
+        # representation across sqrt-sampling
+        n_dense     = emb_tr.shape[1] + stylo_tr.shape[1]   # 384+25 = 409
+        multiplier  = max(1.0, (tfidf_sz / n_dense) ** 0.5)
+
+        dense_tr = np.hstack([emb_tr, stylo_tr]) * multiplier
+        dense_te = np.hstack([emb_te, stylo_te]) * multiplier
+
+        X_tr = hstack([X_tr_tfidf, csr_matrix(dense_tr)])
+        X_te = hstack([X_te_tfidf, csr_matrix(dense_te)])
+
+        for max_depth in MAX_DEPTHS:
+            for max_feat in MAX_FEATURES_RF:
+                trial += 1
+                depth_label = str(max_depth) if max_depth else "None"
+
+                t0 = time.time()
+                rf = RandomForestClassifier(
+                    n_estimators=300,
+                    max_depth=max_depth,
+                    min_samples_split=5,
+                    min_samples_leaf=3,
+                    max_features=max_feat,
+                    class_weight="balanced",
+                    n_jobs=-1,
+                    random_state=42,
+                )
+                rf.fit(X_tr, y_train_arr)
+                y_pred   = rf.predict(X_te)
+                proba    = rf.predict_proba(X_te)
+                elapsed  = time.time() - t0
+
+                acc = accuracy_score(y_test_arr, y_pred)
+                f1  = f1_score(y_test_arr, y_pred, average="weighted", zero_division=0)
+                try:
+                    auc = roc_auc_score(y_test_arr, proba[:, 1])
+                except Exception:
+                    auc = float("nan")
+
+                beat   = "✓ BEAT IT!" if acc > SVM_BASELINE_ACC else ""
+                marker = ">>>" if acc > SVM_BASELINE_ACC else "   "
+
+                row = {
+                    "trial":      trial,
+                    "tfidf_sz":   tfidf_sz,
+                    "max_depth":  max_depth,
+                    "max_feat":   max_feat,
+                    "accuracy":   acc,
+                    "f1":         f1,
+                    "auc":        auc,
+                    "time":       elapsed,
+                    "multiplier": multiplier,
+                }
+                results.append(row)
+
+                if acc > best_acc:
+                    best_acc = acc
+                    best_cfg = row.copy()
+
+                mf_str = f"{max_feat:.2f}" if isinstance(max_feat, float) else max_feat
+                print(
+                    f"{marker} {trial:>3}  {tfidf_sz:>6,}  {depth_label:>9}  {mf_str:>8}  "
+                    f"{acc:>9.4f}  {f1:>8.4f}  {auc:>7.4f}  {elapsed:>5.1f}s  {beat}"
+                )
+
+    # ── Summary ──────────────────────────────────────────────────────────
+    print("\n" + "=" * 65)
+    print("  BEST CONFIGURATION FOUND")
+    print("=" * 65)
+    depth_label = str(best_cfg["max_depth"]) if best_cfg["max_depth"] else "None"
+    mf_label    = (f"{best_cfg['max_feat']:.2f}"
+                   if isinstance(best_cfg["max_feat"], float)
+                   else best_cfg["max_feat"])
+    print(f"  TF-IDF features : {best_cfg['tfidf_sz']:,}")
+    print(f"  max_depth       : {depth_label}")
+    print(f"  max_features    : {mf_label}")
+    print(f"  Dense multiplier: {best_cfg['multiplier']:.2f}")
+    print(f"  Accuracy        : {best_cfg['accuracy']:.4f}  "
+          f"({'BEATS SVM ✓' if best_cfg['accuracy'] > SVM_BASELINE_ACC else 'below SVM ✗'})")
+    print(f"  F1 Weighted     : {best_cfg['f1']:.4f}")
+    print(f"  AUC-ROC         : {best_cfg['auc']:.4f}")
+
+    # ── Save text report ─────────────────────────────────────────────────
+    lines  = []
+    lines.append("=" * 65)
+    lines.append("  HYBRID RF TUNING RESULTS")
+    lines.append(f"  Target: SVM Acc={SVM_BASELINE_ACC:.4f}  F1={SVM_BASELINE_F1:.4f}")
+    lines.append("=" * 65)
+    lines.append(f"\n  {'#':>3}  {'TFIDF':>6}  {'MaxDepth':>9}  {'MaxFeat':>8}  "
+                 f"{'Accuracy':>9}  {'F1(Wtd)':>8}  {'AUC':>7}  {'Time':>6}")
+    lines.append("  " + "-" * 70)
+    for r in results:
+        d  = str(r["max_depth"]) if r["max_depth"] else "None"
+        mf = f"{r['max_feat']:.2f}" if isinstance(r["max_feat"], float) else r["max_feat"]
+        b  = "  ← BEST" if r == best_cfg else ""
+        lines.append(
+            f"  {r['trial']:>3}  {r['tfidf_sz']:>6,}  {d:>9}  {mf:>8}  "
+            f"{r['accuracy']:>9.4f}  {r['f1']:>8.4f}  {r['auc']:>7.4f}  "
+            f"{r['time']:>5.1f}s{b}"
+        )
+
+    lines.append("\n" + "=" * 65)
+    lines.append("  RECOMMENDATION FOR train.py")
+    lines.append("=" * 65)
+    depth_label = str(best_cfg["max_depth"]) if best_cfg["max_depth"] else "None (unlimited)"
+    mf_label    = (f"{best_cfg['max_feat']:.2f}"
+                   if isinstance(best_cfg["max_feat"], float)
+                   else f"'{best_cfg['max_feat']}'")
+    lines.append(f"""
+  In backend/train.py → build_features():
+    TfidfVectorizer(max_features={best_cfg['tfidf_sz']}, ...)
+
+  In backend/train.py → train_model():
+    RandomForestClassifier(
+        n_estimators=500,
+        max_depth={best_cfg['max_depth']},
+        min_samples_split=5,
+        min_samples_leaf=3,
+        max_features={mf_label},
+        class_weight='balanced',
+        n_jobs=-1,
+        random_state=42,
+    )
+""")
+
+    txt = "\n".join(lines)
+    print(txt)
+    out_txt = os.path.join(OUTPUT_DIR, "tune_rf_results.txt")
+    with open(out_txt, "w", encoding="utf-8") as f:
+        f.write(txt)
+    print(f"\n  Saved: {out_txt}")
+
+    # ── Heatmap: Accuracy by (tfidf_size x max_depth) for best max_feat ──
+    try:
+        best_mf    = best_cfg["max_feat"]
+        sub        = [r for r in results if r["max_feat"] == best_mf]
+        depths_u   = [None, 30, 50]
+        tfidf_u    = TFIDF_SIZES
+        grid       = np.zeros((len(depths_u), len(tfidf_u)))
+        for r in sub:
+            di = depths_u.index(r["max_depth"])
+            ti = tfidf_u.index(r["tfidf_sz"])
+            grid[di, ti] = r["accuracy"]
+
+        fig, ax = plt.subplots(figsize=(8, 5))
+        im = ax.imshow(grid, cmap="RdYlGn", vmin=0.85, vmax=1.0)
+        ax.set_xticks(range(len(tfidf_u)))
+        ax.set_xticklabels([f"{t:,}" for t in tfidf_u])
+        ax.set_yticks(range(len(depths_u)))
+        ax.set_yticklabels(["None (unlimited)", "30", "50"])
+        ax.set_xlabel("TF-IDF Vocabulary Size", fontsize=12)
+        ax.set_ylabel("max_depth", fontsize=12)
+        mf_str = f"{best_mf:.2f}" if isinstance(best_mf, float) else best_mf
+        ax.set_title(
+            f"Hybrid RF Accuracy Heatmap  (max_features='{mf_str}')\n"
+            f"Red < {SVM_BASELINE_ACC:.0%} = below SVM baseline   Green = above SVM baseline",
+            fontsize=11, fontweight="bold"
+        )
+        # Annotate cells
+        for i in range(len(depths_u)):
+            for j in range(len(tfidf_u)):
+                v = grid[i, j]
+                ax.text(j, i, f"{v:.4f}", ha="center", va="center",
+                        fontsize=11, fontweight="bold",
+                        color="white" if v < 0.90 else "black")
+        fig.colorbar(im, ax=ax, label="Accuracy")
+        plt.axhline(-0.5, color="gray")
+        plt.tight_layout()
+        hm_path = os.path.join(OUTPUT_DIR, "tune_rf_heatmap.png")
+        fig.savefig(hm_path, dpi=150, bbox_inches="tight")
+        plt.close(fig)
+        print(f"  Saved: {hm_path}")
+    except Exception as e:
+        print(f"  (Heatmap skipped: {e})")
+
+    print("\n  All done. Check evaluation_results/ for the full report.")
+
+
+if __name__ == "__main__":
+    main()

check_app/.flutter-plugins-dependencies

@@ -0,0 +1 @@
+{"info":"This is a generated file; do not edit or check into version control.","plugins":{"ios":[{"name":"path_provider_foundation","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\path_provider_foundation-2.6.0\\\\","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"shared_preferences_foundation","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\shared_preferences_foundation-2.5.6\\\\","shared_darwin_source":true,"native_build":true,"dependencies":[],"dev_dependency":false},{"name":"url_launcher_ios","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\url_launcher_ios-6.4.1\\\\","native_build":true,"dependencies":[],"dev_dependency":false}],"android":[{"name":"jni","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\jni-1.0.0\\\\","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"jni_flutter","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\jni_flutter-1.0.1\\\\","native_build":true,"dependencies":["jni"],"dev_dependency":false},{"name":"path_provider_android","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\path_provider_android-2.3.1\\\\","native_build":false,"dependencies":["jni","jni_flutter"],"dev_dependency":false},{"name":"shared_preferences_android","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\shared_preferences_android-2.4.21\\\\","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"url_launcher_android","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\url_launcher_android-6.3.28\\\\","native_build":true,"dependencies":[],"dev_dependency":false}],"macos":[{"name":"path_provider_foundation","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\path_provider_foundation-2.6.0\\\\","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"shared_preferences_foundation","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\shared_preferences_foundation-2.5.6\\\\","shared_darwin_source":true,"native_build":true,"dependencies":[],"dev_dependency":false},{"name":"url_launcher_macos","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\url_launcher_macos-3.2.5\\\\","native_build":true,"dependencies":[],"dev_dependency":false}],"linux":[{"name":"jni","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\jni-1.0.0\\\\","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"path_provider_linux","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\path_provider_linux-2.2.1\\\\","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"shared_preferences_linux","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\shared_preferences_linux-2.4.1\\\\","native_build":false,"dependencies":["path_provider_linux"],"dev_dependency":false},{"name":"url_launcher_linux","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\url_launcher_linux-3.2.2\\\\","native_build":true,"dependencies":[],"dev_dependency":false}],"windows":[{"name":"jni","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\jni-1.0.0\\\\","native_build":true,"dependencies":[],"dev_dependency":false},{"name":"path_provider_windows","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\path_provider_windows-2.3.0\\\\","native_build":false,"dependencies":[],"dev_dependency":false},{"name":"shared_preferences_windows","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\shared_preferences_windows-2.4.1\\\\","native_build":false,"dependencies":["path_provider_windows"],"dev_dependency":false},{"name":"url_launcher_windows","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\url_launcher_windows-3.1.5\\\\","native_build":true,"dependencies":[],"dev_dependency":false}],"web":[{"name":"shared_preferences_web","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\shared_preferences_web-2.4.3\\\\","dependencies":[],"dev_dependency":false},{"name":"url_launcher_web","path":"C:\\\\Users\\\\Carl P\\\\AppData\\\\Local\\\\Pub\\\\Cache\\\\hosted\\\\pub.dev\\\\url_launcher_web-2.4.2\\\\","dependencies":[],"dev_dependency":false}]},"dependencyGraph":[{"name":"jni","dependencies":[]},{"name":"jni_flutter","dependencies":["jni"]},{"name":"path_provider","dependencies":["path_provider_android","path_provider_foundation","path_provider_linux","path_provider_windows"]},{"name":"path_provider_android","dependencies":["jni","jni_flutter"]},{"name":"path_provider_foundation","dependencies":[]},{"name":"path_provider_linux","dependencies":[]},{"name":"path_provider_windows","dependencies":[]},{"name":"shared_preferences","dependencies":["shared_preferences_android","shared_preferences_foundation","shared_preferences_linux","shared_preferences_web","shared_preferences_windows"]},{"name":"shared_preferences_android","dependencies":[]},{"name":"shared_preferences_foundation","dependencies":[]},{"name":"shared_preferences_linux","dependencies":["path_provider_linux"]},{"name":"shared_preferences_web","dependencies":[]},{"name":"shared_preferences_windows","dependencies":["path_provider_windows"]},{"name":"url_launcher","dependencies":["url_launcher_android","url_launcher_ios","url_launcher_linux","url_launcher_macos","url_launcher_web","url_launcher_windows"]},{"name":"url_launcher_android","dependencies":[]},{"name":"url_launcher_ios","dependencies":[]},{"name":"url_launcher_linux","dependencies":[]},{"name":"url_launcher_macos","dependencies":[]},{"name":"url_launcher_web","dependencies":[]},{"name":"url_launcher_windows","dependencies":[]}],"date_created":"2026-04-10 23:20:38.632272","version":"3.41.2","swift_package_manager_enabled":{"ios":false,"macos":false}}

check_app/.gitignore

@@ -1,45 +1,45 @@
-# Miscellaneous
-*.class
-*.log
-*.pyc
-*.swp
-.DS_Store
-.atom/
-.build/
-.buildlog/
-.history
-.svn/
-.swiftpm/
-migrate_working_dir/
-
-# IntelliJ related
-*.iml
-*.ipr
-*.iws
-.idea/
-
-# The .vscode folder contains launch configuration and tasks you configure in
-# VS Code which you may wish to be included in version control, so this line
-# is commented out by default.
-#.vscode/
-
-# Flutter/Dart/Pub related
-**/doc/api/
-**/ios/Flutter/.last_build_id
-.dart_tool/
-.flutter-plugins-dependencies
-.pub-cache/
-.pub/
-/build/
-/coverage/
-
-# Symbolication related
-app.*.symbols
-
-# Obfuscation related
-app.*.map.json
-
-# Android Studio will place build artifacts here
-/android/app/debug
-/android/app/profile
-/android/app/release
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.build/
+.buildlog/
+.history
+.svn/
+.swiftpm/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+**/doc/api/
+**/ios/Flutter/.last_build_id
+.dart_tool/
+.flutter-plugins-dependencies
+.pub-cache/
+.pub/
+/build/
+/coverage/
+
+# Symbolication related
+app.*.symbols
+
+# Obfuscation related
+app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release

check_app/.metadata

@@ -1,45 +1,45 @@
-# This file tracks properties of this Flutter project.
-# Used by Flutter tool to assess capabilities and perform upgrades etc.
-#
-# This file should be version controlled and should not be manually edited.
-
-version:
-  revision: "90673a4eef275d1a6692c26ac80d6d746d41a73a"
-  channel: "stable"
-
-project_type: app
-
-# Tracks metadata for the flutter migrate command
-migration:
-  platforms:
-    - platform: root
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-    - platform: android
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-    - platform: ios
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-    - platform: linux
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-    - platform: macos
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-    - platform: web
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-    - platform: windows
-      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
-
-  # User provided section
-
-  # List of Local paths (relative to this file) that should be
-  # ignored by the migrate tool.
-  #
-  # Files that are not part of the templates will be ignored by default.
-  unmanaged_files:
-    - 'lib/main.dart'
-    - 'ios/Runner.xcodeproj/project.pbxproj'
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: "90673a4eef275d1a6692c26ac80d6d746d41a73a"
+  channel: "stable"
+
+project_type: app
+
+# Tracks metadata for the flutter migrate command
+migration:
+  platforms:
+    - platform: root
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+    - platform: android
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+    - platform: ios
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+    - platform: linux
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+    - platform: macos
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+    - platform: web
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+    - platform: windows
+      create_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+      base_revision: 90673a4eef275d1a6692c26ac80d6d746d41a73a
+
+  # User provided section
+
+  # List of Local paths (relative to this file) that should be
+  # ignored by the migrate tool.
+  #
+  # Files that are not part of the templates will be ignored by default.
+  unmanaged_files:
+    - 'lib/main.dart'
+    - 'ios/Runner.xcodeproj/project.pbxproj'

check_app/README.md

@@ -1,17 +1,17 @@
-# check_app
-
-A new Flutter project.
-
-## Getting Started
-
-This project is a starting point for a Flutter application.
-
-A few resources to get you started if this is your first Flutter project:
-
-- [Learn Flutter](https://docs.flutter.dev/get-started/learn-flutter)
-- [Write your first Flutter app](https://docs.flutter.dev/get-started/codelab)
-- [Flutter learning resources](https://docs.flutter.dev/reference/learning-resources)
-
-For help getting started with Flutter development, view the
-[online documentation](https://docs.flutter.dev/), which offers tutorials,
-samples, guidance on mobile development, and a full API reference.
+# check_app
+
+A new Flutter project.
+
+## Getting Started
+
+This project is a starting point for a Flutter application.
+
+A few resources to get you started if this is your first Flutter project:
+
+- [Learn Flutter](https://docs.flutter.dev/get-started/learn-flutter)
+- [Write your first Flutter app](https://docs.flutter.dev/get-started/codelab)
+- [Flutter learning resources](https://docs.flutter.dev/reference/learning-resources)
+
+For help getting started with Flutter development, view the
+[online documentation](https://docs.flutter.dev/), which offers tutorials,
+samples, guidance on mobile development, and a full API reference.

check_app/analysis_options.yaml

@@ -1,28 +1,28 @@
-# This file configures the analyzer, which statically analyzes Dart code to
-# check for errors, warnings, and lints.
-#
-# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
-# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
-# invoked from the command line by running `flutter analyze`.
-
-# The following line activates a set of recommended lints for Flutter apps,
-# packages, and plugins designed to encourage good coding practices.
-include: package:flutter_lints/flutter.yaml
-
-linter:
-  # The lint rules applied to this project can be customized in the
-  # section below to disable rules from the `package:flutter_lints/flutter.yaml`
-  # included above or to enable additional rules. A list of all available lints
-  # and their documentation is published at https://dart.dev/lints.
-  #
-  # Instead of disabling a lint rule for the entire project in the
-  # section below, it can also be suppressed for a single line of code
-  # or a specific dart file by using the `// ignore: name_of_lint` and
-  # `// ignore_for_file: name_of_lint` syntax on the line or in the file
-  # producing the lint.
-  rules:
-    # avoid_print: false  # Uncomment to disable the `avoid_print` rule
-    # prefer_single_quotes: true  # Uncomment to enable the `prefer_single_quotes` rule
-
-# Additional information about this file can be found at
-# https://dart.dev/guides/language/analysis-options
+# This file configures the analyzer, which statically analyzes Dart code to
+# check for errors, warnings, and lints.
+#
+# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
+# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
+# invoked from the command line by running `flutter analyze`.
+
+# The following line activates a set of recommended lints for Flutter apps,
+# packages, and plugins designed to encourage good coding practices.
+include: package:flutter_lints/flutter.yaml
+
+linter:
+  # The lint rules applied to this project can be customized in the
+  # section below to disable rules from the `package:flutter_lints/flutter.yaml`
+  # included above or to enable additional rules. A list of all available lints
+  # and their documentation is published at https://dart.dev/lints.
+  #
+  # Instead of disabling a lint rule for the entire project in the
+  # section below, it can also be suppressed for a single line of code
+  # or a specific dart file by using the `// ignore: name_of_lint` and
+  # `// ignore_for_file: name_of_lint` syntax on the line or in the file
+  # producing the lint.
+  rules:
+    # avoid_print: false  # Uncomment to disable the `avoid_print` rule
+    # prefer_single_quotes: true  # Uncomment to enable the `prefer_single_quotes` rule
+
+# Additional information about this file can be found at
+# https://dart.dev/guides/language/analysis-options

check_app/android/.gitignore

@@ -1,14 +1,14 @@
-gradle-wrapper.jar
-/.gradle
-/captures/
-/gradlew
-/gradlew.bat
-/local.properties
-GeneratedPluginRegistrant.java
-.cxx/
-
-# Remember to never publicly share your keystore.
-# See https://flutter.dev/to/reference-keystore
-key.properties
-**/*.keystore
-**/*.jks
+gradle-wrapper.jar
+/.gradle
+/captures/
+/gradlew
+/gradlew.bat
+/local.properties
+GeneratedPluginRegistrant.java
+.cxx/
+
+# Remember to never publicly share your keystore.
+# See https://flutter.dev/to/reference-keystore
+key.properties
+**/*.keystore
+**/*.jks

check_app/android/.gradle/8.14/checksums/sha1-checksums.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f3080ceac71cb4dc7d292b0ec33179e5e7d6de81a9d50c323e33c721005b474
+size 126215

check_app/android/.gradle/8.14/executionHistory/executionHistory.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf6ec48baf09b431f11f0fcc117503713cd528880d5c544427a025f443cd4198
+size 7752405

check_app/android/.gradle/8.14/fileHashes/fileHashes.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d360ea65f31f6b8ea2d31eac54024c603025ba1ffca7845faea655d8fca82819
+size 518941

check_app/android/.gradle/buildOutputCleanup/cache.properties

@@ -0,0 +1,2 @@
+#Sun Mar 08 00:19:03 CST 2026
+gradle.version=8.14

check_app/android/app/build.gradle.kts

@@ -1,44 +1,44 @@
-plugins {
-    id("com.android.application")
-    id("kotlin-android")
-    // The Flutter Gradle Plugin must be applied after the Android and Kotlin Gradle plugins.
-    id("dev.flutter.flutter-gradle-plugin")
-}
-
-android {
-    namespace = "com.example.check_app"
-    compileSdk = flutter.compileSdkVersion
-    ndkVersion = flutter.ndkVersion
-
-    compileOptions {
-        sourceCompatibility = JavaVersion.VERSION_17
-        targetCompatibility = JavaVersion.VERSION_17
-    }
-
-    kotlinOptions {
-        jvmTarget = JavaVersion.VERSION_17.toString()
-    }
-
-    defaultConfig {
-        // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).
-        applicationId = "com.example.check_app"
-        // You can update the following values to match your application needs.
-        // For more information, see: https://flutter.dev/to/review-gradle-config.
-        minSdk = flutter.minSdkVersion
-        targetSdk = flutter.targetSdkVersion
-        versionCode = flutter.versionCode
-        versionName = flutter.versionName
-    }
-
-    buildTypes {
-        release {
-            // TODO: Add your own signing config for the release build.
-            // Signing with the debug keys for now, so `flutter run --release` works.
-            signingConfig = signingConfigs.getByName("debug")
-        }
-    }
-}
-
-flutter {
-    source = "../.."
-}
+plugins {
+    id("com.android.application")
+    id("kotlin-android")
+    // The Flutter Gradle Plugin must be applied after the Android and Kotlin Gradle plugins.
+    id("dev.flutter.flutter-gradle-plugin")
+}
+
+android {
+    namespace = "com.bantaypahayag.app"
+    compileSdk = flutter.compileSdkVersion
+    ndkVersion = flutter.ndkVersion
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_17
+        targetCompatibility = JavaVersion.VERSION_17
+    }
+
+    kotlinOptions {
+        jvmTarget = JavaVersion.VERSION_17.toString()
+    }
+
+    defaultConfig {
+        // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).
+        applicationId = "com.bantaypahayag.app"
+        // You can update the following values to match your application needs.
+        // For more information, see: https://flutter.dev/to/review-gradle-config.
+        minSdk = flutter.minSdkVersion
+        targetSdk = flutter.targetSdkVersion
+        versionCode = flutter.versionCode
+        versionName = flutter.versionName
+    }
+
+    buildTypes {
+        release {
+            // TODO: Add your own signing config for the release build.
+            // Signing with the debug keys for now, so `flutter run --release` works.
+            signingConfig = signingConfigs.getByName("debug")
+        }
+    }
+}
+
+flutter {
+    source = "../.."
+}

check_app/android/app/src/debug/AndroidManifest.xml

@@ -1,7 +1,7 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android">
-    <!-- The INTERNET permission is required for development. Specifically,
-         the Flutter tool needs it to communicate with the running application
-         to allow setting breakpoints, to provide hot reload, etc.
-    -->
-    <uses-permission android:name="android.permission.INTERNET"/>
-</manifest>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <!-- The INTERNET permission is required for development. Specifically,
+         the Flutter tool needs it to communicate with the running application
+         to allow setting breakpoints, to provide hot reload, etc.
+    -->
+    <uses-permission android:name="android.permission.INTERNET"/>
+</manifest>

check_app/android/app/src/main/AndroidManifest.xml

@@ -1,45 +1,46 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android">
-    <application
-        android:label="BantayPahayag"
-        android:name="${applicationName}"
-        android:icon="@mipmap/ic_launcher">
-        <activity
-            android:name=".MainActivity"
-            android:exported="true"
-            android:launchMode="singleTop"
-            android:taskAffinity=""
-            android:theme="@style/LaunchTheme"
-            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode"
-            android:hardwareAccelerated="true"
-            android:windowSoftInputMode="adjustResize">
-            <!-- Specifies an Android theme to apply to this Activity as soon as
-                 the Android process has started. This theme is visible to the user
-                 while the Flutter UI initializes. After that, this theme continues
-                 to determine the Window background behind the Flutter UI. -->
-            <meta-data
-              android:name="io.flutter.embedding.android.NormalTheme"
-              android:resource="@style/NormalTheme"
-              />
-            <intent-filter>
-                <action android:name="android.intent.action.MAIN"/>
-                <category android:name="android.intent.category.LAUNCHER"/>
-            </intent-filter>
-        </activity>
-        <!-- Don't delete the meta-data below.
-             This is used by the Flutter tool to generate GeneratedPluginRegistrant.java -->
-        <meta-data
-            android:name="flutterEmbedding"
-            android:value="2" />
-    </application>
-    <!-- Required to query activities that can process text, see:
-         https://developer.android.com/training/package-visibility and
-         https://developer.android.com/reference/android/content/Intent#ACTION_PROCESS_TEXT.
-
-         In particular, this is used by the Flutter engine in io.flutter.plugin.text.ProcessTextPlugin. -->
-    <queries>
-        <intent>
-            <action android:name="android.intent.action.PROCESS_TEXT"/>
-            <data android:mimeType="text/plain"/>
-        </intent>
-    </queries>
-</manifest>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <uses-permission android:name="android.permission.INTERNET"/>
+    <application
+        android:label="BantayPahayag"
+        android:name="${applicationName}"
+        android:icon="@mipmap/ic_launcher">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:launchMode="singleTop"
+            android:taskAffinity=""
+            android:theme="@style/LaunchTheme"
+            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode"
+            android:hardwareAccelerated="true"
+            android:windowSoftInputMode="adjustResize">
+            <!-- Specifies an Android theme to apply to this Activity as soon as
+                 the Android process has started. This theme is visible to the user
+                 while the Flutter UI initializes. After that, this theme continues
+                 to determine the Window background behind the Flutter UI. -->
+            <meta-data
+              android:name="io.flutter.embedding.android.NormalTheme"
+              android:resource="@style/NormalTheme"
+              />
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN"/>
+                <category android:name="android.intent.category.LAUNCHER"/>
+            </intent-filter>
+        </activity>
+        <!-- Don't delete the meta-data below.
+             This is used by the Flutter tool to generate GeneratedPluginRegistrant.java -->
+        <meta-data
+            android:name="flutterEmbedding"
+            android:value="2" />
+    </application>
+    <!-- Required to query activities that can process text, see:
+         https://developer.android.com/training/package-visibility and
+         https://developer.android.com/reference/android/content/Intent#ACTION_PROCESS_TEXT.
+
+         In particular, this is used by the Flutter engine in io.flutter.plugin.text.ProcessTextPlugin. -->
+    <queries>
+        <intent>
+            <action android:name="android.intent.action.PROCESS_TEXT"/>
+            <data android:mimeType="text/plain"/>
+        </intent>
+    </queries>
+</manifest>

check_app/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java

@@ -0,0 +1,39 @@
+package io.flutter.plugins;
+
+import androidx.annotation.Keep;
+import androidx.annotation.NonNull;
+import io.flutter.Log;
+
+import io.flutter.embedding.engine.FlutterEngine;
+
+/**
+ * Generated file. Do not edit.
+ * This file is generated by the Flutter tool based on the
+ * plugins that support the Android platform.
+ */
+@Keep
+public final class GeneratedPluginRegistrant {
+  private static final String TAG = "GeneratedPluginRegistrant";
+  public static void registerWith(@NonNull FlutterEngine flutterEngine) {
+    try {
+      flutterEngine.getPlugins().add(new com.github.dart_lang.jni.JniPlugin());
+    } catch (Exception e) {
+      Log.e(TAG, "Error registering plugin jni, com.github.dart_lang.jni.JniPlugin", e);
+    }
+    try {
+      flutterEngine.getPlugins().add(new com.github.dart_lang.jni_flutter.JniFlutterPlugin());
+    } catch (Exception e) {
+      Log.e(TAG, "Error registering plugin jni_flutter, com.github.dart_lang.jni_flutter.JniFlutterPlugin", e);
+    }
+    try {
+      flutterEngine.getPlugins().add(new io.flutter.plugins.sharedpreferences.SharedPreferencesPlugin());
+    } catch (Exception e) {
+      Log.e(TAG, "Error registering plugin shared_preferences_android, io.flutter.plugins.sharedpreferences.SharedPreferencesPlugin", e);
+    }
+    try {
+      flutterEngine.getPlugins().add(new io.flutter.plugins.urllauncher.UrlLauncherPlugin());
+    } catch (Exception e) {
+      Log.e(TAG, "Error registering plugin url_launcher_android, io.flutter.plugins.urllauncher.UrlLauncherPlugin", e);
+    }
+  }
+}

check_app/android/app/src/main/kotlin/com/bantaypahayag/app/MainActivity.kt

@@ -0,0 +1,5 @@
+package com.bantaypahayag.app
+
+import io.flutter.embedding.android.FlutterActivity
+
+class MainActivity : FlutterActivity()

check_app/android/app/src/main/res/drawable-v21/launch_background.xml

@@ -1,12 +1,12 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Modify this file to customize your launch splash screen -->
-<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
-    <item android:drawable="?android:colorBackground" />
-
-    <!-- You can insert your own image assets here -->
-    <!-- <item>
-        <bitmap
-            android:gravity="center"
-            android:src="@mipmap/launch_image" />
-    </item> -->
-</layer-list>
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Modify this file to customize your launch splash screen -->
+<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
+    <item android:drawable="?android:colorBackground" />
+
+    <!-- You can insert your own image assets here -->
+    <!-- <item>
+        <bitmap
+            android:gravity="center"
+            android:src="@mipmap/launch_image" />
+    </item> -->
+</layer-list>

check_app/android/app/src/main/res/drawable/launch_background.xml

@@ -1,12 +1,12 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Modify this file to customize your launch splash screen -->
-<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
-    <item android:drawable="@android:color/white" />
-
-    <!-- You can insert your own image assets here -->
-    <!-- <item>
-        <bitmap
-            android:gravity="center"
-            android:src="@mipmap/launch_image" />
-    </item> -->
-</layer-list>
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Modify this file to customize your launch splash screen -->
+<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
+    <item android:drawable="@android:color/white" />
+
+    <!-- You can insert your own image assets here -->
+    <!-- <item>
+        <bitmap
+            android:gravity="center"
+            android:src="@mipmap/launch_image" />
+    </item> -->
+</layer-list>

check_app/android/app/src/main/res/values-night/styles.xml

@@ -1,18 +1,18 @@
-<?xml version="1.0" encoding="utf-8"?>
-<resources>
-    <!-- Theme applied to the Android Window while the process is starting when the OS's Dark Mode setting is on -->
-    <style name="LaunchTheme" parent="@android:style/Theme.Black.NoTitleBar">
-        <!-- Show a splash screen on the activity. Automatically removed when
-             the Flutter engine draws its first frame -->
-        <item name="android:windowBackground">@drawable/launch_background</item>
-    </style>
-    <!-- Theme applied to the Android Window as soon as the process has started.
-         This theme determines the color of the Android Window while your
-         Flutter UI initializes, as well as behind your Flutter UI while its
-         running.
-
-         This Theme is only used starting with V2 of Flutter's Android embedding. -->
-    <style name="NormalTheme" parent="@android:style/Theme.Black.NoTitleBar">
-        <item name="android:windowBackground">?android:colorBackground</item>
-    </style>
-</resources>
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <!-- Theme applied to the Android Window while the process is starting when the OS's Dark Mode setting is on -->
+    <style name="LaunchTheme" parent="@android:style/Theme.Black.NoTitleBar">
+        <!-- Show a splash screen on the activity. Automatically removed when
+             the Flutter engine draws its first frame -->
+        <item name="android:windowBackground">@drawable/launch_background</item>
+    </style>
+    <!-- Theme applied to the Android Window as soon as the process has started.
+         This theme determines the color of the Android Window while your
+         Flutter UI initializes, as well as behind your Flutter UI while its
+         running.
+
+         This Theme is only used starting with V2 of Flutter's Android embedding. -->
+    <style name="NormalTheme" parent="@android:style/Theme.Black.NoTitleBar">
+        <item name="android:windowBackground">?android:colorBackground</item>
+    </style>
+</resources>

check_app/android/app/src/main/res/values/styles.xml

@@ -1,18 +1,18 @@
-<?xml version="1.0" encoding="utf-8"?>
-<resources>
-    <!-- Theme applied to the Android Window while the process is starting when the OS's Dark Mode setting is off -->
-    <style name="LaunchTheme" parent="@android:style/Theme.Light.NoTitleBar">
-        <!-- Show a splash screen on the activity. Automatically removed when
-             the Flutter engine draws its first frame -->
-        <item name="android:windowBackground">@drawable/launch_background</item>
-    </style>
-    <!-- Theme applied to the Android Window as soon as the process has started.
-         This theme determines the color of the Android Window while your
-         Flutter UI initializes, as well as behind your Flutter UI while its
-         running.
-
-         This Theme is only used starting with V2 of Flutter's Android embedding. -->
-    <style name="NormalTheme" parent="@android:style/Theme.Light.NoTitleBar">
-        <item name="android:windowBackground">?android:colorBackground</item>
-    </style>
-</resources>
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <!-- Theme applied to the Android Window while the process is starting when the OS's Dark Mode setting is off -->
+    <style name="LaunchTheme" parent="@android:style/Theme.Light.NoTitleBar">
+        <!-- Show a splash screen on the activity. Automatically removed when
+             the Flutter engine draws its first frame -->
+        <item name="android:windowBackground">@drawable/launch_background</item>
+    </style>
+    <!-- Theme applied to the Android Window as soon as the process has started.
+         This theme determines the color of the Android Window while your
+         Flutter UI initializes, as well as behind your Flutter UI while its
+         running.
+
+         This Theme is only used starting with V2 of Flutter's Android embedding. -->
+    <style name="NormalTheme" parent="@android:style/Theme.Light.NoTitleBar">
+        <item name="android:windowBackground">?android:colorBackground</item>
+    </style>
+</resources>

check_app/android/app/src/profile/AndroidManifest.xml

@@ -1,7 +1,7 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android">
-    <!-- The INTERNET permission is required for development. Specifically,
-         the Flutter tool needs it to communicate with the running application
-         to allow setting breakpoints, to provide hot reload, etc.
-    -->
-    <uses-permission android:name="android.permission.INTERNET"/>
-</manifest>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <!-- The INTERNET permission is required for development. Specifically,
+         the Flutter tool needs it to communicate with the running application
+         to allow setting breakpoints, to provide hot reload, etc.
+    -->
+    <uses-permission android:name="android.permission.INTERNET"/>
+</manifest>

check_app/android/build.gradle.kts

@@ -1,24 +1,24 @@
-allprojects {
-    repositories {
-        google()
-        mavenCentral()
-    }
-}
-
-val newBuildDir: Directory =
-    rootProject.layout.buildDirectory
-        .dir("../../build")
-        .get()
-rootProject.layout.buildDirectory.value(newBuildDir)
-
-subprojects {
-    val newSubprojectBuildDir: Directory = newBuildDir.dir(project.name)
-    project.layout.buildDirectory.value(newSubprojectBuildDir)
-}
-subprojects {
-    project.evaluationDependsOn(":app")
-}
-
-tasks.register<Delete>("clean") {
-    delete(rootProject.layout.buildDirectory)
-}
+allprojects {
+    repositories {
+        google()
+        mavenCentral()
+    }
+}
+
+val newBuildDir: Directory =
+    rootProject.layout.buildDirectory
+        .dir("../../build")
+        .get()
+rootProject.layout.buildDirectory.value(newBuildDir)
+
+subprojects {
+    val newSubprojectBuildDir: Directory = newBuildDir.dir(project.name)
+    project.layout.buildDirectory.value(newSubprojectBuildDir)
+}
+subprojects {
+    project.evaluationDependsOn(":app")
+}
+
+tasks.register<Delete>("clean") {
+    delete(rootProject.layout.buildDirectory)
+}

check_app/android/gradle.properties

@@ -1,2 +1,2 @@
-org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=4G -XX:ReservedCodeCacheSize=512m -XX:+HeapDumpOnOutOfMemoryError
-android.useAndroidX=true
+org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=4G -XX:ReservedCodeCacheSize=512m -XX:+HeapDumpOnOutOfMemoryError
+android.useAndroidX=true

check_app/android/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip