backend/analyze_languages.py

"""
backend/analyze_languages.py
============================
Analyzes the language distribution of every training dataset and prints a
summary table broken down by dataset and language.

Uses langdetect (already pulled in transitively; install with `pip install langdetect`
if missing).

Usage:
    python backend/analyze_languages.py
"""

import sys
import os
import random
from collections import Counter

import pandas as pd

PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, PROJECT_ROOT)

# ── Try importing langdetect ──────────────────────────────────────────────────
try:
    from langdetect import detect, LangDetectException
    from langdetect import DetectorFactory

    DetectorFactory.seed = 42  # make detection deterministic
    HAS_LANGDETECT = True
except ImportError:
    HAS_LANGDETECT = False
    print("WARNING: langdetect not installed. Run: pip install langdetect")
    print("         Falling back to heuristic detection only.\n")


# ── Simple Tagalog heuristic (fast, used as a sanity check) ──────────────────
_TAGALOG_MARKERS = {
    "ang",
    "ng",
    "mga",
    "sa",
    "na",
    "ay",
    "at",
    "hindi",
    "ako",
    "siya",
    "nila",
    "niya",
    "ito",
    "iyon",
    "kung",
    "para",
    "nang",
    "din",
    "rin",
    "kaya",
    "pero",
    "dahil",
    "ayon",
    "noon",
    "ngayon",
    "dito",
    "doon",
    "sinabi",
    "sinasabi",
    "nagpapatunay",
    "araw",
    "taon",
    "buwan",
}

_CEBUANO_MARKERS = {
    "ug",
    "nga",
    "ang",
    "sa",
    "si",
    "nag",
    "mao",
    "kang",
    "usab",
    "man",
    "dayon",
    "gyud",
    "kaayo",
    "lang",
    "pud",
    "adto",
    "kini",
    "sila",
    "niadtong",
    "gitawag",
    "giingon",
    "matud",
    "nasayran",
    "gidakop",
}


def _heuristic_lang(text: str) -> str:
    """Very rough heuristic: count Tagalog vs Cebuano marker hits."""
    words = set(text.lower().split())
    tl_hits = len(words & _TAGALOG_MARKERS)
    ceb_hits = len(words & _CEBUANO_MARKERS)
    if tl_hits == 0 and ceb_hits == 0:
        return "unknown"
    return "tl" if tl_hits >= ceb_hits else "ceb"


def detect_lang(text: str) -> str:
    """Detect language; falls back to heuristic if langdetect fails."""
    if not text or not isinstance(text, str) or len(text.split()) < 5:
        return "unknown"
    if HAS_LANGDETECT:
        try:
            return detect(text[:500])  # only need a snippet
        except LangDetectException:
            pass
    return _heuristic_lang(text)


# ── Dataset loaders (mirrors train.py logic) ─────────────────────────────────


def load_datasets_raw() -> list[tuple[str, pd.DataFrame]]:
    """Return list of (name, df) pairs, df has columns: article, label."""
    result = []

    # 1. jcblaise/fake_news_filipino
    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
    if os.path.exists(csv1):
        # The CSV has a `<<<<<<< HEAD` git conflict marker on line 1;
        # skiprows=1 makes pandas treat the real header (line 2) as the header.
        df1 = pd.read_csv(csv1, skiprows=1)
        # Keep only rows where both columns are valid
        if "article" in df1.columns and "label" in df1.columns:
            df1 = df1[["article", "label"]].dropna()
            # Drop any remaining git conflict marker rows
            df1 = df1[
                ~df1["article"].astype(str).str.startswith(("=======", ">>>>>>>"))
            ]
        result.append(("jcblaise/fake_news_filipino", df1))
        print(f"  [1] Loaded jcblaise: {len(df1)} articles")
    else:
        print(f"  [1] jcblaise not found at {csv1}, skipping.")

    # 2. Philippine Fake News Corpus
    csv2 = os.path.join(
        PROJECT_ROOT,
        "data",
        "raw",
        "philippine_corpus",
        "Philippine Fake News Corpus.csv",
    )
    if os.path.exists(csv2):
        # Same git conflict marker fix — skip line 1
        df2 = pd.read_csv(csv2, skiprows=1)
        df2 = df2.rename(columns={"Content": "article"})
        df2["label"] = df2["Label"].map({"Credible": 0, "Not Credible": 1})
        df2 = df2[["article", "label"]].dropna()
        df2 = df2[~df2["article"].astype(str).str.startswith(("=======", ">>>>>>>"))]
        result.append(("Philippine Fake News Corpus", df2))
        print(f"  [2] Loaded Philippine Corpus: {len(df2)} articles")
    else:
        print(f"  [2] Philippine Corpus not found at {csv2}, skipping.")

    # 3. CebuaNER — definitively Cebuano, no need to run detection on every row
    try:
        from datasets import load_dataset

        print("  [3] Downloading CebuaNER...")
        ds = load_dataset("josephimperial/CebuaNER")
        sentences = []
        for split_data in ds.values():
            for row in split_data:
                # CebuaNER schema: {'text': str}  — one sentence per row
                text = row.get("text") or " ".join(
                    row.get("tokens") or row.get("words") or []
                )
                if text and text.strip():
                    sentences.append(text.strip())

        MIN_CHUNK = 30
        articles, buf, buf_tok = [], [], 0
        for s in sentences:
            buf.append(s)
            buf_tok += len(s.split())
            if buf_tok >= MIN_CHUNK:
                articles.append(" ".join(buf))
                buf, buf_tok = [], 0
        if buf:
            articles.append(" ".join(buf))

        df3 = pd.DataFrame({"article": articles, "label": 0})
        result.append(("josephimperial/CebuaNER", df3))
        print(f"  [3] Loaded CebuaNER: {len(df3)} chunks")
    except ImportError:
        print("  [3] 'datasets' not installed, skipping CebuaNER.")
    except Exception as exc:
        print(f"  [3] CebuaNER error: {exc}")

    return result


# ── Main analysis ─────────────────────────────────────────────────────────────


def analyze(sample_size: int = 500):
    print("=" * 60)
    print("  LANGUAGE DISTRIBUTION ANALYSIS")
    print("=" * 60)
    print("\nLoading datasets...\n")

    datasets = load_datasets_raw()
    if not datasets:
        print("No datasets found.")
        return

    grand_total = 0
    grand_tl = 0

    for name, df in datasets:
        total = len(df)
        grand_total += total

        # CebuaNER is definitively Cebuano — skip expensive detection
        if "CebuaNER" in name:
            lang_counts = Counter({"ceb": total})
            print(f"\n  [{name}]")
            print(f"    Total  : {total:,}")
            print(
                f"    ceb    : {total:,} (100.0%)  [source is Cebuano news by definition]"
            )
            continue

        # Sample for speed on large datasets
        if total > sample_size:
            df_sample = df.sample(n=sample_size, random_state=42)
            sampled = True
        else:
            df_sample = df
            sampled = False

        lang_counts: Counter = Counter()
        for text in df_sample["article"]:
            lang_counts[detect_lang(str(text))] += 1

        # Scale up sample counts to full dataset size
        if sampled:
            scale = total / sample_size
            lang_counts = Counter({k: int(v * scale) for k, v in lang_counts.items()})

        tl_count = lang_counts.get("tl", 0) + lang_counts.get("fil", 0)
        grand_tl += tl_count

        print(f"\n  [{name}]")
        print(
            f"    Total  : {total:,}" + (" (estimate from sample)" if sampled else "")
        )
        for lang, cnt in lang_counts.most_common():
            pct = cnt / total * 100
            print(f"    {lang:<8}: {cnt:>6,}  ({pct:.1f}%)")

    print("\n" + "=" * 60)
    print(f"  GRAND TOTAL articles : {grand_total:,}")
    print(f"  Estimated Tagalog    : {grand_tl:,}  ({grand_tl/grand_total*100:.1f}%)")
    print("=" * 60)
    print("\nNote: 'tl'=Tagalog/Filipino, 'ceb'=Cebuano, 'en'=English")
    print("      langdetect may mis-classify short or code-switched texts.")


if __name__ == "__main__":
    analyze()