src/data_loader.py

"""TikTok dataset loader.

Single entry point: ``load_tiktok_dataset(path)``. Reads the Kaggle CSV
(2,109 rows, semicolon delimiter, UTF-8 double-encoded via cp1252), applies
the mojibake fix (`text.encode('cp1252').decode('utf-8')`), drops rows whose
markers (`ð`/`Ÿ`/`˜`) remain after the round-trip, dedups on `komentar`, and
returns a DataFrame with columns `komentar` (str) and `label` (int 0/1).

Expected final size: 1,990–2,010 rows. Used by every training notebook -
nothing else should read the raw CSV directly.

Labels are Kaggle-canonical: 0 = cyberbullying, 1 = non-cyberbullying.
"""

from __future__ import annotations

import logging
import re
from pathlib import Path

import pandas as pd

logger = logging.getLogger(__name__)

_MOJIBAKE_MARKERS = re.compile(r"[ðŸ˜]")
_EXPECTED_ROW_RANGE = (1990, 2010)


def _fix_mojibake(text: str) -> str:
    """Recover UTF-8 text that was double-encoded via cp1252."""
    if not isinstance(text, str):
        return text
    try:
        return text.encode("cp1252", errors="strict").decode("utf-8", errors="strict")
    except (UnicodeEncodeError, UnicodeDecodeError):
        return text


def _has_unresolved_mojibake(text: str) -> bool:
    """True if the string still contains mojibake markers after the cp1252 round-trip."""
    if not isinstance(text, str):
        return False
    return bool(_MOJIBAKE_MARKERS.search(text))


def load_tiktok_dataset(path: Path | str) -> pd.DataFrame:
    """Load the TikTok dataset, fix mojibake, drop unrecoverable rows, and dedup.

    Returns a DataFrame with exactly two columns: ``komentar`` (str) and ``label`` (int 0/1).
    """
    csv_path = Path(path)
    df = pd.read_csv(csv_path, sep=";", encoding="utf-8")
    logger.info("loaded %d rows from %s", len(df), csv_path)

    df = df[["komentar", "label"]].copy()

    df = df.dropna(subset=["komentar", "label"])
    logger.info("after dropna(komentar, label): %d rows", len(df))

    df["komentar"] = df["komentar"].astype(str).map(_fix_mojibake)
    logger.info("applied mojibake fix")

    unresolved_mask = df["komentar"].map(_has_unresolved_mojibake)
    n_unresolved = int(unresolved_mask.sum())
    df = df[~unresolved_mask].copy()
    logger.info("dropped %d unrecoverable mojibake rows: %d remaining", n_unresolved, len(df))

    before_dedup = len(df)
    df = df.drop_duplicates(subset="komentar").reset_index(drop=True)
    logger.info("dedup on komentar: %d -> %d rows", before_dedup, len(df))

    n_multiline = int(df["komentar"].str.contains("\n", regex=False).sum())
    if n_multiline > 0:
        logger.warning(
            "Detected %d rows with embedded newlines in 'komentar'. "
            "Ensure downstream CSV save uses QUOTE_ALL to avoid row corruption.",
            n_multiline,
        )

    df["label"] = df["label"].astype(int)

    assert df["komentar"].notna().all(), "NaN found in komentar after cleaning"
    assert df["label"].notna().all(), "NaN found in label after cleaning"
    assert set(df["label"].unique()).issubset({0, 1}), (
        f"unexpected label values: {sorted(df['label'].unique())}"
    )
    lo, hi = _EXPECTED_ROW_RANGE
    assert lo <= len(df) <= hi, f"row count {len(df)} outside expected range [{lo}, {hi}]"

    return df