backend/input/dataset.py

"""Input — Dataset loaders pentru LIAR (TSV), FakeNewsNet (JSON), VER-1 (CSV)."""

from __future__ import annotations
from backend.pipeline.graph.models import Article
from backend.config import DATASETS_DIR

import csv
import json
import logging
from datetime import datetime
from pathlib import Path
from typing import Optional

logger = logging.getLogger(__name__)


_LIAR2_LABEL_MAP = {
    "0": "true",
    "1": "mostly-true",
    "2": "half-true",
    "3": "barely-true",
    "4": "false",
    "5": "pants-fire",
}


def load_liar(
    filepath: Optional[Path] = None,
    split: str = "test",
    max_articles: Optional[int] = None,
) -> list[Article]:
    """LIAR / LIAR2: PolitiFact statements.

    Supports both formats:
    - LIAR original: {split}.tsv, tab-delimited, no header, 14+ columns, string labels
    - LIAR2:         {split}.csv, comma-delimited, header on row 0, numeric labels (0-5)
    Column mapping (both formats): col[1]=label, col[2]=statement, col[4]=speaker/source.
    """
    if filepath is None:
        filepath_csv = DATASETS_DIR / "liar" / f"{split}.csv"
        filepath_tsv = DATASETS_DIR / "liar" / f"{split}.tsv"
        filepath = filepath_csv if filepath_csv.exists() else filepath_tsv

    if not filepath.exists():
        logger.warning(f"LIAR file not found: {filepath}")
        return []

    delimiter = "," if filepath.suffix == ".csv" else "\t"
    logger.info(f"LIAR [{split}]: loading from {filepath.name} (delimiter={delimiter!r})")

    articles = []
    with open(filepath, "r", encoding="utf-8") as f:
        reader = csv.reader(f, delimiter=delimiter)
        if filepath.suffix == ".csv":
            next(reader, None)  # LIAR2 CSV has a header row; original TSV does not
        for row in reader:
            if len(row) < 3:
                continue
            source_col = row[4] if len(row) > 4 else "unknown"
            raw_label = row[1].strip()
            label = _LIAR2_LABEL_MAP.get(raw_label, raw_label)
            article = Article(
                text=row[2], title=row[2][:80], publication_date=None,
                source=f"liar-{source_col}", label=label, dataset="LIAR",
            )
            articles.append(article)
            if max_articles and len(articles) >= max_articles:
                break

    logger.info(f"LIAR [{split}]: loaded {len(articles)} articles from {filepath.name}")
    return articles


def load_fakenewsnet(
    base_dir: Optional[Path] = None,
    source: str = "politifact",
    label: str = "fake",
    max_articles: Optional[int] = None,
) -> list[Article]:
    """FakeNewsNet: JSON articles with timestamps (politifact/gossipcop, fake/real)."""
    if base_dir is None:
        base_dir = DATASETS_DIR / "fakenewsnet"
    data_dir = base_dir / source / label
    if not data_dir.exists():
        logger.warning(f"FakeNewsNet directory not found: {data_dir}")
        return []

    articles = []
    for article_dir in sorted(data_dir.iterdir()):
        if not article_dir.is_dir():
            continue
        json_file = article_dir / "news content.json"
        if not json_file.exists():
            continue

        try:
            with open(json_file, "r", encoding="utf-8") as f:
                data = json.load(f)

            pub_date = None
            if data.get("publish_date"):
                try:
                    pub_date = datetime.fromisoformat(str(data["publish_date"]))
                except (ValueError, TypeError):
                    pass

            text = data.get("text", "")
            if not text.strip():
                continue

            article = Article(
                text=text, title=data.get("title", ""), publication_date=pub_date,
                source=source, url=data.get("url", ""), label=label, dataset="FakeNewsNet",
            )
            articles.append(article)
        except (json.JSONDecodeError, KeyError) as e:
            logger.debug(f"Error reading {json_file}: {e}")
            continue

        if max_articles and len(articles) >= max_articles:
            break

    logger.info(f"FakeNewsNet [{source}/{label}]: loaded {len(articles)} articles")
    return articles


def load_ver1(
    filepath: Optional[Path] = None,
    max_articles: Optional[int] = None,
) -> list[Article]:
    """VER-1 (Cheres & Groza): Eastern Europe disinformation, CSV."""
    if filepath is None:
        filepath = DATASETS_DIR / "ver1" / "ver1.csv"
    if not filepath.exists():
        logger.warning(f"VER-1 file not found: {filepath}")
        return []

    articles = []
    with open(filepath, "r", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            text = row.get("text", "")
            if not text.strip():
                continue
            article = Article(
                text=text, title=text[:80], publication_date=None,
                source="veridica.ro", label=row.get("label", "disinformation"),
                dataset="VER-1",
            )
            articles.append(article)
            if max_articles and len(articles) >= max_articles:
                break

    logger.info(f"VER-1: loaded {len(articles)} articles")
    return articles


def load_dataset(name: str, max_articles: Optional[int] = None, **kwargs) -> list[Article]:
    """Dispatcher: load_dataset('liar'), load_dataset('fakenewsnet', source='politifact')."""
    loaders = {"liar": load_liar, "fakenewsnet": load_fakenewsnet, "ver1": load_ver1}
    if name.lower() not in loaders:
        raise ValueError(f"Unknown dataset: {name}. Available: {list(loaders.keys())}")
    return loaders[name.lower()](max_articles=max_articles, **kwargs)