evaluation/error_analysis.py

"""Claim category error analysis."""

from __future__ import annotations

from collections import Counter
from dataclasses import dataclass
from pathlib import Path


@dataclass(frozen=True)
class ErrorExample:
    claim: str
    truth: str
    prediction: str
    category: str


def categorize_claim(claim: str) -> str:
    lowered = claim.lower()
    if any(token in lowered for token in ["not ", "never", "no "]):
        return "negation"
    if any(char.isdigit() for char in claim):
        return "numerical"
    if any(month in lowered for month in [" january", " february", " march", " april", " may ", " june", " july", " august", " september", " october", " november", " december"]):
        return "temporal"
    if any(token in lowered for token in ["who", "where", "which", "what"]):
        return "entity"
    if any(token in lowered for token in ["because", "therefore", "as a result"]):
        return "possible multi-hop"
    return "entity"


def build_error_analysis(examples: list[ErrorExample]) -> dict[str, object]:
    counts = Counter(example.category for example in examples)
    return {"category_counts": dict(counts), "examples": [example.__dict__ for example in examples]}


def write_error_report(examples: list[ErrorExample], output_dir: str | Path) -> Path:
    output = Path(output_dir)
    output.mkdir(parents=True, exist_ok=True)
    path = output / "error_analysis.json"
    path.write_text(_json_dumps(build_error_analysis(examples)), encoding="utf-8")
    return path


def _json_dumps(payload: dict[str, object]) -> str:
    import json

    return json.dumps(payload, indent=2, sort_keys=True)