potato/server_utils/iaa/ordinal.py

"""
Ordinal IAA metrics: weighted kappa (linear + quadratic), Spearman's rho.
"""

from __future__ import annotations

from typing import Sequence

import logging

logger = logging.getLogger(__name__)


def _coerce_ordinal(values: Sequence) -> list:
    """Try to coerce a sequence of (str|int|float) ratings into numeric ranks."""
    coerced = []
    for v in values:
        if isinstance(v, (int, float)):
            coerced.append(float(v))
        else:
            try:
                coerced.append(float(v))
            except (TypeError, ValueError):
                # Fall back to lexical ordering by stable string sort
                coerced.append(str(v))
    if any(isinstance(c, str) for c in coerced):
        rank = {c: i for i, c in enumerate(sorted(set(coerced)))}
        return [rank[c] for c in coerced]
    return coerced


def weighted_kappa(labels_a: Sequence, labels_b: Sequence, weights: str = "quadratic") -> float:
    """
    Cohen's weighted kappa for ordinal categories.

    weights: 'linear' or 'quadratic' (CKD convention).
    """
    if len(labels_a) != len(labels_b):
        raise ValueError("label lists must be the same length")
    if not labels_a:
        return float("nan")
    try:
        from sklearn.metrics import cohen_kappa_score
        a = _coerce_ordinal(labels_a)
        b = _coerce_ordinal(labels_b)
        return float(cohen_kappa_score(a, b, weights=weights))
    except ImportError:  # pragma: no cover
        logger.warning("sklearn unavailable; weighted_kappa returning NaN")
        return float("nan")


def spearman_rho(labels_a: Sequence, labels_b: Sequence) -> float:
    """Spearman rank correlation between two annotators."""
    if len(labels_a) != len(labels_b):
        raise ValueError("label lists must be the same length")
    if len(labels_a) < 2:
        return float("nan")
    try:
        from scipy.stats import spearmanr
        a = _coerce_ordinal(labels_a)
        b = _coerce_ordinal(labels_b)
        rho, _ = spearmanr(a, b)
        return float(rho) if rho == rho else float("nan")  # NaN-safe
    except ImportError:  # pragma: no cover
        logger.warning("scipy unavailable; spearman_rho returning NaN")
        return float("nan")