potato/server_utils/iaa/multilabel.py

"""
Multi-label IAA metrics for schemas where each annotator can select a set of labels
per item (e.g., multiselect, hierarchical_multiselect, card_sort).

Provides MASI distance, Jaccard distance, and pairwise alpha-MASI.
"""

from __future__ import annotations

from typing import Dict, Iterable, Sequence

from potato.server_utils.iaa.alpha import krippendorff_alpha, _masi_distance


def jaccard_distance(set_a: Iterable, set_b: Iterable) -> float:
    a = frozenset(set_a)
    b = frozenset(set_b)
    if not a and not b:
        return 0.0
    union = a | b
    if not union:
        return 0.0
    return 1.0 - len(a & b) / len(union)


def masi_distance(set_a: Iterable, set_b: Iterable) -> float:
    return _masi_distance(set_a, set_b)


def mean_jaccard(label_sets_by_user: Dict[str, Sequence[Iterable]]) -> float:
    """Average pairwise (1 - Jaccard distance) across users and items."""
    users = list(label_sets_by_user)
    if len(users) < 2:
        return float("nan")
    sims = []
    for i in range(len(users)):
        a = list(label_sets_by_user[users[i]])
        for j in range(i + 1, len(users)):
            b = list(label_sets_by_user[users[j]])
            m = min(len(a), len(b))
            if m == 0:
                continue
            for k in range(m):
                sims.append(1.0 - jaccard_distance(a[k], b[k]))
    if not sims:
        return float("nan")
    return sum(sims) / len(sims)


def alpha_masi(long_format_sets) -> float:
    """Krippendorff's alpha with MASI distance on multi-label sets."""
    return krippendorff_alpha(long_format_sets, level="masi")