Hugging Face's logo

Spaces:
aslemen
/
eval-single-classification
Runtime error

App Files Community
eval-single-classification
File size: 4,768 Bytes
0460cdb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d808801
0460cdb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
from typing import TypedDict
import datasets
import evaluate
import numpy as np


class Metrics(TypedDict):
    confusion_matrix: np.ndarray
    confusion_matrix_normalized: np.ndarray
    bias: np.ndarray
    prevalence: np.ndarray
    correct: np.ndarray
    class_precision: np.ndarray
    class_recall: np.ndarray
    accuracy: float
    """
    Also known as micro F1.
    """
    macro_recall: float
    macro_precision: float
    macro_precision_prevalence_calibrated: float
    F1: np.ndarray
    macro_F1: float
    macro_F1_2: float
    weighted_F1: float
    chance: np.ndarray
    MCC: float
    """
    Generalized Matthews Correlation Coefficient [_2].
    """
    KAPPA: float
    """
    Cohen's Kappa.
    """

_DESCRIPTION = """
This module provides various metrics for single label classification.
"""

_KWARGS_DESCRIPTION = """
Args:
    predictions (`int`): Predicted label.
    references (`int`): Ground truth label.
    num_classes (`int`): Number of classes.

Returns:
    metric_dict (`Metrics`)
"""

_CITATION = """
@article{10.1162/tacl_a_00675,
    author = {Opitz, Juri},
    title = {A Closer Look at Classification Evaluation Metrics and a Critical
                    Reflection of Common Evaluation Practice},
    journal = {Transactions of the Association for Computational Linguistics},
    volume = {12},
    pages = {820-836},
    year = {2024},
    month = {06},
    issn = {2307-387X},
    doi = {10.1162/tacl_a_00675},
}
@article{GORODKIN2004367,
    title = {Comparing two K-category assignments by a K-category correlation coefficient},
    journal = {Computational Biology and Chemistry},
    volume = {28},
    number = {5},
    pages = {367-374},
    year = {2004},
    issn = {1476-9271},
    doi = {https://doi.org/10.1016/j.compbiolchem.2004.09.006},
    url = {https://www.sciencedirect.com/science/article/pii/S1476927104000799},
    author = {J. Gorodkin},
}
"""

@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class EvalSingleClassification(evaluate.Metric):
    def _info(self):
        return evaluate.MetricInfo(
            description = _DESCRIPTION,
            citation = _CITATION,
            inputs_description = _KWARGS_DESCRIPTION,
            features = datasets.Features(
                {
                    "predictions": datasets.Value("int32"),
                    "references": datasets.Value("int32"),
                }
            )
        )

    def _compute(self, predictions, references, num_classes: int):
        confusion_matrix = np.zeros((num_classes, num_classes), dtype=int)
        # Shape: (n_classes_pred, n_classes_label)
        for pred, label in zip(predictions, references):
            confusion_matrix[pred, label] += 1

        confusion_matrix_normalized = confusion_matrix / confusion_matrix.sum()

        bias = np.sum(confusion_matrix, axis=1)
        prevalence = np.sum(confusion_matrix, axis=0)
        prevalence_calibration = confusion_matrix.sum() / prevalence / num_classes
        correct = np.diag(confusion_matrix)
        class_precision = correct / bias
        class_recall = correct / prevalence
        accuracy = correct.sum() / confusion_matrix.sum()
        macro_recall = class_recall.mean()
        macro_precision = class_precision.mean()
        macro_precision_prevalence_calibrated = (
            (prevalence_calibration * correct)
            / (confusion_matrix * prevalence_calibration[None, :]).sum(axis=1)
        ) / num_classes
        F1 = correct / (bias + prevalence + EPS)
        macro_F1 = 2 * F1.sum() / num_classes
        macro_F1_2 = (
            2 * (macro_precision * macro_recall) / (macro_precision + macro_recall + EPS)
        )
        weighted_F1 = (prevalence * F1).sum() / (prevalence.sum() + EPS)

        chance = prevalence.T @ bias

        MCC = (accuracy - chance) / np.sqrt(
            (1 - bias.T @ bias) * (1 - prevalence.T @ prevalence)
        )
        KAPPA = (accuracy - chance) / (1 - chance)

        return {
            "confusion_matrix": confusion_matrix,
            "confusion_matrix_normalized": confusion_matrix_normalized,
            "bias": bias,
            "prevalence": prevalence,
            "correct": correct,
            "class_precision": class_precision,
            "class_recall": class_recall,
            "accuracy": accuracy,
            "macro_recall": macro_recall,
            "macro_precision": macro_precision,
            "macro_precision_prevalence_calibrated": macro_precision_prevalence_calibrated,
            "F1": F1,
            "macro_F1": macro_F1,
            "macro_F1_2": macro_F1_2,
            "weighted_F1": weighted_F1,
            "chance": chance,
            "MCC": MCC,
            "KAPPA": KAPPA,
        }