File size: 2,522 Bytes
dddb6a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
"""
Track 1.4 — imbalance-aware evaluation.

Aggregate accuracy flatters the easy classes and hides exactly the failure
mode (e.g. Sad/Neutral confusion) most likely to matter in a chat context.
This script reports PR-AUC and per-class F1 alongside accuracy, and should
be what you lead with in any eval report — not the headline accuracy number.

Usage:
    python scripts/eval_report.py --predictions data/val_predictions.csv \
        --out data/eval_report.json
"""
import argparse
import json

import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score, average_precision_score, f1_score

IDX_TO_EMOTION = {
    0: "angry", 1: "calm", 2: "disgust", 3: "fearful",
    4: "happy", 5: "neutral", 6: "sad", 7: "surprised",
}
NUM_CLASSES = 8


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--predictions", required=True, help="CSV with true_label, prob_0..prob_7 columns")
    ap.add_argument("--out", default="data/eval_report.json")
    ap.add_argument("--corpus", default="unspecified", help="Label this eval run, e.g. RAVDESS / cross-corpus / chat-holdout")
    args = ap.parse_args()

    df = pd.read_csv(args.predictions)
    labels = df["true_label"].values
    probs = df[[f"prob_{i}" for i in range(NUM_CLASSES)]].values
    preds = probs.argmax(axis=1)

    accuracy = accuracy_score(labels, preds)
    per_class_f1 = f1_score(labels, preds, average=None, labels=list(range(NUM_CLASSES)))
    per_class_pr_auc = {}
    for c in range(NUM_CLASSES):
        y_bin = (labels == c).astype(int)
        if y_bin.sum() == 0:
            per_class_pr_auc[IDX_TO_EMOTION[c]] = None
            continue
        per_class_pr_auc[IDX_TO_EMOTION[c]] = float(average_precision_score(y_bin, probs[:, c]))

    class_support = {IDX_TO_EMOTION[c]: int((labels == c).sum()) for c in range(NUM_CLASSES)}

    report = {
        "corpus": args.corpus,
        "n_samples": len(df),
        "accuracy": float(accuracy),
        "per_class_f1": {IDX_TO_EMOTION[c]: float(per_class_f1[c]) for c in range(NUM_CLASSES)},
        "per_class_pr_auc": per_class_pr_auc,
        "class_support": class_support,
        "note": "Lead with per_class_f1 / per_class_pr_auc, not accuracy. "
                "Low-support / low-F1 classes (e.g. sad/neutral) are the ones "
                "most likely to matter for a chat product.",
    }

    print(json.dumps(report, indent=2))
    with open(args.out, "w") as f:
        json.dump(report, f, indent=2)


if __name__ == "__main__":
    main()