Spaces:
Sleeping
Sleeping
Download scripts/eval_report.py from space-cadet/ser-inference: direct link, hf CLI and curl.
- Browser
- Download file 2.52 kB
-
https://huggingface.co/spaces/space-cadet/ser-inference/resolve/main/scripts/eval_report.py
- Command line
-
hf download hf://spaces/space-cadet/ser-inference/scripts/eval_report.py
-
curl -L -o eval_report.py https://huggingface.co/spaces/space-cadet/ser-inference/resolve/main/scripts/eval_report.py
2.52 kB
| """ | |
| Track 1.4 — imbalance-aware evaluation. | |
| Aggregate accuracy flatters the easy classes and hides exactly the failure | |
| mode (e.g. Sad/Neutral confusion) most likely to matter in a chat context. | |
| This script reports PR-AUC and per-class F1 alongside accuracy, and should | |
| be what you lead with in any eval report — not the headline accuracy number. | |
| Usage: | |
| python scripts/eval_report.py --predictions data/val_predictions.csv \ | |
| --out data/eval_report.json | |
| """ | |
| import argparse | |
| import json | |
| import numpy as np | |
| import pandas as pd | |
| from sklearn.metrics import accuracy_score, average_precision_score, f1_score | |
| IDX_TO_EMOTION = { | |
| 0: "angry", 1: "calm", 2: "disgust", 3: "fearful", | |
| 4: "happy", 5: "neutral", 6: "sad", 7: "surprised", | |
| } | |
| NUM_CLASSES = 8 | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--predictions", required=True, help="CSV with true_label, prob_0..prob_7 columns") | |
| ap.add_argument("--out", default="data/eval_report.json") | |
| ap.add_argument("--corpus", default="unspecified", help="Label this eval run, e.g. RAVDESS / cross-corpus / chat-holdout") | |
| args = ap.parse_args() | |
| df = pd.read_csv(args.predictions) | |
| labels = df["true_label"].values | |
| probs = df[[f"prob_{i}" for i in range(NUM_CLASSES)]].values | |
| preds = probs.argmax(axis=1) | |
| accuracy = accuracy_score(labels, preds) | |
| per_class_f1 = f1_score(labels, preds, average=None, labels=list(range(NUM_CLASSES))) | |
| per_class_pr_auc = {} | |
| for c in range(NUM_CLASSES): | |
| y_bin = (labels == c).astype(int) | |
| if y_bin.sum() == 0: | |
| per_class_pr_auc[IDX_TO_EMOTION[c]] = None | |
| continue | |
| per_class_pr_auc[IDX_TO_EMOTION[c]] = float(average_precision_score(y_bin, probs[:, c])) | |
| class_support = {IDX_TO_EMOTION[c]: int((labels == c).sum()) for c in range(NUM_CLASSES)} | |
| report = { | |
| "corpus": args.corpus, | |
| "n_samples": len(df), | |
| "accuracy": float(accuracy), | |
| "per_class_f1": {IDX_TO_EMOTION[c]: float(per_class_f1[c]) for c in range(NUM_CLASSES)}, | |
| "per_class_pr_auc": per_class_pr_auc, | |
| "class_support": class_support, | |
| "note": "Lead with per_class_f1 / per_class_pr_auc, not accuracy. " | |
| "Low-support / low-F1 classes (e.g. sad/neutral) are the ones " | |
| "most likely to matter for a chat product.", | |
| } | |
| print(json.dumps(report, indent=2)) | |
| with open(args.out, "w") as f: | |
| json.dump(report, f, indent=2) | |
| if __name__ == "__main__": | |
| main() | |