| """Fixed-prediction evaluation entry points for the published Test_TSB script.""" |
|
|
| import numpy as np |
| import torch |
| from sklearn.metrics import precision_recall_fscore_support |
|
|
| from .basic_metrics import basic_metricor, generate_curve, improved_cardinality_fn |
|
|
|
|
| def get_metrics(score, labels, slidingWindow=100, pred=None, version="opt", thre=250): |
| """Evaluate scores without selecting thresholds from ground-truth labels. |
| |
| Respect supplied predictions. Otherwise use mean(score) + 3 * std(score), |
| matching Test_TSB.py. Remove labels == -1 before calibration and evaluation. |
| Undefined ranking/event metrics are NaN, rather than a fabricated zero. |
| """ |
| score = np.asarray(score, dtype=float) |
| labels = np.asarray(labels) |
| if score.ndim != 1 or labels.shape != score.shape: |
| raise ValueError("Scores and labels must be one-dimensional with equal lengths.") |
| if pred is not None: |
| pred = np.asarray(pred) |
| if pred.shape != score.shape: |
| raise ValueError("Predictions must have the same shape as scores.") |
| valid = labels != -1 |
| score, labels = score[valid], labels[valid] |
| if not len(score) or not np.isfinite(score).all(): |
| raise ValueError("At least one finite, unpadded score is required.") |
| if not np.isin(labels, [0, 1]).all(): |
| raise ValueError("Labels must be 0, 1, or -1 for padding.") |
| labels = labels.astype(int) |
| if pred is None: |
| pred = score > (score.mean() + 3 * score.std()) |
| else: |
| pred = pred[valid] |
| if not np.isin(pred, [0, 1]).all(): |
| raise ValueError("Predictions must be binary.") |
| pred = pred.astype(int, copy=True) |
| if int(slidingWindow) != slidingWindow or slidingWindow < 1: |
| raise ValueError("slidingWindow must be a positive integer.") |
|
|
| grader = basic_metricor() |
| precision, recall, f1, _ = precision_recall_fscore_support( |
| labels, pred, average="binary", zero_division=0 |
| ) |
| adjusted = grader.adjustment(labels, pred.copy()) |
| pa_p, pa_r, pa_f, _ = precision_recall_fscore_support( |
| labels, adjusted, average="binary", zero_division=0 |
| ) |
| t_p, t_r = grader.ts_precision_and_recall( |
| torch.as_tensor(labels), torch.as_tensor(pred), |
| recall_cardinality_fn=improved_cardinality_fn, |
| weighted_precision=True, |
| ) |
| t_f = 2 * t_p * t_r / (t_p + t_r) if t_p + t_r else 0.0 |
|
|
| auc_roc = auc_pr = vus_roc = vus_pr = float("nan") |
| if len(np.unique(labels)) == 2: |
| auc_roc = grader.metric_ROC(labels, score) |
| auc_pr = grader.metric_PR(labels, score) |
| *_, vus_roc, vus_pr = generate_curve( |
| labels, score, int(slidingWindow), version=version, thre=thre |
| ) |
|
|
| aff_f = aff_p = aff_r = float("nan") |
| if labels.any(): |
| if pred.any(): |
| aff_f, aff_p, aff_r = grader.metric_Affiliation(labels, score, preds=pred) |
| else: |
| aff_f, aff_p, aff_r = 0.0, float("nan"), 0.0 |
|
|
| return {key: float(value) for key, value in { |
| "AUC-PR": auc_pr, "AUC-ROC": auc_roc, |
| "VUS-PR": vus_pr, "VUS-ROC": vus_roc, |
| "Standard-F1": f1, "Standard-Precision": precision, "Standard-Recall": recall, |
| "PA-F1": pa_f, "PA-Precision": pa_p, "PA-Recall": pa_r, |
| "Affiliation-F": aff_f, "Affiliation-P": aff_p, "Affiliation-R": aff_r, |
| "F1_T": t_f, "Precision_T": t_p, "Recall_T": t_r, |
| }.items()} |
|
|
|
|
| def get_metrics_pred(score, labels, pred, slidingWindow=100): |
| """Evaluate a fixed prediction vector through the same public protocol.""" |
| return get_metrics(score, labels, slidingWindow=slidingWindow, pred=pred) |
|
|
