import numpy as np from sklearn.metrics import auc, precision_recall_curve, roc_auc_score def calculate_metrics( y_true: np.ndarray, y_pred: np.ndarray, label_names: list[str] | None = None ) -> tuple[float, float, dict[str, float | None]]: """ Computes Macro-AUROC and Micro-AUPRC across multi-label targets. Protects against constant-label columns by returning None for single-class columns. """ if label_names is None: label_names = [f"Class_{i}" for i in range(y_true.shape[1])] per_class_auroc: dict[str, float | None] = {} valid_aurocs: list[float] = [] for idx, name in enumerate(label_names): col_true = y_true[:, idx] col_pred = y_pred[:, idx] # Guard against single-class constant labels if len(np.unique(col_true)) < 2: per_class_auroc[name] = None else: try: score = float(roc_auc_score(col_true, col_pred)) per_class_auroc[name] = round(score, 4) valid_aurocs.append(score) except Exception: per_class_auroc[name] = None macro_auroc = float(np.mean(valid_aurocs)) if valid_aurocs else 0.5 # Micro-AUPRC try: precision, recall, _ = precision_recall_curve(y_true.ravel(), y_pred.ravel()) micro_auprc = float(auc(recall, precision)) except Exception: micro_auprc = 0.0 return macro_auroc, micro_auprc, per_class_auroc def bootstrap_confidence_intervals( y_true: np.ndarray, y_pred: np.ndarray, n_bootstraps: int = 1000, ci: float = 95.0, seed: int = 42 ) -> dict[str, dict[str, float]]: """ Computes non-parametric percentile bootstrap confidence intervals (2.5% - 97.5%) for Macro-AUROC and Micro-AUPRC over n_bootstraps resamples with replacement. """ rng = np.random.RandomState(seed) n_samples = len(y_true) bootstrapped_aurocs: list[float] = [] bootstrapped_auprcs: list[float] = [] for _ in range(n_bootstraps): indices = rng.choice(n_samples, size=n_samples, replace=True) if len(np.unique(y_true[indices])) < 2: continue macro_auc, micro_pr, _ = calculate_metrics(y_true[indices], y_pred[indices]) bootstrapped_aurocs.append(macro_auc) bootstrapped_auprcs.append(micro_pr) alpha = (100.0 - ci) / 2.0 auroc_lower = float(np.percentile(bootstrapped_aurocs, alpha)) auroc_upper = float(np.percentile(bootstrapped_aurocs, 100.0 - alpha)) auprc_lower = float(np.percentile(bootstrapped_auprcs, alpha)) auprc_upper = float(np.percentile(bootstrapped_auprcs, 100.0 - alpha)) return { "macro_auroc_ci": { "mean": float(np.mean(bootstrapped_aurocs)), "lower": round(auroc_lower, 4), "upper": round(auroc_upper, 4), }, "micro_auprc_ci": { "mean": float(np.mean(bootstrapped_auprcs)), "lower": round(auprc_lower, 4), "upper": round(auprc_upper, 4), } } def calculate_expected_calibration_error( y_true: np.ndarray, y_pred: np.ndarray, n_bins: int = 10 ) -> float: """ Calculates Expected Calibration Error (ECE) for multi-label predictions across n_bins. """ bin_boundaries = np.linspace(0, 1, n_bins + 1) ece = 0.0 for i in range(n_bins): bin_lower = bin_boundaries[i] bin_upper = bin_boundaries[i + 1] in_bin = (y_pred >= bin_lower) & (y_pred < bin_upper) prop_in_bin = float(np.mean(in_bin)) if prop_in_bin > 0: accuracy_in_bin = float(np.mean(y_true[in_bin])) avg_confidence_in_bin = float(np.mean(y_pred[in_bin])) ece += abs(accuracy_in_bin - avg_confidence_in_bin) * prop_in_bin return round(float(ece), 4)