Spaces:
Runtime error
Runtime error
| """Temperature scaling + expected calibration error, numpy/torch only (no sklearn). | |
| The training extras (mlflow/sklearn/matplotlib) are NOT installed in this env, so all | |
| metrics here are plain numpy; only ``fit_temperature`` uses torch (LBFGS on NLL). | |
| """ | |
| from __future__ import annotations | |
| import numpy as np | |
| def softmax(logits: np.ndarray, temperature: float = 1.0) -> np.ndarray: | |
| """Row-wise softmax with optional temperature (numpy, numerically stable).""" | |
| z = logits.astype(np.float64) / max(float(temperature), 1e-8) | |
| z = z - z.max(axis=1, keepdims=True) | |
| e = np.exp(z) | |
| return e / e.sum(axis=1, keepdims=True) | |
| def fit_temperature(logits: np.ndarray, labels: np.ndarray) -> float: | |
| """Fit a single softmax temperature by minimizing NLL on held-out (val) logits. | |
| LBFGS over log-temperature (guarantees T > 0). Returns 1.0 for empty input. | |
| """ | |
| import torch # lazy: keep numpy-only callers (e.g. ece tests) torch-free | |
| logits = np.asarray(logits, dtype=np.float32) | |
| labels = np.asarray(labels) | |
| if logits.ndim != 2 or labels.ndim != 1 or logits.shape[0] != labels.shape[0]: | |
| raise ValueError(f"bad shapes: logits {logits.shape}, labels {labels.shape}") | |
| if logits.shape[0] == 0: | |
| return 1.0 | |
| logits_t = torch.from_numpy(logits) | |
| labels_t = torch.from_numpy(labels.astype(np.int64)) | |
| log_temp = torch.zeros(1, requires_grad=True) | |
| optimizer = torch.optim.LBFGS([log_temp], lr=0.1, max_iter=100) | |
| nll = torch.nn.CrossEntropyLoss() | |
| def closure() -> torch.Tensor: | |
| optimizer.zero_grad() | |
| loss = nll(logits_t / log_temp.exp(), labels_t) | |
| loss.backward() | |
| return loss | |
| optimizer.step(closure) # type: ignore[arg-type] | |
| temperature = float(log_temp.exp().item()) | |
| if not np.isfinite(temperature): | |
| return 1.0 | |
| return float(np.clip(temperature, 0.05, 50.0)) | |
| def ece(probs: np.ndarray, labels: np.ndarray, bins: int = 15) -> float: | |
| """Expected calibration error (equal-width confidence bins), pure numpy. | |
| Perfectly confident + correct predictions give ~0. | |
| """ | |
| probs = np.asarray(probs, dtype=np.float64) | |
| labels = np.asarray(labels) | |
| if probs.ndim != 2 or probs.shape[0] != labels.shape[0]: | |
| raise ValueError(f"bad shapes: probs {probs.shape}, labels {labels.shape}") | |
| if probs.shape[0] == 0: | |
| return 0.0 | |
| confidence = probs.max(axis=1) | |
| correct = (probs.argmax(axis=1) == labels).astype(np.float64) | |
| edges = np.linspace(0.0, 1.0, bins + 1) | |
| total = 0.0 | |
| n = float(len(labels)) | |
| for i in range(bins): | |
| lo, hi = edges[i], edges[i + 1] | |
| mask = (confidence > lo) & (confidence <= hi) if i > 0 else (confidence <= hi) | |
| if mask.any(): | |
| gap = abs(correct[mask].mean() - confidence[mask].mean()) | |
| total += (mask.sum() / n) * gap | |
| return float(total) | |