File size: 4,108 Bytes
8a7c723 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | """
Correct aggregate group shares for classifier error.
If you classify a corpus and report the share of each group, those shares are
biased by classifier error, and the bias is largest exactly where accuracy is
lowest. With a confusion matrix M where
M[i, j] = P(predicted = i | true = j)
observed and true shares are related by p_obs = M @ p_true, so
p_true ~= pinv(M) @ p_obs
This is the standard quantification / prior-correction estimator. Its bias goes
to zero as M approaches the identity, and it is base-rate invariant, so a
confusion matrix estimated on a stratified validation sample is valid here.
Two cautions:
* Estimate M with the SAME pipeline that produced the counts you are
correcting - same detector, same crop margin, same checkpoints. A confusion
matrix from a different preprocessing is the wrong matrix.
* When a class has very low recall the correction is poorly conditioned and
can return a near-zero or clipped estimate. Report those classes as bounds.
On our data the unweighted correction sent Hispanic to 0.0, while the
inverse-probability-weighted version recovered 2.6%. Prefer the weighted
matrix and say which you used.
Licence: MIT.
"""
from __future__ import annotations
import numpy as np
__all__ = [
"confusion_matrix_from_labels",
"correct_proportions",
"correct_counts",
]
def confusion_matrix_from_labels(y_true, y_pred, labels, weights=None) -> np.ndarray:
"""
Column-stochastic confusion matrix M[i, j] = P(pred = labels[i] | true = labels[j]).
weights : optional per-sample weights. Pass inverse selection probabilities
when the validation sample was stratified on predicted labels,
otherwise M is conditioned on the predictions rather than the
population.
"""
labels = list(labels)
idx = {l: i for i, l in enumerate(labels)}
n = len(labels)
if weights is None:
weights = np.ones(len(y_true), dtype=float)
weights = np.asarray(weights, dtype=float)
M = np.zeros((n, n), dtype=float)
for t, p, w in zip(y_true, y_pred, weights):
if t in idx and p in idx:
M[idx[p], idx[t]] += w
for j in range(n):
col = M[:, j].sum()
if col > 0:
M[:, j] /= col
else:
M[j, j] = 1.0 # unobserved class: assume no confusion
return M
def correct_proportions(p_obs, M, clip_negative: bool = True) -> np.ndarray:
"""
Recover true class proportions from observed ones.
p_obs : observed proportions, same order as the confusion matrix labels
M : column-stochastic confusion matrix
"""
p_obs = np.asarray(p_obs, dtype=float)
q = np.linalg.pinv(M) @ p_obs
if clip_negative:
q = np.clip(q, 0.0, None)
total = q.sum()
return q / total if total > 0 else p_obs
def correct_counts(counts, M) -> np.ndarray:
"""Same correction expressed in counts rather than proportions."""
counts = np.asarray(counts, dtype=float)
total = counts.sum()
if total <= 0:
return counts
return correct_proportions(counts / total, M) * total
if __name__ == "__main__":
# Worked example: race shares of real advertisements, using the published
# real-domain IPW matrix from benchmark/confusion_race_real_ipw.csv
labels = ["white", "black", "asian", "hispanic"]
p_obs = [0.6494, 0.1889, 0.1552, 0.0065]
census = [0.6090, 0.1329, 0.0697, 0.1884]
M = np.array([
[0.9603, 0.1138, 0.2073, 0.1376],
[0.0057, 0.8383, 0.0106, 0.2646],
[0.0327, 0.0448, 0.7804, 0.4092],
[0.0013, 0.0031, 0.0017, 0.1886],
])
p_cor = correct_proportions(p_obs, M)
print(f"{'class':10s} {'observed':>9s} {'corrected':>10s} {'census':>8s} "
f"{'dev obs':>8s} {'dev cor':>8s}")
for l, o, c, r in zip(labels, p_obs, p_cor, census):
print(f"{l:10s} {o*100:8.2f}% {c*100:9.2f}% {r*100:7.2f}% "
f"{(o-r)*100:+7.2f} {(c-r)*100:+7.2f}")
print("\nWhite over-representation: +4.04 pp observed -> +0.67 pp corrected.")
|