Spaces:
Configuration error
Configuration error
updates for equal mass binning
Browse files
ece.py
CHANGED
|
@@ -103,8 +103,8 @@ def create_bins(n_bins=10, scheme="equal-range", bin_range=None, P=None):
|
|
| 103 |
|
| 104 |
def discretize_into_bins(P, bins):
|
| 105 |
|
| 106 |
-
contains_rightmost = bool(bins[-1] > 1)
|
| 107 |
-
contains_leftmost = bool(bins[0] == 0)
|
| 108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
| 109 |
|
| 110 |
oneDbins = np.digitize(
|
|
@@ -143,7 +143,7 @@ def discretize_into_bins(P, bins):
|
|
| 143 |
def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
|
| 144 |
bin_assignments = discretize_into_bins(np.expand_dims(P, 0), bins)[0]
|
| 145 |
|
| 146 |
-
# indexed as in julia!
|
| 147 |
result = np.empty([len(bins)], float)
|
| 148 |
result.fill(np.nan) # cannot assume each bin will have observations
|
| 149 |
|
|
@@ -160,8 +160,10 @@ def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
|
|
| 160 |
def bin_calibrated_accuracy(bins, proxy="upper-edge"):
|
| 161 |
assert proxy in ["center", "upper-edge"], f"Unsupported proxy{proxy}"
|
| 162 |
|
|
|
|
|
|
|
| 163 |
if proxy == "upper-edge":
|
| 164 |
-
return bins[1:]
|
| 165 |
|
| 166 |
if proxy == "center":
|
| 167 |
return bins[:-1] + np.diff(bins) / 2
|
|
@@ -175,7 +177,7 @@ def CE_estimate(y_correct, P, bins=None, p=1, proxy="upper-edge", detail=False):
|
|
| 175 |
Summary: weighted average over the accuracy/confidence difference of discrete bins of prediction probability
|
| 176 |
"""
|
| 177 |
|
| 178 |
-
n_bins = len(bins) - 1
|
| 179 |
bin_range = [min(bins), max(bins)]
|
| 180 |
|
| 181 |
# average bin probability #55 for bin 50-60, mean per bin; or right/upper bin edges
|
|
@@ -185,9 +187,6 @@ def CE_estimate(y_correct, P, bins=None, p=1, proxy="upper-edge", detail=False):
|
|
| 185 |
bin_numbers, weights_ece = np.unique(bin_assignment, return_counts=True)
|
| 186 |
anindices = bin_numbers - 1 # reduce bin counts; left edge; indexes right by default
|
| 187 |
|
| 188 |
-
import pdb; pdb.set_trace() # breakpoint 83c9148b //
|
| 189 |
-
|
| 190 |
-
|
| 191 |
# Expected calibration error
|
| 192 |
if p < np.inf: # L^p-CE
|
| 193 |
CE = np.average(
|
|
|
|
| 103 |
|
| 104 |
def discretize_into_bins(P, bins):
|
| 105 |
|
| 106 |
+
contains_rightmost = bool(bins[-1] > 1) # outlier bins
|
| 107 |
+
contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
|
| 108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
| 109 |
|
| 110 |
oneDbins = np.digitize(
|
|
|
|
| 143 |
def manual_binned_statistic(P, y_correct, bins, statistic="mean"):
|
| 144 |
bin_assignments = discretize_into_bins(np.expand_dims(P, 0), bins)[0]
|
| 145 |
|
| 146 |
+
# indexed as in julia!
|
| 147 |
result = np.empty([len(bins)], float)
|
| 148 |
result.fill(np.nan) # cannot assume each bin will have observations
|
| 149 |
|
|
|
|
| 160 |
def bin_calibrated_accuracy(bins, proxy="upper-edge"):
|
| 161 |
assert proxy in ["center", "upper-edge"], f"Unsupported proxy{proxy}"
|
| 162 |
|
| 163 |
+
contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
|
| 164 |
+
|
| 165 |
if proxy == "upper-edge":
|
| 166 |
+
return bins[1:] if contains_leftmost else bins
|
| 167 |
|
| 168 |
if proxy == "center":
|
| 169 |
return bins[:-1] + np.diff(bins) / 2
|
|
|
|
| 177 |
Summary: weighted average over the accuracy/confidence difference of discrete bins of prediction probability
|
| 178 |
"""
|
| 179 |
|
| 180 |
+
n_bins = len(bins) - 1 # true number of bins
|
| 181 |
bin_range = [min(bins), max(bins)]
|
| 182 |
|
| 183 |
# average bin probability #55 for bin 50-60, mean per bin; or right/upper bin edges
|
|
|
|
| 187 |
bin_numbers, weights_ece = np.unique(bin_assignment, return_counts=True)
|
| 188 |
anindices = bin_numbers - 1 # reduce bin counts; left edge; indexes right by default
|
| 189 |
|
|
|
|
|
|
|
|
|
|
| 190 |
# Expected calibration error
|
| 191 |
if p < np.inf: # L^p-CE
|
| 192 |
CE = np.average(
|