Spaces:
Configuration error
Configuration error
small update to support unrealistic perfect classification
Browse files
ece.py
CHANGED
|
@@ -104,7 +104,7 @@ def create_bins(n_bins=10, scheme="equal-range", bin_range=None, P=None):
|
|
| 104 |
def discretize_into_bins(P, bins):
|
| 105 |
|
| 106 |
contains_rightmost = bool(bins[-1] > 1) # outlier bins
|
| 107 |
-
contains_leftmost = bool(bins[0] =
|
| 108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
| 109 |
|
| 110 |
oneDbins = np.digitize(
|
|
@@ -112,7 +112,6 @@ def discretize_into_bins(P, bins):
|
|
| 112 |
) # since bins contains extra righmost (& leftmost bins)
|
| 113 |
if contains_leftmost:
|
| 114 |
oneDbins -= 1
|
| 115 |
-
# oneDbins = np.digitize(P, bins) - 1 # since bins contains extra righmost (& leftmost bins)
|
| 116 |
|
| 117 |
# Fix to scipy.binned_dd_statistic:
|
| 118 |
# Tie-breaking to the left for rightmost bin
|
|
@@ -245,8 +244,8 @@ class ECE(evaluate.EvaluationModule):
|
|
| 245 |
# Homepage of the module for documentation
|
| 246 |
homepage="https://huggingface.co/spaces/jordyvl/ece",
|
| 247 |
# Additional links to the codebase or references
|
| 248 |
-
codebase_urls=["
|
| 249 |
-
reference_urls=["
|
| 250 |
)
|
| 251 |
|
| 252 |
def init_kwargs(
|
|
@@ -341,32 +340,27 @@ def test_equalmass_binning():
|
|
| 341 |
scheme="equal-mass",
|
| 342 |
bin_range=None,
|
| 343 |
proxy="upper-edge",
|
| 344 |
-
# proxy="center",
|
| 345 |
p=1,
|
| 346 |
detail=True,
|
| 347 |
)
|
| 348 |
bins = create_bins(
|
| 349 |
n_bins=kwargs["n_bins"], scheme=kwargs["scheme"], bin_range=kwargs["bin_range"], P=probs
|
| 350 |
)
|
| 351 |
-
|
| 352 |
test_ECE(**kwargs)
|
| 353 |
|
| 354 |
-
|
|
|
|
|
|
|
| 355 |
res = ECE()._compute(
|
| 356 |
-
references=
|
| 357 |
-
predictions=
|
| 358 |
detail=True,
|
| 359 |
)
|
| 360 |
print(f"ECE: {res['ECE']}\n {res}")
|
| 361 |
-
"""
|
| 362 |
-
# need to provide lens
|
| 363 |
-
|
| 364 |
-
import pdb
|
| 365 |
-
|
| 366 |
-
pdb.set_trace() # breakpoint 94583f9f //
|
| 367 |
|
| 368 |
|
| 369 |
if __name__ == "__main__":
|
|
|
|
| 370 |
test_equalmass_binning()
|
| 371 |
test_deterministic()
|
| 372 |
test_ECE()
|
|
|
|
| 104 |
def discretize_into_bins(P, bins):
|
| 105 |
|
| 106 |
contains_rightmost = bool(bins[-1] > 1) # outlier bins
|
| 107 |
+
contains_leftmost = bool(bins[0] <= 0) # beyond [before] bin_range[0]
|
| 108 |
# bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
|
| 109 |
|
| 110 |
oneDbins = np.digitize(
|
|
|
|
| 112 |
) # since bins contains extra righmost (& leftmost bins)
|
| 113 |
if contains_leftmost:
|
| 114 |
oneDbins -= 1
|
|
|
|
| 115 |
|
| 116 |
# Fix to scipy.binned_dd_statistic:
|
| 117 |
# Tie-breaking to the left for rightmost bin
|
|
|
|
| 244 |
# Homepage of the module for documentation
|
| 245 |
homepage="https://huggingface.co/spaces/jordyvl/ece",
|
| 246 |
# Additional links to the codebase or references
|
| 247 |
+
codebase_urls=[""],
|
| 248 |
+
reference_urls=[""],
|
| 249 |
)
|
| 250 |
|
| 251 |
def init_kwargs(
|
|
|
|
| 340 |
scheme="equal-mass",
|
| 341 |
bin_range=None,
|
| 342 |
proxy="upper-edge",
|
|
|
|
| 343 |
p=1,
|
| 344 |
detail=True,
|
| 345 |
)
|
| 346 |
bins = create_bins(
|
| 347 |
n_bins=kwargs["n_bins"], scheme=kwargs["scheme"], bin_range=kwargs["bin_range"], P=probs
|
| 348 |
)
|
|
|
|
| 349 |
test_ECE(**kwargs)
|
| 350 |
|
| 351 |
+
|
| 352 |
+
def test_perfect_predictions(K=3):
|
| 353 |
+
references = [0, 1, 2]
|
| 354 |
res = ECE()._compute(
|
| 355 |
+
references=references,
|
| 356 |
+
predictions=np.eye(K)[references],
|
| 357 |
detail=True,
|
| 358 |
)
|
| 359 |
print(f"ECE: {res['ECE']}\n {res}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
|
| 361 |
|
| 362 |
if __name__ == "__main__":
|
| 363 |
+
test_perfect_predictions()
|
| 364 |
test_equalmass_binning()
|
| 365 |
test_deterministic()
|
| 366 |
test_ECE()
|