bias / eval /bias_eval.json
bogdanraduta's picture
Upload eval/bias_eval.json with huggingface_hub
28bffff verified
Raw
History Blame Contribute Delete
21.4 kB
{
"task": "bias",
"threshold": 0.77,
"per_language": {
"az": {
"support": 80,
"precision": 0.9195,
"recall": 1.0,
"f1": 0.9581,
"fpr": 0.0745,
"accuracy": 0.954,
"n": 174
},
"bg": {
"support": 80,
"precision": 0.9873,
"recall": 0.975,
"f1": 0.9811,
"fpr": 0.0106,
"accuracy": 0.977,
"n": 174
},
"cs": {
"support": 80,
"precision": 0.9877,
"recall": 1.0,
"f1": 0.9938,
"fpr": 0.0106,
"accuracy": 0.9943,
"n": 174
},
"da": {
"support": 80,
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 174
},
"de": {
"support": 80,
"precision": 0.9634,
"recall": 0.9875,
"f1": 0.9753,
"fpr": 0.0319,
"accuracy": 0.9598,
"n": 174
},
"el": {
"support": 80,
"precision": 1.0,
"recall": 0.9875,
"f1": 0.9937,
"fpr": 0.0,
"accuracy": 0.9885,
"n": 174
},
"en": {
"support": 80,
"precision": 0.9634,
"recall": 0.9875,
"f1": 0.9753,
"fpr": 0.0319,
"accuracy": 0.9713,
"n": 174
},
"es": {
"support": 76,
"precision": 1.0,
"recall": 0.9737,
"f1": 0.9867,
"fpr": 0.0,
"accuracy": 0.9882,
"n": 170
},
"et": {
"support": 79,
"precision": 0.9634,
"recall": 1.0,
"f1": 0.9814,
"fpr": 0.0323,
"accuracy": 0.9826,
"n": 172
},
"fi": {
"support": 80,
"precision": 0.9875,
"recall": 0.9875,
"f1": 0.9875,
"fpr": 0.0106,
"accuracy": 0.9885,
"n": 174
},
"fr": {
"support": 80,
"precision": 0.9756,
"recall": 1.0,
"f1": 0.9877,
"fpr": 0.0213,
"accuracy": 0.9885,
"n": 174
},
"ga": {
"support": 77,
"precision": 0.9737,
"recall": 0.961,
"f1": 0.9673,
"fpr": 0.0213,
"accuracy": 0.9591,
"n": 171
},
"hr": {
"support": 80,
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 174
},
"hu": {
"support": 80,
"precision": 0.9518,
"recall": 0.9875,
"f1": 0.9693,
"fpr": 0.0426,
"accuracy": 0.9713,
"n": 174
},
"it": {
"support": 80,
"precision": 1.0,
"recall": 1.0,
"f1": 1.0,
"fpr": 0.0,
"accuracy": 0.9943,
"n": 174
},
"lt": {
"support": 79,
"precision": 0.9873,
"recall": 0.9873,
"f1": 0.9873,
"fpr": 0.0106,
"accuracy": 0.9827,
"n": 173
},
"lv": {
"support": 80,
"precision": 0.9753,
"recall": 0.9875,
"f1": 0.9814,
"fpr": 0.0213,
"accuracy": 0.9713,
"n": 174
},
"mt": {
"support": 78,
"precision": 0.9481,
"recall": 0.9359,
"f1": 0.9419,
"fpr": 0.0426,
"accuracy": 0.9419,
"n": 172
},
"nl": {
"support": 79,
"precision": 0.9875,
"recall": 1.0,
"f1": 0.9937,
"fpr": 0.0106,
"accuracy": 0.9942,
"n": 173
},
"pl": {
"support": 80,
"precision": 0.9756,
"recall": 1.0,
"f1": 0.9877,
"fpr": 0.0213,
"accuracy": 0.9885,
"n": 174
},
"pt": {
"support": 76,
"precision": 0.9737,
"recall": 0.9737,
"f1": 0.9737,
"fpr": 0.0213,
"accuracy": 0.9765,
"n": 170
},
"ro": {
"support": 80,
"precision": 1.0,
"recall": 0.9875,
"f1": 0.9937,
"fpr": 0.0,
"accuracy": 0.9943,
"n": 174
},
"sk": {
"support": 80,
"precision": 0.9875,
"recall": 0.9875,
"f1": 0.9875,
"fpr": 0.0106,
"accuracy": 0.9885,
"n": 174
},
"sl": {
"support": 80,
"precision": 1.0,
"recall": 0.9875,
"f1": 0.9937,
"fpr": 0.0,
"accuracy": 0.9943,
"n": 174
},
"sv": {
"support": 80,
"precision": 0.9875,
"recall": 0.9875,
"f1": 0.9875,
"fpr": 0.0106,
"accuracy": 0.9885,
"n": 174
},
"tr": {
"support": 80,
"precision": 0.9512,
"recall": 0.975,
"f1": 0.963,
"fpr": 0.0426,
"accuracy": 0.9655,
"n": 174
}
},
"per_register": {
"age_assumption": {
"support": 415,
"precision": 1.0,
"recall": 0.9855,
"f1": 0.9927,
"fpr": 0.0,
"accuracy": 0.9807,
"n": 415
},
"counter_stereotype": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.011,
"accuracy": 0.989,
"n": 364
},
"demographic_statistic": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 364
},
"disability_condescension": {
"support": 413,
"precision": 1.0,
"recall": 0.9952,
"f1": 0.9976,
"fpr": 0.0,
"accuracy": 0.9927,
"n": 413
},
"discussing_bias": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0165,
"accuracy": 0.9835,
"n": 364
},
"ethnic_generalisation": {
"support": 411,
"precision": 1.0,
"recall": 0.9903,
"f1": 0.9951,
"fpr": 0.0,
"accuracy": 0.9878,
"n": 411
},
"inclusive_phrasing": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0962,
"accuracy": 0.9038,
"n": 364
},
"mundane_informational": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 208
},
"mundane_operational": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 208
},
"mundane_transactional": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 207
},
"neutral_description": {
"support": 0,
"precision": 0.0,
"recall": 0.0,
"f1": 0.0,
"fpr": 0.0,
"accuracy": 1.0,
"n": 364
},
"occupational_stereotype": {
"support": 415,
"precision": 1.0,
"recall": 0.9687,
"f1": 0.9841,
"fpr": 0.0,
"accuracy": 0.959,
"n": 415
},
"religious_assumption": {
"support": 410,
"precision": 1.0,
"recall": 0.9951,
"f1": 0.9976,
"fpr": 0.0,
"accuracy": 0.9805,
"n": 410
}
},
"per_label": {
"age": {
"support": 415,
"precision": 0.9667,
"recall": 0.9807,
"f1": 0.9737,
"fpr": 0.0034,
"accuracy": 0.0,
"n": 0
},
"disability": {
"support": 413,
"precision": 0.9832,
"recall": 0.9927,
"f1": 0.988,
"fpr": 0.0017,
"accuracy": 0.0,
"n": 0
},
"ethnicity": {
"support": 411,
"precision": 0.9713,
"recall": 0.9878,
"f1": 0.9795,
"fpr": 0.0029,
"accuracy": 0.0,
"n": 0
},
"gender": {
"support": 415,
"precision": 0.9476,
"recall": 0.959,
"f1": 0.9533,
"fpr": 0.0054,
"accuracy": 0.0,
"n": 0
},
"religion": {
"support": 410,
"precision": 0.9901,
"recall": 0.9805,
"f1": 0.9853,
"fpr": 0.001,
"accuracy": 0.0,
"n": 0
}
},
"worst_languages": [
[
"mt",
0.9419354838709677
],
[
"az",
0.9580838323353293
],
[
"tr",
0.9629629629629629
],
[
"ga",
0.9673202614379085
],
[
"hu",
0.9693251533742331
]
],
"calibration": {
"threshold": 0.77,
"objective": "macro_f1",
"value_on_val": 0.9822,
"default_threshold": 0.5,
"default_value_on_val": 0.9811,
"improvement": 0.0011,
"chosen_on": "validation split, never test",
"curve": [
{
"threshold": 0.02,
"macro_f1": 0.97,
"any_label_recall": 0.9966,
"any_label_fpr": 0.027
},
{
"threshold": 0.03,
"macro_f1": 0.9731,
"any_label_recall": 0.9966,
"any_label_fpr": 0.0237
},
{
"threshold": 0.04,
"macro_f1": 0.974,
"any_label_recall": 0.9966,
"any_label_fpr": 0.0237
},
{
"threshold": 0.05,
"macro_f1": 0.9749,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0229
},
{
"threshold": 0.06,
"macro_f1": 0.9763,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0221
},
{
"threshold": 0.07,
"macro_f1": 0.9767,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0221
},
{
"threshold": 0.08,
"macro_f1": 0.9772,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0213
},
{
"threshold": 0.09,
"macro_f1": 0.9775,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0213
},
{
"threshold": 0.1,
"macro_f1": 0.9777,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.11,
"macro_f1": 0.9781,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.12,
"macro_f1": 0.9781,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.13,
"macro_f1": 0.9781,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.14,
"macro_f1": 0.9781,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.15,
"macro_f1": 0.9779,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.16,
"macro_f1": 0.9779,
"any_label_recall": 0.9961,
"any_label_fpr": 0.0209
},
{
"threshold": 0.17,
"macro_f1": 0.9774,
"any_label_recall": 0.9952,
"any_label_fpr": 0.0209
},
{
"threshold": 0.18,
"macro_f1": 0.9779,
"any_label_recall": 0.9947,
"any_label_fpr": 0.0205
},
{
"threshold": 0.19,
"macro_f1": 0.9779,
"any_label_recall": 0.9947,
"any_label_fpr": 0.0205
},
{
"threshold": 0.2,
"macro_f1": 0.9781,
"any_label_recall": 0.9947,
"any_label_fpr": 0.0201
},
{
"threshold": 0.21,
"macro_f1": 0.9783,
"any_label_recall": 0.9947,
"any_label_fpr": 0.0201
},
{
"threshold": 0.22,
"macro_f1": 0.9786,
"any_label_recall": 0.9947,
"any_label_fpr": 0.0196
},
{
"threshold": 0.23,
"macro_f1": 0.9788,
"any_label_recall": 0.9942,
"any_label_fpr": 0.0196
},
{
"threshold": 0.24,
"macro_f1": 0.9793,
"any_label_recall": 0.9942,
"any_label_fpr": 0.0188
},
{
"threshold": 0.25,
"macro_f1": 0.9793,
"any_label_recall": 0.9942,
"any_label_fpr": 0.0188
},
{
"threshold": 0.26,
"macro_f1": 0.9795,
"any_label_recall": 0.9937,
"any_label_fpr": 0.0184
},
{
"threshold": 0.27,
"macro_f1": 0.9795,
"any_label_recall": 0.9932,
"any_label_fpr": 0.0184
},
{
"threshold": 0.28,
"macro_f1": 0.9792,
"any_label_recall": 0.9927,
"any_label_fpr": 0.0184
},
{
"threshold": 0.29,
"macro_f1": 0.9795,
"any_label_recall": 0.9927,
"any_label_fpr": 0.018
},
{
"threshold": 0.3,
"macro_f1": 0.9795,
"any_label_recall": 0.9927,
"any_label_fpr": 0.018
},
{
"threshold": 0.31,
"macro_f1": 0.9795,
"any_label_recall": 0.9927,
"any_label_fpr": 0.018
},
{
"threshold": 0.32,
"macro_f1": 0.9797,
"any_label_recall": 0.9927,
"any_label_fpr": 0.018
},
{
"threshold": 0.33,
"macro_f1": 0.9797,
"any_label_recall": 0.9927,
"any_label_fpr": 0.018
},
{
"threshold": 0.34,
"macro_f1": 0.9797,
"any_label_recall": 0.9927,
"any_label_fpr": 0.018
},
{
"threshold": 0.35,
"macro_f1": 0.9799,
"any_label_recall": 0.9927,
"any_label_fpr": 0.0176
},
{
"threshold": 0.36,
"macro_f1": 0.9799,
"any_label_recall": 0.9927,
"any_label_fpr": 0.0176
},
{
"threshold": 0.37,
"macro_f1": 0.9799,
"any_label_recall": 0.9927,
"any_label_fpr": 0.0176
},
{
"threshold": 0.38,
"macro_f1": 0.9802,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0176
},
{
"threshold": 0.39,
"macro_f1": 0.9802,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0176
},
{
"threshold": 0.4,
"macro_f1": 0.9804,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0172
},
{
"threshold": 0.41,
"macro_f1": 0.9804,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0172
},
{
"threshold": 0.42,
"macro_f1": 0.9804,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0172
},
{
"threshold": 0.43,
"macro_f1": 0.9804,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0172
},
{
"threshold": 0.44,
"macro_f1": 0.9804,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0172
},
{
"threshold": 0.45,
"macro_f1": 0.9806,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0168
},
{
"threshold": 0.46,
"macro_f1": 0.9806,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0168
},
{
"threshold": 0.47,
"macro_f1": 0.9806,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0168
},
{
"threshold": 0.48,
"macro_f1": 0.9806,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0168
},
{
"threshold": 0.49,
"macro_f1": 0.9806,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0168
},
{
"threshold": 0.5,
"macro_f1": 0.9811,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0164
},
{
"threshold": 0.51,
"macro_f1": 0.9813,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0164
},
{
"threshold": 0.52,
"macro_f1": 0.9813,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0164
},
{
"threshold": 0.53,
"macro_f1": 0.9813,
"any_label_recall": 0.9922,
"any_label_fpr": 0.0164
},
{
"threshold": 0.54,
"macro_f1": 0.9811,
"any_label_recall": 0.9918,
"any_label_fpr": 0.0164
},
{
"threshold": 0.55,
"macro_f1": 0.9811,
"any_label_recall": 0.9918,
"any_label_fpr": 0.0164
},
{
"threshold": 0.56,
"macro_f1": 0.9811,
"any_label_recall": 0.9918,
"any_label_fpr": 0.0164
},
{
"threshold": 0.57,
"macro_f1": 0.9811,
"any_label_recall": 0.9918,
"any_label_fpr": 0.0164
},
{
"threshold": 0.58,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.59,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.6,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.61,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.62,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.63,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.64,
"macro_f1": 0.9813,
"any_label_recall": 0.9918,
"any_label_fpr": 0.016
},
{
"threshold": 0.65,
"macro_f1": 0.9811,
"any_label_recall": 0.9913,
"any_label_fpr": 0.016
},
{
"threshold": 0.66,
"macro_f1": 0.9813,
"any_label_recall": 0.9913,
"any_label_fpr": 0.016
},
{
"threshold": 0.67,
"macro_f1": 0.9818,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0156
},
{
"threshold": 0.68,
"macro_f1": 0.9818,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0156
},
{
"threshold": 0.69,
"macro_f1": 0.9818,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0156
},
{
"threshold": 0.7,
"macro_f1": 0.9818,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0156
},
{
"threshold": 0.71,
"macro_f1": 0.982,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0151
},
{
"threshold": 0.72,
"macro_f1": 0.982,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0151
},
{
"threshold": 0.73,
"macro_f1": 0.982,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0151
},
{
"threshold": 0.74,
"macro_f1": 0.982,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0151
},
{
"threshold": 0.75,
"macro_f1": 0.982,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0151
},
{
"threshold": 0.76,
"macro_f1": 0.982,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0151
},
{
"threshold": 0.77,
"macro_f1": 0.9822,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0147
},
{
"threshold": 0.78,
"macro_f1": 0.9822,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0147
},
{
"threshold": 0.79,
"macro_f1": 0.9822,
"any_label_recall": 0.9913,
"any_label_fpr": 0.0147
},
{
"threshold": 0.8,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.81,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.82,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.83,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.84,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.85,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.86,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.87,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.88,
"macro_f1": 0.9817,
"any_label_recall": 0.9903,
"any_label_fpr": 0.0147
},
{
"threshold": 0.89,
"macro_f1": 0.9815,
"any_label_recall": 0.9898,
"any_label_fpr": 0.0147
},
{
"threshold": 0.9,
"macro_f1": 0.9812,
"any_label_recall": 0.9898,
"any_label_fpr": 0.0147
},
{
"threshold": 0.91,
"macro_f1": 0.9812,
"any_label_recall": 0.9893,
"any_label_fpr": 0.0143
},
{
"threshold": 0.92,
"macro_f1": 0.9812,
"any_label_recall": 0.9893,
"any_label_fpr": 0.0143
},
{
"threshold": 0.93,
"macro_f1": 0.9812,
"any_label_recall": 0.9884,
"any_label_fpr": 0.0139
},
{
"threshold": 0.94,
"macro_f1": 0.9815,
"any_label_recall": 0.9879,
"any_label_fpr": 0.0139
},
{
"threshold": 0.95,
"macro_f1": 0.9814,
"any_label_recall": 0.9869,
"any_label_fpr": 0.0131
}
],
"notes": []
}
}