{ "scope": "Exploratory existing splits; test has previously informed development", "calibration": { "global": 1.096478196143185, "by_question": { "150440942f8fd3e04c6ffcb60c0215ffb1265e7d27e071f3ac453fd9817a85d4": 1.513561248436208, "2a95b662ebe5bb2aa612f55ab9490ec200c83aec6e0ce1e9c439d769cb39a93c": 0.1, "330b97e5d96bf0d453555d62465367c41f1c5eb57ca53a1f3e8015c8ebe8548a": 1.513561248436208, "59786fb640773138e83576a42b1baff87c3a12c6924070c867a17f7541f8fd50": 0.9120108393559097 }, "minimum_examples": 50, "unknown_question_policy": "global temperature; calibration not established" }, "splits": { "validation": { "uncalibrated": { "overall": { "count": 800, "accuracy": 0.93, "macro_f1": 0.942767064108638, "nll": 0.14363739399664024, "brier": 0.09649431923038009, "ece_10_bins": 0.014156258404254909, "oracle_expected_nll": 0.13552431792992842, "oracle_distribution_mse": 0.009192354615839761, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 15, "confidence": 0.5693979422251384, "accuracy": 0.4666666666666667 }, { "lower": 0.6, "upper": 0.7, "count": 28, "confidence": 0.6569398194551468, "accuracy": 0.6428571428571429 }, { "lower": 0.7, "upper": 0.8, "count": 61, "confidence": 0.7547330357989327, "accuracy": 0.6721311475409836 }, { "lower": 0.8, "upper": 0.9, "count": 93, "confidence": 0.8439700603485107, "accuracy": 0.8064516129032258 }, { "lower": 0.9, "upper": 1.0, "count": 603, "confidence": 0.9985708261208353, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.06999999999999995 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.99, "macro_f1": 0.9901098901098901, "nll": 0.010185061729229394, "brier": 0.006637792461137025, "ece_10_bins": 0.0073371312022209225, "oracle_expected_nll": 0.010185061729229394, "oracle_distribution_mse": 0.006637792461137025, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 1, "confidence": 0.5138780474662781, "accuracy": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "confidence": 0.6265591382980347, "accuracy": 0.0 }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 198, "confidence": 0.9983485401278794, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.010000000000000009 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.7286704853783539, "nll": 0.5641286429153144, "brier": 0.37933894420616143, "ece_10_bins": 0.05137460917234424, "oracle_expected_nll": 0.531676338648467, "oracle_distribution_mse": 0.030131085748000032, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 14, "confidence": 0.5733636489936283, "accuracy": 0.5 }, { "lower": 0.6, "upper": 0.7, "count": 27, "confidence": 0.6580650298683731, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 61, "confidence": 0.7547330357989327, "accuracy": 0.6721311475409836 }, { "lower": 0.8, "upper": 0.9, "count": 93, "confidence": 0.8439700603485107, "accuracy": 0.8064516129032258 }, { "lower": 0.9, "upper": 1.0, "count": 5, "confidence": 0.9024688482284546, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.12 }, { "coverage": 0.5, "count": 100, "error_rate": 0.18999999999999995 }, { "coverage": 0.75, "count": 150, "error_rate": 0.22666666666666668 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 9.878679824863307e-05, "brier": 2.084608453162e-08, "ece_10_bins": 9.878158569331941e-05, "oracle_expected_nll": 9.878679824863307e-05, "oracle_distribution_mse": 2.084608453162e-08, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9999012184143067, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0001370845437686307, "brier": 5.194081374567017e-07, "ece_10_bins": 0.00013695359230037063, "oracle_expected_nll": 0.0001370845437686307, "oracle_distribution_mse": 5.194081374567017e-07, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9998630464076996, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "global": { "overall": { "count": 800, "accuracy": 0.93, "macro_f1": 0.942767064108638, "nll": 0.14297785222380682, "brier": 0.09584865920230996, "ece_10_bins": 0.009678431376814825, "oracle_expected_nll": 0.13557863835585515, "oracle_distribution_mse": 0.009061730485262987, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 16, "confidence": 0.565223541110754, "accuracy": 0.5 }, { "lower": 0.6, "upper": 0.7, "count": 34, "confidence": 0.6557925483759712, "accuracy": 0.5882352941176471 }, { "lower": 0.7, "upper": 0.8, "count": 76, "confidence": 0.7558186516950005, "accuracy": 0.7105263157894737 }, { "lower": 0.8, "upper": 0.9, "count": 76, "confidence": 0.837152016006018, "accuracy": 0.8421052631578947 }, { "lower": 0.9, "upper": 1.0, "count": 598, "confidence": 0.9990241513603109, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.06999999999999995 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.99, "macro_f1": 0.9901098901098901, "nll": 0.010840845209029908, "brier": 0.006549160572433809, "ece_10_bins": 0.008071925044059772, "oracle_expected_nll": 0.010840845209029908, "oracle_distribution_mse": 0.006549160572433809, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 1, "confidence": 0.5126363039016724, "accuracy": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "confidence": 0.615834653377533, "accuracy": 0.0 }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 198, "confidence": 0.9975458886286225, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.010000000000000009 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.7286704853783539, "nll": 0.5605819633748517, "brier": 0.3768439641098432, "ece_10_bins": 0.03015358328819274, "oracle_expected_nll": 0.530985107903045, "oracle_distribution_mse": 0.02969624924165533, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 15, "confidence": 0.5687293569246928, "accuracy": 0.5333333333333333 }, { "lower": 0.6, "upper": 0.7, "count": 33, "confidence": 0.6570033936789541, "accuracy": 0.6060606060606061 }, { "lower": 0.7, "upper": 0.8, "count": 76, "confidence": 0.7558186516950005, "accuracy": 0.7105263157894737 }, { "lower": 0.8, "upper": 0.9, "count": 76, "confidence": 0.837152016006018, "accuracy": 0.8421052631578947 }, { "lower": 0.9, "upper": 1.0, "count": 0, "confidence": null, "accuracy": null } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.12 }, { "coverage": 0.5, "count": 100, "error_rate": 0.18999999999999995 }, { "coverage": 0.75, "count": 150, "error_rate": 0.22666666666666668 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.00022176956251617814, "brier": 1.0389551787255491e-07, "ece_10_bins": 0.00022174358367921254, "oracle_expected_nll": 0.00022176956251617814, "oracle_distribution_mse": 1.0389551787255491e-07, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9997782564163208, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.00026683074882947707, "brier": 1.4082314449344625e-06, "ece_10_bins": 0.0002664735913276983, "oracle_expected_nll": 0.00026683074882947707, "oracle_distribution_mse": 1.4082314449344625e-06, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9997335264086723, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "per_question": { "overall": { "count": 800, "accuracy": 0.93, "macro_f1": 0.942767064108638, "nll": 0.14733114755095042, "brier": 0.09750962712499976, "ece_10_bins": 0.017871906235814118, "oracle_expected_nll": 0.13843533652518125, "oracle_distribution_mse": 0.009677819048759641, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 11, "confidence": 0.5629869699478149, "accuracy": 0.36363636363636365 }, { "lower": 0.6, "upper": 0.7, "count": 27, "confidence": 0.6522127919726901, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 48, "confidence": 0.7543277218937874, "accuracy": 0.6666666666666666 }, { "lower": 0.8, "upper": 0.9, "count": 105, "confidence": 0.8517835855484008, "accuracy": 0.7904761904761904 }, { "lower": 0.9, "upper": 1.0, "count": 609, "confidence": 0.994959923532013, "accuracy": 0.9967159277504105 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.06999999999999995 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.99, "macro_f1": 0.9901098901098901, "nll": 0.016769024587334803, "brier": 0.006763791666141161, "ece_10_bins": 0.01407569766044616, "oracle_expected_nll": 0.016769024587334803, "oracle_distribution_mse": 0.006763791666141161, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 2, "confidence": 0.5465996563434601, "accuracy": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 3, "confidence": 0.8894220391909281, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 195, "confidence": 0.992870736733461, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.010000000000000009 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.7286704853783539, "nll": 0.5704197966243141, "brier": 0.3832444677877699, "ece_10_bins": 0.06606497406959536, "oracle_expected_nll": 0.5348365525212375, "oracle_distribution_mse": 0.03191723548280944, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 9, "confidence": 0.5666285951932272, "accuracy": 0.4444444444444444 }, { "lower": 0.6, "upper": 0.7, "count": 27, "confidence": 0.6522127919726901, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 48, "confidence": 0.7543277218937874, "accuracy": 0.6666666666666666 }, { "lower": 0.8, "upper": 0.9, "count": 102, "confidence": 0.8506765722059736, "accuracy": 0.7843137254901961 }, { "lower": 0.9, "upper": 1.0, "count": 14, "confidence": 0.9104529874665397, "accuracy": 0.8571428571428571 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.12 }, { "coverage": 0.5, "count": 100, "error_rate": 0.18999999999999995 }, { "coverage": 0.75, "count": 150, "error_rate": 0.22666666666666668 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0, "brier": 5.221318590436242e-77, "ece_10_bins": 0.0, "oracle_expected_nll": 0.0, "oracle_distribution_mse": 5.221318590436242e-77, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 1.0, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0021357689921527212, "brier": 3.0249046087972453e-05, "ece_10_bins": 0.002127710282802564, "oracle_expected_nll": 0.0021357689921527212, "oracle_distribution_mse": 3.0249046087972453e-05, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9978722897171974, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } } }, "test": { "uncalibrated": { "overall": { "count": 800, "accuracy": 0.9325, "macro_f1": 0.945994599459946, "nll": 0.14603000355243925, "brier": 0.0982134248610902, "ece_10_bins": 0.01680564425885677, "oracle_expected_nll": 0.12875951419169415, "oracle_distribution_mse": 0.005807715000243819, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 11, "confidence": 0.5358001806519248, "accuracy": 0.6363636363636364 }, { "lower": 0.6, "upper": 0.7, "count": 18, "confidence": 0.668232325050566, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 67, "confidence": 0.7674324290076299, "accuracy": 0.7313432835820896 }, { "lower": 0.8, "upper": 0.9, "count": 104, "confidence": 0.843749422293443, "accuracy": 0.75 }, { "lower": 0.9, "upper": 1.0, "count": 600, "confidence": 0.9997629618644714, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.0675 } ] }, "by_task": { "department": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0003887213664454956, "brier": 6.573280611448498e-06, "ece_10_bins": 0.00038706064224247605, "oracle_expected_nll": 0.0003887213664454956, "oracle_distribution_mse": 6.573280611448498e-06, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9996129393577575, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.72997299729973, "nll": 0.5834056262796022, "brier": 0.3928407776022817, "ece_10_bins": 0.06651146262884142, "oracle_expected_nll": 0.514323668836622, "oracle_distribution_mse": 0.023217938158896164, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 11, "confidence": 0.5358001806519248, "accuracy": 0.6363636363636364 }, { "lower": 0.6, "upper": 0.7, "count": 18, "confidence": 0.668232325050566, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 67, "confidence": 0.7674324290076299, "accuracy": 0.7313432835820896 }, { "lower": 0.8, "upper": 0.9, "count": 104, "confidence": 0.843749422293443, "accuracy": 0.75 }, { "lower": 0.9, "upper": 1.0, "count": 0, "confidence": null, "accuracy": null } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.19999999999999996 }, { "coverage": 0.5, "count": 100, "error_rate": 0.25 }, { "coverage": 0.75, "count": 150, "error_rate": 0.24 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.00010169231916523476, "brier": 2.3599058819392956e-08, "ece_10_bins": 0.00010168641805652356, "oracle_expected_nll": 0.00010169231916523476, "oracle_distribution_mse": 2.3599058819392956e-08, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9998983135819435, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.00022397424454392842, "brier": 6.324962408847836e-06, "ece_10_bins": 0.00022236734628677812, "oracle_expected_nll": 0.00022397424454392842, "oracle_distribution_mse": 6.324962408847836e-06, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9997776326537132, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "global": { "overall": { "count": 800, "accuracy": 0.9325, "macro_f1": 0.945994599459946, "nll": 0.14461302598233813, "brier": 0.09727346687970145, "ece_10_bins": 0.010928971841931354, "oracle_expected_nll": 0.12886215335419976, "oracle_distribution_mse": 0.005768674131682376, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 12, "confidence": 0.5381810814142227, "accuracy": 0.5833333333333334 }, { "lower": 0.6, "upper": 0.7, "count": 20, "confidence": 0.6634815037250519, "accuracy": 0.65 }, { "lower": 0.7, "upper": 0.8, "count": 88, "confidence": 0.7614993337880481, "accuracy": 0.7386363636363636 }, { "lower": 0.8, "upper": 0.9, "count": 80, "confidence": 0.833437080681324, "accuracy": 0.7625 }, { "lower": 0.9, "upper": 1.0, "count": 600, "confidence": 0.9995919790863991, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.0675 } ] }, "by_task": { "department": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0006354354848862528, "brier": 1.4171445392714965e-05, "ece_10_bins": 0.000631839334964801, "oracle_expected_nll": 0.0006354354848862528, "oracle_distribution_mse": 1.4171445392714965e-05, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9993681606650352, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.72997299729973, "nll": 0.5772212989828499, "brier": 0.389067384510727, "ece_10_bins": 0.042491824626922636, "oracle_expected_nll": 0.5142178084702962, "oracle_distribution_mse": 0.023048213518650672, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 12, "confidence": 0.5381810814142227, "accuracy": 0.5833333333333334 }, { "lower": 0.6, "upper": 0.7, "count": 20, "confidence": 0.6634815037250519, "accuracy": 0.65 }, { "lower": 0.7, "upper": 0.8, "count": 88, "confidence": 0.7614993337880481, "accuracy": 0.7386363636363636 }, { "lower": 0.8, "upper": 0.9, "count": 80, "confidence": 0.833437080681324, "accuracy": 0.7625 }, { "lower": 0.9, "upper": 1.0, "count": 0, "confidence": null, "accuracy": null } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.19999999999999996 }, { "coverage": 0.5, "count": 100, "error_rate": 0.25 }, { "coverage": 0.75, "count": 150, "error_rate": 0.24 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.00022736255682158317, "brier": 1.1458914958364816e-07, "ece_10_bins": 0.00022733390331264314, "oracle_expected_nll": 0.00022736255682158317, "oracle_distribution_mse": 1.1458914958364816e-07, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9997726660966874, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.00036800690479488446, "brier": 1.2196973536534902e-05, "ece_10_bins": 0.00036488950252533403, "oracle_expected_nll": 0.00036800690479488446, "oracle_distribution_mse": 1.2196973536534902e-05, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9996351104974747, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "per_question": { "overall": { "count": 800, "accuracy": 0.9325, "macro_f1": 0.945994599459946, "nll": 0.14949719129215913, "brier": 0.0995688090168633, "ece_10_bins": 0.02478441372513776, "oracle_expected_nll": 0.1305604755579714, "oracle_distribution_mse": 0.006266329786064511, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 11, "confidence": 0.5392229827967557, "accuracy": 0.6363636363636364 }, { "lower": 0.6, "upper": 0.7, "count": 11, "confidence": 0.666460779580203, "accuracy": 0.5454545454545454 }, { "lower": 0.7, "upper": 0.8, "count": 45, "confidence": 0.7596085866292318, "accuracy": 0.7333333333333333 }, { "lower": 0.8, "upper": 0.9, "count": 123, "confidence": 0.8477989275281023, "accuracy": 0.7317073170731707 }, { "lower": 0.9, "upper": 1.0, "count": 610, "confidence": 0.9967766217520979, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.0675 } ] }, "by_task": { "department": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.003119851317236034, "brier": 0.000139220646155405, "ece_10_bins": 0.0030834728479385287, "oracle_expected_nll": 0.003119851317236034, "oracle_distribution_mse": 0.000139220646155405, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9969165271520615, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.72997299729973, "nll": 0.592753167735883, "brier": 0.39804941940743904, "ece_10_bins": 0.09396138578653342, "oracle_expected_nll": 0.5170063047991321, "oracle_distribution_mse": 0.024839502484243964, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 11, "confidence": 0.5392229827967557, "accuracy": 0.6363636363636364 }, { "lower": 0.6, "upper": 0.7, "count": 11, "confidence": 0.666460779580203, "accuracy": 0.5454545454545454 }, { "lower": 0.7, "upper": 0.8, "count": 45, "confidence": 0.7596085866292318, "accuracy": 0.7333333333333333 }, { "lower": 0.8, "upper": 0.9, "count": 123, "confidence": 0.8477989275281023, "accuracy": 0.7317073170731707 }, { "lower": 0.9, "upper": 1.0, "count": 10, "confidence": 0.9068993091583252, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.19999999999999996 }, { "coverage": 0.5, "count": 100, "error_rate": 0.25 }, { "coverage": 0.75, "count": 150, "error_rate": 0.24 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0, "brier": 1.5641410521010998e-69, "ece_10_bins": 0.0, "oracle_expected_nll": 0.0, "oracle_distribution_mse": 1.5641410521010998e-69, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 1.0, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.002115746115517467, "brier": 8.65960138586772e-05, "ece_10_bins": 0.0020927962660789445, "oracle_expected_nll": 0.002115746115517467, "oracle_distribution_mse": 8.65960138586772e-05, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.997907203733921, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } } } } }