| { | |
| "scope": "Exploratory existing splits; test has previously informed development", | |
| "calibration": { | |
| "global": 1.096478196143185, | |
| "by_question": { | |
| "150440942f8fd3e04c6ffcb60c0215ffb1265e7d27e071f3ac453fd9817a85d4": 1.513561248436208, | |
| "2a95b662ebe5bb2aa612f55ab9490ec200c83aec6e0ce1e9c439d769cb39a93c": 0.1, | |
| "330b97e5d96bf0d453555d62465367c41f1c5eb57ca53a1f3e8015c8ebe8548a": 1.513561248436208, | |
| "59786fb640773138e83576a42b1baff87c3a12c6924070c867a17f7541f8fd50": 0.9120108393559097 | |
| }, | |
| "minimum_examples": 50, | |
| "unknown_question_policy": "global temperature; calibration not established" | |
| }, | |
| "splits": { | |
| "validation": { | |
| "uncalibrated": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.93, | |
| "macro_f1": 0.942767064108638, | |
| "nll": 0.14363739399664024, | |
| "brier": 0.09649431923038009, | |
| "ece_10_bins": 0.014156258404254909, | |
| "oracle_expected_nll": 0.13552431792992842, | |
| "oracle_distribution_mse": 0.009192354615839761, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 15, | |
| "confidence": 0.5693979422251384, | |
| "accuracy": 0.4666666666666667 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 28, | |
| "confidence": 0.6569398194551468, | |
| "accuracy": 0.6428571428571429 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 61, | |
| "confidence": 0.7547330357989327, | |
| "accuracy": 0.6721311475409836 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 93, | |
| "confidence": 0.8439700603485107, | |
| "accuracy": 0.8064516129032258 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 603, | |
| "confidence": 0.9985708261208353, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.06999999999999995 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.99, | |
| "macro_f1": 0.9901098901098901, | |
| "nll": 0.010185061729229394, | |
| "brier": 0.006637792461137025, | |
| "ece_10_bins": 0.0073371312022209225, | |
| "oracle_expected_nll": 0.010185061729229394, | |
| "oracle_distribution_mse": 0.006637792461137025, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 1, | |
| "confidence": 0.5138780474662781, | |
| "accuracy": 0.0 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 1, | |
| "confidence": 0.6265591382980347, | |
| "accuracy": 0.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 198, | |
| "confidence": 0.9983485401278794, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.010000000000000009 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.7286704853783539, | |
| "nll": 0.5641286429153144, | |
| "brier": 0.37933894420616143, | |
| "ece_10_bins": 0.05137460917234424, | |
| "oracle_expected_nll": 0.531676338648467, | |
| "oracle_distribution_mse": 0.030131085748000032, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 14, | |
| "confidence": 0.5733636489936283, | |
| "accuracy": 0.5 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 27, | |
| "confidence": 0.6580650298683731, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 61, | |
| "confidence": 0.7547330357989327, | |
| "accuracy": 0.6721311475409836 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 93, | |
| "confidence": 0.8439700603485107, | |
| "accuracy": 0.8064516129032258 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 5, | |
| "confidence": 0.9024688482284546, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.12 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.18999999999999995 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.22666666666666668 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 9.878679824863307e-05, | |
| "brier": 2.084608453162e-08, | |
| "ece_10_bins": 9.878158569331941e-05, | |
| "oracle_expected_nll": 9.878679824863307e-05, | |
| "oracle_distribution_mse": 2.084608453162e-08, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9999012184143067, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0001370845437686307, | |
| "brier": 5.194081374567017e-07, | |
| "ece_10_bins": 0.00013695359230037063, | |
| "oracle_expected_nll": 0.0001370845437686307, | |
| "oracle_distribution_mse": 5.194081374567017e-07, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9998630464076996, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "global": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.93, | |
| "macro_f1": 0.942767064108638, | |
| "nll": 0.14297785222380682, | |
| "brier": 0.09584865920230996, | |
| "ece_10_bins": 0.009678431376814825, | |
| "oracle_expected_nll": 0.13557863835585515, | |
| "oracle_distribution_mse": 0.009061730485262987, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 16, | |
| "confidence": 0.565223541110754, | |
| "accuracy": 0.5 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 34, | |
| "confidence": 0.6557925483759712, | |
| "accuracy": 0.5882352941176471 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 76, | |
| "confidence": 0.7558186516950005, | |
| "accuracy": 0.7105263157894737 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 76, | |
| "confidence": 0.837152016006018, | |
| "accuracy": 0.8421052631578947 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 598, | |
| "confidence": 0.9990241513603109, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.06999999999999995 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.99, | |
| "macro_f1": 0.9901098901098901, | |
| "nll": 0.010840845209029908, | |
| "brier": 0.006549160572433809, | |
| "ece_10_bins": 0.008071925044059772, | |
| "oracle_expected_nll": 0.010840845209029908, | |
| "oracle_distribution_mse": 0.006549160572433809, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 1, | |
| "confidence": 0.5126363039016724, | |
| "accuracy": 0.0 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 1, | |
| "confidence": 0.615834653377533, | |
| "accuracy": 0.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 198, | |
| "confidence": 0.9975458886286225, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.010000000000000009 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.7286704853783539, | |
| "nll": 0.5605819633748517, | |
| "brier": 0.3768439641098432, | |
| "ece_10_bins": 0.03015358328819274, | |
| "oracle_expected_nll": 0.530985107903045, | |
| "oracle_distribution_mse": 0.02969624924165533, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 15, | |
| "confidence": 0.5687293569246928, | |
| "accuracy": 0.5333333333333333 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 33, | |
| "confidence": 0.6570033936789541, | |
| "accuracy": 0.6060606060606061 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 76, | |
| "confidence": 0.7558186516950005, | |
| "accuracy": 0.7105263157894737 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 76, | |
| "confidence": 0.837152016006018, | |
| "accuracy": 0.8421052631578947 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.12 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.18999999999999995 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.22666666666666668 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.00022176956251617814, | |
| "brier": 1.0389551787255491e-07, | |
| "ece_10_bins": 0.00022174358367921254, | |
| "oracle_expected_nll": 0.00022176956251617814, | |
| "oracle_distribution_mse": 1.0389551787255491e-07, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9997782564163208, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.00026683074882947707, | |
| "brier": 1.4082314449344625e-06, | |
| "ece_10_bins": 0.0002664735913276983, | |
| "oracle_expected_nll": 0.00026683074882947707, | |
| "oracle_distribution_mse": 1.4082314449344625e-06, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9997335264086723, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "per_question": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.93, | |
| "macro_f1": 0.942767064108638, | |
| "nll": 0.14733114755095042, | |
| "brier": 0.09750962712499976, | |
| "ece_10_bins": 0.017871906235814118, | |
| "oracle_expected_nll": 0.13843533652518125, | |
| "oracle_distribution_mse": 0.009677819048759641, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 11, | |
| "confidence": 0.5629869699478149, | |
| "accuracy": 0.36363636363636365 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 27, | |
| "confidence": 0.6522127919726901, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 48, | |
| "confidence": 0.7543277218937874, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 105, | |
| "confidence": 0.8517835855484008, | |
| "accuracy": 0.7904761904761904 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 609, | |
| "confidence": 0.994959923532013, | |
| "accuracy": 0.9967159277504105 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.06999999999999995 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.99, | |
| "macro_f1": 0.9901098901098901, | |
| "nll": 0.016769024587334803, | |
| "brier": 0.006763791666141161, | |
| "ece_10_bins": 0.01407569766044616, | |
| "oracle_expected_nll": 0.016769024587334803, | |
| "oracle_distribution_mse": 0.006763791666141161, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 2, | |
| "confidence": 0.5465996563434601, | |
| "accuracy": 0.0 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 3, | |
| "confidence": 0.8894220391909281, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 195, | |
| "confidence": 0.992870736733461, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.010000000000000009 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.7286704853783539, | |
| "nll": 0.5704197966243141, | |
| "brier": 0.3832444677877699, | |
| "ece_10_bins": 0.06606497406959536, | |
| "oracle_expected_nll": 0.5348365525212375, | |
| "oracle_distribution_mse": 0.03191723548280944, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 9, | |
| "confidence": 0.5666285951932272, | |
| "accuracy": 0.4444444444444444 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 27, | |
| "confidence": 0.6522127919726901, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 48, | |
| "confidence": 0.7543277218937874, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 102, | |
| "confidence": 0.8506765722059736, | |
| "accuracy": 0.7843137254901961 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 14, | |
| "confidence": 0.9104529874665397, | |
| "accuracy": 0.8571428571428571 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.12 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.18999999999999995 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.22666666666666668 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0, | |
| "brier": 5.221318590436242e-77, | |
| "ece_10_bins": 0.0, | |
| "oracle_expected_nll": 0.0, | |
| "oracle_distribution_mse": 5.221318590436242e-77, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 1.0, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0021357689921527212, | |
| "brier": 3.0249046087972453e-05, | |
| "ece_10_bins": 0.002127710282802564, | |
| "oracle_expected_nll": 0.0021357689921527212, | |
| "oracle_distribution_mse": 3.0249046087972453e-05, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9978722897171974, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "test": { | |
| "uncalibrated": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.9325, | |
| "macro_f1": 0.945994599459946, | |
| "nll": 0.14603000355243925, | |
| "brier": 0.0982134248610902, | |
| "ece_10_bins": 0.01680564425885677, | |
| "oracle_expected_nll": 0.12875951419169415, | |
| "oracle_distribution_mse": 0.005807715000243819, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 11, | |
| "confidence": 0.5358001806519248, | |
| "accuracy": 0.6363636363636364 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 18, | |
| "confidence": 0.668232325050566, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 67, | |
| "confidence": 0.7674324290076299, | |
| "accuracy": 0.7313432835820896 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 104, | |
| "confidence": 0.843749422293443, | |
| "accuracy": 0.75 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 600, | |
| "confidence": 0.9997629618644714, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.0675 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0003887213664454956, | |
| "brier": 6.573280611448498e-06, | |
| "ece_10_bins": 0.00038706064224247605, | |
| "oracle_expected_nll": 0.0003887213664454956, | |
| "oracle_distribution_mse": 6.573280611448498e-06, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9996129393577575, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.72997299729973, | |
| "nll": 0.5834056262796022, | |
| "brier": 0.3928407776022817, | |
| "ece_10_bins": 0.06651146262884142, | |
| "oracle_expected_nll": 0.514323668836622, | |
| "oracle_distribution_mse": 0.023217938158896164, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 11, | |
| "confidence": 0.5358001806519248, | |
| "accuracy": 0.6363636363636364 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 18, | |
| "confidence": 0.668232325050566, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 67, | |
| "confidence": 0.7674324290076299, | |
| "accuracy": 0.7313432835820896 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 104, | |
| "confidence": 0.843749422293443, | |
| "accuracy": 0.75 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.19999999999999996 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.25 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.24 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.00010169231916523476, | |
| "brier": 2.3599058819392956e-08, | |
| "ece_10_bins": 0.00010168641805652356, | |
| "oracle_expected_nll": 0.00010169231916523476, | |
| "oracle_distribution_mse": 2.3599058819392956e-08, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9998983135819435, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.00022397424454392842, | |
| "brier": 6.324962408847836e-06, | |
| "ece_10_bins": 0.00022236734628677812, | |
| "oracle_expected_nll": 0.00022397424454392842, | |
| "oracle_distribution_mse": 6.324962408847836e-06, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9997776326537132, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "global": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.9325, | |
| "macro_f1": 0.945994599459946, | |
| "nll": 0.14461302598233813, | |
| "brier": 0.09727346687970145, | |
| "ece_10_bins": 0.010928971841931354, | |
| "oracle_expected_nll": 0.12886215335419976, | |
| "oracle_distribution_mse": 0.005768674131682376, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 12, | |
| "confidence": 0.5381810814142227, | |
| "accuracy": 0.5833333333333334 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 20, | |
| "confidence": 0.6634815037250519, | |
| "accuracy": 0.65 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 88, | |
| "confidence": 0.7614993337880481, | |
| "accuracy": 0.7386363636363636 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 80, | |
| "confidence": 0.833437080681324, | |
| "accuracy": 0.7625 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 600, | |
| "confidence": 0.9995919790863991, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.0675 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0006354354848862528, | |
| "brier": 1.4171445392714965e-05, | |
| "ece_10_bins": 0.000631839334964801, | |
| "oracle_expected_nll": 0.0006354354848862528, | |
| "oracle_distribution_mse": 1.4171445392714965e-05, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9993681606650352, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.72997299729973, | |
| "nll": 0.5772212989828499, | |
| "brier": 0.389067384510727, | |
| "ece_10_bins": 0.042491824626922636, | |
| "oracle_expected_nll": 0.5142178084702962, | |
| "oracle_distribution_mse": 0.023048213518650672, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 12, | |
| "confidence": 0.5381810814142227, | |
| "accuracy": 0.5833333333333334 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 20, | |
| "confidence": 0.6634815037250519, | |
| "accuracy": 0.65 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 88, | |
| "confidence": 0.7614993337880481, | |
| "accuracy": 0.7386363636363636 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 80, | |
| "confidence": 0.833437080681324, | |
| "accuracy": 0.7625 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.19999999999999996 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.25 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.24 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.00022736255682158317, | |
| "brier": 1.1458914958364816e-07, | |
| "ece_10_bins": 0.00022733390331264314, | |
| "oracle_expected_nll": 0.00022736255682158317, | |
| "oracle_distribution_mse": 1.1458914958364816e-07, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9997726660966874, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.00036800690479488446, | |
| "brier": 1.2196973536534902e-05, | |
| "ece_10_bins": 0.00036488950252533403, | |
| "oracle_expected_nll": 0.00036800690479488446, | |
| "oracle_distribution_mse": 1.2196973536534902e-05, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9996351104974747, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "per_question": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.9325, | |
| "macro_f1": 0.945994599459946, | |
| "nll": 0.14949719129215913, | |
| "brier": 0.0995688090168633, | |
| "ece_10_bins": 0.02478441372513776, | |
| "oracle_expected_nll": 0.1305604755579714, | |
| "oracle_distribution_mse": 0.006266329786064511, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 11, | |
| "confidence": 0.5392229827967557, | |
| "accuracy": 0.6363636363636364 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 11, | |
| "confidence": 0.666460779580203, | |
| "accuracy": 0.5454545454545454 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 45, | |
| "confidence": 0.7596085866292318, | |
| "accuracy": 0.7333333333333333 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 123, | |
| "confidence": 0.8477989275281023, | |
| "accuracy": 0.7317073170731707 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 610, | |
| "confidence": 0.9967766217520979, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.0675 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.003119851317236034, | |
| "brier": 0.000139220646155405, | |
| "ece_10_bins": 0.0030834728479385287, | |
| "oracle_expected_nll": 0.003119851317236034, | |
| "oracle_distribution_mse": 0.000139220646155405, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9969165271520615, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.72997299729973, | |
| "nll": 0.592753167735883, | |
| "brier": 0.39804941940743904, | |
| "ece_10_bins": 0.09396138578653342, | |
| "oracle_expected_nll": 0.5170063047991321, | |
| "oracle_distribution_mse": 0.024839502484243964, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 11, | |
| "confidence": 0.5392229827967557, | |
| "accuracy": 0.6363636363636364 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 11, | |
| "confidence": 0.666460779580203, | |
| "accuracy": 0.5454545454545454 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 45, | |
| "confidence": 0.7596085866292318, | |
| "accuracy": 0.7333333333333333 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 123, | |
| "confidence": 0.8477989275281023, | |
| "accuracy": 0.7317073170731707 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 10, | |
| "confidence": 0.9068993091583252, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.19999999999999996 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.25 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.24 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0, | |
| "brier": 1.5641410521010998e-69, | |
| "ece_10_bins": 0.0, | |
| "oracle_expected_nll": 0.0, | |
| "oracle_distribution_mse": 1.5641410521010998e-69, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 1.0, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.002115746115517467, | |
| "brier": 8.65960138586772e-05, | |
| "ece_10_bins": 0.0020927962660789445, | |
| "oracle_expected_nll": 0.002115746115517467, | |
| "oracle_distribution_mse": 8.65960138586772e-05, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.997907203733921, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } |