| { | |
| "scope": "Exploratory existing splits; test has previously informed development", | |
| "calibration": { | |
| "global": 1.2589254117941675, | |
| "by_question": { | |
| "150440942f8fd3e04c6ffcb60c0215ffb1265e7d27e071f3ac453fd9817a85d4": 1.7378008287493754, | |
| "2a95b662ebe5bb2aa612f55ab9490ec200c83aec6e0ce1e9c439d769cb39a93c": 0.1, | |
| "330b97e5d96bf0d453555d62465367c41f1c5eb57ca53a1f3e8015c8ebe8548a": 0.1, | |
| "59786fb640773138e83576a42b1baff87c3a12c6924070c867a17f7541f8fd50": 1.0 | |
| }, | |
| "minimum_examples": 50, | |
| "unknown_question_policy": "global temperature; calibration not established" | |
| }, | |
| "splits": { | |
| "validation": { | |
| "uncalibrated": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.93125, | |
| "macro_f1": 0.9441014135596143, | |
| "nll": 0.1488269731871437, | |
| "brier": 0.0975886054619946, | |
| "ece_10_bins": 0.017362924292683653, | |
| "oracle_expected_nll": 0.14322728173785357, | |
| "oracle_distribution_mse": 0.012396862257624472, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 14, | |
| "confidence": 0.5606423233236585, | |
| "accuracy": 0.42857142857142855 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 38, | |
| "confidence": 0.6494223318601909, | |
| "accuracy": 0.7105263157894737 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 48, | |
| "confidence": 0.7586801064511141, | |
| "accuracy": 0.6875 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 76, | |
| "confidence": 0.8506408341621098, | |
| "accuracy": 0.7763157894736842 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 624, | |
| "confidence": 0.9925415913263956, | |
| "accuracy": 0.9935897435897436 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0033333333333332993 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.06874999999999998 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.995, | |
| "macro_f1": 0.9948307988219031, | |
| "nll": 0.02237316982799771, | |
| "brier": 0.010520559354480222, | |
| "ece_10_bins": 0.009969844520092013, | |
| "oracle_expected_nll": 0.02237316982799771, | |
| "oracle_distribution_mse": 0.010520559354480222, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 1, | |
| "confidence": 0.6584500074386597, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 1, | |
| "confidence": 0.7848847508430481, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 5, | |
| "confidence": 0.8563984274864197, | |
| "accuracy": 0.8 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 193, | |
| "confidence": 0.9940139299229637, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0050000000000000044 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.7282608695652174, | |
| "nll": 0.5679232407765421, | |
| "brier": 0.3794293531759783, | |
| "ece_10_bins": 0.07154568254947667, | |
| "oracle_expected_nll": 0.5455244749793817, | |
| "oracle_distribution_mse": 0.0386623803584977, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 14, | |
| "confidence": 0.5606423233236585, | |
| "accuracy": 0.42857142857142855 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 37, | |
| "confidence": 0.6491783406283405, | |
| "accuracy": 0.7027027027027027 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 47, | |
| "confidence": 0.7581225608257537, | |
| "accuracy": 0.6808510638297872 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 71, | |
| "confidence": 0.8502353698434965, | |
| "accuracy": 0.7746478873239436 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 31, | |
| "confidence": 0.9187506764165817, | |
| "accuracy": 0.8709677419354839 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.14 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.18999999999999995 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.23333333333333328 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 9.63352862168806e-05, | |
| "brier": 1.9762492451412852e-08, | |
| "ece_10_bins": 9.633034467693147e-05, | |
| "oracle_expected_nll": 9.63352862168806e-05, | |
| "oracle_distribution_mse": 1.9762492451412852e-08, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9999036696553231, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.004915146857817981, | |
| "brier": 0.0004044895550275171, | |
| "ece_10_bins": 0.00480370193719859, | |
| "oracle_expected_nll": 0.004915146857817981, | |
| "oracle_distribution_mse": 0.0004044895550275171, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9951962980628014, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "global": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.93125, | |
| "macro_f1": 0.9441014135596143, | |
| "nll": 0.1497719984221153, | |
| "brier": 0.09681936358318226, | |
| "ece_10_bins": 0.00701747298240663, | |
| "oracle_expected_nll": 0.14532400513188456, | |
| "oracle_distribution_mse": 0.01255885871902599, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 25, | |
| "confidence": 0.565165548324585, | |
| "accuracy": 0.52 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 42, | |
| "confidence": 0.6500179114795867, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 70, | |
| "confidence": 0.7533530805792127, | |
| "accuracy": 0.7571428571428571 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 80, | |
| "confidence": 0.8482251234352589, | |
| "accuracy": 0.85 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 583, | |
| "confidence": 0.9942052873493672, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0033333333333332993 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.06874999999999998 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.995, | |
| "macro_f1": 0.9948307988219031, | |
| "nll": 0.029297618117778645, | |
| "brier": 0.011847873858568251, | |
| "ece_10_bins": 0.0152635979652405, | |
| "oracle_expected_nll": 0.029297618117778645, | |
| "oracle_distribution_mse": 0.011847873858568251, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 1, | |
| "confidence": 0.6272245645523071, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 2, | |
| "confidence": 0.7533551752567291, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 9, | |
| "confidence": 0.839881075753106, | |
| "accuracy": 0.8888888888888888 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 188, | |
| "confidence": 0.9907149777133414, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0050000000000000044 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.7282608695652174, | |
| "nll": 0.558092843042443, | |
| "brier": 0.37415878347224946, | |
| "ece_10_bins": 0.014070594906806932, | |
| "oracle_expected_nll": 0.54030086988152, | |
| "oracle_distribution_mse": 0.03711676401562454, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 25, | |
| "confidence": 0.565165548324585, | |
| "accuracy": 0.52 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 41, | |
| "confidence": 0.6505738467704959, | |
| "accuracy": 0.6585365853658537 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 68, | |
| "confidence": 0.7533530189710504, | |
| "accuracy": 0.75 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 65, | |
| "confidence": 0.8466363714291499, | |
| "accuracy": 0.8307692307692308 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 1, | |
| "confidence": 0.9008614420890808, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.14 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.18999999999999995 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.23333333333333328 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0006419544265103104, | |
| "brier": 8.562553287209336e-07, | |
| "ece_10_bins": 0.0006417402625084057, | |
| "oracle_expected_nll": 0.0006419544265103104, | |
| "oracle_distribution_mse": 8.562553287209336e-07, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9993582597374916, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.01105557810172927, | |
| "brier": 0.0012699407465824586, | |
| "ece_10_bins": 0.010687653124332395, | |
| "oracle_expected_nll": 0.01105557810172927, | |
| "oracle_distribution_mse": 0.0012699407465824586, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 6, | |
| "confidence": 0.8779526750246683, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 194, | |
| "confidence": 0.9927564604380696, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "per_question": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.93125, | |
| "macro_f1": 0.9441014135596143, | |
| "nll": 0.1545261240547399, | |
| "brier": 0.09900029128593654, | |
| "ece_10_bins": 0.015734072476625435, | |
| "oracle_expected_nll": 0.14892643260544974, | |
| "oracle_distribution_mse": 0.013808548081566407, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 15, | |
| "confidence": 0.5626973231633504, | |
| "accuracy": 0.4666666666666667 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 38, | |
| "confidence": 0.6498527918991289, | |
| "accuracy": 0.7105263157894737 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 56, | |
| "confidence": 0.7574487828782627, | |
| "accuracy": 0.7142857142857143 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 81, | |
| "confidence": 0.8516954691321762, | |
| "accuracy": 0.8024691358024691 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 610, | |
| "confidence": 0.9894479682211016, | |
| "accuracy": 0.9934426229508196 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.00666666666666671 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.06874999999999998 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.995, | |
| "macro_f1": 0.9948307988219031, | |
| "nll": 0.05018125544241732, | |
| "brier": 0.016571811967767927, | |
| "ece_10_bins": 0.03622871160507202, | |
| "oracle_expected_nll": 0.05018125544241732, | |
| "oracle_distribution_mse": 0.016571811967767927, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 1, | |
| "confidence": 0.5914673209190369, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 1, | |
| "confidence": 0.6748074889183044, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 9, | |
| "confidence": 0.7539301647080315, | |
| "accuracy": 0.8888888888888888 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 10, | |
| "confidence": 0.8620621740818024, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 179, | |
| "confidence": 0.9781116739997651, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0050000000000000044 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.73, | |
| "macro_f1": 0.7282608695652174, | |
| "nll": 0.5679232407765421, | |
| "brier": 0.3794293531759783, | |
| "ece_10_bins": 0.07154568254947667, | |
| "oracle_expected_nll": 0.5455244749793817, | |
| "oracle_distribution_mse": 0.0386623803584977, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 14, | |
| "confidence": 0.5606423233236585, | |
| "accuracy": 0.42857142857142855 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 37, | |
| "confidence": 0.6491783406283405, | |
| "accuracy": 0.7027027027027027 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 47, | |
| "confidence": 0.7581225608257537, | |
| "accuracy": 0.6808510638297872 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 71, | |
| "confidence": 0.8502353698434965, | |
| "accuracy": 0.7746478873239436 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 31, | |
| "confidence": 0.9187506764165817, | |
| "accuracy": 0.8709677419354839 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.14 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.18999999999999995 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.23333333333333328 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.27 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0, | |
| "brier": 4.3804362436076516e-76, | |
| "ece_10_bins": 0.0, | |
| "oracle_expected_nll": 0.0, | |
| "oracle_distribution_mse": 4.3804362436076516e-76, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 1.0, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0, | |
| "brier": 7.613875406184365e-24, | |
| "ece_10_bins": 0.0, | |
| "oracle_expected_nll": 0.0, | |
| "oracle_distribution_mse": 7.613875406184365e-24, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 1.0, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "test": { | |
| "uncalibrated": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.925, | |
| "macro_f1": 0.9392552790275188, | |
| "nll": 0.15805856801203613, | |
| "brier": 0.10519841661327942, | |
| "ece_10_bins": 0.025654533132910683, | |
| "oracle_expected_nll": 0.138174928971103, | |
| "oracle_distribution_mse": 0.01090862890356645, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 17, | |
| "confidence": 0.5612775718464571, | |
| "accuracy": 0.23529411764705882 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 16, | |
| "confidence": 0.6538418717682362, | |
| "accuracy": 0.75 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 64, | |
| "confidence": 0.7603475973010063, | |
| "accuracy": 0.765625 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 75, | |
| "confidence": 0.8518771576881409, | |
| "accuracy": 0.68 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 628, | |
| "confidence": 0.9939726705953573, | |
| "accuracy": 0.9936305732484076 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.07499999999999996 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.995, | |
| "macro_f1": 0.9944254755766148, | |
| "nll": 0.013614306728828655, | |
| "brier": 0.006972022766819956, | |
| "ece_10_bins": 0.00483260512351995, | |
| "oracle_expected_nll": 0.013614306728828655, | |
| "oracle_distribution_mse": 0.006972022766819956, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 3, | |
| "confidence": 0.6343480944633484, | |
| "accuracy": 0.6666666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 3, | |
| "confidence": 0.8595178723335266, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 194, | |
| "confidence": 0.9976901086335329, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0050000000000000044 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.705, | |
| "macro_f1": 0.7046381817726715, | |
| "nll": 0.6145763126252539, | |
| "brier": 0.4129853920524647, | |
| "ece_10_bins": 0.11028719127178192, | |
| "oracle_expected_nll": 0.5350417564615215, | |
| "oracle_distribution_mse": 0.03582624121361282, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 17, | |
| "confidence": 0.5612775718464571, | |
| "accuracy": 0.23529411764705882 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 13, | |
| "confidence": 0.6583404357616718, | |
| "accuracy": 0.7692307692307693 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 64, | |
| "confidence": 0.7603475973010063, | |
| "accuracy": 0.765625 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 69, | |
| "confidence": 0.8522450759790946, | |
| "accuracy": 0.6521739130434783 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 37, | |
| "confidence": 0.9170670589885196, | |
| "accuracy": 0.8918918918918919 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.19999999999999996 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.26 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.2733333333333333 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.29500000000000004 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 9.923051486518732e-05, | |
| "brier": 2.306917757809561e-08, | |
| "ece_10_bins": 9.922474622725552e-05, | |
| "oracle_expected_nll": 9.923051486518732e-05, | |
| "oracle_distribution_mse": 2.306917757809561e-08, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9999007752537727, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.003944422179196725, | |
| "brier": 0.000836228564655437, | |
| "ece_10_bins": 0.00370678603649136, | |
| "oracle_expected_nll": 0.003944422179196725, | |
| "oracle_distribution_mse": 0.000836228564655437, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 3, | |
| "confidence": 0.8357743223508199, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 197, | |
| "confidence": 0.9987376640895902, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "global": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.925, | |
| "macro_f1": 0.9392552790275188, | |
| "nll": 0.15451136706332724, | |
| "brier": 0.10279470684407294, | |
| "ece_10_bins": 0.02014885999262334, | |
| "oracle_expected_nll": 0.1387172312235814, | |
| "oracle_distribution_mse": 0.010962450171169045, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 22, | |
| "confidence": 0.5578833384947344, | |
| "accuracy": 0.36363636363636365 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 24, | |
| "confidence": 0.6626042500138283, | |
| "accuracy": 0.7916666666666666 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 86, | |
| "confidence": 0.7429546443528907, | |
| "accuracy": 0.7441860465116279 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 79, | |
| "confidence": 0.8482364740552781, | |
| "accuracy": 0.759493670886076 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 589, | |
| "confidence": 0.9972299244286451, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.07499999999999996 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.995, | |
| "macro_f1": 0.9944254755766148, | |
| "nll": 0.017469275119756628, | |
| "brier": 0.008068493609724506, | |
| "ece_10_bins": 0.010246815681457555, | |
| "oracle_expected_nll": 0.017469275119756628, | |
| "oracle_distribution_mse": 0.008068493609724506, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 2, | |
| "confidence": 0.5836484432220459, | |
| "accuracy": 0.5 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 1, | |
| "confidence": 0.6414172649383545, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 1, | |
| "confidence": 0.7566628456115723, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 4, | |
| "confidence": 0.8481354713439941, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 192, | |
| "confidence": 0.9964964153865973, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0050000000000000044 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.705, | |
| "macro_f1": 0.7046381817726715, | |
| "nll": 0.5920632934974356, | |
| "brier": 0.4015789899739018, | |
| "ece_10_bins": 0.07626515358686448, | |
| "oracle_expected_nll": 0.5288867501384523, | |
| "oracle_distribution_mse": 0.03424996328228628, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 20, | |
| "confidence": 0.5553068280220032, | |
| "accuracy": 0.35 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 23, | |
| "confidence": 0.6635254232779794, | |
| "accuracy": 0.782608695652174 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 83, | |
| "confidence": 0.7421774620033172, | |
| "accuracy": 0.7349397590361446 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 74, | |
| "confidence": 0.848746615487176, | |
| "accuracy": 0.7432432432432432 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.19999999999999996 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.26 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.2733333333333333 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.29500000000000004 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0006544187157477001, | |
| "brier": 9.303064614080757e-07, | |
| "ece_10_bins": 0.0006541860103607133, | |
| "oracle_expected_nll": 0.0006544187157477001, | |
| "oracle_distribution_mse": 9.303064614080757e-07, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9993458139896393, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.007858480920368994, | |
| "brier": 0.0015304134862039825, | |
| "ece_10_bins": 0.007402259111404373, | |
| "oracle_expected_nll": 0.007858480920368994, | |
| "oracle_distribution_mse": 0.0015304134862039825, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 2, | |
| "confidence": 0.7683536112308502, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 1, | |
| "confidence": 0.8108900189399719, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 197, | |
| "confidence": 0.9957967052604947, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "per_question": { | |
| "overall": { | |
| "count": 800, | |
| "accuracy": 0.925, | |
| "macro_f1": 0.9392552790275188, | |
| "nll": 0.16079165909777055, | |
| "brier": 0.10593323494171354, | |
| "ece_10_bins": 0.02878171861171729, | |
| "oracle_expected_nll": 0.14090802005683742, | |
| "oracle_distribution_mse": 0.01164344723200057, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 19, | |
| "confidence": 0.5593467480257938, | |
| "accuracy": 0.2631578947368421 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 15, | |
| "confidence": 0.6569308956464132, | |
| "accuracy": 0.8 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 68, | |
| "confidence": 0.7610192018396714, | |
| "accuracy": 0.7794117647058824 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 73, | |
| "confidence": 0.8525561878125961, | |
| "accuracy": 0.6712328767123288 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 625, | |
| "confidence": 0.9923768733978271, | |
| "accuracy": 0.9936 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 400, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 600, | |
| "error_rate": 0.0016666666666667052 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 800, | |
| "error_rate": 0.07499999999999996 | |
| } | |
| ] | |
| }, | |
| "by_task": { | |
| "department": { | |
| "count": 200, | |
| "accuracy": 0.995, | |
| "macro_f1": 0.9944254755766148, | |
| "nll": 0.02859032138164205, | |
| "brier": 0.010747547714388071, | |
| "ece_10_bins": 0.01983757793903353, | |
| "oracle_expected_nll": 0.02859032138164205, | |
| "oracle_distribution_mse": 0.010747547714388071, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 2, | |
| "confidence": 0.5429347455501556, | |
| "accuracy": 0.5 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 2, | |
| "confidence": 0.6477688848972321, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 4, | |
| "confidence": 0.771764874458313, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 4, | |
| "confidence": 0.8579228669404984, | |
| "accuracy": 1.0 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 188, | |
| "confidence": 0.990979070042042, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0050000000000000044 | |
| } | |
| ] | |
| }, | |
| "escalation": { | |
| "count": 200, | |
| "accuracy": 0.705, | |
| "macro_f1": 0.7046381817726715, | |
| "nll": 0.6145763126252539, | |
| "brier": 0.4129853920524647, | |
| "ece_10_bins": 0.11028719127178192, | |
| "oracle_expected_nll": 0.5350417564615215, | |
| "oracle_distribution_mse": 0.03582624121361282, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 17, | |
| "confidence": 0.5612775718464571, | |
| "accuracy": 0.23529411764705882 | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 13, | |
| "confidence": 0.6583404357616718, | |
| "accuracy": 0.7692307692307693 | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 64, | |
| "confidence": 0.7603475973010063, | |
| "accuracy": 0.765625 | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 69, | |
| "confidence": 0.8522450759790946, | |
| "accuracy": 0.6521739130434783 | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 37, | |
| "confidence": 0.9170670589885196, | |
| "accuracy": 0.8918918918918919 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.19999999999999996 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.26 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.2733333333333333 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.29500000000000004 | |
| } | |
| ] | |
| }, | |
| "refund": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 0.0, | |
| "brier": 6.297547947730489e-70, | |
| "ece_10_bins": 0.0, | |
| "oracle_expected_nll": 0.0, | |
| "oracle_distribution_mse": 6.297547947730489e-70, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 1.0, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| }, | |
| "severity": { | |
| "count": 200, | |
| "accuracy": 1.0, | |
| "macro_f1": 1.0, | |
| "nll": 2.384186146287072e-09, | |
| "brier": 1.3848676160326649e-15, | |
| "ece_10_bins": 2.38418573772492e-09, | |
| "oracle_expected_nll": 2.384186146287072e-09, | |
| "oracle_distribution_mse": 1.3848676160326649e-15, | |
| "reliability": [ | |
| { | |
| "lower": 0.0, | |
| "upper": 0.1, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.1, | |
| "upper": 0.2, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.2, | |
| "upper": 0.3, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.3, | |
| "upper": 0.4, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.4, | |
| "upper": 0.5, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.5, | |
| "upper": 0.6, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.6, | |
| "upper": 0.7, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.7, | |
| "upper": 0.8, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.8, | |
| "upper": 0.9, | |
| "count": 0, | |
| "confidence": null, | |
| "accuracy": null | |
| }, | |
| { | |
| "lower": 0.9, | |
| "upper": 1.0, | |
| "count": 200, | |
| "confidence": 0.9999999976158143, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "risk_coverage": [ | |
| { | |
| "coverage": 0.25, | |
| "count": 50, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.5, | |
| "count": 100, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 0.75, | |
| "count": 150, | |
| "error_rate": 0.0 | |
| }, | |
| { | |
| "coverage": 1.0, | |
| "count": 200, | |
| "error_rate": 0.0 | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } |